diff --git a/docs/swarm/builder.md b/docs/swarm/builder.md index 0f370b6e..0bade76f 100644 --- a/docs/swarm/builder.md +++ b/docs/swarm/builder.md @@ -394,9 +394,16 @@ path nobody added or fetched for 14 days goes at the next garbage collection, which runs every 12 hours (`deploy.attic.retentionPeriod`). The controller [rebuilds a closure the cache lost](#rebuild-on-a-cache-miss). -The gateway terminates TLS with a swarm services certificate and forwards to -`127.0.0.1:8910` (`deploy.attic.port`) with no login in front: attic checks its -own tokens, and a nix client can't follow a login redirect. The container logs +The container sits on the hive bridge in a network namespace of its own. atticd +listens on that namespace's loopback (`deploy.attic.port`), and a socket-activated +relay forwards `/run/swarm-attic/atticd.sock` to it, a directory only the +gateway's nginx and the container can open. The gateway terminates TLS with a +swarm services certificate and forwards to that socket with no login in front: +attic checks its own tokens, and a nix client can't follow a login redirect. +The two host units reach the cache through the gateway, at +`https://`, like every other client; the host resolves the +name through the hive resolver, and where the gateway serves a self-signed +certificate `swarm-bao-attic-tokens` trusts the swarm's CA. The container logs to its journal only; its journal doesn't reach the swarm log store. ### External cache diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 2a0a5fa5..91f480d6 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -459,8 +459,9 @@ it. For an external cache, override `swarm.attic.{domain,cache}` and don't set `deploy.attic.enable` anywhere, or set it to `lib.mkForce false` under `allSwarmServices`. -The container shares the host's network namespace and atticd listens on loopback -only. The gateway serves it at `swarm.attic.domain` (default +The container sits on the hive bridge in a network namespace of its own, and atticd +listens on that namespace's loopback only; the gateway reaches it over a unix socket. +The gateway serves it at `swarm.attic.domain` (default `attic.`, on every host) over TLS, with no login in front, because attic checks its own tokens. Storage is local disk in the container's state. diff --git a/nix/host-modules/lib/attic-init.nix b/nix/host-modules/lib/attic-init.nix index 531c9db7..91c71bfe 100644 --- a/nix/host-modules/lib/attic-init.nix +++ b/nix/host-modules/lib/attic-init.nix @@ -31,7 +31,8 @@ url, # The cache's name on the server. cache, - # Where the host reaches atticd, scheme included. + # Where the host reaches atticd, scheme included: the gateway, which answers + # 502 while atticd is down. endpoint, # The server's config file, which the host's `atticadm` reads as well. configFile, @@ -110,16 +111,23 @@ in # tokens are signed here with the key the store holds; the cache is created # through the API by a token that lives for five minutes and is never stored. provision = '' + # The gateway answers 502 while atticd is down, so only a status below 500 + # is atticd answering. up=0 + code=000 for _ in $(seq 1 30); do - if curl -s -o /dev/null --max-time 2 ${q endpoint}; then - up=1 - break - fi + code="$(curl -s -o /dev/null --max-time 2 -w '%{http_code}' ${q endpoint} || true)" + case "$code" in + "" | 000 | 5??) ;; + *) + up=1 + break + ;; + esac sleep 2 done if [ "$up" != 1 ]; then - echo "atticd did not answer at ${endpoint}." >&2 + echo "atticd did not answer at ${endpoint} (last status $code)." >&2 exit 1 fi diff --git a/nix/host-modules/swarm-attic.nix b/nix/host-modules/swarm-attic.nix index c7b68d66..4c8bf99f 100644 --- a/nix/host-modules/swarm-attic.nix +++ b/nix/host-modules/swarm-attic.nix @@ -10,10 +10,13 @@ # (its address, name and public key), each only while absent # (./lib/attic-init.nix). Neither needs a manual step. # -# The container shares the host's network namespace, as ./swarm-victorialogs.nix -# does, and atticd listens on loopback only: the gateway's vhost is the way in, -# and attic's own tokens are the authentication. Storage is local disk under the -# container's state. +# The container has a network namespace of its own on the hive bridge. atticd +# listens on that namespace's loopback only, because it cannot listen on a unix +# socket. A socket-activated relay in the container forwards a unix socket to it, +# and the gateway's nginx proxies to that socket. The host units reach the cache +# as every client does, at its published name through the gateway. Attic's own +# tokens are the authentication. Storage is local disk under the container's +# state. # # The two `swarm.attic` options every host reads (./swarm-attic-options.nix) # default to this cache's names on every host, whether or not it runs here. @@ -45,8 +48,17 @@ let secretDir = "/run/swarm-attic-secrets"; envFile = "${secretDir}/server.env"; - # Where the host units reach atticd: the shared loopback, not the gateway. - endpoint = "http://127.0.0.1:${toString cfg.port}"; + # Where the host units reach atticd: the gateway, at the published name. + endpoint = "https://${atticCfg.domain}"; + + # The unix socket the gateway's nginx connects to. The directory's mode is + # what keeps it private, so it holds this one socket. + socketDir = "/run/swarm-attic"; + socketPath = "${socketDir}/atticd.sock"; + socketGroup = "gateway-nginx"; + # Only the number crosses into the container, which knows the group by name. + nginxGid = config.ids.gids.nginx; + relayUnit = "swarm-attic-relay"; configFile = (pkgs.formats.toml { }).generate "swarm-attic.toml" { listen = "127.0.0.1:${toString cfg.port}"; @@ -77,7 +89,7 @@ let urlPath = "secret/swarm/services/attic/url"; cachePath = "secret/swarm/services/attic/cache"; publicKeyPath = "secret/swarm/services/attic/public-key"; - url = "https://${atticCfg.domain}"; + url = endpoint; cache = atticCfg.cache; inherit endpoint @@ -100,10 +112,10 @@ let forwarderClientId = atticCfg.otel.clientId; forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}/${machine}"; forwarderUnit = "swarm-journal-forwarder-oidc"; - # In the host netns: distinct from every other collector's there. + # Loopback ports of the forwarder's own metrics and of its OTLP/HTTP metrics + # receiver, which the cache-size exporter pushes to, in the container's own + # netns. forwarderTelemetryPort = 8903; - # The forwarder's OTLP/HTTP metrics receiver, which the cache-size exporter - # pushes to. Not 4318: the hive collector holds that in the host netns. forwarderOtlpPort = 8905; # The exporter's unit and query. `V` is attic's stored value for a valid @@ -144,6 +156,20 @@ let }; in { + # The tokens unit dials the gateway's name, whose leaf the system store lacks + # when the gateway serves a self-signed one. `hostUnit`: it reads the host's + # CA path and waits on `hive-tls-ca.service`. `enable`: `imports` is + # unconditional, so without it a host that runs no cache gets a bundle unit. + imports = [ + (caTrust.trustBundle { + inherit pkgs; + name = "swarm-bao-attic-tokens"; + consumers = [ "swarm-bao-attic-tokens" ]; + hostUnit = true; + inherit (cfg) enable; + }) + ]; + options.services.hyperhive.deploy.attic = { package = lib.mkOption { type = lib.types.package; @@ -163,11 +189,9 @@ in type = lib.types.port; default = 8910; description = '' - Loopback port atticd listens on, which the gateway proxies to. - - ⚠️ Every swarm container shares the host's network namespace, so this - is a swarm-wide claim: two modules picking the same number collide at - runtime with no bind error. No repo-wide check catches it. + Port atticd listens on, on the loopback of the container's own network + namespace. The container's relay forwards the gateway's unix socket to + it; nothing else dials it. ''; }; @@ -276,6 +300,9 @@ in services.hyperhive.gateway.enable = lib.mkDefault true; services.hyperhive.gateway.dns.enable = lib.mkDefault true; + # The hive bridge carries the container's DHCP lease and its resolver. + services.hyperhive.network.enable = lib.mkDefault true; + # `forceSSL`: the tokens ride in a header. No `auth_request`: attic checks # its own tokens, and a nix client cannot follow a login redirect. A store # path is uploaded as one request that can run for minutes, so the body is @@ -283,7 +310,7 @@ in services.hyperhive.gateway.vhosts."${atticCfg.domain}" = { forceSSL = true; locations."/" = { - proxyPass = "http://127.0.0.1:${toString cfg.port}"; + proxyPass = "http://unix:${socketPath}"; extraConfig = '' client_max_body_size 0; proxy_request_buffering off; @@ -295,8 +322,12 @@ in # The key's directory is the container's read-only bind source, so it has # to exist before the container starts even when the store does not answer. - # The delivery unit creates it too, which covers a switch. - systemd.tmpfiles.rules = [ "d ${secretDir} 0700 root root -" ]; + # The delivery unit creates it too, which covers a switch. The socket + # directory is the gateway's way in: its group is nginx's. + systemd.tmpfiles.rules = [ + "d ${secretDir} 0700 root root -" + "d ${socketDir} 0770 root ${toString nginxGid} - -" + ]; # The signing key, created in the store if it holds none and written where # the container reads it. Before the container and wanted by it: a store @@ -344,9 +375,12 @@ in # facts are for the hives, which the controller copies them to. systemd.services.swarm-bao-attic-tokens = { description = "create the swarm cache, and its tokens and public facts in the secret store, if absent"; + # `nginx.service`: the unit dials the cache through the gateway, whose + # resolver answers the name with the bridge address (`localNames` above). after = [ "container@${machine}.service" "swarm-bao-attic-secret.service" + "nginx.service" ]; wants = [ "container@${machine}.service" ]; wantedBy = [ "multi-user.target" ]; @@ -383,12 +417,20 @@ in extraFlags = lib.mapAttrsToList ( id: source: "--load-credential=${id}:${source}" ) forwarderStoreFiles; - privateNetwork = false; + privateNetwork = true; + hostBridge = networkCfg.bridgeName; bindMounts = { ${secretDir} = { hostPath = secretDir; isReadOnly = true; }; + # A bind mount, not a path both sides know: a container's `/run` is an + # nspawn tmpfs the host cannot see into. Connecting to a socket is a + # write to it. + ${socketDir} = { + hostPath = socketDir; + isReadOnly = false; + }; } // caTrust.bindMount; @@ -416,7 +458,41 @@ in }) ]; - services.hyperhive.swarmContainer.privateNetwork = false; + services.hyperhive.swarmContainer.privateNetwork = true; + + # The bridge address comes from the gateway's DHCP. `swarm-bridge-dns` + # owns resolv.conf, so dhcpcd must not write it over that unit. + networking.interfaces.eth0.useDHCP = true; + networking.dhcpcd.extraConfig = "nohook resolv.conf"; + + # The socket's group is nginx's, so the gateway can connect to it. + users.groups.${socketGroup}.gid = nginxGid; + systemd.sockets.${relayUnit} = { + description = "unix socket the gateway reaches atticd through"; + wantedBy = [ "sockets.target" ]; + listenStreams = [ socketPath ]; + socketConfig = { + SocketMode = "0660"; + SocketUser = "root"; + SocketGroup = socketGroup; + }; + }; + systemd.services.${relayUnit} = { + description = "relay the gateway's connections to atticd"; + requires = [ "${relayUnit}.socket" ]; + after = [ "${relayUnit}.socket" ]; + serviceConfig = { + ExecStart = "${pkgs.systemd}/lib/systemd/systemd-socket-proxyd 127.0.0.1:${toString cfg.port}"; + DynamicUser = true; + PrivateTmp = true; + PrivateDevices = true; + NoNewPrivileges = true; + RestrictAddressFamilies = [ + "AF_INET" + "AF_UNIX" + ]; + }; + }; # Authenticated as the forwarder's own OIDC client, with the cache's # store identity: one identity for the whole container. It is a diff --git a/nix/module-eval/builder.nix b/nix/module-eval/builder.nix index 4deb17a5..938ae150 100644 --- a/nix/module-eval/builder.nix +++ b/nix/module-eval/builder.nix @@ -462,7 +462,7 @@ let && lib.elem "swarm-bao-attic-policy.service" atticSecret.after && lib.elem "swarm-bao-attic-policy.service" atticTokens.after && atticHere.containers.swarm-attic.bindMounts."/run/swarm-attic-secrets".isReadOnly - && !atticHere.containers.swarm-attic.privateNetwork + && atticHere.containers.swarm-attic.privateNetwork && atticUnit.serviceConfig.EnvironmentFile == "/run/swarm-attic-secrets/server.env" && lib.hasInfix "--mode monolithic" atticUnit.serviceConfig.ExecStart && atticDeploy.retentionPeriod == "14 days"; @@ -534,7 +534,7 @@ let names = m: m.services.hyperhive.swarm.serviceDomains; in atticVhost.forceSSL - && atticVhost.locations."/".proxyPass == "http://127.0.0.1:8910" + && atticVhost.locations."/".proxyPass == "http://unix:/run/swarm-attic/atticd.sock" && !(lib.hasInfix "auth_request" atticVhost.locations."/".extraConfig) && lib.hasInfix "client_max_body_size 0" atticVhost.locations."/".extraConfig && lib.hasInfix "proxy_request_buffering off" atticVhost.locations."/".extraConfig diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 7bd0b08b..5be89a56 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -174,8 +174,8 @@ let # Every service container at once: the services-only host plus the CI # runner. The CI runner, the review classifier, the notification router, the # builder, the claude login minter, the metrics store, the homeserver, - # authelia, Grafana and the collector are the ten containers on a netns of - # their own. + # authelia, Grafana, the collector and the cache are the eleven containers on + # a netns of their own. serviceContainersWithCi = hive { deploy.hive-controller.enable = false; deploy.allSwarmServices = true; @@ -814,14 +814,14 @@ let } { # Both values come from ../container-modules/swarm-container.nix. - # Five of the 15 service containers share the host netns (hive-forge, bao, - # the queue, the log store, the cache) and ten have one of their own. + # Four of the 15 service containers share the host netns (hive-forge, bao, + # the queue, the log store) and eleven have one of their own. name = "a service container on the host netns runs no firewall or resolvconf of its own"; ok = let shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers); in - lib.length shared == 5 + lib.length shared == 4 && lib.all ( c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable ) shared; @@ -992,6 +992,46 @@ let && lib.all (lib.hasPrefix "http://unix:${sock}:") forwarded && c.bindMounts.${dir}.hostPath == dir; } + { + # atticd listens on the container's own loopback, a relay there forwards + # the socket nginx connects to, and the host unit that drives the + # TCP-only `attic` client dials the published name, not a loopback. The + # `cache create` clause and the trust-bundle clauses are controls: they + # find something in the same script and unit the absences are claimed of. + # `upstreams != []` keeps an empty vhost from passing. + name = "the cache runs on a netns of its own, the gateway reaches it over a socket and the host unit dials its public name"; + ok = + let + host = serviceContainersWithCi; + h = host.services.hyperhive; + c = host.containers.swarm-attic; + dir = "/run/swarm-attic"; + sock = "${dir}/atticd.sock"; + domain = h.swarm.attic.domain; + upstreams = map (l: l.proxyPass) (lib.attrValues h.gateway.vhosts.${domain}.locations); + relay = c.config.systemd.services.swarm-attic-relay; + tokens = host.systemd.services.swarm-bao-attic-tokens; + in + c.privateNetwork + && c.hostBridge == h.network.bridgeName + && c.hostAddress == null + && c.localAddress == null + && c.config.networking.interfaces.eth0.useDHCP + && c.config.networking.firewall.enable + && upstreams != [ ] + && lib.all (p: p == "http://unix:${sock}") upstreams + && c.config.systemd.sockets.swarm-attic-relay.listenStreams == [ sock ] + && lib.hasSuffix "systemd-socket-proxyd 127.0.0.1:${toString h.deploy.attic.port}" relay.serviceConfig.ExecStart + && c.bindMounts.${dir}.hostPath == dir + && lib.elem "d ${dir} 0770 root ${toString host.ids.gids.nginx} - -" host.systemd.tmpfiles.rules + && lib.hasInfix "cache create" tokens.script + && lib.hasInfix "https://${domain}" tokens.script + && !(lib.hasInfix "127.0.0.1" tokens.script) + && lib.elem "nginx.service" tokens.after + && h.gateway.useSelfSigned + && tokens.environment.SSL_CERT_FILE == "/run/swarm-bao-attic-tokens-ca/trust-bundle.pem" + && lib.elem "swarm-bao-attic-tokens-ca-bundle.service" tokens.requires; + } { # On the bridge by DHCP, with every receiver on a socket: the hive's, the # swarm tier's and each service forwarder's. Every gateway location that diff --git a/nix/script-tests/attic-init.nix b/nix/script-tests/attic-init.nix index 1287899d..78d11a46 100644 --- a/nix/script-tests/attic-init.nix +++ b/nix/script-tests/attic-init.nix @@ -18,7 +18,7 @@ let publicKeyPath = "secret/swarm/services/attic/public-key"; url = "http://cache.example"; cache = "grid"; - endpoint = "http://127.0.0.1:8910"; + endpoint = "https://cache.example"; configFile = "@DIR@/server.toml"; envFile = "@DIR@/secrets/server.env"; machine = "swarm-attic"; diff --git a/nix/script-tests/attic-init.sh b/nix/script-tests/attic-init.sh index be8be0bd..db53dd5f 100644 --- a/nix/script-tests/attic-init.sh +++ b/nix/script-tests/attic-init.sh @@ -95,8 +95,14 @@ chmod +x "$dir/bin/"* cat >"$dir/bin/curl" <>"$calls" -[ "\${CURL_FAIL:-0}" != 1 ] +echo "curl \$*" >>"$calls" +# A curl that cannot connect prints 000 and exits 7; CURL_CODE is the status of +# a server that answered. +if [ "\${CURL_FAIL:-0}" = 1 ]; then + printf 000 + exit 7 +fi +printf %s "\${CURL_CODE:-200}" STUB printf '#!%s\nexit 0\n' "$BASH" >"$dir/bin/sleep" # A container that is not running has nothing to refresh. @@ -246,6 +252,25 @@ expect "exit status" $? 1 expect "nothing minted" "$(n '^atticadm')" 0 expect "nothing stored" "$(n '^bao put')" 0 +case="a 502 from the gateway is not atticd answering" +reset +run "$ENSURE_JWT" +: >"$calls" +run "$PROVISION" CURL_CODE=502 +expect "exit status" $? 1 +expect "nothing minted" "$(n '^atticadm')" 0 +expect "nothing stored" "$(n '^bao put')" 0 +expect "the probe asked the gateway's name" "$(n '^curl .*https://cache.example')" 30 + +case="a status from atticd below 500 is an answer, a 404 included" +reset +run "$ENSURE_JWT" +: >"$calls" +run "$PROVISION" CURL_CODE=404 +expect "exit status" $? 0 +expect "the cache is created" "$(n '^attic cache create')" 1 +expect "one probe" "$(n '^curl')" 1 + case="the client reads its token from a file, not from an argument" reset run "$ENSURE_JWT"