run swarm-attic in its own network namespace

the gateway reaches atticd over a unix socket a relay in the container
forwards, and the host unit that creates the cache dials its public name.

docs/swarm/builder.md

@@ -394,9 +394,16 @@ path nobody added or fetched for 14 days goes at the next garbage collection,
 which runs every 12 hours (`deploy.attic.retentionPeriod`). The controller
 [rebuilds a closure the cache lost](#rebuild-on-a-cache-miss).
 
-The gateway terminates TLS with a swarm services certificate and forwards to
+The container sits on the hive bridge in a network namespace of its own. atticd
-`127.0.0.1:8910` (`deploy.attic.port`) with no login in front: attic checks its
+listens on that namespace's loopback (`deploy.attic.port`), and a socket-activated
-own tokens, and a nix client can't follow a login redirect. The container logs
+relay forwards `/run/swarm-attic/atticd.sock` to it, a directory only the
+gateway's nginx and the container can open. The gateway terminates TLS with a
+swarm services certificate and forwards to that socket with no login in front:
+attic checks its own tokens, and a nix client can't follow a login redirect.
+The two host units reach the cache through the gateway, at
+`https://<swarm.attic.domain>`, like every other client; the host resolves the
+name through the hive resolver, and where the gateway serves a self-signed
+certificate `swarm-bao-attic-tokens` trusts the swarm's CA. The container logs
 to its journal only; its journal doesn't reach the swarm log store.
 
 ### External cache

docs/swarm/services.md

@@ -459,8 +459,9 @@ it. For an external cache, override `swarm.attic.{domain,cache}` and don't set
 `deploy.attic.enable` anywhere, or set it to `lib.mkForce false` under
 `allSwarmServices`.
 
-The container shares the host's network namespace and atticd listens on loopback
+The container sits on the hive bridge in a network namespace of its own, and atticd
-only. The gateway serves it at `swarm.attic.domain` (default
+listens on that namespace's loopback only; the gateway reaches it over a unix socket.
+The gateway serves it at `swarm.attic.domain` (default
 `attic.<swarm domain>`, on every host) over TLS, with no login in front, because attic checks
 its own tokens. Storage is local disk in the container's state.
 

nix/host-modules/lib/attic-init.nix

@@ -31,7 +31,8 @@
   url,
   # The cache's name on the server.
   cache,
-  # Where the host reaches atticd, scheme included.
+  # Where the host reaches atticd, scheme included: the gateway, which answers
+  # 502 while atticd is down.
   endpoint,
   # The server's config file, which the host's `atticadm` reads as well.
   configFile,
@@ -110,16 +111,23 @@ in
   # tokens are signed here with the key the store holds; the cache is created
   # through the API by a token that lives for five minutes and is never stored.
   provision = ''
+    # The gateway answers 502 while atticd is down, so only a status below 500
+    # is atticd answering.
     up=0
+    code=000
     for _ in $(seq 1 30); do
-      if curl -s -o /dev/null --max-time 2 ${q endpoint}; then
+      code="$(curl -s -o /dev/null --max-time 2 -w '%{http_code}' ${q endpoint} || true)"
-        up=1
+      case "$code" in
-        break
+        "" | 000 | 5??) ;;
-      fi
+        *)
+          up=1
+          break
+          ;;
+      esac
       sleep 2
     done
     if [ "$up" != 1 ]; then
-      echo "atticd did not answer at ${endpoint}." >&2
+      echo "atticd did not answer at ${endpoint} (last status $code)." >&2
       exit 1
     fi
 

nix/host-modules/swarm-attic.nix

@@ -10,10 +10,13 @@
 # (its address, name and public key), each only while absent
 # (./lib/attic-init.nix). Neither needs a manual step.
 #
-# The container shares the host's network namespace, as ./swarm-victorialogs.nix
+# The container has a network namespace of its own on the hive bridge. atticd
-# does, and atticd listens on loopback only: the gateway's vhost is the way in,
+# listens on that namespace's loopback only, because it cannot listen on a unix
-# and attic's own tokens are the authentication. Storage is local disk under the
+# socket. A socket-activated relay in the container forwards a unix socket to it,
-# container's state.
+# and the gateway's nginx proxies to that socket. The host units reach the cache
+# as every client does, at its published name through the gateway. Attic's own
+# tokens are the authentication. Storage is local disk under the container's
+# state.
 #
 # The two `swarm.attic` options every host reads (./swarm-attic-options.nix)
 # default to this cache's names on every host, whether or not it runs here.
@@ -45,8 +48,17 @@ let
   secretDir = "/run/swarm-attic-secrets";
   envFile = "${secretDir}/server.env";
 
-  # Where the host units reach atticd: the shared loopback, not the gateway.
+  # Where the host units reach atticd: the gateway, at the published name.
-  endpoint = "http://127.0.0.1:${toString cfg.port}";
+  endpoint = "https://${atticCfg.domain}";
+
+  # The unix socket the gateway's nginx connects to. The directory's mode is
+  # what keeps it private, so it holds this one socket.
+  socketDir = "/run/swarm-attic";
+  socketPath = "${socketDir}/atticd.sock";
+  socketGroup = "gateway-nginx";
+  # Only the number crosses into the container, which knows the group by name.
+  nginxGid = config.ids.gids.nginx;
+  relayUnit = "swarm-attic-relay";
 
   configFile = (pkgs.formats.toml { }).generate "swarm-attic.toml" {
     listen = "127.0.0.1:${toString cfg.port}";
@@ -77,7 +89,7 @@ let
     urlPath = "secret/swarm/services/attic/url";
     cachePath = "secret/swarm/services/attic/cache";
     publicKeyPath = "secret/swarm/services/attic/public-key";
-    url = "https://${atticCfg.domain}";
+    url = endpoint;
     cache = atticCfg.cache;
     inherit
       endpoint
@@ -100,10 +112,10 @@ let
   forwarderClientId = atticCfg.otel.clientId;
   forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}/${machine}";
   forwarderUnit = "swarm-journal-forwarder-oidc";
-  # In the host netns: distinct from every other collector's there.
+  # Loopback ports of the forwarder's own metrics and of its OTLP/HTTP metrics
+  # receiver, which the cache-size exporter pushes to, in the container's own
+  # netns.
   forwarderTelemetryPort = 8903;
-  # The forwarder's OTLP/HTTP metrics receiver, which the cache-size exporter
-  # pushes to. Not 4318: the hive collector holds that in the host netns.
   forwarderOtlpPort = 8905;
 
   # The exporter's unit and query. `V` is attic's stored value for a valid
@@ -144,6 +156,20 @@ let
   };
 in
 {
+  # The tokens unit dials the gateway's name, whose leaf the system store lacks
+  # when the gateway serves a self-signed one. `hostUnit`: it reads the host's
+  # CA path and waits on `hive-tls-ca.service`. `enable`: `imports` is
+  # unconditional, so without it a host that runs no cache gets a bundle unit.
+  imports = [
+    (caTrust.trustBundle {
+      inherit pkgs;
+      name = "swarm-bao-attic-tokens";
+      consumers = [ "swarm-bao-attic-tokens" ];
+      hostUnit = true;
+      inherit (cfg) enable;
+    })
+  ];
+
   options.services.hyperhive.deploy.attic = {
     package = lib.mkOption {
       type = lib.types.package;
@@ -163,11 +189,9 @@ in
       type = lib.types.port;
       default = 8910;
       description = ''
-        Loopback port atticd listens on, which the gateway proxies to.
+        Port atticd listens on, on the loopback of the container's own network
-
+        namespace. The container's relay forwards the gateway's unix socket to
-        ⚠️ Every swarm container shares the host's network namespace, so this
+        it; nothing else dials it.
-        is a swarm-wide claim: two modules picking the same number collide at
-        runtime with no bind error. No repo-wide check catches it.
       '';
     };
 
@@ -276,6 +300,9 @@ in
     services.hyperhive.gateway.enable = lib.mkDefault true;
     services.hyperhive.gateway.dns.enable = lib.mkDefault true;
 
+    # The hive bridge carries the container's DHCP lease and its resolver.
+    services.hyperhive.network.enable = lib.mkDefault true;
+
     # `forceSSL`: the tokens ride in a header. No `auth_request`: attic checks
     # its own tokens, and a nix client cannot follow a login redirect. A store
     # path is uploaded as one request that can run for minutes, so the body is
@@ -283,7 +310,7 @@ in
     services.hyperhive.gateway.vhosts."${atticCfg.domain}" = {
       forceSSL = true;
       locations."/" = {
-        proxyPass = "http://127.0.0.1:${toString cfg.port}";
+        proxyPass = "http://unix:${socketPath}";
         extraConfig = ''
           client_max_body_size 0;
           proxy_request_buffering off;
@@ -295,8 +322,12 @@ in
 
     # The key's directory is the container's read-only bind source, so it has
     # to exist before the container starts even when the store does not answer.
-    # The delivery unit creates it too, which covers a switch.
+    # The delivery unit creates it too, which covers a switch. The socket
-    systemd.tmpfiles.rules = [ "d ${secretDir} 0700 root root -" ];
+    # directory is the gateway's way in: its group is nginx's.
+    systemd.tmpfiles.rules = [
+      "d ${secretDir} 0700 root root -"
+      "d ${socketDir} 0770 root ${toString nginxGid} - -"
+    ];
 
     # The signing key, created in the store if it holds none and written where
     # the container reads it. Before the container and wanted by it: a store
@@ -344,9 +375,12 @@ in
     # facts are for the hives, which the controller copies them to.
     systemd.services.swarm-bao-attic-tokens = {
       description = "create the swarm cache, and its tokens and public facts in the secret store, if absent";
+      # `nginx.service`: the unit dials the cache through the gateway, whose
+      # resolver answers the name with the bridge address (`localNames` above).
       after = [
         "container@${machine}.service"
         "swarm-bao-attic-secret.service"
+        "nginx.service"
       ];
       wants = [ "container@${machine}.service" ];
       wantedBy = [ "multi-user.target" ];
@@ -383,12 +417,20 @@ in
       extraFlags = lib.mapAttrsToList (
         id: source: "--load-credential=${id}:${source}"
       ) forwarderStoreFiles;
-      privateNetwork = false;
+      privateNetwork = true;
+      hostBridge = networkCfg.bridgeName;
       bindMounts = {
         ${secretDir} = {
           hostPath = secretDir;
           isReadOnly = true;
         };
+        # A bind mount, not a path both sides know: a container's `/run` is an
+        # nspawn tmpfs the host cannot see into. Connecting to a socket is a
+        # write to it.
+        ${socketDir} = {
+          hostPath = socketDir;
+          isReadOnly = false;
+        };
       }
       // caTrust.bindMount;
 
@@ -416,7 +458,41 @@ in
             })
           ];
 
-          services.hyperhive.swarmContainer.privateNetwork = false;
+          services.hyperhive.swarmContainer.privateNetwork = true;
+
+          # The bridge address comes from the gateway's DHCP. `swarm-bridge-dns`
+          # owns resolv.conf, so dhcpcd must not write it over that unit.
+          networking.interfaces.eth0.useDHCP = true;
+          networking.dhcpcd.extraConfig = "nohook resolv.conf";
+
+          # The socket's group is nginx's, so the gateway can connect to it.
+          users.groups.${socketGroup}.gid = nginxGid;
+          systemd.sockets.${relayUnit} = {
+            description = "unix socket the gateway reaches atticd through";
+            wantedBy = [ "sockets.target" ];
+            listenStreams = [ socketPath ];
+            socketConfig = {
+              SocketMode = "0660";
+              SocketUser = "root";
+              SocketGroup = socketGroup;
+            };
+          };
+          systemd.services.${relayUnit} = {
+            description = "relay the gateway's connections to atticd";
+            requires = [ "${relayUnit}.socket" ];
+            after = [ "${relayUnit}.socket" ];
+            serviceConfig = {
+              ExecStart = "${pkgs.systemd}/lib/systemd/systemd-socket-proxyd 127.0.0.1:${toString cfg.port}";
+              DynamicUser = true;
+              PrivateTmp = true;
+              PrivateDevices = true;
+              NoNewPrivileges = true;
+              RestrictAddressFamilies = [
+                "AF_INET"
+                "AF_UNIX"
+              ];
+            };
+          };
 
           # Authenticated as the forwarder's own OIDC client, with the cache's
           # store identity: one identity for the whole container. It is a

nix/module-eval/builder.nix

@@ -462,7 +462,7 @@ let
         && lib.elem "swarm-bao-attic-policy.service" atticSecret.after
         && lib.elem "swarm-bao-attic-policy.service" atticTokens.after
         && atticHere.containers.swarm-attic.bindMounts."/run/swarm-attic-secrets".isReadOnly
-        && !atticHere.containers.swarm-attic.privateNetwork
+        && atticHere.containers.swarm-attic.privateNetwork
         && atticUnit.serviceConfig.EnvironmentFile == "/run/swarm-attic-secrets/server.env"
         && lib.hasInfix "--mode monolithic" atticUnit.serviceConfig.ExecStart
         && atticDeploy.retentionPeriod == "14 days";
@@ -534,7 +534,7 @@ let
           names = m: m.services.hyperhive.swarm.serviceDomains;
         in
         atticVhost.forceSSL
-        && atticVhost.locations."/".proxyPass == "http://127.0.0.1:8910"
+        && atticVhost.locations."/".proxyPass == "http://unix:/run/swarm-attic/atticd.sock"
         && !(lib.hasInfix "auth_request" atticVhost.locations."/".extraConfig)
         && lib.hasInfix "client_max_body_size 0" atticVhost.locations."/".extraConfig
         && lib.hasInfix "proxy_request_buffering off" atticVhost.locations."/".extraConfig

nix/module-eval/swarm-services-switch.nix

@@ -174,8 +174,8 @@ let
   # Every service container at once: the services-only host plus the CI
   # runner. The CI runner, the review classifier, the notification router, the
   # builder, the claude login minter, the metrics store, the homeserver,
-  # authelia, Grafana and the collector are the ten containers on a netns of
+  # authelia, Grafana, the collector and the cache are the eleven containers on
-  # their own.
+  # a netns of their own.
   serviceContainersWithCi = hive {
     deploy.hive-controller.enable = false;
     deploy.allSwarmServices = true;
@@ -814,14 +814,14 @@ let
     }
     {
       # Both values come from ../container-modules/swarm-container.nix.
-      # Five of the 15 service containers share the host netns (hive-forge, bao,
+      # Four of the 15 service containers share the host netns (hive-forge, bao,
-      # the queue, the log store, the cache) and ten have one of their own.
+      # the queue, the log store) and eleven have one of their own.
       name = "a service container on the host netns runs no firewall or resolvconf of its own";
       ok =
         let
           shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
         in
-        lib.length shared == 5
+        lib.length shared == 4
         && lib.all (
           c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
         ) shared;
@@ -992,6 +992,46 @@ let
         && lib.all (lib.hasPrefix "http://unix:${sock}:") forwarded
         && c.bindMounts.${dir}.hostPath == dir;
     }
+    {
+      # atticd listens on the container's own loopback, a relay there forwards
+      # the socket nginx connects to, and the host unit that drives the
+      # TCP-only `attic` client dials the published name, not a loopback. The
+      # `cache create` clause and the trust-bundle clauses are controls: they
+      # find something in the same script and unit the absences are claimed of.
+      # `upstreams != []` keeps an empty vhost from passing.
+      name = "the cache runs on a netns of its own, the gateway reaches it over a socket and the host unit dials its public name";
+      ok =
+        let
+          host = serviceContainersWithCi;
+          h = host.services.hyperhive;
+          c = host.containers.swarm-attic;
+          dir = "/run/swarm-attic";
+          sock = "${dir}/atticd.sock";
+          domain = h.swarm.attic.domain;
+          upstreams = map (l: l.proxyPass) (lib.attrValues h.gateway.vhosts.${domain}.locations);
+          relay = c.config.systemd.services.swarm-attic-relay;
+          tokens = host.systemd.services.swarm-bao-attic-tokens;
+        in
+        c.privateNetwork
+        && c.hostBridge == h.network.bridgeName
+        && c.hostAddress == null
+        && c.localAddress == null
+        && c.config.networking.interfaces.eth0.useDHCP
+        && c.config.networking.firewall.enable
+        && upstreams != [ ]
+        && lib.all (p: p == "http://unix:${sock}") upstreams
+        && c.config.systemd.sockets.swarm-attic-relay.listenStreams == [ sock ]
+        && lib.hasSuffix "systemd-socket-proxyd 127.0.0.1:${toString h.deploy.attic.port}" relay.serviceConfig.ExecStart
+        && c.bindMounts.${dir}.hostPath == dir
+        && lib.elem "d ${dir} 0770 root ${toString host.ids.gids.nginx} - -" host.systemd.tmpfiles.rules
+        && lib.hasInfix "cache create" tokens.script
+        && lib.hasInfix "https://${domain}" tokens.script
+        && !(lib.hasInfix "127.0.0.1" tokens.script)
+        && lib.elem "nginx.service" tokens.after
+        && h.gateway.useSelfSigned
+        && tokens.environment.SSL_CERT_FILE == "/run/swarm-bao-attic-tokens-ca/trust-bundle.pem"
+        && lib.elem "swarm-bao-attic-tokens-ca-bundle.service" tokens.requires;
+    }
     {
       # On the bridge by DHCP, with every receiver on a socket: the hive's, the
       # swarm tier's and each service forwarder's. Every gateway location that

nix/script-tests/attic-init.nix

@@ -18,7 +18,7 @@ let
     publicKeyPath = "secret/swarm/services/attic/public-key";
     url = "http://cache.example";
     cache = "grid";
-    endpoint = "http://127.0.0.1:8910";
+    endpoint = "https://cache.example";
     configFile = "@DIR@/server.toml";
     envFile = "@DIR@/secrets/server.env";
     machine = "swarm-attic";

nix/script-tests/attic-init.sh

@@ -95,8 +95,14 @@ chmod +x "$dir/bin/"*
 
 cat >"$dir/bin/curl" <<STUB
 #!$BASH
-echo curl >>"$calls"
+echo "curl \$*" >>"$calls"
-[ "\${CURL_FAIL:-0}" != 1 ]
+# A curl that cannot connect prints 000 and exits 7; CURL_CODE is the status of
+# a server that answered.
+if [ "\${CURL_FAIL:-0}" = 1 ]; then
+  printf 000
+  exit 7
+fi
+printf %s "\${CURL_CODE:-200}"
 STUB
 printf '#!%s\nexit 0\n' "$BASH" >"$dir/bin/sleep"
 # A container that is not running has nothing to refresh.
@@ -246,6 +252,25 @@ expect "exit status" $? 1
 expect "nothing minted" "$(n '^atticadm')" 0
 expect "nothing stored" "$(n '^bao put')" 0
 
+case="a 502 from the gateway is not atticd answering"
+reset
+run "$ENSURE_JWT"
+: >"$calls"
+run "$PROVISION" CURL_CODE=502
+expect "exit status" $? 1
+expect "nothing minted" "$(n '^atticadm')" 0
+expect "nothing stored" "$(n '^bao put')" 0
+expect "the probe asked the gateway's name" "$(n '^curl .*https://cache.example')" 30
+
+case="a status from atticd below 500 is an answer, a 404 included"
+reset
+run "$ENSURE_JWT"
+: >"$calls"
+run "$PROVISION" CURL_CODE=404
+expect "exit status" $? 0
+expect "the cache is created" "$(n '^attic cache create')" 1
+expect "one probe" "$(n '^curl')" 1
+
 case="the client reads its token from a file, not from an argument"
 reset
 run "$ENSURE_JWT"