swarm-bao: keep the store forwarder's OIDC secret on the host's /run, not in the container's root

swarm-bao-forwarder-oidc still reads the secret from bao under the
forwarder's own leaf, but now writes it to /run/swarm-bao-forwarder-oidc on
the host (tmpfs), which the swarm-bao container bind-mounts read-only for the
collector's LoadCredential. Before, it wrote it to
/var/lib/nixos-containers/swarm-bao/var/lib/swarm-bao-otel-oidc/<id>.secret on
persistent disk, where a superseded secret stayed after rotation and went
into backups of the container root. Host tmpfiles now removes that old file.

Rotation is unchanged: the ten-minute timer rewrites the file when the value
differs and refresh_consumer restarts the forwarder. After a reboot the
secret is fetched again; the file is not pre-created, so until the store
answers the forwarder stays down (LoadCredential refuses a missing source) and
resumes from its journal cursor afterwards. The container now starts after
systemd-tmpfiles-resetup, since the new bind source is made there.

Part of #4998 (slice 3: forwarder).

docs/swarm/secrets.md

@@ -112,7 +112,7 @@ The **secret store's own** collector — the forwarder inside the `swarm-bao`
 container — needs a delivery step too, and it takes the same route with one
 principal of its own: `swarm-bao-forwarder-oidc.service` reads
 `swarm/services/<client-id>/oidc/client` under a leaf of its own and
-lands it in the container's tree, where `LoadCredential` hands it to the
+writes it to `/run/swarm-bao-forwarder-oidc` on the host's tmpfs, which the container bind-mounts read-only and where `LoadCredential` hands it to the
 collector. The client id is its own
 (`services.hyperhive.swarm.bao.otel.clientId`), registered by
 `glue-swarm-bao-otel-oidc-client.nix` wherever authelia runs, and the swarm
@@ -123,11 +123,11 @@ this forwarder borrows neither the swarm collector's client nor
 ⚠️ **This one has no degraded mode, and the difference is structural.** The
 route it exports over accepts an authenticated request and nothing else, so a
 forwarder without the secret has nothing to fall back to: the unit fails and
-retries, and the collector refuses to start until the file arrives rather than
+retries, and the collector refuses to start until the file arrives, after every boot, rather than
 exporting into a 401 while reporting healthy. Its ordering differs from the
 unit above it for the same structural reason — it reads the store inside the
 container it delivers into, so it runs **after** that container rather than
-before it, and only a first boot ever waits.
+before it. The fetch creates the file, so when the store stays sealed or unreachable after a boot the forwarder stays down, then resumes from its saved journal cursor once the store answers.
 
 ### The queue's callout nkeys
 

nix/host-modules/swarm-bao.nix

@@ -1401,12 +1401,17 @@ let
   # `swarm-forwarder-oidc` role's grant is this one path, so any other answers
   # 403 however correct it looks.
   forwarderStoreSecretPath = "secret/swarm/services/${cfg.otel.clientId}/oidc/client";
-  # At rest in the container's tree, written by the host unit below — under
+  # On the host's /run (tmpfs), so the secret never reaches persistent disk:
-  # /var/lib rather than /run, because a secret that evaporates on reboot
+  # after a reboot `swarm-bao-forwarder-oidc` fetches it again. The container
-  # turns a working export into an intermittent one.
+  # sees the directory read-only at the same path. The file is NOT created
-  forwarderSecretInContainer = "/var/lib/swarm-bao-otel-oidc/${cfg.otel.clientId}.secret";
+  # ahead of that fetch: `LoadCredential` refuses a missing source, so until
-  forwarderHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${forwarderSecretInContainer}";
+  # the store answers the forwarder stays down and resumes from its saved
-  forwarderHostSecretDir = builtins.dirOf forwarderHostSecretPath;
+  # read cursor. A placeholder file would start it, and its refused pushes
+  # would be dropped once the exporter's retry window ends.
+  forwarderSecretDir = "/run/swarm-bao-forwarder-oidc";
+  forwarderSecret = "${forwarderSecretDir}/${cfg.otel.clientId}.secret";
+  # Where the secret sat on persistent disk before it moved to /run.
+  forwarderLegacyHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}/var/lib/swarm-bao-otel-oidc/${cfg.otel.clientId}.secret";
 
   atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
   refreshConsumer = import ./lib/refresh-consumer.nix { };
@@ -2738,8 +2743,9 @@ in
       # THE delivery unit for the forwarder's own OIDC client secret, copied
       # in shape from ./swarm-otel.nix's `swarm-bao-otel-oidc`: a cert login
       # that fails LOUDLY, since every state it fails on is one a retry fixes,
-      # then a read, then a root-owned file in the container's tree that
+      # then a read, then a root-owned file on the host's /run that the
-      # `LoadCredential` re-exposes to the collector's dynamic user.
+      # container bind-mounts read-only and `LoadCredential` re-exposes to the
+      # collector's dynamic user.
       #
       # ⚠️ ORDERED AFTER the store's container and deliberately NOT before it,
       # which is the one line that differs from the unit it copies. That one
@@ -2748,12 +2754,11 @@ in
       # process that cannot start until this finishes. There is no bootstrap
       # cycle behind it — the identity used here is the static
       # `forwarder-oidc.pem` leaf from `swarm-bao-pki`, not anything the store
-      # issues — only an ordering one, and the cost is bounded: the file is
+      # issues — only an ordering one. The file is on tmpfs, so EVERY boot has
-      # under /var/lib, so it survives reboots and only a FIRST boot has the
+      # the collector starting before it exists: `LoadCredential` refuses to
-      # collector starting before it exists. `LoadCredential` refuses to start
+      # start a unit whose source is missing, so the collector restarts, says
-      # a unit whose source is missing, so that boot is a collector that
+      # so each time, and comes up the moment this lands. Loud and
-      # restarts, says so each time, and comes up the moment this lands. Loud
+      # self-healing rather than silently exporting without a credential.
-      # and self-healing rather than silently exporting without a credential.
       systemd.services.swarm-bao-forwarder-oidc = {
         description = "fetch the secret store forwarder's OIDC client secret from the store";
         after = [
@@ -2837,18 +2842,25 @@ in
           # a program: `printf` is bash's own, so the plaintext never becomes
           # an argument in /proc the way `install <<<"$secret"` or an `echo`
           # from `path` would.
-          install -d -m 0755 ${lib.escapeShellArg forwarderHostSecretDir}
+          install -d -m 0700 ${lib.escapeShellArg forwarderSecretDir}
-          if secret_differs ${lib.escapeShellArg forwarderHostSecretPath} "$secret"; then
+          if secret_differs ${lib.escapeShellArg forwarderSecret} "$secret"; then
-            atomic_write_secret 0400 root:root ${lib.escapeShellArg forwarderHostSecretPath} "$secret"
+            atomic_write_secret 0400 root:root ${lib.escapeShellArg forwarderSecret} "$secret"
           fi
 
           # `LoadCredential` copies the file at start only: a rotated secret
           # reaches the forwarder by restarting it.
-          refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg forwarderHostSecretPath}
+          refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg forwarderSecret}
         '';
       };
 
       systemd.timers.swarm-bao-forwarder-oidc = storeRefreshTimer "re-read the secret store forwarder's OIDC client secret from the store";
+
+      systemd.tmpfiles.rules = [
+        # The container's bind source, so it exists when nspawn sets the mount
+        # up. Not the file: see `forwarderSecretDir`.
+        "d ${forwarderSecretDir} 0700 root root -"
+        "r ${forwarderLegacyHostSecretPath}"
+      ];
     })
 
     (lib.mkIf (deployCfg.bao.enable && haveServerTls) {
@@ -3721,7 +3733,15 @@ in
       # The CA bind source is written at runtime by a host unit, so the
       # container has to start after it — otherwise nspawn sets up a mount
       # over a file that does not exist yet.
-      systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;
+      #
+      # The forwarder's secret directory is a bind source too, made by
+      # tmpfiles; on a switch the container restarts in the same transaction
+      # as `systemd-tmpfiles-resetup`, so it starts after it. A container that
+      # cannot start is a store that is down.
+      systemd.services."container@${cfg.machine}" = lib.mkMerge [
+        caTrust.containerOrdering
+        { after = [ "systemd-tmpfiles-resetup.service" ]; }
+      ];
 
       containers.${cfg.machine} = {
         autoStart = true;
@@ -3734,6 +3754,13 @@ in
         # /var, systemd owns `${stateDir}` through `StateDirectory=`, and
         # binding over it is what breaks the unit.
         bindMounts = {
+          # The forwarder's OIDC secret, written by `swarm-bao-forwarder-oidc`
+          # on the host's tmpfs; read-only because the container only loads it.
+          ${forwarderSecretDir} = {
+            hostPath = forwarderSecretDir;
+            isReadOnly = true;
+          };
+
           # Read-only: host units write it (`swarm-bao-certs`, and
           # `swarm-bao-agent-pki` for the agent CA and the listener bundle), and
           # the store has no reason to modify its own identity.
@@ -4230,17 +4257,18 @@ in
 
             # How the secret crosses the sandbox. The host unit
             # `swarm-bao-forwarder-oidc` leaves a root-owned 0400 file at this
-            # path in the container's tree; systemd reads it as root before
+            # path on the host's /run, which the container bind-mounts
-            # the sandbox exists and re-exposes it to whichever uid
+            # read-only; systemd reads it as root before the sandbox exists
-            # `DynamicUser` picked, under the directory the extension above
+            # and re-exposes it to whichever uid `DynamicUser` picked, under
-            # names through `${env:CREDENTIALS_DIRECTORY}`.
+            # the directory the extension above names through
+            # `${env:CREDENTIALS_DIRECTORY}`.
             #
             # ⚠️ Unconditional, and a missing source is a unit that refuses to
             # start. That is the intended shape rather than an oversight: this
             # forwarder has one way to deliver, so a boot where the secret has
             # not arrived yet is a collector that says so and retries, not one
             # that quietly exports into a 401. The restart is what closes the
-            # first-boot window the host unit's own comment describes.
+            # post-boot window the host unit's own comment describes.
             #
             # ⚠️ `Restart` is deliberately NOT set here, same as the sibling
             # collector in ./swarm-otel.nix: nixpkgs' own
@@ -4254,7 +4282,7 @@ in
             # is still delivering the secret, leaving it failed rather than
             # retrying.
             systemd.services.opentelemetry-collector.serviceConfig = {
-              LoadCredential = [ "${forwarderCredentialId}:${forwarderSecretInContainer}" ];
+              LoadCredential = [ "${forwarderCredentialId}:${forwarderSecret}" ];
               RestartSec = 15;
             };
           };

nix/module-eval/bao-otel-collector.nix

@@ -368,9 +368,32 @@ let
         e.client_secret_file == "\${env:CREDENTIALS_DIRECTORY}/oidc-client-secret"
         &&
           unit.LoadCredential == [
-            "oidc-client-secret:/var/lib/swarm-bao-otel-oidc/swarm-bao-collector.secret"
+            "oidc-client-secret:/run/swarm-bao-forwarder-oidc/swarm-bao-collector.secret"
           ];
     }
+    {
+      # A copy on persistent disk outlives the secret's rotation and lands in
+      # backups of the container's root. The file is deliberately not created
+      # ahead of the fetch: a placeholder would start the forwarder and have
+      # its pushes refused instead of leaving it down until the store answers.
+      name = "the forwarder's client secret is written to the host's /run only, and the old copy removed";
+      ok =
+        let
+          m = baoWithCollector;
+          script = m.systemd.services.swarm-bao-forwarder-oidc.script;
+          mount = m.containers.swarm-bao.bindMounts."/run/swarm-bao-forwarder-oidc";
+          rules = m.systemd.tmpfiles.rules;
+        in
+        lib.hasInfix "atomic_write_secret 0400 root:root /run/swarm-bao-forwarder-oidc/swarm-bao-collector.secret" script
+        && !(lib.hasInfix "/var/lib/nixos-containers" script)
+        && !(lib.hasInfix "/var/lib/swarm-bao-otel-oidc" script)
+        && mount.hostPath == "/run/swarm-bao-forwarder-oidc"
+        && mount.isReadOnly
+        && lib.elem "d /run/swarm-bao-forwarder-oidc 0700 root root -" rules
+        && lib.elem "r /var/lib/nixos-containers/swarm-bao/var/lib/swarm-bao-otel-oidc/swarm-bao-collector.secret" rules
+        && !(lib.any (r: lib.hasPrefix "f /run/swarm-bao-forwarder-oidc" r) rules)
+        && lib.elem "systemd-tmpfiles-resetup.service" m.systemd.services."container@swarm-bao".after;
+    }
     {
       # Same 403-not-a-miss property the grafana and matrix readers are pinned
       # for: the reader's grant is one path under `services`, so a secret