From 6f5bd1dc5df6cc07792a3bc775c69241ec3024f0 Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 15 Oct 2026 22:36:28 +0200 Subject: [PATCH] swarm-bao: keep the store forwarder's OIDC secret on the host's /run, not in the container's root swarm-bao-forwarder-oidc still reads the secret from bao under the forwarder's own leaf, but now writes it to /run/swarm-bao-forwarder-oidc on the host (tmpfs), which the swarm-bao container bind-mounts read-only for the collector's LoadCredential. Before, it wrote it to /var/lib/nixos-containers/swarm-bao/var/lib/swarm-bao-otel-oidc/.secret on persistent disk, where a superseded secret stayed after rotation and went into backups of the container root. Host tmpfiles now removes that old file. Rotation is unchanged: the ten-minute timer rewrites the file when the value differs and refresh_consumer restarts the forwarder. After a reboot the secret is fetched again; the file is not pre-created, so until the store answers the forwarder stays down (LoadCredential refuses a missing source) and resumes from its journal cursor afterwards. The container now starts after systemd-tmpfiles-resetup, since the new bind source is made there. Part of #4998 (slice 3: forwarder). --- docs/swarm/secrets.md | 6 +- nix/host-modules/swarm-bao.nix | 78 +++++++++++++++++--------- nix/module-eval/bao-otel-collector.nix | 25 ++++++++- 3 files changed, 80 insertions(+), 29 deletions(-) diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 6ed124d8..f33738d5 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -112,7 +112,7 @@ The **secret store's own** collector — the forwarder inside the `swarm-bao` container — needs a delivery step too, and it takes the same route with one principal of its own: `swarm-bao-forwarder-oidc.service` reads `swarm/services//oidc/client` under a leaf of its own and -lands it in the container's tree, where `LoadCredential` hands it to the +writes it to `/run/swarm-bao-forwarder-oidc` on the host's tmpfs, which the container bind-mounts read-only and where `LoadCredential` hands it to the collector. The client id is its own (`services.hyperhive.swarm.bao.otel.clientId`), registered by `glue-swarm-bao-otel-oidc-client.nix` wherever authelia runs, and the swarm @@ -123,11 +123,11 @@ this forwarder borrows neither the swarm collector's client nor ⚠️ **This one has no degraded mode, and the difference is structural.** The route it exports over accepts an authenticated request and nothing else, so a forwarder without the secret has nothing to fall back to: the unit fails and -retries, and the collector refuses to start until the file arrives rather than +retries, and the collector refuses to start until the file arrives, after every boot, rather than exporting into a 401 while reporting healthy. Its ordering differs from the unit above it for the same structural reason — it reads the store inside the container it delivers into, so it runs **after** that container rather than -before it, and only a first boot ever waits. +before it. The fetch creates the file, so when the store stays sealed or unreachable after a boot the forwarder stays down, then resumes from its saved journal cursor once the store answers. ### The queue's callout nkeys diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 0754e0e5..7c404943 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -1401,12 +1401,17 @@ let # `swarm-forwarder-oidc` role's grant is this one path, so any other answers # 403 however correct it looks. forwarderStoreSecretPath = "secret/swarm/services/${cfg.otel.clientId}/oidc/client"; - # At rest in the container's tree, written by the host unit below — under - # /var/lib rather than /run, because a secret that evaporates on reboot - # turns a working export into an intermittent one. - forwarderSecretInContainer = "/var/lib/swarm-bao-otel-oidc/${cfg.otel.clientId}.secret"; - forwarderHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${forwarderSecretInContainer}"; - forwarderHostSecretDir = builtins.dirOf forwarderHostSecretPath; + # On the host's /run (tmpfs), so the secret never reaches persistent disk: + # after a reboot `swarm-bao-forwarder-oidc` fetches it again. The container + # sees the directory read-only at the same path. The file is NOT created + # ahead of that fetch: `LoadCredential` refuses a missing source, so until + # the store answers the forwarder stays down and resumes from its saved + # read cursor. A placeholder file would start it, and its refused pushes + # would be dropped once the exporter's retry window ends. + forwarderSecretDir = "/run/swarm-bao-forwarder-oidc"; + forwarderSecret = "${forwarderSecretDir}/${cfg.otel.clientId}.secret"; + # Where the secret sat on persistent disk before it moved to /run. + forwarderLegacyHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}/var/lib/swarm-bao-otel-oidc/${cfg.otel.clientId}.secret"; atomicWriteSecret = import ./lib/atomic-write-secret.nix { }; refreshConsumer = import ./lib/refresh-consumer.nix { }; @@ -2738,8 +2743,9 @@ in # THE delivery unit for the forwarder's own OIDC client secret, copied # in shape from ./swarm-otel.nix's `swarm-bao-otel-oidc`: a cert login # that fails LOUDLY, since every state it fails on is one a retry fixes, - # then a read, then a root-owned file in the container's tree that - # `LoadCredential` re-exposes to the collector's dynamic user. + # then a read, then a root-owned file on the host's /run that the + # container bind-mounts read-only and `LoadCredential` re-exposes to the + # collector's dynamic user. # # ⚠️ ORDERED AFTER the store's container and deliberately NOT before it, # which is the one line that differs from the unit it copies. That one @@ -2748,12 +2754,11 @@ in # process that cannot start until this finishes. There is no bootstrap # cycle behind it — the identity used here is the static # `forwarder-oidc.pem` leaf from `swarm-bao-pki`, not anything the store - # issues — only an ordering one, and the cost is bounded: the file is - # under /var/lib, so it survives reboots and only a FIRST boot has the - # collector starting before it exists. `LoadCredential` refuses to start - # a unit whose source is missing, so that boot is a collector that - # restarts, says so each time, and comes up the moment this lands. Loud - # and self-healing rather than silently exporting without a credential. + # issues — only an ordering one. The file is on tmpfs, so EVERY boot has + # the collector starting before it exists: `LoadCredential` refuses to + # start a unit whose source is missing, so the collector restarts, says + # so each time, and comes up the moment this lands. Loud and + # self-healing rather than silently exporting without a credential. systemd.services.swarm-bao-forwarder-oidc = { description = "fetch the secret store forwarder's OIDC client secret from the store"; after = [ @@ -2837,18 +2842,25 @@ in # a program: `printf` is bash's own, so the plaintext never becomes # an argument in /proc the way `install <<<"$secret"` or an `echo` # from `path` would. - install -d -m 0755 ${lib.escapeShellArg forwarderHostSecretDir} - if secret_differs ${lib.escapeShellArg forwarderHostSecretPath} "$secret"; then - atomic_write_secret 0400 root:root ${lib.escapeShellArg forwarderHostSecretPath} "$secret" + install -d -m 0700 ${lib.escapeShellArg forwarderSecretDir} + if secret_differs ${lib.escapeShellArg forwarderSecret} "$secret"; then + atomic_write_secret 0400 root:root ${lib.escapeShellArg forwarderSecret} "$secret" fi # `LoadCredential` copies the file at start only: a rotated secret # reaches the forwarder by restarting it. - refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg forwarderHostSecretPath} + refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg forwarderSecret} ''; }; systemd.timers.swarm-bao-forwarder-oidc = storeRefreshTimer "re-read the secret store forwarder's OIDC client secret from the store"; + + systemd.tmpfiles.rules = [ + # The container's bind source, so it exists when nspawn sets the mount + # up. Not the file: see `forwarderSecretDir`. + "d ${forwarderSecretDir} 0700 root root -" + "r ${forwarderLegacyHostSecretPath}" + ]; }) (lib.mkIf (deployCfg.bao.enable && haveServerTls) { @@ -3721,7 +3733,15 @@ in # The CA bind source is written at runtime by a host unit, so the # container has to start after it — otherwise nspawn sets up a mount # over a file that does not exist yet. - systemd.services."container@${cfg.machine}" = caTrust.containerOrdering; + # + # The forwarder's secret directory is a bind source too, made by + # tmpfiles; on a switch the container restarts in the same transaction + # as `systemd-tmpfiles-resetup`, so it starts after it. A container that + # cannot start is a store that is down. + systemd.services."container@${cfg.machine}" = lib.mkMerge [ + caTrust.containerOrdering + { after = [ "systemd-tmpfiles-resetup.service" ]; } + ]; containers.${cfg.machine} = { autoStart = true; @@ -3734,6 +3754,13 @@ in # /var, systemd owns `${stateDir}` through `StateDirectory=`, and # binding over it is what breaks the unit. bindMounts = { + # The forwarder's OIDC secret, written by `swarm-bao-forwarder-oidc` + # on the host's tmpfs; read-only because the container only loads it. + ${forwarderSecretDir} = { + hostPath = forwarderSecretDir; + isReadOnly = true; + }; + # Read-only: host units write it (`swarm-bao-certs`, and # `swarm-bao-agent-pki` for the agent CA and the listener bundle), and # the store has no reason to modify its own identity. @@ -4230,17 +4257,18 @@ in # How the secret crosses the sandbox. The host unit # `swarm-bao-forwarder-oidc` leaves a root-owned 0400 file at this - # path in the container's tree; systemd reads it as root before - # the sandbox exists and re-exposes it to whichever uid - # `DynamicUser` picked, under the directory the extension above - # names through `${env:CREDENTIALS_DIRECTORY}`. + # path on the host's /run, which the container bind-mounts + # read-only; systemd reads it as root before the sandbox exists + # and re-exposes it to whichever uid `DynamicUser` picked, under + # the directory the extension above names through + # `${env:CREDENTIALS_DIRECTORY}`. # # ⚠️ Unconditional, and a missing source is a unit that refuses to # start. That is the intended shape rather than an oversight: this # forwarder has one way to deliver, so a boot where the secret has # not arrived yet is a collector that says so and retries, not one # that quietly exports into a 401. The restart is what closes the - # first-boot window the host unit's own comment describes. + # post-boot window the host unit's own comment describes. # # ⚠️ `Restart` is deliberately NOT set here, same as the sibling # collector in ./swarm-otel.nix: nixpkgs' own @@ -4254,7 +4282,7 @@ in # is still delivering the secret, leaving it failed rather than # retrying. systemd.services.opentelemetry-collector.serviceConfig = { - LoadCredential = [ "${forwarderCredentialId}:${forwarderSecretInContainer}" ]; + LoadCredential = [ "${forwarderCredentialId}:${forwarderSecret}" ]; RestartSec = 15; }; }; diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix index d255a884..453ab073 100644 --- a/nix/module-eval/bao-otel-collector.nix +++ b/nix/module-eval/bao-otel-collector.nix @@ -368,9 +368,32 @@ let e.client_secret_file == "\${env:CREDENTIALS_DIRECTORY}/oidc-client-secret" && unit.LoadCredential == [ - "oidc-client-secret:/var/lib/swarm-bao-otel-oidc/swarm-bao-collector.secret" + "oidc-client-secret:/run/swarm-bao-forwarder-oidc/swarm-bao-collector.secret" ]; } + { + # A copy on persistent disk outlives the secret's rotation and lands in + # backups of the container's root. The file is deliberately not created + # ahead of the fetch: a placeholder would start the forwarder and have + # its pushes refused instead of leaving it down until the store answers. + name = "the forwarder's client secret is written to the host's /run only, and the old copy removed"; + ok = + let + m = baoWithCollector; + script = m.systemd.services.swarm-bao-forwarder-oidc.script; + mount = m.containers.swarm-bao.bindMounts."/run/swarm-bao-forwarder-oidc"; + rules = m.systemd.tmpfiles.rules; + in + lib.hasInfix "atomic_write_secret 0400 root:root /run/swarm-bao-forwarder-oidc/swarm-bao-collector.secret" script + && !(lib.hasInfix "/var/lib/nixos-containers" script) + && !(lib.hasInfix "/var/lib/swarm-bao-otel-oidc" script) + && mount.hostPath == "/run/swarm-bao-forwarder-oidc" + && mount.isReadOnly + && lib.elem "d /run/swarm-bao-forwarder-oidc 0700 root root -" rules + && lib.elem "r /var/lib/nixos-containers/swarm-bao/var/lib/swarm-bao-otel-oidc/swarm-bao-collector.secret" rules + && !(lib.any (r: lib.hasPrefix "f /run/swarm-bao-forwarder-oidc" r) rules) + && lib.elem "systemd-tmpfiles-resetup.service" m.systemd.services."container@swarm-bao".after; + } { # Same 403-not-a-miss property the grafana and matrix readers are pinned # for: the reader's grant is one path under `services`, so a secret