From 48c64511fe276f65a781a2ae159788a719d4921a Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 16 Aug 2026 20:29:29 +0200 Subject: [PATCH] fix(#3349): let the controller survive a queue credential that is not there yet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A systemd credential named by an absolute path is fatal when the file is missing, and the co-located queue secret is minted by authelia's first boot in another container — which a host unit cannot order against. On a real boot the daemon spent three of systemd's five default starts losing that race before the file appeared; two seconds more would have reached start-limit-hit, which does not self-heal. An empty SetCredential acts as the default that makes the load non-fatal, so the controller starts and serves its HTTP surface with the queue unconfigured — a shape it already reports. That alone would be a regression, though: a credential is snapshotted at unit start, so the placeholder would freeze and the daemon would sit degraded forever instead of recovering the way the restart loop accidentally did. A path unit on the secret file closes it, and closes a second gap in the same stroke — mint_token reads the secret on every call precisely so a rotation takes effect without a restart, and a snapshot in %d silently defeats that. PathChanged and not PathExists: the latter activates immediately when the file is already present at unit start, which would restart a healthy daemon on every boot. The option being consumed already warned about this shape — its own description says a consumer has to wait for the secret because a missing source turns a fresh hive into a boot-order deadlock. That warning was written on the producing side and did not fire while I was writing the consumer. --- nix/host-modules/swarm-controller.nix | 65 +++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index 48e627b6..4f6ff103 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -418,10 +418,31 @@ in # off here" case left for a `mkIf` to express. The forge token # stays optional: a controller with no forge access still serves # its HTTP surface, and that IS a supported shape. + # + # ⚠️ "the path is a value" is not "the file is on disk". The + # co-located secret is minted by authelia's FIRST BOOT, in another + # container, and `hostClientSecretDir`'s own description says a + # consumer has to wait for it. A `LoadCredential=` naming an + # absolute path that is not there yet is fatal (`243/CREDENTIALS`), + # so the daemon spent three of systemd's five default starts losing + # that race on a real boot — two seconds more and it would have hit + # `start-limit-hit`, which does not self-heal. LoadCredential = [ "queue-client.secret:${cfg.queue.clientSecretFile}" ] ++ lib.optional (cfg.forgeTokenFile != null) "forge-token:${cfg.forgeTokenFile}"; + + # The empty default that makes the above non-fatal. `LoadCredential=` + # takes priority over `SetCredential=`, so this is only ever seen + # when the file is missing — and in that case systemd starts the unit + # instead of refusing to. The controller then serves its HTTP surface + # with the queue unconfigured, which is a supported shape it already + # knows how to report. + # + # Safe to put in a unit file precisely because it carries nothing: + # `SetCredential=` values are readable by unprivileged processes over + # IPC, so a real secret must never appear here. + SetCredential = [ "queue-client.secret:" ]; User = "swarm-controller"; Group = "swarm-controller"; Restart = "on-failure"; @@ -495,5 +516,49 @@ in // queueEnv // forgeEnv; }; + + # A systemd credential is a SNAPSHOT: it is materialised into `%d` once, + # at unit start, and never re-read. That is invisible until the file + # underneath it changes — and two ordinary things change it. + # + # - it ARRIVES LATE. The co-located secret is minted by authelia's + # first boot, in another container, which a host unit cannot order + # against. Before this, the unit died at `243/CREDENTIALS` and was + # rescued only by burning restarts until the file showed up. + # - it is ROTATED. `mint_token` deliberately reads the secret file on + # every call so a rotation takes effect without a restart — a + # snapshot in `%d` quietly defeats that, and nothing reports it. + # + # Watching the file closes both: on close-after-write, restart the + # daemon so it re-snapshots. `PathChanged=` and not `PathExists=`, + # measured against the semantics rather than guessed — `PathExists=` + # activates immediately whenever the file is *already there* at unit + # start, which would restart a perfectly healthy daemon on every boot. + # `PathChanged=` requires a write, so it cannot do that and cannot spin. + # + # ⚠️ Known gap, stated rather than papered over: a secret appearing in + # the sub-second window between the daemon starting and this unit + # watching is missed until the next write. Closing it needs + # `PathExists=`, whose cost is the spurious per-boot restart above. + systemd.paths.swarm-controller-credential = { + description = "watch the swarm controller's queue credential"; + wantedBy = [ "multi-user.target" ]; + pathConfig = { + PathChanged = cfg.queue.clientSecretFile; + Unit = "swarm-controller-credential.service"; + }; + }; + + # `try-restart`, not `restart`: if the daemon is stopped — masked, + # disabled, or deliberately down — a secret rotation is not a reason to + # start it. Rotating a credential should never be how a service comes + # back to life. + systemd.services.swarm-controller-credential = { + description = "restart the swarm controller after its queue credential changed"; + serviceConfig = { + Type = "oneshot"; + ExecStart = "${pkgs.systemd}/bin/systemctl try-restart swarm-controller.service"; + }; + }; }; }