diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 7895e78c..6249fc4f 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -557,11 +557,27 @@ in # has run. `requires` as well as `after`: if minting fails there is # nothing to deliver, and a copy that silently succeeds with a stale # or absent seed is worse than not running. - after = lib.optional cfg.autoGenerateCallout "swarm-nats-callout-keys.service"; + after = + lib.optional cfg.autoGenerateCallout "swarm-nats-callout-keys.service" + # The third credential does not come from the generator above — it is + # minted by authelia's FIRST BOOT, inside its own container. Ordering + # after that container is necessary and NOT sufficient: the container + # being up says nothing about whether its in-container secrets unit + # has finished. The wait in the script is what actually closes it; + # this only stops us spinning for the full timeout on every boot. + ++ lib.optional autheliaCfg.enable "container@${autheliaCfg.machine}.service"; requires = lib.optional cfg.autoGenerateCallout "swarm-nats-callout-keys.service"; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; + # Must exceed the script's own wait below, and is set rather than + # left to the default for exactly that reason: systemd's + # `DefaultTimeoutStartSec` is 90s, so a 120s wait would be killed at + # 90 and the operator would get a generic unit timeout instead of + # the message that names the missing file and says what to do. + # The bound that matters belongs in one place, and this makes the + # two visibly related. + TimeoutStartSec = "180s"; SyslogIdentifier = "swarm-nats-auth-secrets"; }; path = [ pkgs.coreutils ]; @@ -572,7 +588,31 @@ in ${lib.escapeShellArg (hostPath "callout-user.seed")} install -m 0400 ${lib.escapeShellArg issuerSeedFile} \ ${lib.escapeShellArg (hostPath "issuer.seed")} - install -m 0400 ${lib.escapeShellArg clientSecretSource} \ + # Wait for authelia's minted secret rather than failing the instant + # it is absent. On a fresh boot this unit and authelia's first-boot + # generator race, and losing that race used to cost the WHOLE QUEUE: + # this exits 1, the responder never starts, and `auth_callout` with + # no responder refuses every client — fail-closed by design, so the + # symptom appears on every queue client and nowhere near the cause. + # + # Bounded, not indefinite. Where authelia runs on another host the + # file is never going to appear, and blocking the queue container + # forever would replace a clear failure with a hang. After the + # timeout this fails exactly as it did before, having first given + # the co-located case the seconds it actually needs. + secret=${lib.escapeShellArg clientSecretSource} + deadline=$(( SECONDS + 120 )) + while [ ! -s "$secret" ]; do + if [ "$SECONDS" -ge "$deadline" ]; then + echo "the swarm queue responder's OIDC secret never appeared at $secret" >&2 + echo "(authelia mints it on first boot; if authelia runs on another host," >&2 + echo " copy the secret there and this unit will pick it up)" >&2 + exit 1 + fi + sleep 2 + done + + install -m 0400 "$secret" \ ${lib.escapeShellArg (hostPath "oidc-client.secret")} ''; };