From 31d221eff8313793bd051ff68c703fd1568456e0 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Sun, 16 Aug 2026 20:48:26 +0200
Subject: [PATCH] fix(#3349): wait for authelia's minted secret instead of
 failing the boot race
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The responder's third credential is not minted by the generator beside
it — authelia produces it on first boot, inside its own container, and
nothing ordered this unit against that. Losing the race cost the entire
queue: install exits 1, the responder never starts, and auth_callout
with no responder refuses every client. Fail-closed by design, so the
symptom lands on every queue client and nowhere near the cause. Seen on
a real boot at 20:31:31.

Now it waits for the file, bounded at two minutes. Where authelia runs
on another host the secret is never going to appear, and blocking the
queue container indefinitely would replace a clear failure with a hang;
after the timeout this fails exactly as it did before, having first
given the co-located case the seconds it needs.

TimeoutStartSec is set rather than left to the default because the
default is ninety seconds — a two-minute wait would be killed at ninety
and the operator would get a generic unit timeout instead of the
message naming the file.

Ordering after the authelia container is necessary and not sufficient:
the container being up says nothing about whether its in-container
secrets unit has finished. It only stops this spinning for the full
timeout on every boot.
---
 nix/host-modules/swarm-nats.nix | 44 +++++++++++++++++++++++++++++++--
 1 file changed, 42 insertions(+), 2 deletions(-)

diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix
index 7895e78c..6249fc4f 100644
--- a/nix/host-modules/swarm-nats.nix
+++ b/nix/host-modules/swarm-nats.nix
@@ -557,11 +557,27 @@ in
       # has run. `requires` as well as `after`: if minting fails there is
       # nothing to deliver, and a copy that silently succeeds with a stale
       # or absent seed is worse than not running.
-      after = lib.optional cfg.autoGenerateCallout "swarm-nats-callout-keys.service";
+      after =
+        lib.optional cfg.autoGenerateCallout "swarm-nats-callout-keys.service"
+        # The third credential does not come from the generator above — it is
+        # minted by authelia's FIRST BOOT, inside its own container. Ordering
+        # after that container is necessary and NOT sufficient: the container
+        # being up says nothing about whether its in-container secrets unit
+        # has finished. The wait in the script is what actually closes it;
+        # this only stops us spinning for the full timeout on every boot.
+        ++ lib.optional autheliaCfg.enable "container@${autheliaCfg.machine}.service";
       requires = lib.optional cfg.autoGenerateCallout "swarm-nats-callout-keys.service";
       serviceConfig = {
         Type = "oneshot";
         RemainAfterExit = true;
+        # Must exceed the script's own wait below, and is set rather than
+        # left to the default for exactly that reason: systemd's
+        # `DefaultTimeoutStartSec` is 90s, so a 120s wait would be killed at
+        # 90 and the operator would get a generic unit timeout instead of
+        # the message that names the missing file and says what to do.
+        # The bound that matters belongs in one place, and this makes the
+        # two visibly related.
+        TimeoutStartSec = "180s";
         SyslogIdentifier = "swarm-nats-auth-secrets";
       };
       path = [ pkgs.coreutils ];
@@ -572,7 +588,31 @@ in
           ${lib.escapeShellArg (hostPath "callout-user.seed")}
         install -m 0400 ${lib.escapeShellArg issuerSeedFile} \
           ${lib.escapeShellArg (hostPath "issuer.seed")}
-        install -m 0400 ${lib.escapeShellArg clientSecretSource} \
+        # Wait for authelia's minted secret rather than failing the instant
+        # it is absent. On a fresh boot this unit and authelia's first-boot
+        # generator race, and losing that race used to cost the WHOLE QUEUE:
+        # this exits 1, the responder never starts, and `auth_callout` with
+        # no responder refuses every client — fail-closed by design, so the
+        # symptom appears on every queue client and nowhere near the cause.
+        #
+        # Bounded, not indefinite. Where authelia runs on another host the
+        # file is never going to appear, and blocking the queue container
+        # forever would replace a clear failure with a hang. After the
+        # timeout this fails exactly as it did before, having first given
+        # the co-located case the seconds it actually needs.
+        secret=${lib.escapeShellArg clientSecretSource}
+        deadline=$(( SECONDS + 120 ))
+        while [ ! -s "$secret" ]; do
+          if [ "$SECONDS" -ge "$deadline" ]; then
+            echo "the swarm queue responder's OIDC secret never appeared at $secret" >&2
+            echo "(authelia mints it on first boot; if authelia runs on another host," >&2
+            echo " copy the secret there and this unit will pick it up)" >&2
+            exit 1
+          fi
+          sleep 2
+        done
+
+        install -m 0400 "$secret" \
           ${lib.escapeShellArg (hostPath "oidc-client.secret")}
       '';
     };