swarm-nats: the auth responder has no start limit, always restarts, and is started after the server's restarts

nix/host-modules/swarm-nats.nix

@@ -345,6 +345,14 @@ let
   # Not agent containers, though: they have a netns of their own and
   # the bridge firewall does not open this port.
   privateNetwork = false;
+
+  # After the host restarts or starts the server in the container: the
+  # responder is cleared of a failed state and started. `start`, not
+  # `restart`: a running responder was already restarted with the server
+  # (`Requires=`) and so has loaded the current keys; a stopped one has not.
+  rearmResponder = ''
+    systemctl --machine=${lib.escapeShellArg machine} reset-failed swarm-nats-auth.service
+    systemctl --machine=${lib.escapeShellArg machine} start --no-block swarm-nats-auth.service'';
 in
 {
   # The swarm's message queue: one NATS server, reached by every hive at
@@ -1020,10 +1028,22 @@ in
               # store with that identity.
               LoadCredential = lib.mapAttrsToList (id: _: "${id}:${inContainer id}") authStoreFiles;
               DynamicUser = true;
-              Restart = "on-failure";
+              # `always`: a clean exit comes back too. The delay grows from
+              # `RestartSec` to `RestartMaxDelaySec` over `RestartSteps`
+              # restarts (systemd.service(5)).
+              Restart = "always";
               RestartSec = "5s";
+              RestartSteps = 4;
+              RestartMaxDelaySec = 60;
               SyslogIdentifier = "swarm-nats-auth";
             };
+            # No start limit: with one, a burst of starts (the server's, the
+            # trust bundle's, its own retries) leaves the responder failed and
+            # the queue refusing every client. A `[Unit]` setting; systemd
+            # ignores it under `[Service]`. `Restart=` does not revive a stop
+            # that came from the server failing (`Requires=`), so the host
+            # starts this unit after it restarts the server.
+            startLimitIntervalSec = 0;
             # The store the responder reads the callout keys and agent
             # credentials from.
             environment = {
@@ -1182,7 +1202,12 @@ in
       # `renderConfig` reads the files as the server starts, and only then.
       + lib.concatMapStrings (k: ''
         refresh_consumer ${lib.escapeShellArg machine} nats.service ${lib.escapeShellArg "${hostRuntimeDir}/${k.file}"}
-      '') calloutPublicKeys;
+      '') calloutPublicKeys
+      + ''
+        if systemctl is-active --quiet ${lib.escapeShellArg "container@${machine}.service"}; then
+          ${rearmResponder}
+        fi
+      '';
     };
 
     # The queue's TLS leaf, issued by the secret store's `pki/roles/swarm-nats`
@@ -1324,6 +1349,7 @@ in
           echo "queue leaf rotated — restarting the server in ${machine}"
           systemctl --machine=${lib.escapeShellArg machine} reset-failed nats.service
           systemctl --machine=${lib.escapeShellArg machine} restart nats.service
+          ${rearmResponder}
         fi
       '';
     };