swarm-nats: the auth responder has no start limit, always restarts, and is started after the server's restarts

docs/4-implementation/swarm/credentials.md

@@ -80,10 +80,20 @@ On the queue's host, `swarm-bao-nats-callout-xkey` runs before the
 The store has to be reachable each time the responder starts. When it's down,
 the fetch unit fails and retries for a day. The server starts with the key
 fetched last time, or not at all on a first boot. A responder that starts
-while the store is down exits and retries every 5 seconds, and until one
-start reads the seed the queue refuses every client, OIDC ones included. The
-retry that lands the file starts or restarts the server, and the responder
-comes up with it.
+while the store is down exits and retries, first after 5 seconds and then
+after longer waits up to a minute, and until one start reads the seed the
+queue refuses every client, OIDC ones included. The retry that lands the file
+starts or restarts the server, and the responder comes up with it.
+
+The responder always comes back. Its unit has `Restart=always` and no start
+limit, so a burst of starts can't leave it failed. A stop that comes from the
+server failing isn't a restart trigger, so every host script that restarts
+the server in `swarm-nats` (`swarm-bao-nats-tls` on a leaf rotation,
+`swarm-bao-nats-callout-xkey` after fetching the keys) also clears the
+responder's failed state and starts it. Those scripts use `start`, not
+`restart`: the server's restart already restarts a running responder, which loads
+the current seeds. Stopping the responder by hand lasts only until the next
+key or leaf run, which starts it again. To keep it down, mask it.
 
 The responder's store leaf is `deploy.nats.authBaoClientCertFile` /
 `authBaoClientKeyFile`. On the store's host both default to the leaf

nix/host-modules/swarm-nats.nix

@@ -345,6 +345,14 @@ let
   # Not agent containers, though: they have a netns of their own and
   # the bridge firewall does not open this port.
   privateNetwork = false;
+
+  # After the host restarts or starts the server in the container: the
+  # responder is cleared of a failed state and started. `start`, not
+  # `restart`: a running responder was already restarted with the server
+  # (`Requires=`) and so has loaded the current keys; a stopped one has not.
+  rearmResponder = ''
+    systemctl --machine=${lib.escapeShellArg machine} reset-failed swarm-nats-auth.service
+    systemctl --machine=${lib.escapeShellArg machine} start --no-block swarm-nats-auth.service'';
 in
 {
   # The swarm's message queue: one NATS server, reached by every hive at
@@ -1020,10 +1028,22 @@ in
               # store with that identity.
               LoadCredential = lib.mapAttrsToList (id: _: "${id}:${inContainer id}") authStoreFiles;
               DynamicUser = true;
-              Restart = "on-failure";
+              # `always`: a clean exit comes back too. The delay grows from
+              # `RestartSec` to `RestartMaxDelaySec` over `RestartSteps`
+              # restarts (systemd.service(5)).
+              Restart = "always";
               RestartSec = "5s";
+              RestartSteps = 4;
+              RestartMaxDelaySec = 60;
               SyslogIdentifier = "swarm-nats-auth";
             };
+            # No start limit: with one, a burst of starts (the server's, the
+            # trust bundle's, its own retries) leaves the responder failed and
+            # the queue refusing every client. A `[Unit]` setting; systemd
+            # ignores it under `[Service]`. `Restart=` does not revive a stop
+            # that came from the server failing (`Requires=`), so the host
+            # starts this unit after it restarts the server.
+            startLimitIntervalSec = 0;
             # The store the responder reads the callout keys and agent
             # credentials from.
             environment = {
@@ -1182,7 +1202,12 @@ in
       # `renderConfig` reads the files as the server starts, and only then.
       + lib.concatMapStrings (k: ''
         refresh_consumer ${lib.escapeShellArg machine} nats.service ${lib.escapeShellArg "${hostRuntimeDir}/${k.file}"}
-      '') calloutPublicKeys;
+      '') calloutPublicKeys
+      + ''
+        if systemctl is-active --quiet ${lib.escapeShellArg "container@${machine}.service"}; then
+          ${rearmResponder}
+        fi
+      '';
     };
 
     # The queue's TLS leaf, issued by the secret store's `pki/roles/swarm-nats`
@@ -1324,6 +1349,7 @@ in
           echo "queue leaf rotated — restarting the server in ${machine}"
           systemctl --machine=${lib.escapeShellArg machine} reset-failed nats.service
           systemctl --machine=${lib.escapeShellArg machine} restart nats.service
+          ${rearmResponder}
         fi
       '';
     };

nix/module-eval/nats-authelia.nix

@@ -381,6 +381,22 @@ let
             natsOnStoreHost
           ];
     }
+    {
+      # A burst of starts must not leave the responder failed, and a stop
+      # from the server failing must not leave it down: no start limit and
+      # `Restart=always`. `RestartSec` is the control that the lookup reaches
+      # the unit's real `serviceConfig`; the callout key fetch starts the
+      # responder after it restarts the server.
+      name = "the queue's responder has no start limit, always restarts, and is started after the key fetch";
+      ok =
+        let
+          r = responderOf natsQueueOnly;
+        in
+        r.startLimitIntervalSec == 0
+        && r.serviceConfig.Restart == "always"
+        && r.serviceConfig.RestartSec == "5s"
+        && lib.hasInfix "start --no-block swarm-nats-auth.service" natsWithStoreIdentity.systemd.services.swarm-bao-nats-callout-xkey.script;
+    }
     {
       # A server that starts late, once its xkey arrives, must bring the
       # responder with it: the responder requires the server, so a server that

nix/module-eval/nats-tls.nix

@@ -225,6 +225,18 @@ let
         && lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats.pem ]" allLocal.systemd.services.swarm-bao-pki.script
         && lib.hasInfix "swarm-nats \"\" clientAuth" allLocal.systemd.services.swarm-bao-pki.script;
     }
+    {
+      # The leaf rotation restarts the server; the responder, stopped when
+      # the server failed, is cleared and started after it.
+      name = "the leaf rotation starts the responder after restarting the server";
+      ok =
+        let
+          s = leafUnit.script;
+        in
+        lib.hasInfix "restart nats.service" s
+        && lib.hasInfix "reset-failed swarm-nats-auth.service" s
+        && lib.hasInfix "start --no-block swarm-nats-auth.service" s;
+    }
     {
       # The hive's own reader is not the queue's identity, and no path is
       # derived from it: a queue host without the store that holds only the