diff --git a/docs/4-implementation/swarm/credentials.md b/docs/4-implementation/swarm/credentials.md index 0857b09c1..500d60f58 100644 --- a/docs/4-implementation/swarm/credentials.md +++ b/docs/4-implementation/swarm/credentials.md @@ -80,10 +80,20 @@ On the queue's host, `swarm-bao-nats-callout-xkey` runs before the The store has to be reachable each time the responder starts. When it's down, the fetch unit fails and retries for a day. The server starts with the key fetched last time, or not at all on a first boot. A responder that starts -while the store is down exits and retries every 5 seconds, and until one -start reads the seed the queue refuses every client, OIDC ones included. The -retry that lands the file starts or restarts the server, and the responder -comes up with it. +while the store is down exits and retries, first after 5 seconds and then +after longer waits up to a minute, and until one start reads the seed the +queue refuses every client, OIDC ones included. The retry that lands the file +starts or restarts the server, and the responder comes up with it. + +The responder always comes back. Its unit has `Restart=always` and no start +limit, so a burst of starts can't leave it failed. A stop that comes from the +server failing isn't a restart trigger, so every host script that restarts +the server in `swarm-nats` (`swarm-bao-nats-tls` on a leaf rotation, +`swarm-bao-nats-callout-xkey` after fetching the keys) also clears the +responder's failed state and starts it. Those scripts use `start`, not +`restart`: the server's restart already restarts a running responder, which loads +the current seeds. Stopping the responder by hand lasts only until the next +key or leaf run, which starts it again. To keep it down, mask it. The responder's store leaf is `deploy.nats.authBaoClientCertFile` / `authBaoClientKeyFile`. On the store's host both default to the leaf diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 89510cd3d..320f9645f 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -345,6 +345,14 @@ let # Not agent containers, though: they have a netns of their own and # the bridge firewall does not open this port. privateNetwork = false; + + # After the host restarts or starts the server in the container: the + # responder is cleared of a failed state and started. `start`, not + # `restart`: a running responder was already restarted with the server + # (`Requires=`) and so has loaded the current keys; a stopped one has not. + rearmResponder = '' + systemctl --machine=${lib.escapeShellArg machine} reset-failed swarm-nats-auth.service + systemctl --machine=${lib.escapeShellArg machine} start --no-block swarm-nats-auth.service''; in { # The swarm's message queue: one NATS server, reached by every hive at @@ -1020,10 +1028,22 @@ in # store with that identity. LoadCredential = lib.mapAttrsToList (id: _: "${id}:${inContainer id}") authStoreFiles; DynamicUser = true; - Restart = "on-failure"; + # `always`: a clean exit comes back too. The delay grows from + # `RestartSec` to `RestartMaxDelaySec` over `RestartSteps` + # restarts (systemd.service(5)). + Restart = "always"; RestartSec = "5s"; + RestartSteps = 4; + RestartMaxDelaySec = 60; SyslogIdentifier = "swarm-nats-auth"; }; + # No start limit: with one, a burst of starts (the server's, the + # trust bundle's, its own retries) leaves the responder failed and + # the queue refusing every client. A `[Unit]` setting; systemd + # ignores it under `[Service]`. `Restart=` does not revive a stop + # that came from the server failing (`Requires=`), so the host + # starts this unit after it restarts the server. + startLimitIntervalSec = 0; # The store the responder reads the callout keys and agent # credentials from. environment = { @@ -1182,7 +1202,12 @@ in # `renderConfig` reads the files as the server starts, and only then. + lib.concatMapStrings (k: '' refresh_consumer ${lib.escapeShellArg machine} nats.service ${lib.escapeShellArg "${hostRuntimeDir}/${k.file}"} - '') calloutPublicKeys; + '') calloutPublicKeys + + '' + if systemctl is-active --quiet ${lib.escapeShellArg "container@${machine}.service"}; then + ${rearmResponder} + fi + ''; }; # The queue's TLS leaf, issued by the secret store's `pki/roles/swarm-nats` @@ -1324,6 +1349,7 @@ in echo "queue leaf rotated — restarting the server in ${machine}" systemctl --machine=${lib.escapeShellArg machine} reset-failed nats.service systemctl --machine=${lib.escapeShellArg machine} restart nats.service + ${rearmResponder} fi ''; }; diff --git a/nix/module-eval/nats-authelia.nix b/nix/module-eval/nats-authelia.nix index 72ca0c04e..c0b069c2b 100644 --- a/nix/module-eval/nats-authelia.nix +++ b/nix/module-eval/nats-authelia.nix @@ -381,6 +381,22 @@ let natsOnStoreHost ]; } + { + # A burst of starts must not leave the responder failed, and a stop + # from the server failing must not leave it down: no start limit and + # `Restart=always`. `RestartSec` is the control that the lookup reaches + # the unit's real `serviceConfig`; the callout key fetch starts the + # responder after it restarts the server. + name = "the queue's responder has no start limit, always restarts, and is started after the key fetch"; + ok = + let + r = responderOf natsQueueOnly; + in + r.startLimitIntervalSec == 0 + && r.serviceConfig.Restart == "always" + && r.serviceConfig.RestartSec == "5s" + && lib.hasInfix "start --no-block swarm-nats-auth.service" natsWithStoreIdentity.systemd.services.swarm-bao-nats-callout-xkey.script; + } { # A server that starts late, once its xkey arrives, must bring the # responder with it: the responder requires the server, so a server that diff --git a/nix/module-eval/nats-tls.nix b/nix/module-eval/nats-tls.nix index ec3985ae8..38fe502ab 100644 --- a/nix/module-eval/nats-tls.nix +++ b/nix/module-eval/nats-tls.nix @@ -225,6 +225,18 @@ let && lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats.pem ]" allLocal.systemd.services.swarm-bao-pki.script && lib.hasInfix "swarm-nats \"\" clientAuth" allLocal.systemd.services.swarm-bao-pki.script; } + { + # The leaf rotation restarts the server; the responder, stopped when + # the server failed, is cleared and started after it. + name = "the leaf rotation starts the responder after restarting the server"; + ok = + let + s = leafUnit.script; + in + lib.hasInfix "restart nats.service" s + && lib.hasInfix "reset-failed swarm-nats-auth.service" s + && lib.hasInfix "start --no-block swarm-nats-auth.service" s; + } { # The hive's own reader is not the queue's identity, and no path is # derived from it: a queue host without the store that holds only the