diff --git a/docs/4-implementation/swarm/credentials.md b/docs/4-implementation/swarm/credentials.md index 0d835f938..bf1703aeb 100644 --- a/docs/4-implementation/swarm/credentials.md +++ b/docs/4-implementation/swarm/credentials.md @@ -92,8 +92,17 @@ the server in `swarm-nats` (`swarm-bao-nats-tls` on a leaf rotation, `swarm-bao-nats-callout-xkey` after fetching the keys) also clears the responder's failed state and starts it. Those scripts use `start`, not `restart`: the server's restart already restarts a running responder, which loads -the current seeds. Stopping the responder by hand lasts only until the next -key or leaf run, which starts it again. To keep it down, mask it. +the current seeds. + +Those starts can still fail: a start job fails when a unit the responder +requires, the server or the container's trust bundle, fails first, and a +failed job leaves the responder inactive, out of `Restart=`'s reach. Both +units therefore uphold it (systemd's `Upholds=`): while either one runs, systemd starts +an inactive or failed responder, and the unit that comes up last starts it with +both up. This also means stopping the responder or the server by hand inside +the container lasts only as long as the stop itself: systemd starts the +responder again, and the responder pulls the server back in. To keep it down, +mask it, or stop the container. The responder's store leaf is `deploy.nats.authBaoClientCertFile` / `authBaoClientKeyFile`. On the store's host both default to the leaf diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 4230f1499..e7b66b49f 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -742,7 +742,7 @@ in }; }; config = - { ... }: + { ... }@container: { imports = [ ../container-modules/swarm-container.nix @@ -1068,6 +1068,13 @@ in # that came from the server failing (`Requires=`), so the host # starts this unit after it restarts the server. startLimitIntervalSec = 0; + # Started whenever it is inactive or failed while any unit it + # `Requires=` is active (`Upholds=` on each). A start job that fails + # on a dependency leaves the unit inactive, which `Restart=` never + # sees, so whichever requirement comes up last starts it with all of + # them up. It also undoes a manual `systemctl stop` of the responder, + # or of the server it pulls back in: stop the container instead. + upheldBy = container.config.systemd.services.swarm-nats-auth.requires; # The store the responder reads the callout keys and agent # credentials from. environment = { diff --git a/nix/module-eval/nats-authelia.nix b/nix/module-eval/nats-authelia.nix index 587561aaf..781b47b92 100644 --- a/nix/module-eval/nats-authelia.nix +++ b/nix/module-eval/nats-authelia.nix @@ -410,6 +410,22 @@ let name = "starting the queue's server starts its responder"; ok = lib.elem "nats.service" (responderOf natsQueueOnly).wantedBy; } + { + # A start job that fails on a dependency leaves the responder inactive, + # out of `Restart=`'s reach, so every unit it requires upholds it. The + # rendered unit is read, and both upholders are units the container has. + name = "the server and the trust bundle the queue's responder requires both uphold it"; + ok = + let + c = natsQueueOnly.containers.swarm-nats.config; + in + lib.sort lib.lessThan c.systemd.units."swarm-nats-auth.service".upheldBy == [ + "nats.service" + "swarm-nats-ca-bundle.service" + ] + && c.systemd.units ? "nats.service" + && c.systemd.units ? "swarm-nats-ca-bundle.service"; + } { # The keys are created before anything needs them, by the responder's # own verb under its own leaf, then each public half is read back from