swarm-nats: the units the responder requires uphold it
A start job of swarm-nats-auth that fails on a dependency (the server, or the container's trust bundle, failing first) leaves the unit inactive rather than failed, so Restart=always never applies. Its recovery was a set of one-shot host kicks; when the last of them still found a dependency down, the responder stayed dead and the queue refused every client (nightly full-setup, runs 8058 and 7917). upheldBy the units it requires: whichever comes up last starts it.
This commit is contained in:
parent
7c2f7443fb
commit
887ea26c56
2 changed files with 35 additions and 3 deletions
|
|
@ -742,7 +742,7 @@ in
|
|||
};
|
||||
};
|
||||
config =
|
||||
{ ... }:
|
||||
{ ... }@container:
|
||||
{
|
||||
imports = [
|
||||
../container-modules/swarm-container.nix
|
||||
|
|
@ -1068,6 +1068,13 @@ in
|
|||
# that came from the server failing (`Requires=`), so the host
|
||||
# starts this unit after it restarts the server.
|
||||
startLimitIntervalSec = 0;
|
||||
# Started whenever it is inactive or failed while any unit it
|
||||
# `Requires=` is active (`Upholds=` on each). A start job that fails
|
||||
# on a dependency leaves the unit inactive, which `Restart=` never
|
||||
# sees, so whichever requirement comes up last starts it with all of
|
||||
# them up. It also undoes a manual `systemctl stop` of the responder,
|
||||
# or of the server it pulls back in: stop the container instead.
|
||||
upheldBy = container.config.systemd.services.swarm-nats-auth.requires;
|
||||
# The store the responder reads the callout keys and agent
|
||||
# credentials from.
|
||||
environment = {
|
||||
|
|
|
|||
|
|
@ -410,6 +410,22 @@ let
|
|||
name = "starting the queue's server starts its responder";
|
||||
ok = lib.elem "nats.service" (responderOf natsQueueOnly).wantedBy;
|
||||
}
|
||||
{
|
||||
# A start job that fails on a dependency leaves the responder inactive,
|
||||
# out of `Restart=`'s reach, so every unit it requires upholds it. The
|
||||
# rendered unit is read, and both upholders are units the container has.
|
||||
name = "the server and the trust bundle the queue's responder requires both uphold it";
|
||||
ok =
|
||||
let
|
||||
c = natsQueueOnly.containers.swarm-nats.config;
|
||||
in
|
||||
lib.sort lib.lessThan c.systemd.units."swarm-nats-auth.service".upheldBy == [
|
||||
"nats.service"
|
||||
"swarm-nats-ca-bundle.service"
|
||||
]
|
||||
&& c.systemd.units ? "nats.service"
|
||||
&& c.systemd.units ? "swarm-nats-ca-bundle.service";
|
||||
}
|
||||
{
|
||||
# The keys are created before anything needs them, by the responder's
|
||||
# own verb under its own leaf, then each public half is read back from
|
||||
|
|
|
|||
Loading…
Reference in a new issue