make bao readers want swarm-bao-pki and drop its start limit
A pki start refused by its start limit failed every reader that Requires= it with `dependency`. That leaves a unit inactive, not failed, so Restart=on-failure never fired and readers without a refresh timer never ran again (#5381: grafana 502 for ~43 min). Readers and granting units now Wants= + After= pki; one whose leaf is still missing fails its login with exit 1 into its existing retry. pki gets StartLimitIntervalSec=0: it is deliberately not RemainAfterExit, so each reader's start re-runs it.
This commit is contained in:
parent
141be16c6d
commit
265f8a47cc
12 changed files with 95 additions and 46 deletions
|
|
@ -1,18 +1,23 @@
|
|||
# Glue: where the store and one of its readers share a host, the reader waits
|
||||
# for the store's container and requires `swarm-bao-pki`, which mints the leaf
|
||||
# the reader logs in with.
|
||||
# for the store's container and `swarm-bao-pki`, which mints the leaf the
|
||||
# reader logs in with.
|
||||
#
|
||||
# ONE PAIRING PER FILE — the store's container and PKI unit ← the readers that
|
||||
# log in to it, and nothing else. The waits on the store's policy units are
|
||||
# ./glue-bao-readers-policy-order.nix's.
|
||||
#
|
||||
# `Wants=` on `swarm-bao-pki`, not `Requires=`: a pki start that fails or is
|
||||
# refused fails each requirer's job with `dependency`, which leaves the reader
|
||||
# inactive rather than failed, so its `Restart=on-failure` never fires. Here
|
||||
# the reader runs anyway; with its leaf still missing, its login exits 1 into
|
||||
# ./lib/store-retry.nix's retries.
|
||||
#
|
||||
# ⚠️ Gated on BOTH the store and that reader being here. Off the store's host
|
||||
# neither unit exists, and `Requires=` on an absent unit fails the job, so
|
||||
# there the reader's own start timeout bounds the attempt instead. On the
|
||||
# store's host without the reader, setting `systemd.services.<reader>.after`
|
||||
# would define a unit with no ExecStart, so each gate below restates the one
|
||||
# the reader's own module puts on it. A reader whose gate changes must change
|
||||
# here too.
|
||||
# neither unit exists, and the reader's own start timeout bounds the attempt
|
||||
# instead. On the store's host without the reader, setting
|
||||
# `systemd.services.<reader>.after` would define a unit with no ExecStart, so
|
||||
# each gate below restates the one the reader's own module puts on it. A
|
||||
# reader whose gate changes must change here too.
|
||||
{
|
||||
lib,
|
||||
config,
|
||||
|
|
@ -65,8 +70,10 @@ let
|
|||
"swarm-bao-pki.service"
|
||||
"container@${baoCfg.machine}.service"
|
||||
];
|
||||
wants = [ "container@${baoCfg.machine}.service" ];
|
||||
requires = [ "swarm-bao-pki.service" ];
|
||||
wants = [
|
||||
"swarm-bao-pki.service"
|
||||
"container@${baoCfg.machine}.service"
|
||||
];
|
||||
};
|
||||
};
|
||||
in
|
||||
|
|
|
|||
|
|
@ -149,6 +149,11 @@ in
|
|||
before = [ "swarm-bao-certs.service" ];
|
||||
requiredBy = [ "swarm-bao-certs.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
# No start limit: a mass container start pulls this in once per unit it
|
||||
# starts, which outruns systemd's default of 5 starts in 10s. No
|
||||
# `Restart=`, so nothing here loops. `[Unit]` setting, hence not in
|
||||
# `serviceConfig`.
|
||||
startLimitIntervalSec = 0;
|
||||
path = [
|
||||
pkgs.openssl
|
||||
pkgs.coreutils
|
||||
|
|
|
|||
|
|
@ -158,16 +158,19 @@ in
|
|||
(lib.mkIf (deployCfg.hive-controller.enable && haveClientIdentity) {
|
||||
systemd.services.swarm-bao-queue-agent = {
|
||||
description = "fetch this hive's agent queue credential from the swarm secret store";
|
||||
# Every one of these names a unit that exists only where the store runs.
|
||||
# `Requires=` on an absent unit fails the job outright, so the ordering is
|
||||
# conditional even though the read is not: off-host there is nothing local
|
||||
# to wait for, and the timeout below is what bounds the attempt instead.
|
||||
# Every one of these names a unit that exists only where the store runs,
|
||||
# so the ordering is conditional even though the read is not: off-host
|
||||
# there is nothing local to wait for, and the timeout below is what
|
||||
# bounds the attempt instead. `Wants=` pki, for
|
||||
# ./glue-bao-readers-store-order.nix's reason.
|
||||
after = lib.optionals baoDeploy.enable [
|
||||
"swarm-bao-pki.service"
|
||||
"container@${baoCfg.machine}.service"
|
||||
];
|
||||
wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
|
||||
requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
|
||||
wants = lib.optionals baoDeploy.enable [
|
||||
"swarm-bao-pki.service"
|
||||
"container@${baoCfg.machine}.service"
|
||||
];
|
||||
# Ordered before hive-c0re, so no agent container renders ahead of an
|
||||
# attempt at its credential. `Wants=`, not `Requires=`: a store this
|
||||
# unit can't reach delays hive-c0re's start by one attempt
|
||||
|
|
|
|||
|
|
@ -332,6 +332,10 @@ let
|
|||
|
||||
# `swarm-bao-pki` mints the granter's leaf; the granter's role is written by
|
||||
# `swarm-bao-granter-role`, which normally skips, hence ordering only there.
|
||||
# Each granting unit `Wants=` pki and never `Requires=` it: a refused or
|
||||
# failed pki start fails a requirer's job with `dependency`, which no
|
||||
# `Restart=` retries, while a leaf still missing fails `granterLogin` into
|
||||
# the unit's own retry.
|
||||
granterAfter = [
|
||||
"swarm-bao-pki.service"
|
||||
"swarm-bao-granter-role.service"
|
||||
|
|
@ -1166,7 +1170,7 @@ let
|
|||
"swarm-bao-controller-policy.service"
|
||||
]
|
||||
++ granterAfter;
|
||||
requires = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
baoCli
|
||||
|
|
@ -2790,8 +2794,11 @@ in
|
|||
"swarm-bao-pki.service"
|
||||
"container@${cfg.machine}.service"
|
||||
];
|
||||
wants = [ "container@${cfg.machine}.service" ];
|
||||
requires = [ "swarm-bao-pki.service" ];
|
||||
# `Wants=` pki, for ./glue-bao-readers-store-order.nix's reason.
|
||||
wants = [
|
||||
"swarm-bao-pki.service"
|
||||
"container@${cfg.machine}.service"
|
||||
];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
baoDeploy.package
|
||||
|
|
@ -3137,7 +3144,7 @@ in
|
|||
systemd.services.swarm-bao-controller-policy = lib.mkIf haveGranter {
|
||||
description = "write the swarm controller's bao policy and cert-auth role";
|
||||
after = [ "container@${cfg.machine}.service" ] ++ granterAfter;
|
||||
requires = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
# The wrapper rather than the package: it carries the address and the
|
||||
# CA, which is what makes running here cheaper than shipping an
|
||||
|
|
@ -3563,7 +3570,7 @@ in
|
|||
"swarm-bao-forge-publisher-policy.service"
|
||||
]
|
||||
++ granterAfter;
|
||||
requires = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
baoCli
|
||||
|
|
|
|||
Loading…
Reference in a new issue