Watch
0
0
Fork
You've already forked hyperhive
0

make bao readers want swarm-bao-pki and drop its start limit

A pki start refused by its start limit failed every reader that
Requires= it with `dependency`. That leaves a unit inactive, not
failed, so Restart=on-failure never fired and readers without a
refresh timer never ran again (#5381: grafana 502 for ~43 min).

Readers and granting units now Wants= + After= pki; one whose leaf is
still missing fails its login with exit 1 into its existing retry.
pki gets StartLimitIntervalSec=0: it is deliberately not
RemainAfterExit, so each reader's start re-runs it.
This commit is contained in:
atlas 2026-10-16 18:29:35 +02:00
commit 265f8a47cc
12 changed files with 95 additions and 46 deletions

View file

@ -1,18 +1,23 @@
# Glue: where the store and one of its readers share a host, the reader waits
# for the store's container and requires `swarm-bao-pki`, which mints the leaf
# the reader logs in with.
# for the store's container and `swarm-bao-pki`, which mints the leaf the
# reader logs in with.
#
# ONE PAIRING PER FILE — the store's container and PKI unit ← the readers that
# log in to it, and nothing else. The waits on the store's policy units are
# ./glue-bao-readers-policy-order.nix's.
#
# `Wants=` on `swarm-bao-pki`, not `Requires=`: a pki start that fails or is
# refused fails each requirer's job with `dependency`, which leaves the reader
# inactive rather than failed, so its `Restart=on-failure` never fires. Here
# the reader runs anyway; with its leaf still missing, its login exits 1 into
# ./lib/store-retry.nix's retries.
#
# ⚠️ Gated on BOTH the store and that reader being here. Off the store's host
# neither unit exists, and `Requires=` on an absent unit fails the job, so
# there the reader's own start timeout bounds the attempt instead. On the
# store's host without the reader, setting `systemd.services.<reader>.after`
# would define a unit with no ExecStart, so each gate below restates the one
# the reader's own module puts on it. A reader whose gate changes must change
# here too.
# neither unit exists, and the reader's own start timeout bounds the attempt
# instead. On the store's host without the reader, setting
# `systemd.services.<reader>.after` would define a unit with no ExecStart, so
# each gate below restates the one the reader's own module puts on it. A
# reader whose gate changes must change here too.
{
lib,
config,
@ -65,8 +70,10 @@ let
"swarm-bao-pki.service"
"container@${baoCfg.machine}.service"
];
wants = [ "container@${baoCfg.machine}.service" ];
requires = [ "swarm-bao-pki.service" ];
wants = [
"swarm-bao-pki.service"
"container@${baoCfg.machine}.service"
];
};
};
in

View file

@ -149,6 +149,11 @@ in
before = [ "swarm-bao-certs.service" ];
requiredBy = [ "swarm-bao-certs.service" ];
wantedBy = [ "multi-user.target" ];
# No start limit: a mass container start pulls this in once per unit it
# starts, which outruns systemd's default of 5 starts in 10s. No
# `Restart=`, so nothing here loops. `[Unit]` setting, hence not in
# `serviceConfig`.
startLimitIntervalSec = 0;
path = [
pkgs.openssl
pkgs.coreutils

View file

@ -158,16 +158,19 @@ in
(lib.mkIf (deployCfg.hive-controller.enable && haveClientIdentity) {
systemd.services.swarm-bao-queue-agent = {
description = "fetch this hive's agent queue credential from the swarm secret store";
# Every one of these names a unit that exists only where the store runs.
# `Requires=` on an absent unit fails the job outright, so the ordering is
# conditional even though the read is not: off-host there is nothing local
# to wait for, and the timeout below is what bounds the attempt instead.
# Every one of these names a unit that exists only where the store runs,
# so the ordering is conditional even though the read is not: off-host
# there is nothing local to wait for, and the timeout below is what
# bounds the attempt instead. `Wants=` pki, for
# ./glue-bao-readers-store-order.nix's reason.
after = lib.optionals baoDeploy.enable [
"swarm-bao-pki.service"
"container@${baoCfg.machine}.service"
];
wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
wants = lib.optionals baoDeploy.enable [
"swarm-bao-pki.service"
"container@${baoCfg.machine}.service"
];
# Ordered before hive-c0re, so no agent container renders ahead of an
# attempt at its credential. `Wants=`, not `Requires=`: a store this
# unit can't reach delays hive-c0re's start by one attempt

View file

@ -332,6 +332,10 @@ let
# `swarm-bao-pki` mints the granter's leaf; the granter's role is written by
# `swarm-bao-granter-role`, which normally skips, hence ordering only there.
# Each granting unit `Wants=` pki and never `Requires=` it: a refused or
# failed pki start fails a requirer's job with `dependency`, which no
# `Restart=` retries, while a leaf still missing fails `granterLogin` into
# the unit's own retry.
granterAfter = [
"swarm-bao-pki.service"
"swarm-bao-granter-role.service"
@ -1166,7 +1170,7 @@ let
"swarm-bao-controller-policy.service"
]
++ granterAfter;
requires = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];
wantedBy = [ "multi-user.target" ];
path = [
baoCli
@ -2790,8 +2794,11 @@ in
"swarm-bao-pki.service"
"container@${cfg.machine}.service"
];
wants = [ "container@${cfg.machine}.service" ];
requires = [ "swarm-bao-pki.service" ];
# `Wants=` pki, for ./glue-bao-readers-store-order.nix's reason.
wants = [
"swarm-bao-pki.service"
"container@${cfg.machine}.service"
];
wantedBy = [ "multi-user.target" ];
path = [
baoDeploy.package
@ -3137,7 +3144,7 @@ in
systemd.services.swarm-bao-controller-policy = lib.mkIf haveGranter {
description = "write the swarm controller's bao policy and cert-auth role";
after = [ "container@${cfg.machine}.service" ] ++ granterAfter;
requires = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];
wantedBy = [ "multi-user.target" ];
# The wrapper rather than the package: it carries the address and the
# CA, which is what makes running here cheaper than shipping an
@ -3563,7 +3570,7 @@ in
"swarm-bao-forge-publisher-policy.service"
]
++ granterAfter;
requires = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];
wantedBy = [ "multi-user.target" ];
path = [
baoCli