vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

make bao readers want swarm-bao-pki and drop its start limit

Browse source 
A pki start refused by its start limit failed every reader that
Requires= it with `dependency`. That leaves a unit inactive, not
failed, so Restart=on-failure never fired and readers without a
refresh timer never ran again (#5381: grafana 502 for ~43 min).

Readers and granting units now Wants= + After= pki; one whose leaf is
still missing fails its login with exit 1 into its existing retry.
pki gets StartLimitIntervalSec=0: it is deliberately not
RemainAfterExit, so each reader's start re-runs it.
This commit is contained in:
atlas 2026-10-16 18:29:35 +02:00
commit 265f8a47cc
12 changed files with 95 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

4
docs/swarm/bao.md
View file

@ -100,6 +100,10 @@ principal — and points `deploy.bao.serverCertFile`, `.serverKeyFile` and
It runs at every boot and every deploy, and before each start of a container
that loads one of its leaves, and signs only the files that are missing, so a
leaf a deploy adds, or a service a deploy enables, has its file on that deploy.
The store's readers and granting units on this host order after it and want it,
but none requires it: when a run fails, each of them still starts, and one
whose leaf is missing fails its login into its own retry. It has no start
limit, since a mass container start pulls it in once for each unit it starts.
Those are `mkDefault`s, so naming your own paths wins. Do that when your
certificates come from a real internal CA; the store has no opinion about

27
nix/host-modules/glue-bao-readers-store-order.nix
View file

@ -1,18 +1,23 @@
# Glue: where the store and one of its readers share a host, the reader waits
# for the store's container and requires `swarm-bao-pki`, which mints the leaf
# the reader logs in with.
# for the store's container and `swarm-bao-pki`, which mints the leaf the
# reader logs in with.
#
# ONE PAIRING PER FILE — the store's container and PKI unit ← the readers that
# log in to it, and nothing else. The waits on the store's policy units are
# ./glue-bao-readers-policy-order.nix's.
#
# `Wants=` on `swarm-bao-pki`, not `Requires=`: a pki start that fails or is
# refused fails each requirer's job with `dependency`, which leaves the reader
# inactive rather than failed, so its `Restart=on-failure` never fires. Here
# the reader runs anyway; with its leaf still missing, its login exits 1 into
# ./lib/store-retry.nix's retries.
#
# ⚠️ Gated on BOTH the store and that reader being here. Off the store's host
# neither unit exists, and `Requires=` on an absent unit fails the job, so
# there the reader's own start timeout bounds the attempt instead. On the
# store's host without the reader, setting `systemd.services.<reader>.after`
# would define a unit with no ExecStart, so each gate below restates the one
# the reader's own module puts on it. A reader whose gate changes must change
# here too.
# neither unit exists, and the reader's own start timeout bounds the attempt
# instead. On the store's host without the reader, setting
# `systemd.services.<reader>.after` would define a unit with no ExecStart, so
# each gate below restates the one the reader's own module puts on it. A
# reader whose gate changes must change here too.
{
lib,
config,
@ -65,8 +70,10 @@ let
"swarm-bao-pki.service"
"container@${baoCfg.machine}.service"
];
wants = [ "container@${baoCfg.machine}.service" ];
requires = [ "swarm-bao-pki.service" ];
wants = [
"swarm-bao-pki.service"
"container@${baoCfg.machine}.service"
];
};
};
in

5
nix/host-modules/glue-bao-tls.nix
View file

@ -149,6 +149,11 @@ in
before = [ "swarm-bao-certs.service" ];
requiredBy = [ "swarm-bao-certs.service" ];
wantedBy = [ "multi-user.target" ];
# No start limit: a mass container start pulls this in once per unit it
# starts, which outruns systemd's default of 5 starts in 10s. No
# `Restart=`, so nothing here loops. `[Unit]` setting, hence not in
# `serviceConfig`.
startLimitIntervalSec = 0;
path = [
pkgs.openssl
pkgs.coreutils

15
nix/host-modules/glue-queue-agent-credential.nix
View file

@ -158,16 +158,19 @@ in
(lib.mkIf (deployCfg.hive-controller.enable && haveClientIdentity) {
systemd.services.swarm-bao-queue-agent = {
description = "fetch this hive's agent queue credential from the swarm secret store";
# Every one of these names a unit that exists only where the store runs.
# `Requires=` on an absent unit fails the job outright, so the ordering is
# conditional even though the read is not: off-host there is nothing local
# to wait for, and the timeout below is what bounds the attempt instead.
# Every one of these names a unit that exists only where the store runs,
# so the ordering is conditional even though the read is not: off-host
# there is nothing local to wait for, and the timeout below is what
# bounds the attempt instead. `Wants=` pki, for
# ./glue-bao-readers-store-order.nix's reason.
after = lib.optionals baoDeploy.enable [
"swarm-bao-pki.service"
"container@${baoCfg.machine}.service"
];
wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
wants = lib.optionals baoDeploy.enable [
"swarm-bao-pki.service"
"container@${baoCfg.machine}.service"
];
# Ordered before hive-c0re, so no agent container renders ahead of an
# attempt at its credential. `Wants=`, not `Requires=`: a store this
# unit can't reach delays hive-c0re's start by one attempt

17
nix/host-modules/swarm-bao.nix
View file

@ -332,6 +332,10 @@ let
# `swarm-bao-pki` mints the granter's leaf; the granter's role is written by
# `swarm-bao-granter-role`, which normally skips, hence ordering only there.
# Each granting unit `Wants=` pki and never `Requires=` it: a refused or
# failed pki start fails a requirer's job with `dependency`, which no
# `Restart=` retries, while a leaf still missing fails `granterLogin` into
# the unit's own retry.
granterAfter = [
"swarm-bao-pki.service"
"swarm-bao-granter-role.service"
@ -1166,7 +1170,7 @@ let
"swarm-bao-controller-policy.service"
]
++ granterAfter;
requires = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];
wantedBy = [ "multi-user.target" ];
path = [
baoCli
@ -2790,8 +2794,11 @@ in
"swarm-bao-pki.service"
"container@${cfg.machine}.service"
];
wants = [ "container@${cfg.machine}.service" ];
requires = [ "swarm-bao-pki.service" ];
# `Wants=` pki, for ./glue-bao-readers-store-order.nix's reason.
wants = [
"swarm-bao-pki.service"
"container@${cfg.machine}.service"
];
wantedBy = [ "multi-user.target" ];
path = [
baoDeploy.package
@ -3137,7 +3144,7 @@ in
systemd.services.swarm-bao-controller-policy = lib.mkIf haveGranter {
description = "write the swarm controller's bao policy and cert-auth role";
after = [ "container@${cfg.machine}.service" ] ++ granterAfter;
requires = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];
wantedBy = [ "multi-user.target" ];
# The wrapper rather than the package: it carries the address and the
# CA, which is what makes running here cheaper than shipping an
@ -3563,7 +3570,7 @@ in
"swarm-bao-forge-publisher-policy.service"
]
++ granterAfter;
requires = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];
wantedBy = [ "multi-user.target" ];
path = [
baoCli

16
nix/module-eval/bao-grants.nix
View file

@ -1097,14 +1097,16 @@ let
{
# Every granting unit retries a sealed or late store for a day, in the
# `[Unit]` section systemd reads it from, and waits for the unit that
# mints the granter's leaf.
name = "each granting unit requires the PKI unit and retries 2880 times at 30s";
# mints the granter's leaf. Wanted, never required: a refused pki start
# fails a requirer's job with `dependency`, which no `Restart=` retries.
name = "each granting unit wants, not requires, the PKI unit and retries 2880 times at 30s";
ok = lib.all (
unit:
let
u = baoGrantHere.systemd.services.${unit};
in
lib.elem "swarm-bao-pki.service" u.requires
lib.elem "swarm-bao-pki.service" u.wants
&& !(lib.elem "swarm-bao-pki.service" u.requires)
&& lib.elem "swarm-bao-pki.service" u.after
&& lib.elem "swarm-bao-granter-role.service" u.after
&& !(lib.elem "swarm-bao-granter-role.service" (u.requires ++ u.wants))
@ -1114,6 +1116,14 @@ let
&& u.serviceConfig.Restart == "on-failure"
) grantingUnitNames;
}
{
# Every reader and granting unit's start pulls the PKI unit in, so a mass
# container start outruns systemd's default limit of 5 starts in 10s.
name = "the PKI unit has no start limit";
ok =
toString (baoGrantHere.systemd.services.swarm-bao-pki.unitConfig.StartLimitIntervalSec or "unset")
== "0";
}
{
# A oneshot has no start timeout by default, so an attempt that never
# ends never fails into a retry.

20
nix/module-eval/bao-matrix-reader.nix
View file

@ -162,26 +162,32 @@ let
&& !(builtins.elem "hive-c0re.service" (u.requires or [ ]));
}
{
# `Requires=` on a unit that does not exist fails the job, and nothing
# local mints certificates off-host — so this orders against nothing.
# Eval cannot see that failure; only the empty list here stands in for it.
name = "an off-host reader requires no unit the store's host would have provided";
# Nothing local mints certificates off-host, so this orders against
# nothing there.
name = "an off-host reader names no unit the store's host would have provided";
# Membership first, then the value: indexing a missing unit throws, and a
# table that reports which property broke must not be the thing that dies.
ok =
let
s = baoRemoteReader.systemd.services;
u = s.swarm-bao-queue-agent;
in
s ? swarm-bao-queue-agent && s.swarm-bao-queue-agent.requires == [ ];
s ? swarm-bao-queue-agent && !(lib.elem "swarm-bao-pki.service" (u.after ++ u.wants ++ u.requires));
}
{
# Presence control for the case above: the list is conditional, not gone.
name = "a co-located reader still orders after the local pki unit";
# Wanted, never required: a refused pki start fails a requirer's job with
# `dependency`, which its `Restart=on-failure` never retries.
name = "a co-located reader still orders after and wants the local pki unit, and does not require it";
ok =
let
s = baoWithMatrix.systemd.services;
u = s.swarm-bao-queue-agent;
in
s ? swarm-bao-queue-agent && s.swarm-bao-queue-agent.requires == [ "swarm-bao-pki.service" ];
s ? swarm-bao-queue-agent
&& lib.elem "swarm-bao-pki.service" u.after
&& lib.elem "swarm-bao-pki.service" u.wants
&& u.requires == [ ];
}
{
# Nothing asserted this script before, which is how it kept a branch that

3
nix/module-eval/bao-otel-collector.nix
View file

@ -420,7 +420,8 @@ let
in
lib.elem "container@swarm-bao.service" u.after
&& !(lib.elem "container@swarm-bao.service" (u.before or [ ]))
&& lib.elem "swarm-bao-pki.service" u.requires;
&& lib.elem "swarm-bao-pki.service" u.wants
&& !(lib.elem "swarm-bao-pki.service" u.requires);
}
{
# The far end of the credential. An authenticator checks ONE audience,

4
nix/module-eval/forge-placement.nix
View file

@ -190,8 +190,8 @@ let
== "/etc/pki/bao-ci-runner.pem";
}
{
# The store's units exist only on its host; `Requires=` on an absent one
# fails the fetch's start. The arm where the store is here is in
# The store's units exist only on its host, so elsewhere the fetch waits
# on nothing local. The arm where the store is here is in
# ./swarm-services-switch.nix.
name = "the CI runner's token fetch names none of the store's units on a host without the store";
ok =

7
nix/module-eval/hive-otel.nix
View file

@ -309,8 +309,8 @@ let
lib.hasInfix "bao kv get -field=value ${path}" fetch && lib.hasInfix "bao kv put ${path} " publish;
}
{
# The store's units exist only on its host; `Requires=` on an absent one
# fails the fetch's start.
# The store's units exist only on its host. The PKI unit is wanted, never
# required: a refused pki start fails a requirer's job with `dependency`.
name = "the hive collector's fetch waits for the store's container and PKI unit only where the store runs here";
ok =
let
@ -322,7 +322,8 @@ let
elsewhere = hiveOtelFetch hiveOtelLeaf;
in
lib.all (n: lib.elem n here.after) (storeUnits hiveOtelAllLocal)
&& lib.elem "swarm-bao-pki.service" here.requires
&& lib.elem "swarm-bao-pki.service" here.wants
&& !(lib.elem "swarm-bao-pki.service" here.requires)
&& lib.elem "container@${hiveOtelAllLocal.services.hyperhive.swarm.bao.machine}.service" here.wants
&& !(lib.any (n: lib.elem n (elsewhere.after ++ elsewhere.wants ++ elsewhere.requires)) (
storeUnits hiveOtelLeaf

4
nix/module-eval/swarm-otel-identity.nix
View file

@ -740,8 +740,8 @@ let
&& !(refusedOver otelRemoteAuthelia "otelOidcClientCertFile");
}
{
# The store's units exist only on its host; `Requires=` on an absent one
# fails the fetch's start. The arm where the store is here is in
# The store's units exist only on its host, so elsewhere the fetch waits
# on nothing local. The arm where the store is here is in
# ./swarm-services-switch.nix.
name = "the remote reader's fetch names none of the store's units";
ok =

19
nix/module-eval/swarm-services-switch.nix
View file

@ -804,12 +804,13 @@ let
}
{
# Where the store runs here, each reader waits for its container and
# requires the PKI unit minting the leaf it logs in with. Elsewhere
# neither unit exists, and `Requires=` on an absent unit fails the
# reader's start. On the store's host without the readers, none of them
# is rendered as a stub. Rosters are counted: an empty one holds
# vacuously.
name = "a store reader waits for the store's container and PKI unit only where the store runs here";
# the PKI unit minting the leaf it logs in with, and wants both. It
# never requires the PKI unit: a refused pki start fails a requirer's
# job with `dependency`, which its `Restart=on-failure` never retries.
# Elsewhere neither unit exists. On the store's host without the
# readers, none of them is rendered as a stub. Rosters are counted: an
# empty one holds vacuously.
name = "a store reader waits for and wants the store's container and PKI unit only where the store runs here, and requires neither";
ok =
let
machine = serviceContainersWithCi.services.hyperhive.swarm.bao.machine;
@ -831,12 +832,14 @@ let
];
here = serviceContainersWithCi.systemd.services;
elsewhere = swarmServicesBaoElsewhere.systemd.services;
readersHere = lib.filter (u: here ? ${u}) readers;
readersElsewhere = lib.filter (u: elsewhere ? ${u}) readers;
waits =
u:
here ? ${u}
&& lib.elem "swarm-bao-pki.service" here.${u}.after
&& lib.elem "swarm-bao-pki.service" here.${u}.requires
&& lib.elem "swarm-bao-pki.service" here.${u}.wants
&& !(lib.any (n: lib.elem n here.${u}.requires) storeUnits)
&& lib.elem "container@${machine}.service" here.${u}.after
&& lib.elem "container@${machine}.service" here.${u}.wants;
namesNoStoreUnit =
@ -847,6 +850,8 @@ let
!(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits);
in
lib.all waits readers
&& lib.length readersHere == 10
&& lib.elem "swarm-bao-grafana-admin" readersHere
&& lib.length readersElsewhere == 7
&& lib.all namesNoStoreUnit readersElsewhere
&& !(lib.any (u: storeOnlyNoHive.systemd.services ? ${u}) readers);