vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

make bao readers want swarm-bao-pki and drop its start limit

Browse source 
A pki start refused by its start limit failed every reader that
Requires= it with `dependency`. That leaves a unit inactive, not
failed, so Restart=on-failure never fired and readers without a
refresh timer never ran again (#5381: grafana 502 for ~43 min).

Readers and granting units now Wants= + After= pki; one whose leaf is
still missing fails its login with exit 1 into its existing retry.
pki gets StartLimitIntervalSec=0: it is deliberately not
RemainAfterExit, so each reader's start re-runs it.
This commit is contained in:
atlas 2026-10-16 18:29:35 +02:00
commit 265f8a47cc
12 changed files with 95 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

16
nix/module-eval/bao-grants.nix
View file

@ -1097,14 +1097,16 @@ let
{
# Every granting unit retries a sealed or late store for a day, in the
# `[Unit]` section systemd reads it from, and waits for the unit that
# mints the granter's leaf.
name = "each granting unit requires the PKI unit and retries 2880 times at 30s";
# mints the granter's leaf. Wanted, never required: a refused pki start
# fails a requirer's job with `dependency`, which no `Restart=` retries.
name = "each granting unit wants, not requires, the PKI unit and retries 2880 times at 30s";
ok = lib.all (
unit:
let
u = baoGrantHere.systemd.services.${unit};
in
lib.elem "swarm-bao-pki.service" u.requires
lib.elem "swarm-bao-pki.service" u.wants
&& !(lib.elem "swarm-bao-pki.service" u.requires)
&& lib.elem "swarm-bao-pki.service" u.after
&& lib.elem "swarm-bao-granter-role.service" u.after
&& !(lib.elem "swarm-bao-granter-role.service" (u.requires ++ u.wants))
@ -1114,6 +1116,14 @@ let
&& u.serviceConfig.Restart == "on-failure"
) grantingUnitNames;
}
{
# Every reader and granting unit's start pulls the PKI unit in, so a mass
# container start outruns systemd's default limit of 5 starts in 10s.
name = "the PKI unit has no start limit";
ok =
toString (baoGrantHere.systemd.services.swarm-bao-pki.unitConfig.StartLimitIntervalSec or "unset")
== "0";
}
{
# A oneshot has no start timeout by default, so an attempt that never
# ends never fails into a retry.

20
nix/module-eval/bao-matrix-reader.nix
View file

@ -162,26 +162,32 @@ let
&& !(builtins.elem "hive-c0re.service" (u.requires or [ ]));
}
{
# `Requires=` on a unit that does not exist fails the job, and nothing
# local mints certificates off-host — so this orders against nothing.
# Eval cannot see that failure; only the empty list here stands in for it.
name = "an off-host reader requires no unit the store's host would have provided";
# Nothing local mints certificates off-host, so this orders against
# nothing there.
name = "an off-host reader names no unit the store's host would have provided";
# Membership first, then the value: indexing a missing unit throws, and a
# table that reports which property broke must not be the thing that dies.
ok =
let
s = baoRemoteReader.systemd.services;
u = s.swarm-bao-queue-agent;
in
s ? swarm-bao-queue-agent && s.swarm-bao-queue-agent.requires == [ ];
s ? swarm-bao-queue-agent && !(lib.elem "swarm-bao-pki.service" (u.after ++ u.wants ++ u.requires));
}
{
# Presence control for the case above: the list is conditional, not gone.
name = "a co-located reader still orders after the local pki unit";
# Wanted, never required: a refused pki start fails a requirer's job with
# `dependency`, which its `Restart=on-failure` never retries.
name = "a co-located reader still orders after and wants the local pki unit, and does not require it";
ok =
let
s = baoWithMatrix.systemd.services;
u = s.swarm-bao-queue-agent;
in
s ? swarm-bao-queue-agent && s.swarm-bao-queue-agent.requires == [ "swarm-bao-pki.service" ];
s ? swarm-bao-queue-agent
&& lib.elem "swarm-bao-pki.service" u.after
&& lib.elem "swarm-bao-pki.service" u.wants
&& u.requires == [ ];
}
{
# Nothing asserted this script before, which is how it kept a branch that

3
nix/module-eval/bao-otel-collector.nix
View file

@ -420,7 +420,8 @@ let
in
lib.elem "container@swarm-bao.service" u.after
&& !(lib.elem "container@swarm-bao.service" (u.before or [ ]))
&& lib.elem "swarm-bao-pki.service" u.requires;
&& lib.elem "swarm-bao-pki.service" u.wants
&& !(lib.elem "swarm-bao-pki.service" u.requires);
}
{
# The far end of the credential. An authenticator checks ONE audience,

4
nix/module-eval/forge-placement.nix
View file

@ -190,8 +190,8 @@ let
== "/etc/pki/bao-ci-runner.pem";
}
{
# The store's units exist only on its host; `Requires=` on an absent one
# fails the fetch's start. The arm where the store is here is in
# The store's units exist only on its host, so elsewhere the fetch waits
# on nothing local. The arm where the store is here is in
# ./swarm-services-switch.nix.
name = "the CI runner's token fetch names none of the store's units on a host without the store";
ok =

7
nix/module-eval/hive-otel.nix
View file

@ -309,8 +309,8 @@ let
lib.hasInfix "bao kv get -field=value ${path}" fetch && lib.hasInfix "bao kv put ${path} " publish;
}
{
# The store's units exist only on its host; `Requires=` on an absent one
# fails the fetch's start.
# The store's units exist only on its host. The PKI unit is wanted, never
# required: a refused pki start fails a requirer's job with `dependency`.
name = "the hive collector's fetch waits for the store's container and PKI unit only where the store runs here";
ok =
let
@ -322,7 +322,8 @@ let
elsewhere = hiveOtelFetch hiveOtelLeaf;
in
lib.all (n: lib.elem n here.after) (storeUnits hiveOtelAllLocal)
&& lib.elem "swarm-bao-pki.service" here.requires
&& lib.elem "swarm-bao-pki.service" here.wants
&& !(lib.elem "swarm-bao-pki.service" here.requires)
&& lib.elem "container@${hiveOtelAllLocal.services.hyperhive.swarm.bao.machine}.service" here.wants
&& !(lib.any (n: lib.elem n (elsewhere.after ++ elsewhere.wants ++ elsewhere.requires)) (
storeUnits hiveOtelLeaf

4
nix/module-eval/swarm-otel-identity.nix
View file

@ -740,8 +740,8 @@ let
&& !(refusedOver otelRemoteAuthelia "otelOidcClientCertFile");
}
{
# The store's units exist only on its host; `Requires=` on an absent one
# fails the fetch's start. The arm where the store is here is in
# The store's units exist only on its host, so elsewhere the fetch waits
# on nothing local. The arm where the store is here is in
# ./swarm-services-switch.nix.
name = "the remote reader's fetch names none of the store's units";
ok =

19
nix/module-eval/swarm-services-switch.nix
View file

@ -804,12 +804,13 @@ let
}
{
# Where the store runs here, each reader waits for its container and
# requires the PKI unit minting the leaf it logs in with. Elsewhere
# neither unit exists, and `Requires=` on an absent unit fails the
# reader's start. On the store's host without the readers, none of them
# is rendered as a stub. Rosters are counted: an empty one holds
# vacuously.
name = "a store reader waits for the store's container and PKI unit only where the store runs here";
# the PKI unit minting the leaf it logs in with, and wants both. It
# never requires the PKI unit: a refused pki start fails a requirer's
# job with `dependency`, which its `Restart=on-failure` never retries.
# Elsewhere neither unit exists. On the store's host without the
# readers, none of them is rendered as a stub. Rosters are counted: an
# empty one holds vacuously.
name = "a store reader waits for and wants the store's container and PKI unit only where the store runs here, and requires neither";
ok =
let
machine = serviceContainersWithCi.services.hyperhive.swarm.bao.machine;
@ -831,12 +832,14 @@ let
];
here = serviceContainersWithCi.systemd.services;
elsewhere = swarmServicesBaoElsewhere.systemd.services;
readersHere = lib.filter (u: here ? ${u}) readers;
readersElsewhere = lib.filter (u: elsewhere ? ${u}) readers;
waits =
u:
here ? ${u}
&& lib.elem "swarm-bao-pki.service" here.${u}.after
&& lib.elem "swarm-bao-pki.service" here.${u}.requires
&& lib.elem "swarm-bao-pki.service" here.${u}.wants
&& !(lib.any (n: lib.elem n here.${u}.requires) storeUnits)
&& lib.elem "container@${machine}.service" here.${u}.after
&& lib.elem "container@${machine}.service" here.${u}.wants;
namesNoStoreUnit =
@ -847,6 +850,8 @@ let
!(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits);
in
lib.all waits readers
&& lib.length readersHere == 10
&& lib.elem "swarm-bao-grafana-admin" readersHere
&& lib.length readersElsewhere == 7
&& lib.all namesNoStoreUnit readersElsewhere
&& !(lib.any (u: storeOnlyNoHive.systemd.services ? ${u}) readers);