From 265f8a47ccdf40d47dc532a91692e6e56e43e0ef Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 16 Oct 2026 18:29:35 +0200 Subject: [PATCH] make bao readers want swarm-bao-pki and drop its start limit A pki start refused by its start limit failed every reader that Requires= it with `dependency`. That leaves a unit inactive, not failed, so Restart=on-failure never fired and readers without a refresh timer never ran again (#5381: grafana 502 for ~43 min). Readers and granting units now Wants= + After= pki; one whose leaf is still missing fails its login with exit 1 into its existing retry. pki gets StartLimitIntervalSec=0: it is deliberately not RemainAfterExit, so each reader's start re-runs it. --- docs/swarm/bao.md | 4 +++ .../glue-bao-readers-store-order.nix | 27 ++++++++++++------- nix/host-modules/glue-bao-tls.nix | 5 ++++ .../glue-queue-agent-credential.nix | 15 ++++++----- nix/host-modules/swarm-bao.nix | 17 ++++++++---- nix/module-eval/bao-grants.nix | 16 ++++++++--- nix/module-eval/bao-matrix-reader.nix | 20 +++++++++----- nix/module-eval/bao-otel-collector.nix | 3 ++- nix/module-eval/forge-placement.nix | 4 +-- nix/module-eval/hive-otel.nix | 7 ++--- nix/module-eval/swarm-otel-identity.nix | 4 +-- nix/module-eval/swarm-services-switch.nix | 19 ++++++++----- 12 files changed, 95 insertions(+), 46 deletions(-) diff --git a/docs/swarm/bao.md b/docs/swarm/bao.md index 42e0ff9d..268d307f 100644 --- a/docs/swarm/bao.md +++ b/docs/swarm/bao.md @@ -100,6 +100,10 @@ principal — and points `deploy.bao.serverCertFile`, `.serverKeyFile` and It runs at every boot and every deploy, and before each start of a container that loads one of its leaves, and signs only the files that are missing, so a leaf a deploy adds, or a service a deploy enables, has its file on that deploy. +The store's readers and granting units on this host order after it and want it, +but none requires it: when a run fails, each of them still starts, and one +whose leaf is missing fails its login into its own retry. It has no start +limit, since a mass container start pulls it in once for each unit it starts. Those are `mkDefault`s, so naming your own paths wins. Do that when your certificates come from a real internal CA; the store has no opinion about diff --git a/nix/host-modules/glue-bao-readers-store-order.nix b/nix/host-modules/glue-bao-readers-store-order.nix index 47689399..cd04e6d3 100644 --- a/nix/host-modules/glue-bao-readers-store-order.nix +++ b/nix/host-modules/glue-bao-readers-store-order.nix @@ -1,18 +1,23 @@ # Glue: where the store and one of its readers share a host, the reader waits -# for the store's container and requires `swarm-bao-pki`, which mints the leaf -# the reader logs in with. +# for the store's container and `swarm-bao-pki`, which mints the leaf the +# reader logs in with. # # ONE PAIRING PER FILE — the store's container and PKI unit ← the readers that # log in to it, and nothing else. The waits on the store's policy units are # ./glue-bao-readers-policy-order.nix's. # +# `Wants=` on `swarm-bao-pki`, not `Requires=`: a pki start that fails or is +# refused fails each requirer's job with `dependency`, which leaves the reader +# inactive rather than failed, so its `Restart=on-failure` never fires. Here +# the reader runs anyway; with its leaf still missing, its login exits 1 into +# ./lib/store-retry.nix's retries. +# # ⚠️ Gated on BOTH the store and that reader being here. Off the store's host -# neither unit exists, and `Requires=` on an absent unit fails the job, so -# there the reader's own start timeout bounds the attempt instead. On the -# store's host without the reader, setting `systemd.services..after` -# would define a unit with no ExecStart, so each gate below restates the one -# the reader's own module puts on it. A reader whose gate changes must change -# here too. +# neither unit exists, and the reader's own start timeout bounds the attempt +# instead. On the store's host without the reader, setting +# `systemd.services..after` would define a unit with no ExecStart, so +# each gate below restates the one the reader's own module puts on it. A +# reader whose gate changes must change here too. { lib, config, @@ -65,8 +70,10 @@ let "swarm-bao-pki.service" "container@${baoCfg.machine}.service" ]; - wants = [ "container@${baoCfg.machine}.service" ]; - requires = [ "swarm-bao-pki.service" ]; + wants = [ + "swarm-bao-pki.service" + "container@${baoCfg.machine}.service" + ]; }; }; in diff --git a/nix/host-modules/glue-bao-tls.nix b/nix/host-modules/glue-bao-tls.nix index e8d8b96c..387f8926 100644 --- a/nix/host-modules/glue-bao-tls.nix +++ b/nix/host-modules/glue-bao-tls.nix @@ -149,6 +149,11 @@ in before = [ "swarm-bao-certs.service" ]; requiredBy = [ "swarm-bao-certs.service" ]; wantedBy = [ "multi-user.target" ]; + # No start limit: a mass container start pulls this in once per unit it + # starts, which outruns systemd's default of 5 starts in 10s. No + # `Restart=`, so nothing here loops. `[Unit]` setting, hence not in + # `serviceConfig`. + startLimitIntervalSec = 0; path = [ pkgs.openssl pkgs.coreutils diff --git a/nix/host-modules/glue-queue-agent-credential.nix b/nix/host-modules/glue-queue-agent-credential.nix index c08a9a85..970dfbad 100644 --- a/nix/host-modules/glue-queue-agent-credential.nix +++ b/nix/host-modules/glue-queue-agent-credential.nix @@ -158,16 +158,19 @@ in (lib.mkIf (deployCfg.hive-controller.enable && haveClientIdentity) { systemd.services.swarm-bao-queue-agent = { description = "fetch this hive's agent queue credential from the swarm secret store"; - # Every one of these names a unit that exists only where the store runs. - # `Requires=` on an absent unit fails the job outright, so the ordering is - # conditional even though the read is not: off-host there is nothing local - # to wait for, and the timeout below is what bounds the attempt instead. + # Every one of these names a unit that exists only where the store runs, + # so the ordering is conditional even though the read is not: off-host + # there is nothing local to wait for, and the timeout below is what + # bounds the attempt instead. `Wants=` pki, for + # ./glue-bao-readers-store-order.nix's reason. after = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" "container@${baoCfg.machine}.service" ]; - wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; - requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; + wants = lib.optionals baoDeploy.enable [ + "swarm-bao-pki.service" + "container@${baoCfg.machine}.service" + ]; # Ordered before hive-c0re, so no agent container renders ahead of an # attempt at its credential. `Wants=`, not `Requires=`: a store this # unit can't reach delays hive-c0re's start by one attempt diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index ba214c6e..b7e51fbb 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -332,6 +332,10 @@ let # `swarm-bao-pki` mints the granter's leaf; the granter's role is written by # `swarm-bao-granter-role`, which normally skips, hence ordering only there. + # Each granting unit `Wants=` pki and never `Requires=` it: a refused or + # failed pki start fails a requirer's job with `dependency`, which no + # `Restart=` retries, while a leaf still missing fails `granterLogin` into + # the unit's own retry. granterAfter = [ "swarm-bao-pki.service" "swarm-bao-granter-role.service" @@ -1166,7 +1170,7 @@ let "swarm-bao-controller-policy.service" ] ++ granterAfter; - requires = [ "swarm-bao-pki.service" ]; + wants = [ "swarm-bao-pki.service" ]; wantedBy = [ "multi-user.target" ]; path = [ baoCli @@ -2790,8 +2794,11 @@ in "swarm-bao-pki.service" "container@${cfg.machine}.service" ]; - wants = [ "container@${cfg.machine}.service" ]; - requires = [ "swarm-bao-pki.service" ]; + # `Wants=` pki, for ./glue-bao-readers-store-order.nix's reason. + wants = [ + "swarm-bao-pki.service" + "container@${cfg.machine}.service" + ]; wantedBy = [ "multi-user.target" ]; path = [ baoDeploy.package @@ -3137,7 +3144,7 @@ in systemd.services.swarm-bao-controller-policy = lib.mkIf haveGranter { description = "write the swarm controller's bao policy and cert-auth role"; after = [ "container@${cfg.machine}.service" ] ++ granterAfter; - requires = [ "swarm-bao-pki.service" ]; + wants = [ "swarm-bao-pki.service" ]; wantedBy = [ "multi-user.target" ]; # The wrapper rather than the package: it carries the address and the # CA, which is what makes running here cheaper than shipping an @@ -3563,7 +3570,7 @@ in "swarm-bao-forge-publisher-policy.service" ] ++ granterAfter; - requires = [ "swarm-bao-pki.service" ]; + wants = [ "swarm-bao-pki.service" ]; wantedBy = [ "multi-user.target" ]; path = [ baoCli diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index 5e87d6c0..0fec9a92 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -1097,14 +1097,16 @@ let { # Every granting unit retries a sealed or late store for a day, in the # `[Unit]` section systemd reads it from, and waits for the unit that - # mints the granter's leaf. - name = "each granting unit requires the PKI unit and retries 2880 times at 30s"; + # mints the granter's leaf. Wanted, never required: a refused pki start + # fails a requirer's job with `dependency`, which no `Restart=` retries. + name = "each granting unit wants, not requires, the PKI unit and retries 2880 times at 30s"; ok = lib.all ( unit: let u = baoGrantHere.systemd.services.${unit}; in - lib.elem "swarm-bao-pki.service" u.requires + lib.elem "swarm-bao-pki.service" u.wants + && !(lib.elem "swarm-bao-pki.service" u.requires) && lib.elem "swarm-bao-pki.service" u.after && lib.elem "swarm-bao-granter-role.service" u.after && !(lib.elem "swarm-bao-granter-role.service" (u.requires ++ u.wants)) @@ -1114,6 +1116,14 @@ let && u.serviceConfig.Restart == "on-failure" ) grantingUnitNames; } + { + # Every reader and granting unit's start pulls the PKI unit in, so a mass + # container start outruns systemd's default limit of 5 starts in 10s. + name = "the PKI unit has no start limit"; + ok = + toString (baoGrantHere.systemd.services.swarm-bao-pki.unitConfig.StartLimitIntervalSec or "unset") + == "0"; + } { # A oneshot has no start timeout by default, so an attempt that never # ends never fails into a retry. diff --git a/nix/module-eval/bao-matrix-reader.nix b/nix/module-eval/bao-matrix-reader.nix index 6b505092..e8e41192 100644 --- a/nix/module-eval/bao-matrix-reader.nix +++ b/nix/module-eval/bao-matrix-reader.nix @@ -162,26 +162,32 @@ let && !(builtins.elem "hive-c0re.service" (u.requires or [ ])); } { - # `Requires=` on a unit that does not exist fails the job, and nothing - # local mints certificates off-host — so this orders against nothing. - # Eval cannot see that failure; only the empty list here stands in for it. - name = "an off-host reader requires no unit the store's host would have provided"; + # Nothing local mints certificates off-host, so this orders against + # nothing there. + name = "an off-host reader names no unit the store's host would have provided"; # Membership first, then the value: indexing a missing unit throws, and a # table that reports which property broke must not be the thing that dies. ok = let s = baoRemoteReader.systemd.services; + u = s.swarm-bao-queue-agent; in - s ? swarm-bao-queue-agent && s.swarm-bao-queue-agent.requires == [ ]; + s ? swarm-bao-queue-agent && !(lib.elem "swarm-bao-pki.service" (u.after ++ u.wants ++ u.requires)); } { # Presence control for the case above: the list is conditional, not gone. - name = "a co-located reader still orders after the local pki unit"; + # Wanted, never required: a refused pki start fails a requirer's job with + # `dependency`, which its `Restart=on-failure` never retries. + name = "a co-located reader still orders after and wants the local pki unit, and does not require it"; ok = let s = baoWithMatrix.systemd.services; + u = s.swarm-bao-queue-agent; in - s ? swarm-bao-queue-agent && s.swarm-bao-queue-agent.requires == [ "swarm-bao-pki.service" ]; + s ? swarm-bao-queue-agent + && lib.elem "swarm-bao-pki.service" u.after + && lib.elem "swarm-bao-pki.service" u.wants + && u.requires == [ ]; } { # Nothing asserted this script before, which is how it kept a branch that diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix index 453ab073..c05e56aa 100644 --- a/nix/module-eval/bao-otel-collector.nix +++ b/nix/module-eval/bao-otel-collector.nix @@ -420,7 +420,8 @@ let in lib.elem "container@swarm-bao.service" u.after && !(lib.elem "container@swarm-bao.service" (u.before or [ ])) - && lib.elem "swarm-bao-pki.service" u.requires; + && lib.elem "swarm-bao-pki.service" u.wants + && !(lib.elem "swarm-bao-pki.service" u.requires); } { # The far end of the credential. An authenticator checks ONE audience, diff --git a/nix/module-eval/forge-placement.nix b/nix/module-eval/forge-placement.nix index ddc39f5b..a6cf76e0 100644 --- a/nix/module-eval/forge-placement.nix +++ b/nix/module-eval/forge-placement.nix @@ -190,8 +190,8 @@ let == "/etc/pki/bao-ci-runner.pem"; } { - # The store's units exist only on its host; `Requires=` on an absent one - # fails the fetch's start. The arm where the store is here is in + # The store's units exist only on its host, so elsewhere the fetch waits + # on nothing local. The arm where the store is here is in # ./swarm-services-switch.nix. name = "the CI runner's token fetch names none of the store's units on a host without the store"; ok = diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 3d2defcc..d9acdd47 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -309,8 +309,8 @@ let lib.hasInfix "bao kv get -field=value ${path}" fetch && lib.hasInfix "bao kv put ${path} " publish; } { - # The store's units exist only on its host; `Requires=` on an absent one - # fails the fetch's start. + # The store's units exist only on its host. The PKI unit is wanted, never + # required: a refused pki start fails a requirer's job with `dependency`. name = "the hive collector's fetch waits for the store's container and PKI unit only where the store runs here"; ok = let @@ -322,7 +322,8 @@ let elsewhere = hiveOtelFetch hiveOtelLeaf; in lib.all (n: lib.elem n here.after) (storeUnits hiveOtelAllLocal) - && lib.elem "swarm-bao-pki.service" here.requires + && lib.elem "swarm-bao-pki.service" here.wants + && !(lib.elem "swarm-bao-pki.service" here.requires) && lib.elem "container@${hiveOtelAllLocal.services.hyperhive.swarm.bao.machine}.service" here.wants && !(lib.any (n: lib.elem n (elsewhere.after ++ elsewhere.wants ++ elsewhere.requires)) ( storeUnits hiveOtelLeaf diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index 1b5cde81..c970a2b8 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -740,8 +740,8 @@ let && !(refusedOver otelRemoteAuthelia "otelOidcClientCertFile"); } { - # The store's units exist only on its host; `Requires=` on an absent one - # fails the fetch's start. The arm where the store is here is in + # The store's units exist only on its host, so elsewhere the fetch waits + # on nothing local. The arm where the store is here is in # ./swarm-services-switch.nix. name = "the remote reader's fetch names none of the store's units"; ok = diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 7e63a99e..be3f35fe 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -804,12 +804,13 @@ let } { # Where the store runs here, each reader waits for its container and - # requires the PKI unit minting the leaf it logs in with. Elsewhere - # neither unit exists, and `Requires=` on an absent unit fails the - # reader's start. On the store's host without the readers, none of them - # is rendered as a stub. Rosters are counted: an empty one holds - # vacuously. - name = "a store reader waits for the store's container and PKI unit only where the store runs here"; + # the PKI unit minting the leaf it logs in with, and wants both. It + # never requires the PKI unit: a refused pki start fails a requirer's + # job with `dependency`, which its `Restart=on-failure` never retries. + # Elsewhere neither unit exists. On the store's host without the + # readers, none of them is rendered as a stub. Rosters are counted: an + # empty one holds vacuously. + name = "a store reader waits for and wants the store's container and PKI unit only where the store runs here, and requires neither"; ok = let machine = serviceContainersWithCi.services.hyperhive.swarm.bao.machine; @@ -831,12 +832,14 @@ let ]; here = serviceContainersWithCi.systemd.services; elsewhere = swarmServicesBaoElsewhere.systemd.services; + readersHere = lib.filter (u: here ? ${u}) readers; readersElsewhere = lib.filter (u: elsewhere ? ${u}) readers; waits = u: here ? ${u} && lib.elem "swarm-bao-pki.service" here.${u}.after - && lib.elem "swarm-bao-pki.service" here.${u}.requires + && lib.elem "swarm-bao-pki.service" here.${u}.wants + && !(lib.any (n: lib.elem n here.${u}.requires) storeUnits) && lib.elem "container@${machine}.service" here.${u}.after && lib.elem "container@${machine}.service" here.${u}.wants; namesNoStoreUnit = @@ -847,6 +850,8 @@ let !(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits); in lib.all waits readers + && lib.length readersHere == 10 + && lib.elem "swarm-bao-grafana-admin" readersHere && lib.length readersElsewhere == 7 && lib.all namesNoStoreUnit readersElsewhere && !(lib.any (u: storeOnlyNoHive.systemd.services ? ${u}) readers);