make bao readers want swarm-bao-pki and drop its start limit

A pki start refused by its start limit failed every reader that
Requires= it with `dependency`. That leaves a unit inactive, not
failed, so Restart=on-failure never fired and readers without a
refresh timer never ran again (#5381: grafana 502 for ~43 min).

Readers and granting units now Wants= + After= pki; one whose leaf is
still missing fails its login with exit 1 into its existing retry.
pki gets StartLimitIntervalSec=0: it is deliberately not
RemainAfterExit, so each reader's start re-runs it.

docs/swarm/bao.md

@@ -100,6 +100,10 @@ principal — and points `deploy.bao.serverCertFile`, `.serverKeyFile` and
 It runs at every boot and every deploy, and before each start of a container
 that loads one of its leaves, and signs only the files that are missing, so a
 leaf a deploy adds, or a service a deploy enables, has its file on that deploy.
+The store's readers and granting units on this host order after it and want it,
+but none requires it: when a run fails, each of them still starts, and one
+whose leaf is missing fails its login into its own retry. It has no start
+limit, since a mass container start pulls it in once for each unit it starts.
 
 Those are `mkDefault`s, so naming your own paths wins. Do that when your
 certificates come from a real internal CA; the store has no opinion about

nix/host-modules/glue-bao-readers-store-order.nix

@@ -1,18 +1,23 @@
 # Glue: where the store and one of its readers share a host, the reader waits
-# for the store's container and requires `swarm-bao-pki`, which mints the leaf
+# for the store's container and `swarm-bao-pki`, which mints the leaf the
-# the reader logs in with.
+# reader logs in with.
 #
 # ONE PAIRING PER FILE — the store's container and PKI unit ← the readers that
 # log in to it, and nothing else. The waits on the store's policy units are
 # ./glue-bao-readers-policy-order.nix's.
 #
+# `Wants=` on `swarm-bao-pki`, not `Requires=`: a pki start that fails or is
+# refused fails each requirer's job with `dependency`, which leaves the reader
+# inactive rather than failed, so its `Restart=on-failure` never fires. Here
+# the reader runs anyway; with its leaf still missing, its login exits 1 into
+# ./lib/store-retry.nix's retries.
+#
 # ⚠️ Gated on BOTH the store and that reader being here. Off the store's host
-# neither unit exists, and `Requires=` on an absent unit fails the job, so
+# neither unit exists, and the reader's own start timeout bounds the attempt
-# there the reader's own start timeout bounds the attempt instead. On the
+# instead. On the store's host without the reader, setting
-# store's host without the reader, setting `systemd.services.<reader>.after`
+# `systemd.services.<reader>.after` would define a unit with no ExecStart, so
-# would define a unit with no ExecStart, so each gate below restates the one
+# each gate below restates the one the reader's own module puts on it. A
-# the reader's own module puts on it. A reader whose gate changes must change
+# reader whose gate changes must change here too.
-# here too.
 {
   lib,
   config,
@@ -65,8 +70,10 @@ let
           "swarm-bao-pki.service"
           "container@${baoCfg.machine}.service"
         ];
-        wants = [ "container@${baoCfg.machine}.service" ];
+        wants = [
-        requires = [ "swarm-bao-pki.service" ];
+          "swarm-bao-pki.service"
+          "container@${baoCfg.machine}.service"
+        ];
       };
     };
 in

nix/host-modules/glue-bao-tls.nix

@@ -149,6 +149,11 @@ in
       before = [ "swarm-bao-certs.service" ];
       requiredBy = [ "swarm-bao-certs.service" ];
       wantedBy = [ "multi-user.target" ];
+      # No start limit: a mass container start pulls this in once per unit it
+      # starts, which outruns systemd's default of 5 starts in 10s. No
+      # `Restart=`, so nothing here loops. `[Unit]` setting, hence not in
+      # `serviceConfig`.
+      startLimitIntervalSec = 0;
       path = [
         pkgs.openssl
         pkgs.coreutils

nix/host-modules/glue-queue-agent-credential.nix

@@ -158,16 +158,19 @@ in
     (lib.mkIf (deployCfg.hive-controller.enable && haveClientIdentity) {
       systemd.services.swarm-bao-queue-agent = {
         description = "fetch this hive's agent queue credential from the swarm secret store";
-        # Every one of these names a unit that exists only where the store runs.
+        # Every one of these names a unit that exists only where the store runs,
-        # `Requires=` on an absent unit fails the job outright, so the ordering is
+        # so the ordering is conditional even though the read is not: off-host
-        # conditional even though the read is not: off-host there is nothing local
+        # there is nothing local to wait for, and the timeout below is what
-        # to wait for, and the timeout below is what bounds the attempt instead.
+        # bounds the attempt instead. `Wants=` pki, for
+        # ./glue-bao-readers-store-order.nix's reason.
         after = lib.optionals baoDeploy.enable [
           "swarm-bao-pki.service"
           "container@${baoCfg.machine}.service"
         ];
-        wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
+        wants = lib.optionals baoDeploy.enable [
-        requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
+          "swarm-bao-pki.service"
+          "container@${baoCfg.machine}.service"
+        ];
         # Ordered before hive-c0re, so no agent container renders ahead of an
         # attempt at its credential. `Wants=`, not `Requires=`: a store this
         # unit can't reach delays hive-c0re's start by one attempt

nix/host-modules/swarm-bao.nix

@@ -332,6 +332,10 @@ let
 
   # `swarm-bao-pki` mints the granter's leaf; the granter's role is written by
   # `swarm-bao-granter-role`, which normally skips, hence ordering only there.
+  # Each granting unit `Wants=` pki and never `Requires=` it: a refused or
+  # failed pki start fails a requirer's job with `dependency`, which no
+  # `Restart=` retries, while a leaf still missing fails `granterLogin` into
+  # the unit's own retry.
   granterAfter = [
     "swarm-bao-pki.service"
     "swarm-bao-granter-role.service"
@@ -1166,7 +1170,7 @@ let
         "swarm-bao-controller-policy.service"
       ]
       ++ granterAfter;
-      requires = [ "swarm-bao-pki.service" ];
+      wants = [ "swarm-bao-pki.service" ];
       wantedBy = [ "multi-user.target" ];
       path = [
         baoCli
@@ -2790,8 +2794,11 @@ in
           "swarm-bao-pki.service"
           "container@${cfg.machine}.service"
         ];
-        wants = [ "container@${cfg.machine}.service" ];
+        # `Wants=` pki, for ./glue-bao-readers-store-order.nix's reason.
-        requires = [ "swarm-bao-pki.service" ];
+        wants = [
+          "swarm-bao-pki.service"
+          "container@${cfg.machine}.service"
+        ];
         wantedBy = [ "multi-user.target" ];
         path = [
           baoDeploy.package
@@ -3137,7 +3144,7 @@ in
       systemd.services.swarm-bao-controller-policy = lib.mkIf haveGranter {
         description = "write the swarm controller's bao policy and cert-auth role";
         after = [ "container@${cfg.machine}.service" ] ++ granterAfter;
-        requires = [ "swarm-bao-pki.service" ];
+        wants = [ "swarm-bao-pki.service" ];
         wantedBy = [ "multi-user.target" ];
         # The wrapper rather than the package: it carries the address and the
         # CA, which is what makes running here cheaper than shipping an
@@ -3563,7 +3570,7 @@ in
           "swarm-bao-forge-publisher-policy.service"
         ]
         ++ granterAfter;
-        requires = [ "swarm-bao-pki.service" ];
+        wants = [ "swarm-bao-pki.service" ];
         wantedBy = [ "multi-user.target" ];
         path = [
           baoCli

nix/module-eval/bao-grants.nix

@@ -1097,14 +1097,16 @@ let
     {
       # Every granting unit retries a sealed or late store for a day, in the
       # `[Unit]` section systemd reads it from, and waits for the unit that
-      # mints the granter's leaf.
+      # mints the granter's leaf. Wanted, never required: a refused pki start
-      name = "each granting unit requires the PKI unit and retries 2880 times at 30s";
+      # fails a requirer's job with `dependency`, which no `Restart=` retries.
+      name = "each granting unit wants, not requires, the PKI unit and retries 2880 times at 30s";
       ok = lib.all (
         unit:
         let
           u = baoGrantHere.systemd.services.${unit};
         in
-        lib.elem "swarm-bao-pki.service" u.requires
+        lib.elem "swarm-bao-pki.service" u.wants
+        && !(lib.elem "swarm-bao-pki.service" u.requires)
         && lib.elem "swarm-bao-pki.service" u.after
         && lib.elem "swarm-bao-granter-role.service" u.after
         && !(lib.elem "swarm-bao-granter-role.service" (u.requires ++ u.wants))
@@ -1114,6 +1116,14 @@ let
         && u.serviceConfig.Restart == "on-failure"
       ) grantingUnitNames;
     }
+    {
+      # Every reader and granting unit's start pulls the PKI unit in, so a mass
+      # container start outruns systemd's default limit of 5 starts in 10s.
+      name = "the PKI unit has no start limit";
+      ok =
+        toString (baoGrantHere.systemd.services.swarm-bao-pki.unitConfig.StartLimitIntervalSec or "unset")
+        == "0";
+    }
     {
       # A oneshot has no start timeout by default, so an attempt that never
       # ends never fails into a retry.

nix/module-eval/bao-matrix-reader.nix

@@ -162,26 +162,32 @@ let
         && !(builtins.elem "hive-c0re.service" (u.requires or [ ]));
     }
     {
-      # `Requires=` on a unit that does not exist fails the job, and nothing
+      # Nothing local mints certificates off-host, so this orders against
-      # local mints certificates off-host — so this orders against nothing.
+      # nothing there.
-      # Eval cannot see that failure; only the empty list here stands in for it.
+      name = "an off-host reader names no unit the store's host would have provided";
-      name = "an off-host reader requires no unit the store's host would have provided";
       # Membership first, then the value: indexing a missing unit throws, and a
       # table that reports which property broke must not be the thing that dies.
       ok =
         let
           s = baoRemoteReader.systemd.services;
+          u = s.swarm-bao-queue-agent;
         in
-        s ? swarm-bao-queue-agent && s.swarm-bao-queue-agent.requires == [ ];
+        s ? swarm-bao-queue-agent && !(lib.elem "swarm-bao-pki.service" (u.after ++ u.wants ++ u.requires));
     }
     {
       # Presence control for the case above: the list is conditional, not gone.
-      name = "a co-located reader still orders after the local pki unit";
+      # Wanted, never required: a refused pki start fails a requirer's job with
+      # `dependency`, which its `Restart=on-failure` never retries.
+      name = "a co-located reader still orders after and wants the local pki unit, and does not require it";
       ok =
         let
           s = baoWithMatrix.systemd.services;
+          u = s.swarm-bao-queue-agent;
         in
-        s ? swarm-bao-queue-agent && s.swarm-bao-queue-agent.requires == [ "swarm-bao-pki.service" ];
+        s ? swarm-bao-queue-agent
+        && lib.elem "swarm-bao-pki.service" u.after
+        && lib.elem "swarm-bao-pki.service" u.wants
+        && u.requires == [ ];
     }
     {
       # Nothing asserted this script before, which is how it kept a branch that

nix/module-eval/bao-otel-collector.nix

@@ -420,7 +420,8 @@ let
         in
         lib.elem "container@swarm-bao.service" u.after
         && !(lib.elem "container@swarm-bao.service" (u.before or [ ]))
-        && lib.elem "swarm-bao-pki.service" u.requires;
+        && lib.elem "swarm-bao-pki.service" u.wants
+        && !(lib.elem "swarm-bao-pki.service" u.requires);
     }
     {
       # The far end of the credential. An authenticator checks ONE audience,

nix/module-eval/forge-placement.nix

@@ -190,8 +190,8 @@ let
           == "/etc/pki/bao-ci-runner.pem";
     }
     {
-      # The store's units exist only on its host; `Requires=` on an absent one
+      # The store's units exist only on its host, so elsewhere the fetch waits
-      # fails the fetch's start. The arm where the store is here is in
+      # on nothing local. The arm where the store is here is in
       # ./swarm-services-switch.nix.
       name = "the CI runner's token fetch names none of the store's units on a host without the store";
       ok =

nix/module-eval/hive-otel.nix

@@ -309,8 +309,8 @@ let
         lib.hasInfix "bao kv get -field=value ${path}" fetch && lib.hasInfix "bao kv put ${path} " publish;
     }
     {
-      # The store's units exist only on its host; `Requires=` on an absent one
+      # The store's units exist only on its host. The PKI unit is wanted, never
-      # fails the fetch's start.
+      # required: a refused pki start fails a requirer's job with `dependency`.
       name = "the hive collector's fetch waits for the store's container and PKI unit only where the store runs here";
       ok =
         let
@@ -322,7 +322,8 @@ let
           elsewhere = hiveOtelFetch hiveOtelLeaf;
         in
         lib.all (n: lib.elem n here.after) (storeUnits hiveOtelAllLocal)
-        && lib.elem "swarm-bao-pki.service" here.requires
+        && lib.elem "swarm-bao-pki.service" here.wants
+        && !(lib.elem "swarm-bao-pki.service" here.requires)
         && lib.elem "container@${hiveOtelAllLocal.services.hyperhive.swarm.bao.machine}.service" here.wants
         && !(lib.any (n: lib.elem n (elsewhere.after ++ elsewhere.wants ++ elsewhere.requires)) (
           storeUnits hiveOtelLeaf

nix/module-eval/swarm-otel-identity.nix

@@ -740,8 +740,8 @@ let
         && !(refusedOver otelRemoteAuthelia "otelOidcClientCertFile");
     }
     {
-      # The store's units exist only on its host; `Requires=` on an absent one
+      # The store's units exist only on its host, so elsewhere the fetch waits
-      # fails the fetch's start. The arm where the store is here is in
+      # on nothing local. The arm where the store is here is in
       # ./swarm-services-switch.nix.
       name = "the remote reader's fetch names none of the store's units";
       ok =

nix/module-eval/swarm-services-switch.nix

@@ -804,12 +804,13 @@ let
     }
     {
       # Where the store runs here, each reader waits for its container and
-      # requires the PKI unit minting the leaf it logs in with. Elsewhere
+      # the PKI unit minting the leaf it logs in with, and wants both. It
-      # neither unit exists, and `Requires=` on an absent unit fails the
+      # never requires the PKI unit: a refused pki start fails a requirer's
-      # reader's start. On the store's host without the readers, none of them
+      # job with `dependency`, which its `Restart=on-failure` never retries.
-      # is rendered as a stub. Rosters are counted: an empty one holds
+      # Elsewhere neither unit exists. On the store's host without the
-      # vacuously.
+      # readers, none of them is rendered as a stub. Rosters are counted: an
-      name = "a store reader waits for the store's container and PKI unit only where the store runs here";
+      # empty one holds vacuously.
+      name = "a store reader waits for and wants the store's container and PKI unit only where the store runs here, and requires neither";
       ok =
         let
           machine = serviceContainersWithCi.services.hyperhive.swarm.bao.machine;
@@ -831,12 +832,14 @@ let
           ];
           here = serviceContainersWithCi.systemd.services;
           elsewhere = swarmServicesBaoElsewhere.systemd.services;
+          readersHere = lib.filter (u: here ? ${u}) readers;
           readersElsewhere = lib.filter (u: elsewhere ? ${u}) readers;
           waits =
             u:
             here ? ${u}
             && lib.elem "swarm-bao-pki.service" here.${u}.after
-            && lib.elem "swarm-bao-pki.service" here.${u}.requires
+            && lib.elem "swarm-bao-pki.service" here.${u}.wants
+            && !(lib.any (n: lib.elem n here.${u}.requires) storeUnits)
             && lib.elem "container@${machine}.service" here.${u}.after
             && lib.elem "container@${machine}.service" here.${u}.wants;
           namesNoStoreUnit =
@@ -847,6 +850,8 @@ let
             !(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits);
         in
         lib.all waits readers
+        && lib.length readersHere == 10
+        && lib.elem "swarm-bao-grafana-admin" readersHere
         && lib.length readersElsewhere == 7
         && lib.all namesNoStoreUnit readersElsewhere
         && !(lib.any (u: storeOnlyNoHive.systemd.services ? ${u}) readers);