swarm-bao: say "run the bootstrap step" when the bootstrap token is dead, not "sealed"

swarm-bao-granter-role used its token for `bao auth list` with no check,
so an expired, revoked or policy-less token in bootstrapTokenFile exited
2 with a raw 403 and no hint. It now checks whether the store is up when
that call fails: if it is, the token is at fault, and the unit prints the
one-time bootstrap step and exits 4. A missing token file is still a
ConditionPathExists skip, so the two read differently in the journal.

The twelve granterLogin units printed "the store is sealed or
unreachable" on a healthy store, because `bao status` exited 1 there:
the CLI resolves a token helper under $HOME before asking, systemd sets
no HOME for a unit without User=, and the fallback shells out to
`getent`/`sh`, neither of which is on the unit's PATH ("failed to get
token helper: error expanding config path "": exec: "sh": executable
file not found in $PATH"). The check now runs with HOME=/var/empty and
keeps its stderr, so a genuinely unreachable store says why. When the
store is up and the login is refused, the units now name
swarm-bao-granter-role as the unit that writes the missing role.

setup.md's post-step restart used 'swarm-bao-*-policy.service', which
misses swarm-bao-agent-pki. It now names that unit too, and a
module-eval case fails when the restart misses any unit that logs in as
the granter.

Refs #4704

docs/getting-started/setup.md

@@ -123,11 +123,13 @@ the side that has one.
 
 **Confirm with `systemctl status swarm-bao-granter-role`**, which should log
 `Uploaded policy: bao-granter` and `Data written to: auth/cert/certs/bao-granter`.
-Then restart the granting units that failed while they waited:
+Then restart the granting units that failed while they waited. These two
+names cover every unit that logs in as the granter, and CI fails when one
+doesn't:
 
 ```bash
-systemctl reset-failed 'swarm-bao-*-policy.service'
+systemctl reset-failed 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service
-systemctl restart 'swarm-bao-*-policy.service'
+systemctl restart 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service
 systemctl status swarm-bao-controller-policy   # Uploaded policy, Data written to: auth/cert/certs/swarm-controller
 rm /var/lib/swarm-bao-bootstrap/grant.token
 ```

nix/host-modules/swarm-bao.nix

@@ -285,17 +285,24 @@ let
       ]
   );
 
+  # Exit 0 only on a reachable, unsealed store; its error, if any, is appended
+  # to `$err`. `status` needs no token, but the CLI resolves a token helper
+  # under `$HOME` before it asks, and systemd sets no `HOME` for a unit without
+  # `User=`: there it exits 1 on a healthy store. An empty home holds no helper
+  # config and no token.
+  baoStoreUp = ''HOME=/var/empty bao status >/dev/null 2>>"$err"'';
+
   # The login every granting unit starts with. It FAILS rather than skips: a
   # grant that was not written is otherwise invisible until whatever needs it
-  # fails somewhere else. `bao status` exits 0 only on a reachable, unsealed
+  # fails somewhere else. `baoStoreUp` separates "the granter is not set up"
-  # store, which separates "the granter is not set up" from "retry later";
+  # from "retry later"; either way bao's own message follows.
-  # either way bao's own message follows.
   granterLogin = ''
     err="$(mktemp)"
     trap 'rm -f "$err"' EXIT
     if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
-      if bao status >/dev/null 2>&1; then
+      if ${baoStoreUp}; then
         echo ${lib.escapeShellArg "the store is unsealed but refused the granter's certificate (CN ${granterCn}): the ${granterPolicyName} role is not set up."} >&2
+        ${lib.optionalString haveBootstrapToken ''echo "swarm-bao-granter-role writes that role; journalctl -u swarm-bao-granter-role says why it has not." >&2''}
         echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2
         ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg "  ${l}"} >&2") granterSetupSteps}
       else
@@ -2379,13 +2386,31 @@ in
           BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})"
           export BAO_TOKEN
 
+          err="$(mktemp)"
+          trap 'rm -f "$err"' EXIT
+
           # Every cert-auth role in this file lives under `auth/cert/`, and
           # nothing else creates that mount.
           #
           # Asked rather than attempted: `auth enable` errors on a mount
           # that already exists, and recognising that would tie a rebuild
           # to an error string we have never seen this store emit.
-          mounted="$(bao auth list -format=json)"
+          #
+          # The token's first use, and the bootstrap policy grants this read,
+          # so a refusal from a store that is up is the token's: expired,
+          # revoked, or minted without that policy.
+          if ! mounted="$(bao auth list -format=json 2>"$err")"; then
+            if ${baoStoreUp}; then
+              echo ${lib.escapeShellArg "the store is unsealed but refused the bootstrap token in ${baoDeploy.bootstrapTokenFile}: it has expired, been revoked, or does not carry the bao-bootstrap policy."} >&2
+              echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2
+              ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg "  ${l}"} >&2") granterSetupSteps}
+              cat "$err" >&2
+              exit 4
+            fi
+            echo "the store is sealed or unreachable; retrying." >&2
+            cat "$err" >&2
+            exit 1
+          fi
           case "$mounted" in
             *'"cert/"'*) ;;
             *) bao auth enable cert ;;

nix/module-eval/bao-grants.nix

@@ -821,6 +821,23 @@ let
         && lib.hasInfix "systemctl restart swarm-bao-granter-role" sc
       ) grantingUnitNames;
     }
+    {
+      # A token the store refuses gets the same step, under an exit status of
+      # its own. Both store checks run with a HOME: without one `bao status`
+      # exits 1 on a healthy store and every refusal reads as "sealed".
+      name = "a refused bootstrap token prints the one-time step and exits 4, and every store check has a HOME";
+      ok =
+        let
+          s = baoGrantHere.systemd.services;
+          g = s.swarm-bao-granter-role.script;
+        in
+        lib.hasInfix "refused the bootstrap token in /run/secrets/bao-bootstrap.token" g
+        && lib.hasInfix "bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl" g
+        && lib.hasInfix "exit 4" g
+        && lib.all (unit: lib.hasInfix "HOME=/var/empty bao status" s.${unit}.script) (
+          [ "swarm-bao-granter-role" ] ++ grantingUnitNames
+        );
+    }
     {
       # Every granting unit retries a sealed or late store for a day, in the
       # `[Unit]` section systemd reads it from, and waits for the unit that
@@ -1164,6 +1181,34 @@ let
         && lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits)
         && lib.all (u: baoCalls u.script != [ ]) (lib.attrValues bootstrapUnits);
     }
+    {
+      # setup.md's restart after the one-time step: a granting unit it misses
+      # stays failed once its start limit is hit.
+      name = "setup.md's reset-failed and restart after the one-time step reach every granting unit";
+      ok =
+        let
+          lines = lib.splitString "\n" (builtins.readFile ../../docs/getting-started/setup.md);
+          # The unit patterns on the one `systemctl <verb> 'swarm-bao-…` line.
+          argsOf =
+            verb:
+            map (l: map (lib.replaceStrings [ "'" ] [ "" ]) (lib.drop 2 (lib.splitString " " l))) (
+              lib.filter (lib.hasPrefix "systemctl ${verb} 'swarm-bao-") lines
+            );
+          covers =
+            pats:
+            lib.all (
+              unit:
+              lib.any (
+                p: builtins.match (lib.replaceStrings [ "." "*" ] [ "[.]" ".*" ] p) "${unit}.service" != null
+              ) pats
+            ) (lib.attrNames granterUnits);
+        in
+        lib.length (argsOf "restart") == 1
+        && argsOf "reset-failed" == argsOf "restart"
+        && covers (lib.head (argsOf "restart"))
+        # The control: the policy glob alone misses one.
+        && !(covers [ "swarm-bao-*-policy.service" ]);
+    }
     {
       # And the grants side: a stanza the parser skipped would read as a
       # grant that is not there.