report the real error when the granter role's bootstrap step fails, and retry on a standby store

nix/host-modules/lib/bootstrap-auth-list.nix

@@ -0,0 +1,57 @@
+# The bootstrap token's first use in `swarm-bao-granter-role`, as shell text
+# that leaves `mounted` set to the store's auth listing, or ends the script.
+# Same split as ./granter-login.nix, applied to `bao auth list`:
+#
+# - A refusal ("permission denied", HTTP 403: openbao answers an expired,
+#   revoked or underprivileged token alike) is the token's. If the granter's
+#   own login works the role is already in place and the dead token is
+#   deleted; otherwise the unit prints the one-time steps and exits 4, which
+#   `RestartPreventExitStatus` keeps from retrying.
+# - Any other failure from a store that answers `status` is not the token's:
+#   a node that is not yet active answers with a 500 for the first moments
+#   after an unseal. The message says so and exits 1, which `Restart=` retries.
+# - A store that does not answer `status` is sealed or unreachable; exit 1.
+#
+# In every case bao's own error follows.
+#
+# Pure function — NOT a NixOS module. `baoStoreUp` is as for
+# ./granter-login.nix. Needs `bao`, `coreutils` and `gnugrep` on the caller's
+# `path`, and `$err` to be a scratch file.
+{
+  lib,
+  bootstrapTokenFile,
+  granterClientCertFile,
+  granterClientKeyFile,
+  baoStoreUp,
+  setupSteps,
+}:
+''
+  if ! mounted="$(bao auth list -format=json 2>"$err")"; then
+    if grep -qF 'permission denied' "$err"; then
+      # A dead token beside a working granter is the steady state of
+      # a host that finished bootstrap and kept the file. The
+      # granter's own login is the proof. A fresh token still takes
+      # the full path below, since the granter cannot read back its
+      # own policy to compare it. The refused token is dead,
+      # so it is deleted here and the unit does not start again.
+      if BAO_CLIENT_CERT=${lib.escapeShellArg granterClientCertFile} \
+        BAO_CLIENT_KEY=${lib.escapeShellArg granterClientKeyFile} \
+        bao login -method=cert -token-only >/dev/null 2>&1; then
+        echo "granter role already in place; bootstrap token not needed"
+        rm -f ${lib.escapeShellArg bootstrapTokenFile}
+        exit 0
+      fi
+      echo ${lib.escapeShellArg "the store refused the bootstrap token in ${bootstrapTokenFile}: it has expired, been revoked, or does not carry the bao-bootstrap policy."} >&2
+      echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2
+      ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg "  ${l}"} >&2") setupSteps}
+      cat "$err" >&2
+      exit 4
+    elif ${baoStoreUp}; then
+      echo "the store is unsealed but listing its auth mounts failed without a refused token, as a node that is not active yet answers; retrying." >&2
+    else
+      echo "the store is sealed or unreachable; retrying." >&2
+    fi
+    cat "$err" >&2
+    exit 1
+  fi
+''

nix/host-modules/swarm-bao.nix

@@ -324,6 +324,17 @@ let
     setupSteps = granterSetupSteps;
   };
 
+  # The bootstrap token's first use, in `swarm-bao-granter-role`.
+  bootstrapAuthList = import ./lib/bootstrap-auth-list.nix {
+    inherit lib baoStoreUp;
+    inherit (baoDeploy)
+      bootstrapTokenFile
+      granterClientCertFile
+      granterClientKeyFile
+      ;
+    setupSteps = granterSetupSteps;
+  };
+
   # The granter's certificate for the granting units. The `baoCli` wrapper
   # only defaults these, so the unit's environment wins.
   granterEnv = {
@@ -3374,6 +3385,7 @@ in
         path = [
           baoCli
           pkgs.coreutils
+          pkgs.gnugrep
         ];
         # Named but not placed is a legitimate state: all-local supplies the
         # path as a default and the operator drops the file there after
@@ -3426,34 +3438,9 @@ in
           # that already exists, and recognising that would tie a rebuild
           # to an error string we have never seen this store emit.
           #
-          # The token's first use, and the bootstrap policy grants this read,
-          # so a refusal from a store that is up is the token's: expired,
-          # revoked, or minted without that policy.
-          if ! mounted="$(bao auth list -format=json 2>"$err")"; then
-            if ${baoStoreUp}; then
-              # A dead token beside a working granter is the steady state of
-              # a host that finished bootstrap and kept the file. The
-              # granter's own login is the proof. A fresh token still takes
-              # the full path below, since the granter cannot read back its
-              # own policy to compare it. The refused token is dead,
-              # so it is deleted here and the unit does not start again.
-              if BAO_CLIENT_CERT=${lib.escapeShellArg baoDeploy.granterClientCertFile} \
-                BAO_CLIENT_KEY=${lib.escapeShellArg baoDeploy.granterClientKeyFile} \
-                bao login -method=cert -token-only >/dev/null 2>&1; then
-                echo "granter role already in place; bootstrap token not needed"
-                rm -f ${lib.escapeShellArg baoDeploy.bootstrapTokenFile}
-                exit 0
-              fi
-              echo ${lib.escapeShellArg "the store is unsealed but refused the bootstrap token in ${baoDeploy.bootstrapTokenFile}: it has expired, been revoked, or does not carry the bao-bootstrap policy."} >&2
-              echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2
-              ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg "  ${l}"} >&2") granterSetupSteps}
-              cat "$err" >&2
-              exit 4
-            fi
-            echo "the store is sealed or unreachable; retrying." >&2
-            cat "$err" >&2
-            exit 1
-          fi
+          # The token's first use; ./lib/bootstrap-auth-list.nix says which
+          # failures are the token's.
+          ${bootstrapAuthList}
           case "$mounted" in
             *'"cert/"'*) ;;
             *) bao auth enable cert ;;