report the granter login's real error and retry on a standby store

nix/host-modules/lib/granter-login.nix

@@ -0,0 +1,57 @@
+# The cert login every unit that writes bao policies starts with, as shell text
+# that leaves `BAO_TOKEN` exported. It FAILS rather than skips: a grant that
+# was not written is otherwise invisible until whatever needs it fails
+# somewhere else.
+#
+# What the failure says follows from what the store answered:
+#
+# - A refused certificate ("invalid certificate or no client certificate
+#   supplied", or "no chain matching all constraints"; openbao's cert-auth
+#   login in `builtin/credential/cert/path_login.go`, HTTP 400) is the granter
+#   role missing or not matching. Only that answer prints the one-time setup
+#   steps.
+# - Any other failure from a store that answers `status` is not the role's:
+#   a node that is not yet active answers the login with a 500 ("local node
+#   not active but active cluster node not found") for the first moments after
+#   an unseal. The message says so and the unit's `Restart=` retries.
+# - A store that does not answer `status` is sealed or unreachable.
+#
+# In every case bao's own error follows.
+#
+# Pure function — NOT a NixOS module. Call it from a module's `let`:
+#
+#   granterLogin = import ./lib/granter-login.nix {
+#     inherit lib granterCn granterPolicyName haveBootstrapToken baoStoreUp;
+#     setupSteps = granterSetupSteps;
+#   };
+#
+# `baoStoreUp` is shell text that exits 0 on a reachable, unsealed store and
+# appends its error to `$err`. Needs `bao`, `coreutils` and `gnugrep` on the
+# caller's `path`. Takes the EXIT trap, to remove its scratch file.
+{
+  lib,
+  granterCn,
+  granterPolicyName,
+  haveBootstrapToken,
+  baoStoreUp,
+  setupSteps,
+}:
+''
+  err="$(mktemp)"
+  trap 'rm -f "$err"' EXIT
+  if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
+    if grep -qE 'invalid certificate or no client certificate supplied|no chain matching all constraints' "$err"; then
+      echo ${lib.escapeShellArg "the store refused the granter's certificate (CN ${granterCn}): the ${granterPolicyName} role is not set up."} >&2
+      ${lib.optionalString haveBootstrapToken ''echo "swarm-bao-granter-role writes that role; journalctl -u swarm-bao-granter-role says why it has not." >&2''}
+      echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2
+      ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg "  ${l}"} >&2") setupSteps}
+    elif ${baoStoreUp}; then
+      echo "the store is unsealed but the granter's login failed without a refused certificate, as a node that is not active yet answers; retrying." >&2
+    else
+      echo "the store is sealed or unreachable; retrying." >&2
+    fi
+    cat "$err" >&2
+    exit 1
+  fi
+  export BAO_TOKEN
+''

nix/host-modules/swarm-bao.nix

@@ -311,27 +311,18 @@ let
   # config and no token.
   baoStoreUp = ''HOME=/var/empty bao status >/dev/null 2>>"$err"'';
 
-  # The login every granting unit starts with. It FAILS rather than skips: a
-  # grant that was not written is otherwise invisible until whatever needs it
-  # fails somewhere else. `baoStoreUp` separates "the granter is not set up"
-  # from "retry later"; either way bao's own message follows.
-  granterLogin = ''
-    err="$(mktemp)"
-    trap 'rm -f "$err"' EXIT
-    if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
-      if ${baoStoreUp}; then
-        echo ${lib.escapeShellArg "the store is unsealed but refused the granter's certificate (CN ${granterCn}): the ${granterPolicyName} role is not set up."} >&2
-        ${lib.optionalString haveBootstrapToken ''echo "swarm-bao-granter-role writes that role; journalctl -u swarm-bao-granter-role says why it has not." >&2''}
-        echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2
-        ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg "  ${l}"} >&2") granterSetupSteps}
-      else
-        echo "the store is sealed or unreachable; retrying." >&2
-      fi
-      cat "$err" >&2
-      exit 1
-    fi
-    export BAO_TOKEN
-  '';
+  # The login every granting unit starts with; ./lib/granter-login.nix says
+  # which failures it reports as a missing role.
+  granterLogin = import ./lib/granter-login.nix {
+    inherit
+      lib
+      granterCn
+      granterPolicyName
+      haveBootstrapToken
+      baoStoreUp
+      ;
+    setupSteps = granterSetupSteps;
+  };
 
   # The granter's certificate for the granting units. The `baoCli` wrapper
   # only defaults these, so the unit's environment wins.