report the real error when the granter role's bootstrap step fails, and retry on a standby store

docs/swarm/bao.md

@@ -40,8 +40,10 @@ the bootstrap commands. It never skips. `swarm-bao-granter-role` skips while
 the token file is absent, which is the steady state afterwards; the token's
 TTL means a forgotten file expires rather than lingering.
 
-A sealed or unreachable store makes the unit retry every 30 s. A store that
+A sealed or unreachable store, or one whose listing fails with something other
-refuses the token (expired, revoked, or missing the `bao-bootstrap` policy)
+than a refusal (a node still becoming active), makes the unit retry every
+30 s with the store's error in the log. A store that refuses the token (expired,
+revoked, or missing the `bao-bootstrap` policy)
 makes it exit 4 and stay `failed` with the one-time step in its log; mint a
 fresh token or remove the file, then restart it. If the granter's own
 certificate login already works, the unit logs that the bootstrap token isn't

nix/host-modules/lib/bootstrap-auth-list.nix

@@ -0,0 +1,57 @@
+# The bootstrap token's first use in `swarm-bao-granter-role`, as shell text
+# that leaves `mounted` set to the store's auth listing, or ends the script.
+# Same split as ./granter-login.nix, applied to `bao auth list`:
+#
+# - A refusal ("permission denied", HTTP 403: openbao answers an expired,
+#   revoked or underprivileged token alike) is the token's. If the granter's
+#   own login works the role is already in place and the dead token is
+#   deleted; otherwise the unit prints the one-time steps and exits 4, which
+#   `RestartPreventExitStatus` keeps from retrying.
+# - Any other failure from a store that answers `status` is not the token's:
+#   a node that is not yet active answers with a 500 for the first moments
+#   after an unseal. The message says so and exits 1, which `Restart=` retries.
+# - A store that does not answer `status` is sealed or unreachable; exit 1.
+#
+# In every case bao's own error follows.
+#
+# Pure function — NOT a NixOS module. `baoStoreUp` is as for
+# ./granter-login.nix. Needs `bao`, `coreutils` and `gnugrep` on the caller's
+# `path`, and `$err` to be a scratch file.
+{
+  lib,
+  bootstrapTokenFile,
+  granterClientCertFile,
+  granterClientKeyFile,
+  baoStoreUp,
+  setupSteps,
+}:
+''
+  if ! mounted="$(bao auth list -format=json 2>"$err")"; then
+    if grep -qF 'permission denied' "$err"; then
+      # A dead token beside a working granter is the steady state of
+      # a host that finished bootstrap and kept the file. The
+      # granter's own login is the proof. A fresh token still takes
+      # the full path below, since the granter cannot read back its
+      # own policy to compare it. The refused token is dead,
+      # so it is deleted here and the unit does not start again.
+      if BAO_CLIENT_CERT=${lib.escapeShellArg granterClientCertFile} \
+        BAO_CLIENT_KEY=${lib.escapeShellArg granterClientKeyFile} \
+        bao login -method=cert -token-only >/dev/null 2>&1; then
+        echo "granter role already in place; bootstrap token not needed"
+        rm -f ${lib.escapeShellArg bootstrapTokenFile}
+        exit 0
+      fi
+      echo ${lib.escapeShellArg "the store refused the bootstrap token in ${bootstrapTokenFile}: it has expired, been revoked, or does not carry the bao-bootstrap policy."} >&2
+      echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2
+      ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg "  ${l}"} >&2") setupSteps}
+      cat "$err" >&2
+      exit 4
+    elif ${baoStoreUp}; then
+      echo "the store is unsealed but listing its auth mounts failed without a refused token, as a node that is not active yet answers; retrying." >&2
+    else
+      echo "the store is sealed or unreachable; retrying." >&2
+    fi
+    cat "$err" >&2
+    exit 1
+  fi
+''

nix/host-modules/swarm-bao.nix

@@ -324,6 +324,17 @@ let
     setupSteps = granterSetupSteps;
   };
 
+  # The bootstrap token's first use, in `swarm-bao-granter-role`.
+  bootstrapAuthList = import ./lib/bootstrap-auth-list.nix {
+    inherit lib baoStoreUp;
+    inherit (baoDeploy)
+      bootstrapTokenFile
+      granterClientCertFile
+      granterClientKeyFile
+      ;
+    setupSteps = granterSetupSteps;
+  };
+
   # The granter's certificate for the granting units. The `baoCli` wrapper
   # only defaults these, so the unit's environment wins.
   granterEnv = {
@@ -3374,6 +3385,7 @@ in
         path = [
           baoCli
           pkgs.coreutils
+          pkgs.gnugrep
         ];
         # Named but not placed is a legitimate state: all-local supplies the
         # path as a default and the operator drops the file there after
@@ -3426,34 +3438,9 @@ in
           # that already exists, and recognising that would tie a rebuild
           # to an error string we have never seen this store emit.
           #
-          # The token's first use, and the bootstrap policy grants this read,
+          # The token's first use; ./lib/bootstrap-auth-list.nix says which
-          # so a refusal from a store that is up is the token's: expired,
+          # failures are the token's.
-          # revoked, or minted without that policy.
+          ${bootstrapAuthList}
-          if ! mounted="$(bao auth list -format=json 2>"$err")"; then
-            if ${baoStoreUp}; then
-              # A dead token beside a working granter is the steady state of
-              # a host that finished bootstrap and kept the file. The
-              # granter's own login is the proof. A fresh token still takes
-              # the full path below, since the granter cannot read back its
-              # own policy to compare it. The refused token is dead,
-              # so it is deleted here and the unit does not start again.
-              if BAO_CLIENT_CERT=${lib.escapeShellArg baoDeploy.granterClientCertFile} \
-                BAO_CLIENT_KEY=${lib.escapeShellArg baoDeploy.granterClientKeyFile} \
-                bao login -method=cert -token-only >/dev/null 2>&1; then
-                echo "granter role already in place; bootstrap token not needed"
-                rm -f ${lib.escapeShellArg baoDeploy.bootstrapTokenFile}
-                exit 0
-              fi
-              echo ${lib.escapeShellArg "the store is unsealed but refused the bootstrap token in ${baoDeploy.bootstrapTokenFile}: it has expired, been revoked, or does not carry the bao-bootstrap policy."} >&2
-              echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2
-              ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg "  ${l}"} >&2") granterSetupSteps}
-              cat "$err" >&2
-              exit 4
-            fi
-            echo "the store is sealed or unreachable; retrying." >&2
-            cat "$err" >&2
-            exit 1
-          fi
           case "$mounted" in
             *'"cert/"'*) ;;
             *) bao auth enable cert ;;

nix/script-tests/granter-login-fake-bao.sh

@@ -1,10 +1,11 @@
 #!/usr/bin/env bash
-# The stub `bao` for ./granter-login.nix. `FAKE_BAO_LOGIN` picks the login's
+# The stub `bao` for ./granter-login.nix and ./lib/bootstrap-auth-list.nix. `FAKE_BAO_LOGIN` picks the login's
 # answer, in openbao's wording: `ok`, `standby` (500, a node that is not active
 # yet), `no-role` (400, no trusted chain), `no-chain-match` (400, a chain that
 # no role accepts), `sealed` (503). `FAKE_BAO_STATUS_RC` is `status`'s exit
 # status. `FAKE_BAO_LOGIN_FIRST`, when set, answers the first login with it and
 # later ones with `FAKE_BAO_LOGIN`; `FAKE_BAO_COUNT` is the file counting them.
+# `auth list` is answered per the `auth)` arm below.
 echo "$*" >>"$FAKE_BAO_LOG"
 case "$1" in
   login)
@@ -33,6 +34,31 @@ case "$1" in
         ;;
     esac
     ;;
+  auth)
+    # `FAKE_BAO_LIST` picks `auth list`'s answer: `ok`, `standby` (500),
+    # `denied` (403, a token the store refuses), `sealed` (503).
+    # `FAKE_BAO_LIST_FIRST` answers only the first one with it.
+    n=$(($(cat "$FAKE_BAO_LIST_COUNT" 2>/dev/null || echo 0) + 1))
+    echo "$n" >"$FAKE_BAO_LIST_COUNT"
+    answer=${FAKE_BAO_LIST-ok}
+    if [ "$n" = 1 ] && [ -n "${FAKE_BAO_LIST_FIRST-}" ]; then answer=$FAKE_BAO_LIST_FIRST; fi
+    url='URL: GET https://bao.t.local:8200/v1/sys/auth'
+    case "$answer" in
+      ok) printf '{"cert/":{},"oidc/":{}}' ;;
+      standby)
+        printf 'Error listing enabled authentications: Error making API request.\n\n%s\nCode: 500. Errors:\n\n* local node not active but active cluster node not found\n' "$url" >&2
+        exit 2
+        ;;
+      denied)
+        printf 'Error listing enabled authentications: Error making API request.\n\n%s\nCode: 403. Errors:\n\n* permission denied\n' "$url" >&2
+        exit 2
+        ;;
+      sealed)
+        printf 'Error listing enabled authentications: Error making API request.\n\n%s\nCode: 503. Errors:\n\n* Vault is sealed\n' "$url" >&2
+        exit 2
+        ;;
+    esac
+    ;;
   status)
     [ "${FAKE_BAO_STATUS_RC-0}" = 0 ] || {
       echo "fake bao: status: Vault is sealed" >&2

nix/script-tests/granter-login.nix

@@ -3,7 +3,9 @@
 # cases are in ./granter-login.sh and the stub is ./granter-login-fake-bao.sh.
 #
 # What runs is the rendered login, with the same `baoStoreUp` text the unit
-# passes, rendered once with the bootstrap-token hint and once without.
+# passes, rendered once with the bootstrap-token hint and once without. The
+# bootstrap token's `auth list` step (../host-modules/lib/bootstrap-auth-list.nix)
+# is rendered the same way and run against the same stub.
 {
   pkgs,
   ...
@@ -31,6 +33,16 @@ pkgs.runCommand "hyperhive-script-test-granter-login"
   {
     LOGIN = render false;
     LOGIN_WITH_TOKEN = render true;
+    BOOTSTRAP_LIST = pkgs.writeText "bootstrap-auth-list.sh" (
+      import ../host-modules/lib/bootstrap-auth-list.nix {
+        inherit (pkgs) lib;
+        bootstrapTokenFile = "/tmp/granter-bootstrap-test.token";
+        granterClientCertFile = "/nonexistent/granter.crt";
+        granterClientKeyFile = "/nonexistent/granter.key";
+        baoStoreUp = ''HOME=/var/empty bao status >/dev/null 2>>"$err"'';
+        setupSteps = [ "bao policy write bao-bootstrap /etc/bootstrap.hcl" ];
+      }
+    );
     FAKE_BAO_BIN = "${fakeBao}/bin";
     nativeBuildInputs = [
       pkgs.coreutils

nix/script-tests/granter-login.sh

@@ -99,5 +99,75 @@ err_has "the store is sealed or unreachable; retrying."
 err_has "Vault is sealed"
 err_lacks "role is not set up"
 
+# The bootstrap token's `auth list` step: the rendered snippet runs under
+# `set -e` with the scratch file the unit makes. Sets $rc, $got (the listing it
+# left in `mounted`) and $dir/err.
+token=/tmp/granter-bootstrap-test.token
+
+setup_list() {
+  setup "$1"
+  export FAKE_BAO_LIST_COUNT=$dir/list-count FAKE_BAO_LIST=ok
+  unset FAKE_BAO_LIST_FIRST
+  echo dead >"$token"
+}
+
+run_list() {
+  got=$(
+    (
+      set -e
+      PATH="$FAKE_BAO_BIN:$PATH"
+      err=$(mktemp)
+      # shellcheck source=/dev/null
+      . "$BOOTSTRAP_LIST"
+      echo "$mounted"
+    ) 2>"$dir/err"
+  )
+  rc=$?
+}
+
+setup_list "a listing that succeeds leaves it in mounted and prints nothing"
+run_list
+expect 0 '{"cert/":{},"oidc/":{}}'
+if [ -s "$dir/err" ]; then fail "printed: $(cat "$dir/err")"; fi
+
+setup_list "a standby store fails the listing with its own error, does not blame the token, keeps the token file, and the next attempt succeeds"
+export FAKE_BAO_LIST_FIRST=standby
+run_list
+expect 1 ""
+err_has "local node not active but active cluster node not found"
+err_has "listing its auth mounts failed without a refused token"
+err_lacks "refused the bootstrap token"
+err_lacks "one-time step"
+if [ ! -e "$token" ]; then fail "token file deleted"; fi
+run_list
+expect 0 '{"cert/":{},"oidc/":{}}'
+
+setup_list "a refused token with no working granter names the token and the setup steps, then bao's error, and exits 4"
+export FAKE_BAO_LIST=denied FAKE_BAO_LOGIN=no-role
+run_list
+expect 4 ""
+err_has "the store refused the bootstrap token in /tmp/granter-bootstrap-test.token"
+err_has "one-time step, as root on this host"
+err_has "  bao policy write bao-bootstrap /etc/bootstrap.hcl"
+err_has "permission denied"
+if [ ! -e "$token" ]; then fail "token file deleted"; fi
+
+setup_list "a refused token beside a working granter exits 0 and deletes the token"
+export FAKE_BAO_LIST=denied FAKE_BAO_LOGIN=ok
+run_list
+expect 0 "granter role already in place; bootstrap token not needed"
+err_lacks "refused the bootstrap token"
+if [ -e "$token" ]; then fail "token file kept"; fi
+
+setup_list "a sealed store is reported as sealed or unreachable, with bao's error"
+export FAKE_BAO_LIST=sealed FAKE_BAO_STATUS_RC=2
+run_list
+expect 1 ""
+err_has "the store is sealed or unreachable; retrying."
+err_has "Vault is sealed"
+err_lacks "refused the bootstrap token"
+
+rm -f "$token"
+
 echo "granter-login: $count cases, $failures failures"
 [ "$failures" = 0 ]