report the granter login's real error and retry on a standby store

nix/checks.nix

@@ -101,6 +101,10 @@ in
   # `bao`: a store that holds a key wins, an empty one takes the seed, and an
   # unreachable one mints nothing.
   script-test-grafana-secret-key = import ./script-tests/grafana-secret-key.nix { inherit pkgs; };
+  # Runs ./host-modules/lib/granter-login.nix's login against a stub `bao`: a
+  # refused certificate names the missing role, any other failure (a standby
+  # node, a sealed store) does not.
+  script-test-granter-login = import ./script-tests/granter-login.nix { inherit pkgs; };
   module-eval-nats-tls = import ./module-eval/nats-tls.nix {
     inherit pkgs self nixosSystem;
     inherit (pkgs) lib;

nix/host-modules/lib/granter-login.nix

@@ -0,0 +1,57 @@
+# The cert login every unit that writes bao policies starts with, as shell text
+# that leaves `BAO_TOKEN` exported. It FAILS rather than skips: a grant that
+# was not written is otherwise invisible until whatever needs it fails
+# somewhere else.
+#
+# What the failure says follows from what the store answered:
+#
+# - A refused certificate ("invalid certificate or no client certificate
+#   supplied", or "no chain matching all constraints"; openbao's cert-auth
+#   login in `builtin/credential/cert/path_login.go`, HTTP 400) is the granter
+#   role missing or not matching. Only that answer prints the one-time setup
+#   steps.
+# - Any other failure from a store that answers `status` is not the role's:
+#   a node that is not yet active answers the login with a 500 ("local node
+#   not active but active cluster node not found") for the first moments after
+#   an unseal. The message says so and the unit's `Restart=` retries.
+# - A store that does not answer `status` is sealed or unreachable.
+#
+# In every case bao's own error follows.
+#
+# Pure function — NOT a NixOS module. Call it from a module's `let`:
+#
+#   granterLogin = import ./lib/granter-login.nix {
+#     inherit lib granterCn granterPolicyName haveBootstrapToken baoStoreUp;
+#     setupSteps = granterSetupSteps;
+#   };
+#
+# `baoStoreUp` is shell text that exits 0 on a reachable, unsealed store and
+# appends its error to `$err`. Needs `bao`, `coreutils` and `gnugrep` on the
+# caller's `path`. Takes the EXIT trap, to remove its scratch file.
+{
+  lib,
+  granterCn,
+  granterPolicyName,
+  haveBootstrapToken,
+  baoStoreUp,
+  setupSteps,
+}:
+''
+  err="$(mktemp)"
+  trap 'rm -f "$err"' EXIT
+  if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
+    if grep -qE 'invalid certificate or no client certificate supplied|no chain matching all constraints' "$err"; then
+      echo ${lib.escapeShellArg "the store refused the granter's certificate (CN ${granterCn}): the ${granterPolicyName} role is not set up."} >&2
+      ${lib.optionalString haveBootstrapToken ''echo "swarm-bao-granter-role writes that role; journalctl -u swarm-bao-granter-role says why it has not." >&2''}
+      echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2
+      ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg "  ${l}"} >&2") setupSteps}
+    elif ${baoStoreUp}; then
+      echo "the store is unsealed but the granter's login failed without a refused certificate, as a node that is not active yet answers; retrying." >&2
+    else
+      echo "the store is sealed or unreachable; retrying." >&2
+    fi
+    cat "$err" >&2
+    exit 1
+  fi
+  export BAO_TOKEN
+''

nix/host-modules/swarm-bao.nix

@@ -311,27 +311,18 @@ let
   # config and no token.
   baoStoreUp = ''HOME=/var/empty bao status >/dev/null 2>>"$err"'';
 
-  # The login every granting unit starts with. It FAILS rather than skips: a
+  # The login every granting unit starts with; ./lib/granter-login.nix says
-  # grant that was not written is otherwise invisible until whatever needs it
+  # which failures it reports as a missing role.
-  # fails somewhere else. `baoStoreUp` separates "the granter is not set up"
+  granterLogin = import ./lib/granter-login.nix {
-  # from "retry later"; either way bao's own message follows.
+    inherit
-  granterLogin = ''
+      lib
-    err="$(mktemp)"
+      granterCn
-    trap 'rm -f "$err"' EXIT
+      granterPolicyName
-    if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
+      haveBootstrapToken
-      if ${baoStoreUp}; then
+      baoStoreUp
-        echo ${lib.escapeShellArg "the store is unsealed but refused the granter's certificate (CN ${granterCn}): the ${granterPolicyName} role is not set up."} >&2
+      ;
-        ${lib.optionalString haveBootstrapToken ''echo "swarm-bao-granter-role writes that role; journalctl -u swarm-bao-granter-role says why it has not." >&2''}
+    setupSteps = granterSetupSteps;
-        echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2
+  };
-        ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg "  ${l}"} >&2") granterSetupSteps}
-      else
-        echo "the store is sealed or unreachable; retrying." >&2
-      fi
-      cat "$err" >&2
-      exit 1
-    fi
-    export BAO_TOKEN
-  '';
 
   # The granter's certificate for the granting units. The `baoCli` wrapper
   # only defaults these, so the unit's environment wins.

nix/script-tests/granter-login-fake-bao.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# The stub `bao` for ./granter-login.nix. `FAKE_BAO_LOGIN` picks the login's
+# answer, in openbao's wording: `ok`, `standby` (500, a node that is not active
+# yet), `no-role` (400, no trusted chain), `no-chain-match` (400, a chain that
+# no role accepts), `sealed` (503). `FAKE_BAO_STATUS_RC` is `status`'s exit
+# status. `FAKE_BAO_LOGIN_FIRST`, when set, answers the first login with it and
+# later ones with `FAKE_BAO_LOGIN`; `FAKE_BAO_COUNT` is the file counting them.
+echo "$*" >>"$FAKE_BAO_LOG"
+case "$1" in
+  login)
+    n=$(($(cat "$FAKE_BAO_COUNT" 2>/dev/null || echo 0) + 1))
+    echo "$n" >"$FAKE_BAO_COUNT"
+    answer=$FAKE_BAO_LOGIN
+    if [ "$n" = 1 ] && [ -n "${FAKE_BAO_LOGIN_FIRST-}" ]; then answer=$FAKE_BAO_LOGIN_FIRST; fi
+    url='URL: PUT https://bao.t.local:8200/v1/auth/cert/login'
+    case "$answer" in
+      ok) printf 's.fake' ;;
+      standby)
+        printf 'Error authenticating: Error making API request.\n\n%s\nCode: 500. Errors:\n\n* local node not active but active cluster node not found\n' "$url" >&2
+        exit 2
+        ;;
+      no-role)
+        printf 'Error authenticating: Error making API request.\n\n%s\nCode: 400. Errors:\n\n* invalid certificate or no client certificate supplied\n' "$url" >&2
+        exit 2
+        ;;
+      no-chain-match)
+        printf 'Error authenticating: Error making API request.\n\n%s\nCode: 400. Errors:\n\n* no chain matching all constraints could be found for this login certificate\n' "$url" >&2
+        exit 2
+        ;;
+      sealed)
+        printf 'Error authenticating: Error making API request.\n\n%s\nCode: 503. Errors:\n\n* Vault is sealed\n' "$url" >&2
+        exit 2
+        ;;
+    esac
+    ;;
+  status)
+    [ "${FAKE_BAO_STATUS_RC-0}" = 0 ] || {
+      echo "fake bao: status: Vault is sealed" >&2
+      exit 2
+    }
+    ;;
+  *)
+    echo "fake bao: unexpected call: $*" >&2
+    exit 98
+    ;;
+esac

nix/script-tests/granter-login.nix

@@ -0,0 +1,43 @@
+# `checks.script-test-granter-login` — runs ../host-modules/lib/granter-login.nix
+# against a stub `bao` whose login and status answers are set per case. The
+# cases are in ./granter-login.sh and the stub is ./granter-login-fake-bao.sh.
+#
+# What runs is the rendered login, with the same `baoStoreUp` text the unit
+# passes, rendered once with the bootstrap-token hint and once without.
+{
+  pkgs,
+  ...
+}:
+let
+  render =
+    haveBootstrapToken:
+    pkgs.writeText "granter-login.sh" (
+      import ../host-modules/lib/granter-login.nix {
+        inherit (pkgs) lib;
+        granterCn = "bao-granter";
+        granterPolicyName = "bao-granter";
+        inherit haveBootstrapToken;
+        baoStoreUp = ''HOME=/var/empty bao status >/dev/null 2>>"$err"'';
+        setupSteps = [ "bao policy write bao-bootstrap /etc/bootstrap.hcl" ];
+      }
+    );
+
+  fakeBao = pkgs.runCommand "fake-bao" { } ''
+    install -Dm755 ${./granter-login-fake-bao.sh} $out/bin/bao
+    patchShebangs $out/bin
+  '';
+in
+pkgs.runCommand "hyperhive-script-test-granter-login"
+  {
+    LOGIN = render false;
+    LOGIN_WITH_TOKEN = render true;
+    FAKE_BAO_BIN = "${fakeBao}/bin";
+    nativeBuildInputs = [
+      pkgs.coreutils
+      pkgs.gnugrep
+    ];
+  }
+  ''
+    ${pkgs.bash}/bin/bash ${./granter-login.sh}
+    touch "$out"
+  ''

nix/script-tests/granter-login.sh

@@ -0,0 +1,103 @@
+# Cases for ./granter-login.nix. Each case sets the stub's answers, runs the
+# rendered login under `set -e` as a unit's script does, then compares its exit
+# status, what it printed to stderr and the token it left exported to children.
+set -uo pipefail
+
+failures=0
+count=0
+
+fail() {
+  echo "FAILED: $case: $*" >&2
+  failures=$((failures + 1))
+}
+
+# setup <description>
+setup() {
+  case=$1
+  count=$((count + 1))
+  dir=$(mktemp -d)
+  export FAKE_BAO_LOG=$dir/bao.log FAKE_BAO_COUNT=$dir/count FAKE_BAO_LOGIN=ok FAKE_BAO_STATUS_RC=0
+  unset FAKE_BAO_LOGIN_FIRST
+  : >"$FAKE_BAO_LOG"
+  login=$LOGIN
+}
+
+# run — the login, then the token it exported. Sets $rc, $got, and $dir/err.
+run() {
+  got=$(
+    (
+      set -e
+      PATH="$FAKE_BAO_BIN:$PATH"
+      unset BAO_TOKEN
+      # shellcheck source=/dev/null
+      . "$login"
+      env | sed -n "s/^BAO_TOKEN=//p"
+    ) 2>"$dir/err"
+  )
+  rc=$?
+}
+
+# expect <status> <token>
+expect() {
+  if [ "$rc" != "$1" ]; then fail "exited $rc, wanted $1: $(cat "$dir/err")"; fi
+  if [ "$got" != "$2" ]; then fail "token '$got', wanted '$2'"; fi
+}
+
+err_has() {
+  if ! grep -qF -- "$1" "$dir/err"; then fail "stderr lacks '$1': $(cat "$dir/err")"; fi
+}
+
+err_lacks() {
+  if grep -qF -- "$1" "$dir/err"; then fail "stderr has '$1': $(cat "$dir/err")"; fi
+}
+
+setup "a successful login exports the token and prints nothing"
+run
+expect 0 s.fake
+if [ -s "$dir/err" ]; then fail "printed: $(cat "$dir/err")"; fi
+
+setup "a standby store fails the login with its own error, does not blame the role, and the next attempt succeeds"
+export FAKE_BAO_LOGIN_FIRST=standby
+run
+expect 1 ""
+err_has "local node not active but active cluster node not found"
+err_has "login failed without a refused certificate"
+err_lacks "role is not set up"
+err_lacks "one-time step"
+run
+expect 0 s.fake
+
+setup "an unknown role names the role and the setup steps, then bao's error"
+export FAKE_BAO_LOGIN=no-role
+run
+expect 1 ""
+err_has "refused the granter's certificate (CN bao-granter): the bao-granter role is not set up."
+err_has "one-time step, as root on this host"
+err_has "  bao policy write bao-bootstrap /etc/bootstrap.hcl"
+err_has "invalid certificate or no client certificate supplied"
+err_lacks "swarm-bao-granter-role writes that role"
+
+setup "a certificate no role accepts is a missing role too"
+export FAKE_BAO_LOGIN=no-chain-match
+run
+expect 1 ""
+err_has "the bao-granter role is not set up."
+err_has "no chain matching all constraints"
+
+setup "a unit with a bootstrap token also points at the unit that writes the role"
+export FAKE_BAO_LOGIN=no-role
+login=$LOGIN_WITH_TOKEN
+run
+expect 1 ""
+err_has "swarm-bao-granter-role writes that role"
+
+setup "a sealed store is reported as sealed or unreachable, with bao's error"
+export FAKE_BAO_LOGIN=sealed FAKE_BAO_STATUS_RC=2
+run
+expect 1 ""
+err_has "the store is sealed or unreachable; retrying."
+err_has "Vault is sealed"
+err_lacks "role is not set up"
+
+echo "granter-login: $count cases, $failures failures"
+[ "$failures" = 0 ]