From cb5a9b2bd7f30d7a7b0ba89902e522f4ce3c843e Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 5 Nov 2026 08:57:10 +0100 Subject: [PATCH] report the real error when the granter role's bootstrap step fails, and retry on a standby store --- docs/swarm/bao.md | 6 +- nix/host-modules/lib/bootstrap-auth-list.nix | 57 ++++++++++++++++ nix/host-modules/swarm-bao.nix | 43 +++++------- nix/script-tests/granter-login-fake-bao.sh | 28 +++++++- nix/script-tests/granter-login.nix | 14 +++- nix/script-tests/granter-login.sh | 70 ++++++++++++++++++++ 6 files changed, 186 insertions(+), 32 deletions(-) create mode 100644 nix/host-modules/lib/bootstrap-auth-list.nix diff --git a/docs/swarm/bao.md b/docs/swarm/bao.md index e5b41f13..707c377b 100644 --- a/docs/swarm/bao.md +++ b/docs/swarm/bao.md @@ -40,8 +40,10 @@ the bootstrap commands. It never skips. `swarm-bao-granter-role` skips while the token file is absent, which is the steady state afterwards; the token's TTL means a forgotten file expires rather than lingering. -A sealed or unreachable store makes the unit retry every 30 s. A store that -refuses the token (expired, revoked, or missing the `bao-bootstrap` policy) +A sealed or unreachable store, or one whose listing fails with something other +than a refusal (a node still becoming active), makes the unit retry every +30 s with the store's error in the log. A store that refuses the token (expired, +revoked, or missing the `bao-bootstrap` policy) makes it exit 4 and stay `failed` with the one-time step in its log; mint a fresh token or remove the file, then restart it. If the granter's own certificate login already works, the unit logs that the bootstrap token isn't diff --git a/nix/host-modules/lib/bootstrap-auth-list.nix b/nix/host-modules/lib/bootstrap-auth-list.nix new file mode 100644 index 00000000..c80a0ef9 --- /dev/null +++ b/nix/host-modules/lib/bootstrap-auth-list.nix @@ -0,0 +1,57 @@ +# The bootstrap token's first use in `swarm-bao-granter-role`, as shell text +# that leaves `mounted` set to the store's auth listing, or ends the script. +# Same split as ./granter-login.nix, applied to `bao auth list`: +# +# - A refusal ("permission denied", HTTP 403: openbao answers an expired, +# revoked or underprivileged token alike) is the token's. If the granter's +# own login works the role is already in place and the dead token is +# deleted; otherwise the unit prints the one-time steps and exits 4, which +# `RestartPreventExitStatus` keeps from retrying. +# - Any other failure from a store that answers `status` is not the token's: +# a node that is not yet active answers with a 500 for the first moments +# after an unseal. The message says so and exits 1, which `Restart=` retries. +# - A store that does not answer `status` is sealed or unreachable; exit 1. +# +# In every case bao's own error follows. +# +# Pure function — NOT a NixOS module. `baoStoreUp` is as for +# ./granter-login.nix. Needs `bao`, `coreutils` and `gnugrep` on the caller's +# `path`, and `$err` to be a scratch file. +{ + lib, + bootstrapTokenFile, + granterClientCertFile, + granterClientKeyFile, + baoStoreUp, + setupSteps, +}: +'' + if ! mounted="$(bao auth list -format=json 2>"$err")"; then + if grep -qF 'permission denied' "$err"; then + # A dead token beside a working granter is the steady state of + # a host that finished bootstrap and kept the file. The + # granter's own login is the proof. A fresh token still takes + # the full path below, since the granter cannot read back its + # own policy to compare it. The refused token is dead, + # so it is deleted here and the unit does not start again. + if BAO_CLIENT_CERT=${lib.escapeShellArg granterClientCertFile} \ + BAO_CLIENT_KEY=${lib.escapeShellArg granterClientKeyFile} \ + bao login -method=cert -token-only >/dev/null 2>&1; then + echo "granter role already in place; bootstrap token not needed" + rm -f ${lib.escapeShellArg bootstrapTokenFile} + exit 0 + fi + echo ${lib.escapeShellArg "the store refused the bootstrap token in ${bootstrapTokenFile}: it has expired, been revoked, or does not carry the bao-bootstrap policy."} >&2 + echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2 + ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg " ${l}"} >&2") setupSteps} + cat "$err" >&2 + exit 4 + elif ${baoStoreUp}; then + echo "the store is unsealed but listing its auth mounts failed without a refused token, as a node that is not active yet answers; retrying." >&2 + else + echo "the store is sealed or unreachable; retrying." >&2 + fi + cat "$err" >&2 + exit 1 + fi +'' diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 2c93a69a..aa4c63c5 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -324,6 +324,17 @@ let setupSteps = granterSetupSteps; }; + # The bootstrap token's first use, in `swarm-bao-granter-role`. + bootstrapAuthList = import ./lib/bootstrap-auth-list.nix { + inherit lib baoStoreUp; + inherit (baoDeploy) + bootstrapTokenFile + granterClientCertFile + granterClientKeyFile + ; + setupSteps = granterSetupSteps; + }; + # The granter's certificate for the granting units. The `baoCli` wrapper # only defaults these, so the unit's environment wins. granterEnv = { @@ -3374,6 +3385,7 @@ in path = [ baoCli pkgs.coreutils + pkgs.gnugrep ]; # Named but not placed is a legitimate state: all-local supplies the # path as a default and the operator drops the file there after @@ -3426,34 +3438,9 @@ in # that already exists, and recognising that would tie a rebuild # to an error string we have never seen this store emit. # - # The token's first use, and the bootstrap policy grants this read, - # so a refusal from a store that is up is the token's: expired, - # revoked, or minted without that policy. - if ! mounted="$(bao auth list -format=json 2>"$err")"; then - if ${baoStoreUp}; then - # A dead token beside a working granter is the steady state of - # a host that finished bootstrap and kept the file. The - # granter's own login is the proof. A fresh token still takes - # the full path below, since the granter cannot read back its - # own policy to compare it. The refused token is dead, - # so it is deleted here and the unit does not start again. - if BAO_CLIENT_CERT=${lib.escapeShellArg baoDeploy.granterClientCertFile} \ - BAO_CLIENT_KEY=${lib.escapeShellArg baoDeploy.granterClientKeyFile} \ - bao login -method=cert -token-only >/dev/null 2>&1; then - echo "granter role already in place; bootstrap token not needed" - rm -f ${lib.escapeShellArg baoDeploy.bootstrapTokenFile} - exit 0 - fi - echo ${lib.escapeShellArg "the store is unsealed but refused the bootstrap token in ${baoDeploy.bootstrapTokenFile}: it has expired, been revoked, or does not carry the bao-bootstrap policy."} >&2 - echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2 - ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg " ${l}"} >&2") granterSetupSteps} - cat "$err" >&2 - exit 4 - fi - echo "the store is sealed or unreachable; retrying." >&2 - cat "$err" >&2 - exit 1 - fi + # The token's first use; ./lib/bootstrap-auth-list.nix says which + # failures are the token's. + ${bootstrapAuthList} case "$mounted" in *'"cert/"'*) ;; *) bao auth enable cert ;; diff --git a/nix/script-tests/granter-login-fake-bao.sh b/nix/script-tests/granter-login-fake-bao.sh index 482d5db6..82cfd9e7 100755 --- a/nix/script-tests/granter-login-fake-bao.sh +++ b/nix/script-tests/granter-login-fake-bao.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash -# The stub `bao` for ./granter-login.nix. `FAKE_BAO_LOGIN` picks the login's +# The stub `bao` for ./granter-login.nix and ./lib/bootstrap-auth-list.nix. `FAKE_BAO_LOGIN` picks the login's # answer, in openbao's wording: `ok`, `standby` (500, a node that is not active # yet), `no-role` (400, no trusted chain), `no-chain-match` (400, a chain that # no role accepts), `sealed` (503). `FAKE_BAO_STATUS_RC` is `status`'s exit # status. `FAKE_BAO_LOGIN_FIRST`, when set, answers the first login with it and # later ones with `FAKE_BAO_LOGIN`; `FAKE_BAO_COUNT` is the file counting them. +# `auth list` is answered per the `auth)` arm below. echo "$*" >>"$FAKE_BAO_LOG" case "$1" in login) @@ -33,6 +34,31 @@ case "$1" in ;; esac ;; + auth) + # `FAKE_BAO_LIST` picks `auth list`'s answer: `ok`, `standby` (500), + # `denied` (403, a token the store refuses), `sealed` (503). + # `FAKE_BAO_LIST_FIRST` answers only the first one with it. + n=$(($(cat "$FAKE_BAO_LIST_COUNT" 2>/dev/null || echo 0) + 1)) + echo "$n" >"$FAKE_BAO_LIST_COUNT" + answer=${FAKE_BAO_LIST-ok} + if [ "$n" = 1 ] && [ -n "${FAKE_BAO_LIST_FIRST-}" ]; then answer=$FAKE_BAO_LIST_FIRST; fi + url='URL: GET https://bao.t.local:8200/v1/sys/auth' + case "$answer" in + ok) printf '{"cert/":{},"oidc/":{}}' ;; + standby) + printf 'Error listing enabled authentications: Error making API request.\n\n%s\nCode: 500. Errors:\n\n* local node not active but active cluster node not found\n' "$url" >&2 + exit 2 + ;; + denied) + printf 'Error listing enabled authentications: Error making API request.\n\n%s\nCode: 403. Errors:\n\n* permission denied\n' "$url" >&2 + exit 2 + ;; + sealed) + printf 'Error listing enabled authentications: Error making API request.\n\n%s\nCode: 503. Errors:\n\n* Vault is sealed\n' "$url" >&2 + exit 2 + ;; + esac + ;; status) [ "${FAKE_BAO_STATUS_RC-0}" = 0 ] || { echo "fake bao: status: Vault is sealed" >&2 diff --git a/nix/script-tests/granter-login.nix b/nix/script-tests/granter-login.nix index d196717b..553f90be 100644 --- a/nix/script-tests/granter-login.nix +++ b/nix/script-tests/granter-login.nix @@ -3,7 +3,9 @@ # cases are in ./granter-login.sh and the stub is ./granter-login-fake-bao.sh. # # What runs is the rendered login, with the same `baoStoreUp` text the unit -# passes, rendered once with the bootstrap-token hint and once without. +# passes, rendered once with the bootstrap-token hint and once without. The +# bootstrap token's `auth list` step (../host-modules/lib/bootstrap-auth-list.nix) +# is rendered the same way and run against the same stub. { pkgs, ... @@ -31,6 +33,16 @@ pkgs.runCommand "hyperhive-script-test-granter-login" { LOGIN = render false; LOGIN_WITH_TOKEN = render true; + BOOTSTRAP_LIST = pkgs.writeText "bootstrap-auth-list.sh" ( + import ../host-modules/lib/bootstrap-auth-list.nix { + inherit (pkgs) lib; + bootstrapTokenFile = "/tmp/granter-bootstrap-test.token"; + granterClientCertFile = "/nonexistent/granter.crt"; + granterClientKeyFile = "/nonexistent/granter.key"; + baoStoreUp = ''HOME=/var/empty bao status >/dev/null 2>>"$err"''; + setupSteps = [ "bao policy write bao-bootstrap /etc/bootstrap.hcl" ]; + } + ); FAKE_BAO_BIN = "${fakeBao}/bin"; nativeBuildInputs = [ pkgs.coreutils diff --git a/nix/script-tests/granter-login.sh b/nix/script-tests/granter-login.sh index 20cd17a0..ee80a56f 100644 --- a/nix/script-tests/granter-login.sh +++ b/nix/script-tests/granter-login.sh @@ -99,5 +99,75 @@ err_has "the store is sealed or unreachable; retrying." err_has "Vault is sealed" err_lacks "role is not set up" +# The bootstrap token's `auth list` step: the rendered snippet runs under +# `set -e` with the scratch file the unit makes. Sets $rc, $got (the listing it +# left in `mounted`) and $dir/err. +token=/tmp/granter-bootstrap-test.token + +setup_list() { + setup "$1" + export FAKE_BAO_LIST_COUNT=$dir/list-count FAKE_BAO_LIST=ok + unset FAKE_BAO_LIST_FIRST + echo dead >"$token" +} + +run_list() { + got=$( + ( + set -e + PATH="$FAKE_BAO_BIN:$PATH" + err=$(mktemp) + # shellcheck source=/dev/null + . "$BOOTSTRAP_LIST" + echo "$mounted" + ) 2>"$dir/err" + ) + rc=$? +} + +setup_list "a listing that succeeds leaves it in mounted and prints nothing" +run_list +expect 0 '{"cert/":{},"oidc/":{}}' +if [ -s "$dir/err" ]; then fail "printed: $(cat "$dir/err")"; fi + +setup_list "a standby store fails the listing with its own error, does not blame the token, keeps the token file, and the next attempt succeeds" +export FAKE_BAO_LIST_FIRST=standby +run_list +expect 1 "" +err_has "local node not active but active cluster node not found" +err_has "listing its auth mounts failed without a refused token" +err_lacks "refused the bootstrap token" +err_lacks "one-time step" +if [ ! -e "$token" ]; then fail "token file deleted"; fi +run_list +expect 0 '{"cert/":{},"oidc/":{}}' + +setup_list "a refused token with no working granter names the token and the setup steps, then bao's error, and exits 4" +export FAKE_BAO_LIST=denied FAKE_BAO_LOGIN=no-role +run_list +expect 4 "" +err_has "the store refused the bootstrap token in /tmp/granter-bootstrap-test.token" +err_has "one-time step, as root on this host" +err_has " bao policy write bao-bootstrap /etc/bootstrap.hcl" +err_has "permission denied" +if [ ! -e "$token" ]; then fail "token file deleted"; fi + +setup_list "a refused token beside a working granter exits 0 and deletes the token" +export FAKE_BAO_LIST=denied FAKE_BAO_LOGIN=ok +run_list +expect 0 "granter role already in place; bootstrap token not needed" +err_lacks "refused the bootstrap token" +if [ -e "$token" ]; then fail "token file kept"; fi + +setup_list "a sealed store is reported as sealed or unreachable, with bao's error" +export FAKE_BAO_LIST=sealed FAKE_BAO_STATUS_RC=2 +run_list +expect 1 "" +err_has "the store is sealed or unreachable; retrying." +err_has "Vault is sealed" +err_lacks "refused the bootstrap token" + +rm -f "$token" + echo "granter-login: $count cases, $failures failures" [ "$failures" = 0 ]