report the real error when the granter role's bootstrap step fails, and retry on a standby store
This commit is contained in:
atlas
parent
3ff4dc7be0
commit
cb5a9b2bd7
6 changed files with 186 additions and 32 deletions
|
|
@ -40,8 +40,10 @@ the bootstrap commands. It never skips. `swarm-bao-granter-role` skips while
|
|||
the token file is absent, which is the steady state afterwards; the token's
|
||||
TTL means a forgotten file expires rather than lingering.
|
||||
|
||||
A sealed or unreachable store makes the unit retry every 30 s. A store that
|
||||
refuses the token (expired, revoked, or missing the `bao-bootstrap` policy)
|
||||
A sealed or unreachable store, or one whose listing fails with something other
|
||||
than a refusal (a node still becoming active), makes the unit retry every
|
||||
30 s with the store's error in the log. A store that refuses the token (expired,
|
||||
revoked, or missing the `bao-bootstrap` policy)
|
||||
makes it exit 4 and stay `failed` with the one-time step in its log; mint a
|
||||
fresh token or remove the file, then restart it. If the granter's own
|
||||
certificate login already works, the unit logs that the bootstrap token isn't
|
||||
|
|
|
|||
57
nix/host-modules/lib/bootstrap-auth-list.nix
Normal file
57
nix/host-modules/lib/bootstrap-auth-list.nix
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
# The bootstrap token's first use in `swarm-bao-granter-role`, as shell text
|
||||
# that leaves `mounted` set to the store's auth listing, or ends the script.
|
||||
# Same split as ./granter-login.nix, applied to `bao auth list`:
|
||||
#
|
||||
# - A refusal ("permission denied", HTTP 403: openbao answers an expired,
|
||||
# revoked or underprivileged token alike) is the token's. If the granter's
|
||||
# own login works the role is already in place and the dead token is
|
||||
# deleted; otherwise the unit prints the one-time steps and exits 4, which
|
||||
# `RestartPreventExitStatus` keeps from retrying.
|
||||
# - Any other failure from a store that answers `status` is not the token's:
|
||||
# a node that is not yet active answers with a 500 for the first moments
|
||||
# after an unseal. The message says so and exits 1, which `Restart=` retries.
|
||||
# - A store that does not answer `status` is sealed or unreachable; exit 1.
|
||||
#
|
||||
# In every case bao's own error follows.
|
||||
#
|
||||
# Pure function — NOT a NixOS module. `baoStoreUp` is as for
|
||||
# ./granter-login.nix. Needs `bao`, `coreutils` and `gnugrep` on the caller's
|
||||
# `path`, and `$err` to be a scratch file.
|
||||
{
|
||||
lib,
|
||||
bootstrapTokenFile,
|
||||
granterClientCertFile,
|
||||
granterClientKeyFile,
|
||||
baoStoreUp,
|
||||
setupSteps,
|
||||
}:
|
||||
''
|
||||
if ! mounted="$(bao auth list -format=json 2>"$err")"; then
|
||||
if grep -qF 'permission denied' "$err"; then
|
||||
# A dead token beside a working granter is the steady state of
|
||||
# a host that finished bootstrap and kept the file. The
|
||||
# granter's own login is the proof. A fresh token still takes
|
||||
# the full path below, since the granter cannot read back its
|
||||
# own policy to compare it. The refused token is dead,
|
||||
# so it is deleted here and the unit does not start again.
|
||||
if BAO_CLIENT_CERT=${lib.escapeShellArg granterClientCertFile} \
|
||||
BAO_CLIENT_KEY=${lib.escapeShellArg granterClientKeyFile} \
|
||||
bao login -method=cert -token-only >/dev/null 2>&1; then
|
||||
echo "granter role already in place; bootstrap token not needed"
|
||||
rm -f ${lib.escapeShellArg bootstrapTokenFile}
|
||||
exit 0
|
||||
fi
|
||||
echo ${lib.escapeShellArg "the store refused the bootstrap token in ${bootstrapTokenFile}: it has expired, been revoked, or does not carry the bao-bootstrap policy."} >&2
|
||||
echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2
|
||||
${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg " ${l}"} >&2") setupSteps}
|
||||
cat "$err" >&2
|
||||
exit 4
|
||||
elif ${baoStoreUp}; then
|
||||
echo "the store is unsealed but listing its auth mounts failed without a refused token, as a node that is not active yet answers; retrying." >&2
|
||||
else
|
||||
echo "the store is sealed or unreachable; retrying." >&2
|
||||
fi
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
''
|
||||
|
|
@ -324,6 +324,17 @@ let
|
|||
setupSteps = granterSetupSteps;
|
||||
};
|
||||
|
||||
# The bootstrap token's first use, in `swarm-bao-granter-role`.
|
||||
bootstrapAuthList = import ./lib/bootstrap-auth-list.nix {
|
||||
inherit lib baoStoreUp;
|
||||
inherit (baoDeploy)
|
||||
bootstrapTokenFile
|
||||
granterClientCertFile
|
||||
granterClientKeyFile
|
||||
;
|
||||
setupSteps = granterSetupSteps;
|
||||
};
|
||||
|
||||
# The granter's certificate for the granting units. The `baoCli` wrapper
|
||||
# only defaults these, so the unit's environment wins.
|
||||
granterEnv = {
|
||||
|
|
@ -3374,6 +3385,7 @@ in
|
|||
path = [
|
||||
baoCli
|
||||
pkgs.coreutils
|
||||
pkgs.gnugrep
|
||||
];
|
||||
# Named but not placed is a legitimate state: all-local supplies the
|
||||
# path as a default and the operator drops the file there after
|
||||
|
|
@ -3426,34 +3438,9 @@ in
|
|||
# that already exists, and recognising that would tie a rebuild
|
||||
# to an error string we have never seen this store emit.
|
||||
#
|
||||
# The token's first use, and the bootstrap policy grants this read,
|
||||
# so a refusal from a store that is up is the token's: expired,
|
||||
# revoked, or minted without that policy.
|
||||
if ! mounted="$(bao auth list -format=json 2>"$err")"; then
|
||||
if ${baoStoreUp}; then
|
||||
# A dead token beside a working granter is the steady state of
|
||||
# a host that finished bootstrap and kept the file. The
|
||||
# granter's own login is the proof. A fresh token still takes
|
||||
# the full path below, since the granter cannot read back its
|
||||
# own policy to compare it. The refused token is dead,
|
||||
# so it is deleted here and the unit does not start again.
|
||||
if BAO_CLIENT_CERT=${lib.escapeShellArg baoDeploy.granterClientCertFile} \
|
||||
BAO_CLIENT_KEY=${lib.escapeShellArg baoDeploy.granterClientKeyFile} \
|
||||
bao login -method=cert -token-only >/dev/null 2>&1; then
|
||||
echo "granter role already in place; bootstrap token not needed"
|
||||
rm -f ${lib.escapeShellArg baoDeploy.bootstrapTokenFile}
|
||||
exit 0
|
||||
fi
|
||||
echo ${lib.escapeShellArg "the store is unsealed but refused the bootstrap token in ${baoDeploy.bootstrapTokenFile}: it has expired, been revoked, or does not carry the bao-bootstrap policy."} >&2
|
||||
echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2
|
||||
${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg " ${l}"} >&2") granterSetupSteps}
|
||||
cat "$err" >&2
|
||||
exit 4
|
||||
fi
|
||||
echo "the store is sealed or unreachable; retrying." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
# The token's first use; ./lib/bootstrap-auth-list.nix says which
|
||||
# failures are the token's.
|
||||
${bootstrapAuthList}
|
||||
case "$mounted" in
|
||||
*'"cert/"'*) ;;
|
||||
*) bao auth enable cert ;;
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
#!/usr/bin/env bash
|
||||
# The stub `bao` for ./granter-login.nix. `FAKE_BAO_LOGIN` picks the login's
|
||||
# The stub `bao` for ./granter-login.nix and ./lib/bootstrap-auth-list.nix. `FAKE_BAO_LOGIN` picks the login's
|
||||
# answer, in openbao's wording: `ok`, `standby` (500, a node that is not active
|
||||
# yet), `no-role` (400, no trusted chain), `no-chain-match` (400, a chain that
|
||||
# no role accepts), `sealed` (503). `FAKE_BAO_STATUS_RC` is `status`'s exit
|
||||
# status. `FAKE_BAO_LOGIN_FIRST`, when set, answers the first login with it and
|
||||
# later ones with `FAKE_BAO_LOGIN`; `FAKE_BAO_COUNT` is the file counting them.
|
||||
# `auth list` is answered per the `auth)` arm below.
|
||||
echo "$*" >>"$FAKE_BAO_LOG"
|
||||
case "$1" in
|
||||
login)
|
||||
|
|
@ -33,6 +34,31 @@ case "$1" in
|
|||
;;
|
||||
esac
|
||||
;;
|
||||
auth)
|
||||
# `FAKE_BAO_LIST` picks `auth list`'s answer: `ok`, `standby` (500),
|
||||
# `denied` (403, a token the store refuses), `sealed` (503).
|
||||
# `FAKE_BAO_LIST_FIRST` answers only the first one with it.
|
||||
n=$(($(cat "$FAKE_BAO_LIST_COUNT" 2>/dev/null || echo 0) + 1))
|
||||
echo "$n" >"$FAKE_BAO_LIST_COUNT"
|
||||
answer=${FAKE_BAO_LIST-ok}
|
||||
if [ "$n" = 1 ] && [ -n "${FAKE_BAO_LIST_FIRST-}" ]; then answer=$FAKE_BAO_LIST_FIRST; fi
|
||||
url='URL: GET https://bao.t.local:8200/v1/sys/auth'
|
||||
case "$answer" in
|
||||
ok) printf '{"cert/":{},"oidc/":{}}' ;;
|
||||
standby)
|
||||
printf 'Error listing enabled authentications: Error making API request.\n\n%s\nCode: 500. Errors:\n\n* local node not active but active cluster node not found\n' "$url" >&2
|
||||
exit 2
|
||||
;;
|
||||
denied)
|
||||
printf 'Error listing enabled authentications: Error making API request.\n\n%s\nCode: 403. Errors:\n\n* permission denied\n' "$url" >&2
|
||||
exit 2
|
||||
;;
|
||||
sealed)
|
||||
printf 'Error listing enabled authentications: Error making API request.\n\n%s\nCode: 503. Errors:\n\n* Vault is sealed\n' "$url" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
status)
|
||||
[ "${FAKE_BAO_STATUS_RC-0}" = 0 ] || {
|
||||
echo "fake bao: status: Vault is sealed" >&2
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@
|
|||
# cases are in ./granter-login.sh and the stub is ./granter-login-fake-bao.sh.
|
||||
#
|
||||
# What runs is the rendered login, with the same `baoStoreUp` text the unit
|
||||
# passes, rendered once with the bootstrap-token hint and once without.
|
||||
# passes, rendered once with the bootstrap-token hint and once without. The
|
||||
# bootstrap token's `auth list` step (../host-modules/lib/bootstrap-auth-list.nix)
|
||||
# is rendered the same way and run against the same stub.
|
||||
{
|
||||
pkgs,
|
||||
...
|
||||
|
|
@ -31,6 +33,16 @@ pkgs.runCommand "hyperhive-script-test-granter-login"
|
|||
{
|
||||
LOGIN = render false;
|
||||
LOGIN_WITH_TOKEN = render true;
|
||||
BOOTSTRAP_LIST = pkgs.writeText "bootstrap-auth-list.sh" (
|
||||
import ../host-modules/lib/bootstrap-auth-list.nix {
|
||||
inherit (pkgs) lib;
|
||||
bootstrapTokenFile = "/tmp/granter-bootstrap-test.token";
|
||||
granterClientCertFile = "/nonexistent/granter.crt";
|
||||
granterClientKeyFile = "/nonexistent/granter.key";
|
||||
baoStoreUp = ''HOME=/var/empty bao status >/dev/null 2>>"$err"'';
|
||||
setupSteps = [ "bao policy write bao-bootstrap /etc/bootstrap.hcl" ];
|
||||
}
|
||||
);
|
||||
FAKE_BAO_BIN = "${fakeBao}/bin";
|
||||
nativeBuildInputs = [
|
||||
pkgs.coreutils
|
||||
|
|
|
|||
|
|
@ -99,5 +99,75 @@ err_has "the store is sealed or unreachable; retrying."
|
|||
err_has "Vault is sealed"
|
||||
err_lacks "role is not set up"
|
||||
|
||||
# The bootstrap token's `auth list` step: the rendered snippet runs under
|
||||
# `set -e` with the scratch file the unit makes. Sets $rc, $got (the listing it
|
||||
# left in `mounted`) and $dir/err.
|
||||
token=/tmp/granter-bootstrap-test.token
|
||||
|
||||
setup_list() {
|
||||
setup "$1"
|
||||
export FAKE_BAO_LIST_COUNT=$dir/list-count FAKE_BAO_LIST=ok
|
||||
unset FAKE_BAO_LIST_FIRST
|
||||
echo dead >"$token"
|
||||
}
|
||||
|
||||
run_list() {
|
||||
got=$(
|
||||
(
|
||||
set -e
|
||||
PATH="$FAKE_BAO_BIN:$PATH"
|
||||
err=$(mktemp)
|
||||
# shellcheck source=/dev/null
|
||||
. "$BOOTSTRAP_LIST"
|
||||
echo "$mounted"
|
||||
) 2>"$dir/err"
|
||||
)
|
||||
rc=$?
|
||||
}
|
||||
|
||||
setup_list "a listing that succeeds leaves it in mounted and prints nothing"
|
||||
run_list
|
||||
expect 0 '{"cert/":{},"oidc/":{}}'
|
||||
if [ -s "$dir/err" ]; then fail "printed: $(cat "$dir/err")"; fi
|
||||
|
||||
setup_list "a standby store fails the listing with its own error, does not blame the token, keeps the token file, and the next attempt succeeds"
|
||||
export FAKE_BAO_LIST_FIRST=standby
|
||||
run_list
|
||||
expect 1 ""
|
||||
err_has "local node not active but active cluster node not found"
|
||||
err_has "listing its auth mounts failed without a refused token"
|
||||
err_lacks "refused the bootstrap token"
|
||||
err_lacks "one-time step"
|
||||
if [ ! -e "$token" ]; then fail "token file deleted"; fi
|
||||
run_list
|
||||
expect 0 '{"cert/":{},"oidc/":{}}'
|
||||
|
||||
setup_list "a refused token with no working granter names the token and the setup steps, then bao's error, and exits 4"
|
||||
export FAKE_BAO_LIST=denied FAKE_BAO_LOGIN=no-role
|
||||
run_list
|
||||
expect 4 ""
|
||||
err_has "the store refused the bootstrap token in /tmp/granter-bootstrap-test.token"
|
||||
err_has "one-time step, as root on this host"
|
||||
err_has " bao policy write bao-bootstrap /etc/bootstrap.hcl"
|
||||
err_has "permission denied"
|
||||
if [ ! -e "$token" ]; then fail "token file deleted"; fi
|
||||
|
||||
setup_list "a refused token beside a working granter exits 0 and deletes the token"
|
||||
export FAKE_BAO_LIST=denied FAKE_BAO_LOGIN=ok
|
||||
run_list
|
||||
expect 0 "granter role already in place; bootstrap token not needed"
|
||||
err_lacks "refused the bootstrap token"
|
||||
if [ -e "$token" ]; then fail "token file kept"; fi
|
||||
|
||||
setup_list "a sealed store is reported as sealed or unreachable, with bao's error"
|
||||
export FAKE_BAO_LIST=sealed FAKE_BAO_STATUS_RC=2
|
||||
run_list
|
||||
expect 1 ""
|
||||
err_has "the store is sealed or unreachable; retrying."
|
||||
err_has "Vault is sealed"
|
||||
err_lacks "refused the bootstrap token"
|
||||
|
||||
rm -f "$token"
|
||||
|
||||
echo "granter-login: $count cases, $failures failures"
|
||||
[ "$failures" = 0 ]
|
||||
|
|
|
|||
Loading…
Reference in a new issue