report the granter login's real error and retry on a standby store
This commit is contained in:
atlas
mara
parent
1665436347
commit
1e2e93da59
6 changed files with 265 additions and 21 deletions
|
|
@ -101,6 +101,10 @@ in
|
|||
# `bao`: a store that holds a key wins, an empty one takes the seed, and an
|
||||
# unreachable one mints nothing.
|
||||
script-test-grafana-secret-key = import ./script-tests/grafana-secret-key.nix { inherit pkgs; };
|
||||
# Runs ./host-modules/lib/granter-login.nix's login against a stub `bao`: a
|
||||
# refused certificate names the missing role, any other failure (a standby
|
||||
# node, a sealed store) does not.
|
||||
script-test-granter-login = import ./script-tests/granter-login.nix { inherit pkgs; };
|
||||
module-eval-nats-tls = import ./module-eval/nats-tls.nix {
|
||||
inherit pkgs self nixosSystem;
|
||||
inherit (pkgs) lib;
|
||||
|
|
|
|||
57
nix/host-modules/lib/granter-login.nix
Normal file
57
nix/host-modules/lib/granter-login.nix
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
# The cert login every unit that writes bao policies starts with, as shell text
|
||||
# that leaves `BAO_TOKEN` exported. It FAILS rather than skips: a grant that
|
||||
# was not written is otherwise invisible until whatever needs it fails
|
||||
# somewhere else.
|
||||
#
|
||||
# What the failure says follows from what the store answered:
|
||||
#
|
||||
# - A refused certificate ("invalid certificate or no client certificate
|
||||
# supplied", or "no chain matching all constraints"; openbao's cert-auth
|
||||
# login in `builtin/credential/cert/path_login.go`, HTTP 400) is the granter
|
||||
# role missing or not matching. Only that answer prints the one-time setup
|
||||
# steps.
|
||||
# - Any other failure from a store that answers `status` is not the role's:
|
||||
# a node that is not yet active answers the login with a 500 ("local node
|
||||
# not active but active cluster node not found") for the first moments after
|
||||
# an unseal. The message says so and the unit's `Restart=` retries.
|
||||
# - A store that does not answer `status` is sealed or unreachable.
|
||||
#
|
||||
# In every case bao's own error follows.
|
||||
#
|
||||
# Pure function — NOT a NixOS module. Call it from a module's `let`:
|
||||
#
|
||||
# granterLogin = import ./lib/granter-login.nix {
|
||||
# inherit lib granterCn granterPolicyName haveBootstrapToken baoStoreUp;
|
||||
# setupSteps = granterSetupSteps;
|
||||
# };
|
||||
#
|
||||
# `baoStoreUp` is shell text that exits 0 on a reachable, unsealed store and
|
||||
# appends its error to `$err`. Needs `bao`, `coreutils` and `gnugrep` on the
|
||||
# caller's `path`. Takes the EXIT trap, to remove its scratch file.
|
||||
{
|
||||
lib,
|
||||
granterCn,
|
||||
granterPolicyName,
|
||||
haveBootstrapToken,
|
||||
baoStoreUp,
|
||||
setupSteps,
|
||||
}:
|
||||
''
|
||||
err="$(mktemp)"
|
||||
trap 'rm -f "$err"' EXIT
|
||||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||||
if grep -qE 'invalid certificate or no client certificate supplied|no chain matching all constraints' "$err"; then
|
||||
echo ${lib.escapeShellArg "the store refused the granter's certificate (CN ${granterCn}): the ${granterPolicyName} role is not set up."} >&2
|
||||
${lib.optionalString haveBootstrapToken ''echo "swarm-bao-granter-role writes that role; journalctl -u swarm-bao-granter-role says why it has not." >&2''}
|
||||
echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2
|
||||
${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg " ${l}"} >&2") setupSteps}
|
||||
elif ${baoStoreUp}; then
|
||||
echo "the store is unsealed but the granter's login failed without a refused certificate, as a node that is not active yet answers; retrying." >&2
|
||||
else
|
||||
echo "the store is sealed or unreachable; retrying." >&2
|
||||
fi
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
export BAO_TOKEN
|
||||
''
|
||||
|
|
@ -311,27 +311,18 @@ let
|
|||
# config and no token.
|
||||
baoStoreUp = ''HOME=/var/empty bao status >/dev/null 2>>"$err"'';
|
||||
|
||||
# The login every granting unit starts with. It FAILS rather than skips: a
|
||||
# grant that was not written is otherwise invisible until whatever needs it
|
||||
# fails somewhere else. `baoStoreUp` separates "the granter is not set up"
|
||||
# from "retry later"; either way bao's own message follows.
|
||||
granterLogin = ''
|
||||
err="$(mktemp)"
|
||||
trap 'rm -f "$err"' EXIT
|
||||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||||
if ${baoStoreUp}; then
|
||||
echo ${lib.escapeShellArg "the store is unsealed but refused the granter's certificate (CN ${granterCn}): the ${granterPolicyName} role is not set up."} >&2
|
||||
${lib.optionalString haveBootstrapToken ''echo "swarm-bao-granter-role writes that role; journalctl -u swarm-bao-granter-role says why it has not." >&2''}
|
||||
echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2
|
||||
${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg " ${l}"} >&2") granterSetupSteps}
|
||||
else
|
||||
echo "the store is sealed or unreachable; retrying." >&2
|
||||
fi
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
export BAO_TOKEN
|
||||
'';
|
||||
# The login every granting unit starts with; ./lib/granter-login.nix says
|
||||
# which failures it reports as a missing role.
|
||||
granterLogin = import ./lib/granter-login.nix {
|
||||
inherit
|
||||
lib
|
||||
granterCn
|
||||
granterPolicyName
|
||||
haveBootstrapToken
|
||||
baoStoreUp
|
||||
;
|
||||
setupSteps = granterSetupSteps;
|
||||
};
|
||||
|
||||
# The granter's certificate for the granting units. The `baoCli` wrapper
|
||||
# only defaults these, so the unit's environment wins.
|
||||
|
|
|
|||
46
nix/script-tests/granter-login-fake-bao.sh
Executable file
46
nix/script-tests/granter-login-fake-bao.sh
Executable file
|
|
@ -0,0 +1,46 @@
|
|||
#!/usr/bin/env bash
|
||||
# The stub `bao` for ./granter-login.nix. `FAKE_BAO_LOGIN` picks the login's
|
||||
# answer, in openbao's wording: `ok`, `standby` (500, a node that is not active
|
||||
# yet), `no-role` (400, no trusted chain), `no-chain-match` (400, a chain that
|
||||
# no role accepts), `sealed` (503). `FAKE_BAO_STATUS_RC` is `status`'s exit
|
||||
# status. `FAKE_BAO_LOGIN_FIRST`, when set, answers the first login with it and
|
||||
# later ones with `FAKE_BAO_LOGIN`; `FAKE_BAO_COUNT` is the file counting them.
|
||||
echo "$*" >>"$FAKE_BAO_LOG"
|
||||
case "$1" in
|
||||
login)
|
||||
n=$(($(cat "$FAKE_BAO_COUNT" 2>/dev/null || echo 0) + 1))
|
||||
echo "$n" >"$FAKE_BAO_COUNT"
|
||||
answer=$FAKE_BAO_LOGIN
|
||||
if [ "$n" = 1 ] && [ -n "${FAKE_BAO_LOGIN_FIRST-}" ]; then answer=$FAKE_BAO_LOGIN_FIRST; fi
|
||||
url='URL: PUT https://bao.t.local:8200/v1/auth/cert/login'
|
||||
case "$answer" in
|
||||
ok) printf 's.fake' ;;
|
||||
standby)
|
||||
printf 'Error authenticating: Error making API request.\n\n%s\nCode: 500. Errors:\n\n* local node not active but active cluster node not found\n' "$url" >&2
|
||||
exit 2
|
||||
;;
|
||||
no-role)
|
||||
printf 'Error authenticating: Error making API request.\n\n%s\nCode: 400. Errors:\n\n* invalid certificate or no client certificate supplied\n' "$url" >&2
|
||||
exit 2
|
||||
;;
|
||||
no-chain-match)
|
||||
printf 'Error authenticating: Error making API request.\n\n%s\nCode: 400. Errors:\n\n* no chain matching all constraints could be found for this login certificate\n' "$url" >&2
|
||||
exit 2
|
||||
;;
|
||||
sealed)
|
||||
printf 'Error authenticating: Error making API request.\n\n%s\nCode: 503. Errors:\n\n* Vault is sealed\n' "$url" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
status)
|
||||
[ "${FAKE_BAO_STATUS_RC-0}" = 0 ] || {
|
||||
echo "fake bao: status: Vault is sealed" >&2
|
||||
exit 2
|
||||
}
|
||||
;;
|
||||
*)
|
||||
echo "fake bao: unexpected call: $*" >&2
|
||||
exit 98
|
||||
;;
|
||||
esac
|
||||
43
nix/script-tests/granter-login.nix
Normal file
43
nix/script-tests/granter-login.nix
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
# `checks.script-test-granter-login` — runs ../host-modules/lib/granter-login.nix
|
||||
# against a stub `bao` whose login and status answers are set per case. The
|
||||
# cases are in ./granter-login.sh and the stub is ./granter-login-fake-bao.sh.
|
||||
#
|
||||
# What runs is the rendered login, with the same `baoStoreUp` text the unit
|
||||
# passes, rendered once with the bootstrap-token hint and once without.
|
||||
{
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
render =
|
||||
haveBootstrapToken:
|
||||
pkgs.writeText "granter-login.sh" (
|
||||
import ../host-modules/lib/granter-login.nix {
|
||||
inherit (pkgs) lib;
|
||||
granterCn = "bao-granter";
|
||||
granterPolicyName = "bao-granter";
|
||||
inherit haveBootstrapToken;
|
||||
baoStoreUp = ''HOME=/var/empty bao status >/dev/null 2>>"$err"'';
|
||||
setupSteps = [ "bao policy write bao-bootstrap /etc/bootstrap.hcl" ];
|
||||
}
|
||||
);
|
||||
|
||||
fakeBao = pkgs.runCommand "fake-bao" { } ''
|
||||
install -Dm755 ${./granter-login-fake-bao.sh} $out/bin/bao
|
||||
patchShebangs $out/bin
|
||||
'';
|
||||
in
|
||||
pkgs.runCommand "hyperhive-script-test-granter-login"
|
||||
{
|
||||
LOGIN = render false;
|
||||
LOGIN_WITH_TOKEN = render true;
|
||||
FAKE_BAO_BIN = "${fakeBao}/bin";
|
||||
nativeBuildInputs = [
|
||||
pkgs.coreutils
|
||||
pkgs.gnugrep
|
||||
];
|
||||
}
|
||||
''
|
||||
${pkgs.bash}/bin/bash ${./granter-login.sh}
|
||||
touch "$out"
|
||||
''
|
||||
103
nix/script-tests/granter-login.sh
Normal file
103
nix/script-tests/granter-login.sh
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
# Cases for ./granter-login.nix. Each case sets the stub's answers, runs the
|
||||
# rendered login under `set -e` as a unit's script does, then compares its exit
|
||||
# status, what it printed to stderr and the token it left exported to children.
|
||||
set -uo pipefail
|
||||
|
||||
failures=0
|
||||
count=0
|
||||
|
||||
fail() {
|
||||
echo "FAILED: $case: $*" >&2
|
||||
failures=$((failures + 1))
|
||||
}
|
||||
|
||||
# setup <description>
|
||||
setup() {
|
||||
case=$1
|
||||
count=$((count + 1))
|
||||
dir=$(mktemp -d)
|
||||
export FAKE_BAO_LOG=$dir/bao.log FAKE_BAO_COUNT=$dir/count FAKE_BAO_LOGIN=ok FAKE_BAO_STATUS_RC=0
|
||||
unset FAKE_BAO_LOGIN_FIRST
|
||||
: >"$FAKE_BAO_LOG"
|
||||
login=$LOGIN
|
||||
}
|
||||
|
||||
# run — the login, then the token it exported. Sets $rc, $got, and $dir/err.
|
||||
run() {
|
||||
got=$(
|
||||
(
|
||||
set -e
|
||||
PATH="$FAKE_BAO_BIN:$PATH"
|
||||
unset BAO_TOKEN
|
||||
# shellcheck source=/dev/null
|
||||
. "$login"
|
||||
env | sed -n "s/^BAO_TOKEN=//p"
|
||||
) 2>"$dir/err"
|
||||
)
|
||||
rc=$?
|
||||
}
|
||||
|
||||
# expect <status> <token>
|
||||
expect() {
|
||||
if [ "$rc" != "$1" ]; then fail "exited $rc, wanted $1: $(cat "$dir/err")"; fi
|
||||
if [ "$got" != "$2" ]; then fail "token '$got', wanted '$2'"; fi
|
||||
}
|
||||
|
||||
err_has() {
|
||||
if ! grep -qF -- "$1" "$dir/err"; then fail "stderr lacks '$1': $(cat "$dir/err")"; fi
|
||||
}
|
||||
|
||||
err_lacks() {
|
||||
if grep -qF -- "$1" "$dir/err"; then fail "stderr has '$1': $(cat "$dir/err")"; fi
|
||||
}
|
||||
|
||||
setup "a successful login exports the token and prints nothing"
|
||||
run
|
||||
expect 0 s.fake
|
||||
if [ -s "$dir/err" ]; then fail "printed: $(cat "$dir/err")"; fi
|
||||
|
||||
setup "a standby store fails the login with its own error, does not blame the role, and the next attempt succeeds"
|
||||
export FAKE_BAO_LOGIN_FIRST=standby
|
||||
run
|
||||
expect 1 ""
|
||||
err_has "local node not active but active cluster node not found"
|
||||
err_has "login failed without a refused certificate"
|
||||
err_lacks "role is not set up"
|
||||
err_lacks "one-time step"
|
||||
run
|
||||
expect 0 s.fake
|
||||
|
||||
setup "an unknown role names the role and the setup steps, then bao's error"
|
||||
export FAKE_BAO_LOGIN=no-role
|
||||
run
|
||||
expect 1 ""
|
||||
err_has "refused the granter's certificate (CN bao-granter): the bao-granter role is not set up."
|
||||
err_has "one-time step, as root on this host"
|
||||
err_has " bao policy write bao-bootstrap /etc/bootstrap.hcl"
|
||||
err_has "invalid certificate or no client certificate supplied"
|
||||
err_lacks "swarm-bao-granter-role writes that role"
|
||||
|
||||
setup "a certificate no role accepts is a missing role too"
|
||||
export FAKE_BAO_LOGIN=no-chain-match
|
||||
run
|
||||
expect 1 ""
|
||||
err_has "the bao-granter role is not set up."
|
||||
err_has "no chain matching all constraints"
|
||||
|
||||
setup "a unit with a bootstrap token also points at the unit that writes the role"
|
||||
export FAKE_BAO_LOGIN=no-role
|
||||
login=$LOGIN_WITH_TOKEN
|
||||
run
|
||||
expect 1 ""
|
||||
err_has "swarm-bao-granter-role writes that role"
|
||||
|
||||
setup "a sealed store is reported as sealed or unreachable, with bao's error"
|
||||
export FAKE_BAO_LOGIN=sealed FAKE_BAO_STATUS_RC=2
|
||||
run
|
||||
expect 1 ""
|
||||
err_has "the store is sealed or unreachable; retrying."
|
||||
err_has "Vault is sealed"
|
||||
err_lacks "role is not set up"
|
||||
|
||||
echo "granter-login: $count cases, $failures failures"
|
||||
[ "$failures" = 0 ]
|
||||
Loading…
Reference in a new issue