diff --git a/nix/checks.nix b/nix/checks.nix index a749da24..1d7a48b0 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -101,6 +101,10 @@ in # `bao`: a store that holds a key wins, an empty one takes the seed, and an # unreachable one mints nothing. script-test-grafana-secret-key = import ./script-tests/grafana-secret-key.nix { inherit pkgs; }; + # Runs ./host-modules/lib/granter-login.nix's login against a stub `bao`: a + # refused certificate names the missing role, any other failure (a standby + # node, a sealed store) does not. + script-test-granter-login = import ./script-tests/granter-login.nix { inherit pkgs; }; module-eval-nats-tls = import ./module-eval/nats-tls.nix { inherit pkgs self nixosSystem; inherit (pkgs) lib; diff --git a/nix/host-modules/lib/granter-login.nix b/nix/host-modules/lib/granter-login.nix new file mode 100644 index 00000000..6ee02aa2 --- /dev/null +++ b/nix/host-modules/lib/granter-login.nix @@ -0,0 +1,57 @@ +# The cert login every unit that writes bao policies starts with, as shell text +# that leaves `BAO_TOKEN` exported. It FAILS rather than skips: a grant that +# was not written is otherwise invisible until whatever needs it fails +# somewhere else. +# +# What the failure says follows from what the store answered: +# +# - A refused certificate ("invalid certificate or no client certificate +# supplied", or "no chain matching all constraints"; openbao's cert-auth +# login in `builtin/credential/cert/path_login.go`, HTTP 400) is the granter +# role missing or not matching. Only that answer prints the one-time setup +# steps. +# - Any other failure from a store that answers `status` is not the role's: +# a node that is not yet active answers the login with a 500 ("local node +# not active but active cluster node not found") for the first moments after +# an unseal. The message says so and the unit's `Restart=` retries. +# - A store that does not answer `status` is sealed or unreachable. +# +# In every case bao's own error follows. +# +# Pure function — NOT a NixOS module. Call it from a module's `let`: +# +# granterLogin = import ./lib/granter-login.nix { +# inherit lib granterCn granterPolicyName haveBootstrapToken baoStoreUp; +# setupSteps = granterSetupSteps; +# }; +# +# `baoStoreUp` is shell text that exits 0 on a reachable, unsealed store and +# appends its error to `$err`. Needs `bao`, `coreutils` and `gnugrep` on the +# caller's `path`. Takes the EXIT trap, to remove its scratch file. +{ + lib, + granterCn, + granterPolicyName, + haveBootstrapToken, + baoStoreUp, + setupSteps, +}: +'' + err="$(mktemp)" + trap 'rm -f "$err"' EXIT + if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then + if grep -qE 'invalid certificate or no client certificate supplied|no chain matching all constraints' "$err"; then + echo ${lib.escapeShellArg "the store refused the granter's certificate (CN ${granterCn}): the ${granterPolicyName} role is not set up."} >&2 + ${lib.optionalString haveBootstrapToken ''echo "swarm-bao-granter-role writes that role; journalctl -u swarm-bao-granter-role says why it has not." >&2''} + echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2 + ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg " ${l}"} >&2") setupSteps} + elif ${baoStoreUp}; then + echo "the store is unsealed but the granter's login failed without a refused certificate, as a node that is not active yet answers; retrying." >&2 + else + echo "the store is sealed or unreachable; retrying." >&2 + fi + cat "$err" >&2 + exit 1 + fi + export BAO_TOKEN +'' diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 47e1d935..2c93a69a 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -311,27 +311,18 @@ let # config and no token. baoStoreUp = ''HOME=/var/empty bao status >/dev/null 2>>"$err"''; - # The login every granting unit starts with. It FAILS rather than skips: a - # grant that was not written is otherwise invisible until whatever needs it - # fails somewhere else. `baoStoreUp` separates "the granter is not set up" - # from "retry later"; either way bao's own message follows. - granterLogin = '' - err="$(mktemp)" - trap 'rm -f "$err"' EXIT - if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then - if ${baoStoreUp}; then - echo ${lib.escapeShellArg "the store is unsealed but refused the granter's certificate (CN ${granterCn}): the ${granterPolicyName} role is not set up."} >&2 - ${lib.optionalString haveBootstrapToken ''echo "swarm-bao-granter-role writes that role; journalctl -u swarm-bao-granter-role says why it has not." >&2''} - echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2 - ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg " ${l}"} >&2") granterSetupSteps} - else - echo "the store is sealed or unreachable; retrying." >&2 - fi - cat "$err" >&2 - exit 1 - fi - export BAO_TOKEN - ''; + # The login every granting unit starts with; ./lib/granter-login.nix says + # which failures it reports as a missing role. + granterLogin = import ./lib/granter-login.nix { + inherit + lib + granterCn + granterPolicyName + haveBootstrapToken + baoStoreUp + ; + setupSteps = granterSetupSteps; + }; # The granter's certificate for the granting units. The `baoCli` wrapper # only defaults these, so the unit's environment wins. diff --git a/nix/script-tests/granter-login-fake-bao.sh b/nix/script-tests/granter-login-fake-bao.sh new file mode 100755 index 00000000..482d5db6 --- /dev/null +++ b/nix/script-tests/granter-login-fake-bao.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# The stub `bao` for ./granter-login.nix. `FAKE_BAO_LOGIN` picks the login's +# answer, in openbao's wording: `ok`, `standby` (500, a node that is not active +# yet), `no-role` (400, no trusted chain), `no-chain-match` (400, a chain that +# no role accepts), `sealed` (503). `FAKE_BAO_STATUS_RC` is `status`'s exit +# status. `FAKE_BAO_LOGIN_FIRST`, when set, answers the first login with it and +# later ones with `FAKE_BAO_LOGIN`; `FAKE_BAO_COUNT` is the file counting them. +echo "$*" >>"$FAKE_BAO_LOG" +case "$1" in + login) + n=$(($(cat "$FAKE_BAO_COUNT" 2>/dev/null || echo 0) + 1)) + echo "$n" >"$FAKE_BAO_COUNT" + answer=$FAKE_BAO_LOGIN + if [ "$n" = 1 ] && [ -n "${FAKE_BAO_LOGIN_FIRST-}" ]; then answer=$FAKE_BAO_LOGIN_FIRST; fi + url='URL: PUT https://bao.t.local:8200/v1/auth/cert/login' + case "$answer" in + ok) printf 's.fake' ;; + standby) + printf 'Error authenticating: Error making API request.\n\n%s\nCode: 500. Errors:\n\n* local node not active but active cluster node not found\n' "$url" >&2 + exit 2 + ;; + no-role) + printf 'Error authenticating: Error making API request.\n\n%s\nCode: 400. Errors:\n\n* invalid certificate or no client certificate supplied\n' "$url" >&2 + exit 2 + ;; + no-chain-match) + printf 'Error authenticating: Error making API request.\n\n%s\nCode: 400. Errors:\n\n* no chain matching all constraints could be found for this login certificate\n' "$url" >&2 + exit 2 + ;; + sealed) + printf 'Error authenticating: Error making API request.\n\n%s\nCode: 503. Errors:\n\n* Vault is sealed\n' "$url" >&2 + exit 2 + ;; + esac + ;; + status) + [ "${FAKE_BAO_STATUS_RC-0}" = 0 ] || { + echo "fake bao: status: Vault is sealed" >&2 + exit 2 + } + ;; + *) + echo "fake bao: unexpected call: $*" >&2 + exit 98 + ;; +esac diff --git a/nix/script-tests/granter-login.nix b/nix/script-tests/granter-login.nix new file mode 100644 index 00000000..d196717b --- /dev/null +++ b/nix/script-tests/granter-login.nix @@ -0,0 +1,43 @@ +# `checks.script-test-granter-login` — runs ../host-modules/lib/granter-login.nix +# against a stub `bao` whose login and status answers are set per case. The +# cases are in ./granter-login.sh and the stub is ./granter-login-fake-bao.sh. +# +# What runs is the rendered login, with the same `baoStoreUp` text the unit +# passes, rendered once with the bootstrap-token hint and once without. +{ + pkgs, + ... +}: +let + render = + haveBootstrapToken: + pkgs.writeText "granter-login.sh" ( + import ../host-modules/lib/granter-login.nix { + inherit (pkgs) lib; + granterCn = "bao-granter"; + granterPolicyName = "bao-granter"; + inherit haveBootstrapToken; + baoStoreUp = ''HOME=/var/empty bao status >/dev/null 2>>"$err"''; + setupSteps = [ "bao policy write bao-bootstrap /etc/bootstrap.hcl" ]; + } + ); + + fakeBao = pkgs.runCommand "fake-bao" { } '' + install -Dm755 ${./granter-login-fake-bao.sh} $out/bin/bao + patchShebangs $out/bin + ''; +in +pkgs.runCommand "hyperhive-script-test-granter-login" + { + LOGIN = render false; + LOGIN_WITH_TOKEN = render true; + FAKE_BAO_BIN = "${fakeBao}/bin"; + nativeBuildInputs = [ + pkgs.coreutils + pkgs.gnugrep + ]; + } + '' + ${pkgs.bash}/bin/bash ${./granter-login.sh} + touch "$out" + '' diff --git a/nix/script-tests/granter-login.sh b/nix/script-tests/granter-login.sh new file mode 100644 index 00000000..20cd17a0 --- /dev/null +++ b/nix/script-tests/granter-login.sh @@ -0,0 +1,103 @@ +# Cases for ./granter-login.nix. Each case sets the stub's answers, runs the +# rendered login under `set -e` as a unit's script does, then compares its exit +# status, what it printed to stderr and the token it left exported to children. +set -uo pipefail + +failures=0 +count=0 + +fail() { + echo "FAILED: $case: $*" >&2 + failures=$((failures + 1)) +} + +# setup +setup() { + case=$1 + count=$((count + 1)) + dir=$(mktemp -d) + export FAKE_BAO_LOG=$dir/bao.log FAKE_BAO_COUNT=$dir/count FAKE_BAO_LOGIN=ok FAKE_BAO_STATUS_RC=0 + unset FAKE_BAO_LOGIN_FIRST + : >"$FAKE_BAO_LOG" + login=$LOGIN +} + +# run — the login, then the token it exported. Sets $rc, $got, and $dir/err. +run() { + got=$( + ( + set -e + PATH="$FAKE_BAO_BIN:$PATH" + unset BAO_TOKEN + # shellcheck source=/dev/null + . "$login" + env | sed -n "s/^BAO_TOKEN=//p" + ) 2>"$dir/err" + ) + rc=$? +} + +# expect +expect() { + if [ "$rc" != "$1" ]; then fail "exited $rc, wanted $1: $(cat "$dir/err")"; fi + if [ "$got" != "$2" ]; then fail "token '$got', wanted '$2'"; fi +} + +err_has() { + if ! grep -qF -- "$1" "$dir/err"; then fail "stderr lacks '$1': $(cat "$dir/err")"; fi +} + +err_lacks() { + if grep -qF -- "$1" "$dir/err"; then fail "stderr has '$1': $(cat "$dir/err")"; fi +} + +setup "a successful login exports the token and prints nothing" +run +expect 0 s.fake +if [ -s "$dir/err" ]; then fail "printed: $(cat "$dir/err")"; fi + +setup "a standby store fails the login with its own error, does not blame the role, and the next attempt succeeds" +export FAKE_BAO_LOGIN_FIRST=standby +run +expect 1 "" +err_has "local node not active but active cluster node not found" +err_has "login failed without a refused certificate" +err_lacks "role is not set up" +err_lacks "one-time step" +run +expect 0 s.fake + +setup "an unknown role names the role and the setup steps, then bao's error" +export FAKE_BAO_LOGIN=no-role +run +expect 1 "" +err_has "refused the granter's certificate (CN bao-granter): the bao-granter role is not set up." +err_has "one-time step, as root on this host" +err_has " bao policy write bao-bootstrap /etc/bootstrap.hcl" +err_has "invalid certificate or no client certificate supplied" +err_lacks "swarm-bao-granter-role writes that role" + +setup "a certificate no role accepts is a missing role too" +export FAKE_BAO_LOGIN=no-chain-match +run +expect 1 "" +err_has "the bao-granter role is not set up." +err_has "no chain matching all constraints" + +setup "a unit with a bootstrap token also points at the unit that writes the role" +export FAKE_BAO_LOGIN=no-role +login=$LOGIN_WITH_TOKEN +run +expect 1 "" +err_has "swarm-bao-granter-role writes that role" + +setup "a sealed store is reported as sealed or unreachable, with bao's error" +export FAKE_BAO_LOGIN=sealed FAKE_BAO_STATUS_RC=2 +run +expect 1 "" +err_has "the store is sealed or unreachable; retrying." +err_has "Vault is sealed" +err_lacks "role is not set up" + +echo "granter-login: $count cases, $failures failures" +[ "$failures" = 0 ]