checks: run the agent bao-fetch unit scripts against a stub bao
`script-test-agent-bao-fetch` executes the rendered `ExecStart` of `hive-agent-forge-token` and `hive-agent-queue-credential` under `umask 0377`, with a stub `bao` first on the unit's own PATH. It covers every error branch, the happy path, forge rotation/unchanged, and 0400 files already in place: the redirect failure #4736 fixed, whose live symptom was `bao.err: Permission denied` reported as a refused certificate. The TLS-alert fixture is the `unknown certificate authority` error h-atlas's identity check got from the store. #4736 claimed this test but never committed it; the two module-eval suites for these units only evaluate the config. Closes #4748
This commit is contained in:
atlas
mara
parent
916c441e82
commit
202f7f7c83
3 changed files with 346 additions and 0 deletions
|
|
@ -120,6 +120,12 @@ in
|
|||
inherit pkgs self nixosSystem;
|
||||
inherit (pkgs) lib;
|
||||
};
|
||||
# Executes what the two `module-eval-agent-*-bao` suites above only
|
||||
# evaluate: both fetch units' rendered scripts, against a stub `bao`.
|
||||
script-test-agent-bao-fetch = import ./script-tests/agent-bao-fetch.nix {
|
||||
inherit pkgs self nixosSystem;
|
||||
inherit (pkgs) lib;
|
||||
};
|
||||
module-eval-agent-memory = import ./module-eval/agent-memory.nix {
|
||||
inherit pkgs self nixosSystem;
|
||||
inherit (pkgs) lib;
|
||||
|
|
|
|||
82
nix/script-tests/agent-bao-fetch.nix
Normal file
82
nix/script-tests/agent-bao-fetch.nix
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
# `checks.script-test-agent-bao-fetch` — runs the two agent units that log in
|
||||
# to the swarm secret store and fetch a secret, ../agent-modules/forge-token.nix
|
||||
# and ../agent-modules/queue-identity.nix, against a stub `bao`. The cases are
|
||||
# in ./agent-bao-fetch.sh.
|
||||
#
|
||||
# What runs is each unit's rendered `ExecStart`, on the unit's own `PATH` with
|
||||
# the stub in front. The one edit is the unit's `/run/<unit>/` prefix, moved
|
||||
# under the build directory because the sandbox has no writable `/run`.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
self,
|
||||
nixosSystem,
|
||||
}:
|
||||
let
|
||||
inherit
|
||||
(import ../module-eval/lib.nix {
|
||||
inherit
|
||||
pkgs
|
||||
lib
|
||||
self
|
||||
nixosSystem
|
||||
;
|
||||
})
|
||||
agentWith
|
||||
;
|
||||
|
||||
machine = agentWith { services.hyperhive.agent.bao.addr = "https://bao.t.local:8200"; };
|
||||
|
||||
unitEnv =
|
||||
prefix: name:
|
||||
let
|
||||
u = machine.systemd.services.${name};
|
||||
in
|
||||
{
|
||||
"${prefix}_UNIT" = name;
|
||||
# `removeSuffix`, not `trim`: `trim` drops the string context, and with it
|
||||
# the script's store path from this check's inputs.
|
||||
"${prefix}_SCRIPT" = lib.removeSuffix " " u.serviceConfig.ExecStart;
|
||||
"${prefix}_PATH" = u.environment.PATH;
|
||||
"${prefix}_BAO_ADDR" = u.environment.BAO_ADDR;
|
||||
};
|
||||
|
||||
# Answers `login` and `kv get` from `FAKE_BAO_*` variables and logs every
|
||||
# call. A `kv` call without the token `login` printed fails, so a script that
|
||||
# drops `BAO_TOKEN` between the two cannot pass.
|
||||
fakeBao = pkgs.writeShellScriptBin "bao" ''
|
||||
echo "$*" >> "$FAKE_BAO_LOG"
|
||||
case "$1" in
|
||||
login)
|
||||
printf '%s' "$FAKE_BAO_LOGIN_ERR" >&2
|
||||
printf '%s' "$FAKE_BAO_LOGIN_OUT"
|
||||
exit "$FAKE_BAO_LOGIN_RC"
|
||||
;;
|
||||
kv)
|
||||
if [ "''${BAO_TOKEN-}" != "$FAKE_BAO_LOGIN_OUT" ]; then
|
||||
echo "fake bao: kv called without the login's token" >&2
|
||||
exit 97
|
||||
fi
|
||||
printf '%s' "$FAKE_BAO_KV_ERR" >&2
|
||||
printf '%s' "$FAKE_BAO_KV_OUT"
|
||||
exit "$FAKE_BAO_KV_RC"
|
||||
;;
|
||||
*)
|
||||
echo "fake bao: unexpected call: $*" >&2
|
||||
exit 98
|
||||
;;
|
||||
esac
|
||||
'';
|
||||
in
|
||||
pkgs.runCommand "hyperhive-script-test-agent-bao-fetch"
|
||||
(
|
||||
unitEnv "FORGE" "hive-agent-forge-token"
|
||||
// unitEnv "QUEUE" "hive-agent-queue-credential"
|
||||
// {
|
||||
FAKE_BAO_BIN = "${fakeBao}/bin";
|
||||
}
|
||||
)
|
||||
''
|
||||
${pkgs.bash}/bin/bash ${./agent-bao-fetch.sh}
|
||||
touch "$out"
|
||||
''
|
||||
258
nix/script-tests/agent-bao-fetch.sh
Normal file
258
nix/script-tests/agent-bao-fetch.sh
Normal file
|
|
@ -0,0 +1,258 @@
|
|||
# Cases for ./agent-bao-fetch.nix. Each case gets a fresh runtime directory and
|
||||
# credentials directory, runs one unit's script under the unit's `UMask=0377`
|
||||
# with a clean environment, and asserts on the exit code, the output, the
|
||||
# files left behind and the `bao` calls made.
|
||||
set -uo pipefail
|
||||
|
||||
failures=0
|
||||
count=0
|
||||
|
||||
fail() {
|
||||
echo "FAILED: $case: $*" >&2
|
||||
failures=$((failures + 1))
|
||||
}
|
||||
|
||||
# The umask cases are only meaningful if a 0400 file cannot be reopened for
|
||||
# writing, which is not so for root.
|
||||
probe=$TMPDIR/umask-probe
|
||||
: >"$probe"
|
||||
chmod 0400 "$probe"
|
||||
if (: >"$probe") 2>/dev/null; then
|
||||
echo "a 0400 file is writable by this builder, so the UMask=0377 cases would prove nothing" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# setup FORGE|QUEUE <description>
|
||||
setup() {
|
||||
prefix=$1
|
||||
case="$prefix: $2"
|
||||
local unit_var=${prefix}_UNIT script_var=${prefix}_SCRIPT path_var=${prefix}_PATH addr_var=${prefix}_BAO_ADDR
|
||||
unit=${!unit_var}
|
||||
unit_path=${!path_var}
|
||||
bao_addr=${!addr_var}
|
||||
dir=$(mktemp -d)
|
||||
rt=$dir/rt
|
||||
creds=$dir/creds
|
||||
log=$dir/bao.log
|
||||
mkdir -m 0700 "$rt"
|
||||
mkdir "$creds"
|
||||
printf cert >"$creds/hive-agent-bao-cert"
|
||||
printf key >"$creds/hive-agent-bao-key"
|
||||
: >"$log"
|
||||
if ! sed "s|/run/$unit/|$rt/|g" "${!script_var}" >"$dir/script"; then
|
||||
echo "cannot read the rendered script for $unit" >&2
|
||||
exit 1
|
||||
fi
|
||||
chmod +x "$dir/script"
|
||||
if grep -q '/run/' "$dir/script"; then fail "the script still names /run after the rewrite"; fi
|
||||
if ! grep -qF "$rt/bao.err" "$dir/script"; then fail "the rewrite did not reach the script's error file"; fi
|
||||
|
||||
login_rc=0
|
||||
login_out=s.fake-token
|
||||
login_err=
|
||||
kv_rc=0
|
||||
kv_out=the-secret
|
||||
kv_err=
|
||||
}
|
||||
|
||||
go() {
|
||||
count=$((count + 1))
|
||||
(
|
||||
umask 0377
|
||||
cd "$dir" || exit 99
|
||||
exec env -i \
|
||||
PATH="$FAKE_BAO_BIN:$unit_path" \
|
||||
BAO_ADDR="$bao_addr" \
|
||||
CREDENTIALS_DIRECTORY="$creds" \
|
||||
FAKE_BAO_LOG="$log" \
|
||||
FAKE_BAO_LOGIN_RC="$login_rc" \
|
||||
FAKE_BAO_LOGIN_OUT="$login_out" \
|
||||
FAKE_BAO_LOGIN_ERR="$login_err" \
|
||||
FAKE_BAO_KV_RC="$kv_rc" \
|
||||
FAKE_BAO_KV_OUT="$kv_out" \
|
||||
FAKE_BAO_KV_ERR="$kv_err" \
|
||||
"$dir/script"
|
||||
) >"$dir/out" 2>"$dir/err"
|
||||
rc=$?
|
||||
}
|
||||
|
||||
expect_rc() {
|
||||
if [ "$rc" != "$1" ]; then fail "exit $rc, expected $1; stderr: $(<"$dir/err")"; fi
|
||||
}
|
||||
|
||||
expect_err() {
|
||||
if ! grep -qF -- "$1" "$dir/err"; then fail "stderr lacks '$1'; stderr: $(<"$dir/err")"; fi
|
||||
}
|
||||
|
||||
expect_no_err() {
|
||||
if grep -qF -- "$1" "$dir/err"; then fail "stderr has '$1'; stderr: $(<"$dir/err")"; fi
|
||||
}
|
||||
|
||||
expect_out() {
|
||||
if ! grep -qF -- "$1" "$dir/out"; then fail "stdout lacks '$1'; stdout: $(<"$dir/out")"; fi
|
||||
}
|
||||
|
||||
# expect_calls <line>... — the exact `bao` argument lines, in order.
|
||||
expect_calls() {
|
||||
local want
|
||||
want=$(printf '%s\n' "$@")
|
||||
if [ "$(<"$log")" != "${want%$'\n'}" ]; then fail "bao calls were '$(<"$log")', expected '$*'"; fi
|
||||
}
|
||||
|
||||
expect_file() {
|
||||
if [ ! -e "$1" ]; then
|
||||
fail "$1 missing"
|
||||
return
|
||||
fi
|
||||
if [ "$(<"$1")" != "$2" ]; then fail "$1 holds '$(<"$1")', expected '$2'"; fi
|
||||
}
|
||||
|
||||
expect_no_file() {
|
||||
if [ -e "$1" ]; then fail "$1 left behind"; fi
|
||||
}
|
||||
|
||||
for prefix in FORGE QUEUE; do
|
||||
if [ "$prefix" = FORGE ]; then
|
||||
secret_name=token
|
||||
path=secret/swarm/agents/a1/forge-token
|
||||
missing_msg="no forge token at $path yet"
|
||||
else
|
||||
secret_name=secret
|
||||
path=secret/swarm/agents/a1/queue
|
||||
missing_msg="no per-agent queue credential at $path yet"
|
||||
fi
|
||||
login_call="login -method=cert -token-only"
|
||||
kv_call="kv get -field=value $path"
|
||||
|
||||
setup "$prefix" "no store identity delivered"
|
||||
: >"$creds/hive-agent-bao-cert"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_err "has no store identity"
|
||||
expect_calls
|
||||
|
||||
setup "$prefix" "a credential that cannot be read"
|
||||
chmod 0000 "$creds/hive-agent-bao-key"
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "cannot read $creds/hive-agent-bao-key"
|
||||
expect_calls
|
||||
|
||||
setup "$prefix" "bao's stderr file cannot be created"
|
||||
chmod 0500 "$rt"
|
||||
go
|
||||
chmod 0700 "$rt"
|
||||
expect_rc 1
|
||||
expect_err "could not create $rt/bao.err, so bao never ran"
|
||||
expect_calls
|
||||
|
||||
setup "$prefix" "the store refuses the login with an HTTP 4xx"
|
||||
login_rc=2
|
||||
login_err=$'Error authenticating: Error making API request.\n\nURL: PUT '"$bao_addr"$'/v1/auth/cert/login\nCode: 403. Errors:\n\n* permission denied\n'
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "refused this agent's certificate login with HTTP 403:"
|
||||
expect_err "* permission denied"
|
||||
expect_calls "$login_call"
|
||||
|
||||
setup "$prefix" "the store fails the login with another HTTP status"
|
||||
login_rc=2
|
||||
login_err=$'Error authenticating: Error making API request.\n\nCode: 500. Errors:\n\n* internal error\n'
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "failed this agent's certificate login with HTTP 500:"
|
||||
expect_err "* internal error"
|
||||
expect_calls "$login_call"
|
||||
|
||||
setup "$prefix" "the store refuses the certificate with a TLS alert"
|
||||
login_rc=2
|
||||
login_err="Error authenticating: Put \"$bao_addr/v1/auth/cert/login\": remote error: tls: unknown certificate authority"
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "refused this agent's certificate in the TLS handshake:"
|
||||
expect_err "remote error: tls: unknown certificate authority"
|
||||
expect_calls "$login_call"
|
||||
|
||||
setup "$prefix" "the store does not answer"
|
||||
login_rc=2
|
||||
login_err="Error authenticating: Put \"$bao_addr/v1/auth/cert/login\": dial tcp: lookup bao.t.local: no such host"
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "got no answer from the swarm secret store at $bao_addr"
|
||||
expect_err "no such host"
|
||||
expect_calls "$login_call"
|
||||
|
||||
# Only the forge unit keeps its runtime directory between runs; the queue
|
||||
# unit starts each run with an empty one.
|
||||
setup "$prefix" "nothing minted at the agent's path yet"
|
||||
if [ "$prefix" = FORGE ]; then
|
||||
printf old >"$rt/$secret_name"
|
||||
chmod 0400 "$rt/$secret_name"
|
||||
fi
|
||||
kv_rc=2
|
||||
kv_err="No value found at secret/data/swarm/agents/a1"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_err "$missing_msg"
|
||||
expect_err "No value found"
|
||||
expect_calls "$login_call" "$kv_call"
|
||||
if [ "$prefix" = FORGE ]; then
|
||||
expect_file "$rt/$secret_name" old
|
||||
else
|
||||
expect_no_file "$rt/$secret_name"
|
||||
fi
|
||||
|
||||
setup "$prefix" "a first fetch writes the secret 0400"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "fetched this agent's"
|
||||
expect_no_err "Permission denied"
|
||||
expect_calls "$login_call" "$kv_call"
|
||||
expect_file "$rt/$secret_name" the-secret
|
||||
if [ "$(stat -c %a "$rt/$secret_name")" != 400 ]; then fail "$secret_name is mode $(stat -c %a "$rt/$secret_name"), expected 400"; fi
|
||||
expect_no_file "$rt/bao.err"
|
||||
expect_no_file "$rt/token.new"
|
||||
|
||||
# `UMask=0377` makes every file the script creates 0400, the login's own
|
||||
# `bao.err` included, and a redirect into a 0400 file fails before bao starts.
|
||||
setup "$prefix" "0400 files already in place do not block the fetch"
|
||||
printf stale >"$rt/bao.err"
|
||||
chmod 0400 "$rt/bao.err"
|
||||
if [ "$prefix" = FORGE ]; then
|
||||
printf stale >"$rt/token.new"
|
||||
chmod 0400 "$rt/token.new"
|
||||
fi
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "fetched this agent's"
|
||||
expect_no_err "Permission denied"
|
||||
expect_no_err "refused"
|
||||
expect_calls "$login_call" "$kv_call"
|
||||
expect_file "$rt/$secret_name" the-secret
|
||||
done
|
||||
|
||||
setup FORGE "an unchanged token is left in place"
|
||||
printf the-secret >"$rt/token"
|
||||
chmod 0400 "$rt/token"
|
||||
before=$(stat -c %i "$rt/token")
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "is unchanged"
|
||||
expect_file "$rt/token" the-secret
|
||||
if [ "$(stat -c %i "$rt/token")" != "$before" ]; then fail "the unchanged token was replaced"; fi
|
||||
expect_no_file "$rt/token.new"
|
||||
|
||||
setup FORGE "a rotated token replaces the old one"
|
||||
printf old >"$rt/token"
|
||||
chmod 0400 "$rt/token"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "fetched this agent's forge token"
|
||||
expect_file "$rt/token" the-secret
|
||||
expect_no_file "$rt/token.new"
|
||||
|
||||
if [ "$failures" -ne 0 ]; then
|
||||
echo "script-test-agent-bao-fetch: $failures failed assertions over $count runs" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "script-test-agent-bao-fetch: $count runs, all assertions hold"
|
||||
Loading…
Reference in a new issue