checks: run the agent bao-fetch unit scripts against a stub bao
`script-test-agent-bao-fetch` executes the rendered `ExecStart` of `hive-agent-forge-token` and `hive-agent-queue-credential` under `umask 0377`, with a stub `bao` first on the unit's own PATH. It covers every error branch, the happy path, forge rotation/unchanged, and 0400 files already in place: the redirect failure #4736 fixed, whose live symptom was `bao.err: Permission denied` reported as a refused certificate. The TLS-alert fixture is the `unknown certificate authority` error h-atlas's identity check got from the store. #4736 claimed this test but never committed it; the two module-eval suites for these units only evaluate the config. Closes #4748
This commit is contained in:
atlas
mara
parent
916c441e82
commit
202f7f7c83
3 changed files with 346 additions and 0 deletions
|
|
@ -120,6 +120,12 @@ in
|
||||||
inherit pkgs self nixosSystem;
|
inherit pkgs self nixosSystem;
|
||||||
inherit (pkgs) lib;
|
inherit (pkgs) lib;
|
||||||
};
|
};
|
||||||
|
# Executes what the two `module-eval-agent-*-bao` suites above only
|
||||||
|
# evaluate: both fetch units' rendered scripts, against a stub `bao`.
|
||||||
|
script-test-agent-bao-fetch = import ./script-tests/agent-bao-fetch.nix {
|
||||||
|
inherit pkgs self nixosSystem;
|
||||||
|
inherit (pkgs) lib;
|
||||||
|
};
|
||||||
module-eval-agent-memory = import ./module-eval/agent-memory.nix {
|
module-eval-agent-memory = import ./module-eval/agent-memory.nix {
|
||||||
inherit pkgs self nixosSystem;
|
inherit pkgs self nixosSystem;
|
||||||
inherit (pkgs) lib;
|
inherit (pkgs) lib;
|
||||||
|
|
|
||||||
82
nix/script-tests/agent-bao-fetch.nix
Normal file
82
nix/script-tests/agent-bao-fetch.nix
Normal file
|
|
@ -0,0 +1,82 @@
|
||||||
|
# `checks.script-test-agent-bao-fetch` — runs the two agent units that log in
|
||||||
|
# to the swarm secret store and fetch a secret, ../agent-modules/forge-token.nix
|
||||||
|
# and ../agent-modules/queue-identity.nix, against a stub `bao`. The cases are
|
||||||
|
# in ./agent-bao-fetch.sh.
|
||||||
|
#
|
||||||
|
# What runs is each unit's rendered `ExecStart`, on the unit's own `PATH` with
|
||||||
|
# the stub in front. The one edit is the unit's `/run/<unit>/` prefix, moved
|
||||||
|
# under the build directory because the sandbox has no writable `/run`.
|
||||||
|
{
|
||||||
|
pkgs,
|
||||||
|
lib,
|
||||||
|
self,
|
||||||
|
nixosSystem,
|
||||||
|
}:
|
||||||
|
let
|
||||||
|
inherit
|
||||||
|
(import ../module-eval/lib.nix {
|
||||||
|
inherit
|
||||||
|
pkgs
|
||||||
|
lib
|
||||||
|
self
|
||||||
|
nixosSystem
|
||||||
|
;
|
||||||
|
})
|
||||||
|
agentWith
|
||||||
|
;
|
||||||
|
|
||||||
|
machine = agentWith { services.hyperhive.agent.bao.addr = "https://bao.t.local:8200"; };
|
||||||
|
|
||||||
|
unitEnv =
|
||||||
|
prefix: name:
|
||||||
|
let
|
||||||
|
u = machine.systemd.services.${name};
|
||||||
|
in
|
||||||
|
{
|
||||||
|
"${prefix}_UNIT" = name;
|
||||||
|
# `removeSuffix`, not `trim`: `trim` drops the string context, and with it
|
||||||
|
# the script's store path from this check's inputs.
|
||||||
|
"${prefix}_SCRIPT" = lib.removeSuffix " " u.serviceConfig.ExecStart;
|
||||||
|
"${prefix}_PATH" = u.environment.PATH;
|
||||||
|
"${prefix}_BAO_ADDR" = u.environment.BAO_ADDR;
|
||||||
|
};
|
||||||
|
|
||||||
|
# Answers `login` and `kv get` from `FAKE_BAO_*` variables and logs every
|
||||||
|
# call. A `kv` call without the token `login` printed fails, so a script that
|
||||||
|
# drops `BAO_TOKEN` between the two cannot pass.
|
||||||
|
fakeBao = pkgs.writeShellScriptBin "bao" ''
|
||||||
|
echo "$*" >> "$FAKE_BAO_LOG"
|
||||||
|
case "$1" in
|
||||||
|
login)
|
||||||
|
printf '%s' "$FAKE_BAO_LOGIN_ERR" >&2
|
||||||
|
printf '%s' "$FAKE_BAO_LOGIN_OUT"
|
||||||
|
exit "$FAKE_BAO_LOGIN_RC"
|
||||||
|
;;
|
||||||
|
kv)
|
||||||
|
if [ "''${BAO_TOKEN-}" != "$FAKE_BAO_LOGIN_OUT" ]; then
|
||||||
|
echo "fake bao: kv called without the login's token" >&2
|
||||||
|
exit 97
|
||||||
|
fi
|
||||||
|
printf '%s' "$FAKE_BAO_KV_ERR" >&2
|
||||||
|
printf '%s' "$FAKE_BAO_KV_OUT"
|
||||||
|
exit "$FAKE_BAO_KV_RC"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "fake bao: unexpected call: $*" >&2
|
||||||
|
exit 98
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
'';
|
||||||
|
in
|
||||||
|
pkgs.runCommand "hyperhive-script-test-agent-bao-fetch"
|
||||||
|
(
|
||||||
|
unitEnv "FORGE" "hive-agent-forge-token"
|
||||||
|
// unitEnv "QUEUE" "hive-agent-queue-credential"
|
||||||
|
// {
|
||||||
|
FAKE_BAO_BIN = "${fakeBao}/bin";
|
||||||
|
}
|
||||||
|
)
|
||||||
|
''
|
||||||
|
${pkgs.bash}/bin/bash ${./agent-bao-fetch.sh}
|
||||||
|
touch "$out"
|
||||||
|
''
|
||||||
258
nix/script-tests/agent-bao-fetch.sh
Normal file
258
nix/script-tests/agent-bao-fetch.sh
Normal file
|
|
@ -0,0 +1,258 @@
|
||||||
|
# Cases for ./agent-bao-fetch.nix. Each case gets a fresh runtime directory and
|
||||||
|
# credentials directory, runs one unit's script under the unit's `UMask=0377`
|
||||||
|
# with a clean environment, and asserts on the exit code, the output, the
|
||||||
|
# files left behind and the `bao` calls made.
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
failures=0
|
||||||
|
count=0
|
||||||
|
|
||||||
|
fail() {
|
||||||
|
echo "FAILED: $case: $*" >&2
|
||||||
|
failures=$((failures + 1))
|
||||||
|
}
|
||||||
|
|
||||||
|
# The umask cases are only meaningful if a 0400 file cannot be reopened for
|
||||||
|
# writing, which is not so for root.
|
||||||
|
probe=$TMPDIR/umask-probe
|
||||||
|
: >"$probe"
|
||||||
|
chmod 0400 "$probe"
|
||||||
|
if (: >"$probe") 2>/dev/null; then
|
||||||
|
echo "a 0400 file is writable by this builder, so the UMask=0377 cases would prove nothing" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# setup FORGE|QUEUE <description>
|
||||||
|
setup() {
|
||||||
|
prefix=$1
|
||||||
|
case="$prefix: $2"
|
||||||
|
local unit_var=${prefix}_UNIT script_var=${prefix}_SCRIPT path_var=${prefix}_PATH addr_var=${prefix}_BAO_ADDR
|
||||||
|
unit=${!unit_var}
|
||||||
|
unit_path=${!path_var}
|
||||||
|
bao_addr=${!addr_var}
|
||||||
|
dir=$(mktemp -d)
|
||||||
|
rt=$dir/rt
|
||||||
|
creds=$dir/creds
|
||||||
|
log=$dir/bao.log
|
||||||
|
mkdir -m 0700 "$rt"
|
||||||
|
mkdir "$creds"
|
||||||
|
printf cert >"$creds/hive-agent-bao-cert"
|
||||||
|
printf key >"$creds/hive-agent-bao-key"
|
||||||
|
: >"$log"
|
||||||
|
if ! sed "s|/run/$unit/|$rt/|g" "${!script_var}" >"$dir/script"; then
|
||||||
|
echo "cannot read the rendered script for $unit" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
chmod +x "$dir/script"
|
||||||
|
if grep -q '/run/' "$dir/script"; then fail "the script still names /run after the rewrite"; fi
|
||||||
|
if ! grep -qF "$rt/bao.err" "$dir/script"; then fail "the rewrite did not reach the script's error file"; fi
|
||||||
|
|
||||||
|
login_rc=0
|
||||||
|
login_out=s.fake-token
|
||||||
|
login_err=
|
||||||
|
kv_rc=0
|
||||||
|
kv_out=the-secret
|
||||||
|
kv_err=
|
||||||
|
}
|
||||||
|
|
||||||
|
go() {
|
||||||
|
count=$((count + 1))
|
||||||
|
(
|
||||||
|
umask 0377
|
||||||
|
cd "$dir" || exit 99
|
||||||
|
exec env -i \
|
||||||
|
PATH="$FAKE_BAO_BIN:$unit_path" \
|
||||||
|
BAO_ADDR="$bao_addr" \
|
||||||
|
CREDENTIALS_DIRECTORY="$creds" \
|
||||||
|
FAKE_BAO_LOG="$log" \
|
||||||
|
FAKE_BAO_LOGIN_RC="$login_rc" \
|
||||||
|
FAKE_BAO_LOGIN_OUT="$login_out" \
|
||||||
|
FAKE_BAO_LOGIN_ERR="$login_err" \
|
||||||
|
FAKE_BAO_KV_RC="$kv_rc" \
|
||||||
|
FAKE_BAO_KV_OUT="$kv_out" \
|
||||||
|
FAKE_BAO_KV_ERR="$kv_err" \
|
||||||
|
"$dir/script"
|
||||||
|
) >"$dir/out" 2>"$dir/err"
|
||||||
|
rc=$?
|
||||||
|
}
|
||||||
|
|
||||||
|
expect_rc() {
|
||||||
|
if [ "$rc" != "$1" ]; then fail "exit $rc, expected $1; stderr: $(<"$dir/err")"; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
expect_err() {
|
||||||
|
if ! grep -qF -- "$1" "$dir/err"; then fail "stderr lacks '$1'; stderr: $(<"$dir/err")"; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
expect_no_err() {
|
||||||
|
if grep -qF -- "$1" "$dir/err"; then fail "stderr has '$1'; stderr: $(<"$dir/err")"; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
expect_out() {
|
||||||
|
if ! grep -qF -- "$1" "$dir/out"; then fail "stdout lacks '$1'; stdout: $(<"$dir/out")"; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# expect_calls <line>... — the exact `bao` argument lines, in order.
|
||||||
|
expect_calls() {
|
||||||
|
local want
|
||||||
|
want=$(printf '%s\n' "$@")
|
||||||
|
if [ "$(<"$log")" != "${want%$'\n'}" ]; then fail "bao calls were '$(<"$log")', expected '$*'"; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
expect_file() {
|
||||||
|
if [ ! -e "$1" ]; then
|
||||||
|
fail "$1 missing"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
if [ "$(<"$1")" != "$2" ]; then fail "$1 holds '$(<"$1")', expected '$2'"; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
expect_no_file() {
|
||||||
|
if [ -e "$1" ]; then fail "$1 left behind"; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
for prefix in FORGE QUEUE; do
|
||||||
|
if [ "$prefix" = FORGE ]; then
|
||||||
|
secret_name=token
|
||||||
|
path=secret/swarm/agents/a1/forge-token
|
||||||
|
missing_msg="no forge token at $path yet"
|
||||||
|
else
|
||||||
|
secret_name=secret
|
||||||
|
path=secret/swarm/agents/a1/queue
|
||||||
|
missing_msg="no per-agent queue credential at $path yet"
|
||||||
|
fi
|
||||||
|
login_call="login -method=cert -token-only"
|
||||||
|
kv_call="kv get -field=value $path"
|
||||||
|
|
||||||
|
setup "$prefix" "no store identity delivered"
|
||||||
|
: >"$creds/hive-agent-bao-cert"
|
||||||
|
go
|
||||||
|
expect_rc 0
|
||||||
|
expect_err "has no store identity"
|
||||||
|
expect_calls
|
||||||
|
|
||||||
|
setup "$prefix" "a credential that cannot be read"
|
||||||
|
chmod 0000 "$creds/hive-agent-bao-key"
|
||||||
|
go
|
||||||
|
expect_rc 1
|
||||||
|
expect_err "cannot read $creds/hive-agent-bao-key"
|
||||||
|
expect_calls
|
||||||
|
|
||||||
|
setup "$prefix" "bao's stderr file cannot be created"
|
||||||
|
chmod 0500 "$rt"
|
||||||
|
go
|
||||||
|
chmod 0700 "$rt"
|
||||||
|
expect_rc 1
|
||||||
|
expect_err "could not create $rt/bao.err, so bao never ran"
|
||||||
|
expect_calls
|
||||||
|
|
||||||
|
setup "$prefix" "the store refuses the login with an HTTP 4xx"
|
||||||
|
login_rc=2
|
||||||
|
login_err=$'Error authenticating: Error making API request.\n\nURL: PUT '"$bao_addr"$'/v1/auth/cert/login\nCode: 403. Errors:\n\n* permission denied\n'
|
||||||
|
go
|
||||||
|
expect_rc 1
|
||||||
|
expect_err "refused this agent's certificate login with HTTP 403:"
|
||||||
|
expect_err "* permission denied"
|
||||||
|
expect_calls "$login_call"
|
||||||
|
|
||||||
|
setup "$prefix" "the store fails the login with another HTTP status"
|
||||||
|
login_rc=2
|
||||||
|
login_err=$'Error authenticating: Error making API request.\n\nCode: 500. Errors:\n\n* internal error\n'
|
||||||
|
go
|
||||||
|
expect_rc 1
|
||||||
|
expect_err "failed this agent's certificate login with HTTP 500:"
|
||||||
|
expect_err "* internal error"
|
||||||
|
expect_calls "$login_call"
|
||||||
|
|
||||||
|
setup "$prefix" "the store refuses the certificate with a TLS alert"
|
||||||
|
login_rc=2
|
||||||
|
login_err="Error authenticating: Put \"$bao_addr/v1/auth/cert/login\": remote error: tls: unknown certificate authority"
|
||||||
|
go
|
||||||
|
expect_rc 1
|
||||||
|
expect_err "refused this agent's certificate in the TLS handshake:"
|
||||||
|
expect_err "remote error: tls: unknown certificate authority"
|
||||||
|
expect_calls "$login_call"
|
||||||
|
|
||||||
|
setup "$prefix" "the store does not answer"
|
||||||
|
login_rc=2
|
||||||
|
login_err="Error authenticating: Put \"$bao_addr/v1/auth/cert/login\": dial tcp: lookup bao.t.local: no such host"
|
||||||
|
go
|
||||||
|
expect_rc 1
|
||||||
|
expect_err "got no answer from the swarm secret store at $bao_addr"
|
||||||
|
expect_err "no such host"
|
||||||
|
expect_calls "$login_call"
|
||||||
|
|
||||||
|
# Only the forge unit keeps its runtime directory between runs; the queue
|
||||||
|
# unit starts each run with an empty one.
|
||||||
|
setup "$prefix" "nothing minted at the agent's path yet"
|
||||||
|
if [ "$prefix" = FORGE ]; then
|
||||||
|
printf old >"$rt/$secret_name"
|
||||||
|
chmod 0400 "$rt/$secret_name"
|
||||||
|
fi
|
||||||
|
kv_rc=2
|
||||||
|
kv_err="No value found at secret/data/swarm/agents/a1"
|
||||||
|
go
|
||||||
|
expect_rc 0
|
||||||
|
expect_err "$missing_msg"
|
||||||
|
expect_err "No value found"
|
||||||
|
expect_calls "$login_call" "$kv_call"
|
||||||
|
if [ "$prefix" = FORGE ]; then
|
||||||
|
expect_file "$rt/$secret_name" old
|
||||||
|
else
|
||||||
|
expect_no_file "$rt/$secret_name"
|
||||||
|
fi
|
||||||
|
|
||||||
|
setup "$prefix" "a first fetch writes the secret 0400"
|
||||||
|
go
|
||||||
|
expect_rc 0
|
||||||
|
expect_out "fetched this agent's"
|
||||||
|
expect_no_err "Permission denied"
|
||||||
|
expect_calls "$login_call" "$kv_call"
|
||||||
|
expect_file "$rt/$secret_name" the-secret
|
||||||
|
if [ "$(stat -c %a "$rt/$secret_name")" != 400 ]; then fail "$secret_name is mode $(stat -c %a "$rt/$secret_name"), expected 400"; fi
|
||||||
|
expect_no_file "$rt/bao.err"
|
||||||
|
expect_no_file "$rt/token.new"
|
||||||
|
|
||||||
|
# `UMask=0377` makes every file the script creates 0400, the login's own
|
||||||
|
# `bao.err` included, and a redirect into a 0400 file fails before bao starts.
|
||||||
|
setup "$prefix" "0400 files already in place do not block the fetch"
|
||||||
|
printf stale >"$rt/bao.err"
|
||||||
|
chmod 0400 "$rt/bao.err"
|
||||||
|
if [ "$prefix" = FORGE ]; then
|
||||||
|
printf stale >"$rt/token.new"
|
||||||
|
chmod 0400 "$rt/token.new"
|
||||||
|
fi
|
||||||
|
go
|
||||||
|
expect_rc 0
|
||||||
|
expect_out "fetched this agent's"
|
||||||
|
expect_no_err "Permission denied"
|
||||||
|
expect_no_err "refused"
|
||||||
|
expect_calls "$login_call" "$kv_call"
|
||||||
|
expect_file "$rt/$secret_name" the-secret
|
||||||
|
done
|
||||||
|
|
||||||
|
setup FORGE "an unchanged token is left in place"
|
||||||
|
printf the-secret >"$rt/token"
|
||||||
|
chmod 0400 "$rt/token"
|
||||||
|
before=$(stat -c %i "$rt/token")
|
||||||
|
go
|
||||||
|
expect_rc 0
|
||||||
|
expect_out "is unchanged"
|
||||||
|
expect_file "$rt/token" the-secret
|
||||||
|
if [ "$(stat -c %i "$rt/token")" != "$before" ]; then fail "the unchanged token was replaced"; fi
|
||||||
|
expect_no_file "$rt/token.new"
|
||||||
|
|
||||||
|
setup FORGE "a rotated token replaces the old one"
|
||||||
|
printf old >"$rt/token"
|
||||||
|
chmod 0400 "$rt/token"
|
||||||
|
go
|
||||||
|
expect_rc 0
|
||||||
|
expect_out "fetched this agent's forge token"
|
||||||
|
expect_file "$rt/token" the-secret
|
||||||
|
expect_no_file "$rt/token.new"
|
||||||
|
|
||||||
|
if [ "$failures" -ne 0 ]; then
|
||||||
|
echo "script-test-agent-bao-fetch: $failures failed assertions over $count runs" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "script-test-agent-bao-fetch: $count runs, all assertions hold"
|
||||||
Loading…
Reference in a new issue