diff --git a/nix/checks.nix b/nix/checks.nix index 339d229b..5565e53b 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -120,6 +120,12 @@ in inherit pkgs self nixosSystem; inherit (pkgs) lib; }; + # Executes what the two `module-eval-agent-*-bao` suites above only + # evaluate: both fetch units' rendered scripts, against a stub `bao`. + script-test-agent-bao-fetch = import ./script-tests/agent-bao-fetch.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; module-eval-agent-memory = import ./module-eval/agent-memory.nix { inherit pkgs self nixosSystem; inherit (pkgs) lib; diff --git a/nix/script-tests/agent-bao-fetch.nix b/nix/script-tests/agent-bao-fetch.nix new file mode 100644 index 00000000..00156389 --- /dev/null +++ b/nix/script-tests/agent-bao-fetch.nix @@ -0,0 +1,82 @@ +# `checks.script-test-agent-bao-fetch` — runs the two agent units that log in +# to the swarm secret store and fetch a secret, ../agent-modules/forge-token.nix +# and ../agent-modules/queue-identity.nix, against a stub `bao`. The cases are +# in ./agent-bao-fetch.sh. +# +# What runs is each unit's rendered `ExecStart`, on the unit's own `PATH` with +# the stub in front. The one edit is the unit's `/run//` prefix, moved +# under the build directory because the sandbox has no writable `/run`. +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ../module-eval/lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + agentWith + ; + + machine = agentWith { services.hyperhive.agent.bao.addr = "https://bao.t.local:8200"; }; + + unitEnv = + prefix: name: + let + u = machine.systemd.services.${name}; + in + { + "${prefix}_UNIT" = name; + # `removeSuffix`, not `trim`: `trim` drops the string context, and with it + # the script's store path from this check's inputs. + "${prefix}_SCRIPT" = lib.removeSuffix " " u.serviceConfig.ExecStart; + "${prefix}_PATH" = u.environment.PATH; + "${prefix}_BAO_ADDR" = u.environment.BAO_ADDR; + }; + + # Answers `login` and `kv get` from `FAKE_BAO_*` variables and logs every + # call. A `kv` call without the token `login` printed fails, so a script that + # drops `BAO_TOKEN` between the two cannot pass. + fakeBao = pkgs.writeShellScriptBin "bao" '' + echo "$*" >> "$FAKE_BAO_LOG" + case "$1" in + login) + printf '%s' "$FAKE_BAO_LOGIN_ERR" >&2 + printf '%s' "$FAKE_BAO_LOGIN_OUT" + exit "$FAKE_BAO_LOGIN_RC" + ;; + kv) + if [ "''${BAO_TOKEN-}" != "$FAKE_BAO_LOGIN_OUT" ]; then + echo "fake bao: kv called without the login's token" >&2 + exit 97 + fi + printf '%s' "$FAKE_BAO_KV_ERR" >&2 + printf '%s' "$FAKE_BAO_KV_OUT" + exit "$FAKE_BAO_KV_RC" + ;; + *) + echo "fake bao: unexpected call: $*" >&2 + exit 98 + ;; + esac + ''; +in +pkgs.runCommand "hyperhive-script-test-agent-bao-fetch" + ( + unitEnv "FORGE" "hive-agent-forge-token" + // unitEnv "QUEUE" "hive-agent-queue-credential" + // { + FAKE_BAO_BIN = "${fakeBao}/bin"; + } + ) + '' + ${pkgs.bash}/bin/bash ${./agent-bao-fetch.sh} + touch "$out" + '' diff --git a/nix/script-tests/agent-bao-fetch.sh b/nix/script-tests/agent-bao-fetch.sh new file mode 100644 index 00000000..a69efc47 --- /dev/null +++ b/nix/script-tests/agent-bao-fetch.sh @@ -0,0 +1,258 @@ +# Cases for ./agent-bao-fetch.nix. Each case gets a fresh runtime directory and +# credentials directory, runs one unit's script under the unit's `UMask=0377` +# with a clean environment, and asserts on the exit code, the output, the +# files left behind and the `bao` calls made. +set -uo pipefail + +failures=0 +count=0 + +fail() { + echo "FAILED: $case: $*" >&2 + failures=$((failures + 1)) +} + +# The umask cases are only meaningful if a 0400 file cannot be reopened for +# writing, which is not so for root. +probe=$TMPDIR/umask-probe +: >"$probe" +chmod 0400 "$probe" +if (: >"$probe") 2>/dev/null; then + echo "a 0400 file is writable by this builder, so the UMask=0377 cases would prove nothing" >&2 + exit 1 +fi + +# setup FORGE|QUEUE +setup() { + prefix=$1 + case="$prefix: $2" + local unit_var=${prefix}_UNIT script_var=${prefix}_SCRIPT path_var=${prefix}_PATH addr_var=${prefix}_BAO_ADDR + unit=${!unit_var} + unit_path=${!path_var} + bao_addr=${!addr_var} + dir=$(mktemp -d) + rt=$dir/rt + creds=$dir/creds + log=$dir/bao.log + mkdir -m 0700 "$rt" + mkdir "$creds" + printf cert >"$creds/hive-agent-bao-cert" + printf key >"$creds/hive-agent-bao-key" + : >"$log" + if ! sed "s|/run/$unit/|$rt/|g" "${!script_var}" >"$dir/script"; then + echo "cannot read the rendered script for $unit" >&2 + exit 1 + fi + chmod +x "$dir/script" + if grep -q '/run/' "$dir/script"; then fail "the script still names /run after the rewrite"; fi + if ! grep -qF "$rt/bao.err" "$dir/script"; then fail "the rewrite did not reach the script's error file"; fi + + login_rc=0 + login_out=s.fake-token + login_err= + kv_rc=0 + kv_out=the-secret + kv_err= +} + +go() { + count=$((count + 1)) + ( + umask 0377 + cd "$dir" || exit 99 + exec env -i \ + PATH="$FAKE_BAO_BIN:$unit_path" \ + BAO_ADDR="$bao_addr" \ + CREDENTIALS_DIRECTORY="$creds" \ + FAKE_BAO_LOG="$log" \ + FAKE_BAO_LOGIN_RC="$login_rc" \ + FAKE_BAO_LOGIN_OUT="$login_out" \ + FAKE_BAO_LOGIN_ERR="$login_err" \ + FAKE_BAO_KV_RC="$kv_rc" \ + FAKE_BAO_KV_OUT="$kv_out" \ + FAKE_BAO_KV_ERR="$kv_err" \ + "$dir/script" + ) >"$dir/out" 2>"$dir/err" + rc=$? +} + +expect_rc() { + if [ "$rc" != "$1" ]; then fail "exit $rc, expected $1; stderr: $(<"$dir/err")"; fi +} + +expect_err() { + if ! grep -qF -- "$1" "$dir/err"; then fail "stderr lacks '$1'; stderr: $(<"$dir/err")"; fi +} + +expect_no_err() { + if grep -qF -- "$1" "$dir/err"; then fail "stderr has '$1'; stderr: $(<"$dir/err")"; fi +} + +expect_out() { + if ! grep -qF -- "$1" "$dir/out"; then fail "stdout lacks '$1'; stdout: $(<"$dir/out")"; fi +} + +# expect_calls ... — the exact `bao` argument lines, in order. +expect_calls() { + local want + want=$(printf '%s\n' "$@") + if [ "$(<"$log")" != "${want%$'\n'}" ]; then fail "bao calls were '$(<"$log")', expected '$*'"; fi +} + +expect_file() { + if [ ! -e "$1" ]; then + fail "$1 missing" + return + fi + if [ "$(<"$1")" != "$2" ]; then fail "$1 holds '$(<"$1")', expected '$2'"; fi +} + +expect_no_file() { + if [ -e "$1" ]; then fail "$1 left behind"; fi +} + +for prefix in FORGE QUEUE; do + if [ "$prefix" = FORGE ]; then + secret_name=token + path=secret/swarm/agents/a1/forge-token + missing_msg="no forge token at $path yet" + else + secret_name=secret + path=secret/swarm/agents/a1/queue + missing_msg="no per-agent queue credential at $path yet" + fi + login_call="login -method=cert -token-only" + kv_call="kv get -field=value $path" + + setup "$prefix" "no store identity delivered" + : >"$creds/hive-agent-bao-cert" + go + expect_rc 0 + expect_err "has no store identity" + expect_calls + + setup "$prefix" "a credential that cannot be read" + chmod 0000 "$creds/hive-agent-bao-key" + go + expect_rc 1 + expect_err "cannot read $creds/hive-agent-bao-key" + expect_calls + + setup "$prefix" "bao's stderr file cannot be created" + chmod 0500 "$rt" + go + chmod 0700 "$rt" + expect_rc 1 + expect_err "could not create $rt/bao.err, so bao never ran" + expect_calls + + setup "$prefix" "the store refuses the login with an HTTP 4xx" + login_rc=2 + login_err=$'Error authenticating: Error making API request.\n\nURL: PUT '"$bao_addr"$'/v1/auth/cert/login\nCode: 403. Errors:\n\n* permission denied\n' + go + expect_rc 1 + expect_err "refused this agent's certificate login with HTTP 403:" + expect_err "* permission denied" + expect_calls "$login_call" + + setup "$prefix" "the store fails the login with another HTTP status" + login_rc=2 + login_err=$'Error authenticating: Error making API request.\n\nCode: 500. Errors:\n\n* internal error\n' + go + expect_rc 1 + expect_err "failed this agent's certificate login with HTTP 500:" + expect_err "* internal error" + expect_calls "$login_call" + + setup "$prefix" "the store refuses the certificate with a TLS alert" + login_rc=2 + login_err="Error authenticating: Put \"$bao_addr/v1/auth/cert/login\": remote error: tls: unknown certificate authority" + go + expect_rc 1 + expect_err "refused this agent's certificate in the TLS handshake:" + expect_err "remote error: tls: unknown certificate authority" + expect_calls "$login_call" + + setup "$prefix" "the store does not answer" + login_rc=2 + login_err="Error authenticating: Put \"$bao_addr/v1/auth/cert/login\": dial tcp: lookup bao.t.local: no such host" + go + expect_rc 1 + expect_err "got no answer from the swarm secret store at $bao_addr" + expect_err "no such host" + expect_calls "$login_call" + + # Only the forge unit keeps its runtime directory between runs; the queue + # unit starts each run with an empty one. + setup "$prefix" "nothing minted at the agent's path yet" + if [ "$prefix" = FORGE ]; then + printf old >"$rt/$secret_name" + chmod 0400 "$rt/$secret_name" + fi + kv_rc=2 + kv_err="No value found at secret/data/swarm/agents/a1" + go + expect_rc 0 + expect_err "$missing_msg" + expect_err "No value found" + expect_calls "$login_call" "$kv_call" + if [ "$prefix" = FORGE ]; then + expect_file "$rt/$secret_name" old + else + expect_no_file "$rt/$secret_name" + fi + + setup "$prefix" "a first fetch writes the secret 0400" + go + expect_rc 0 + expect_out "fetched this agent's" + expect_no_err "Permission denied" + expect_calls "$login_call" "$kv_call" + expect_file "$rt/$secret_name" the-secret + if [ "$(stat -c %a "$rt/$secret_name")" != 400 ]; then fail "$secret_name is mode $(stat -c %a "$rt/$secret_name"), expected 400"; fi + expect_no_file "$rt/bao.err" + expect_no_file "$rt/token.new" + + # `UMask=0377` makes every file the script creates 0400, the login's own + # `bao.err` included, and a redirect into a 0400 file fails before bao starts. + setup "$prefix" "0400 files already in place do not block the fetch" + printf stale >"$rt/bao.err" + chmod 0400 "$rt/bao.err" + if [ "$prefix" = FORGE ]; then + printf stale >"$rt/token.new" + chmod 0400 "$rt/token.new" + fi + go + expect_rc 0 + expect_out "fetched this agent's" + expect_no_err "Permission denied" + expect_no_err "refused" + expect_calls "$login_call" "$kv_call" + expect_file "$rt/$secret_name" the-secret +done + +setup FORGE "an unchanged token is left in place" +printf the-secret >"$rt/token" +chmod 0400 "$rt/token" +before=$(stat -c %i "$rt/token") +go +expect_rc 0 +expect_out "is unchanged" +expect_file "$rt/token" the-secret +if [ "$(stat -c %i "$rt/token")" != "$before" ]; then fail "the unchanged token was replaced"; fi +expect_no_file "$rt/token.new" + +setup FORGE "a rotated token replaces the old one" +printf old >"$rt/token" +chmod 0400 "$rt/token" +go +expect_rc 0 +expect_out "fetched this agent's forge token" +expect_file "$rt/token" the-secret +expect_no_file "$rt/token.new" + +if [ "$failures" -ne 0 ]; then + echo "script-test-agent-bao-fetch: $failures failed assertions over $count runs" >&2 + exit 1 +fi +echo "script-test-agent-bao-fetch: $count runs, all assertions hold"