c6717bccb0
swarm-controller re-issues an agent's store leaf at half its validity, but the running container kept presenting the copy its hive delivered at start, so a re-issued leaf reached the agent only on its next restart. hive-agent-bao-identity now seeds a live copy of the pair in its own runtime directory from the delivered credentials, and re-runs every 10 minutes: it logs in with the live leaf, reads its own swarm/agents/<agent>/bao-mtls (a read render_agent already grants), and when the leaf changed, logs in with the new pair and swaps it in only if the store accepts it. Every unit in the container that logs in to the store presents the live pair: hive-agent (queue-identity.nix), hive-matrix-daemon, hive-subagent-daemon, hive-agent-forge-token, hive-agent-forge-accounts and hive-agent-github-token. They keep loading the server CA as a credential. The long-running daemons are ordered after the refresher so the live pair exists when they start. It renews only while the held leaf is still valid: the login that reads the new leaf is made with the current one. Refs #5017
314 lines
9.1 KiB
Shell
314 lines
9.1 KiB
Shell
# Cases for ./agent-bao-fetch.nix. Each case gets a fresh runtime directory and
|
|
# credentials directory, runs one unit's script under the unit's `UMask=0377`
|
|
# with a clean environment, and asserts on the exit code, the output, the
|
|
# files left behind and the `bao` calls made.
|
|
set -uo pipefail
|
|
|
|
failures=0
|
|
count=0
|
|
|
|
fail() {
|
|
echo "FAILED: $case: $*" >&2
|
|
failures=$((failures + 1))
|
|
}
|
|
|
|
# The umask cases are only meaningful if a 0400 file cannot be reopened for
|
|
# writing, which is not so for root.
|
|
probe=$TMPDIR/umask-probe
|
|
: >"$probe"
|
|
chmod 0400 "$probe"
|
|
if (: >"$probe") 2>/dev/null; then
|
|
echo "a 0400 file is writable by this builder, so the UMask=0377 cases would prove nothing" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# setup FORGE <description>
|
|
setup() {
|
|
prefix=$1
|
|
case="$prefix: $2"
|
|
local unit_var=${prefix}_UNIT script_var=${prefix}_SCRIPT path_var=${prefix}_PATH addr_var=${prefix}_BAO_ADDR
|
|
unit=${!unit_var}
|
|
unit_path=${!path_var}
|
|
bao_addr=${!addr_var}
|
|
dir=$(mktemp -d)
|
|
rt=$dir/rt
|
|
creds=$dir/creds
|
|
log=$dir/bao.log
|
|
mkdir -m 0700 "$rt"
|
|
mkdir "$creds"
|
|
printf cert >"$creds/hive-agent-bao-cert"
|
|
printf key >"$creds/hive-agent-bao-key"
|
|
# The identity pair the unit presents: the delivered one for the fetch, the
|
|
# live copy it keeps for the identity unit.
|
|
if [ "$prefix" = IDENT ]; then
|
|
client_cert=$rt/cert client_key=$rt/key
|
|
else
|
|
client_cert=$creds/hive-agent-bao-cert client_key=$creds/hive-agent-bao-key
|
|
fi
|
|
: >"$log"
|
|
if ! sed "s|/run/$unit/|$rt/|g" "${!script_var}" >"$dir/script"; then
|
|
echo "cannot read the rendered script for $unit" >&2
|
|
exit 1
|
|
fi
|
|
chmod +x "$dir/script"
|
|
if grep -q '/run/' "$dir/script"; then fail "the script still names /run after the rewrite"; fi
|
|
if ! grep -qF "$rt/bao.err" "$dir/script"; then fail "the rewrite did not reach the script's error file"; fi
|
|
|
|
login_rc=0
|
|
login_out=s.fake-token
|
|
login_err=
|
|
kv_rc=0
|
|
kv_out=the-secret
|
|
kv_err=
|
|
refuse_cert=
|
|
}
|
|
|
|
go() {
|
|
count=$((count + 1))
|
|
(
|
|
umask 0377
|
|
cd "$dir" || exit 99
|
|
exec env -i \
|
|
PATH="$FAKE_BAO_BIN:$unit_path" \
|
|
BAO_ADDR="$bao_addr" \
|
|
BAO_CLIENT_CERT="$client_cert" \
|
|
BAO_CLIENT_KEY="$client_key" \
|
|
CREDENTIALS_DIRECTORY="$creds" \
|
|
FAKE_BAO_LOG="$log" \
|
|
FAKE_BAO_LOGIN_RC="$login_rc" \
|
|
FAKE_BAO_LOGIN_OUT="$login_out" \
|
|
FAKE_BAO_LOGIN_ERR="$login_err" \
|
|
FAKE_BAO_KV_RC="$kv_rc" \
|
|
FAKE_BAO_KV_OUT="$kv_out" \
|
|
FAKE_BAO_KV_ERR="$kv_err" \
|
|
FAKE_BAO_REFUSE_CERT="$refuse_cert" \
|
|
"$dir/script"
|
|
) >"$dir/out" 2>"$dir/err"
|
|
rc=$?
|
|
}
|
|
|
|
expect_rc() {
|
|
if [ "$rc" != "$1" ]; then fail "exit $rc, expected $1; stderr: $(<"$dir/err")"; fi
|
|
}
|
|
|
|
expect_err() {
|
|
if ! grep -qF -- "$1" "$dir/err"; then fail "stderr lacks '$1'; stderr: $(<"$dir/err")"; fi
|
|
}
|
|
|
|
expect_no_err() {
|
|
if grep -qF -- "$1" "$dir/err"; then fail "stderr has '$1'; stderr: $(<"$dir/err")"; fi
|
|
}
|
|
|
|
expect_out() {
|
|
if ! grep -qF -- "$1" "$dir/out"; then fail "stdout lacks '$1'; stdout: $(<"$dir/out")"; fi
|
|
}
|
|
|
|
# expect_calls <line>... — the exact `bao` argument lines, in order.
|
|
expect_calls() {
|
|
local want
|
|
want=$(printf '%s\n' "$@")
|
|
if [ "$(<"$log")" != "${want%$'\n'}" ]; then fail "bao calls were '$(<"$log")', expected '$*'"; fi
|
|
}
|
|
|
|
expect_file() {
|
|
if [ ! -e "$1" ]; then
|
|
fail "$1 missing"
|
|
return
|
|
fi
|
|
if [ "$(<"$1")" != "$2" ]; then fail "$1 holds '$(<"$1")', expected '$2'"; fi
|
|
}
|
|
|
|
expect_no_file() {
|
|
if [ -e "$1" ]; then fail "$1 left behind"; fi
|
|
}
|
|
|
|
secret_name=token
|
|
path=secret/swarm/agents/a1/forge-token
|
|
missing_msg="no forge token at $path yet"
|
|
login_call="login -method=cert -token-only"
|
|
kv_call="kv get -field=value $path"
|
|
|
|
setup FORGE "no store identity delivered"
|
|
: >"$creds/hive-agent-bao-cert"
|
|
go
|
|
expect_rc 0
|
|
expect_err "has no store identity"
|
|
expect_calls
|
|
|
|
setup FORGE "a credential that cannot be read"
|
|
chmod 0000 "$creds/hive-agent-bao-key"
|
|
go
|
|
expect_rc 1
|
|
expect_err "cannot read $creds/hive-agent-bao-key"
|
|
expect_calls
|
|
|
|
setup FORGE "bao's stderr file cannot be created"
|
|
chmod 0500 "$rt"
|
|
go
|
|
chmod 0700 "$rt"
|
|
expect_rc 1
|
|
expect_err "could not create $rt/bao.err, so bao never ran"
|
|
expect_calls
|
|
|
|
setup FORGE "the store refuses the login with an HTTP 4xx"
|
|
login_rc=2
|
|
login_err=$'Error authenticating: Error making API request.\n\nURL: PUT '"$bao_addr"$'/v1/auth/cert/login\nCode: 403. Errors:\n\n* permission denied\n'
|
|
go
|
|
expect_rc 1
|
|
expect_err "refused this agent's certificate login with HTTP 403:"
|
|
expect_err "* permission denied"
|
|
expect_calls "$login_call"
|
|
|
|
setup FORGE "the store fails the login with another HTTP status"
|
|
login_rc=2
|
|
login_err=$'Error authenticating: Error making API request.\n\nCode: 500. Errors:\n\n* internal error\n'
|
|
go
|
|
expect_rc 1
|
|
expect_err "failed this agent's certificate login with HTTP 500:"
|
|
expect_err "* internal error"
|
|
expect_calls "$login_call"
|
|
|
|
setup FORGE "the store refuses the certificate with a TLS alert"
|
|
login_rc=2
|
|
login_err="Error authenticating: Put \"$bao_addr/v1/auth/cert/login\": remote error: tls: unknown certificate authority"
|
|
go
|
|
expect_rc 1
|
|
expect_err "refused this agent's certificate in the TLS handshake:"
|
|
expect_err "remote error: tls: unknown certificate authority"
|
|
expect_calls "$login_call"
|
|
|
|
setup FORGE "the store does not answer"
|
|
login_rc=2
|
|
login_err="Error authenticating: Put \"$bao_addr/v1/auth/cert/login\": dial tcp: lookup bao.t.local: no such host"
|
|
go
|
|
expect_rc 1
|
|
expect_err "got no answer from the swarm secret store at $bao_addr"
|
|
expect_err "no such host"
|
|
expect_calls "$login_call"
|
|
|
|
# The unit keeps its runtime directory between runs.
|
|
setup FORGE "nothing minted at the agent's path yet"
|
|
printf old >"$rt/$secret_name"
|
|
chmod 0400 "$rt/$secret_name"
|
|
kv_rc=2
|
|
kv_err="No value found at secret/data/swarm/agents/a1"
|
|
go
|
|
expect_rc 0
|
|
expect_err "$missing_msg"
|
|
expect_err "No value found"
|
|
expect_calls "$login_call" "$kv_call"
|
|
expect_file "$rt/$secret_name" old
|
|
|
|
setup FORGE "a first fetch writes the secret 0400"
|
|
go
|
|
expect_rc 0
|
|
expect_out "fetched this agent's"
|
|
expect_no_err "Permission denied"
|
|
expect_calls "$login_call" "$kv_call"
|
|
expect_file "$rt/$secret_name" the-secret
|
|
if [ "$(stat -c %a "$rt/$secret_name")" != 400 ]; then fail "$secret_name is mode $(stat -c %a "$rt/$secret_name"), expected 400"; fi
|
|
expect_no_file "$rt/bao.err"
|
|
expect_no_file "$rt/token.new"
|
|
|
|
# `UMask=0377` makes every file the script creates 0400, the login's own
|
|
# `bao.err` included, and a redirect into a 0400 file fails before bao starts.
|
|
setup FORGE "0400 files already in place do not block the fetch"
|
|
printf stale >"$rt/bao.err"
|
|
chmod 0400 "$rt/bao.err"
|
|
printf stale >"$rt/token.new"
|
|
chmod 0400 "$rt/token.new"
|
|
go
|
|
expect_rc 0
|
|
expect_out "fetched this agent's"
|
|
expect_no_err "Permission denied"
|
|
expect_no_err "refused"
|
|
expect_calls "$login_call" "$kv_call"
|
|
expect_file "$rt/$secret_name" the-secret
|
|
|
|
setup FORGE "an unchanged token is left in place"
|
|
printf the-secret >"$rt/token"
|
|
chmod 0400 "$rt/token"
|
|
before=$(stat -c %i "$rt/token")
|
|
go
|
|
expect_rc 0
|
|
expect_out "is unchanged"
|
|
expect_file "$rt/token" the-secret
|
|
if [ "$(stat -c %i "$rt/token")" != "$before" ]; then fail "the unchanged token was replaced"; fi
|
|
expect_no_file "$rt/token.new"
|
|
|
|
setup FORGE "a rotated token replaces the old one"
|
|
printf old >"$rt/token"
|
|
chmod 0400 "$rt/token"
|
|
go
|
|
expect_rc 0
|
|
expect_out "fetched this agent's forge token"
|
|
expect_file "$rt/token" the-secret
|
|
expect_no_file "$rt/token.new"
|
|
|
|
ident_path=secret/swarm/agents/a1/bao-mtls
|
|
ident_kv_call="kv get -format=json $ident_path"
|
|
leaf() { printf '{"data":{"data":{"cert":"%s","key":"%s"}}}' "$1" "$2"; }
|
|
|
|
setup IDENT "no store identity delivered"
|
|
: >"$creds/hive-agent-bao-cert"
|
|
go
|
|
expect_rc 1
|
|
expect_err "has no store identity"
|
|
expect_calls
|
|
expect_no_file "$rt/cert"
|
|
|
|
setup IDENT "a first run seeds the live pair from the delivered one"
|
|
kv_out=$(leaf cert key)
|
|
go
|
|
expect_rc 0
|
|
expect_out "is unchanged"
|
|
expect_calls "$login_call" "$ident_kv_call"
|
|
expect_file "$rt/cert" cert
|
|
expect_file "$rt/key" key
|
|
if [ "$(stat -c %a "$rt/key")" != 400 ]; then fail "key is mode $(stat -c %a "$rt/key"), expected 400"; fi
|
|
expect_no_file "$rt/bao-mtls.json"
|
|
|
|
setup IDENT "the live pair outlives the delivered one"
|
|
printf cert2 >"$rt/cert"
|
|
printf key2 >"$rt/key"
|
|
chmod 0400 "$rt/cert" "$rt/key"
|
|
kv_out=$(leaf cert2 key2)
|
|
go
|
|
expect_rc 0
|
|
expect_out "is unchanged"
|
|
expect_file "$rt/cert" cert2
|
|
|
|
setup IDENT "a re-issued leaf replaces the live pair once a login with it succeeds"
|
|
kv_out=$(leaf cert2 key2)
|
|
go
|
|
expect_rc 0
|
|
expect_out "replaced this agent's store certificate"
|
|
expect_calls "$login_call" "$ident_kv_call" "$login_call"
|
|
expect_file "$rt/cert" cert2
|
|
expect_file "$rt/key" key2
|
|
expect_no_file "$rt/cert.new"
|
|
expect_no_file "$rt/key.new"
|
|
|
|
setup IDENT "a re-issued leaf the store refuses is not swapped in"
|
|
kv_out=$(leaf cert2 key2)
|
|
refuse_cert=cert2
|
|
go
|
|
expect_rc 1
|
|
expect_err "refused the certificate at $ident_path"
|
|
expect_file "$rt/cert" cert
|
|
expect_file "$rt/key" key
|
|
expect_no_file "$rt/cert.new"
|
|
|
|
setup IDENT "an object without a key keeps the live pair"
|
|
kv_out='{"data":{"data":{"cert":"cert2"}}}'
|
|
go
|
|
expect_rc 1
|
|
expect_err "holds no certificate and key"
|
|
expect_file "$rt/cert" cert
|
|
expect_file "$rt/key" key
|
|
|
|
if [ "$failures" -ne 0 ]; then
|
|
echo "script-test-agent-bao-fetch: $failures failed assertions over $count runs" >&2
|
|
exit 1
|
|
fi
|
|
echo "script-test-agent-bao-fetch: $count runs, all assertions hold"
|