agents: swap the agent's bao leaf in with one rename
The live pair was replaced with two renames, key then cert. A run stopped between them (timeout, container stop, power loss) left a new key beside the old cert, and every later run logged in with that pair, so the agent could not re-pull until its container restarted. Each pair now goes into a fresh directory under /run/hive-agent-bao-identity, and `current`, a symlink to it, is replaced by one rename. Wherever a run stops, `current` names one whole pair; the next run removes whatever a stopped run left. Consumers read `current/cert` and `current/key` through the same read-only options and still re-read them on every login. The script test covers a swap stopped at each of its link and rename steps: the live pair is always one whole pair, and the next run puts the new one live. Refs #5017
This commit is contained in:
atlas
mara
parent
c6717bccb0
commit
4c4ecb8ed9
7 changed files with 166 additions and 58 deletions
|
|
@ -36,12 +36,14 @@ setup() {
|
|||
log=$dir/bao.log
|
||||
mkdir -m 0700 "$rt"
|
||||
mkdir "$creds"
|
||||
# Per-case stubs that shadow the unit's own tools; empty unless a case adds one.
|
||||
mkdir "$dir/bin"
|
||||
printf cert >"$creds/hive-agent-bao-cert"
|
||||
printf key >"$creds/hive-agent-bao-key"
|
||||
# The identity pair the unit presents: the delivered one for the fetch, the
|
||||
# live copy it keeps for the identity unit.
|
||||
if [ "$prefix" = IDENT ]; then
|
||||
client_cert=$rt/cert client_key=$rt/key
|
||||
client_cert=$rt/current/cert client_key=$rt/current/key
|
||||
else
|
||||
client_cert=$creds/hive-agent-bao-cert client_key=$creds/hive-agent-bao-key
|
||||
fi
|
||||
|
|
@ -69,7 +71,7 @@ go() {
|
|||
umask 0377
|
||||
cd "$dir" || exit 99
|
||||
exec env -i \
|
||||
PATH="$FAKE_BAO_BIN:$unit_path" \
|
||||
PATH="$dir/bin:$FAKE_BAO_BIN:$unit_path" \
|
||||
BAO_ADDR="$bao_addr" \
|
||||
BAO_CLIENT_CERT="$client_cert" \
|
||||
BAO_CLIENT_KEY="$client_key" \
|
||||
|
|
@ -249,63 +251,136 @@ ident_path=secret/swarm/agents/a1/bao-mtls
|
|||
ident_kv_call="kv get -format=json $ident_path"
|
||||
leaf() { printf '{"data":{"data":{"cert":"%s","key":"%s"}}}' "$1" "$2"; }
|
||||
|
||||
expect_pair() {
|
||||
expect_file "$rt/current/cert" "$1"
|
||||
expect_file "$rt/current/key" "$2"
|
||||
}
|
||||
|
||||
expect_pairs() {
|
||||
local d n=0
|
||||
for d in "$rt"/pair.*; do
|
||||
if [ -d "$d" ]; then n=$((n + 1)); fi
|
||||
done
|
||||
if [ "$n" != "$1" ]; then fail "$n pair directories left, expected $1"; fi
|
||||
}
|
||||
|
||||
# live_pair <name> <cert> <key> — a pair an earlier run made live.
|
||||
live_pair() {
|
||||
mkdir -m 0700 "$rt/pair.$1"
|
||||
printf '%s' "$2" >"$rt/pair.$1/cert"
|
||||
printf '%s' "$3" >"$rt/pair.$1/key"
|
||||
chmod 0400 "$rt/pair.$1/cert" "$rt/pair.$1/key"
|
||||
ln -sfn "pair.$1" "$rt/current"
|
||||
}
|
||||
|
||||
# stop_at <n> — the n-th `ln` or `mv` the script runs fails, which leaves
|
||||
# things as a run killed just before that step would.
|
||||
stop_at() {
|
||||
local tool real
|
||||
mkdir "$dir/steps"
|
||||
for tool in ln mv; do
|
||||
real=$(PATH=$unit_path command -v "$tool")
|
||||
cat >"$dir/bin/$tool" <<STUB
|
||||
#!$BASH
|
||||
mktemp -p "$dir/steps" >/dev/null
|
||||
steps=("$dir/steps"/*)
|
||||
if [ "\${#steps[@]}" -ge "$1" ]; then exit 1; fi
|
||||
exec "$real" "\$@"
|
||||
STUB
|
||||
chmod +x "$dir/bin/$tool"
|
||||
done
|
||||
}
|
||||
|
||||
setup IDENT "no store identity delivered"
|
||||
: >"$creds/hive-agent-bao-cert"
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "has no store identity"
|
||||
expect_calls
|
||||
expect_no_file "$rt/cert"
|
||||
if [ -e "$rt/current" ]; then fail "a live pair was made from nothing"; fi
|
||||
|
||||
setup IDENT "a first run seeds the live pair from the delivered one"
|
||||
setup IDENT "a first run makes the delivered pair live"
|
||||
kv_out=$(leaf cert key)
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "is unchanged"
|
||||
expect_calls "$login_call" "$ident_kv_call"
|
||||
expect_file "$rt/cert" cert
|
||||
expect_file "$rt/key" key
|
||||
if [ "$(stat -c %a "$rt/key")" != 400 ]; then fail "key is mode $(stat -c %a "$rt/key"), expected 400"; fi
|
||||
if [ ! -L "$rt/current" ]; then fail "$rt/current is not a symlink"; fi
|
||||
expect_pair cert key
|
||||
if [ "$(stat -L -c %a "$rt/current/key")" != 400 ]; then fail "key is mode $(stat -L -c %a "$rt/current/key"), expected 400"; fi
|
||||
expect_pairs 1
|
||||
expect_no_file "$rt/bao-mtls.json"
|
||||
|
||||
setup IDENT "the live pair outlives the delivered one"
|
||||
printf cert2 >"$rt/cert"
|
||||
printf key2 >"$rt/key"
|
||||
chmod 0400 "$rt/cert" "$rt/key"
|
||||
live_pair old cert2 key2
|
||||
kv_out=$(leaf cert2 key2)
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "is unchanged"
|
||||
expect_file "$rt/cert" cert2
|
||||
expect_pair cert2 key2
|
||||
expect_pairs 1
|
||||
|
||||
setup IDENT "a re-issued leaf replaces the live pair once a login with it succeeds"
|
||||
setup IDENT "a re-issued leaf goes live once a login with it succeeds"
|
||||
kv_out=$(leaf cert2 key2)
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "replaced this agent's store certificate"
|
||||
expect_calls "$login_call" "$ident_kv_call" "$login_call"
|
||||
expect_file "$rt/cert" cert2
|
||||
expect_file "$rt/key" key2
|
||||
expect_no_file "$rt/cert.new"
|
||||
expect_no_file "$rt/key.new"
|
||||
expect_pair cert2 key2
|
||||
# The replaced pair stays for a reader that resolved the link before the swap.
|
||||
expect_pairs 2
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "is unchanged"
|
||||
expect_pairs 1
|
||||
|
||||
setup IDENT "a re-issued leaf the store refuses is not swapped in"
|
||||
setup IDENT "a re-issued leaf the store refuses does not go live"
|
||||
kv_out=$(leaf cert2 key2)
|
||||
refuse_cert=cert2
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "refused the certificate at $ident_path"
|
||||
expect_file "$rt/cert" cert
|
||||
expect_file "$rt/key" key
|
||||
expect_no_file "$rt/cert.new"
|
||||
expect_pair cert key
|
||||
expect_pairs 1
|
||||
|
||||
setup IDENT "an object without a key keeps the live pair"
|
||||
kv_out='{"data":{"data":{"cert":"cert2"}}}'
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "holds no certificate and key"
|
||||
expect_file "$rt/cert" cert
|
||||
expect_file "$rt/key" key
|
||||
expect_pair cert key
|
||||
expect_pairs 1
|
||||
|
||||
setup IDENT "what a stopped run left behind is removed"
|
||||
live_pair old cert key
|
||||
mkdir -m 0700 "$rt/pair.stale"
|
||||
printf cert2 >"$rt/pair.stale/cert"
|
||||
ln -s pair.stale "$rt/current.new"
|
||||
kv_out=$(leaf cert key)
|
||||
go
|
||||
expect_rc 0
|
||||
expect_pair cert key
|
||||
expect_pairs 1
|
||||
if [ -L "$rt/current.new" ]; then fail "current.new left behind"; fi
|
||||
|
||||
# The swap is atomic: wherever a run stops, the link names one whole pair,
|
||||
# and the next run finishes the job.
|
||||
for n in 1 2 3; do
|
||||
setup IDENT "a swap stopped at its step $n leaves one whole pair live"
|
||||
live_pair old cert key
|
||||
kv_out=$(leaf cert2 key2)
|
||||
stop_at "$n"
|
||||
go
|
||||
got="$(<"$rt/current/cert") $(<"$rt/current/key")"
|
||||
case "$got" in
|
||||
"cert key" | "cert2 key2") ;;
|
||||
*) fail "the live pair is '$got', one file of each" ;;
|
||||
esac
|
||||
rm -f "$dir/bin/ln" "$dir/bin/mv"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_pair cert2 key2
|
||||
done
|
||||
|
||||
if [ "$failures" -ne 0 ]; then
|
||||
echo "script-test-agent-bao-fetch: $failures failed assertions over $count runs" >&2
|
||||
|
|
|
|||
Loading…
Reference in a new issue