agents: swap the agent's bao leaf in with one rename
The live pair was replaced with two renames, key then cert. A run stopped between them (timeout, container stop, power loss) left a new key beside the old cert, and every later run logged in with that pair, so the agent could not re-pull until its container restarted. Each pair now goes into a fresh directory under /run/hive-agent-bao-identity, and `current`, a symlink to it, is replaced by one rename. Wherever a run stops, `current` names one whole pair; the next run removes whatever a stopped run left. Consumers read `current/cert` and `current/key` through the same read-only options and still re-read them on every login. The script test covers a swap stopped at each of its link and rename steps: the live pair is always one whole pair, and the next run puts the new one live. Refs #5017
This commit is contained in:
atlas
mara
parent
c6717bccb0
commit
4c4ecb8ed9
7 changed files with 166 additions and 58 deletions
|
|
@ -50,12 +50,12 @@ let
|
|||
unitName = "hive-agent-bao-identity";
|
||||
|
||||
runtimeDir = unitName;
|
||||
certFile = "/run/${runtimeDir}/cert";
|
||||
keyFile = "/run/${runtimeDir}/key";
|
||||
# Beside the live pair, so each rename that replaces it stays in one
|
||||
# directory.
|
||||
stagedCert = "/run/${runtimeDir}/cert.new";
|
||||
stagedKey = "/run/${runtimeDir}/key.new";
|
||||
# A symlink to the directory that holds the live pair. Each pair is written
|
||||
# into a fresh directory and goes live with one rename of this link, so the
|
||||
# link always names one whole pair, old or new.
|
||||
liveLink = "/run/${runtimeDir}/current";
|
||||
certFile = "${liveLink}/cert";
|
||||
keyFile = "${liveLink}/key";
|
||||
# The store's whole answer, key included, in the unit's own `0700` directory.
|
||||
rawFile = "/run/${runtimeDir}/bao-mtls.json";
|
||||
errFile = "/run/${runtimeDir}/bao.err";
|
||||
|
|
@ -196,14 +196,48 @@ in
|
|||
# is removed before the command that creates it.
|
||||
err=${lib.escapeShellArg errFile}
|
||||
raw=${lib.escapeShellArg rawFile}
|
||||
trap 'rm -f "$err" "$raw" ${lib.escapeShellArg stagedCert} ${lib.escapeShellArg stagedKey}' EXIT
|
||||
trap 'rm -f "$err" "$raw"' EXIT
|
||||
|
||||
live=${lib.escapeShellArg liveLink}
|
||||
dir=''${live%/*}
|
||||
|
||||
# A fresh directory for one pair. `chmod`, because the umask cuts
|
||||
# mktemp's `0700` down to a `0400` no file can be created in.
|
||||
new_pair() {
|
||||
local d
|
||||
d=$(mktemp -d -p "$dir" pair.XXXXXX)
|
||||
chmod 0700 "$d"
|
||||
echo "$d"
|
||||
}
|
||||
|
||||
# The only step that changes the live pair: one rename of the link. A
|
||||
# reader whose two opens straddle it gets one file of each pair for
|
||||
# that one login; the next login reads a whole pair.
|
||||
go_live() {
|
||||
rm -f "$live.new"
|
||||
ln -s "''${1##*/}" "$live.new"
|
||||
mv -Tf "$live.new" "$live"
|
||||
}
|
||||
|
||||
# Every pair directory but the live one is left over: the pair the
|
||||
# last swap replaced, kept until now for a reader that resolved the
|
||||
# link before the swap, or a pair a stopped run staged.
|
||||
current=$(readlink "$live" || true)
|
||||
for d in "$dir"/pair.*; do
|
||||
if [ -d "$d" ] && [ "''${d##*/}" != "$current" ]; then
|
||||
chmod 0700 "$d"
|
||||
rm -rf "$d"
|
||||
fi
|
||||
done
|
||||
rm -f "$live.new"
|
||||
|
||||
# The delivered pair is the start of every container's life: /run is
|
||||
# empty at boot and kept across this unit's runs.
|
||||
if [ ! -s "$BAO_CLIENT_CERT" ] || [ ! -s "$BAO_CLIENT_KEY" ]; then
|
||||
rm -f "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY"
|
||||
cp "$CREDENTIALS_DIRECTORY/${keyCredential}" "$BAO_CLIENT_KEY"
|
||||
cp "$CREDENTIALS_DIRECTORY/${certCredential}" "$BAO_CLIENT_CERT"
|
||||
seed=$(new_pair)
|
||||
cp "$CREDENTIALS_DIRECTORY/${keyCredential}" "$seed/key"
|
||||
cp "$CREDENTIALS_DIRECTORY/${certCredential}" "$seed/cert"
|
||||
go_live "$seed"
|
||||
fi
|
||||
|
||||
# Cert auth is a login, not a transport setting. The `BAO_CLIENT_*`
|
||||
|
|
@ -247,31 +281,30 @@ in
|
|||
|
||||
# `-j`: the bytes as the store holds them, which is what the hive
|
||||
# delivered, so an unchanged leaf compares equal to the seeded copy.
|
||||
rm -f ${lib.escapeShellArg stagedCert} ${lib.escapeShellArg stagedKey}
|
||||
if ! jq -ej '.data.data.cert | strings' "$raw" >${lib.escapeShellArg stagedCert} \
|
||||
|| ! jq -ej '.data.data.key | strings' "$raw" >${lib.escapeShellArg stagedKey}; then
|
||||
staged=$(new_pair)
|
||||
if ! jq -ej '.data.data.cert | strings' "$raw" >"$staged/cert" \
|
||||
|| ! jq -ej '.data.data.key | strings' "$raw" >"$staged/key"; then
|
||||
rm -rf "$staged"
|
||||
echo "${identityPath} holds no certificate and key; this agent keeps the one it has." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if cmp -s ${lib.escapeShellArg stagedCert} "$BAO_CLIENT_CERT" \
|
||||
&& cmp -s ${lib.escapeShellArg stagedKey} "$BAO_CLIENT_KEY"; then
|
||||
if cmp -s "$staged/cert" "$BAO_CLIENT_CERT" && cmp -s "$staged/key" "$BAO_CLIENT_KEY"; then
|
||||
rm -rf "$staged"
|
||||
echo "authenticated to the swarm secret store at $BAO_ADDR as ${agentName}; its certificate at ${identityPath} is unchanged."
|
||||
else
|
||||
# This unit's next login uses whatever is swapped in, so a pair the
|
||||
# This unit's next login uses whatever goes live, so a pair the
|
||||
# store refuses must never replace the one it accepts.
|
||||
rm -f "$err"
|
||||
if ! BAO_CLIENT_CERT=${lib.escapeShellArg stagedCert} BAO_CLIENT_KEY=${lib.escapeShellArg stagedKey} \
|
||||
if ! BAO_CLIENT_CERT="$staged/cert" BAO_CLIENT_KEY="$staged/key" \
|
||||
bao login -method=cert -token-only >/dev/null 2>"$err"; then
|
||||
rm -rf "$staged"
|
||||
echo "the swarm secret store refused the certificate at ${identityPath}; this agent keeps the one it has." >&2
|
||||
if [ -s "$err" ]; then cat "$err" >&2; fi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Two renames: a reader that opens the pair between them gets a
|
||||
# mismatched cert and key and fails that one login.
|
||||
mv -f ${lib.escapeShellArg stagedKey} "$BAO_CLIENT_KEY"
|
||||
mv -f ${lib.escapeShellArg stagedCert} "$BAO_CLIENT_CERT"
|
||||
go_live "$staged"
|
||||
echo "replaced this agent's store certificate with the one at ${identityPath}."
|
||||
fi
|
||||
'';
|
||||
|
|
|
|||
|
|
@ -84,8 +84,8 @@ let
|
|||
e = u.environment;
|
||||
in
|
||||
e.BAO_ADDR == baoAddr
|
||||
&& e.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert"
|
||||
&& e.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key";
|
||||
&& e.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
|
||||
&& e.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key";
|
||||
}
|
||||
{
|
||||
# The field is the secret. It goes to a file by redirect and is never
|
||||
|
|
@ -199,7 +199,7 @@ let
|
|||
in
|
||||
u.serviceConfig.User == name
|
||||
&& u.serviceConfig.UMask == "0077"
|
||||
&& u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert";
|
||||
&& u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert";
|
||||
}
|
||||
{
|
||||
# Files a hive wrote keep working until the operator re-links them.
|
||||
|
|
|
|||
|
|
@ -83,8 +83,8 @@ let
|
|||
u.serviceConfig.User == name
|
||||
&& u.serviceConfig.UMask == "0077"
|
||||
&& u.environment.BAO_ADDR == baoAddr
|
||||
&& u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert"
|
||||
&& u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key";
|
||||
&& u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
|
||||
&& u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key";
|
||||
}
|
||||
{
|
||||
# A file a hive wrote keeps working until the operator links a token in
|
||||
|
|
|
|||
|
|
@ -124,8 +124,8 @@ let
|
|||
in
|
||||
u.serviceConfig.LoadCredential == [ "hive-agent-bao-server-ca" ]
|
||||
&& u.environment.BAO_ADDR == "https://bao.t.local:8200"
|
||||
&& u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert"
|
||||
&& u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key"
|
||||
&& u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
|
||||
&& u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key"
|
||||
&& builtins.elem "hive-agent-bao-identity.service" u.after;
|
||||
}
|
||||
{
|
||||
|
|
|
|||
|
|
@ -97,8 +97,8 @@ let
|
|||
u = agentBaoIdentity agentBao;
|
||||
e = u.environment;
|
||||
in
|
||||
e.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert"
|
||||
&& e.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key"
|
||||
e.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
|
||||
&& e.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key"
|
||||
&& u.serviceConfig.RuntimeDirectory == "hive-agent-bao-identity"
|
||||
&& u.serviceConfig.RuntimeDirectoryPreserve == "yes"
|
||||
&& e.BAO_ADDR == agentBao.services.hyperhive.agent.bao.addr;
|
||||
|
|
@ -177,8 +177,8 @@ let
|
|||
let
|
||||
u = agentHarness agentBao;
|
||||
in
|
||||
u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert"
|
||||
&& u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key"
|
||||
u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
|
||||
&& u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key"
|
||||
&& builtins.elem "hive-agent-bao-identity.service" u.after
|
||||
&& u.environment.BAO_ADDR == agentBao.services.hyperhive.agent.bao.addr;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -116,8 +116,8 @@ let
|
|||
u.serviceConfig.LoadCredential == [ "hive-agent-bao-server-ca" ]
|
||||
&& u.environment.HIVE_AGENT_NAME == "a1"
|
||||
&& u.environment.BAO_ADDR == "https://bao.t.local:8200"
|
||||
&& u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert"
|
||||
&& u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key"
|
||||
&& u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
|
||||
&& u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key"
|
||||
)
|
||||
[
|
||||
(subagent claudeBao)
|
||||
|
|
@ -134,8 +134,8 @@ let
|
|||
u.serviceConfig.LoadCredential == [ "hive-agent-bao-server-ca" ]
|
||||
&& u.environment.HIVE_AGENT_NAME == "a1"
|
||||
&& u.environment.BAO_ADDR == "https://bao.t.local:8200"
|
||||
&& u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert"
|
||||
&& u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key";
|
||||
&& u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
|
||||
&& u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key";
|
||||
}
|
||||
{
|
||||
name = "an opencode agent with no store hands its subagent daemon no store identity";
|
||||
|
|
|
|||
|
|
@ -36,12 +36,14 @@ setup() {
|
|||
log=$dir/bao.log
|
||||
mkdir -m 0700 "$rt"
|
||||
mkdir "$creds"
|
||||
# Per-case stubs that shadow the unit's own tools; empty unless a case adds one.
|
||||
mkdir "$dir/bin"
|
||||
printf cert >"$creds/hive-agent-bao-cert"
|
||||
printf key >"$creds/hive-agent-bao-key"
|
||||
# The identity pair the unit presents: the delivered one for the fetch, the
|
||||
# live copy it keeps for the identity unit.
|
||||
if [ "$prefix" = IDENT ]; then
|
||||
client_cert=$rt/cert client_key=$rt/key
|
||||
client_cert=$rt/current/cert client_key=$rt/current/key
|
||||
else
|
||||
client_cert=$creds/hive-agent-bao-cert client_key=$creds/hive-agent-bao-key
|
||||
fi
|
||||
|
|
@ -69,7 +71,7 @@ go() {
|
|||
umask 0377
|
||||
cd "$dir" || exit 99
|
||||
exec env -i \
|
||||
PATH="$FAKE_BAO_BIN:$unit_path" \
|
||||
PATH="$dir/bin:$FAKE_BAO_BIN:$unit_path" \
|
||||
BAO_ADDR="$bao_addr" \
|
||||
BAO_CLIENT_CERT="$client_cert" \
|
||||
BAO_CLIENT_KEY="$client_key" \
|
||||
|
|
@ -249,63 +251,136 @@ ident_path=secret/swarm/agents/a1/bao-mtls
|
|||
ident_kv_call="kv get -format=json $ident_path"
|
||||
leaf() { printf '{"data":{"data":{"cert":"%s","key":"%s"}}}' "$1" "$2"; }
|
||||
|
||||
expect_pair() {
|
||||
expect_file "$rt/current/cert" "$1"
|
||||
expect_file "$rt/current/key" "$2"
|
||||
}
|
||||
|
||||
expect_pairs() {
|
||||
local d n=0
|
||||
for d in "$rt"/pair.*; do
|
||||
if [ -d "$d" ]; then n=$((n + 1)); fi
|
||||
done
|
||||
if [ "$n" != "$1" ]; then fail "$n pair directories left, expected $1"; fi
|
||||
}
|
||||
|
||||
# live_pair <name> <cert> <key> — a pair an earlier run made live.
|
||||
live_pair() {
|
||||
mkdir -m 0700 "$rt/pair.$1"
|
||||
printf '%s' "$2" >"$rt/pair.$1/cert"
|
||||
printf '%s' "$3" >"$rt/pair.$1/key"
|
||||
chmod 0400 "$rt/pair.$1/cert" "$rt/pair.$1/key"
|
||||
ln -sfn "pair.$1" "$rt/current"
|
||||
}
|
||||
|
||||
# stop_at <n> — the n-th `ln` or `mv` the script runs fails, which leaves
|
||||
# things as a run killed just before that step would.
|
||||
stop_at() {
|
||||
local tool real
|
||||
mkdir "$dir/steps"
|
||||
for tool in ln mv; do
|
||||
real=$(PATH=$unit_path command -v "$tool")
|
||||
cat >"$dir/bin/$tool" <<STUB
|
||||
#!$BASH
|
||||
mktemp -p "$dir/steps" >/dev/null
|
||||
steps=("$dir/steps"/*)
|
||||
if [ "\${#steps[@]}" -ge "$1" ]; then exit 1; fi
|
||||
exec "$real" "\$@"
|
||||
STUB
|
||||
chmod +x "$dir/bin/$tool"
|
||||
done
|
||||
}
|
||||
|
||||
setup IDENT "no store identity delivered"
|
||||
: >"$creds/hive-agent-bao-cert"
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "has no store identity"
|
||||
expect_calls
|
||||
expect_no_file "$rt/cert"
|
||||
if [ -e "$rt/current" ]; then fail "a live pair was made from nothing"; fi
|
||||
|
||||
setup IDENT "a first run seeds the live pair from the delivered one"
|
||||
setup IDENT "a first run makes the delivered pair live"
|
||||
kv_out=$(leaf cert key)
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "is unchanged"
|
||||
expect_calls "$login_call" "$ident_kv_call"
|
||||
expect_file "$rt/cert" cert
|
||||
expect_file "$rt/key" key
|
||||
if [ "$(stat -c %a "$rt/key")" != 400 ]; then fail "key is mode $(stat -c %a "$rt/key"), expected 400"; fi
|
||||
if [ ! -L "$rt/current" ]; then fail "$rt/current is not a symlink"; fi
|
||||
expect_pair cert key
|
||||
if [ "$(stat -L -c %a "$rt/current/key")" != 400 ]; then fail "key is mode $(stat -L -c %a "$rt/current/key"), expected 400"; fi
|
||||
expect_pairs 1
|
||||
expect_no_file "$rt/bao-mtls.json"
|
||||
|
||||
setup IDENT "the live pair outlives the delivered one"
|
||||
printf cert2 >"$rt/cert"
|
||||
printf key2 >"$rt/key"
|
||||
chmod 0400 "$rt/cert" "$rt/key"
|
||||
live_pair old cert2 key2
|
||||
kv_out=$(leaf cert2 key2)
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "is unchanged"
|
||||
expect_file "$rt/cert" cert2
|
||||
expect_pair cert2 key2
|
||||
expect_pairs 1
|
||||
|
||||
setup IDENT "a re-issued leaf replaces the live pair once a login with it succeeds"
|
||||
setup IDENT "a re-issued leaf goes live once a login with it succeeds"
|
||||
kv_out=$(leaf cert2 key2)
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "replaced this agent's store certificate"
|
||||
expect_calls "$login_call" "$ident_kv_call" "$login_call"
|
||||
expect_file "$rt/cert" cert2
|
||||
expect_file "$rt/key" key2
|
||||
expect_no_file "$rt/cert.new"
|
||||
expect_no_file "$rt/key.new"
|
||||
expect_pair cert2 key2
|
||||
# The replaced pair stays for a reader that resolved the link before the swap.
|
||||
expect_pairs 2
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "is unchanged"
|
||||
expect_pairs 1
|
||||
|
||||
setup IDENT "a re-issued leaf the store refuses is not swapped in"
|
||||
setup IDENT "a re-issued leaf the store refuses does not go live"
|
||||
kv_out=$(leaf cert2 key2)
|
||||
refuse_cert=cert2
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "refused the certificate at $ident_path"
|
||||
expect_file "$rt/cert" cert
|
||||
expect_file "$rt/key" key
|
||||
expect_no_file "$rt/cert.new"
|
||||
expect_pair cert key
|
||||
expect_pairs 1
|
||||
|
||||
setup IDENT "an object without a key keeps the live pair"
|
||||
kv_out='{"data":{"data":{"cert":"cert2"}}}'
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "holds no certificate and key"
|
||||
expect_file "$rt/cert" cert
|
||||
expect_file "$rt/key" key
|
||||
expect_pair cert key
|
||||
expect_pairs 1
|
||||
|
||||
setup IDENT "what a stopped run left behind is removed"
|
||||
live_pair old cert key
|
||||
mkdir -m 0700 "$rt/pair.stale"
|
||||
printf cert2 >"$rt/pair.stale/cert"
|
||||
ln -s pair.stale "$rt/current.new"
|
||||
kv_out=$(leaf cert key)
|
||||
go
|
||||
expect_rc 0
|
||||
expect_pair cert key
|
||||
expect_pairs 1
|
||||
if [ -L "$rt/current.new" ]; then fail "current.new left behind"; fi
|
||||
|
||||
# The swap is atomic: wherever a run stops, the link names one whole pair,
|
||||
# and the next run finishes the job.
|
||||
for n in 1 2 3; do
|
||||
setup IDENT "a swap stopped at its step $n leaves one whole pair live"
|
||||
live_pair old cert key
|
||||
kv_out=$(leaf cert2 key2)
|
||||
stop_at "$n"
|
||||
go
|
||||
got="$(<"$rt/current/cert") $(<"$rt/current/key")"
|
||||
case "$got" in
|
||||
"cert key" | "cert2 key2") ;;
|
||||
*) fail "the live pair is '$got', one file of each" ;;
|
||||
esac
|
||||
rm -f "$dir/bin/ln" "$dir/bin/mv"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_pair cert2 key2
|
||||
done
|
||||
|
||||
if [ "$failures" -ne 0 ]; then
|
||||
echo "script-test-agent-bao-fetch: $failures failed assertions over $count runs" >&2
|
||||
|
|
|
|||
Loading…
Reference in a new issue