From 4c4ecb8ed9a37915d19792069cf98918a8857048 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 11 Oct 2026 03:24:49 +0200 Subject: [PATCH] agents: swap the agent's bao leaf in with one rename The live pair was replaced with two renames, key then cert. A run stopped between them (timeout, container stop, power loss) left a new key beside the old cert, and every later run logged in with that pair, so the agent could not re-pull until its container restarted. Each pair now goes into a fresh directory under /run/hive-agent-bao-identity, and `current`, a symlink to it, is replaced by one rename. Wherever a run stops, `current` names one whole pair; the next run removes whatever a stopped run left. Consumers read `current/cert` and `current/key` through the same read-only options and still re-read them on every login. The script test covers a swap stopped at each of its link and rename steps: the live pair is always one whole pair, and the next run puts the new one live. Refs #5017 --- nix/agent-modules/bao.nix | 75 ++++++++++++----- nix/module-eval/agent-forge-bao.nix | 6 +- nix/module-eval/agent-github-bao.nix | 4 +- nix/module-eval/agent-matrix.nix | 4 +- nix/module-eval/agent-queue-bao.nix | 8 +- nix/module-eval/agent-runtime.nix | 8 +- nix/script-tests/agent-bao-fetch.sh | 119 ++++++++++++++++++++++----- 7 files changed, 166 insertions(+), 58 deletions(-) diff --git a/nix/agent-modules/bao.nix b/nix/agent-modules/bao.nix index 8380abd1..121a756a 100644 --- a/nix/agent-modules/bao.nix +++ b/nix/agent-modules/bao.nix @@ -50,12 +50,12 @@ let unitName = "hive-agent-bao-identity"; runtimeDir = unitName; - certFile = "/run/${runtimeDir}/cert"; - keyFile = "/run/${runtimeDir}/key"; - # Beside the live pair, so each rename that replaces it stays in one - # directory. - stagedCert = "/run/${runtimeDir}/cert.new"; - stagedKey = "/run/${runtimeDir}/key.new"; + # A symlink to the directory that holds the live pair. Each pair is written + # into a fresh directory and goes live with one rename of this link, so the + # link always names one whole pair, old or new. + liveLink = "/run/${runtimeDir}/current"; + certFile = "${liveLink}/cert"; + keyFile = "${liveLink}/key"; # The store's whole answer, key included, in the unit's own `0700` directory. rawFile = "/run/${runtimeDir}/bao-mtls.json"; errFile = "/run/${runtimeDir}/bao.err"; @@ -196,14 +196,48 @@ in # is removed before the command that creates it. err=${lib.escapeShellArg errFile} raw=${lib.escapeShellArg rawFile} - trap 'rm -f "$err" "$raw" ${lib.escapeShellArg stagedCert} ${lib.escapeShellArg stagedKey}' EXIT + trap 'rm -f "$err" "$raw"' EXIT + + live=${lib.escapeShellArg liveLink} + dir=''${live%/*} + + # A fresh directory for one pair. `chmod`, because the umask cuts + # mktemp's `0700` down to a `0400` no file can be created in. + new_pair() { + local d + d=$(mktemp -d -p "$dir" pair.XXXXXX) + chmod 0700 "$d" + echo "$d" + } + + # The only step that changes the live pair: one rename of the link. A + # reader whose two opens straddle it gets one file of each pair for + # that one login; the next login reads a whole pair. + go_live() { + rm -f "$live.new" + ln -s "''${1##*/}" "$live.new" + mv -Tf "$live.new" "$live" + } + + # Every pair directory but the live one is left over: the pair the + # last swap replaced, kept until now for a reader that resolved the + # link before the swap, or a pair a stopped run staged. + current=$(readlink "$live" || true) + for d in "$dir"/pair.*; do + if [ -d "$d" ] && [ "''${d##*/}" != "$current" ]; then + chmod 0700 "$d" + rm -rf "$d" + fi + done + rm -f "$live.new" # The delivered pair is the start of every container's life: /run is # empty at boot and kept across this unit's runs. if [ ! -s "$BAO_CLIENT_CERT" ] || [ ! -s "$BAO_CLIENT_KEY" ]; then - rm -f "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY" - cp "$CREDENTIALS_DIRECTORY/${keyCredential}" "$BAO_CLIENT_KEY" - cp "$CREDENTIALS_DIRECTORY/${certCredential}" "$BAO_CLIENT_CERT" + seed=$(new_pair) + cp "$CREDENTIALS_DIRECTORY/${keyCredential}" "$seed/key" + cp "$CREDENTIALS_DIRECTORY/${certCredential}" "$seed/cert" + go_live "$seed" fi # Cert auth is a login, not a transport setting. The `BAO_CLIENT_*` @@ -247,31 +281,30 @@ in # `-j`: the bytes as the store holds them, which is what the hive # delivered, so an unchanged leaf compares equal to the seeded copy. - rm -f ${lib.escapeShellArg stagedCert} ${lib.escapeShellArg stagedKey} - if ! jq -ej '.data.data.cert | strings' "$raw" >${lib.escapeShellArg stagedCert} \ - || ! jq -ej '.data.data.key | strings' "$raw" >${lib.escapeShellArg stagedKey}; then + staged=$(new_pair) + if ! jq -ej '.data.data.cert | strings' "$raw" >"$staged/cert" \ + || ! jq -ej '.data.data.key | strings' "$raw" >"$staged/key"; then + rm -rf "$staged" echo "${identityPath} holds no certificate and key; this agent keeps the one it has." >&2 exit 1 fi - if cmp -s ${lib.escapeShellArg stagedCert} "$BAO_CLIENT_CERT" \ - && cmp -s ${lib.escapeShellArg stagedKey} "$BAO_CLIENT_KEY"; then + if cmp -s "$staged/cert" "$BAO_CLIENT_CERT" && cmp -s "$staged/key" "$BAO_CLIENT_KEY"; then + rm -rf "$staged" echo "authenticated to the swarm secret store at $BAO_ADDR as ${agentName}; its certificate at ${identityPath} is unchanged." else - # This unit's next login uses whatever is swapped in, so a pair the + # This unit's next login uses whatever goes live, so a pair the # store refuses must never replace the one it accepts. rm -f "$err" - if ! BAO_CLIENT_CERT=${lib.escapeShellArg stagedCert} BAO_CLIENT_KEY=${lib.escapeShellArg stagedKey} \ + if ! BAO_CLIENT_CERT="$staged/cert" BAO_CLIENT_KEY="$staged/key" \ bao login -method=cert -token-only >/dev/null 2>"$err"; then + rm -rf "$staged" echo "the swarm secret store refused the certificate at ${identityPath}; this agent keeps the one it has." >&2 if [ -s "$err" ]; then cat "$err" >&2; fi exit 1 fi - # Two renames: a reader that opens the pair between them gets a - # mismatched cert and key and fails that one login. - mv -f ${lib.escapeShellArg stagedKey} "$BAO_CLIENT_KEY" - mv -f ${lib.escapeShellArg stagedCert} "$BAO_CLIENT_CERT" + go_live "$staged" echo "replaced this agent's store certificate with the one at ${identityPath}." fi ''; diff --git a/nix/module-eval/agent-forge-bao.nix b/nix/module-eval/agent-forge-bao.nix index 3a71384e..be6be7f4 100644 --- a/nix/module-eval/agent-forge-bao.nix +++ b/nix/module-eval/agent-forge-bao.nix @@ -84,8 +84,8 @@ let e = u.environment; in e.BAO_ADDR == baoAddr - && e.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert" - && e.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key"; + && e.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert" + && e.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key"; } { # The field is the secret. It goes to a file by redirect and is never @@ -199,7 +199,7 @@ let in u.serviceConfig.User == name && u.serviceConfig.UMask == "0077" - && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert"; + && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"; } { # Files a hive wrote keep working until the operator re-links them. diff --git a/nix/module-eval/agent-github-bao.nix b/nix/module-eval/agent-github-bao.nix index 6d2e0dcd..6b787911 100644 --- a/nix/module-eval/agent-github-bao.nix +++ b/nix/module-eval/agent-github-bao.nix @@ -83,8 +83,8 @@ let u.serviceConfig.User == name && u.serviceConfig.UMask == "0077" && u.environment.BAO_ADDR == baoAddr - && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert" - && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key"; + && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert" + && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key"; } { # A file a hive wrote keeps working until the operator links a token in diff --git a/nix/module-eval/agent-matrix.nix b/nix/module-eval/agent-matrix.nix index 45586759..0ca35876 100644 --- a/nix/module-eval/agent-matrix.nix +++ b/nix/module-eval/agent-matrix.nix @@ -124,8 +124,8 @@ let in u.serviceConfig.LoadCredential == [ "hive-agent-bao-server-ca" ] && u.environment.BAO_ADDR == "https://bao.t.local:8200" - && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert" - && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key" + && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert" + && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key" && builtins.elem "hive-agent-bao-identity.service" u.after; } { diff --git a/nix/module-eval/agent-queue-bao.nix b/nix/module-eval/agent-queue-bao.nix index 725b2f88..85f11f42 100644 --- a/nix/module-eval/agent-queue-bao.nix +++ b/nix/module-eval/agent-queue-bao.nix @@ -97,8 +97,8 @@ let u = agentBaoIdentity agentBao; e = u.environment; in - e.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert" - && e.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key" + e.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert" + && e.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key" && u.serviceConfig.RuntimeDirectory == "hive-agent-bao-identity" && u.serviceConfig.RuntimeDirectoryPreserve == "yes" && e.BAO_ADDR == agentBao.services.hyperhive.agent.bao.addr; @@ -177,8 +177,8 @@ let let u = agentHarness agentBao; in - u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert" - && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key" + u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert" + && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key" && builtins.elem "hive-agent-bao-identity.service" u.after && u.environment.BAO_ADDR == agentBao.services.hyperhive.agent.bao.addr; } diff --git a/nix/module-eval/agent-runtime.nix b/nix/module-eval/agent-runtime.nix index 7f061e60..83dd4a31 100644 --- a/nix/module-eval/agent-runtime.nix +++ b/nix/module-eval/agent-runtime.nix @@ -116,8 +116,8 @@ let u.serviceConfig.LoadCredential == [ "hive-agent-bao-server-ca" ] && u.environment.HIVE_AGENT_NAME == "a1" && u.environment.BAO_ADDR == "https://bao.t.local:8200" - && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert" - && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key" + && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert" + && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key" ) [ (subagent claudeBao) @@ -134,8 +134,8 @@ let u.serviceConfig.LoadCredential == [ "hive-agent-bao-server-ca" ] && u.environment.HIVE_AGENT_NAME == "a1" && u.environment.BAO_ADDR == "https://bao.t.local:8200" - && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert" - && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key"; + && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert" + && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key"; } { name = "an opencode agent with no store hands its subagent daemon no store identity"; diff --git a/nix/script-tests/agent-bao-fetch.sh b/nix/script-tests/agent-bao-fetch.sh index 988e60a4..74ad8476 100644 --- a/nix/script-tests/agent-bao-fetch.sh +++ b/nix/script-tests/agent-bao-fetch.sh @@ -36,12 +36,14 @@ setup() { log=$dir/bao.log mkdir -m 0700 "$rt" mkdir "$creds" + # Per-case stubs that shadow the unit's own tools; empty unless a case adds one. + mkdir "$dir/bin" printf cert >"$creds/hive-agent-bao-cert" printf key >"$creds/hive-agent-bao-key" # The identity pair the unit presents: the delivered one for the fetch, the # live copy it keeps for the identity unit. if [ "$prefix" = IDENT ]; then - client_cert=$rt/cert client_key=$rt/key + client_cert=$rt/current/cert client_key=$rt/current/key else client_cert=$creds/hive-agent-bao-cert client_key=$creds/hive-agent-bao-key fi @@ -69,7 +71,7 @@ go() { umask 0377 cd "$dir" || exit 99 exec env -i \ - PATH="$FAKE_BAO_BIN:$unit_path" \ + PATH="$dir/bin:$FAKE_BAO_BIN:$unit_path" \ BAO_ADDR="$bao_addr" \ BAO_CLIENT_CERT="$client_cert" \ BAO_CLIENT_KEY="$client_key" \ @@ -249,63 +251,136 @@ ident_path=secret/swarm/agents/a1/bao-mtls ident_kv_call="kv get -format=json $ident_path" leaf() { printf '{"data":{"data":{"cert":"%s","key":"%s"}}}' "$1" "$2"; } +expect_pair() { + expect_file "$rt/current/cert" "$1" + expect_file "$rt/current/key" "$2" +} + +expect_pairs() { + local d n=0 + for d in "$rt"/pair.*; do + if [ -d "$d" ]; then n=$((n + 1)); fi + done + if [ "$n" != "$1" ]; then fail "$n pair directories left, expected $1"; fi +} + +# live_pair — a pair an earlier run made live. +live_pair() { + mkdir -m 0700 "$rt/pair.$1" + printf '%s' "$2" >"$rt/pair.$1/cert" + printf '%s' "$3" >"$rt/pair.$1/key" + chmod 0400 "$rt/pair.$1/cert" "$rt/pair.$1/key" + ln -sfn "pair.$1" "$rt/current" +} + +# stop_at — the n-th `ln` or `mv` the script runs fails, which leaves +# things as a run killed just before that step would. +stop_at() { + local tool real + mkdir "$dir/steps" + for tool in ln mv; do + real=$(PATH=$unit_path command -v "$tool") + cat >"$dir/bin/$tool" </dev/null +steps=("$dir/steps"/*) +if [ "\${#steps[@]}" -ge "$1" ]; then exit 1; fi +exec "$real" "\$@" +STUB + chmod +x "$dir/bin/$tool" + done +} + setup IDENT "no store identity delivered" : >"$creds/hive-agent-bao-cert" go expect_rc 1 expect_err "has no store identity" expect_calls -expect_no_file "$rt/cert" +if [ -e "$rt/current" ]; then fail "a live pair was made from nothing"; fi -setup IDENT "a first run seeds the live pair from the delivered one" +setup IDENT "a first run makes the delivered pair live" kv_out=$(leaf cert key) go expect_rc 0 expect_out "is unchanged" expect_calls "$login_call" "$ident_kv_call" -expect_file "$rt/cert" cert -expect_file "$rt/key" key -if [ "$(stat -c %a "$rt/key")" != 400 ]; then fail "key is mode $(stat -c %a "$rt/key"), expected 400"; fi +if [ ! -L "$rt/current" ]; then fail "$rt/current is not a symlink"; fi +expect_pair cert key +if [ "$(stat -L -c %a "$rt/current/key")" != 400 ]; then fail "key is mode $(stat -L -c %a "$rt/current/key"), expected 400"; fi +expect_pairs 1 expect_no_file "$rt/bao-mtls.json" setup IDENT "the live pair outlives the delivered one" -printf cert2 >"$rt/cert" -printf key2 >"$rt/key" -chmod 0400 "$rt/cert" "$rt/key" +live_pair old cert2 key2 kv_out=$(leaf cert2 key2) go expect_rc 0 expect_out "is unchanged" -expect_file "$rt/cert" cert2 +expect_pair cert2 key2 +expect_pairs 1 -setup IDENT "a re-issued leaf replaces the live pair once a login with it succeeds" +setup IDENT "a re-issued leaf goes live once a login with it succeeds" kv_out=$(leaf cert2 key2) go expect_rc 0 expect_out "replaced this agent's store certificate" expect_calls "$login_call" "$ident_kv_call" "$login_call" -expect_file "$rt/cert" cert2 -expect_file "$rt/key" key2 -expect_no_file "$rt/cert.new" -expect_no_file "$rt/key.new" +expect_pair cert2 key2 +# The replaced pair stays for a reader that resolved the link before the swap. +expect_pairs 2 +go +expect_rc 0 +expect_out "is unchanged" +expect_pairs 1 -setup IDENT "a re-issued leaf the store refuses is not swapped in" +setup IDENT "a re-issued leaf the store refuses does not go live" kv_out=$(leaf cert2 key2) refuse_cert=cert2 go expect_rc 1 expect_err "refused the certificate at $ident_path" -expect_file "$rt/cert" cert -expect_file "$rt/key" key -expect_no_file "$rt/cert.new" +expect_pair cert key +expect_pairs 1 setup IDENT "an object without a key keeps the live pair" kv_out='{"data":{"data":{"cert":"cert2"}}}' go expect_rc 1 expect_err "holds no certificate and key" -expect_file "$rt/cert" cert -expect_file "$rt/key" key +expect_pair cert key +expect_pairs 1 + +setup IDENT "what a stopped run left behind is removed" +live_pair old cert key +mkdir -m 0700 "$rt/pair.stale" +printf cert2 >"$rt/pair.stale/cert" +ln -s pair.stale "$rt/current.new" +kv_out=$(leaf cert key) +go +expect_rc 0 +expect_pair cert key +expect_pairs 1 +if [ -L "$rt/current.new" ]; then fail "current.new left behind"; fi + +# The swap is atomic: wherever a run stops, the link names one whole pair, +# and the next run finishes the job. +for n in 1 2 3; do + setup IDENT "a swap stopped at its step $n leaves one whole pair live" + live_pair old cert key + kv_out=$(leaf cert2 key2) + stop_at "$n" + go + got="$(<"$rt/current/cert") $(<"$rt/current/key")" + case "$got" in + "cert key" | "cert2 key2") ;; + *) fail "the live pair is '$got', one file of each" ;; + esac + rm -f "$dir/bin/ln" "$dir/bin/mv" + go + expect_rc 0 + expect_pair cert2 key2 +done if [ "$failures" -ne 0 ]; then echo "script-test-agent-bao-fetch: $failures failed assertions over $count runs" >&2