agents: swap the agent's bao leaf in with one rename

The live pair was replaced with two renames, key then cert. A run
stopped between them (timeout, container stop, power loss) left a new
key beside the old cert, and every later run logged in with that pair,
so the agent could not re-pull until its container restarted.

Each pair now goes into a fresh directory under
/run/hive-agent-bao-identity, and `current`, a symlink to it, is
replaced by one rename. Wherever a run stops, `current` names one whole
pair; the next run removes whatever a stopped run left. Consumers read
`current/cert` and `current/key` through the same read-only options and
still re-read them on every login.

The script test covers a swap stopped at each of its link and rename
steps: the live pair is always one whole pair, and the next run puts
the new one live.

Refs #5017

nix/agent-modules/bao.nix

@@ -50,12 +50,12 @@ let
   unitName = "hive-agent-bao-identity";
 
   runtimeDir = unitName;
-  certFile = "/run/${runtimeDir}/cert";
+  # A symlink to the directory that holds the live pair. Each pair is written
-  keyFile = "/run/${runtimeDir}/key";
+  # into a fresh directory and goes live with one rename of this link, so the
-  # Beside the live pair, so each rename that replaces it stays in one
+  # link always names one whole pair, old or new.
-  # directory.
+  liveLink = "/run/${runtimeDir}/current";
-  stagedCert = "/run/${runtimeDir}/cert.new";
+  certFile = "${liveLink}/cert";
-  stagedKey = "/run/${runtimeDir}/key.new";
+  keyFile = "${liveLink}/key";
   # The store's whole answer, key included, in the unit's own `0700` directory.
   rawFile = "/run/${runtimeDir}/bao-mtls.json";
   errFile = "/run/${runtimeDir}/bao.err";
@@ -196,14 +196,48 @@ in
         # is removed before the command that creates it.
         err=${lib.escapeShellArg errFile}
         raw=${lib.escapeShellArg rawFile}
-        trap 'rm -f "$err" "$raw" ${lib.escapeShellArg stagedCert} ${lib.escapeShellArg stagedKey}' EXIT
+        trap 'rm -f "$err" "$raw"' EXIT
+
+        live=${lib.escapeShellArg liveLink}
+        dir=''${live%/*}
+
+        # A fresh directory for one pair. `chmod`, because the umask cuts
+        # mktemp's `0700` down to a `0400` no file can be created in.
+        new_pair() {
+          local d
+          d=$(mktemp -d -p "$dir" pair.XXXXXX)
+          chmod 0700 "$d"
+          echo "$d"
+        }
+
+        # The only step that changes the live pair: one rename of the link. A
+        # reader whose two opens straddle it gets one file of each pair for
+        # that one login; the next login reads a whole pair.
+        go_live() {
+          rm -f "$live.new"
+          ln -s "''${1##*/}" "$live.new"
+          mv -Tf "$live.new" "$live"
+        }
+
+        # Every pair directory but the live one is left over: the pair the
+        # last swap replaced, kept until now for a reader that resolved the
+        # link before the swap, or a pair a stopped run staged.
+        current=$(readlink "$live" || true)
+        for d in "$dir"/pair.*; do
+          if [ -d "$d" ] && [ "''${d##*/}" != "$current" ]; then
+            chmod 0700 "$d"
+            rm -rf "$d"
+          fi
+        done
+        rm -f "$live.new"
 
         # The delivered pair is the start of every container's life: /run is
         # empty at boot and kept across this unit's runs.
         if [ ! -s "$BAO_CLIENT_CERT" ] || [ ! -s "$BAO_CLIENT_KEY" ]; then
-          rm -f "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY"
+          seed=$(new_pair)
-          cp "$CREDENTIALS_DIRECTORY/${keyCredential}" "$BAO_CLIENT_KEY"
+          cp "$CREDENTIALS_DIRECTORY/${keyCredential}" "$seed/key"
-          cp "$CREDENTIALS_DIRECTORY/${certCredential}" "$BAO_CLIENT_CERT"
+          cp "$CREDENTIALS_DIRECTORY/${certCredential}" "$seed/cert"
+          go_live "$seed"
         fi
 
         # Cert auth is a login, not a transport setting. The `BAO_CLIENT_*`
@@ -247,31 +281,30 @@ in
 
         # `-j`: the bytes as the store holds them, which is what the hive
         # delivered, so an unchanged leaf compares equal to the seeded copy.
-        rm -f ${lib.escapeShellArg stagedCert} ${lib.escapeShellArg stagedKey}
+        staged=$(new_pair)
-        if ! jq -ej '.data.data.cert | strings' "$raw" >${lib.escapeShellArg stagedCert} \
+        if ! jq -ej '.data.data.cert | strings' "$raw" >"$staged/cert" \
-          || ! jq -ej '.data.data.key | strings' "$raw" >${lib.escapeShellArg stagedKey}; then
+          || ! jq -ej '.data.data.key | strings' "$raw" >"$staged/key"; then
+          rm -rf "$staged"
           echo "${identityPath} holds no certificate and key; this agent keeps the one it has." >&2
           exit 1
         fi
 
-        if cmp -s ${lib.escapeShellArg stagedCert} "$BAO_CLIENT_CERT" \
+        if cmp -s "$staged/cert" "$BAO_CLIENT_CERT" && cmp -s "$staged/key" "$BAO_CLIENT_KEY"; then
-          && cmp -s ${lib.escapeShellArg stagedKey} "$BAO_CLIENT_KEY"; then
+          rm -rf "$staged"
           echo "authenticated to the swarm secret store at $BAO_ADDR as ${agentName}; its certificate at ${identityPath} is unchanged."
         else
-          # This unit's next login uses whatever is swapped in, so a pair the
+          # This unit's next login uses whatever goes live, so a pair the
           # store refuses must never replace the one it accepts.
           rm -f "$err"
-          if ! BAO_CLIENT_CERT=${lib.escapeShellArg stagedCert} BAO_CLIENT_KEY=${lib.escapeShellArg stagedKey} \
+          if ! BAO_CLIENT_CERT="$staged/cert" BAO_CLIENT_KEY="$staged/key" \
             bao login -method=cert -token-only >/dev/null 2>"$err"; then
+            rm -rf "$staged"
             echo "the swarm secret store refused the certificate at ${identityPath}; this agent keeps the one it has." >&2
             if [ -s "$err" ]; then cat "$err" >&2; fi
             exit 1
           fi
 
-          # Two renames: a reader that opens the pair between them gets a
+          go_live "$staged"
-          # mismatched cert and key and fails that one login.
-          mv -f ${lib.escapeShellArg stagedKey} "$BAO_CLIENT_KEY"
-          mv -f ${lib.escapeShellArg stagedCert} "$BAO_CLIENT_CERT"
           echo "replaced this agent's store certificate with the one at ${identityPath}."
         fi
       '';

nix/module-eval/agent-forge-bao.nix

@@ -84,8 +84,8 @@ let
           e = u.environment;
         in
         e.BAO_ADDR == baoAddr
-        && e.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert"
+        && e.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
-        && e.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key";
+        && e.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key";
     }
     {
       # The field is the secret. It goes to a file by redirect and is never
@@ -199,7 +199,7 @@ let
         in
         u.serviceConfig.User == name
         && u.serviceConfig.UMask == "0077"
-        && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert";
+        && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert";
     }
     {
       # Files a hive wrote keep working until the operator re-links them.

nix/module-eval/agent-github-bao.nix

@@ -83,8 +83,8 @@ let
         u.serviceConfig.User == name
         && u.serviceConfig.UMask == "0077"
         && u.environment.BAO_ADDR == baoAddr
-        && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert"
+        && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
-        && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key";
+        && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key";
     }
     {
       # A file a hive wrote keeps working until the operator links a token in

nix/module-eval/agent-matrix.nix

@@ -124,8 +124,8 @@ let
         in
         u.serviceConfig.LoadCredential == [ "hive-agent-bao-server-ca" ]
         && u.environment.BAO_ADDR == "https://bao.t.local:8200"
-        && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert"
+        && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
-        && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key"
+        && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key"
         && builtins.elem "hive-agent-bao-identity.service" u.after;
     }
     {

nix/module-eval/agent-queue-bao.nix

@@ -97,8 +97,8 @@ let
           u = agentBaoIdentity agentBao;
           e = u.environment;
         in
-        e.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert"
+        e.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
-        && e.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key"
+        && e.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key"
         && u.serviceConfig.RuntimeDirectory == "hive-agent-bao-identity"
         && u.serviceConfig.RuntimeDirectoryPreserve == "yes"
         && e.BAO_ADDR == agentBao.services.hyperhive.agent.bao.addr;
@@ -177,8 +177,8 @@ let
         let
           u = agentHarness agentBao;
         in
-        u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert"
+        u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
-        && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key"
+        && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key"
         && builtins.elem "hive-agent-bao-identity.service" u.after
         && u.environment.BAO_ADDR == agentBao.services.hyperhive.agent.bao.addr;
     }

nix/module-eval/agent-runtime.nix

@@ -116,8 +116,8 @@ let
             u.serviceConfig.LoadCredential == [ "hive-agent-bao-server-ca" ]
             && u.environment.HIVE_AGENT_NAME == "a1"
             && u.environment.BAO_ADDR == "https://bao.t.local:8200"
-            && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert"
+            && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
-            && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key"
+            && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key"
           )
           [
             (subagent claudeBao)
@@ -134,8 +134,8 @@ let
         u.serviceConfig.LoadCredential == [ "hive-agent-bao-server-ca" ]
         && u.environment.HIVE_AGENT_NAME == "a1"
         && u.environment.BAO_ADDR == "https://bao.t.local:8200"
-        && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/cert"
+        && u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
-        && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/key";
+        && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key";
     }
     {
       name = "an opencode agent with no store hands its subagent daemon no store identity";

nix/script-tests/agent-bao-fetch.sh

@@ -36,12 +36,14 @@ setup() {
   log=$dir/bao.log
   mkdir -m 0700 "$rt"
   mkdir "$creds"
+  # Per-case stubs that shadow the unit's own tools; empty unless a case adds one.
+  mkdir "$dir/bin"
   printf cert >"$creds/hive-agent-bao-cert"
   printf key >"$creds/hive-agent-bao-key"
   # The identity pair the unit presents: the delivered one for the fetch, the
   # live copy it keeps for the identity unit.
   if [ "$prefix" = IDENT ]; then
-    client_cert=$rt/cert client_key=$rt/key
+    client_cert=$rt/current/cert client_key=$rt/current/key
   else
     client_cert=$creds/hive-agent-bao-cert client_key=$creds/hive-agent-bao-key
   fi
@@ -69,7 +71,7 @@ go() {
     umask 0377
     cd "$dir" || exit 99
     exec env -i \
-      PATH="$FAKE_BAO_BIN:$unit_path" \
+      PATH="$dir/bin:$FAKE_BAO_BIN:$unit_path" \
       BAO_ADDR="$bao_addr" \
       BAO_CLIENT_CERT="$client_cert" \
       BAO_CLIENT_KEY="$client_key" \
@@ -249,63 +251,136 @@ ident_path=secret/swarm/agents/a1/bao-mtls
 ident_kv_call="kv get -format=json $ident_path"
 leaf() { printf '{"data":{"data":{"cert":"%s","key":"%s"}}}' "$1" "$2"; }
 
+expect_pair() {
+  expect_file "$rt/current/cert" "$1"
+  expect_file "$rt/current/key" "$2"
+}
+
+expect_pairs() {
+  local d n=0
+  for d in "$rt"/pair.*; do
+    if [ -d "$d" ]; then n=$((n + 1)); fi
+  done
+  if [ "$n" != "$1" ]; then fail "$n pair directories left, expected $1"; fi
+}
+
+# live_pair <name> <cert> <key> — a pair an earlier run made live.
+live_pair() {
+  mkdir -m 0700 "$rt/pair.$1"
+  printf '%s' "$2" >"$rt/pair.$1/cert"
+  printf '%s' "$3" >"$rt/pair.$1/key"
+  chmod 0400 "$rt/pair.$1/cert" "$rt/pair.$1/key"
+  ln -sfn "pair.$1" "$rt/current"
+}
+
+# stop_at <n> — the n-th `ln` or `mv` the script runs fails, which leaves
+# things as a run killed just before that step would.
+stop_at() {
+  local tool real
+  mkdir "$dir/steps"
+  for tool in ln mv; do
+    real=$(PATH=$unit_path command -v "$tool")
+    cat >"$dir/bin/$tool" <<STUB
+#!$BASH
+mktemp -p "$dir/steps" >/dev/null
+steps=("$dir/steps"/*)
+if [ "\${#steps[@]}" -ge "$1" ]; then exit 1; fi
+exec "$real" "\$@"
+STUB
+    chmod +x "$dir/bin/$tool"
+  done
+}
+
 setup IDENT "no store identity delivered"
 : >"$creds/hive-agent-bao-cert"
 go
 expect_rc 1
 expect_err "has no store identity"
 expect_calls
-expect_no_file "$rt/cert"
+if [ -e "$rt/current" ]; then fail "a live pair was made from nothing"; fi
 
-setup IDENT "a first run seeds the live pair from the delivered one"
+setup IDENT "a first run makes the delivered pair live"
 kv_out=$(leaf cert key)
 go
 expect_rc 0
 expect_out "is unchanged"
 expect_calls "$login_call" "$ident_kv_call"
-expect_file "$rt/cert" cert
+if [ ! -L "$rt/current" ]; then fail "$rt/current is not a symlink"; fi
-expect_file "$rt/key" key
+expect_pair cert key
-if [ "$(stat -c %a "$rt/key")" != 400 ]; then fail "key is mode $(stat -c %a "$rt/key"), expected 400"; fi
+if [ "$(stat -L -c %a "$rt/current/key")" != 400 ]; then fail "key is mode $(stat -L -c %a "$rt/current/key"), expected 400"; fi
+expect_pairs 1
 expect_no_file "$rt/bao-mtls.json"
 
 setup IDENT "the live pair outlives the delivered one"
-printf cert2 >"$rt/cert"
+live_pair old cert2 key2
-printf key2 >"$rt/key"
-chmod 0400 "$rt/cert" "$rt/key"
 kv_out=$(leaf cert2 key2)
 go
 expect_rc 0
 expect_out "is unchanged"
-expect_file "$rt/cert" cert2
+expect_pair cert2 key2
+expect_pairs 1
 
-setup IDENT "a re-issued leaf replaces the live pair once a login with it succeeds"
+setup IDENT "a re-issued leaf goes live once a login with it succeeds"
 kv_out=$(leaf cert2 key2)
 go
 expect_rc 0
 expect_out "replaced this agent's store certificate"
 expect_calls "$login_call" "$ident_kv_call" "$login_call"
-expect_file "$rt/cert" cert2
+expect_pair cert2 key2
-expect_file "$rt/key" key2
+# The replaced pair stays for a reader that resolved the link before the swap.
-expect_no_file "$rt/cert.new"
+expect_pairs 2
-expect_no_file "$rt/key.new"
+go
+expect_rc 0
+expect_out "is unchanged"
+expect_pairs 1
 
-setup IDENT "a re-issued leaf the store refuses is not swapped in"
+setup IDENT "a re-issued leaf the store refuses does not go live"
 kv_out=$(leaf cert2 key2)
 refuse_cert=cert2
 go
 expect_rc 1
 expect_err "refused the certificate at $ident_path"
-expect_file "$rt/cert" cert
+expect_pair cert key
-expect_file "$rt/key" key
+expect_pairs 1
-expect_no_file "$rt/cert.new"
 
 setup IDENT "an object without a key keeps the live pair"
 kv_out='{"data":{"data":{"cert":"cert2"}}}'
 go
 expect_rc 1
 expect_err "holds no certificate and key"
-expect_file "$rt/cert" cert
+expect_pair cert key
-expect_file "$rt/key" key
+expect_pairs 1
+
+setup IDENT "what a stopped run left behind is removed"
+live_pair old cert key
+mkdir -m 0700 "$rt/pair.stale"
+printf cert2 >"$rt/pair.stale/cert"
+ln -s pair.stale "$rt/current.new"
+kv_out=$(leaf cert key)
+go
+expect_rc 0
+expect_pair cert key
+expect_pairs 1
+if [ -L "$rt/current.new" ]; then fail "current.new left behind"; fi
+
+# The swap is atomic: wherever a run stops, the link names one whole pair,
+# and the next run finishes the job.
+for n in 1 2 3; do
+  setup IDENT "a swap stopped at its step $n leaves one whole pair live"
+  live_pair old cert key
+  kv_out=$(leaf cert2 key2)
+  stop_at "$n"
+  go
+  got="$(<"$rt/current/cert") $(<"$rt/current/key")"
+  case "$got" in
+    "cert key" | "cert2 key2") ;;
+    *) fail "the live pair is '$got', one file of each" ;;
+  esac
+  rm -f "$dir/bin/ln" "$dir/bin/mv"
+  go
+  expect_rc 0
+  expect_pair cert2 key2
+done
 
 if [ "$failures" -ne 0 ]; then
   echo "script-test-agent-bao-fetch: $failures failed assertions over $count runs" >&2