vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

agents: swap the agent's bao leaf in with one rename

Browse source 
The live pair was replaced with two renames, key then cert. A run
stopped between them (timeout, container stop, power loss) left a new
key beside the old cert, and every later run logged in with that pair,
so the agent could not re-pull until its container restarted.

Each pair now goes into a fresh directory under
/run/hive-agent-bao-identity, and `current`, a symlink to it, is
replaced by one rename. Wherever a run stops, `current` names one whole
pair; the next run removes whatever a stopped run left. Consumers read
`current/cert` and `current/key` through the same read-only options and
still re-read them on every login.

The script test covers a swap stopped at each of its link and rename
steps: the live pair is always one whole pair, and the next run puts
the new one live.

Refs #5017
This commit is contained in:
atlas 2026-10-11 03:24:49 +02:00 • committed by mara
commit 4c4ecb8ed9
7 changed files with 166 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

75
nix/agent-modules/bao.nix
View file

@ -50,12 +50,12 @@ let
unitName = "hive-agent-bao-identity";
runtimeDir = unitName;
certFile = "/run/${runtimeDir}/cert";
keyFile = "/run/${runtimeDir}/key";
# Beside the live pair, so each rename that replaces it stays in one
# directory.
stagedCert = "/run/${runtimeDir}/cert.new";
stagedKey = "/run/${runtimeDir}/key.new";
# A symlink to the directory that holds the live pair. Each pair is written
# into a fresh directory and goes live with one rename of this link, so the
# link always names one whole pair, old or new.
liveLink = "/run/${runtimeDir}/current";
certFile = "${liveLink}/cert";
keyFile = "${liveLink}/key";
# The store's whole answer, key included, in the unit's own `0700` directory.
rawFile = "/run/${runtimeDir}/bao-mtls.json";
errFile = "/run/${runtimeDir}/bao.err";
@ -196,14 +196,48 @@ in
# is removed before the command that creates it.
err=${lib.escapeShellArg errFile}
raw=${lib.escapeShellArg rawFile}
trap 'rm -f "$err" "$raw" ${lib.escapeShellArg stagedCert} ${lib.escapeShellArg stagedKey}' EXIT
trap 'rm -f "$err" "$raw"' EXIT
live=${lib.escapeShellArg liveLink}
dir=''${live%/*}
# A fresh directory for one pair. `chmod`, because the umask cuts
# mktemp's `0700` down to a `0400` no file can be created in.
new_pair() {
local d
d=$(mktemp -d -p "$dir" pair.XXXXXX)
chmod 0700 "$d"
echo "$d"
}
# The only step that changes the live pair: one rename of the link. A
# reader whose two opens straddle it gets one file of each pair for
# that one login; the next login reads a whole pair.
go_live() {
rm -f "$live.new"
ln -s "''${1##*/}" "$live.new"
mv -Tf "$live.new" "$live"
}
# Every pair directory but the live one is left over: the pair the
# last swap replaced, kept until now for a reader that resolved the
# link before the swap, or a pair a stopped run staged.
current=$(readlink "$live" || true)
for d in "$dir"/pair.*; do
if [ -d "$d" ] && [ "''${d##*/}" != "$current" ]; then
chmod 0700 "$d"
rm -rf "$d"
fi
done
rm -f "$live.new"
# The delivered pair is the start of every container's life: /run is
# empty at boot and kept across this unit's runs.
if [ ! -s "$BAO_CLIENT_CERT" ] || [ ! -s "$BAO_CLIENT_KEY" ]; then
rm -f "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY"
cp "$CREDENTIALS_DIRECTORY/${keyCredential}" "$BAO_CLIENT_KEY"
cp "$CREDENTIALS_DIRECTORY/${certCredential}" "$BAO_CLIENT_CERT"
seed=$(new_pair)
cp "$CREDENTIALS_DIRECTORY/${keyCredential}" "$seed/key"
cp "$CREDENTIALS_DIRECTORY/${certCredential}" "$seed/cert"
go_live "$seed"
fi
# Cert auth is a login, not a transport setting. The `BAO_CLIENT_*`
@ -247,31 +281,30 @@ in
# `-j`: the bytes as the store holds them, which is what the hive
# delivered, so an unchanged leaf compares equal to the seeded copy.
rm -f ${lib.escapeShellArg stagedCert} ${lib.escapeShellArg stagedKey}
if ! jq -ej '.data.data.cert | strings' "$raw" >${lib.escapeShellArg stagedCert} \
|| ! jq -ej '.data.data.key | strings' "$raw" >${lib.escapeShellArg stagedKey}; then
staged=$(new_pair)
if ! jq -ej '.data.data.cert | strings' "$raw" >"$staged/cert" \
|| ! jq -ej '.data.data.key | strings' "$raw" >"$staged/key"; then
rm -rf "$staged"
echo "${identityPath} holds no certificate and key; this agent keeps the one it has." >&2
exit 1
fi
if cmp -s ${lib.escapeShellArg stagedCert} "$BAO_CLIENT_CERT" \
&& cmp -s ${lib.escapeShellArg stagedKey} "$BAO_CLIENT_KEY"; then
if cmp -s "$staged/cert" "$BAO_CLIENT_CERT" && cmp -s "$staged/key" "$BAO_CLIENT_KEY"; then
rm -rf "$staged"
echo "authenticated to the swarm secret store at $BAO_ADDR as ${agentName}; its certificate at ${identityPath} is unchanged."
else
# This unit's next login uses whatever is swapped in, so a pair the
# This unit's next login uses whatever goes live, so a pair the
# store refuses must never replace the one it accepts.
rm -f "$err"
if ! BAO_CLIENT_CERT=${lib.escapeShellArg stagedCert} BAO_CLIENT_KEY=${lib.escapeShellArg stagedKey} \
if ! BAO_CLIENT_CERT="$staged/cert" BAO_CLIENT_KEY="$staged/key" \
bao login -method=cert -token-only >/dev/null 2>"$err"; then
rm -rf "$staged"
echo "the swarm secret store refused the certificate at ${identityPath}; this agent keeps the one it has." >&2
if [ -s "$err" ]; then cat "$err" >&2; fi
exit 1
fi
# Two renames: a reader that opens the pair between them gets a
# mismatched cert and key and fails that one login.
mv -f ${lib.escapeShellArg stagedKey} "$BAO_CLIENT_KEY"
mv -f ${lib.escapeShellArg stagedCert} "$BAO_CLIENT_CERT"
go_live "$staged"
echo "replaced this agent's store certificate with the one at ${identityPath}."
fi
'';