hive-agent: read the per-agent queue secret from bao in process
The harness now reads swarm/agents/<agent>/queue from the store itself, under the agent's own store certificate, and holds it in memory only. It reads once before the first connect and again on every reconnect attempt (async-nats `ConnectOptions::with_auth_callback`), so an agent whose secret was re-minted reconnects with the new value instead of being refused until the container restarts. hive-agent-queue-credential.service, the /run file it wrote, and HIVE_AGENT_QUEUE_AGENT_SECRET_FILE are gone; queue-identity.nix now hands hive-agent.service the store address, its certificate paths and the agent name. A failed or empty read before the first connect still falls back to the hive's shared client. Each read is bounded by a 10s timeout, and retries wait out the existing reconnect backoff (500ms doubling, capped at 60s). Closes #4783
This commit is contained in:
atlas
mara
parent
f7437a4773
commit
ccb5bd3b38
10 changed files with 770 additions and 529 deletions
|
|
@ -1,9 +1,8 @@
|
|||
# `checks.script-test-agent-bao-fetch` — runs the two agent units that log in
|
||||
# to the swarm secret store and fetch a secret, ../agent-modules/forge-token.nix
|
||||
# and ../agent-modules/queue-identity.nix, against a stub `bao`. The cases are
|
||||
# in ./agent-bao-fetch.sh.
|
||||
# `checks.script-test-agent-bao-fetch` — runs the agent unit that logs in to
|
||||
# the swarm secret store and fetches a secret, ../agent-modules/forge-token.nix,
|
||||
# against a stub `bao`. The cases are in ./agent-bao-fetch.sh.
|
||||
#
|
||||
# What runs is each unit's rendered `ExecStart`, on the unit's own `PATH` with
|
||||
# What runs is the unit's rendered `ExecStart`, on the unit's own `PATH` with
|
||||
# the stub in front. The one edit is the unit's `/run/<unit>/` prefix, moved
|
||||
# under the build directory because the sandbox has no writable `/run`.
|
||||
{
|
||||
|
|
@ -71,7 +70,6 @@ in
|
|||
pkgs.runCommand "hyperhive-script-test-agent-bao-fetch"
|
||||
(
|
||||
unitEnv "FORGE" "hive-agent-forge-token"
|
||||
// unitEnv "QUEUE" "hive-agent-queue-credential"
|
||||
// {
|
||||
FAKE_BAO_BIN = "${fakeBao}/bin";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ if (: >"$probe") 2>/dev/null; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# setup FORGE|QUEUE <description>
|
||||
# setup FORGE <description>
|
||||
setup() {
|
||||
prefix=$1
|
||||
case="$prefix: $2"
|
||||
|
|
@ -111,125 +111,108 @@ expect_no_file() {
|
|||
if [ -e "$1" ]; then fail "$1 left behind"; fi
|
||||
}
|
||||
|
||||
for prefix in FORGE QUEUE; do
|
||||
if [ "$prefix" = FORGE ]; then
|
||||
secret_name=token
|
||||
path=secret/swarm/agents/a1/forge-token
|
||||
missing_msg="no forge token at $path yet"
|
||||
else
|
||||
secret_name=secret
|
||||
path=secret/swarm/agents/a1/queue
|
||||
missing_msg="no per-agent queue credential at $path yet"
|
||||
fi
|
||||
login_call="login -method=cert -token-only"
|
||||
kv_call="kv get -field=value $path"
|
||||
secret_name=token
|
||||
path=secret/swarm/agents/a1/forge-token
|
||||
missing_msg="no forge token at $path yet"
|
||||
login_call="login -method=cert -token-only"
|
||||
kv_call="kv get -field=value $path"
|
||||
|
||||
setup "$prefix" "no store identity delivered"
|
||||
: >"$creds/hive-agent-bao-cert"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_err "has no store identity"
|
||||
expect_calls
|
||||
setup FORGE "no store identity delivered"
|
||||
: >"$creds/hive-agent-bao-cert"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_err "has no store identity"
|
||||
expect_calls
|
||||
|
||||
setup "$prefix" "a credential that cannot be read"
|
||||
chmod 0000 "$creds/hive-agent-bao-key"
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "cannot read $creds/hive-agent-bao-key"
|
||||
expect_calls
|
||||
setup FORGE "a credential that cannot be read"
|
||||
chmod 0000 "$creds/hive-agent-bao-key"
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "cannot read $creds/hive-agent-bao-key"
|
||||
expect_calls
|
||||
|
||||
setup "$prefix" "bao's stderr file cannot be created"
|
||||
chmod 0500 "$rt"
|
||||
go
|
||||
chmod 0700 "$rt"
|
||||
expect_rc 1
|
||||
expect_err "could not create $rt/bao.err, so bao never ran"
|
||||
expect_calls
|
||||
setup FORGE "bao's stderr file cannot be created"
|
||||
chmod 0500 "$rt"
|
||||
go
|
||||
chmod 0700 "$rt"
|
||||
expect_rc 1
|
||||
expect_err "could not create $rt/bao.err, so bao never ran"
|
||||
expect_calls
|
||||
|
||||
setup "$prefix" "the store refuses the login with an HTTP 4xx"
|
||||
login_rc=2
|
||||
login_err=$'Error authenticating: Error making API request.\n\nURL: PUT '"$bao_addr"$'/v1/auth/cert/login\nCode: 403. Errors:\n\n* permission denied\n'
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "refused this agent's certificate login with HTTP 403:"
|
||||
expect_err "* permission denied"
|
||||
expect_calls "$login_call"
|
||||
setup FORGE "the store refuses the login with an HTTP 4xx"
|
||||
login_rc=2
|
||||
login_err=$'Error authenticating: Error making API request.\n\nURL: PUT '"$bao_addr"$'/v1/auth/cert/login\nCode: 403. Errors:\n\n* permission denied\n'
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "refused this agent's certificate login with HTTP 403:"
|
||||
expect_err "* permission denied"
|
||||
expect_calls "$login_call"
|
||||
|
||||
setup "$prefix" "the store fails the login with another HTTP status"
|
||||
login_rc=2
|
||||
login_err=$'Error authenticating: Error making API request.\n\nCode: 500. Errors:\n\n* internal error\n'
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "failed this agent's certificate login with HTTP 500:"
|
||||
expect_err "* internal error"
|
||||
expect_calls "$login_call"
|
||||
setup FORGE "the store fails the login with another HTTP status"
|
||||
login_rc=2
|
||||
login_err=$'Error authenticating: Error making API request.\n\nCode: 500. Errors:\n\n* internal error\n'
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "failed this agent's certificate login with HTTP 500:"
|
||||
expect_err "* internal error"
|
||||
expect_calls "$login_call"
|
||||
|
||||
setup "$prefix" "the store refuses the certificate with a TLS alert"
|
||||
login_rc=2
|
||||
login_err="Error authenticating: Put \"$bao_addr/v1/auth/cert/login\": remote error: tls: unknown certificate authority"
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "refused this agent's certificate in the TLS handshake:"
|
||||
expect_err "remote error: tls: unknown certificate authority"
|
||||
expect_calls "$login_call"
|
||||
setup FORGE "the store refuses the certificate with a TLS alert"
|
||||
login_rc=2
|
||||
login_err="Error authenticating: Put \"$bao_addr/v1/auth/cert/login\": remote error: tls: unknown certificate authority"
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "refused this agent's certificate in the TLS handshake:"
|
||||
expect_err "remote error: tls: unknown certificate authority"
|
||||
expect_calls "$login_call"
|
||||
|
||||
setup "$prefix" "the store does not answer"
|
||||
login_rc=2
|
||||
login_err="Error authenticating: Put \"$bao_addr/v1/auth/cert/login\": dial tcp: lookup bao.t.local: no such host"
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "got no answer from the swarm secret store at $bao_addr"
|
||||
expect_err "no such host"
|
||||
expect_calls "$login_call"
|
||||
setup FORGE "the store does not answer"
|
||||
login_rc=2
|
||||
login_err="Error authenticating: Put \"$bao_addr/v1/auth/cert/login\": dial tcp: lookup bao.t.local: no such host"
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "got no answer from the swarm secret store at $bao_addr"
|
||||
expect_err "no such host"
|
||||
expect_calls "$login_call"
|
||||
|
||||
# Only the forge unit keeps its runtime directory between runs; the queue
|
||||
# unit starts each run with an empty one.
|
||||
setup "$prefix" "nothing minted at the agent's path yet"
|
||||
if [ "$prefix" = FORGE ]; then
|
||||
printf old >"$rt/$secret_name"
|
||||
chmod 0400 "$rt/$secret_name"
|
||||
fi
|
||||
kv_rc=2
|
||||
kv_err="No value found at secret/data/swarm/agents/a1"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_err "$missing_msg"
|
||||
expect_err "No value found"
|
||||
expect_calls "$login_call" "$kv_call"
|
||||
if [ "$prefix" = FORGE ]; then
|
||||
expect_file "$rt/$secret_name" old
|
||||
else
|
||||
expect_no_file "$rt/$secret_name"
|
||||
fi
|
||||
# The unit keeps its runtime directory between runs.
|
||||
setup FORGE "nothing minted at the agent's path yet"
|
||||
printf old >"$rt/$secret_name"
|
||||
chmod 0400 "$rt/$secret_name"
|
||||
kv_rc=2
|
||||
kv_err="No value found at secret/data/swarm/agents/a1"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_err "$missing_msg"
|
||||
expect_err "No value found"
|
||||
expect_calls "$login_call" "$kv_call"
|
||||
expect_file "$rt/$secret_name" old
|
||||
|
||||
setup "$prefix" "a first fetch writes the secret 0400"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "fetched this agent's"
|
||||
expect_no_err "Permission denied"
|
||||
expect_calls "$login_call" "$kv_call"
|
||||
expect_file "$rt/$secret_name" the-secret
|
||||
if [ "$(stat -c %a "$rt/$secret_name")" != 400 ]; then fail "$secret_name is mode $(stat -c %a "$rt/$secret_name"), expected 400"; fi
|
||||
expect_no_file "$rt/bao.err"
|
||||
expect_no_file "$rt/token.new"
|
||||
setup FORGE "a first fetch writes the secret 0400"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "fetched this agent's"
|
||||
expect_no_err "Permission denied"
|
||||
expect_calls "$login_call" "$kv_call"
|
||||
expect_file "$rt/$secret_name" the-secret
|
||||
if [ "$(stat -c %a "$rt/$secret_name")" != 400 ]; then fail "$secret_name is mode $(stat -c %a "$rt/$secret_name"), expected 400"; fi
|
||||
expect_no_file "$rt/bao.err"
|
||||
expect_no_file "$rt/token.new"
|
||||
|
||||
# `UMask=0377` makes every file the script creates 0400, the login's own
|
||||
# `bao.err` included, and a redirect into a 0400 file fails before bao starts.
|
||||
setup "$prefix" "0400 files already in place do not block the fetch"
|
||||
printf stale >"$rt/bao.err"
|
||||
chmod 0400 "$rt/bao.err"
|
||||
if [ "$prefix" = FORGE ]; then
|
||||
printf stale >"$rt/token.new"
|
||||
chmod 0400 "$rt/token.new"
|
||||
fi
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "fetched this agent's"
|
||||
expect_no_err "Permission denied"
|
||||
expect_no_err "refused"
|
||||
expect_calls "$login_call" "$kv_call"
|
||||
expect_file "$rt/$secret_name" the-secret
|
||||
done
|
||||
# `UMask=0377` makes every file the script creates 0400, the login's own
|
||||
# `bao.err` included, and a redirect into a 0400 file fails before bao starts.
|
||||
setup FORGE "0400 files already in place do not block the fetch"
|
||||
printf stale >"$rt/bao.err"
|
||||
chmod 0400 "$rt/bao.err"
|
||||
printf stale >"$rt/token.new"
|
||||
chmod 0400 "$rt/token.new"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "fetched this agent's"
|
||||
expect_no_err "Permission denied"
|
||||
expect_no_err "refused"
|
||||
expect_calls "$login_call" "$kv_call"
|
||||
expect_file "$rt/$secret_name" the-secret
|
||||
|
||||
setup FORGE "an unchanged token is left in place"
|
||||
printf the-secret >"$rt/token"
|
||||
|
|
|
|||
Loading…
Reference in a new issue