fetch the agent's backend env file from the store, seeded from the existing file
the agent's own root unit reads swarm/agents/<agent>/backend-env into /run and creates it from backendEnvironmentFile with -cas=0 when the store holds nothing. the shared agent policy gains create (not update) on that one leaf. the harness and the subagent daemon load the old file first and the fetched one after it.
This commit is contained in:
atlas
parent
5486fc0333
commit
75e9e205e3
9 changed files with 933 additions and 31 deletions
|
|
@ -201,6 +201,15 @@ in
|
|||
and on an agent with `useApiKey` set, so its subagents authenticate
|
||||
the same way the harness does.
|
||||
|
||||
On an agent with a store (`services.hyperhive.agent.bao.addr`) that uses
|
||||
`useApiKey` or the `acp` runtime, `hive-agent-backend-env.service` also
|
||||
keeps this file's text at `swarm/agents/<agent>/backend-env`, field `env`.
|
||||
A leaf nobody has written is created from this file once, with the same
|
||||
bytes, and this file is never rewritten or deleted. The harness and the
|
||||
subagent daemon load this file first and the store's copy after it, so
|
||||
a variable the store holds wins and this file carries the agent while
|
||||
the store is down. Both read at their own start.
|
||||
|
||||
On an ACP agent using the `opencode` preset, a provider key this file
|
||||
leaves unset is read from the swarm secret store at
|
||||
`swarm/agents/<agent>/acp-provider`, field `api_key`, if one is stored.
|
||||
|
|
@ -552,10 +561,11 @@ in
|
|||
User = userName;
|
||||
Group = userName;
|
||||
}
|
||||
// lib.optionalAttrs (config.services.hyperhive.agent.backendEnvironmentFile != null) {
|
||||
// lib.optionalAttrs (config.services.hyperhive.agent.backendEnvironment.environmentFiles != [ ]) {
|
||||
# See `services.hyperhive.agent.backendEnvironmentFile`'s own description for
|
||||
# the file shape and the leading-`-` rationale.
|
||||
EnvironmentFile = "-${config.services.hyperhive.agent.backendEnvironmentFile}";
|
||||
# the file shape and the leading-`-` rationale; ./backend-env.nix for
|
||||
# the second entry.
|
||||
EnvironmentFile = config.services.hyperhive.agent.backendEnvironment.environmentFiles;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
|
|
|||
205
nix/agent-modules/backend-env.nix
Normal file
205
nix/agent-modules/backend-env.nix
Normal file
|
|
@ -0,0 +1,205 @@
|
|||
# This agent's backend environment (`ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`,
|
||||
# an ACP provider key), fetched from the swarm secret store by the agent
|
||||
# itself into the file the harness and the subagent daemon load as
|
||||
# `EnvironmentFile=`.
|
||||
#
|
||||
# The store holds the file's text at `swarm/agents/<agent>/backend-env`, field
|
||||
# `env` (`swarm_secret_client::backend_env`). Each run does one of three things:
|
||||
#
|
||||
# - the store has a value: it is written to `/run/hive-agent-backend-env/env`;
|
||||
# - the store has none and `backendEnvironmentFile` is a non-empty file: the
|
||||
# file's exact bytes are created at the leaf with `-cas=0`, then read back
|
||||
# and written the same way. A run that loses the race to another writer, or
|
||||
# whose create the store's policy refuses, uses whatever the read-back finds;
|
||||
# - neither: nothing is written, which is an agent with no backend file.
|
||||
#
|
||||
# It never rotates a value and never touches `backendEnvironmentFile`. That
|
||||
# file stays an `EnvironmentFile=` of its own, listed before the fetched one:
|
||||
# systemd lets a later file override an earlier one, so the store's value wins
|
||||
# when there is one, and the file carries the agent when the store is down.
|
||||
#
|
||||
# The unit runs as root because the file is root-owned `0600` where an operator
|
||||
# placed it. Root can still read the agent's `0400` store identity.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
agentCfg = config.services.hyperhive.agent;
|
||||
cfg = agentCfg.bao;
|
||||
|
||||
agentName = agentCfg.user.name;
|
||||
|
||||
# The server CA id ./bao.nix loads.
|
||||
serverCaCredential = "hive-agent-bao-server-ca";
|
||||
|
||||
unitName = "hive-agent-backend-env";
|
||||
|
||||
# The nix half of `swarm_secret_client::backend_env::env_path` plus
|
||||
# `path::MOUNT`.
|
||||
leaf = "secret/swarm/agents/${agentName}/backend-env";
|
||||
|
||||
runtimeDir = unitName;
|
||||
envFile = "/run/${runtimeDir}/env";
|
||||
stagedFile = "/run/${runtimeDir}/env.new";
|
||||
errFile = "/run/${runtimeDir}/bao.err";
|
||||
|
||||
# An OAuth claude agent has no backend credentials to fetch.
|
||||
fetches = cfg.addr != null && (agentCfg.useApiKey || agentCfg.runtime == "acp");
|
||||
|
||||
earlier = agentCfg.backendEnvironmentFile;
|
||||
|
||||
storeRetry = import ../host-modules/lib/store-retry.nix { };
|
||||
in
|
||||
{
|
||||
options.services.hyperhive.agent.backendEnvironment.environmentFiles = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
readOnly = true;
|
||||
default = lib.optional (earlier != null) "-${earlier}" ++ lib.optional fetches "-${envFile}";
|
||||
description = ''
|
||||
The `EnvironmentFile=` entries that carry this agent's backend
|
||||
environment, in load order: `backendEnvironmentFile` when set, then the
|
||||
file `${unitName}.service` keeps when the agent has a store and uses an
|
||||
API key. Read-only: the harness and the subagent daemon take it as it is.
|
||||
'';
|
||||
};
|
||||
|
||||
config = lib.mkIf fetches {
|
||||
systemd.services.${unitName} = {
|
||||
description = "fetch this agent's backend environment from the secret store";
|
||||
after = [
|
||||
"network.target"
|
||||
"hive-agent-bao-identity.service"
|
||||
];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
pkgs.openbao
|
||||
pkgs.coreutils
|
||||
pkgs.diffutils
|
||||
pkgs.gnugrep
|
||||
];
|
||||
# ../host-modules/lib/store-retry.nix.
|
||||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||||
serviceConfig = storeRetry.serviceConfig // {
|
||||
Type = "oneshot";
|
||||
# Not `RemainAfterExit`, so the timer below can start it again.
|
||||
# `RuntimeDirectoryPreserve` keeps the file between runs and after the
|
||||
# last one: the consumers read it at their own start.
|
||||
RemainAfterExit = false;
|
||||
TimeoutStartSec = 60;
|
||||
RuntimeDirectory = runtimeDir;
|
||||
RuntimeDirectoryMode = "0700";
|
||||
RuntimeDirectoryPreserve = "yes";
|
||||
# `0600`, the mode of the file an operator placed.
|
||||
UMask = "0077";
|
||||
LoadCredential = [ serverCaCredential ];
|
||||
};
|
||||
environment = {
|
||||
BAO_ADDR = cfg.addr;
|
||||
BAO_CLIENT_CERT = cfg.clientCertFile;
|
||||
BAO_CLIENT_KEY = cfg.clientKeyFile;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
# No identity delivered: ./bao.nix's check reports that.
|
||||
for f in "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY"; do
|
||||
if [ ! -s "$f" ]; then
|
||||
echo "this agent has no store identity, so it cannot fetch its backend environment." >&2
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then
|
||||
export BAO_CACERT="$CREDENTIALS_DIRECTORY/${serverCaCredential}"
|
||||
fi
|
||||
|
||||
err=${lib.escapeShellArg errFile}
|
||||
staged=${lib.escapeShellArg stagedFile}
|
||||
live=${lib.escapeShellArg envFile}
|
||||
earlier=${lib.escapeShellArg (if earlier == null then "" else earlier)}
|
||||
trap 'rm -f "$err" "$staged"' EXIT
|
||||
|
||||
# Bounded, so a store that drops packets cannot hold the unit past its
|
||||
# start timeout.
|
||||
store() { timeout 15 bao "$@"; }
|
||||
|
||||
if ! BAO_TOKEN="$(store login -method=cert -token-only 2>"$err")"; then
|
||||
echo "the swarm secret store at $BAO_ADDR did not accept this agent's certificate login:" >&2
|
||||
if [ -s "$err" ]; then cat "$err" >&2; fi
|
||||
exit 1
|
||||
fi
|
||||
export BAO_TOKEN
|
||||
|
||||
# Sets READ to have, missing or unusable, and leaves the value in
|
||||
# $staged. `kv get` exits 2 with "No value found at <path>" for a leaf
|
||||
# nobody wrote, and 2 again for a refused read, so the message tells
|
||||
# them apart. The value goes from the store's answer into a file and
|
||||
# is never in a variable or an argument.
|
||||
read_leaf() {
|
||||
rm -f "$staged"
|
||||
if store kv get -field=env ${lib.escapeShellArg leaf} >"$staged" 2>"$err"; then
|
||||
if [ -s "$staged" ]; then READ=have; else READ=unusable; fi
|
||||
elif grep -q '^No value found at ' "$err"; then
|
||||
READ=missing
|
||||
elif grep -q '^Field "env" not present' "$err"; then
|
||||
READ=unusable
|
||||
else
|
||||
echo "could not read ${leaf}:" >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
read_leaf
|
||||
if [ "$READ" = missing ]; then
|
||||
if [ -n "$earlier" ] && [ -s "$earlier" ]; then
|
||||
echo "${leaf} holds nothing; creating it from $earlier."
|
||||
if ! store kv put -cas=0 ${lib.escapeShellArg leaf} env=@"$earlier" >/dev/null 2>"$err"; then
|
||||
echo "could not create ${leaf}; reading it back in case another run did:" >&2
|
||||
cat "$err" >&2
|
||||
fi
|
||||
read_leaf
|
||||
else
|
||||
echo "${leaf} holds nothing and there is no file to create it from."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
if [ "$READ" != have ]; then
|
||||
echo "${leaf} holds no usable env; the file in place, if any, is used." >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if cmp -s "$staged" "$live"; then
|
||||
echo "this agent's backend environment at ${leaf} is unchanged."
|
||||
exit 0
|
||||
fi
|
||||
mv -f "$staged" "$live"
|
||||
echo "fetched this agent's backend environment from ${leaf}."
|
||||
'';
|
||||
};
|
||||
|
||||
# The same cadence as ./github-token.nix.
|
||||
systemd.timers.${unitName} = {
|
||||
description = "re-fetch this agent's backend environment from the secret store";
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnUnitInactiveSec = "2min";
|
||||
RandomizedDelaySec = "20s";
|
||||
};
|
||||
};
|
||||
|
||||
# `wants`, not `requires`: a store outage must not stop the harness, whose
|
||||
# `backendEnvironmentFile` still loads. `after` waits for one attempt.
|
||||
systemd.services.hive-agent = {
|
||||
wants = [ "${unitName}.service" ];
|
||||
after = [ "${unitName}.service" ];
|
||||
};
|
||||
systemd.services.hive-subagent-daemon = {
|
||||
wants = [ "${unitName}.service" ];
|
||||
after = [ "${unitName}.service" ];
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -66,6 +66,7 @@ in
|
|||
imports = [
|
||||
./agent-service.nix
|
||||
./bao.nix
|
||||
./backend-env.nix
|
||||
./bash-env.nix
|
||||
./claude-settings.nix
|
||||
./dashboard-links.nix
|
||||
|
|
|
|||
|
|
@ -479,14 +479,14 @@ in
|
|||
lib.optionalAttrs
|
||||
(
|
||||
(config.services.hyperhive.agent.runtime == "acp" || config.services.hyperhive.agent.useApiKey)
|
||||
&& config.services.hyperhive.agent.backendEnvironmentFile != null
|
||||
&& config.services.hyperhive.agent.backendEnvironment.environmentFiles != [ ]
|
||||
)
|
||||
{
|
||||
# A subagent authenticates with the same credentials as the
|
||||
# harness: an ACP subagent to its provider, a claude subagent of
|
||||
# an api-key agent with `ANTHROPIC_API_KEY`, as it has no OAuth
|
||||
# session to fall back on.
|
||||
EnvironmentFile = "-${config.services.hyperhive.agent.backendEnvironmentFile}";
|
||||
EnvironmentFile = config.services.hyperhive.agent.backendEnvironment.environmentFiles;
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -184,6 +184,12 @@ in
|
|||
inherit pkgs self nixosSystem;
|
||||
inherit (pkgs) lib;
|
||||
};
|
||||
# Runs ./agent-modules/backend-env.nix's fetch-and-seed unit against a stub
|
||||
# `bao` that keeps its leaves in files.
|
||||
script-test-agent-backend-env = import ./script-tests/agent-backend-env.nix {
|
||||
inherit pkgs self nixosSystem;
|
||||
inherit (pkgs) lib;
|
||||
};
|
||||
# Runs ./agent-modules/hive-facts.nix's generator and login-shell snippet
|
||||
# against sample facts files.
|
||||
script-test-agent-hive-facts = import ./script-tests/agent-hive-facts.nix {
|
||||
|
|
|
|||
|
|
@ -61,6 +61,14 @@ let
|
|||
};
|
||||
opencode = opencodeWith { };
|
||||
opencodeBao = opencodeWith { bao.addr = "https://bao.t.local:8200"; };
|
||||
claudeApiKeyBao = agentWith {
|
||||
services.hyperhive.agent = {
|
||||
runtime = "claude";
|
||||
useApiKey = true;
|
||||
backendEnvironmentFile = backendEnv;
|
||||
bao.addr = "https://bao.t.local:8200";
|
||||
};
|
||||
};
|
||||
claudeBao = agentWith {
|
||||
services.hyperhive.agent = {
|
||||
runtime = "claude";
|
||||
|
|
@ -133,13 +141,13 @@ let
|
|||
}
|
||||
{
|
||||
name = "an ACP agent's subagent daemon loads the backend credentials";
|
||||
ok = (subagent acp).serviceConfig.EnvironmentFile or null == "-${backendEnv}";
|
||||
ok = (subagent acp).serviceConfig.EnvironmentFile or null == [ "-${backendEnv}" ];
|
||||
}
|
||||
{
|
||||
# The nested `claude` inherits the daemon's environment, so this file
|
||||
# is where its `ANTHROPIC_API_KEY` comes from.
|
||||
name = "an api-key claude agent's subagent daemon loads the backend credentials";
|
||||
ok = (subagent claudeApiKey).serviceConfig.EnvironmentFile or null == "-${backendEnv}";
|
||||
ok = (subagent claudeApiKey).serviceConfig.EnvironmentFile or null == [ "-${backendEnv}" ];
|
||||
}
|
||||
{
|
||||
# Unset is what `RuntimeSpec` reads as claude, and a claude subagent
|
||||
|
|
@ -149,6 +157,79 @@ let
|
|||
lib.all (var: (subagent claude).environment.${var} or null == null) runtimeVars
|
||||
&& !((subagent claude).serviceConfig ? EnvironmentFile);
|
||||
}
|
||||
{
|
||||
# Later files override earlier ones, so the operator's file carries the
|
||||
# agent when the store is down and the store's copy wins when it is up.
|
||||
name = "an api-key agent with a store loads its backend file, then the store's copy, in the harness and the subagent daemon";
|
||||
ok =
|
||||
let
|
||||
want = [
|
||||
"-${backendEnv}"
|
||||
"-/run/hive-agent-backend-env/env"
|
||||
];
|
||||
in
|
||||
(harness claudeApiKeyBao).serviceConfig.EnvironmentFile == want
|
||||
&& (subagent claudeApiKeyBao).serviceConfig.EnvironmentFile == want
|
||||
&& (harness acpBao).serviceConfig.EnvironmentFile == [ "-/run/hive-agent-backend-env/env" ]
|
||||
&& (subagent acpBao).serviceConfig.EnvironmentFile == [ "-/run/hive-agent-backend-env/env" ];
|
||||
}
|
||||
{
|
||||
# The case above is the presence control: the same lookup finds the
|
||||
# fetched file where it is loaded.
|
||||
name = "an agent with no store, or a claude agent on OAuth, loads no fetched backend file";
|
||||
ok =
|
||||
let
|
||||
fetched =
|
||||
machine:
|
||||
lib.elem "-/run/hive-agent-backend-env/env" (
|
||||
lib.toList (harness machine).serviceConfig.EnvironmentFile or [ ]
|
||||
);
|
||||
in
|
||||
!(fetched claudeApiKey)
|
||||
&& !(fetched acp)
|
||||
&& !(fetched opencode)
|
||||
&& !(fetched claudeBao)
|
||||
&& !(claudeBao.systemd.services ? hive-agent-backend-env)
|
||||
&& !(claudeApiKey.systemd.services ? hive-agent-backend-env)
|
||||
&& claudeApiKeyBao.systemd.services ? hive-agent-backend-env
|
||||
&& claudeApiKeyBao.systemd.timers ? hive-agent-backend-env;
|
||||
}
|
||||
{
|
||||
name = "the backend environment fetch runs as root, keeps its file, and the consumers want it without requiring it";
|
||||
ok =
|
||||
let
|
||||
u = claudeApiKeyBao.systemd.services.hive-agent-backend-env;
|
||||
orders =
|
||||
unit:
|
||||
lib.elem "hive-agent-backend-env.service" unit.wants
|
||||
&& lib.elem "hive-agent-backend-env.service" unit.after
|
||||
&& !(lib.elem "hive-agent-backend-env.service" unit.requires);
|
||||
in
|
||||
!(u.serviceConfig ? User)
|
||||
&& u.serviceConfig.UMask == "0077"
|
||||
&& u.serviceConfig.RuntimeDirectoryPreserve == "yes"
|
||||
&& u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
|
||||
&& orders (harness claudeApiKeyBao)
|
||||
&& orders (subagent claudeApiKeyBao)
|
||||
&& claudeApiKeyBao.systemd.timers.hive-agent-backend-env.timerConfig.OnUnitInactiveSec == "2min";
|
||||
}
|
||||
{
|
||||
# The nix half of `swarm_secret_client::backend_env::env_path`, and the
|
||||
# three promises the unit makes about the operator's file.
|
||||
name = "the backend environment fetch reads the agent's own leaf, seeds it create-only from the operator's file and never writes that file";
|
||||
ok =
|
||||
let
|
||||
s = claudeApiKeyBao.systemd.services.hive-agent-backend-env.script;
|
||||
in
|
||||
lib.hasInfix "kv get -field=env secret/swarm/agents/a1/backend-env" s
|
||||
&& lib.hasInfix "kv put -cas=0 secret/swarm/agents/a1/backend-env env=@\"$earlier\"" s
|
||||
&& lib.hasInfix "earlier=${backendEnv}" s
|
||||
&& !(lib.hasInfix "kv put secret" s)
|
||||
&& !(lib.hasInfix "> \"$earlier\"" s)
|
||||
&& !(lib.hasInfix ">\"$earlier\"" s)
|
||||
&& !(lib.hasInfix "rm -f \"$earlier\"" s)
|
||||
&& !(lib.hasInfix "mv -f \"$staged\" \"$earlier\"" s);
|
||||
}
|
||||
];
|
||||
in
|
||||
runGroup "agent-runtime" cases
|
||||
|
|
|
|||
77
nix/script-tests/agent-backend-env-fake-bao.sh
Executable file
77
nix/script-tests/agent-backend-env-fake-bao.sh
Executable file
|
|
@ -0,0 +1,77 @@
|
|||
#!/usr/bin/env bash
|
||||
# The stub `bao` for ./agent-backend-env.nix. Leaves are files under
|
||||
# `$FAKE_BAO_KV`, one per path. Every call is logged, a `kv` call without the
|
||||
# token `login` printed fails, and the answers are the ones openbao gives:
|
||||
# `kv get` of a leaf nobody wrote exits 2 with "No value found at <path>", and
|
||||
# a policy granting `create` and `read` only refuses a second `-cas=0` write
|
||||
# with a 403.
|
||||
#
|
||||
# `FAKE_BAO_LOGIN_RC` makes the login fail, `FAKE_BAO_DENY_READ` and
|
||||
# `FAKE_BAO_DENY_WRITE` answer 403 as a policy that grants neither does, and
|
||||
# `FAKE_BAO_RACE` has another writer create the leaf with that text between
|
||||
# this run's first read and its write.
|
||||
echo "$*" >>"$FAKE_BAO_LOG"
|
||||
case "$1 $2" in
|
||||
"login -method=cert")
|
||||
[ "${FAKE_BAO_LOGIN_RC-0}" = 0 ] || {
|
||||
echo 'Error authenticating: Put "https://bao.t.local:8200/v1/auth/cert/login": dial tcp: lookup bao.t.local: no such host' >&2
|
||||
exit 2
|
||||
}
|
||||
printf 's.fake'
|
||||
;;
|
||||
"kv get" | "kv put")
|
||||
[ "${BAO_TOKEN-}" = s.fake ] || {
|
||||
echo "fake bao: kv called without the login's token" >&2
|
||||
exit 97
|
||||
}
|
||||
;;&
|
||||
"kv get")
|
||||
[ "$3" = -field=env ] || {
|
||||
echo "fake bao: get without -field=env: $*" >&2
|
||||
exit 95
|
||||
}
|
||||
leaf=$FAKE_BAO_KV/$(printf '%s' "$4" | tr / _)
|
||||
[ -z "${FAKE_BAO_DENY_READ-}" ] || {
|
||||
printf 'Error reading %s: Error making API request.\n\nURL: GET https://bao.t.local:8200/v1/%s\nCode: 403. Errors:\n\n* 1 error occurred:\n\t* permission denied\n' "$4" "$4" >&2
|
||||
exit 2
|
||||
}
|
||||
if [ -n "${FAKE_BAO_RACE-}" ] && [ ! -e "$FAKE_BAO_KV/.raced" ]; then
|
||||
: >"$FAKE_BAO_KV/.raced"
|
||||
printf '%s' "$FAKE_BAO_RACE" >"$leaf"
|
||||
echo "No value found at ${4/secret\//secret/data/}" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [ ! -e "$leaf" ]; then
|
||||
echo "No value found at ${4/secret\//secret/data/}" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [ "$(cat "$leaf.field" 2>/dev/null)" = other ]; then
|
||||
echo 'Field "env" not present in secret' >&2
|
||||
exit 1
|
||||
fi
|
||||
cat "$leaf"
|
||||
;;
|
||||
"kv put")
|
||||
[ "$3" = -cas=0 ] || {
|
||||
echo "fake bao: put without -cas=0: $*" >&2
|
||||
exit 96
|
||||
}
|
||||
leaf=$FAKE_BAO_KV/$(printf '%s' "$4" | tr / _)
|
||||
case "$5" in
|
||||
env=@*) ;;
|
||||
*)
|
||||
echo "fake bao: put of something other than env=@file: $*" >&2
|
||||
exit 94
|
||||
;;
|
||||
esac
|
||||
if [ -n "${FAKE_BAO_DENY_WRITE-}" ] || [ -e "$leaf" ]; then
|
||||
printf 'Error writing data to %s: Error making API request.\n\nURL: PUT https://bao.t.local:8200/v1/%s\nCode: 403. Errors:\n\n* 1 error occurred:\n\t* permission denied\n' "$4" "$4" >&2
|
||||
exit 2
|
||||
fi
|
||||
cp "${5#env=@}" "$leaf"
|
||||
;;
|
||||
*)
|
||||
echo "fake bao: unexpected call: $*" >&2
|
||||
exit 98
|
||||
;;
|
||||
esac
|
||||
56
nix/script-tests/agent-backend-env.nix
Normal file
56
nix/script-tests/agent-backend-env.nix
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
# `checks.script-test-agent-backend-env` — runs ../agent-modules/backend-env.nix's
|
||||
# unit against a stub `bao` that keeps its leaves in files. The cases are in
|
||||
# ./agent-backend-env.sh and the stub is ./agent-backend-env-fake-bao.sh.
|
||||
#
|
||||
# What runs is the unit's rendered `ExecStart`, on the unit's own `PATH` with
|
||||
# the stub in front. The edits are the unit's `/run/<unit>/` prefix and the
|
||||
# operator's file, both moved under the build directory because the sandbox has
|
||||
# no writable `/run` and no `/agents`.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
self,
|
||||
nixosSystem,
|
||||
}:
|
||||
let
|
||||
inherit
|
||||
(import ../module-eval/lib.nix {
|
||||
inherit
|
||||
pkgs
|
||||
lib
|
||||
self
|
||||
nixosSystem
|
||||
;
|
||||
})
|
||||
agentWith
|
||||
;
|
||||
|
||||
earlierFile = "/agents/a1/harness/backend.env";
|
||||
|
||||
machine = agentWith {
|
||||
services.hyperhive.agent = {
|
||||
bao.addr = "https://bao.t.local:8200";
|
||||
useApiKey = true;
|
||||
backendEnvironmentFile = earlierFile;
|
||||
};
|
||||
};
|
||||
|
||||
unit = machine.systemd.services.hive-agent-backend-env;
|
||||
|
||||
fakeBao = pkgs.runCommand "fake-bao" { } ''
|
||||
install -Dm755 ${./agent-backend-env-fake-bao.sh} $out/bin/bao
|
||||
patchShebangs $out/bin
|
||||
'';
|
||||
in
|
||||
pkgs.runCommand "hyperhive-script-test-agent-backend-env"
|
||||
{
|
||||
UNIT_SCRIPT = lib.removeSuffix " " unit.serviceConfig.ExecStart;
|
||||
UNIT_PATH = unit.environment.PATH;
|
||||
UNIT_BAO_ADDR = unit.environment.BAO_ADDR;
|
||||
EARLIER_FILE = earlierFile;
|
||||
FAKE_BAO_BIN = "${fakeBao}/bin";
|
||||
}
|
||||
''
|
||||
${pkgs.bash}/bin/bash ${./agent-backend-env.sh}
|
||||
touch "$out"
|
||||
''
|
||||
306
nix/script-tests/agent-backend-env.sh
Normal file
306
nix/script-tests/agent-backend-env.sh
Normal file
|
|
@ -0,0 +1,306 @@
|
|||
# Cases for ./agent-backend-env.nix. Each case gets a fresh runtime directory,
|
||||
# credentials directory, operator's file and store, runs the unit's script under
|
||||
# the unit's `UMask=0077` with a clean environment, and asserts on the exit
|
||||
# code, the output, the files left behind and the `bao` calls made.
|
||||
set -uo pipefail
|
||||
|
||||
failures=0
|
||||
count=0
|
||||
|
||||
fail() {
|
||||
echo "FAILED: $case: $*" >&2
|
||||
failures=$((failures + 1))
|
||||
}
|
||||
|
||||
leaf=secret/swarm/agents/a1/backend-env
|
||||
login_call="login -method=cert -token-only"
|
||||
get_call="kv get -field=env $leaf"
|
||||
|
||||
# setup <description>
|
||||
setup() {
|
||||
case=$1
|
||||
dir=$(mktemp -d)
|
||||
rt=$dir/rt
|
||||
creds=$dir/creds
|
||||
log=$dir/bao.log
|
||||
kv=$dir/kv
|
||||
earlier=$dir/earlier.env
|
||||
mkdir -m 0700 "$rt"
|
||||
mkdir "$creds" "$kv"
|
||||
printf cert >"$creds/hive-agent-bao-cert"
|
||||
printf key >"$creds/hive-agent-bao-key"
|
||||
: >"$log"
|
||||
if ! sed -e "s|/run/hive-agent-backend-env/|$rt/|g" -e "s|$EARLIER_FILE|$earlier|g" \
|
||||
"$UNIT_SCRIPT" >"$dir/script"; then
|
||||
echo "cannot read the rendered script" >&2
|
||||
exit 1
|
||||
fi
|
||||
chmod +x "$dir/script"
|
||||
if grep -q '/run/' "$dir/script"; then fail "the script still names /run after the rewrite"; fi
|
||||
if ! grep -qF "$earlier" "$dir/script"; then fail "the rewrite did not reach the operator's file"; fi
|
||||
login_rc=0
|
||||
deny_read=
|
||||
deny_write=
|
||||
race=
|
||||
stored=$kv/$(printf '%s' "$leaf" | tr / _)
|
||||
}
|
||||
|
||||
go() {
|
||||
count=$((count + 1))
|
||||
(
|
||||
umask 0077
|
||||
cd "$dir" || exit 99
|
||||
exec env -i \
|
||||
PATH="$FAKE_BAO_BIN:$UNIT_PATH" \
|
||||
BAO_ADDR="$UNIT_BAO_ADDR" \
|
||||
BAO_CLIENT_CERT="$creds/hive-agent-bao-cert" \
|
||||
BAO_CLIENT_KEY="$creds/hive-agent-bao-key" \
|
||||
CREDENTIALS_DIRECTORY="$creds" \
|
||||
FAKE_BAO_LOG="$log" \
|
||||
FAKE_BAO_KV="$kv" \
|
||||
FAKE_BAO_LOGIN_RC="$login_rc" \
|
||||
FAKE_BAO_DENY_READ="$deny_read" \
|
||||
FAKE_BAO_DENY_WRITE="$deny_write" \
|
||||
FAKE_BAO_RACE="$race" \
|
||||
"$dir/script"
|
||||
) >"$dir/out" 2>"$dir/err"
|
||||
rc=$?
|
||||
}
|
||||
|
||||
expect_rc() {
|
||||
if [ "$rc" != "$1" ]; then fail "exit $rc, expected $1; stderr: $(<"$dir/err")"; fi
|
||||
}
|
||||
|
||||
expect_err() {
|
||||
if ! grep -qF -- "$1" "$dir/err"; then fail "stderr lacks '$1'; stderr: $(<"$dir/err")"; fi
|
||||
}
|
||||
|
||||
expect_out() {
|
||||
if ! grep -qF -- "$1" "$dir/out"; then fail "stdout lacks '$1'; stdout: $(<"$dir/out")"; fi
|
||||
}
|
||||
|
||||
# expect_calls <line>... — the exact `bao` argument lines, in order.
|
||||
expect_calls() {
|
||||
local want
|
||||
want=$(printf '%s\n' "$@")
|
||||
if [ "$(<"$log")" != "${want%$'\n'}" ]; then fail "bao calls were '$(<"$log")', expected '$*'"; fi
|
||||
}
|
||||
|
||||
expect_no_calls() {
|
||||
if [ -s "$log" ]; then fail "bao was called: $(<"$log")"; fi
|
||||
}
|
||||
|
||||
# expect_same <path> <expected text> — byte for byte, trailing newlines included.
|
||||
expect_same() {
|
||||
if [ ! -e "$1" ]; then
|
||||
fail "$1 missing"
|
||||
return
|
||||
fi
|
||||
if ! printf '%s' "$2" | cmp -s - "$1"; then fail "$1 holds '$(<"$1")', expected '$2'"; fi
|
||||
}
|
||||
|
||||
expect_no_file() {
|
||||
if [ -e "$1" ]; then fail "$1 left behind"; fi
|
||||
}
|
||||
|
||||
# The operator's file, kept to compare after the run.
|
||||
put_earlier() {
|
||||
printf '%s' "$1" >"$earlier"
|
||||
chmod 0600 "$earlier"
|
||||
earlier_inode=$(stat -c %i "$earlier")
|
||||
}
|
||||
|
||||
expect_earlier_untouched() {
|
||||
expect_same "$earlier" "$1"
|
||||
if [ "$(stat -c %i "$earlier")" != "$earlier_inode" ]; then fail "the operator's file was replaced"; fi
|
||||
}
|
||||
|
||||
# A leaf holding <text> before the run.
|
||||
put_leaf() {
|
||||
printf '%s' "$1" >"$stored"
|
||||
}
|
||||
|
||||
expect_leaf() {
|
||||
expect_same "$stored" "$1"
|
||||
}
|
||||
|
||||
file_text=$'ANTHROPIC_API_KEY=k-file\nANTHROPIC_BASE_URL=http://x\n'
|
||||
|
||||
setup "no store identity delivered"
|
||||
: >"$creds/hive-agent-bao-cert"
|
||||
put_earlier "$file_text"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_err "has no store identity"
|
||||
expect_no_calls
|
||||
expect_no_file "$rt/env"
|
||||
|
||||
setup "a store holding nothing is seeded from the file, bytes unchanged, and the store's value is used"
|
||||
put_earlier "$file_text"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_calls "$login_call" "$get_call" "kv put -cas=0 $leaf env=@$earlier" "$get_call"
|
||||
expect_leaf "$file_text"
|
||||
expect_same "$rt/env" "$file_text"
|
||||
expect_earlier_untouched "$file_text"
|
||||
if [ "$(stat -c %a "$rt/env")" != 600 ]; then fail "env is mode $(stat -c %a "$rt/env"), expected 600"; fi
|
||||
expect_no_file "$rt/env.new"
|
||||
expect_no_file "$rt/bao.err"
|
||||
|
||||
setup "a file with no final newline is seeded exactly"
|
||||
put_earlier 'ANTHROPIC_API_KEY=k-file'
|
||||
go
|
||||
expect_rc 0
|
||||
expect_leaf 'ANTHROPIC_API_KEY=k-file'
|
||||
expect_same "$rt/env" 'ANTHROPIC_API_KEY=k-file'
|
||||
expect_earlier_untouched 'ANTHROPIC_API_KEY=k-file'
|
||||
|
||||
setup "a value in the store wins over the file and is not written back to it"
|
||||
put_earlier "$file_text"
|
||||
put_leaf $'ANTHROPIC_API_KEY=k-bao\n'
|
||||
go
|
||||
expect_rc 0
|
||||
expect_calls "$login_call" "$get_call"
|
||||
expect_leaf $'ANTHROPIC_API_KEY=k-bao\n'
|
||||
expect_same "$rt/env" $'ANTHROPIC_API_KEY=k-bao\n'
|
||||
expect_earlier_untouched "$file_text"
|
||||
|
||||
setup "an unchanged store value leaves the file in place"
|
||||
put_leaf "$file_text"
|
||||
printf '%s' "$file_text" >"$rt/env"
|
||||
chmod 0600 "$rt/env"
|
||||
before=$(stat -c %i "$rt/env")
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "is unchanged"
|
||||
expect_same "$rt/env" "$file_text"
|
||||
if [ "$(stat -c %i "$rt/env")" != "$before" ]; then fail "the unchanged env was replaced"; fi
|
||||
|
||||
setup "a changed store value replaces the file in place"
|
||||
put_leaf $'ANTHROPIC_API_KEY=k-new\n'
|
||||
printf old >"$rt/env"
|
||||
chmod 0600 "$rt/env"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "fetched this agent's backend environment"
|
||||
expect_same "$rt/env" $'ANTHROPIC_API_KEY=k-new\n'
|
||||
expect_no_file "$rt/env.new"
|
||||
|
||||
setup "no file and nothing in the store writes nothing"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_out "no file to create it from"
|
||||
expect_calls "$login_call" "$get_call"
|
||||
expect_no_file "$rt/env"
|
||||
expect_no_file "$stored"
|
||||
|
||||
setup "an empty file is never stored"
|
||||
put_earlier ""
|
||||
go
|
||||
expect_rc 0
|
||||
expect_calls "$login_call" "$get_call"
|
||||
expect_no_file "$rt/env"
|
||||
expect_no_file "$stored"
|
||||
|
||||
setup "nothing in the store leaves an env fetched earlier alone"
|
||||
printf kept >"$rt/env"
|
||||
chmod 0600 "$rt/env"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_same "$rt/env" kept
|
||||
|
||||
setup "a store that refuses the login with a file in place changes nothing and fails the unit"
|
||||
put_earlier "$file_text"
|
||||
login_rc=2
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "did not accept this agent's certificate login"
|
||||
expect_err "no such host"
|
||||
expect_calls "$login_call"
|
||||
expect_no_file "$rt/env"
|
||||
expect_no_file "$stored"
|
||||
expect_earlier_untouched "$file_text"
|
||||
|
||||
setup "a lost race reads back the winner's value and keeps the file"
|
||||
put_earlier "$file_text"
|
||||
race=$'ANTHROPIC_API_KEY=k-winner\n'
|
||||
go
|
||||
expect_rc 0
|
||||
expect_calls "$login_call" "$get_call" "kv put -cas=0 $leaf env=@$earlier" "$get_call"
|
||||
expect_err "could not create $leaf"
|
||||
expect_leaf $'ANTHROPIC_API_KEY=k-winner\n'
|
||||
expect_same "$rt/env" $'ANTHROPIC_API_KEY=k-winner\n'
|
||||
expect_earlier_untouched "$file_text"
|
||||
|
||||
setup "a policy that refuses the create leaves the file to carry the agent"
|
||||
put_earlier "$file_text"
|
||||
deny_write=1
|
||||
go
|
||||
expect_rc 0
|
||||
expect_calls "$login_call" "$get_call" "kv put -cas=0 $leaf env=@$earlier" "$get_call"
|
||||
expect_err "permission denied"
|
||||
expect_err "holds no usable env"
|
||||
expect_no_file "$rt/env"
|
||||
expect_no_file "$stored"
|
||||
expect_earlier_untouched "$file_text"
|
||||
|
||||
setup "a leaf that exists but holds no env is not overwritten"
|
||||
put_earlier "$file_text"
|
||||
put_leaf ""
|
||||
go
|
||||
expect_rc 0
|
||||
expect_calls "$login_call" "$get_call"
|
||||
expect_err "holds no usable env"
|
||||
expect_leaf ""
|
||||
expect_no_file "$rt/env"
|
||||
|
||||
setup "a leaf without the env field is not overwritten"
|
||||
put_earlier "$file_text"
|
||||
put_leaf "x"
|
||||
printf other >"$stored.field"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_calls "$login_call" "$get_call"
|
||||
expect_leaf "x"
|
||||
expect_no_file "$rt/env"
|
||||
|
||||
setup "a refused read fails the unit and writes nothing"
|
||||
put_earlier "$file_text"
|
||||
deny_read=1
|
||||
go
|
||||
expect_rc 1
|
||||
expect_err "could not read $leaf"
|
||||
expect_err "permission denied"
|
||||
expect_calls "$login_call" "$get_call"
|
||||
expect_no_file "$rt/env"
|
||||
expect_earlier_untouched "$file_text"
|
||||
|
||||
# `UMask=0077` makes every file the script creates 0600; a stale leftover of a
|
||||
# stopped run must not block the next.
|
||||
setup "leftovers of a stopped run do not block the fetch"
|
||||
put_leaf $'ANTHROPIC_API_KEY=k-bao\n'
|
||||
printf stale >"$rt/env.new"
|
||||
printf stale >"$rt/bao.err"
|
||||
go
|
||||
expect_rc 0
|
||||
expect_same "$rt/env" $'ANTHROPIC_API_KEY=k-bao\n'
|
||||
expect_no_file "$rt/env.new"
|
||||
expect_no_file "$rt/bao.err"
|
||||
|
||||
# The control: the stub's create-only answer is what the lost-race and
|
||||
# refused-create cases lean on, so show it refuses a second create.
|
||||
setup "the stub refuses a create over an existing leaf"
|
||||
put_leaf kept
|
||||
printf x >"$dir/x"
|
||||
(
|
||||
PATH="$FAKE_BAO_BIN:$PATH" BAO_TOKEN=s.fake FAKE_BAO_LOG=$log FAKE_BAO_KV=$kv \
|
||||
bao kv put -cas=0 "$leaf" env=@"$dir/x"
|
||||
) >/dev/null 2>"$dir/err"
|
||||
if [ $? -eq 0 ]; then fail "the stub took a second create"; fi
|
||||
expect_leaf kept
|
||||
|
||||
if [ "$failures" -gt 0 ]; then
|
||||
echo "script-test-agent-backend-env: $failures failed assertions over $count runs" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "script-test-agent-backend-env: $count runs, all assertions hold"
|
||||
Loading…
Reference in a new issue