Watch
0
0
Fork
You've already forked hyperhive
0

agents keep their github token in /run, fetched on every start, not in the state dir

This commit is contained in:
atlas 2026-11-15 02:04:04 +01:00 • committed by mara
commit 4bd02ea14d
5 changed files with 310 additions and 78 deletions

View file

@ -4,12 +4,18 @@
# An operator links the token in the swarm UI; `swarm-controller` stores it at
# `swarm/agents/<agent>/github-token` (`swarm_secret_client::github`). The
# agent's own read grant covers that path, so this unit reads it and writes
# `<state>/github-token`, which the `gh` wrapper, the git credential helper and
# `hive-github-notify` read.
# `services.hyperhive.agent.github.tokenFile`, under `/run`, which the `gh`
# wrapper, the git credential helper and `hive-github-notify` read.
#
# It never deletes. A `github-token` already in place stays when the store has
# none or cannot be read. The file is replaced by rename, and only when its
# bytes changed.
# `/run` is empty at every start, so the token is fetched anew each time and a
# token removed from the store is gone at the next start. A store that does not
# answer leaves the agent without a token until it does; the timer below asks
# again. Nothing reads the `github-token` file an earlier version kept in the
# state dir, and nothing deletes it.
#
# Within one run, a token already fetched stays when the store has none or
# cannot be read. The file is replaced by rename, and only when its bytes
# changed.
{
pkgs,
lib,
@ -20,7 +26,6 @@ let
cfg = config.services.hyperhive.agent.bao;
agentName = config.services.hyperhive.agent.user.name;
stateDir = "/agents/${agentName}/state";
# The server CA id ./bao.nix loads.
serverCaCredential = "hive-agent-bao-server-ca";
@ -32,19 +37,36 @@ let
tokenPath = "secret/swarm/agents/${agentName}/github-token";
runtimeDir = unitName;
tokenFile = "/run/${runtimeDir}/token";
# Beside the token, so the rename that replaces it stays in one directory.
stagedFile = "/run/${runtimeDir}/token.new";
# The store's whole answer, token included: kept in the unit's own `0700`
# directory, never in the state dir.
# directory.
rawFile = "/run/${runtimeDir}/account.json";
errFile = "/run/${runtimeDir}/bao.err";
tokenFile = "${stateDir}/github-token";
stagedFile = "${stateDir}/.github-token.new";
configured = cfg.addr != null && config.services.hyperhive.agent.github.enable;
storeRetry = import ../host-modules/lib/store-retry.nix { };
in
{
options.services.hyperhive.agent.github.tokenFile = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = tokenFile;
description = ''
Path this agent's GitHub token is fetched to. Read-only: it is a fact
about where `${unitName}.service` writes, not a knob.
🩸 A PATH and never a value. The file is `0400` to the agent user and
lives on `/run`, so it is gone whenever the agent's container restarts.
The file exists only once an operator has linked a token and the agent
has a store identity to fetch it with. Until then `gh` and `git push`
run unauthenticated.
'';
};
config = lib.mkIf configured {
systemd.services.${unitName} = {
description = "fetch this agent's GitHub token from the secret store";
@ -67,14 +89,19 @@ in
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
# Not `RemainAfterExit`, so the timer below can start it again.
# `RuntimeDirectoryPreserve` is what keeps the directory, and the
# token in it, alive between runs instead.
RemainAfterExit = false;
TimeoutStartSec = 30;
User = agentName;
Group = agentName;
RuntimeDirectory = runtimeDir;
RuntimeDirectoryMode = "0700";
# `0600`, the mode `github-token` has.
UMask = "0077";
RuntimeDirectoryPreserve = "yes";
# Every file this script creates is `0400`, so only the redirect that
# creates a file can write to it: each is removed before it is
# written.
UMask = "0377";
LoadCredential = [ serverCaCredential ];
};
environment = {
@ -88,7 +115,7 @@ in
# No identity delivered: ./bao.nix's check reports that.
for f in "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY"; do
if [ ! -s "$f" ]; then
echo "this agent has no store identity, so it cannot fetch its GitHub token." >&2
echo "this agent has no store identity, so it runs without a GitHub token." >&2
exit 0
fi
done
@ -103,17 +130,19 @@ in
token=${lib.escapeShellArg tokenFile}
trap 'rm -f "$err" "$raw" "$staged"' EXIT
rm -f "$err"
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "the swarm secret store at $BAO_ADDR did not accept this agent's certificate login:" >&2
echo "the swarm secret store at $BAO_ADDR did not accept this agent's certificate login; this agent runs without a GitHub token until it does:" >&2
if [ -s "$err" ]; then cat "$err" >&2; fi
exit 1
fi
export BAO_TOKEN
# No token linked and a store that cannot answer look alike here, and
# either way the file in place, if any, is kept.
# either way a token fetched earlier in this run, if any, is kept.
rm -f "$err" "$raw"
if ! bao kv get -format=json ${lib.escapeShellArg tokenPath} >"$raw" 2>"$err"; then
echo "no GitHub token read from ${tokenPath}; github-token left as it is:" >&2
echo "no GitHub token read from ${tokenPath}; this agent runs without one until the store has one:" >&2
if [ -s "$err" ]; then cat "$err" >&2; fi
exit 0
fi
@ -123,7 +152,7 @@ in
# failing the unit would only restart it into the same answer.
rm -f "$staged"
if ! jq -er '.data.data.value | strings' "$raw" >"$staged"; then
echo "${tokenPath} holds no string value; github-token left as it is." >&2
echo "${tokenPath} holds no string value; this agent runs without a GitHub token." >&2
exit 0
fi

View file

@ -1,13 +1,12 @@
# GitHub integration (services.hyperhive.agent.github.enable): a `gh` wrapper + a git
# credential helper, both reading the PAT from the agent's
# `github-token` state file at invocation, so a token ./github-token.nix
# fetches takes effect with no rebuild. The token PATH is baked in at build
# time (nix knows `userName`) — NOT read from `$HIVE_GITHUB_TOKEN_FILE`,
# because claude's Bash tool runs `bash -c` in a minimal env that
# doesn't source `/etc/set-environment`, so the env var isn't present
# where `gh`/`git` actually run. The token value never enters the nix
# store (only its path). github.com only; git auths as `x-access-token`
# + PAT.
# credential helper, both reading the PAT from `github.tokenFile` (the file
# ./github-token.nix fetches into under `/run`) at invocation, so a fetched
# token takes effect with no rebuild. The token PATH is baked in at build
# time — NOT read from an env var, because claude's Bash tool runs `bash -c`
# in a minimal env that doesn't source `/etc/set-environment`, so the env var
# isn't present where `gh`/`git` actually run. The token value never enters
# the nix store (only its path). github.com only; git auths as
# `x-access-token` + PAT.
{
pkgs,
lib,
@ -16,9 +15,10 @@
}:
let
userName = config.services.hyperhive.agent.user.name;
tokenFile = config.services.hyperhive.agent.github.tokenFile;
ghWrapper = pkgs.writeShellScriptBin "gh" ''
if [ -r "/agents/${userName}/state/github-token" ]; then
GH_TOKEN="$(cat "/agents/${userName}/state/github-token")"
if [ -r "${tokenFile}" ]; then
GH_TOKEN="$(cat "${tokenFile}")"
export GH_TOKEN
fi
exec ${pkgs.gh}/bin/gh "$@"
@ -26,11 +26,11 @@ let
gitCredHelper = pkgs.writeShellScriptBin "git-credential-hive-github" ''
# git credential-helper protocol: only the `get` action needs an answer.
[ "''${1:-}" = "get" ] || exit 0
if [ -r "/agents/${userName}/state/github-token" ]; then
if [ -r "${tokenFile}" ]; then
# GitHub ignores the username for PAT auth — `x-access-token` is the
# conventional placeholder; the PAT is the password.
printf 'username=x-access-token\n'
printf 'password=%s\n' "$(cat "/agents/${userName}/state/github-token")"
printf 'password=%s\n' "$(cat "${tokenFile}")"
fi
'';
in
@ -43,7 +43,7 @@ in
git credential helper for `https://github.com`, both authenticated from
an operator-supplied personal access token (PAT). The operator links the
PAT in the swarm UI, and the agent fetches it from the swarm secret store
into `<state>/github-token` (needs `services.hyperhive.agent.bao.addr`),
into `/run` (needs `services.hyperhive.agent.bao.addr`),
so giving an agent GitHub is a runtime action, no per-agent config or
rebuild. The wrappers read the token file at invocation, so a fetched
PAT takes effect immediately; until one exists, `gh` / `git push` just
@ -72,7 +72,7 @@ in
# Wire the GitHub credential helper for `git push` over HTTPS. Host-scoped
# to `https://github.com`, so it never touches the forge (localhost:3000)
# or any other remote. The helper reads the PAT from the agent's
# `github-token` state file at invocation and auths as `x-access-token` +
# `github.tokenFile` at invocation and auths as `x-access-token` +
# the PAT. System /etc/gitconfig merges under the agent's ~/.gitconfig
# (safe.directory), so this is additive.
# Nested-path binding + mkIf (matching the other `environment.etc."…"`
@ -110,8 +110,8 @@ in
# (agent-service.nix) — one todo stream, two producers.
HIVE_AGENT_SOCKET = "/run/hive-agent/${userName}/agent.sock";
RUST_LOG = "info";
# HYPERHIVE_STATE_DIR comes from systemd.globalEnvironment; the
# poller reads the agent's `github-token` from under it.
# The file ./github-token.nix fetches the PAT into; a path, never the value.
HIVE_GITHUB_TOKEN_FILE = tokenFile;
};
serviceConfig = {
ExecStart = "${config.services.hyperhive.agent.packages.hive-github-notify}/bin/hive-github-notify";
@ -138,7 +138,7 @@ in
systemd.paths.hive-github-notify = lib.mkIf config.services.hyperhive.agent.github.enable {
description = "start hive-github-notify when the GitHub token appears";
wantedBy = [ "multi-user.target" ];
pathConfig.PathChanged = "/agents/${userName}/state/github-token";
pathConfig.PathChanged = tokenFile;
};
};
}

View file

@ -3,7 +3,7 @@
# not executes").
#
# The agent side of the swarm-stored GitHub token: ../agent-modules/github-token.nix
# fetches it into the file ../agent-modules/github.nix's readers use.
# fetches it into the `/run` file ../agent-modules/github.nix's readers use.
{
pkgs,
lib,
@ -44,6 +44,7 @@ let
hasTimer = machine: machine.systemd.timers ? hive-agent-github-token;
in
let
tokenFile = "/run/hive-agent-github-token/token";
cases = [
{
name = "an agent with a store address and the integration on fetches its github token";
@ -60,19 +61,49 @@ let
{
# The nix half of `swarm_secret_client::github::account_path`, and the
# file ./github.nix's `gh` wrapper, credential helper and poller read.
name = "the fetch reads the agent's own github-token path into its state-dir file";
name = "the fetch reads the agent's own github-token path into a file under /run";
ok =
let
name = agentGithubBao.services.hyperhive.agent.user.name;
s = (fetchUnit agentGithubBao).script;
in
lib.hasInfix "bao kv get -format=json secret/swarm/agents/${name}/github-token >" s
&& lib.hasInfix "token=/agents/${name}/state/github-token" s
&& lib.hasInfix ".data.data.value | strings" s;
&& lib.hasInfix "token=${tokenFile}" s
&& lib.hasInfix ".data.data.value | strings" s
&& agentGithubBao.services.hyperhive.agent.github.tokenFile == tokenFile;
}
{
# The agent user owns its state dir (./user.nix), and the file keeps
# its `0600` mode.
# The presence arms are the `${tokenFile}` matches above and below: the
# unit and the path unit do name the `/run` file, so a state-dir path
# missing here is a fact and not a misspelled lookup.
name = "neither the fetch nor the poller's units name the state-dir github-token";
ok =
let
name = agentGithubBao.services.hyperhive.agent.user.name;
stateToken = "/agents/${name}/state/github-token";
notify = agentGithubBao.systemd.services.hive-github-notify;
path = agentGithubBao.systemd.paths.hive-github-notify;
in
!(lib.hasInfix "/state" (fetchUnit agentGithubBao).script)
&& notify.environment.HIVE_GITHUB_TOKEN_FILE == tokenFile
&& path.pathConfig.PathChanged == tokenFile
&& path.pathConfig.PathChanged != stateToken;
}
{
# `/run` is empty at every start, so a token the store no longer has is
# gone then; the preserved directory keeps it between this unit's runs.
name = "the token file lives in a preserved 0700 runtime directory and is 0400";
ok =
let
u = fetchUnit agentGithubBao;
in
u.serviceConfig.RuntimeDirectory == "hive-agent-github-token"
&& u.serviceConfig.RuntimeDirectoryMode == "0700"
&& u.serviceConfig.RuntimeDirectoryPreserve == "yes"
&& u.serviceConfig.UMask == "0377";
}
{
# The runtime directory is the agent user's.
name = "the fetch runs as the agent, with its own store identity";
ok =
let
@ -80,15 +111,14 @@ let
name = agentGithubBao.services.hyperhive.agent.user.name;
in
u.serviceConfig.User == name
&& u.serviceConfig.UMask == "0077"
&& u.environment.BAO_ADDR == baoAddr
&& u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
&& u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key";
}
{
# A file a hive wrote keeps working until the operator links a token in
# the swarm UI.
name = "the fetch never deletes the state-dir token, and swaps it in only on a change";
# The fetch itself is run, against a stub store, by
# `checks.script-test-agent-bao-fetch`.
name = "the fetch never deletes the token, and swaps it in only on a change";
ok =
let
s = (fetchUnit agentGithubBao).script;
@ -108,10 +138,9 @@ let
name = "a token that lands after the github poller gave up starts it again";
ok =
let
name = agentGithubBao.services.hyperhive.agent.user.name;
path = agentGithubBao.systemd.paths.hive-github-notify;
in
path.pathConfig.PathChanged == "/agents/${name}/state/github-token"
path.pathConfig.PathChanged == tokenFile
&& path.wantedBy == [ "multi-user.target" ]
&& agentGithubBao.systemd.services ? hive-github-notify
&& !(agentGithubOff.systemd.paths ? hive-github-notify)

View file

@ -1,7 +1,8 @@
# `checks.script-test-agent-bao-fetch` — runs the agent units that log in to
# the swarm secret store and fetch a secret, ../agent-modules/forge-token.nix,
# ../agent-modules/bao.nix and ../agent-modules/egress-proxy.nix, against a stub
# `bao`. The cases are in ./agent-bao-fetch.sh.
# ../agent-modules/github-token.nix, ../agent-modules/bao.nix and
# ../agent-modules/egress-proxy.nix, against a stub `bao`. The cases are in
# ./agent-bao-fetch.sh.
#
# What runs is the unit's rendered `ExecStart`, on the unit's own `PATH` with
# the stub in front. The one edit is the unit's `/run/<unit>/` prefix, moved
@ -41,6 +42,9 @@ let
"${prefix}_BAO_ADDR" = u.environment.BAO_ADDR;
};
# The `gh` wrapper and the git credential helper ./github.nix installs.
systemPackage = n: lib.findFirst (p: (p.name or "") == n) null machine.environment.systemPackages;
# Answers `login` and `kv get` from `FAKE_BAO_*` variables and logs every
# call. A `kv` call without the token `login` printed fails, so a script that
# drops `BAO_TOKEN` between the two cannot pass. A login presenting the
@ -76,10 +80,13 @@ in
pkgs.runCommand "hyperhive-script-test-agent-bao-fetch"
(
unitEnv "FORGE" "hive-agent-forge-token"
// unitEnv "GITHUB" "hive-agent-github-token"
// unitEnv "IDENT" "hive-agent-bao-identity"
// unitEnv "PROXY" "hive-agent-proxy-credential"
// {
FAKE_BAO_BIN = "${fakeBao}/bin";
GH_WRAPPER = "${systemPackage "gh"}/bin/gh";
GIT_CRED_HELPER = "${systemPackage "git-credential-hive-github"}/bin/git-credential-hive-github";
}
)
''

View file

@ -593,6 +593,137 @@ go
expect_rc 0
expect_file "$rt/proxy.env" "$(proxy_url=http://a1:the-secret@10.9.8.7:3129 proxy_env localhost,127.0.0.1,::1,10.9.8.7,x.t.hive,y.t.hive)"
# The GitHub token fetch. The token goes to `$rt/token`, the unit's own
# directory under `/run`, and the unit has no other place to write it.
gh_path=secret/swarm/agents/a1/github-token
gh_kv_call="kv get -format=json $gh_path"
gh_answer() { printf '{"data":{"data":{"value":"%s"}}}' "$1"; }
setup GITHUB "a first fetch writes the token 0400 under the unit's runtime directory"
kv_out=$(gh_answer ghp_fake)
go
expect_rc 0
expect_out "fetched this agent's GitHub token"
expect_calls "$login_call" "$gh_kv_call"
expect_file "$rt/token" ghp_fake
if [ "$(stat -c %a "$rt/token")" != 400 ]; then fail "token is mode $(stat -c %a "$rt/token"), expected 400"; fi
expect_no_file "$rt/token.new"
expect_no_file "$rt/account.json"
expect_no_file "$rt/bao.err"
# The control for the next case: this script names the runtime directory.
if ! grep -qF "$rt/token" "$dir/script"; then fail "the script does not name $rt/token"; fi
setup GITHUB "the script names no state directory"
if ! grep -qF "$rt/token" "$dir/script"; then fail "the script does not name $rt/token"; fi
if grep -qF "/state" "$dir/script"; then fail "the script names a state directory"; fi
setup GITHUB "a store that does not answer at start leaves no token and exits 1 for the retry"
login_rc=2
login_err="Error authenticating: Put \"$bao_addr/v1/auth/cert/login\": dial tcp: lookup bao.t.local: no such host"
go
expect_rc 1
expect_err "this agent runs without a GitHub token until it does"
expect_err "no such host"
expect_calls "$login_call"
expect_no_file "$rt/token"
expect_no_file "$rt/token.new"
expect_no_file "$rt/account.json"
setup GITHUB "a read that fails at start leaves no token and does not fail the unit"
kv_rc=2
kv_err="Error reading secret/data/swarm/agents/a1: Get \"$bao_addr/v1/secret/data/swarm/agents/a1\": dial tcp: connect: connection refused"
go
expect_rc 0
expect_err "this agent runs without one until the store has one"
expect_err "connection refused"
expect_calls "$login_call" "$gh_kv_call"
expect_no_file "$rt/token"
expect_no_file "$rt/account.json"
setup GITHUB "no identity delivered leaves no token and does not fail the unit"
: >"$creds/hive-agent-bao-cert"
go
expect_rc 0
expect_err "runs without a GitHub token"
expect_calls
expect_no_file "$rt/token"
setup GITHUB "an answer without a string value leaves no token"
kv_out='{"data":{"data":{"value":7}}}'
go
expect_rc 0
expect_err "holds no string value"
expect_no_file "$rt/token"
expect_no_file "$rt/token.new"
setup GITHUB "a token fetched earlier in the run stays when the store stops answering"
printf old >"$rt/token"
chmod 0400 "$rt/token"
kv_rc=2
kv_err="dial tcp: connect: connection refused"
go
expect_rc 0
expect_file "$rt/token" old
# `UMask=0377` makes every file the script creates 0400, and a redirect into
# a 0400 file fails.
setup GITHUB "0400 files left by a killed run do not block the fetch"
for f in bao.err account.json token.new; do
printf stale >"$rt/$f"
chmod 0400 "$rt/$f"
done
kv_out=$(gh_answer ghp_fake)
go
expect_rc 0
expect_out "fetched this agent's GitHub token"
expect_no_err "Permission denied"
expect_file "$rt/token" ghp_fake
expect_no_file "$rt/account.json"
setup GITHUB "an unchanged token is left in place"
printf 'ghp_fake\n' >"$rt/token"
chmod 0400 "$rt/token"
before=$(stat -c %i "$rt/token")
kv_out=$(gh_answer ghp_fake)
go
expect_rc 0
expect_out "is unchanged"
if [ "$(stat -c %i "$rt/token")" != "$before" ]; then fail "the unchanged token was replaced"; fi
setup GITHUB "a rotated token replaces the old one"
printf old >"$rt/token"
chmod 0400 "$rt/token"
kv_out=$(gh_answer ghp_new)
go
expect_rc 0
expect_out "fetched this agent's GitHub token"
expect_file "$rt/token" ghp_new
expect_no_file "$rt/token.new"
# The readers, with the token path and the state directory moved under the
# build directory the way the units' `/run` is. The state directory holds a
# token no reader may use.
setup GITHUB "the gh wrapper and the credential helper read the runtime file and not the state directory"
mkdir -p "$dir/state"
printf legacy >"$dir/state/github-token"
reader() {
sed -e "s|/run/$unit/|$rt/|g" -e "s|/agents/a1/state/|$dir/state/|g" -e 's|^exec .*|printenv GH_TOKEN|' "$1" >"$dir/reader"
chmod +x "$dir/reader"
}
reader "$GH_WRAPPER"
if ! grep -qF "$rt/token" "$dir/reader"; then fail "the gh wrapper does not name $rt/token"; fi
git_cred() { env -i PATH="$unit_path" "$dir/reader" get; }
printf ghp_fake >"$rt/token"
chmod 0400 "$rt/token"
if [ "$(env -u GH_TOKEN "$dir/reader")" != ghp_fake ]; then fail "the gh wrapper did not use the runtime token"; fi
reader "$GIT_CRED_HELPER"
if ! grep -qF "$rt/token" "$dir/reader"; then fail "the credential helper does not name $rt/token"; fi
if [ "$(git_cred)" != $'username=x-access-token\npassword=ghp_fake' ]; then fail "the credential helper did not answer with the runtime token: $(git_cred)"; fi
rm "$rt/token"
if [ -n "$(git_cred)" ]; then fail "the credential helper answered with no token fetched: $(git_cred)"; fi
reader "$GH_WRAPPER"
if [ -n "$(env -u GH_TOKEN "$dir/reader" 2>&1)" ]; then fail "the gh wrapper exported a token with none fetched"; fi
if [ "$failures" -ne 0 ]; then
echo "script-test-agent-bao-fetch: $failures failed assertions over $count runs" >&2
exit 1