97cf8a1b2b
hive-agent-forge-token and hive-agent-queue-credential both run with UMask=0377. Their scripts captured bao's stderr in `err="$(mktemp)"`, which under that umask is created 0400; the very next `2>"$err"` on the `bao login` line cannot reopen it for writing, so bash fails the redirect with "Permission denied" before bao ever runs. The `if !` around the login then took the only error branch it had and printed "this agent's certificate was refused by the swarm secret store" — the store was never contacted. No agent has fetched either credential. The stderr file now lives in each unit's own 0700 RuntimeDirectory and is removed before every redirect into it, so the redirect creates it — the idiom forge-token.nix already used for its staging file. The login's error branch now says which of these happened, then quotes bao's output: - `$err` could not be created, so bao never ran; - the store answered with HTTP 4xx (refusal) or another status; - the store sent a TLS alert rejecting the certificate; - no answer at all (network, DNS, or local TLS). Unreadable cert/key credentials are reported before bao runs. bao.nix has the same fetch shape but no UMask=, so its mktemp file is 0600 and writable; it is untouched. Closes #4735
207 lines
8.5 KiB
Nix
207 lines
8.5 KiB
Nix
# This agent's own forge access token, fetched from the swarm secret store by
|
||
# the agent itself.
|
||
#
|
||
# `swarm-controller` mints the token with the forge's admin API and writes it
|
||
# to `swarm/agents/<agent>/forge-token`
|
||
# (`swarm_secret_client::forge::agent_token_path`); no hive is in that chain.
|
||
# This unit logs in to the store with the certificate ./bao.nix already proves
|
||
# it can log in with, and reads its own path. The shape is ./queue-identity.nix's,
|
||
# for the same reason: a hive handing the token over would be the hive reading
|
||
# a secret on the agent's behalf.
|
||
#
|
||
# Unlike the queue credential this one rotates: the controller replaces the
|
||
# token when the forge's copy stops matching the stored one. So the unit is
|
||
# re-run by a timer, and it swaps the file in by rename, only when the value
|
||
# changed, so a reader never sees half a token and a watcher on the file
|
||
# (forge-avatar-sync.path) fires only on a real change.
|
||
#
|
||
# Consumers read `services.hyperhive.agent.forge.tokenFile` first and fall back
|
||
# to `<state>/forge-token`, the file the hive wrote before this existed.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.agent.bao;
|
||
|
||
# The name `swarm-controller` minted the token under — see ./queue-identity.nix.
|
||
agentName = config.services.hyperhive.agent.user.name;
|
||
|
||
# The same three ids ./bao.nix and ./queue-identity.nix load.
|
||
certCredential = "hive-agent-bao-cert";
|
||
keyCredential = "hive-agent-bao-key";
|
||
serverCaCredential = "hive-agent-bao-server-ca";
|
||
|
||
unitName = "hive-agent-forge-token";
|
||
|
||
# The nix half of `swarm_secret_client::forge::agent_token_path` plus
|
||
# `path::MOUNT`.
|
||
tokenPath = "secret/swarm/agents/${agentName}/forge-token";
|
||
|
||
runtimeDir = unitName;
|
||
tokenFile = "/run/${runtimeDir}/token";
|
||
# Beside the token, so the rename that replaces it stays in one directory.
|
||
stagingFile = "/run/${runtimeDir}/token.new";
|
||
# bao's stderr, in the unit's own `0700` directory rather than `/tmp`.
|
||
errFile = "/run/${runtimeDir}/bao.err";
|
||
|
||
# The store's address is the whole switch, as in ./bao.nix and
|
||
# ./queue-identity.nix.
|
||
configured = cfg.addr != null;
|
||
in
|
||
{
|
||
options.services.hyperhive.agent.forge.tokenFile = lib.mkOption {
|
||
type = lib.types.str;
|
||
readOnly = true;
|
||
default = tokenFile;
|
||
description = ''
|
||
Path this agent's own forge token is fetched to. Read-only: it is a
|
||
fact about where `${unitName}.service` writes, not a knob.
|
||
|
||
🩸 A PATH and never a value. The file is `0400` to the agent user.
|
||
|
||
The file exists only once the swarm has minted a token for this agent
|
||
and the agent has a store identity to fetch it with. Until then every
|
||
consumer falls back to `$HYPERHIVE_STATE_DIR/forge-token`.
|
||
'';
|
||
};
|
||
|
||
config = lib.mkIf configured {
|
||
# Every unit and the bash-task runner (where `hive-forge` and `git` run)
|
||
# resolve the token through this, the same way they find
|
||
# `$HYPERHIVE_STATE_DIR`. A path, never the value.
|
||
systemd.globalEnvironment.HIVE_FORGE_TOKEN_FILE = tokenFile;
|
||
environment.variables.HIVE_FORGE_TOKEN_FILE = tokenFile;
|
||
|
||
systemd.services.${unitName} = {
|
||
description = "fetch this agent's own forge token from the secret store";
|
||
after = [
|
||
"network.target"
|
||
# Ordering only, for the reason ./queue-identity.nix gives.
|
||
"hive-agent-bao-identity.service"
|
||
];
|
||
before = [ "hive-forge-notify.service" ];
|
||
wantedBy = [ "multi-user.target" ];
|
||
path = [
|
||
pkgs.openbao
|
||
pkgs.coreutils
|
||
pkgs.diffutils
|
||
];
|
||
startLimitBurst = 4;
|
||
startLimitIntervalSec = 300;
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
# Not `RemainAfterExit`, unlike the queue fetch: the timer below has to
|
||
# be able to start this unit again, and an active unit cannot be
|
||
# started. `RuntimeDirectoryPreserve` is what keeps the directory, and
|
||
# the token in it, alive between runs instead.
|
||
RemainAfterExit = false;
|
||
TimeoutStartSec = 30;
|
||
Restart = "on-failure";
|
||
RestartSec = 15;
|
||
User = agentName;
|
||
Group = agentName;
|
||
RuntimeDirectory = runtimeDir;
|
||
RuntimeDirectoryMode = "0700";
|
||
RuntimeDirectoryPreserve = "yes";
|
||
UMask = "0377";
|
||
LoadCredential = [
|
||
certCredential
|
||
keyCredential
|
||
serverCaCredential
|
||
];
|
||
};
|
||
environment = {
|
||
BAO_ADDR = cfg.addr;
|
||
BAO_CLIENT_CERT = "%d/${certCredential}";
|
||
BAO_CLIENT_KEY = "%d/${keyCredential}";
|
||
};
|
||
script = ''
|
||
set -euo pipefail
|
||
|
||
# No identity delivered: ./bao.nix's check reports that; saying it
|
||
# twice adds nothing.
|
||
for id in ${lib.escapeShellArg certCredential} ${lib.escapeShellArg keyCredential}; do
|
||
if [ ! -s "$CREDENTIALS_DIRECTORY/$id" ]; then
|
||
echo "this agent has no store identity, so it cannot fetch its own forge token." >&2
|
||
exit 0
|
||
fi
|
||
if [ ! -r "$CREDENTIALS_DIRECTORY/$id" ]; then
|
||
echo "cannot read $CREDENTIALS_DIRECTORY/$id, so this agent cannot present its store identity." >&2
|
||
exit 1
|
||
fi
|
||
done
|
||
|
||
if [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then
|
||
export BAO_CACERT="$CREDENTIALS_DIRECTORY/${serverCaCredential}"
|
||
fi
|
||
|
||
# `UMask=0377` makes every file this script creates `0400`, so only
|
||
# the redirect that creates a file can write to it: `$err` is removed
|
||
# before each redirect into it.
|
||
err=${lib.escapeShellArg errFile}
|
||
trap 'rm -f "$err" ${lib.escapeShellArg stagingFile}' EXIT
|
||
|
||
rm -f "$err"
|
||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||
# The redirect creates `$err` before bao starts, so no file means
|
||
# bao never ran. bao prints `Code: <status>` only for an HTTP
|
||
# answer, and `remote error: tls:` only for an alert the store sent.
|
||
re='Code: ([0-9]{3})'
|
||
if [ ! -e "$err" ]; then
|
||
echo "could not create $err, so bao never ran and the store at $BAO_ADDR was not asked." >&2
|
||
elif [[ "$(<"$err")" =~ $re ]]; then
|
||
case "''${BASH_REMATCH[1]}" in
|
||
4*) echo "the swarm secret store at $BAO_ADDR refused this agent's certificate login with HTTP ''${BASH_REMATCH[1]}:" >&2 ;;
|
||
*) echo "the swarm secret store at $BAO_ADDR failed this agent's certificate login with HTTP ''${BASH_REMATCH[1]}:" >&2 ;;
|
||
esac
|
||
elif [[ "$(<"$err")" == *"remote error: tls:"* ]]; then
|
||
echo "the swarm secret store at $BAO_ADDR refused this agent's certificate in the TLS handshake:" >&2
|
||
else
|
||
echo "bao got no answer from the swarm secret store at $BAO_ADDR (network, DNS, or TLS on this side):" >&2
|
||
fi
|
||
if [ -s "$err" ]; then cat "$err" >&2; fi
|
||
exit 1
|
||
fi
|
||
export BAO_TOKEN
|
||
|
||
# 🩸 Degrades rather than fails, for the reason ./queue-identity.nix
|
||
# gives: the policy stanza that let the login read `bao-mtls` covers
|
||
# this path too, so a refusal here is a token not minted yet. The
|
||
# file already in place, if any, is kept: a store that is briefly
|
||
# unreachable must not take a working token away.
|
||
#
|
||
# ⚠️ Written by redirect, never echoed: the field is the secret. The
|
||
# staging file is removed first so the redirect creates it — a file
|
||
# `UMask=0377` left behind is `0400` and could not be reopened for
|
||
# writing.
|
||
rm -f "$err" ${lib.escapeShellArg stagingFile}
|
||
if ! bao kv get -field=value ${lib.escapeShellArg tokenPath} > ${lib.escapeShellArg stagingFile} 2>"$err"; then
|
||
echo "no forge token at ${tokenPath} yet; consumers keep using the state-dir token if there is one." >&2
|
||
if [ -s "$err" ]; then cat "$err" >&2; fi
|
||
exit 0
|
||
fi
|
||
|
||
if cmp -s ${lib.escapeShellArg stagingFile} ${lib.escapeShellArg tokenFile}; then
|
||
echo "this agent's forge token at ${tokenPath} is unchanged."
|
||
exit 0
|
||
fi
|
||
mv -f ${lib.escapeShellArg stagingFile} ${lib.escapeShellArg tokenFile}
|
||
echo "fetched this agent's forge token from ${tokenPath}."
|
||
'';
|
||
};
|
||
|
||
# The controller re-checks every agent's token every five minutes; this
|
||
# picks up a rotation within about ten more.
|
||
systemd.timers.${unitName} = {
|
||
description = "re-fetch this agent's forge token from the secret store";
|
||
wantedBy = [ "timers.target" ];
|
||
timerConfig = {
|
||
OnUnitInactiveSec = "10min";
|
||
RandomizedDelaySec = "1min";
|
||
};
|
||
};
|
||
};
|
||
}
|