97cf8a1b2b
hive-agent-forge-token and hive-agent-queue-credential both run with UMask=0377. Their scripts captured bao's stderr in `err="$(mktemp)"`, which under that umask is created 0400; the very next `2>"$err"` on the `bao login` line cannot reopen it for writing, so bash fails the redirect with "Permission denied" before bao ever runs. The `if !` around the login then took the only error branch it had and printed "this agent's certificate was refused by the swarm secret store" — the store was never contacted. No agent has fetched either credential. The stderr file now lives in each unit's own 0700 RuntimeDirectory and is removed before every redirect into it, so the redirect creates it — the idiom forge-token.nix already used for its staging file. The login's error branch now says which of these happened, then quotes bao's output: - `$err` could not be created, so bao never ran; - the store answered with HTTP 4xx (refusal) or another status; - the store sent a TLS alert rejecting the certificate; - no answer at all (network, DNS, or local TLS). Unreadable cert/key credentials are reported before bao runs. bao.nix has the same fetch shape but no UMask=, so its mktemp file is 0600 and writable; it is untouched. Closes #4735
229 lines
10 KiB
Nix
229 lines
10 KiB
Nix
# This agent's own credential for the swarm queue, fetched from the store by
|
||
# the agent itself.
|
||
#
|
||
# The credential beside it in ./queue.nix is keyed per **hive**: one OIDC
|
||
# client minted at deploy time and handed to every agent container on the
|
||
# hive, so at the queue's auth callout one agent is indistinguishable from its
|
||
# co-hived neighbours. This one is keyed per **agent** — `swarm-controller`
|
||
# mints it at agent creation into `swarm/agents/<agent>/queue`
|
||
# (`swarm_secret_client::queue::agent_queue_path`), at swarm level, with no
|
||
# hive anywhere in the chain.
|
||
#
|
||
# That is also why this unit *fetches* rather than being handed a credential:
|
||
# a hive courier in the path would be the hive vouching for which agent this
|
||
# is, which is the property the per-agent credential exists to remove. The
|
||
# agent authenticates to the store as itself, with the certificate ./bao.nix
|
||
# already proves it can log in with, and reads its own path.
|
||
#
|
||
# ⛔ The store certificate is for reaching the store and nothing else. It is
|
||
# never presented to the queue: what goes to the queue is the secret read
|
||
# back from this path.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.agent.bao;
|
||
|
||
# This container's agent name — the same string `swarm-controller` minted
|
||
# the credential under, because the agent's unix user is named for the
|
||
# agent (see ./user.nix).
|
||
agentName = config.services.hyperhive.agent.user.name;
|
||
|
||
# The same three ids ./bao.nix loads. Both units present the same
|
||
# certificate because there is one store identity per agent; the ids are
|
||
# `hive_c0re::lifecycle::agent_identity`'s and a rename is a rename there
|
||
# too.
|
||
certCredential = "hive-agent-bao-cert";
|
||
keyCredential = "hive-agent-bao-key";
|
||
serverCaCredential = "hive-agent-bao-server-ca";
|
||
|
||
unitName = "hive-agent-queue-credential";
|
||
|
||
# The nix half of `swarm_secret_client::queue::agent_queue_path` plus
|
||
# `path::MOUNT`, spelled exactly as ./bao.nix spells its own sibling path.
|
||
queuePath = "secret/swarm/agents/${agentName}/queue";
|
||
|
||
# `RuntimeDirectory=` under the unit's own `User=`, so the file is owned by
|
||
# the agent and readable by the harness without a mode change. /run and not
|
||
# the state dir on purpose: a secret fetched at boot has no business
|
||
# surviving one.
|
||
runtimeDir = "${unitName}";
|
||
secretFile = "/run/${runtimeDir}/secret";
|
||
# bao's stderr, in the unit's own `0700` directory rather than `/tmp`.
|
||
errFile = "/run/${runtimeDir}/bao.err";
|
||
|
||
# The store's address is the whole switch, exactly as in ./bao.nix — and
|
||
# deliberately *not* the queue coordinates in ./queue.nix. Those are the
|
||
# hive's, and gating a swarm-minted per-agent credential on a hive-level
|
||
# option would put the hive back in a path whose entire purpose is not
|
||
# having one.
|
||
configured = cfg.addr != null;
|
||
in
|
||
{
|
||
options.services.hyperhive.agent.queue.agentSecretFile = lib.mkOption {
|
||
type = lib.types.str;
|
||
readOnly = true;
|
||
default = secretFile;
|
||
description = ''
|
||
Path this agent's own swarm-queue secret is fetched to, for a consumer
|
||
outside the harness unit. Read-only for the same reason
|
||
{option}`services.hyperhive.agent.queue.clientSecretFile` is: it is a
|
||
fact about where the fetch writes, not a knob.
|
||
|
||
🩸 A PATH and never a value. The file is `0400` to the agent user and
|
||
is read at the moment it is needed; nothing in this tree puts its
|
||
contents in an environment variable, where `/proc/<pid>/environ` would
|
||
publish them to every process in the container.
|
||
|
||
The file exists only once the swarm has minted a credential for this
|
||
agent. An agent created before its swarm did so has none, and
|
||
`${unitName}.service` says so in the journal rather than failing —
|
||
see that unit.
|
||
'';
|
||
};
|
||
|
||
config = lib.mkIf configured {
|
||
systemd.services.${unitName} = {
|
||
description = "fetch this agent's own swarm-queue credential from the secret store";
|
||
after = [
|
||
"network.target"
|
||
# Ordering only, not a requirement: that unit is the one that reports
|
||
# a store this agent cannot reach as itself, and it should get to say
|
||
# so before this one reports a path it could not read.
|
||
"hive-agent-bao-identity.service"
|
||
];
|
||
before = [ "hive-agent.service" ];
|
||
wantedBy = [ "multi-user.target" ];
|
||
path = [
|
||
pkgs.openbao
|
||
pkgs.coreutils
|
||
];
|
||
# Same sizing and the same `[Unit]`-not-`[Service]` placement as
|
||
# ./bao.nix's check: a few short attempts cover a store that comes up
|
||
# alongside this container, and a longer window only delays the report.
|
||
startLimitBurst = 4;
|
||
startLimitIntervalSec = 300;
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
# Keeps the unit active, which is what keeps `RuntimeDirectory=`
|
||
# from being removed out from under the harness.
|
||
RemainAfterExit = true;
|
||
TimeoutStartSec = 30;
|
||
Restart = "on-failure";
|
||
RestartSec = 15;
|
||
User = agentName;
|
||
Group = agentName;
|
||
RuntimeDirectory = runtimeDir;
|
||
RuntimeDirectoryMode = "0700";
|
||
UMask = "0377";
|
||
# Bare ids, no paths: the terse `LoadCredential=` form that inherits
|
||
# a credential the service *manager* received, which is what the
|
||
# container manager passed in. ./bao.nix and ./queue.nix state the
|
||
# same shape.
|
||
LoadCredential = [
|
||
certCredential
|
||
keyCredential
|
||
serverCaCredential
|
||
];
|
||
};
|
||
environment = {
|
||
BAO_ADDR = cfg.addr;
|
||
# `%d` is `$CREDENTIALS_DIRECTORY`, per-unit and owned by `User=`.
|
||
BAO_CLIENT_CERT = "%d/${certCredential}";
|
||
BAO_CLIENT_KEY = "%d/${keyCredential}";
|
||
};
|
||
script = ''
|
||
set -euo pipefail
|
||
|
||
# No identity delivered at all. ./bao.nix's check reports this as the
|
||
# failure it is; there is nothing for this unit to add, and failing
|
||
# here too would be the same cause stated twice.
|
||
for id in ${lib.escapeShellArg certCredential} ${lib.escapeShellArg keyCredential}; do
|
||
if [ ! -s "$CREDENTIALS_DIRECTORY/$id" ]; then
|
||
echo "this agent has no store identity, so it cannot fetch its own queue credential." >&2
|
||
exit 0
|
||
fi
|
||
if [ ! -r "$CREDENTIALS_DIRECTORY/$id" ]; then
|
||
echo "cannot read $CREDENTIALS_DIRECTORY/$id, so this agent cannot present its store identity." >&2
|
||
exit 1
|
||
fi
|
||
done
|
||
|
||
# Only when one was delivered — absent means verify the store's
|
||
# listener against the container's own trust store, which is what a
|
||
# deployment with a real CA wants. ./bao.nix says the same.
|
||
if [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then
|
||
export BAO_CACERT="$CREDENTIALS_DIRECTORY/${serverCaCredential}"
|
||
fi
|
||
|
||
# `UMask=0377` makes every file this script creates `0400`, so only
|
||
# the redirect that creates a file can write to it: `$err` is removed
|
||
# before each redirect into it.
|
||
err=${lib.escapeShellArg errFile}
|
||
trap 'rm -f "$err"' EXIT
|
||
|
||
# Cert auth is a login, not a transport setting: the `BAO_CLIENT_*`
|
||
# variables above only pick the certificate the handshake presents.
|
||
# `-token-only` answers on stdout and skips the token helper, which
|
||
# is a `sh` this unit's `path` does not carry.
|
||
rm -f "$err"
|
||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||
# The redirect creates `$err` before bao starts, so no file means
|
||
# bao never ran. bao prints `Code: <status>` only for an HTTP
|
||
# answer, and `remote error: tls:` only for an alert the store sent.
|
||
re='Code: ([0-9]{3})'
|
||
if [ ! -e "$err" ]; then
|
||
echo "could not create $err, so bao never ran and the store at $BAO_ADDR was not asked." >&2
|
||
elif [[ "$(<"$err")" =~ $re ]]; then
|
||
case "''${BASH_REMATCH[1]}" in
|
||
4*) echo "the swarm secret store at $BAO_ADDR refused this agent's certificate login with HTTP ''${BASH_REMATCH[1]}:" >&2 ;;
|
||
*) echo "the swarm secret store at $BAO_ADDR failed this agent's certificate login with HTTP ''${BASH_REMATCH[1]}:" >&2 ;;
|
||
esac
|
||
elif [[ "$(<"$err")" == *"remote error: tls:"* ]]; then
|
||
echo "the swarm secret store at $BAO_ADDR refused this agent's certificate in the TLS handshake:" >&2
|
||
else
|
||
echo "bao got no answer from the swarm secret store at $BAO_ADDR (network, DNS, or TLS on this side):" >&2
|
||
fi
|
||
if [ -s "$err" ]; then cat "$err" >&2; fi
|
||
exit 1
|
||
fi
|
||
export BAO_TOKEN
|
||
|
||
# 🩸 Degrades where ./bao.nix's check fails, and the reason is that
|
||
# the two reads are governed by the *same* policy stanza:
|
||
# `swarm_secret_client::policy::render_agent` grants read on
|
||
# `secret/data/swarm/agents/<agent>/*`, which covers this path and
|
||
# the `bao-mtls` one beside it alike. So a refusal this unit sees and
|
||
# that check did not cannot be a policy that drifted — it is an
|
||
# object that has not been minted, which is the ordinary state of
|
||
# every agent created before its swarm knew to mint one. A unit that
|
||
# failed at every boot over that would be loud about a deployment
|
||
# doing nothing wrong.
|
||
#
|
||
# ⚠️ Written by redirect into the runtime directory, never echoed:
|
||
# the field is the secret itself.
|
||
rm -f "$err"
|
||
if ! bao kv get -field=value ${lib.escapeShellArg queuePath} > ${lib.escapeShellArg secretFile} 2>"$err"; then
|
||
rm -f ${lib.escapeShellArg secretFile}
|
||
echo "no per-agent queue credential at ${queuePath} yet; this agent falls back to its hive's shared one." >&2
|
||
if [ -s "$err" ]; then cat "$err" >&2; fi
|
||
exit 0
|
||
fi
|
||
|
||
echo "fetched this agent's own queue credential from ${queuePath}."
|
||
'';
|
||
};
|
||
|
||
# The harness reads a path and never a value, the same shape ./queue.nix
|
||
# hands it the hive-scoped secret in. Not `%d` here: this credential is
|
||
# not a systemd credential at all — it is a file this container fetched
|
||
# for itself, which is the whole point.
|
||
systemd.services.hive-agent = {
|
||
after = [ "${unitName}.service" ];
|
||
environment.HIVE_AGENT_QUEUE_AGENT_SECRET_FILE = secretFile;
|
||
};
|
||
};
|
||
}
|