c6717bccb0
swarm-controller re-issues an agent's store leaf at half its validity, but the running container kept presenting the copy its hive delivered at start, so a re-issued leaf reached the agent only on its next restart. hive-agent-bao-identity now seeds a live copy of the pair in its own runtime directory from the delivered credentials, and re-runs every 10 minutes: it logs in with the live leaf, reads its own swarm/agents/<agent>/bao-mtls (a read render_agent already grants), and when the leaf changed, logs in with the new pair and swaps it in only if the store accepts it. Every unit in the container that logs in to the store presents the live pair: hive-agent (queue-identity.nix), hive-matrix-daemon, hive-subagent-daemon, hive-agent-forge-token, hive-agent-forge-accounts and hive-agent-github-token. They keep loading the server CA as a credential. The long-running daemons are ordered after the refresher so the live pair exists when they start. It renews only while the held leaf is still valid: the login that reads the new leaf is made with the current one. Refs #5017
291 lines
13 KiB
Nix
291 lines
13 KiB
Nix
# This agent's own identity at the swarm secret store: the check that proves
|
||
# it works, and the live copy every store reader in the container presents.
|
||
#
|
||
# `swarm-controller` mints the leaf at agent creation and publishes it; this
|
||
# agent's hive collects it under the hive's own certificate and hands it in as
|
||
# systemd credentials (`hive_c0re::lifecycle::agent_identity`). That copy is
|
||
# fixed for the life of the container, and the controller re-issues the leaf
|
||
# at half its validity, so this unit seeds a live copy from it and, on a timer,
|
||
# reads the current leaf back from the store under the one it holds. Because
|
||
# that login needs the held leaf to still be valid, a container whose leaf
|
||
# expired while every pull failed recovers only at its next start.
|
||
#
|
||
# ⚠️ The identity arrives as credentials and NOT as a bind mount, and the mode
|
||
# is why: the host file is `0600` to the hive daemon and this unit runs as the
|
||
# unprivileged agent user. nspawn's `--load-credential` is read by the
|
||
# container manager as root and re-exposed under this unit's own `User=`; a
|
||
# bind would deliver a private key this user cannot open. Same answer, same
|
||
# reason, as ./queue.nix's credential pair.
|
||
#
|
||
# This is the courier's other end: the hive-side collector has no purpose
|
||
# without it, so the two ship together and neither is reachable alone.
|
||
#
|
||
# 🩸 This unit fails LOUDLY where the hive-side readers degrade quietly, and
|
||
# that is the opposite default on purpose. A missing OIDC client secret means
|
||
# a hive whose publisher has yet to run; a missing or refused certificate
|
||
# means an agent that believes it can reach the store and cannot, which every
|
||
# later pull would report as its own unrelated failure. The one place that
|
||
# knows the real cause is the login itself, so this is where it is said.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.agent.bao;
|
||
|
||
# This container's agent name. The same string the hive published the
|
||
# identity under, because the agent's unix user is named for the agent —
|
||
# see ./user.nix.
|
||
agentName = config.services.hyperhive.agent.user.name;
|
||
|
||
# The three ids `hive_c0re::lifecycle::agent_identity` forwards under.
|
||
# Neither side can discover the other's spelling, so a rename is a rename
|
||
# there too.
|
||
certCredential = "hive-agent-bao-cert";
|
||
keyCredential = "hive-agent-bao-key";
|
||
serverCaCredential = "hive-agent-bao-server-ca";
|
||
|
||
unitName = "hive-agent-bao-identity";
|
||
|
||
runtimeDir = unitName;
|
||
certFile = "/run/${runtimeDir}/cert";
|
||
keyFile = "/run/${runtimeDir}/key";
|
||
# Beside the live pair, so each rename that replaces it stays in one
|
||
# directory.
|
||
stagedCert = "/run/${runtimeDir}/cert.new";
|
||
stagedKey = "/run/${runtimeDir}/key.new";
|
||
# The store's whole answer, key included, in the unit's own `0700` directory.
|
||
rawFile = "/run/${runtimeDir}/bao-mtls.json";
|
||
errFile = "/run/${runtimeDir}/bao.err";
|
||
|
||
# Where the identity lives in the store, spelled from the same pieces the
|
||
# publisher uses. `swarm_secret_client::mtls::identity_path` builds
|
||
# `swarm/agents/<agent>/bao-mtls` and `path::MOUNT` is `secret`; this
|
||
# literal is the nix half of that one agreement, exactly as
|
||
# ../host-modules/glue-queue-agent-credential.nix spells its own.
|
||
identityPath = "secret/swarm/agents/${agentName}/bao-mtls";
|
||
|
||
# The address is the whole switch — no separate `enable`, the same shape
|
||
# ./queue.nix and ./logs.nix gate themselves with. A hive that has a store
|
||
# forwards its address and every agent on it gets the check; a hive that has
|
||
# none forwards nothing and no agent does. An `enable` beside it would be a
|
||
# knob whose only correct setting is whatever the address already says, and
|
||
# its default would decide whether the hive-side courier delivers into a
|
||
# container that reads what it is given or into one that never looks.
|
||
configured = cfg.addr != null;
|
||
|
||
storeRetry = import ../host-modules/lib/store-retry.nix { };
|
||
in
|
||
{
|
||
options.services.hyperhive.agent.bao = {
|
||
addr = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "https://bao.example.com:8200";
|
||
description = ''
|
||
Where the swarm secret store listens, as this container reaches it.
|
||
|
||
Set by the generated meta flake from the host's own `BAO_ADDR`, which
|
||
is the address this hive already uses, and setting it is what generates
|
||
`${unitName}.service`: at boot that unit logs in with the certificate
|
||
its hive delivered and reads this agent's own path back, failing if
|
||
either step does not succeed. `systemctl status ${unitName}` inside the
|
||
container is then the answer to "can this agent reach the store as
|
||
itself", which nothing else in the tree reports.
|
||
|
||
`null` means the hive was given no store. No unit is generated then,
|
||
because an agent whose swarm never minted an identity has nothing to
|
||
log in with, and a failed unit at every boot would say that in the
|
||
loudest possible way about a deployment that never asked for it.
|
||
'';
|
||
};
|
||
|
||
clientCertFile = lib.mkOption {
|
||
type = lib.types.str;
|
||
readOnly = true;
|
||
default = certFile;
|
||
description = ''
|
||
This agent's current store certificate, kept by `${unitName}.service`.
|
||
Read-only: it is where that unit writes, not a knob. Every unit that
|
||
logs in to the store as this agent points `BAO_CLIENT_CERT` here.
|
||
'';
|
||
};
|
||
|
||
clientKeyFile = lib.mkOption {
|
||
type = lib.types.str;
|
||
readOnly = true;
|
||
default = keyFile;
|
||
description = ''
|
||
The private key for `clientCertFile`, `0400` to the agent user.
|
||
🩸 A PATH and never a value.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkIf configured {
|
||
systemd.services.${unitName} = {
|
||
description = "keep this agent's swarm secret store identity current, and prove it works";
|
||
after = [ "network.target" ];
|
||
wantedBy = [ "multi-user.target" ];
|
||
path = [
|
||
pkgs.openbao
|
||
pkgs.coreutils
|
||
pkgs.diffutils
|
||
pkgs.jq
|
||
];
|
||
# ../host-modules/lib/store-retry.nix. From an agent the store is reached
|
||
# through the gateway's stream passthrough, so either being down fails
|
||
# the login below.
|
||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||
serviceConfig = storeRetry.serviceConfig // {
|
||
Type = "oneshot";
|
||
# Not `RemainAfterExit`, so the timer below can start it again.
|
||
# `RuntimeDirectoryPreserve` keeps the live pair between runs.
|
||
RemainAfterExit = false;
|
||
TimeoutStartSec = 30;
|
||
User = agentName;
|
||
Group = agentName;
|
||
RuntimeDirectory = runtimeDir;
|
||
RuntimeDirectoryMode = "0700";
|
||
RuntimeDirectoryPreserve = "yes";
|
||
UMask = "0377";
|
||
# Bare ids, no paths: the terse `LoadCredential=` form that inherits a
|
||
# credential the service *manager* received, which is what the
|
||
# container manager passed in. ./queue.nix states the same shape.
|
||
LoadCredential = [
|
||
certCredential
|
||
keyCredential
|
||
serverCaCredential
|
||
];
|
||
};
|
||
environment = {
|
||
BAO_ADDR = cfg.addr;
|
||
BAO_CLIENT_CERT = certFile;
|
||
BAO_CLIENT_KEY = keyFile;
|
||
};
|
||
script = ''
|
||
set -euo pipefail
|
||
|
||
# Absent credentials are the first thing checked, because every later
|
||
# message would blame the store for a delivery that never happened.
|
||
# 🩸 The files are named and never read here: what went wrong is a
|
||
# property of the path, and the bytes at it are a private key.
|
||
missing=
|
||
for id in ${lib.escapeShellArg certCredential} ${lib.escapeShellArg keyCredential}; do
|
||
if [ ! -s "$CREDENTIALS_DIRECTORY/$id" ]; then
|
||
missing="$missing $id"
|
||
fi
|
||
done
|
||
if [ -n "$missing" ]; then
|
||
echo "this agent has no store identity: the hive delivered no$missing." >&2
|
||
echo "swarm-controller publishes it at ${identityPath} when the agent is created, and the hive collects it from there." >&2
|
||
exit 1
|
||
fi
|
||
|
||
# Only when one was delivered. Absent means the container verifies the
|
||
# store's listener against its own trust store, which is what a
|
||
# deployment with a real CA wants; pointing BAO_CACERT at a file that
|
||
# is not there would fail the handshake and name the wrong cause.
|
||
if [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then
|
||
export BAO_CACERT="$CREDENTIALS_DIRECTORY/${serverCaCredential}"
|
||
fi
|
||
|
||
# `UMask=0377` makes every file this script creates `0400`, so each one
|
||
# is removed before the command that creates it.
|
||
err=${lib.escapeShellArg errFile}
|
||
raw=${lib.escapeShellArg rawFile}
|
||
trap 'rm -f "$err" "$raw" ${lib.escapeShellArg stagedCert} ${lib.escapeShellArg stagedKey}' EXIT
|
||
|
||
# The delivered pair is the start of every container's life: /run is
|
||
# empty at boot and kept across this unit's runs.
|
||
if [ ! -s "$BAO_CLIENT_CERT" ] || [ ! -s "$BAO_CLIENT_KEY" ]; then
|
||
rm -f "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY"
|
||
cp "$CREDENTIALS_DIRECTORY/${keyCredential}" "$BAO_CLIENT_KEY"
|
||
cp "$CREDENTIALS_DIRECTORY/${certCredential}" "$BAO_CLIENT_CERT"
|
||
fi
|
||
|
||
# Cert auth is a login, not a transport setting. The `BAO_CLIENT_*`
|
||
# variables above only decide which certificate the TLS handshake
|
||
# presents; without a token `bao` asks its token helper instead, and
|
||
# that is a `sh` this unit's `path` does not carry. `-token-only`
|
||
# answers on stdout and skips the helper on both sides.
|
||
#
|
||
# No `name=`: a cert role pins both its authority and the common name
|
||
# it accepts, so this agent's leaf matches its own role and no other.
|
||
# Naming the role here would be a third copy of a string
|
||
# `swarm_secret_client::policy::agent_object_name` already owns.
|
||
rm -f "$err"
|
||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||
echo "this agent's certificate was refused by the swarm secret store at $BAO_ADDR." >&2
|
||
if [ -s "$err" ]; then
|
||
cat "$err" >&2
|
||
else
|
||
echo "bao failed without writing a diagnostic." >&2
|
||
fi
|
||
exit 1
|
||
fi
|
||
export BAO_TOKEN
|
||
|
||
# A token is not yet an answer: the login proves the certificate, this
|
||
# proves the policy attached to it, and the answer is the current
|
||
# leaf. One read, so the certificate and key come from one version.
|
||
#
|
||
# ⚠️ Written by redirect, never printed: the object holds a private
|
||
# key.
|
||
rm -f "$err" "$raw"
|
||
if ! bao kv get -format=json ${lib.escapeShellArg identityPath} >"$raw" 2>"$err"; then
|
||
echo "this agent logged in to the swarm secret store but cannot read ${identityPath}, so its policy does not cover its own path." >&2
|
||
if [ -s "$err" ]; then
|
||
cat "$err" >&2
|
||
else
|
||
echo "bao failed without writing a diagnostic." >&2
|
||
fi
|
||
exit 1
|
||
fi
|
||
|
||
# `-j`: the bytes as the store holds them, which is what the hive
|
||
# delivered, so an unchanged leaf compares equal to the seeded copy.
|
||
rm -f ${lib.escapeShellArg stagedCert} ${lib.escapeShellArg stagedKey}
|
||
if ! jq -ej '.data.data.cert | strings' "$raw" >${lib.escapeShellArg stagedCert} \
|
||
|| ! jq -ej '.data.data.key | strings' "$raw" >${lib.escapeShellArg stagedKey}; then
|
||
echo "${identityPath} holds no certificate and key; this agent keeps the one it has." >&2
|
||
exit 1
|
||
fi
|
||
|
||
if cmp -s ${lib.escapeShellArg stagedCert} "$BAO_CLIENT_CERT" \
|
||
&& cmp -s ${lib.escapeShellArg stagedKey} "$BAO_CLIENT_KEY"; then
|
||
echo "authenticated to the swarm secret store at $BAO_ADDR as ${agentName}; its certificate at ${identityPath} is unchanged."
|
||
else
|
||
# This unit's next login uses whatever is swapped in, so a pair the
|
||
# store refuses must never replace the one it accepts.
|
||
rm -f "$err"
|
||
if ! BAO_CLIENT_CERT=${lib.escapeShellArg stagedCert} BAO_CLIENT_KEY=${lib.escapeShellArg stagedKey} \
|
||
bao login -method=cert -token-only >/dev/null 2>"$err"; then
|
||
echo "the swarm secret store refused the certificate at ${identityPath}; this agent keeps the one it has." >&2
|
||
if [ -s "$err" ]; then cat "$err" >&2; fi
|
||
exit 1
|
||
fi
|
||
|
||
# Two renames: a reader that opens the pair between them gets a
|
||
# mismatched cert and key and fails that one login.
|
||
mv -f ${lib.escapeShellArg stagedKey} "$BAO_CLIENT_KEY"
|
||
mv -f ${lib.escapeShellArg stagedCert} "$BAO_CLIENT_CERT"
|
||
echo "replaced this agent's store certificate with the one at ${identityPath}."
|
||
fi
|
||
'';
|
||
};
|
||
|
||
# The controller re-issues a leaf at half its validity, so ten minutes is
|
||
# small against the time the old one stays valid.
|
||
systemd.timers.${unitName} = {
|
||
description = "re-read this agent's store certificate from the secret store";
|
||
wantedBy = [ "timers.target" ];
|
||
timerConfig = {
|
||
OnUnitInactiveSec = "10min";
|
||
RandomizedDelaySec = "1min";
|
||
};
|
||
};
|
||
};
|
||
}
|