4c4ecb8ed9
The live pair was replaced with two renames, key then cert. A run stopped between them (timeout, container stop, power loss) left a new key beside the old cert, and every later run logged in with that pair, so the agent could not re-pull until its container restarted. Each pair now goes into a fresh directory under /run/hive-agent-bao-identity, and `current`, a symlink to it, is replaced by one rename. Wherever a run stops, `current` names one whole pair; the next run removes whatever a stopped run left. Consumers read `current/cert` and `current/key` through the same read-only options and still re-read them on every login. The script test covers a swap stopped at each of its link and rename steps: the live pair is always one whole pair, and the next run puts the new one live. Refs #5017
324 lines
14 KiB
Nix
324 lines
14 KiB
Nix
# This agent's own identity at the swarm secret store: the check that proves
|
||
# it works, and the live copy every store reader in the container presents.
|
||
#
|
||
# `swarm-controller` mints the leaf at agent creation and publishes it; this
|
||
# agent's hive collects it under the hive's own certificate and hands it in as
|
||
# systemd credentials (`hive_c0re::lifecycle::agent_identity`). That copy is
|
||
# fixed for the life of the container, and the controller re-issues the leaf
|
||
# at half its validity, so this unit seeds a live copy from it and, on a timer,
|
||
# reads the current leaf back from the store under the one it holds. Because
|
||
# that login needs the held leaf to still be valid, a container whose leaf
|
||
# expired while every pull failed recovers only at its next start.
|
||
#
|
||
# ⚠️ The identity arrives as credentials and NOT as a bind mount, and the mode
|
||
# is why: the host file is `0600` to the hive daemon and this unit runs as the
|
||
# unprivileged agent user. nspawn's `--load-credential` is read by the
|
||
# container manager as root and re-exposed under this unit's own `User=`; a
|
||
# bind would deliver a private key this user cannot open. Same answer, same
|
||
# reason, as ./queue.nix's credential pair.
|
||
#
|
||
# This is the courier's other end: the hive-side collector has no purpose
|
||
# without it, so the two ship together and neither is reachable alone.
|
||
#
|
||
# 🩸 This unit fails LOUDLY where the hive-side readers degrade quietly, and
|
||
# that is the opposite default on purpose. A missing OIDC client secret means
|
||
# a hive whose publisher has yet to run; a missing or refused certificate
|
||
# means an agent that believes it can reach the store and cannot, which every
|
||
# later pull would report as its own unrelated failure. The one place that
|
||
# knows the real cause is the login itself, so this is where it is said.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.agent.bao;
|
||
|
||
# This container's agent name. The same string the hive published the
|
||
# identity under, because the agent's unix user is named for the agent —
|
||
# see ./user.nix.
|
||
agentName = config.services.hyperhive.agent.user.name;
|
||
|
||
# The three ids `hive_c0re::lifecycle::agent_identity` forwards under.
|
||
# Neither side can discover the other's spelling, so a rename is a rename
|
||
# there too.
|
||
certCredential = "hive-agent-bao-cert";
|
||
keyCredential = "hive-agent-bao-key";
|
||
serverCaCredential = "hive-agent-bao-server-ca";
|
||
|
||
unitName = "hive-agent-bao-identity";
|
||
|
||
runtimeDir = unitName;
|
||
# A symlink to the directory that holds the live pair. Each pair is written
|
||
# into a fresh directory and goes live with one rename of this link, so the
|
||
# link always names one whole pair, old or new.
|
||
liveLink = "/run/${runtimeDir}/current";
|
||
certFile = "${liveLink}/cert";
|
||
keyFile = "${liveLink}/key";
|
||
# The store's whole answer, key included, in the unit's own `0700` directory.
|
||
rawFile = "/run/${runtimeDir}/bao-mtls.json";
|
||
errFile = "/run/${runtimeDir}/bao.err";
|
||
|
||
# Where the identity lives in the store, spelled from the same pieces the
|
||
# publisher uses. `swarm_secret_client::mtls::identity_path` builds
|
||
# `swarm/agents/<agent>/bao-mtls` and `path::MOUNT` is `secret`; this
|
||
# literal is the nix half of that one agreement, exactly as
|
||
# ../host-modules/glue-queue-agent-credential.nix spells its own.
|
||
identityPath = "secret/swarm/agents/${agentName}/bao-mtls";
|
||
|
||
# The address is the whole switch — no separate `enable`, the same shape
|
||
# ./queue.nix and ./logs.nix gate themselves with. A hive that has a store
|
||
# forwards its address and every agent on it gets the check; a hive that has
|
||
# none forwards nothing and no agent does. An `enable` beside it would be a
|
||
# knob whose only correct setting is whatever the address already says, and
|
||
# its default would decide whether the hive-side courier delivers into a
|
||
# container that reads what it is given or into one that never looks.
|
||
configured = cfg.addr != null;
|
||
|
||
storeRetry = import ../host-modules/lib/store-retry.nix { };
|
||
in
|
||
{
|
||
options.services.hyperhive.agent.bao = {
|
||
addr = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "https://bao.example.com:8200";
|
||
description = ''
|
||
Where the swarm secret store listens, as this container reaches it.
|
||
|
||
Set by the generated meta flake from the host's own `BAO_ADDR`, which
|
||
is the address this hive already uses, and setting it is what generates
|
||
`${unitName}.service`: at boot that unit logs in with the certificate
|
||
its hive delivered and reads this agent's own path back, failing if
|
||
either step does not succeed. `systemctl status ${unitName}` inside the
|
||
container is then the answer to "can this agent reach the store as
|
||
itself", which nothing else in the tree reports.
|
||
|
||
`null` means the hive was given no store. No unit is generated then,
|
||
because an agent whose swarm never minted an identity has nothing to
|
||
log in with, and a failed unit at every boot would say that in the
|
||
loudest possible way about a deployment that never asked for it.
|
||
'';
|
||
};
|
||
|
||
clientCertFile = lib.mkOption {
|
||
type = lib.types.str;
|
||
readOnly = true;
|
||
default = certFile;
|
||
description = ''
|
||
This agent's current store certificate, kept by `${unitName}.service`.
|
||
Read-only: it is where that unit writes, not a knob. Every unit that
|
||
logs in to the store as this agent points `BAO_CLIENT_CERT` here.
|
||
'';
|
||
};
|
||
|
||
clientKeyFile = lib.mkOption {
|
||
type = lib.types.str;
|
||
readOnly = true;
|
||
default = keyFile;
|
||
description = ''
|
||
The private key for `clientCertFile`, `0400` to the agent user.
|
||
🩸 A PATH and never a value.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkIf configured {
|
||
systemd.services.${unitName} = {
|
||
description = "keep this agent's swarm secret store identity current, and prove it works";
|
||
after = [ "network.target" ];
|
||
wantedBy = [ "multi-user.target" ];
|
||
path = [
|
||
pkgs.openbao
|
||
pkgs.coreutils
|
||
pkgs.diffutils
|
||
pkgs.jq
|
||
];
|
||
# ../host-modules/lib/store-retry.nix. From an agent the store is reached
|
||
# through the gateway's stream passthrough, so either being down fails
|
||
# the login below.
|
||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||
serviceConfig = storeRetry.serviceConfig // {
|
||
Type = "oneshot";
|
||
# Not `RemainAfterExit`, so the timer below can start it again.
|
||
# `RuntimeDirectoryPreserve` keeps the live pair between runs.
|
||
RemainAfterExit = false;
|
||
TimeoutStartSec = 30;
|
||
User = agentName;
|
||
Group = agentName;
|
||
RuntimeDirectory = runtimeDir;
|
||
RuntimeDirectoryMode = "0700";
|
||
RuntimeDirectoryPreserve = "yes";
|
||
UMask = "0377";
|
||
# Bare ids, no paths: the terse `LoadCredential=` form that inherits a
|
||
# credential the service *manager* received, which is what the
|
||
# container manager passed in. ./queue.nix states the same shape.
|
||
LoadCredential = [
|
||
certCredential
|
||
keyCredential
|
||
serverCaCredential
|
||
];
|
||
};
|
||
environment = {
|
||
BAO_ADDR = cfg.addr;
|
||
BAO_CLIENT_CERT = certFile;
|
||
BAO_CLIENT_KEY = keyFile;
|
||
};
|
||
script = ''
|
||
set -euo pipefail
|
||
|
||
# Absent credentials are the first thing checked, because every later
|
||
# message would blame the store for a delivery that never happened.
|
||
# 🩸 The files are named and never read here: what went wrong is a
|
||
# property of the path, and the bytes at it are a private key.
|
||
missing=
|
||
for id in ${lib.escapeShellArg certCredential} ${lib.escapeShellArg keyCredential}; do
|
||
if [ ! -s "$CREDENTIALS_DIRECTORY/$id" ]; then
|
||
missing="$missing $id"
|
||
fi
|
||
done
|
||
if [ -n "$missing" ]; then
|
||
echo "this agent has no store identity: the hive delivered no$missing." >&2
|
||
echo "swarm-controller publishes it at ${identityPath} when the agent is created, and the hive collects it from there." >&2
|
||
exit 1
|
||
fi
|
||
|
||
# Only when one was delivered. Absent means the container verifies the
|
||
# store's listener against its own trust store, which is what a
|
||
# deployment with a real CA wants; pointing BAO_CACERT at a file that
|
||
# is not there would fail the handshake and name the wrong cause.
|
||
if [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then
|
||
export BAO_CACERT="$CREDENTIALS_DIRECTORY/${serverCaCredential}"
|
||
fi
|
||
|
||
# `UMask=0377` makes every file this script creates `0400`, so each one
|
||
# is removed before the command that creates it.
|
||
err=${lib.escapeShellArg errFile}
|
||
raw=${lib.escapeShellArg rawFile}
|
||
trap 'rm -f "$err" "$raw"' EXIT
|
||
|
||
live=${lib.escapeShellArg liveLink}
|
||
dir=''${live%/*}
|
||
|
||
# A fresh directory for one pair. `chmod`, because the umask cuts
|
||
# mktemp's `0700` down to a `0400` no file can be created in.
|
||
new_pair() {
|
||
local d
|
||
d=$(mktemp -d -p "$dir" pair.XXXXXX)
|
||
chmod 0700 "$d"
|
||
echo "$d"
|
||
}
|
||
|
||
# The only step that changes the live pair: one rename of the link. A
|
||
# reader whose two opens straddle it gets one file of each pair for
|
||
# that one login; the next login reads a whole pair.
|
||
go_live() {
|
||
rm -f "$live.new"
|
||
ln -s "''${1##*/}" "$live.new"
|
||
mv -Tf "$live.new" "$live"
|
||
}
|
||
|
||
# Every pair directory but the live one is left over: the pair the
|
||
# last swap replaced, kept until now for a reader that resolved the
|
||
# link before the swap, or a pair a stopped run staged.
|
||
current=$(readlink "$live" || true)
|
||
for d in "$dir"/pair.*; do
|
||
if [ -d "$d" ] && [ "''${d##*/}" != "$current" ]; then
|
||
chmod 0700 "$d"
|
||
rm -rf "$d"
|
||
fi
|
||
done
|
||
rm -f "$live.new"
|
||
|
||
# The delivered pair is the start of every container's life: /run is
|
||
# empty at boot and kept across this unit's runs.
|
||
if [ ! -s "$BAO_CLIENT_CERT" ] || [ ! -s "$BAO_CLIENT_KEY" ]; then
|
||
seed=$(new_pair)
|
||
cp "$CREDENTIALS_DIRECTORY/${keyCredential}" "$seed/key"
|
||
cp "$CREDENTIALS_DIRECTORY/${certCredential}" "$seed/cert"
|
||
go_live "$seed"
|
||
fi
|
||
|
||
# Cert auth is a login, not a transport setting. The `BAO_CLIENT_*`
|
||
# variables above only decide which certificate the TLS handshake
|
||
# presents; without a token `bao` asks its token helper instead, and
|
||
# that is a `sh` this unit's `path` does not carry. `-token-only`
|
||
# answers on stdout and skips the helper on both sides.
|
||
#
|
||
# No `name=`: a cert role pins both its authority and the common name
|
||
# it accepts, so this agent's leaf matches its own role and no other.
|
||
# Naming the role here would be a third copy of a string
|
||
# `swarm_secret_client::policy::agent_object_name` already owns.
|
||
rm -f "$err"
|
||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||
echo "this agent's certificate was refused by the swarm secret store at $BAO_ADDR." >&2
|
||
if [ -s "$err" ]; then
|
||
cat "$err" >&2
|
||
else
|
||
echo "bao failed without writing a diagnostic." >&2
|
||
fi
|
||
exit 1
|
||
fi
|
||
export BAO_TOKEN
|
||
|
||
# A token is not yet an answer: the login proves the certificate, this
|
||
# proves the policy attached to it, and the answer is the current
|
||
# leaf. One read, so the certificate and key come from one version.
|
||
#
|
||
# ⚠️ Written by redirect, never printed: the object holds a private
|
||
# key.
|
||
rm -f "$err" "$raw"
|
||
if ! bao kv get -format=json ${lib.escapeShellArg identityPath} >"$raw" 2>"$err"; then
|
||
echo "this agent logged in to the swarm secret store but cannot read ${identityPath}, so its policy does not cover its own path." >&2
|
||
if [ -s "$err" ]; then
|
||
cat "$err" >&2
|
||
else
|
||
echo "bao failed without writing a diagnostic." >&2
|
||
fi
|
||
exit 1
|
||
fi
|
||
|
||
# `-j`: the bytes as the store holds them, which is what the hive
|
||
# delivered, so an unchanged leaf compares equal to the seeded copy.
|
||
staged=$(new_pair)
|
||
if ! jq -ej '.data.data.cert | strings' "$raw" >"$staged/cert" \
|
||
|| ! jq -ej '.data.data.key | strings' "$raw" >"$staged/key"; then
|
||
rm -rf "$staged"
|
||
echo "${identityPath} holds no certificate and key; this agent keeps the one it has." >&2
|
||
exit 1
|
||
fi
|
||
|
||
if cmp -s "$staged/cert" "$BAO_CLIENT_CERT" && cmp -s "$staged/key" "$BAO_CLIENT_KEY"; then
|
||
rm -rf "$staged"
|
||
echo "authenticated to the swarm secret store at $BAO_ADDR as ${agentName}; its certificate at ${identityPath} is unchanged."
|
||
else
|
||
# This unit's next login uses whatever goes live, so a pair the
|
||
# store refuses must never replace the one it accepts.
|
||
rm -f "$err"
|
||
if ! BAO_CLIENT_CERT="$staged/cert" BAO_CLIENT_KEY="$staged/key" \
|
||
bao login -method=cert -token-only >/dev/null 2>"$err"; then
|
||
rm -rf "$staged"
|
||
echo "the swarm secret store refused the certificate at ${identityPath}; this agent keeps the one it has." >&2
|
||
if [ -s "$err" ]; then cat "$err" >&2; fi
|
||
exit 1
|
||
fi
|
||
|
||
go_live "$staged"
|
||
echo "replaced this agent's store certificate with the one at ${identityPath}."
|
||
fi
|
||
'';
|
||
};
|
||
|
||
# The controller re-issues a leaf at half its validity, so ten minutes is
|
||
# small against the time the old one stays valid.
|
||
systemd.timers.${unitName} = {
|
||
description = "re-read this agent's store certificate from the secret store";
|
||
wantedBy = [ "timers.target" ];
|
||
timerConfig = {
|
||
OnUnitInactiveSec = "10min";
|
||
RandomizedDelaySec = "1min";
|
||
};
|
||
};
|
||
};
|
||
}
|