d3e4951cc8
The four-way client-cert split gives each store reader its own leaf, and three of the four readers render only where their own leaf exists. On a host that mints its own PKI glue-bao-tls.nix defaults all eight, so there is nothing to do; on a hand-configured remote-store hive, omitting one pair used to mean that unit silently did not render — a privilege- narrowing unit absent from a green build, with the missing unit as the only evidence. Each of the three now asserts its own pair, shaped after swarm-grafana.nix's haveClientIdentity assertion and named to the pair it needs. What differs from Grafana's is the gate: these fire only where the host demonstrably reads the store (it holds deploy.bao.clientCertFile and clientKeyFile) and the consumer is on. A host with no store identity is the supported no-store deployment and still evaluates; the collector's no-secret degrade is untouched, because that host holds no clientCertFile either. Also rewords three passive-voice sentences in docs/swarm/secrets.md that vale flagged, and documents what the refusal costs and where it stays silent.
293 lines
14 KiB
Nix
293 lines
14 KiB
Nix
# Glue: this hive's agent queue credential comes out of the secret store.
|
||
#
|
||
# The store's second reader, and deliberately the same shape as its first
|
||
# (./glue-matrix-bao-token.nix): a cert login that fails LOUDLY because every
|
||
# state it fails on is one a retry fixes, then a read that degrades QUIETLY
|
||
# because no retry turns "no value there" into a value.
|
||
#
|
||
# ⚠️ Two files, not one, and that is the consumer's shape rather than a
|
||
# preference. `swarm_queue_client::QueueConfig::from_env` takes the secret as
|
||
# a PATH (`<prefix>_OIDC_CLIENT_SECRET_FILE`) and the client id as a VALUE
|
||
# (`<prefix>_OIDC_CLIENT_ID`), so splitting them here is what lets the next
|
||
# slice hand both to an agent without parsing anything.
|
||
#
|
||
# ⚠️ An ABSENT file means this hive's agents do not connect to the queue, and
|
||
# that is correct rather than degraded. The publisher runs on the authelia
|
||
# host and authelia mints on its first boot, so "nothing at that path yet" is
|
||
# the ordinary early state of a swarm. Nothing here writes a local stand-in:
|
||
# unlike a matrix appservice token there is no such thing as a locally valid
|
||
# OIDC client secret, so a placeholder would turn a hive that cannot connect
|
||
# into one that is refused, which reaches the agent as a timeout.
|
||
#
|
||
# 📌 This runs wherever a client identity is configured, NOT only where the
|
||
# store is — the rule ./glue-matrix-bao-token.nix states in full. There is no
|
||
# "this hive runs agents" condition to gate it on as well: agent containers
|
||
# are created at runtime by hive-c0re, so every hyperhive host is a host that
|
||
# may run one.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
hyperhiveCfg = config.services.hyperhive;
|
||
deployCfg = hyperhiveCfg.deploy;
|
||
baoCfg = hyperhiveCfg.swarm.bao;
|
||
|
||
baoDeploy = deployCfg.bao;
|
||
|
||
# What decides whether this unit exists at all. A reader is defined by holding
|
||
# a certificate the store accepts, and that is true on the store's own host
|
||
# and on a hive three networks away for exactly the same reason.
|
||
#
|
||
# 🩸 This principal's OWN leaf, not `clientCertFile` — the hive's, which four
|
||
# units used to share. Bao matches a cert-auth role on the CN, so one leaf for
|
||
# four readers was ONE principal holding the union of four grants: read on
|
||
# `swarm/agents/*` AND `swarm/hives/<hive>/*` AND `swarm/services/*`, when
|
||
# this unit reads one queue credential and nothing else. Its own leaf carries
|
||
# `<queueAgentCommonNamePrefix>-<hive>` and its role grants the single path
|
||
# below — still this hive's own, so the narrowing costs no reach.
|
||
haveClientIdentity =
|
||
baoDeploy.queueAgentClientCertFile != null && baoDeploy.queueAgentClientKeyFile != null;
|
||
|
||
# Does this host read the store at all — the hive's own leaf, which is the
|
||
# one thing a remote-store deployment has always had to place by hand. Only
|
||
# used to decide whether a missing per-principal leaf is a mistake or a
|
||
# deployment that has no store: a host holding neither is the supported
|
||
# no-store shape, and one holding this pair but not the pair above named
|
||
# seven of the eight options and stopped.
|
||
hiveReaderIdentity = baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null;
|
||
|
||
credentialDir = toString deployCfg.hive-controller.queue.agentCredentialDir;
|
||
secretFile = "${credentialDir}/secret";
|
||
clientIdFile = "${credentialDir}/client_id";
|
||
|
||
# Where the credential lives in the store, spelled from the same pieces the
|
||
# writer uses. Whoever writes it and whoever reads it must agree, and the
|
||
# agreement belongs in one visible place.
|
||
#
|
||
# ⚠️ The `hives/<name>` segment is not decoration — it is what the reader's
|
||
# own grant covers, so a path outside it is a 403 rather than a miss, however
|
||
# correct it looks. `swarm-secret-client`'s `queue::agent_client_path` builds
|
||
# the same string from the same pieces; this literal is the nix half of that
|
||
# one agreement.
|
||
#
|
||
# `hiveName` has no fallback here for the reason ./glue-bao-tls.nix gives at
|
||
# its own use of it: it is asserted set for every hyperhive host.
|
||
credentialPath = "secret/swarm/hives/${hyperhiveCfg.hiveName}/queue/agent";
|
||
in
|
||
{
|
||
options.services.hyperhive.deploy.hive-controller.queue = {
|
||
agentCredentialDir = lib.mkOption {
|
||
type = lib.types.path;
|
||
default = "/var/lib/hyperhive/queue-agent";
|
||
description = ''
|
||
Host directory holding this hive's agent queue credential: `secret`
|
||
(the OIDC client secret, `0600`) and `client_id` (the client that
|
||
secret authenticates, `0644` — it is sent to the token endpoint on
|
||
every connection and is not itself a secret).
|
||
|
||
Both files are written by `swarm-bao-queue-agent.service`, which also
|
||
creates this directory. Named here because nothing else creates it:
|
||
the parent `/var/lib/hyperhive` belongs to `hive-c0re.service`'s
|
||
`StateDirectory=`, which re-applies its own mode on every start, and a
|
||
second declaration of *that* path is the two-mechanisms-one-path trap
|
||
./hive-gateway/default.nix records — so this directory is the unit's
|
||
to make and the parent stays c0re's.
|
||
|
||
Absent files mean this hive has no queue credential yet, which is what
|
||
a swarm looks like before the publisher on the authelia host has run.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkMerge [
|
||
# ⚠️ A SEPARATE arm from the unit below, and that separation is the whole
|
||
# mechanism: the unit's arm is gated on `haveClientIdentity`, so an
|
||
# assertion written inside it could never be reached in the state it
|
||
# exists to report.
|
||
#
|
||
# Shaped after ./swarm-grafana.nix's `haveClientIdentity` assertion — the
|
||
# same refusal, named to this principal's own pair. Where Grafana's gate
|
||
# is `deploy.grafana.enable`, this reader has no toggle of its own to
|
||
# check: every hive runs one for its own agents, so reading the store at
|
||
# all is what asks for it. Hence `hiveReaderIdentity` alone — a host
|
||
# holding no hive leaf has no store to read and nothing is missing.
|
||
(lib.mkIf (hyperhiveCfg.enable && hiveReaderIdentity) {
|
||
assertions = [
|
||
{
|
||
assertion = haveClientIdentity;
|
||
message = ''
|
||
This host reads the swarm secret store (services.hyperhive.deploy.bao.clientCertFile
|
||
is set), so it needs the agent queue credential reader's own client
|
||
identity: set both
|
||
|
||
services.hyperhive.deploy.bao.queueAgentClientCertFile
|
||
services.hyperhive.deploy.bao.queueAgentClientKeyFile
|
||
|
||
swarm-bao-queue-agent.service fetches this hive's agent queue
|
||
credential out of the store, and without these it is not rendered
|
||
at all — leaving hive-c0re with no queue credential and no unit
|
||
that would ever have written one.
|
||
|
||
Every hive runs this reader for its own agents, so unlike the other
|
||
three principals there is no per-service toggle that turns it off:
|
||
a hive that reads the store at all is a hive that needs this pair.
|
||
|
||
⚠️ This reader's OWN leaf, not deploy.bao.clientCertFile. That one
|
||
is the hive's, and its grant reads every secret in the store; this
|
||
role reads the one queue-credential path. Pointing this option at
|
||
the hive's leaf would evaluate, deploy and log in — and undo the
|
||
split.
|
||
|
||
On a hive that runs the store, glue-bao-tls.nix supplies both as
|
||
defaults and there is nothing to do. Elsewhere the leaf is issued
|
||
from that CA out of band and named here — see docs/swarm/secrets.md.
|
||
'';
|
||
}
|
||
];
|
||
})
|
||
|
||
(lib.mkIf (hyperhiveCfg.enable && haveClientIdentity) {
|
||
# Same rule as the unit's own gate: this reader exists on any host holding
|
||
# a client identity, which is not every host that runs the store, so the
|
||
# store's module cannot name it.
|
||
services.hyperhive.swarm.otel.journaldUnits = [ "swarm-bao-queue-agent" ];
|
||
|
||
systemd.services.swarm-bao-queue-agent = {
|
||
description = "fetch this hive's agent queue credential from the swarm secret store";
|
||
# Every one of these names a unit that exists only where the store runs.
|
||
# `Requires=` on an absent unit fails the job outright, so the ordering is
|
||
# conditional even though the read is not: off-host there is nothing local
|
||
# to wait for, and the timeout below is what bounds the attempt instead.
|
||
after = lib.optionals baoDeploy.enable [
|
||
"swarm-bao-pki.service"
|
||
"container@${baoCfg.machine}.service"
|
||
];
|
||
wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
|
||
requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
|
||
# Ordered before hive-c0re, so no agent container renders ahead of an
|
||
# attempt at its credential. `Wants=`, not `Requires=`: a store this
|
||
# unit can't reach delays hive-c0re's start by its own start-limit
|
||
# window (`TimeoutStartSec`, retried up to `startLimitBurst` times
|
||
# below) rather than failing it — hive-c0re starts once that window
|
||
# elapses, whatever credential is or isn't on disk by then.
|
||
before = [ "hive-c0re.service" ];
|
||
wantedBy = [
|
||
"multi-user.target"
|
||
"hive-c0re.service"
|
||
];
|
||
path = [
|
||
baoDeploy.package
|
||
pkgs.coreutils
|
||
];
|
||
# Sized for the race this loses, not for an unseal: `swarm-bao` comes up
|
||
# seconds before this unit asks, and the cert-auth role it logs in
|
||
# against is written seconds after, so a few short attempts cover it.
|
||
# An hours-long window would be a bet on a store that is sealed, and the
|
||
# degrade below is already correct for that.
|
||
#
|
||
# `StartLimit*` are `[Unit]` settings, so they go here and not in
|
||
# `serviceConfig` — systemd ignores them under `[Service]`. The window
|
||
# has to exceed `RestartSec × burst`.
|
||
startLimitBurst = 4;
|
||
startLimitIntervalSec = 300;
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
# What actually bounds the reads below. Stated here rather than left to
|
||
# systemd's default, so the number a boot waits on is in the file that
|
||
# waits.
|
||
TimeoutStartSec = 30;
|
||
Restart = "on-failure";
|
||
RestartSec = 15;
|
||
};
|
||
environment = {
|
||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
BAO_CLIENT_CERT = baoDeploy.queueAgentClientCertFile;
|
||
BAO_CLIENT_KEY = baoDeploy.queueAgentClientKeyFile;
|
||
}
|
||
# Absent means the system trust store, which is what a deployment with a
|
||
# real CA wants and what a self-signed one must not be left with.
|
||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||
BAO_CACERT = baoDeploy.serverCaFile;
|
||
};
|
||
script = ''
|
||
set -euo pipefail
|
||
|
||
# `bao`'s own message is the only thing separating a missing value from
|
||
# a refused identity from an unreachable host. This unit's degraded
|
||
# mode is correct for all three, so it reports which one rather than
|
||
# asserting all three in a sentence of ours.
|
||
err="$(mktemp)"
|
||
trap 'rm -f "$err"' EXIT
|
||
|
||
# Cert auth is a login, not a transport setting. The `BAO_CLIENT_*`
|
||
# variables above only decide which certificate the TLS handshake
|
||
# presents; without a token `bao` asks its token helper instead, and
|
||
# that is a `sh` this unit's `path` does not carry. `-token-only`
|
||
# answers on stdout and skips the helper on both sides.
|
||
#
|
||
# Fails LOUDLY, unlike the reads below: the three states a login
|
||
# failure covers — store not up, sealed, role not written yet — are all
|
||
# things a retry fixes, and `Restart=on-failure` above is what retries.
|
||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||
echo "could not log in to swarm-bao with this host's certificate; leaving the queue credential in ${credentialDir} as it is." >&2
|
||
if [ -s "$err" ]; then
|
||
cat "$err" >&2
|
||
else
|
||
echo "bao failed without writing a diagnostic." >&2
|
||
fi
|
||
exit 1
|
||
fi
|
||
export BAO_TOKEN
|
||
|
||
# Two reads of one object rather than one `-format=json` parsed with
|
||
# `jq`: no sibling unit carries `jq` on its `path`, and the pair cannot
|
||
# actually disagree — the client id is derived from this hive's name, so
|
||
# a rotation landing between these two calls changes the secret and
|
||
# rewrites the same id.
|
||
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg credentialPath} 2>"$err")"; then
|
||
echo "swarm-bao did not return ${credentialPath}; this hive's agents have no queue credential yet." >&2
|
||
if [ -s "$err" ]; then
|
||
cat "$err" >&2
|
||
else
|
||
echo "bao failed without writing a diagnostic." >&2
|
||
fi
|
||
exit 0
|
||
fi
|
||
|
||
if ! client_id="$(bao kv get -field=client_id ${lib.escapeShellArg credentialPath} 2>"$err")"; then
|
||
echo "${credentialPath} holds no client_id; the secret alone is not a usable credential, so nothing is written." >&2
|
||
if [ -s "$err" ]; then
|
||
cat "$err" >&2
|
||
else
|
||
echo "bao failed without writing a diagnostic." >&2
|
||
fi
|
||
exit 0
|
||
fi
|
||
|
||
# Both or neither, for the reason `QueueConfig::from_env` refuses a
|
||
# half-set environment: a client that finds one of the two comes up
|
||
# "fine" and never connects.
|
||
if [ -z "$secret" ] || [ -z "$client_id" ]; then
|
||
echo "swarm-bao returned an empty field of ${credentialPath}; leaving the files as they are." >&2
|
||
exit 0
|
||
fi
|
||
|
||
install -d -m 0755 ${lib.escapeShellArg credentialDir}
|
||
|
||
umask 077
|
||
printf '%s\n' "$secret" > ${lib.escapeShellArg secretFile}
|
||
chmod 0600 ${lib.escapeShellArg secretFile}
|
||
|
||
# `0644` on purpose: an OIDC client id is presented to the token
|
||
# endpoint on every connection and is public by construction.
|
||
printf '%s\n' "$client_id" > ${lib.escapeShellArg clientIdFile}
|
||
chmod 0644 ${lib.escapeShellArg clientIdFile}
|
||
'';
|
||
};
|
||
})
|
||
];
|
||
}
|