vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

swarm-bao: refuse a remote reader that named seven of the eight leaves

Browse source 
The four-way client-cert split gives each store reader its own leaf, and
three of the four readers render only where their own leaf exists. On a
host that mints its own PKI glue-bao-tls.nix defaults all eight, so there
is nothing to do; on a hand-configured remote-store hive, omitting one
pair used to mean that unit silently did not render — a privilege-
narrowing unit absent from a green build, with the missing unit as the
only evidence.

Each of the three now asserts its own pair, shaped after
swarm-grafana.nix's haveClientIdentity assertion and named to the pair it
needs. What differs from Grafana's is the gate: these fire only where the
host demonstrably reads the store (it holds deploy.bao.clientCertFile and
clientKeyFile) and the consumer is on. A host with no store identity is
the supported no-store deployment and still evaluates; the collector's
no-secret degrade is untouched, because that host holds no clientCertFile
either.

Also rewords three passive-voice sentences in docs/swarm/secrets.md that
vale flagged, and documents what the refusal costs and where it stays
silent.
This commit is contained in:
atlas 2026-09-23 09:56:43 +02:00 • committed by mara
commit d3e4951cc8
6 changed files with 627 additions and 276 deletions
Download patch file Download diff file Expand all files Collapse all files

310
nix/host-modules/glue-queue-agent-credential.nix
View file

@ -51,6 +51,14 @@ let
haveClientIdentity =
baoDeploy.queueAgentClientCertFile != null && baoDeploy.queueAgentClientKeyFile != null;
# Does this host read the store at all — the hive's own leaf, which is the
# one thing a remote-store deployment has always had to place by hand. Only
# used to decide whether a missing per-principal leaf is a mistake or a
# deployment that has no store: a host holding neither is the supported
# no-store shape, and one holding this pair but not the pair above named
# seven of the eight options and stopped.
hiveReaderIdentity = baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null;
credentialDir = toString deployCfg.hive-controller.queue.agentCredentialDir;
secretFile = "${credentialDir}/secret";
clientIdFile = "${credentialDir}/client_id";
@ -94,144 +102,192 @@ in
};
};
config = lib.mkIf (hyperhiveCfg.enable && haveClientIdentity) {
# Same rule as the unit's own gate: this reader exists on any host holding
# a client identity, which is not every host that runs the store, so the
# store's module cannot name it.
services.hyperhive.swarm.otel.journaldUnits = [ "swarm-bao-queue-agent" ];
config = lib.mkMerge [
# ⚠️ A SEPARATE arm from the unit below, and that separation is the whole
# mechanism: the unit's arm is gated on `haveClientIdentity`, so an
# assertion written inside it could never be reached in the state it
# exists to report.
#
# Shaped after ./swarm-grafana.nix's `haveClientIdentity` assertion — the
# same refusal, named to this principal's own pair. Where Grafana's gate
# is `deploy.grafana.enable`, this reader has no toggle of its own to
# check: every hive runs one for its own agents, so reading the store at
# all is what asks for it. Hence `hiveReaderIdentity` alone — a host
# holding no hive leaf has no store to read and nothing is missing.
(lib.mkIf (hyperhiveCfg.enable && hiveReaderIdentity) {
assertions = [
{
assertion = haveClientIdentity;
message = ''
This host reads the swarm secret store (services.hyperhive.deploy.bao.clientCertFile
is set), so it needs the agent queue credential reader's own client
identity: set both
systemd.services.swarm-bao-queue-agent = {
description = "fetch this hive's agent queue credential from the swarm secret store";
# Every one of these names a unit that exists only where the store runs.
# `Requires=` on an absent unit fails the job outright, so the ordering is
# conditional even though the read is not: off-host there is nothing local
# to wait for, and the timeout below is what bounds the attempt instead.
after = lib.optionals baoDeploy.enable [
"swarm-bao-pki.service"
"container@${baoCfg.machine}.service"
];
wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
# Ordered before hive-c0re, so no agent container renders ahead of an
# attempt at its credential. `Wants=`, not `Requires=`: a store this
# unit can't reach delays hive-c0re's start by its own start-limit
# window (`TimeoutStartSec`, retried up to `startLimitBurst` times
# below) rather than failing it — hive-c0re starts once that window
# elapses, whatever credential is or isn't on disk by then.
before = [ "hive-c0re.service" ];
wantedBy = [
"multi-user.target"
"hive-c0re.service"
];
path = [
baoDeploy.package
pkgs.coreutils
];
# Sized for the race this loses, not for an unseal: `swarm-bao` comes up
# seconds before this unit asks, and the cert-auth role it logs in
# against is written seconds after, so a few short attempts cover it.
# An hours-long window would be a bet on a store that is sealed, and the
# degrade below is already correct for that.
#
# `StartLimit*` are `[Unit]` settings, so they go here and not in
# `serviceConfig` — systemd ignores them under `[Service]`. The window
# has to exceed `RestartSec × burst`.
startLimitBurst = 4;
startLimitIntervalSec = 300;
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
# What actually bounds the reads below. Stated here rather than left to
# systemd's default, so the number a boot waits on is in the file that
# waits.
TimeoutStartSec = 30;
Restart = "on-failure";
RestartSec = 15;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = baoDeploy.queueAgentClientCertFile;
BAO_CLIENT_KEY = baoDeploy.queueAgentClientKeyFile;
}
# Absent means the system trust store, which is what a deployment with a
# real CA wants and what a self-signed one must not be left with.
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
script = ''
set -euo pipefail
services.hyperhive.deploy.bao.queueAgentClientCertFile
services.hyperhive.deploy.bao.queueAgentClientKeyFile
# `bao`'s own message is the only thing separating a missing value from
# a refused identity from an unreachable host. This unit's degraded
# mode is correct for all three, so it reports which one rather than
# asserting all three in a sentence of ours.
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
swarm-bao-queue-agent.service fetches this hive's agent queue
credential out of the store, and without these it is not rendered
at all — leaving hive-c0re with no queue credential and no unit
that would ever have written one.
# Cert auth is a login, not a transport setting. The `BAO_CLIENT_*`
# variables above only decide which certificate the TLS handshake
# presents; without a token `bao` asks its token helper instead, and
# that is a `sh` this unit's `path` does not carry. `-token-only`
# answers on stdout and skips the helper on both sides.
Every hive runs this reader for its own agents, so unlike the other
three principals there is no per-service toggle that turns it off:
a hive that reads the store at all is a hive that needs this pair.
⚠️ This reader's OWN leaf, not deploy.bao.clientCertFile. That one
is the hive's, and its grant reads every secret in the store; this
role reads the one queue-credential path. Pointing this option at
the hive's leaf would evaluate, deploy and log in — and undo the
split.
On a hive that runs the store, glue-bao-tls.nix supplies both as
defaults and there is nothing to do. Elsewhere the leaf is issued
from that CA out of band and named here — see docs/swarm/secrets.md.
'';
}
];
})
(lib.mkIf (hyperhiveCfg.enable && haveClientIdentity) {
# Same rule as the unit's own gate: this reader exists on any host holding
# a client identity, which is not every host that runs the store, so the
# store's module cannot name it.
services.hyperhive.swarm.otel.journaldUnits = [ "swarm-bao-queue-agent" ];
systemd.services.swarm-bao-queue-agent = {
description = "fetch this hive's agent queue credential from the swarm secret store";
# Every one of these names a unit that exists only where the store runs.
# `Requires=` on an absent unit fails the job outright, so the ordering is
# conditional even though the read is not: off-host there is nothing local
# to wait for, and the timeout below is what bounds the attempt instead.
after = lib.optionals baoDeploy.enable [
"swarm-bao-pki.service"
"container@${baoCfg.machine}.service"
];
wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
# Ordered before hive-c0re, so no agent container renders ahead of an
# attempt at its credential. `Wants=`, not `Requires=`: a store this
# unit can't reach delays hive-c0re's start by its own start-limit
# window (`TimeoutStartSec`, retried up to `startLimitBurst` times
# below) rather than failing it — hive-c0re starts once that window
# elapses, whatever credential is or isn't on disk by then.
before = [ "hive-c0re.service" ];
wantedBy = [
"multi-user.target"
"hive-c0re.service"
];
path = [
baoDeploy.package
pkgs.coreutils
];
# Sized for the race this loses, not for an unseal: `swarm-bao` comes up
# seconds before this unit asks, and the cert-auth role it logs in
# against is written seconds after, so a few short attempts cover it.
# An hours-long window would be a bet on a store that is sealed, and the
# degrade below is already correct for that.
#
# Fails LOUDLY, unlike the reads below: the three states a login
# failure covers — store not up, sealed, role not written yet — are all
# things a retry fixes, and `Restart=on-failure` above is what retries.
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with this host's certificate; leaving the queue credential in ${credentialDir} as it is." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
# `StartLimit*` are `[Unit]` settings, so they go here and not in
# `serviceConfig` — systemd ignores them under `[Service]`. The window
# has to exceed `RestartSec × burst`.
startLimitBurst = 4;
startLimitIntervalSec = 300;
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
# What actually bounds the reads below. Stated here rather than left to
# systemd's default, so the number a boot waits on is in the file that
# waits.
TimeoutStartSec = 30;
Restart = "on-failure";
RestartSec = 15;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = baoDeploy.queueAgentClientCertFile;
BAO_CLIENT_KEY = baoDeploy.queueAgentClientKeyFile;
}
# Absent means the system trust store, which is what a deployment with a
# real CA wants and what a self-signed one must not be left with.
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
script = ''
set -euo pipefail
# `bao`'s own message is the only thing separating a missing value from
# a refused identity from an unreachable host. This unit's degraded
# mode is correct for all three, so it reports which one rather than
# asserting all three in a sentence of ours.
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
# Cert auth is a login, not a transport setting. The `BAO_CLIENT_*`
# variables above only decide which certificate the TLS handshake
# presents; without a token `bao` asks its token helper instead, and
# that is a `sh` this unit's `path` does not carry. `-token-only`
# answers on stdout and skips the helper on both sides.
#
# Fails LOUDLY, unlike the reads below: the three states a login
# failure covers — store not up, sealed, role not written yet — are all
# things a retry fixes, and `Restart=on-failure` above is what retries.
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with this host's certificate; leaving the queue credential in ${credentialDir} as it is." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
exit 1
fi
export BAO_TOKEN
export BAO_TOKEN
# Two reads of one object rather than one `-format=json` parsed with
# `jq`: no sibling unit carries `jq` on its `path`, and the pair cannot
# actually disagree — the client id is derived from this hive's name, so
# a rotation landing between these two calls changes the secret and
# rewrites the same id.
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg credentialPath} 2>"$err")"; then
echo "swarm-bao did not return ${credentialPath}; this hive's agents have no queue credential yet." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
# Two reads of one object rather than one `-format=json` parsed with
# `jq`: no sibling unit carries `jq` on its `path`, and the pair cannot
# actually disagree — the client id is derived from this hive's name, so
# a rotation landing between these two calls changes the secret and
# rewrites the same id.
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg credentialPath} 2>"$err")"; then
echo "swarm-bao did not return ${credentialPath}; this hive's agents have no queue credential yet." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 0
fi
exit 0
fi
if ! client_id="$(bao kv get -field=client_id ${lib.escapeShellArg credentialPath} 2>"$err")"; then
echo "${credentialPath} holds no client_id; the secret alone is not a usable credential, so nothing is written." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
if ! client_id="$(bao kv get -field=client_id ${lib.escapeShellArg credentialPath} 2>"$err")"; then
echo "${credentialPath} holds no client_id; the secret alone is not a usable credential, so nothing is written." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 0
fi
exit 0
fi
# Both or neither, for the reason `QueueConfig::from_env` refuses a
# half-set environment: a client that finds one of the two comes up
# "fine" and never connects.
if [ -z "$secret" ] || [ -z "$client_id" ]; then
echo "swarm-bao returned an empty field of ${credentialPath}; leaving the files as they are." >&2
exit 0
fi
# Both or neither, for the reason `QueueConfig::from_env` refuses a
# half-set environment: a client that finds one of the two comes up
# "fine" and never connects.
if [ -z "$secret" ] || [ -z "$client_id" ]; then
echo "swarm-bao returned an empty field of ${credentialPath}; leaving the files as they are." >&2
exit 0
fi
install -d -m 0755 ${lib.escapeShellArg credentialDir}
install -d -m 0755 ${lib.escapeShellArg credentialDir}
umask 077
printf '%s\n' "$secret" > ${lib.escapeShellArg secretFile}
chmod 0600 ${lib.escapeShellArg secretFile}
umask 077
printf '%s\n' "$secret" > ${lib.escapeShellArg secretFile}
chmod 0600 ${lib.escapeShellArg secretFile}
# `0644` on purpose: an OIDC client id is presented to the token
# endpoint on every connection and is public by construction.
printf '%s\n' "$client_id" > ${lib.escapeShellArg clientIdFile}
chmod 0644 ${lib.escapeShellArg clientIdFile}
'';
};
};
# `0644` on purpose: an OIDC client id is presented to the token
# endpoint on every connection and is public by construction.
printf '%s\n' "$client_id" > ${lib.escapeShellArg clientIdFile}
chmod 0644 ${lib.escapeShellArg clientIdFile}
'';
};
})
];
}