vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm: OIDC clients registered wherever authelia runs; nats + matrix secrets from bao

Browse source 
The three client registrations move into glue-{controller,nats,matrix}-oidc-client.nix,
gated only on deploy.authelia.enable and reading their ids from swarm.* options,
so the host running authelia registers every one of them whichever host runs
the service. The matrix callback is a new read-only swarm.matrix.sso.redirectUri,
read by both the homeserver's config and its client entry.

swarm-secret-publish now also publishes the queue's and the homeserver's minted
secrets to swarm/services/<id>/oidc/client. The queue host reads its responder's
secret there with swarm-bao-nats-auth-oidc under the responder's own leaf
(swarm-nats-auth role gains that one read path), and the homeserver's host with
swarm-bao-matrix-oidc under matrix-ctl's leaf (swarm-matrix-ctl role gains its
one read path). The host-file copies are gone: swarm-nats-auth-secrets now
delivers only the callout seeds and the store identity, and
hive-matrix-oidc-secret is removed. A fetch unit whose leaf is unset still
renders and fails naming the option pair.

Closes #4943
This commit is contained in:
atlas 2026-10-05 02:56:18 +02:00 • committed by mara
commit 962da3b784
20 changed files with 702 additions and 293 deletions
Download patch file Download diff file Expand all files Collapse all files

219
nix/host-modules/swarm-nats.nix
View file

@ -57,7 +57,10 @@ let
responderConfigured =
deployCfg.nats.autoGenerateCallout
|| (deployCfg.nats.calloutUserSeedFile != "" && deployCfg.nats.calloutIssuerSeedFile != "");
clientSecretSource = "${deployCfg.authelia.hostClientSecretDir}/${cfg.clientId}.secret";
# Where swarm-secret-publish on authelia's host leaves this client's secret,
# composed from the same swarm-wide client id it publishes under. The
# responder's store role grants read on this one path.
storeSecretPath = "secret/swarm/services/${cfg.clientId}/oidc/client";
introspectionUrl = "${toString autheliaUrl}/api/oidc/introspection";
# Where the responder's seeds actually come from. One name for two
@ -234,6 +237,10 @@ let
);
authCredential = id: "/run/credentials/swarm-nats-auth.service/${id}";
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
refreshConsumer = import ./lib/refresh-consumer.nix { };
storeRetry = import ./lib/store-retry.nix { };
# Re-issue once the leaf is past half of the 720h `pki/roles/swarm-nats`
# grants it (./swarm-bao.nix), so the daily timer below has two weeks of
# retries before it lapses.
@ -430,13 +437,17 @@ in
example = "/var/lib/swarm-bao-pki/nats-auth.pem";
description = ''
Client certificate the auth-callout responder presents to the swarm's
secret store to read agent queue credentials. Its subject must be
secret store to read agent queue credentials, and that
`swarm-bao-nats-auth-oidc` presents to read the responder's own OIDC
client secret. Its subject must be
{option}`services.hyperhive.deploy.bao.natsAuthCommonName`.
No default. ./glue-nats-auth-bao-identity.nix points it at the leaf
./glue-bao-tls.nix mints, where this host mints one. Unset, or set to
a file that does not exist, the responder denies every agent token and
admits OIDC clients as before.
./glue-bao-tls.nix mints, where this host mints one; on any other
queue host it is a file the operator places. Unset, or set to a file
that does not exist, `swarm-bao-nats-auth-oidc` fails; a responder
with no OIDC client secret on disk cannot start, and the queue then
refuses every client.
A path, never a value.
'';
@ -532,33 +543,6 @@ in
}
];
# One declaration, two readers. The queue knows which client id it
# authenticates under; making the operator restate it in authelia's
# client list would be a second source of truth for a string whose
# mismatch is an opaque 401 from the token endpoint.
#
# `client_credentials` rather than an authorization-code flow: a
# queue's clients are daemons with nobody to redirect, so a
# non-interactive grant is what makes a hive able to authenticate at
# all. No redirect URI exists or is wanted.
#
# ⚠️ `kind` says that, and an empty `redirectUris` does NOT. This
# declaration carried the comment above while rendering as an
# interactive client, because authelia permits only the grants a
# client names and an omitted `grant_types` means authorization-code
# alone. Measured against 4.39.20: the token endpoint answered
# `unauthorized_client: The OAuth 2.0 Client is not allowed to use
# authorization grant 'client_credentials'`. Introspection — which
# is all the responder needs today — worked throughout, which is why
# nothing was visibly broken while the comment was untrue.
services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf deployCfg.authelia.enable [
{
id = cfg.clientId;
description = "HyperHive swarm queue";
kind = "machine";
}
];
# Declared here rather than in the collector's module: an entry then
# exists only where the service that named it runs. Gated on a
# collector, because a target nobody reads asserts a collection that is
@ -882,17 +866,16 @@ in
};
};
# Deliver the responder's three credentials into the container before
# it starts. Same shape as `hive-matrix-oidc-secret`, and for the same
# reason it is a copy rather than a `bindMounts` entry: nixos-container
# refuses to start when a bind source is missing, so one absent seed
# would take down the **whole container including the queue**, not
# merely the responder. A far larger blast radius than the fault.
# Order the container after the host CA generator, so the bind source
# exists before nspawn sets the mount up. Without it a late CA fails the
# container start outright rather than degrading.
systemd.services."container@${machine}" = caTrust.containerOrdering;
# Deliver the responder's seeds and store identity into the container
# before it starts. A copy rather than a `bindMounts` entry:
# nixos-container refuses to start when a bind source is missing, so one
# absent seed would take down the **whole container including the queue**,
# not merely the responder.
systemd.services.swarm-nats-auth-secrets = lib.mkIf responderConfigured {
description = "deliver the swarm queue responder's credentials";
before = [ "container@swarm-nats.service" ];
@ -903,13 +886,6 @@ in
# or absent seed is worse than not running.
after =
lib.optional deployCfg.nats.autoGenerateCallout "swarm-nats-callout-keys.service"
# The third credential does not come from the generator above — it is
# minted by authelia's FIRST BOOT, inside its own container. Ordering
# after that container is necessary and NOT sufficient: the container
# being up says nothing about whether its in-container secrets unit
# has finished. The wait in the script is what actually closes it;
# this only stops us spinning for the full timeout on every boot.
++ lib.optional deployCfg.authelia.enable "container@${autheliaCfg.machine}.service"
# Where the store's PKI is minted on this host, the responder's leaf is
# one of its files. Ordering only: elsewhere the unit does not exist.
++ lib.optional authStoreActive "swarm-bao-pki.service";
@ -917,14 +893,6 @@ in
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
# Must exceed the script's own wait below, and is set rather than
# left to the default for exactly that reason: systemd's
# `DefaultTimeoutStartSec` is 90s, so a 120s wait would be killed at
# 90 and the operator would get a generic unit timeout instead of
# the message that names the missing file and says what to do.
# The bound that matters belongs in one place, and this makes the
# two visibly related.
TimeoutStartSec = "180s";
SyslogIdentifier = "swarm-nats-auth-secrets";
};
path = [ pkgs.coreutils ];
@ -935,38 +903,10 @@ in
${lib.escapeShellArg (hostPath "callout-user.seed")}
install -m 0400 ${lib.escapeShellArg issuerSeedFile} \
${lib.escapeShellArg (hostPath "issuer.seed")}
# Wait for authelia's minted secret rather than failing the instant
# it is absent. On a fresh boot this unit and authelia's first-boot
# generator race, and losing that race used to cost the WHOLE QUEUE:
# this exits 1, the responder never starts, and `auth_callout` with
# no responder refuses every client — fail-closed by design, so the
# symptom appears on every queue client and nowhere near the cause.
#
# Bounded, not indefinite. Where authelia runs on another host the
# file is never going to appear, and blocking the queue container
# forever would replace a clear failure with a hang. After the
# timeout this fails exactly as it did before, having first given
# the co-located case the seconds it actually needs.
secret=${lib.escapeShellArg clientSecretSource}
deadline=$(( SECONDS + 120 ))
while [ ! -s "$secret" ]; do
if [ "$SECONDS" -ge "$deadline" ]; then
echo "the swarm queue responder's OIDC secret never appeared at $secret" >&2
echo "(authelia mints it on first boot; if authelia runs on another host," >&2
echo " copy the secret there and this unit will pick it up)" >&2
exit 1
fi
sleep 2
done
install -m 0400 "$secret" \
${lib.escapeShellArg (hostPath "oidc-client.secret")}
''
# The store identity is optional to the responder, so an absent source
# is delivered as an empty file rather than failing this unit: a missing
# `LoadCredential` source stops the responder starting, and a responder
# that does not start denies every client. An empty identity fails only
# the store login, which denies agent tokens.
# The store identity is optional to the responder's agent-token path, so
# an absent source is delivered as an empty file rather than failing this
# unit: a missing `LoadCredential` source stops the responder starting.
+ lib.concatStrings (
lib.mapAttrsToList (id: source: ''
if [ -s ${lib.escapeShellArg source} ]; then
@ -979,6 +919,117 @@ in
);
};
# The responder's OIDC client secret, out of the swarm secret store under
# the responder's own leaf — the one route, wherever authelia runs. Same
# shape as ./swarm-grafana.nix's `swarm-bao-grafana-oidc`: a login that
# fails loudly, since a retry fixes every state it fails on, then a read
# that degrades quietly, since an absent value is "not published yet".
#
# ⚠️ Without the secret the responder does not start (its `LoadCredential`
# source is missing), and `auth_callout` with no responder refuses every
# client. A host with no responder leaf therefore fails here, naming the
# options, rather than rendering no unit.
systemd.services.swarm-bao-nats-auth-oidc = {
description = "fetch the swarm queue responder's OIDC client secret from the swarm secret store";
# The store's units exist only where the store runs; `Requires=` on an
# absent unit fails the job, so off-host the timeout below bounds the
# attempt instead.
after = [
"swarm-nats-auth-secrets.service"
]
++ lib.optionals baoDeploy.enable [
"swarm-bao-pki.service"
"container@${baoCfg.machine}.service"
];
wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
before = [ "container@${machine}.service" ];
wantedBy = [
"multi-user.target"
"container@${machine}.service"
];
path = [
baoDeploy.package
pkgs.coreutils
pkgs.systemd
];
# ./lib/store-retry.nix. The container waits for one attempt; a secret a
# later attempt lands restarts the responder (below).
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "swarm-bao-nats-auth-oidc";
TimeoutStartSec = 30;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
}
// lib.optionalAttrs authStoreActive {
BAO_CLIENT_CERT = deployCfg.nats.authBaoClientCertFile;
BAO_CLIENT_KEY = deployCfg.nats.authBaoClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
script =
if !authStoreActive then
''
echo "the swarm queue responder has no store identity, so it cannot read its OIDC client secret: set services.hyperhive.deploy.nats.authBaoClientCertFile and authBaoClientKeyFile" >&2
exit 1
''
else
''
set -euo pipefail
${atomicWriteSecret}
${refreshConsumer}
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
# `-token-only` answers on stdout and skips the token helper, a
# `sh` this unit's `path` does not carry.
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with the queue responder's certificate; leaving its OIDC client secret as it is." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
export BAO_TOKEN
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg storeSecretPath} 2>"$err")"; then
echo "swarm-bao did not return ${storeSecretPath}; the queue responder has no OIDC client secret yet." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 0
fi
if [ -z "$secret" ]; then
echo "swarm-bao returned an empty ${storeSecretPath}; leaving the file as it is." >&2
exit 0
fi
# Root-owned 0400: `LoadCredential` reads it as root before the
# responder's `DynamicUser` exists. `printf` inside
# `atomic_write_secret` is a builtin, so the plaintext never
# becomes an argument in /proc.
install -d -m 0700 ${lib.escapeShellArg secretDir}
if secret_differs ${lib.escapeShellArg (hostPath "oidc-client.secret")} "$secret"; then
atomic_write_secret 0400 "" ${lib.escapeShellArg (hostPath "oidc-client.secret")} "$secret"
fi
# `LoadCredential` copies the file once, at the responder's start.
refresh_consumer ${lib.escapeShellArg machine} swarm-nats-auth.service ${lib.escapeShellArg (hostPath "oidc-client.secret")}
'';
};
# ⚠️ Minted on the HOST, not in the container, because the responder is
# a separate unit that needs the user seed: generating it inside would
# trap it there and require a secret-export path back out — the exact