vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm: OIDC clients registered wherever authelia runs; nats + matrix secrets from bao

Browse source 
The three client registrations move into glue-{controller,nats,matrix}-oidc-client.nix,
gated only on deploy.authelia.enable and reading their ids from swarm.* options,
so the host running authelia registers every one of them whichever host runs
the service. The matrix callback is a new read-only swarm.matrix.sso.redirectUri,
read by both the homeserver's config and its client entry.

swarm-secret-publish now also publishes the queue's and the homeserver's minted
secrets to swarm/services/<id>/oidc/client. The queue host reads its responder's
secret there with swarm-bao-nats-auth-oidc under the responder's own leaf
(swarm-nats-auth role gains that one read path), and the homeserver's host with
swarm-bao-matrix-oidc under matrix-ctl's leaf (swarm-matrix-ctl role gains its
one read path). The host-file copies are gone: swarm-nats-auth-secrets now
delivers only the callout seeds and the store identity, and
hive-matrix-oidc-secret is removed. A fetch unit whose leaf is unset still
renders and fails naming the option pair.

Closes #4943
This commit is contained in:
atlas 2026-10-05 02:56:18 +02:00 • committed by mara
commit 962da3b784
20 changed files with 702 additions and 293 deletions
Download patch file Download diff file Expand all files Collapse all files

3
nix/host-modules/default.nix
View file

@ -30,12 +30,15 @@
./glue-bao-tls.nix
./glue-bao-ui-oidc-client.nix
./glue-controller-bao-identity.nix
./glue-controller-oidc-client.nix
./glue-forge-oidc-client.nix
./glue-grafana-oidc-client.nix
./glue-hive-tls-bao-identity.nix
./glue-matrix-ctl-bao-identity.nix
./glue-matrix-oidc-client.nix
./glue-nats-auth-bao-identity.nix
./glue-nats-bao-identity.nix
./glue-nats-oidc-client.nix
./glue-queue-agent-credential.nix
./glue-secret-publisher-bao-identity.nix
./glue-services-issuer-bao-identity.nix

26
nix/host-modules/glue-bao-readers-policy-order.nix
View file

@ -47,15 +47,33 @@ let
swarm-bao-nats-tls = deployCfg.nats.enable;
};
# Readers that log in under another unit's principal: reader unit → the
# gate its own module defines it under, and that principal's policy unit.
readersOfOtherPolicy = {
# ./swarm-nats.nix, under the queue responder's leaf.
swarm-bao-nats-auth-oidc = {
here = deployCfg.nats.enable;
policy = "swarm-bao-nats-auth-policy";
};
# ./hive-matrix.nix, under matrix-ctl's leaf.
swarm-bao-matrix-oidc = {
here = deployCfg.matrix.enable;
policy = "swarm-bao-matrix-ctl-policy";
};
};
orderAfterPolicy =
reader: here:
reader: here: policy:
lib.mkIf (baoDeploy.enable && here) {
systemd.services.${reader} = {
after = [ "${reader}-policy.service" ];
wants = [ "${reader}-policy.service" ];
after = [ "${policy}.service" ];
wants = [ "${policy}.service" ];
};
};
in
{
config = lib.mkMerge (lib.mapAttrsToList orderAfterPolicy readersHere);
config = lib.mkMerge (
lib.mapAttrsToList (reader: here: orderAfterPolicy reader here "${reader}-policy") readersHere
++ lib.mapAttrsToList (reader: r: orderAfterPolicy reader r.here r.policy) readersOfOtherPolicy
);
}

48
nix/host-modules/glue-controller-oidc-client.nix Normal file
View file

@ -0,0 +1,48 @@
# Glue: register the swarm controller as an OIDC client wherever authelia runs.
#
# ONE PAIRING PER FILE — swarm-controller ← authelia, and nothing else. Deleting
# this leaves a controller whose token requests authelia answers with a 401, and
# no minted secret for swarm-secret-publish to put in the store.
#
# ⚠️ Gated on authelia being HERE, and NOT on this host running the controller.
# A client is a row in THIS host's provider config, so it is declared where that
# config is rendered; ./swarm-controller.nix's `config` hangs off
# `deploy.swarm-controller.enable`, which answers a question about a different
# host. Same shape as ./glue-grafana-oidc-client.nix.
#
# Registered whether or not the swarm runs a controller: nothing in `swarm.*`
# records that, so the cost of a swarm without one is one unused client and one
# unused minted secret.
{
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
deployCfg = hyperhiveCfg.deploy;
queueClientId = hyperhiveCfg.swarm.controller.queueClientId;
in
{
config = lib.mkIf deployCfg.authelia.enable {
# One declaration, two readers: the id is the read-only option the
# controller authenticates under, so a mismatch cannot be introduced here.
services.hyperhive.swarm.authelia.oidc.clients = [
{
id = queueClientId;
description = "HyperHive swarm controller";
# `client_credentials`: a daemon authenticating as itself, with nobody
# to redirect. Declared rather than inferred from an empty redirect
# list, because authelia permits only the grants a client names and an
# omitted `grant_types` means authorization-code alone.
kind = "machine";
# Its own id as its own permitted audience. The controller mints a
# token for `SWARM_CONTROLLER_OTEL_AUDIENCE` (./swarm-controller.nix),
# and ./swarm-otel.nix's `oidc/swarm` authenticator checks for this
# same id; either half without the other is a 401 several layers from
# this line.
audience = [ queueClientId ];
}
];
};
}

46
nix/host-modules/glue-matrix-oidc-client.nix Normal file
View file

@ -0,0 +1,46 @@
# Glue: register the swarm's homeserver as an OIDC client wherever authelia
# runs.
#
# ONE PAIRING PER FILE — matrix ← authelia, and nothing else. Deleting this
# leaves a homeserver whose SSO login authelia has never heard of, and no
# minted secret for swarm-secret-publish to put in the store.
#
# ⚠️ Gated on authelia being HERE, and NOT on this host running the
# homeserver. A client is a row in THIS host's provider config, so it is
# declared where that config is rendered; ./hive-matrix.nix's `config` hangs
# off `deploy.matrix.enable`, which answers a question about a different host.
# Same shape as ./glue-grafana-oidc-client.nix.
#
# Registered whether or not the swarm runs a homeserver: nothing in `swarm.*`
# records that. The one swarm-level fact that rules a login out is a null
# `gatewayHost`, which leaves no callback a browser can reach, so that alone
# skips the client.
{
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
deployCfg = hyperhiveCfg.deploy;
matrixCfg = hyperhiveCfg.swarm.matrix;
in
{
config = lib.mkIf (deployCfg.authelia.enable && matrixCfg.sso.redirectUri != null) {
# One declaration, two readers: the callback is the read-only option the
# homeserver's own config renders, so it is spelled once.
services.hyperhive.swarm.authelia.oidc.clients = [
{
id = matrixCfg.sso.clientId;
description = "HyperHive matrix";
redirectUris = [ matrixCfg.sso.redirectUri ];
# tuwunel authenticates at the token endpoint by putting the secret in
# the POST body. Authelia enforces the *registered* method, and its
# default is `client_secret_basic` — so without this the browser flow
# completes, consent is granted, and the last hop fails with a 401
# that names neither the secret nor the redirect.
tokenEndpointAuthMethod = "client_secret_post";
}
];
};
}

45
nix/host-modules/glue-nats-oidc-client.nix Normal file
View file

@ -0,0 +1,45 @@
# Glue: register the swarm queue as an OIDC client wherever authelia runs.
#
# ONE PAIRING PER FILE — swarm-nats ← authelia, and nothing else. Deleting this
# leaves a queue responder whose introspection calls authelia refuses, so it
# denies every client, and no minted secret for swarm-secret-publish to put in
# the store.
#
# ⚠️ Gated on authelia being HERE, and NOT on this host running the queue. A
# client is a row in THIS host's provider config, so it is declared where that
# config is rendered; ./swarm-nats.nix's `config` hangs off
# `deploy.nats.enable`, which answers a question about a different host. Same
# shape as ./glue-grafana-oidc-client.nix.
#
# Registered whether or not the swarm runs a queue: nothing in `swarm.*` records
# that, so the cost of a swarm without one is one unused client and one unused
# minted secret.
{
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
deployCfg = hyperhiveCfg.deploy;
natsCfg = hyperhiveCfg.swarm.nats;
in
{
config = lib.mkIf deployCfg.authelia.enable {
services.hyperhive.swarm.authelia.oidc.clients = [
{
id = natsCfg.clientId;
description = "HyperHive swarm queue";
# `client_credentials`: a queue's clients are daemons with nobody to
# redirect, and no redirect URI exists.
#
# ⚠️ `kind` says that, and an empty `redirectUris` does NOT: authelia
# permits only the grants a client names, and an omitted `grant_types`
# means authorization-code alone. Introspection, which is all the
# responder calls, works either way, so the token endpoint's
# `unauthorized_client` is the only place a wrong kind shows.
kind = "machine";
}
];
};
}

28
nix/host-modules/hive-matrix-service.nix
View file

@ -178,14 +178,34 @@ in
type = lib.types.str;
default = "tuwunel";
description = ''
OAuth2 client id this homeserver identifies itself with. Must
match the `id` of the corresponding entry in
`services.hyperhive.swarm.authelia.oidc.clients`.
OAuth2 client id this homeserver identifies itself with. The host
running authelia registers it, whichever host runs the homeserver.
The secret it pairs with is a host path, so it lives at
The homeserver's host reads the secret it pairs with out of the
swarm secret store, into
`services.hyperhive.deploy.matrix.sso.clientSecretFile`.
'';
};
redirectUri = lib.mkOption {
type = lib.types.nullOr lib.types.str;
readOnly = true;
default =
if cfg.gatewayHost == null then
null
else
"https://${cfg.gatewayHost}/_matrix/client/unstable/login/sso/callback/${cfg.sso.clientId}";
defaultText = lib.literalExpression ''"https://''${services.hyperhive.swarm.matrix.gatewayHost}/_matrix/client/unstable/login/sso/callback/''${services.hyperhive.swarm.matrix.sso.clientId}"'';
description = ''
SSO callback authelia sends the browser back to, and the URI it
matches **exactly**. Format-locked by tuwunel: the callback host
must point directly at the homeserver, and the path is fixed.
Read-only: the homeserver's config and its authelia client entry
both read it from here. `null` when `gatewayHost` is, because the
homeserver then has no name a browser can reach.
'';
};
};
};
}

225
nix/host-modules/hive-matrix.nix
View file

@ -33,18 +33,25 @@ let
baoCfg = config.services.hyperhive.swarm.bao;
autheliaUrl = autheliaCfg.url;
# The all-local case: this host runs BOTH the homeserver and the swarm's
# authelia, so the secret can be moved without an operator. The other
# two cases (swarm side, remote hive) leave `clientSecretFile` to be set
# explicitly — same split the forge module documents.
ssoLocal = deployCfg.authelia.enable;
# Where the plaintext lands inside the matrix container. Under /var/lib
# rather than /run: the homeserver may start before the delivery unit on
# a later boot, and a secret that evaporates on reboot turns a working
# login into an intermittent one.
matrixSecretPath = "/var/lib/tuwunel-oidc/${cfg.sso.clientId}.secret";
# The same file seen from the host, where `swarm-bao-matrix-oidc` writes it.
# Follows `clientSecretFile`, so an operator-chosen path is where the
# fetched secret lands.
hostSecretPath = "/var/lib/nixos-containers/hive-matrix${toString deployCfg.matrix.sso.clientSecretFile}";
# Where swarm-secret-publish on authelia's host leaves this client's secret.
# matrix-ctl's store role grants read on this one path.
storeSecretPath = "secret/swarm/services/${cfg.sso.clientId}/oidc/client";
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
refreshConsumer = import ./lib/refresh-consumer.nix { };
storeRetry = import ./lib/store-retry.nix { };
# ⚠️ tuwunel does NOT read the host path directly, and this indirection
# is not ceremony. Upstream's own words: "under systemd the path must be
# visible to the service after sandboxing (ReadWritePaths / ProtectHome),
@ -141,13 +148,6 @@ let
# create these names, not a monopoly on them.
appserviceUserRegex = "^@[a-z0-9._=/-]+:${lib.escapeRegex effectiveServerName}$";
# Format-locked by tuwunel, not chosen here: the callback host must point
# directly at the matrix server and the path is fixed at
# `/_matrix/client/unstable/login/sso/callback/<client_id>`. Built once
# and read twice — by the homeserver's own config and by the client entry
# handed to authelia — because a redirect-URI mismatch is a rejected
# login with no error text worth reading.
ssoCallbackUrl = "https://${toString cfg.gatewayHost}/_matrix/client/unstable/login/sso/callback/${cfg.sso.clientId}";
# Falls back to the SWARM domain: a swarm runs one homeserver, so its
# identifier belongs to the swarm rather than to whichever hive happens
# to host it — otherwise moving the container between hives would look
@ -385,13 +385,11 @@ in
needs the plaintext), and a literal written here would be
rendered into the world-readable nix store.
Deliberately no fallback — a homeserver that boots with SSO
half-configured is worse than one that fails to evaluate:
tuwunel reads OIDC from its config file rather than a database
row, so a malformed block can stop the server outright instead
of merely hiding a button. The id this pairs with is
`swarm.matrix.sso.clientId`, which is swarm-wide because it must
match authelia's register.
Set to `/var/lib/tuwunel-oidc/<client id>.secret` wherever the
homeserver runs. `swarm-bao-matrix-oidc` writes the secret here
from the swarm secret store, whichever path this names. The id
this pairs with is `swarm.matrix.sso.clientId`, which is
swarm-wide because authelia's host registers it.
'';
};
@ -559,9 +557,9 @@ in
(inside the matrix container) holding the OIDC client secret's
plaintext.
On a hive that also runs the swarm's authelia this is wired up
for you. Set it explicitly when authelia lives on another
host: see docs/swarm/ for which secret goes where.
It is set by default wherever the homeserver runs, and
swarm-bao-matrix-oidc writes the secret there from the swarm
secret store. Remove the definition that set it to null.
'';
}
{
@ -582,95 +580,114 @@ in
}
];
# One declaration, two readers. The homeserver knows its own callback
# URL; making the operator restate it in authelia's client list would
# be a second source of truth for a string whose mismatch is a silent
# rejected login.
services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf ssoLocal [
{
id = cfg.sso.clientId;
description = "HyperHive matrix";
redirectUris = [ ssoCallbackUrl ];
# tuwunel authenticates at the token endpoint by putting the
# secret in the POST body. Authelia enforces the *registered*
# method rather than accepting whichever one arrives, and its
# default is `client_secret_basic` — so without this the browser
# flow completes, consent is granted, and the very last hop fails
# with a 401 that names neither the secret nor the redirect.
tokenEndpointAuthMethod = "client_secret_post";
}
];
# Where the homeserver finds its OIDC client secret, inside the container.
services.hyperhive.deploy.matrix.sso.clientSecretFile = lib.mkDefault matrixSecretPath;
# Same case, same reasoning: this host minted the secret, so it can say
# where the homeserver will find it.
services.hyperhive.deploy.matrix.sso.clientSecretFile = lib.mkIf ssoLocal (
lib.mkDefault matrixSecretPath
);
# The delivery. It runs on the HOST because that is the only place both
# container trees are addressable: they share this host's network
# namespace, which makes them feel co-located, but their filesystem
# roots are separate — the homeserver cannot open a path inside
# authelia's tree however local the port looks.
# The homeserver's OIDC client secret, out of the swarm secret store under
# matrix-ctl's leaf — the one route, wherever authelia runs. Same shape as
# ./swarm-grafana.nix's `swarm-bao-grafana-oidc`: a login that fails
# loudly, since a retry fixes every state it fails on, then a read that
# degrades quietly, since an absent value is "not published yet".
#
# ⚠️ Deliberately a copy and not a `bindMounts` entry.
# nixos-container refuses to start when a bind source is missing, and
# this secret does not exist until authelia's first boot has minted it
# — so binding it would make the homeserver wait on a file that waits
# on a container that starts after it. On a fresh hive that is a
# permanent stall presenting as "matrix is broken", several layers from
# its cause.
#
# ⚠️ Rendering a placeholder first is no way around it: tuwunel
# requires the secret file to exist *and be non-empty*, so a zero-byte
# placeholder would satisfy the bind mount and then stop the homeserver
# from starting.
systemd.services.hive-matrix-oidc-secret = lib.mkIf ssoLocal {
description = "deliver the homeserver's OIDC client secret from authelia";
after = [ "container@${autheliaCfg.machine}.service" ];
requires = [ "container@${autheliaCfg.machine}.service" ];
# ⚠️ tuwunel requires the file to exist and be non-empty, and its
# `LoadCredential` source must exist for the unit to start, so a host with
# no matrix-ctl leaf fails here, naming the options, rather than rendering
# no unit.
systemd.services.swarm-bao-matrix-oidc = {
description = "fetch the homeserver's OIDC client secret from the swarm secret store";
# The store's units exist only where the store runs; `Requires=` on an
# absent unit fails the job, so off-host the timeout below bounds the
# attempt instead.
after = lib.optionals deployCfg.bao.enable [
"swarm-bao-pki.service"
"container@${baoCfg.machine}.service"
];
wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ];
requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ];
before = [ "container@hive-matrix.service" ];
wantedBy = [ "container@hive-matrix.service" ];
serviceConfig = {
wantedBy = [
"multi-user.target"
"container@hive-matrix.service"
];
path = [
deployCfg.bao.package
pkgs.coreutils
pkgs.systemd
];
# ./lib/store-retry.nix. The container waits for one attempt; a secret a
# later attempt lands restarts tuwunel (below).
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "hive-matrix-oidc-secret";
# Longer than the 120s bounded wait below, and that is the whole
# point: `DefaultTimeoutStartSec` is 90s, so without this systemd
# kills the unit at 90 — before it can emit the message naming the
# file it was waiting for. The failure then reads as a timeout with
# no cause rather than "authelia has not minted <path>", which is
# the one line that makes a fresh-hive SSO stall diagnosable.
TimeoutStartSec = "180s";
SyslogIdentifier = "swarm-bao-matrix-oidc";
TimeoutStartSec = 30;
};
path = [ pkgs.coreutils ];
script = ''
set -euo pipefail
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
}
// lib.optionalAttrs ctlActive {
BAO_CLIENT_CERT = deployCfg.matrix.ctlBaoClientCertFile;
BAO_CLIENT_KEY = deployCfg.matrix.ctlBaoClientKeyFile;
}
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
BAO_CACERT = deployCfg.bao.serverCaFile;
};
script =
if !ctlActive then
''
echo "the homeserver's host has no matrix-ctl store identity, so it cannot read the homeserver's OIDC client secret: set services.hyperhive.deploy.matrix.ctlBaoClientCertFile and ctlBaoClientKeyFile" >&2
exit 1
''
else
''
set -euo pipefail
src=${lib.escapeShellArg "${deployCfg.authelia.hostClientSecretDir}/${cfg.sso.clientId}.secret"}
dst=${lib.escapeShellArg "/var/lib/nixos-containers/hive-matrix${toString deployCfg.matrix.sso.clientSecretFile}"}
${atomicWriteSecret}
${refreshConsumer}
# authelia's container is up, but its first-boot generator may
# still be minting. Bounded wait, then fail: a silent skip here
# produces a homeserver whose SSO login dead-ends, which is the
# failure this whole design is trying not to ship.
for _ in $(seq 1 60); do
[ -s "$src" ] && break
sleep 2
done
if [ ! -s "$src" ]; then
echo "authelia has not minted $src after 120s" >&2
exit 1
fi
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
# root-owned 0400, and deliberately NOT the forge's `stat -c %u`
# uid discovery: that reads the service's state dir to learn which
# uid to hand the file to, and tuwunel runs under `DynamicUser`, so
# there is no stable uid to discover. It never reads this path
# directly anyway — `LoadCredential` does, as root, before the
# sandbox and the dynamic user exist.
install -D -m 0400 -o root -g root "$src" "$dst"
'';
# `-token-only` answers on stdout and skips the token helper, a
# `sh` this unit's `path` does not carry.
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with matrix-ctl's certificate; leaving the homeserver's OIDC client secret as it is." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
export BAO_TOKEN
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg storeSecretPath} 2>"$err")"; then
echo "swarm-bao did not return ${storeSecretPath}; the homeserver has no OIDC client secret yet." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 0
fi
if [ -z "$secret" ]; then
echo "swarm-bao returned an empty ${storeSecretPath}; leaving the file as it is." >&2
exit 0
fi
# Root-owned 0400: tuwunel runs under `DynamicUser`, so there is no
# stable uid to hand the file to, and `LoadCredential` reads it as
# root before the sandbox exists.
install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf hostSecretPath)}
if secret_differs ${lib.escapeShellArg hostSecretPath} "$secret"; then
atomic_write_secret 0400 "" ${lib.escapeShellArg hostSecretPath} "$secret"
fi
# `LoadCredential` copies the file once, at tuwunel's start.
refresh_consumer hive-matrix tuwunel.service ${lib.escapeShellArg hostSecretPath}
'';
};
# ⚠️ Deliberately NO `networking.hosts` entry for authelia's name. This
@ -887,7 +904,7 @@ in
client_id = cfg.sso.clientId;
client_secret_file = matrixSecretCredential;
issuer_url = toString autheliaUrl;
callback_url = ssoCallbackUrl;
callback_url = cfg.sso.redirectUri;
# Explicit, though a lone provider is auto-defaulted:
# relying on that logs a warning every startup, and a
# recurring warning that is expected is one nobody reads.

24
nix/host-modules/swarm-bao.nix
View file

@ -485,23 +485,28 @@ let
matrixCtlPolicyName = "swarm-matrix-ctl";
matrixCtlCn = baoDeploy.matrixCtlCommonName;
# ONE path, and every narrowing in it is load-bearing.
# Two paths, each spelled to the leaf, and every narrowing in them is
# load-bearing.
#
# `secret/data/` is KV v2's ACL prefix, inserted by the engine rather than
# written by the caller — the same trap as the two grants above.
#
# Not `swarm/services/*` like the publisher's: a homeserver is not entitled
# to overwrite Grafana's OIDC client. The path is spelled to the leaf for
# to overwrite Grafana's OIDC client. The paths are spelled to the leaf for
# that reason, not for tidiness.
#
# It is the swarm appservice's token, which matrix-ctl mints inside the
# container and publishes here for the controller. `read` as well as write,
# unlike either sibling, because publish compares before it writes.
# The first is the swarm appservice's token, which matrix-ctl mints inside
# the container and publishes here for the controller. `read` as well as
# write, unlike either sibling, because publish compares before it writes.
#
# The second is the homeserver's own OIDC client secret, read-only:
# `swarm-bao-matrix-oidc` on the homeserver's host reads it under this leaf.
matrixCtlPolicyText = ''
path "${credentialMountPath}/data/${swarmAppserviceTokenLeaf}" {
capabilities = ["create", "update", "read"]
}
'';
''
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.matrix.sso.clientId}/oidc/client";
# The swarm appservice token's leaf, the nix half of
# `swarm_secret_client::matrix::swarm_appservice_token_path`. Under
@ -836,12 +841,15 @@ let
# The queue's auth-callout responder, which checks the secret an agent
# presents against the one stored for it. One leaf under every agent: `+` is
# one path segment, where `*` globs only at the end and would reach every
# other credential an agent holds.
# other credential an agent holds. Plus the responder's own OIDC client
# secret, which `swarm-bao-nats-auth-oidc` reads under this same leaf.
natsAuthReaders = [
{
name = "swarm-nats-auth";
cn = baoDeploy.natsAuthCommonName;
policyText = readStanza "${credentialMountPath}/data/swarm/agents/+/queue";
policyText =
readStanza "${credentialMountPath}/data/swarm/agents/+/queue"
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.nats.clientId}/oidc/client";
}
];

38
nix/host-modules/swarm-controller.nix
View file

@ -301,7 +301,8 @@ let
OTEL_EXPORTER_OTLP_ENDPOINT = "https://${otelSwarmCfg.domain}/${otelSwarmCfg.producerName}";
# Checked by `swarm-otel.nix`'s `oidc/${producerName}` authenticator
# against exactly this controller's own registered audience (see the
# `audience = [ queueClientId ]` client entry below) — the two have
# `audience = [ queueClientId ]` client entry in
# ./glue-controller-oidc-client.nix) — the two have
# to name the same value or the exporter mints a token the receiver
# was never told to accept.
SWARM_CONTROLLER_OTEL_AUDIENCE = queueClientId;
@ -360,9 +361,10 @@ in
default = "swarm-controller";
description = ''
The OAuth2 client id the controller presents to the swarm queue.
Read-only: it is what this module registers, published so the
auth-callout responder can be told which client may read every
hive's key without repeating the string.
Read-only: it is the client the host running authelia registers,
whichever host runs the controller, published so the auth-callout
responder can be told which client may read every hive's key without
repeating the string.
The responder decides that from a client id, and a client id it
does not recognise is **denied**. A denial reaches a NATS client
@ -704,34 +706,6 @@ in
# exactly what it must not inherit.
environment.systemPackages = [ swarmctlConfigured ];
# One declaration, two readers — the controller knows which client id it
# authenticates under, so making the operator restate it in authelia's
# client list would be a second source of truth for a string whose
# mismatch is an opaque 401 from the token endpoint. Same shape as the
# queue's own client declaration.
services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf deployCfg.authelia.enable [
{
id = queueClientId;
description = "HyperHive swarm controller";
# `client_credentials`: a daemon authenticating as itself, with
# nobody to redirect. Declared rather than inferred from an empty
# redirect list, because authelia permits only the grants a client
# names and an omitted `grant_types` means authorization-code alone.
kind = "machine";
# Its own id as its own permitted audience — same "client's own id
# is the value it may ask for" shape `swarm-authelia.nix`'s
# `hiveClients` uses, just self-referential instead of prefixed:
# this controller is not a hive (see `queueClientId`'s own doc
# comment), so it has no `hiveClientPrefix` name to reuse. This is
# what lets `vcs_metrics::authenticated_http_client()` mint a token
# for `SWARM_CONTROLLER_OTEL_AUDIENCE` and have `swarm-otel.nix`'s
# `oidc/swarm` authenticator (which checks for this same id) accept
# it — registering here without requesting there, or the reverse,
# both fail as a healthy-looking 401 several layers from this line.
audience = [ queueClientId ];
}
];
# A controller without a queue is not a lighter controller, it is a
# broken one: `/api/hives/status` reads the KV, so its whole aggregate
# half is off. The operator's rule is that the controller needs a queue

6
nix/host-modules/swarm-nats-service.nix
View file

@ -107,9 +107,9 @@ in
default = "swarm-nats";
description = ''
OAuth2 client id the queue's authentication path identifies
itself with. Must match the `id` of the corresponding entry in
`services.hyperhive.swarm.authelia.oidc.clients` — which this
module contributes for you when both run on this host.
itself with. The host running authelia registers it, whichever
host runs the queue, and the queue's host reads its secret out of
the swarm secret store.
'';
};

219
nix/host-modules/swarm-nats.nix
View file

@ -57,7 +57,10 @@ let
responderConfigured =
deployCfg.nats.autoGenerateCallout
|| (deployCfg.nats.calloutUserSeedFile != "" && deployCfg.nats.calloutIssuerSeedFile != "");
clientSecretSource = "${deployCfg.authelia.hostClientSecretDir}/${cfg.clientId}.secret";
# Where swarm-secret-publish on authelia's host leaves this client's secret,
# composed from the same swarm-wide client id it publishes under. The
# responder's store role grants read on this one path.
storeSecretPath = "secret/swarm/services/${cfg.clientId}/oidc/client";
introspectionUrl = "${toString autheliaUrl}/api/oidc/introspection";
# Where the responder's seeds actually come from. One name for two
@ -234,6 +237,10 @@ let
);
authCredential = id: "/run/credentials/swarm-nats-auth.service/${id}";
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
refreshConsumer = import ./lib/refresh-consumer.nix { };
storeRetry = import ./lib/store-retry.nix { };
# Re-issue once the leaf is past half of the 720h `pki/roles/swarm-nats`
# grants it (./swarm-bao.nix), so the daily timer below has two weeks of
# retries before it lapses.
@ -430,13 +437,17 @@ in
example = "/var/lib/swarm-bao-pki/nats-auth.pem";
description = ''
Client certificate the auth-callout responder presents to the swarm's
secret store to read agent queue credentials. Its subject must be
secret store to read agent queue credentials, and that
`swarm-bao-nats-auth-oidc` presents to read the responder's own OIDC
client secret. Its subject must be
{option}`services.hyperhive.deploy.bao.natsAuthCommonName`.
No default. ./glue-nats-auth-bao-identity.nix points it at the leaf
./glue-bao-tls.nix mints, where this host mints one. Unset, or set to
a file that does not exist, the responder denies every agent token and
admits OIDC clients as before.
./glue-bao-tls.nix mints, where this host mints one; on any other
queue host it is a file the operator places. Unset, or set to a file
that does not exist, `swarm-bao-nats-auth-oidc` fails; a responder
with no OIDC client secret on disk cannot start, and the queue then
refuses every client.
A path, never a value.
'';
@ -532,33 +543,6 @@ in
}
];
# One declaration, two readers. The queue knows which client id it
# authenticates under; making the operator restate it in authelia's
# client list would be a second source of truth for a string whose
# mismatch is an opaque 401 from the token endpoint.
#
# `client_credentials` rather than an authorization-code flow: a
# queue's clients are daemons with nobody to redirect, so a
# non-interactive grant is what makes a hive able to authenticate at
# all. No redirect URI exists or is wanted.
#
# ⚠️ `kind` says that, and an empty `redirectUris` does NOT. This
# declaration carried the comment above while rendering as an
# interactive client, because authelia permits only the grants a
# client names and an omitted `grant_types` means authorization-code
# alone. Measured against 4.39.20: the token endpoint answered
# `unauthorized_client: The OAuth 2.0 Client is not allowed to use
# authorization grant 'client_credentials'`. Introspection — which
# is all the responder needs today — worked throughout, which is why
# nothing was visibly broken while the comment was untrue.
services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf deployCfg.authelia.enable [
{
id = cfg.clientId;
description = "HyperHive swarm queue";
kind = "machine";
}
];
# Declared here rather than in the collector's module: an entry then
# exists only where the service that named it runs. Gated on a
# collector, because a target nobody reads asserts a collection that is
@ -882,17 +866,16 @@ in
};
};
# Deliver the responder's three credentials into the container before
# it starts. Same shape as `hive-matrix-oidc-secret`, and for the same
# reason it is a copy rather than a `bindMounts` entry: nixos-container
# refuses to start when a bind source is missing, so one absent seed
# would take down the **whole container including the queue**, not
# merely the responder. A far larger blast radius than the fault.
# Order the container after the host CA generator, so the bind source
# exists before nspawn sets the mount up. Without it a late CA fails the
# container start outright rather than degrading.
systemd.services."container@${machine}" = caTrust.containerOrdering;
# Deliver the responder's seeds and store identity into the container
# before it starts. A copy rather than a `bindMounts` entry:
# nixos-container refuses to start when a bind source is missing, so one
# absent seed would take down the **whole container including the queue**,
# not merely the responder.
systemd.services.swarm-nats-auth-secrets = lib.mkIf responderConfigured {
description = "deliver the swarm queue responder's credentials";
before = [ "container@swarm-nats.service" ];
@ -903,13 +886,6 @@ in
# or absent seed is worse than not running.
after =
lib.optional deployCfg.nats.autoGenerateCallout "swarm-nats-callout-keys.service"
# The third credential does not come from the generator above — it is
# minted by authelia's FIRST BOOT, inside its own container. Ordering
# after that container is necessary and NOT sufficient: the container
# being up says nothing about whether its in-container secrets unit
# has finished. The wait in the script is what actually closes it;
# this only stops us spinning for the full timeout on every boot.
++ lib.optional deployCfg.authelia.enable "container@${autheliaCfg.machine}.service"
# Where the store's PKI is minted on this host, the responder's leaf is
# one of its files. Ordering only: elsewhere the unit does not exist.
++ lib.optional authStoreActive "swarm-bao-pki.service";
@ -917,14 +893,6 @@ in
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
# Must exceed the script's own wait below, and is set rather than
# left to the default for exactly that reason: systemd's
# `DefaultTimeoutStartSec` is 90s, so a 120s wait would be killed at
# 90 and the operator would get a generic unit timeout instead of
# the message that names the missing file and says what to do.
# The bound that matters belongs in one place, and this makes the
# two visibly related.
TimeoutStartSec = "180s";
SyslogIdentifier = "swarm-nats-auth-secrets";
};
path = [ pkgs.coreutils ];
@ -935,38 +903,10 @@ in
${lib.escapeShellArg (hostPath "callout-user.seed")}
install -m 0400 ${lib.escapeShellArg issuerSeedFile} \
${lib.escapeShellArg (hostPath "issuer.seed")}
# Wait for authelia's minted secret rather than failing the instant
# it is absent. On a fresh boot this unit and authelia's first-boot
# generator race, and losing that race used to cost the WHOLE QUEUE:
# this exits 1, the responder never starts, and `auth_callout` with
# no responder refuses every client — fail-closed by design, so the
# symptom appears on every queue client and nowhere near the cause.
#
# Bounded, not indefinite. Where authelia runs on another host the
# file is never going to appear, and blocking the queue container
# forever would replace a clear failure with a hang. After the
# timeout this fails exactly as it did before, having first given
# the co-located case the seconds it actually needs.
secret=${lib.escapeShellArg clientSecretSource}
deadline=$(( SECONDS + 120 ))
while [ ! -s "$secret" ]; do
if [ "$SECONDS" -ge "$deadline" ]; then
echo "the swarm queue responder's OIDC secret never appeared at $secret" >&2
echo "(authelia mints it on first boot; if authelia runs on another host," >&2
echo " copy the secret there and this unit will pick it up)" >&2
exit 1
fi
sleep 2
done
install -m 0400 "$secret" \
${lib.escapeShellArg (hostPath "oidc-client.secret")}
''
# The store identity is optional to the responder, so an absent source
# is delivered as an empty file rather than failing this unit: a missing
# `LoadCredential` source stops the responder starting, and a responder
# that does not start denies every client. An empty identity fails only
# the store login, which denies agent tokens.
# The store identity is optional to the responder's agent-token path, so
# an absent source is delivered as an empty file rather than failing this
# unit: a missing `LoadCredential` source stops the responder starting.
+ lib.concatStrings (
lib.mapAttrsToList (id: source: ''
if [ -s ${lib.escapeShellArg source} ]; then
@ -979,6 +919,117 @@ in
);
};
# The responder's OIDC client secret, out of the swarm secret store under
# the responder's own leaf — the one route, wherever authelia runs. Same
# shape as ./swarm-grafana.nix's `swarm-bao-grafana-oidc`: a login that
# fails loudly, since a retry fixes every state it fails on, then a read
# that degrades quietly, since an absent value is "not published yet".
#
# ⚠️ Without the secret the responder does not start (its `LoadCredential`
# source is missing), and `auth_callout` with no responder refuses every
# client. A host with no responder leaf therefore fails here, naming the
# options, rather than rendering no unit.
systemd.services.swarm-bao-nats-auth-oidc = {
description = "fetch the swarm queue responder's OIDC client secret from the swarm secret store";
# The store's units exist only where the store runs; `Requires=` on an
# absent unit fails the job, so off-host the timeout below bounds the
# attempt instead.
after = [
"swarm-nats-auth-secrets.service"
]
++ lib.optionals baoDeploy.enable [
"swarm-bao-pki.service"
"container@${baoCfg.machine}.service"
];
wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
before = [ "container@${machine}.service" ];
wantedBy = [
"multi-user.target"
"container@${machine}.service"
];
path = [
baoDeploy.package
pkgs.coreutils
pkgs.systemd
];
# ./lib/store-retry.nix. The container waits for one attempt; a secret a
# later attempt lands restarts the responder (below).
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "swarm-bao-nats-auth-oidc";
TimeoutStartSec = 30;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
}
// lib.optionalAttrs authStoreActive {
BAO_CLIENT_CERT = deployCfg.nats.authBaoClientCertFile;
BAO_CLIENT_KEY = deployCfg.nats.authBaoClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
script =
if !authStoreActive then
''
echo "the swarm queue responder has no store identity, so it cannot read its OIDC client secret: set services.hyperhive.deploy.nats.authBaoClientCertFile and authBaoClientKeyFile" >&2
exit 1
''
else
''
set -euo pipefail
${atomicWriteSecret}
${refreshConsumer}
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
# `-token-only` answers on stdout and skips the token helper, a
# `sh` this unit's `path` does not carry.
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with the queue responder's certificate; leaving its OIDC client secret as it is." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
export BAO_TOKEN
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg storeSecretPath} 2>"$err")"; then
echo "swarm-bao did not return ${storeSecretPath}; the queue responder has no OIDC client secret yet." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 0
fi
if [ -z "$secret" ]; then
echo "swarm-bao returned an empty ${storeSecretPath}; leaving the file as it is." >&2
exit 0
fi
# Root-owned 0400: `LoadCredential` reads it as root before the
# responder's `DynamicUser` exists. `printf` inside
# `atomic_write_secret` is a builtin, so the plaintext never
# becomes an argument in /proc.
install -d -m 0700 ${lib.escapeShellArg secretDir}
if secret_differs ${lib.escapeShellArg (hostPath "oidc-client.secret")} "$secret"; then
atomic_write_secret 0400 "" ${lib.escapeShellArg (hostPath "oidc-client.secret")} "$secret"
fi
# `LoadCredential` copies the file once, at the responder's start.
refresh_consumer ${lib.escapeShellArg machine} swarm-nats-auth.service ${lib.escapeShellArg (hostPath "oidc-client.secret")}
'';
};
# ⚠️ Minted on the HOST, not in the container, because the responder is
# a separate unit that needs the user seed: generating it inside would
# trap it there and require a secret-export path back out — the exact

2
nix/host-modules/swarm-otel.nix
View file

@ -132,7 +132,7 @@ let
# Upstream's collector unit runs with `DynamicUser = true`, and the
# prometheus receiver opens `client_secret_file` ITSELF, at runtime, as that
# user — so there is no stable uid to hand a file to, and the root-owned
# 0400 shape `hive-matrix-oidc-secret` delivers to would be unreadable.
# 0400 shape `swarm-bao-matrix-oidc` delivers to would be unreadable.
# (Matrix gets away with it because `LoadCredential` reads the file as root
# before the sandbox exists, and tuwunel never opens that path itself.)
#

12
nix/host-modules/swarm-secret-publisher.nix
View file

@ -79,6 +79,11 @@ let
# The store's browser UI, whose `oidc` auth method the store's host
# configures with this secret.
hyperhiveCfg.swarm.bao.ui.oidc.clientId
# The queue's auth-callout responder, read by `swarm-bao-nats-auth-oidc`
# on the queue's host.
hyperhiveCfg.swarm.nats.clientId
# The homeserver, read by `swarm-bao-matrix-oidc` on its host.
hyperhiveCfg.swarm.matrix.sso.clientId
];
# The swarm controller's OIDC client, which it reads back from the store
@ -110,7 +115,8 @@ in
Turning it off leaves every hive but this one without its agents'
credential, the swarm controller unable to start, the swarm's Grafana
without any login at all, and its collector pushing unauthenticated.
without any login at all, its collector pushing unauthenticated, the
queue's responder unable to start, and the homeserver unable to start.
Each secret has exactly one route and this is the producer's
end of it, so the honest reason to set it false is a deployment
delivering those secrets by some other mechanism it owns.
@ -249,8 +255,8 @@ in
value=@"$src"
published=$((published + 1))
else
# Authelia mints it only where the controller registers its client,
# which is where the controller runs beside it.
# Not an error: authelia mints on its first boot, and the path unit
# above re-runs this when it lands.
echo "no minted secret at $src yet; the path unit will re-run this" >&2
skipped=$((skipped + 1))
fi