Watch
0
0
Fork
You've already forked hyperhive
0
hyperhive/nix/module-eval/secret-publisher.nix
atlas b68a8ef290 swarm-authelia: run the secret publisher and a journal forwarder inside the container
swarm-secret-publish and its path unit now run inside authelia's container,
beside the mint, and read each client secret at the path authelia writes it.
The host unit and host path unit, which read the secrets through the
container's root on the host, are gone, and so is
deploy.authelia.hostClientSecretDir, which only they read. The publisher keeps
its store identity, the secret-publisher leaf. The host hands the leaf and the
store's CA into the container as nspawn system credentials, and the unit loads
them; where this host mints the leaf, the container waits for swarm-bao-pki.
Authelia does not wait for the publisher, which retries every 30 seconds while
the store is down.

The container also runs the journal forwarder the other service containers
run, pushing to a receiver of its own on the swarm collector (serviceProducers
swarm-authelia) as a new machine OIDC client (swarm.authelia.otel.clientId)
that authelia registers for itself and the publisher carries to the store. The
fetch unit reads it back under the publisher's leaf, also handed to it as a
system credential, so the swarm-secret-publisher policy gains read on that one
leaf; its stanza keeps create and update, which the exact path would otherwise
take from the services/* prefix. Without the leaf the forwarder is off.

The container drops --link-journal=host, and the host collector stops reading
it. The services' "nothing at <path>" errors name the publisher in authelia's
container.
2026-10-17 00:28:16 +02:00

519 lines
24 KiB
Nix

# `checks.module-eval-secret-publisher` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
pkgs,
lib,
self,
nixosSystem,
shared,
}:
let
inherit
(import ./lib.nix {
inherit
pkgs
lib
self
nixosSystem
;
})
hive
runGroup
;
# The IdP and the store on one machine: the shape where minted plaintext and
# a store identity are both present without an operator placing anything.
# Two hives in the roster, because the publisher walks it — an arm written
# against a single-hive fixture passes on a hardcoded name.
secretPublisherHere = hive {
deploy.bao.enable = true;
deploy.authelia.enable = true;
swarm.hives.h2.domain = "h2.t.local";
};
# The IdP with no store on the box and a leaf placed by hand, which is the
# deployment this unit exists for: authelia is the one host the store is
# guaranteed not to share once either has a machine of its own.
#
# ⚠️ `enable` is deliberately NOT set here: leaving it unset is what makes
# this fixture exercise the default rather than mask it.
secretPublisherRemote = hive {
deploy.authelia.enable = true;
deploy.swarm-secret-publisher.baoClientCertFile = "/etc/pki/publisher.pem";
deploy.swarm-secret-publisher.baoClientKeyFile = "/etc/pki/publisher-key.pem";
};
# The same IdP with the identity taken away. Minting the secrets is not being
# able to publish them, and this is the arm that separates the two.
secretPublisherNoIdentity = shared.autheliaOnly;
# Duplicated from grafana.nix — a case here needs it too.
# The same UI with the IdP on ANOTHER host and a store leaf placed by hand.
# Knowing an IdP is not running one: `swarm.authelia.url` is what says this
# swarm has SSO, and nothing about this host does. Identical to the fixture
# above in everything the delivery path reads, which is the point.
grafanaRemoteAuthelia = hive {
deploy.grafana.enable = true;
deploy.grafana.plugins = [ ];
deploy.grafana.package = pkgs.emptyDirectory;
swarm.authelia.url = "https://auth.example.invalid";
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
};
# The same collector with the IdP on ANOTHER host and a store leaf placed by
# hand. Identical to the fixture above in everything the delivery path
# reads, which is the point.
otelBaoRemoteAuthelia = shared.otelReaderMissingLeaf;
inherit (shared) bare;
# The store and a service that reads from it, versus the store alone. The
# pair is what makes the reader's absence arm mean anything.
inherit (shared) baoWithMatrix;
# The publisher runs inside authelia's container.
inAuthelia = m: m.containers.${m.services.hyperhive.swarm.authelia.machine}.config;
cases = [
{
# Each hive's hive-c0re reads this exact path
# (`swarm_secret_client::queue::hive_client_path`, pinned by that crate's
# own test), for every hive in the roster, not only the one this host is.
# The agents' client beside it stays a separate path.
name = "the publisher writes every hive's own client under that hive's prefix";
ok =
let
s = (inAuthelia secretPublisherHere).systemd.services.swarm-secret-publish.script;
in
lib.all
(h: lib.hasInfix "secret/swarm/hives/${h}/oidc/client" s && lib.hasInfix "/hive-${h}.secret" s)
[
"h1"
"h2"
]
&& lib.hasInfix "secret/swarm/hives/h1/queue/agent" s;
}
{
# The controller reads this exact path
# (`swarm_secret_client::queue::controller_client_path`, pinned by that
# crate's own test). Under `controller/`, which no hive reads: this
# client may read every hive's status.
name = "the publisher writes the controller's client under controller/, never services/";
ok =
let
s = (inAuthelia secretPublisherHere).systemd.services.swarm-secret-publish.script;
in
lib.hasInfix "secret/swarm/controller/swarm-controller/oidc/client" s
&& lib.hasInfix "/swarm-controller.secret" s
&& !(lib.hasInfix "secret/swarm/services/swarm-controller" s);
}
{
# Both ends of a wire nothing at eval time carries end to end: the
# publisher in authelia's container writes the path the reader on
# Grafana's host reads, and the two files agree only because both compose
# it from the same swarm-wide client id.
name = "the publisher writes the swarm service path grafana reads";
ok = lib.hasInfix "secret/swarm/services/swarm-grafana/oidc/client" (
(inAuthelia secretPublisherHere).systemd.services.swarm-secret-publish.script
);
}
{
# Registering the client cannot live where the rest of grafana's module
# lives: that block is gated on this host RUNNING grafana, so on the split
# deployment nothing registered the client, authelia minted no secret, and
# every layer below had nothing to carry. The second arm is the control —
# a host with no IdP registers nothing.
name = "the swarm's grafana client is registered wherever authelia runs";
ok =
let
clients = m: map (c: c.id) m.services.hyperhive.swarm.authelia.oidc.clients;
in
lib.elem "swarm-grafana" (clients secretPublisherHere)
&& !(lib.elem "swarm-grafana" (clients grafanaRemoteAuthelia));
}
{
# The collector's half of the same property: the client is not gated on
# `deploy.swarm-otel.enable`, which would let a split deployment register
# it nowhere, so authelia would mint nothing to publish.
name = "the swarm's collector client is registered wherever authelia runs";
ok =
let
clients = m: map (c: c.id) m.services.hyperhive.swarm.authelia.oidc.clients;
in
lib.elem "swarm-collector" (clients secretPublisherHere)
&& !(lib.elem "swarm-collector" (clients otelBaoRemoteAuthelia));
}
{
# Registration follows authelia, not the service: the IdP host below runs
# no controller, queue or homeserver (the first arms), and still lists all
# three clients. The last arm is the control — a host with no IdP lists
# none of them.
name = "an authelia host running no controller, queue or homeserver registers all three clients";
ok =
let
d = secretPublisherRemote.services.hyperhive.deploy;
clients = m: map (c: c.id) m.services.hyperhive.swarm.authelia.oidc.clients;
byId =
id:
lib.findFirst (
c: c.id == id
) null secretPublisherRemote.services.hyperhive.swarm.authelia.oidc.clients;
controller = byId "swarm-controller";
nats = byId "swarm-nats";
matrix = byId "tuwunel";
in
!d.swarm-controller.enable
&& !d.nats.enable
&& !d.matrix.enable
&& controller.kind == "machine"
&& controller.audience == [ "swarm-controller" ]
&& nats.kind == "machine"
&&
matrix.redirectUris == [
"https://chat.t.local/_matrix/client/unstable/login/sso/callback/tuwunel"
]
&& matrix.tokenEndpointAuthMethod == "client_secret_post"
&& !(lib.any (id: lib.elem id (clients grafanaRemoteAuthelia)) [
"swarm-controller"
"swarm-nats"
"tuwunel"
]);
}
{
# The other half of the queue's, the homeserver's and the forge's one
# route: the publisher on an IdP host that runs none of them carries each
# secret to the path its reader fetches.
name = "the publisher on an authelia host carries the queue's, homeserver's and forge's secrets";
ok =
let
s = (inAuthelia secretPublisherRemote).systemd.services.swarm-secret-publish.script;
in
lib.hasInfix "secret/swarm/services/swarm-nats/oidc/client" s
&& lib.hasInfix "/swarm-nats.secret" s
&& lib.hasInfix "secret/swarm/services/tuwunel/oidc/client" s
&& lib.hasInfix "/tuwunel.secret" s
&& !secretPublisherRemote.services.hyperhive.deploy.forgejo.enable
&& lib.hasInfix "secret/swarm/services/forgejo/oidc/client" s
&& lib.hasInfix "/forgejo.secret" s;
}
{
# Both ends of a wire nothing at eval time carries end to end: the
# publisher in authelia's container writes the path the reader on the
# collector's host reads, and the two files agree only because both
# compose it from the same swarm-wide client id. `secretPublisherHere`
# already grew this client when `serviceClientIds` did.
name = "the publisher writes the swarm service path the collector reads";
ok = lib.hasInfix "secret/swarm/services/swarm-collector/oidc/client" (
(inAuthelia secretPublisherHere).systemd.services.swarm-secret-publish.script
);
}
{
# The third swarm service on that list, and the one whose absence is
# hardest to see: the store's own forwarder holds a client authelia
# registers and mints for, so every layer above looks complete while
# the one hop that CARRIES the value skips it and the reader on the
# store's host waits forever on a path nothing writes.
name = "the publisher carries the store forwarder's secret too";
ok = lib.hasInfix "secret/swarm/services/swarm-bao-collector/oidc/client" (
(inAuthelia secretPublisherHere).systemd.services.swarm-secret-publish.script
);
}
{
# The forwarders in the CI runner's, Grafana's, the log store's, the
# forge's, the homeserver's and authelia's containers read their secrets
# from these paths, and the metrics store's host reads its container's.
name = "the publisher carries the CI runner, Grafana, log store, forge, homeserver, metrics store and authelia forwarders' secrets";
ok =
lib.all
(
id:
lib.hasInfix "secret/swarm/services/${id}/oidc/client" (
(inAuthelia secretPublisherHere).systemd.services.swarm-secret-publish.script
)
)
[
"swarm-ci-runner-collector"
"swarm-grafana-collector"
"swarm-victorialogs-collector"
"swarm-forge-collector"
"swarm-matrix-collector"
"swarm-victoriametrics-collector"
"swarm-authelia-collector"
];
}
{
# The review classifier reads its queue client secret from the store
# inside its container; authelia registers it on every IdP host.
name = "the review classifier's client is registered as a machine, and the publisher carries its secret";
ok =
let
c = lib.findFirst (
c: c.id == "swarm-review-classifier"
) null secretPublisherHere.services.hyperhive.swarm.authelia.oidc.clients;
in
c != null
&& c.kind == "machine"
&& lib.hasInfix "secret/swarm/services/swarm-review-classifier/oidc/client" (
(inAuthelia secretPublisherHere).systemd.services.swarm-secret-publish.script
);
}
{
# The store's UI login: authelia registers the client with the callback
# the store's `oidc` role allows, and the publisher carries its secret
# to the path `swarm-bao-operator-viewer-policy` reads. Not registered
# where authelia is not.
name = "the store UI's client is registered interactive at its callback, and the publisher carries its secret";
ok =
let
ui = lib.findFirst (
c: c.id == "swarm-bao-ui"
) null secretPublisherHere.services.hyperhive.swarm.authelia.oidc.clients;
in
ui != null
&& ui.kind == "interactive"
&& ui.redirectUris == [ "https://bao-ui.t.local/ui/vault/auth/oidc/oidc/callback" ]
&& lib.hasInfix "secret/swarm/services/swarm-bao-ui/oidc/client" (
(inAuthelia secretPublisherHere).systemd.services.swarm-secret-publish.script
)
&& !(lib.any (
c: c.id == "swarm-bao-ui"
) grafanaRemoteAuthelia.services.hyperhive.swarm.authelia.oidc.clients);
}
{
# The same hole the controller's case above names, open a second time: the
# PKI script grew a third leaf and no case read it.
name = "the store mints a leaf for the secret publisher, and the publisher is pointed at it";
ok =
let
m = secretPublisherHere;
p = m.services.hyperhive.deploy.swarm-secret-publisher;
in
lib.hasInfix "secret-publisher.pem" m.systemd.services.swarm-bao-pki.script
&& p.baoClientCertFile == "/var/lib/swarm-bao-pki/secret-publisher.pem"
&& p.baoClientKeyFile == "/var/lib/swarm-bao-pki/secret-publisher-key.pem";
}
{
# mara caught this by reading, which means no arm existed for it: the
# default asked `authelia.enable && bao.enable`, so the split deployment
# this unit is FOR defaulted off and published nothing, silently. The
# default is set by ../host-modules/glue-secret-publisher-authelia.nix.
#
# The second clause is the control. Without it this passes on a default
# of plain `true`, which would be a different bug with the same symptom
# — an IdP-less host claiming it publishes secrets it never mints.
name = "the publisher defaults on where secrets are minted, whether or not the store is local";
ok =
secretPublisherRemote.services.hyperhive.deploy.swarm-secret-publisher.enable
&& !bare.services.hyperhive.deploy.swarm-secret-publisher.enable;
}
{
# The one security property of this unit, and why its push cannot be
# rewritten into the obvious shape: `bao` is an external binary, so an
# argument is world-readable in /proc for the life of the call.
# `value=@<path>` hands it the path and bao opens the file itself.
#
# The second arm is what makes the first mean anything — `value=@` can
# sit one line above a command substitution that put the plaintext in
# argv anyway.
#
# ⚠️ Comments are stripped first, and that is not tidiness. A `script`
# renders its own comments into the text, and this unit's comments name
# the hazard verbatim so the next editor does not reintroduce it. Without
# the strip this case reads that warning and fails — a check the artifact
# defeats by DESCRIBING the thing it is checked for.
name = "the publisher hands bao the secret's path, never the secret";
ok =
let
s = (inAuthelia secretPublisherHere).systemd.services.swarm-secret-publish.script;
code = lib.concatStringsSep "\n" (
lib.filter (l: builtins.match "[[:space:]]*#.*" l == null) (lib.splitString "\n" s)
);
in
lib.hasInfix "value=@" code && !(lib.hasInfix "$(cat" code);
}
{
# Two ends of a wire nothing at eval time carries end to end: this is the
# path `swarm_secret_client::queue` resolves for the reader. Both hives
# are asserted, so a publisher that knew one name rather than the roster
# fails here rather than on the second hive ever added to a swarm.
name = "the publisher writes every hive in the roster to that hive's own queue path";
ok =
let
s = (inAuthelia secretPublisherHere).systemd.services.swarm-secret-publish.script;
in
lib.hasInfix "secret/swarm/hives/h1/queue/agent" s
&& lib.hasInfix "secret/swarm/hives/h2/queue/agent" s;
}
{
# A property of the SET, not of one unit: both of these authenticate by
# certificate, and `BAO_CLIENT_CERT` is transport rather than identity, so
# a script that reaches `bao kv` without a token asks a token helper this
# host does not carry and fails before the store ever answers. `-token-only`
# is what keeps the token off the helper on the way back out.
#
# Ordering, not presence: the login has to come first, so the check is
# that nothing before it is a data command. Comments are stripped because
# both units explain this in prose directly above the code.
name = "the cert-identity bao units log in before their first read or write, and keep the token out of the helper";
ok =
let
code =
s:
lib.concatStringsSep "\n" (
lib.filter (l: builtins.match "[[:space:]]*#.*" l == null) (lib.splitString "\n" s)
);
holdsTokenFirst =
s:
let
c = code s;
in
lib.hasInfix "bao login" c
&& lib.hasInfix "-token-only" c
&& !(lib.hasInfix "bao kv" (lib.head (lib.splitString "bao login" c)));
in
holdsTokenFirst (inAuthelia secretPublisherHere).systemd.services.swarm-secret-publish.script
&& holdsTokenFirst baoWithMatrix.systemd.services.swarm-bao-queue-agent.script
# Controls, so a clean verdict above means something. In order: a bare
# read is refused, a read placed before the login is refused, and a
# login that exists only in a comment is refused — that last one is the
# arm the comment-stripping exists for.
&& !(holdsTokenFirst "bao kv get -field=value secret/x")
&& !(holdsTokenFirst "bao kv get secret/x\nBAO_TOKEN=\"$(bao login -method=cert -token-only)\"")
&& !(holdsTokenFirst "# bao login -method=cert -token-only goes here\nbao kv get secret/x")
&& holdsTokenFirst "BAO_TOKEN=\"$(bao login -method=cert -token-only)\"\nbao kv get secret/x";
}
{
# The doctrine three glue files state, as a property a rewrite has to
# keep: a client is defined by holding a certificate the store accepts,
# never by standing next to the store. Gating this on `deploy.bao.enable`
# would have left the unit rendering only on the one deployment that has
# no use for it. It runs beside the mint, in authelia's container, and
# never on the host.
name = "a publisher holding an identity runs in authelia's container on a host with no store";
ok =
let
m = secretPublisherRemote;
c = inAuthelia m;
in
!m.services.hyperhive.deploy.bao.enable
&& (c.systemd.services ? swarm-secret-publish)
&& (c.systemd.paths ? swarm-secret-publish)
&& !(m.systemd.services ? swarm-secret-publish)
&& !(m.systemd.paths ? swarm-secret-publish)
# Control: the host's own unit set is the one read above.
&& (m.systemd.services ? "container@swarm-authelia");
}
{
# What makes the arm above able to fail. Minting the secrets is not being
# able to publish them: with no certificate the unit would fail a TLS
# handshake on every rotation, so it must not exist at all, and nspawn
# gets no credential to load.
name = "an IdP host with no store identity renders no publisher";
ok =
let
m = secretPublisherNoIdentity;
c = inAuthelia m;
in
m.services.hyperhive.deploy.authelia.enable
&& !(c.systemd.services ? swarm-secret-publish)
&& !(c.systemd.paths ? swarm-secret-publish)
&& !(m.systemd.services ? swarm-secret-publish)
&& !(lib.any (lib.hasPrefix "--load-credential=") m.containers.swarm-authelia.extraFlags)
# Controls: with an identity, the same lookups find both.
&& ((inAuthelia secretPublisherHere).systemd.services ? swarm-secret-publish)
&& lib.any (lib.hasPrefix "--load-credential=") secretPublisherHere.containers.swarm-authelia.extraFlags;
}
{
# The path unit's start is a no-op while its target is `active (exited)`,
# so a secret minted after the first run would never be published.
# `Type` is read too, so a missing `serviceConfig` cannot pass the arm.
# The watched directory is the mint's own, inside the container.
name = "the publisher's path unit watches the mint's directory and starts a oneshot that does not remain active";
ok =
let
c = inAuthelia secretPublisherHere;
sc = c.systemd.services.swarm-secret-publish.serviceConfig;
in
c.systemd.paths.swarm-secret-publish.pathConfig.Unit == "swarm-secret-publish.service"
&&
c.systemd.paths.swarm-secret-publish.pathConfig.PathChanged
== "/var/lib/authelia-swarm/oidc-clients"
&& lib.hasInfix "/var/lib/authelia-swarm/oidc-clients/swarm-grafana.secret" c.systemd.services.swarm-secret-publish.script
&& !(lib.hasInfix "/var/lib/nixos-containers" c.systemd.services.swarm-secret-publish.script)
&& sc.Type == "oneshot"
&& !(sc.RemainAfterExit or false);
}
{
# The leaf crosses into the container as nspawn system credentials, and
# the unit loads them and names them in its environment. On the store's
# host the container waits for the unit that mints the leaf, since nspawn
# refuses a missing credential source; with the leaf placed by hand,
# those paths cross.
name = "the publisher's store identity reaches authelia's container as system credentials";
ok =
let
dir = "/run/credentials/swarm-secret-publish.service";
unit = m: (inAuthelia m).systemd.services.swarm-secret-publish;
flags = m: m.containers.swarm-authelia.extraFlags;
here = unit secretPublisherHere;
in
lib.elem "--load-credential=publisher-bao-cert:/var/lib/swarm-bao-pki/secret-publisher.pem" (
flags secretPublisherHere
)
&& lib.elem "--load-credential=publisher-bao-key:/var/lib/swarm-bao-pki/secret-publisher-key.pem" (
flags secretPublisherHere
)
&& lib.elem "--load-credential=publisher-bao-server-ca:/var/lib/swarm-bao-pki/ca.pem" (
flags secretPublisherHere
)
&&
lib.sort lib.lessThan here.serviceConfig.LoadCredential == [
"publisher-bao-cert"
"publisher-bao-key"
"publisher-bao-server-ca"
]
&& here.environment.BAO_CLIENT_CERT == "${dir}/publisher-bao-cert"
&& here.environment.BAO_CLIENT_KEY == "${dir}/publisher-bao-key"
&& here.environment.BAO_CACERT == "${dir}/publisher-bao-server-ca"
&&
lib.elem "swarm-bao-pki.service"
secretPublisherHere.systemd.services."container@swarm-authelia".after
&& lib.elem "--load-credential=publisher-bao-cert:/etc/pki/publisher.pem" (
flags secretPublisherRemote
)
&& lib.elem "--load-credential=publisher-bao-key:/etc/pki/publisher-key.pem" (
flags secretPublisherRemote
)
&& (unit secretPublisherRemote).environment.BAO_CLIENT_CERT == "${dir}/publisher-bao-cert";
}
{
# Authelia is the swarm's IdP and has to come up while the store is down.
# Nothing it starts after or needs may be the publisher, and the
# publisher may not order itself in front of authelia.
name = "authelia does not wait for the secret publisher";
ok =
let
c = inAuthelia secretPublisherHere;
authelia = c.systemd.services.authelia-swarm;
publisher = c.systemd.services.swarm-secret-publish;
unit = "swarm-secret-publish.service";
in
!(lib.any (l: lib.elem unit (authelia.${l} or [ ])) [
"after"
"requires"
"wants"
"bindsTo"
"requisite"
])
&& !(lib.any (l: lib.elem "authelia-swarm.service" (publisher.${l} or [ ])) [
"before"
"requiredBy"
])
# Controls: both lookups find the real units.
&& authelia.serviceConfig ? ExecStart
&& lib.elem "authelia-swarm-secrets.service" publisher.after;
}
];
in
runGroup "secret-publisher" cases