vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-authelia: run the secret publisher and a journal forwarder inside the container

Browse source 
swarm-secret-publish and its path unit now run inside authelia's container,
beside the mint, and read each client secret at the path authelia writes it.
The host unit and host path unit, which read the secrets through the
container's root on the host, are gone, and so is
deploy.authelia.hostClientSecretDir, which only they read. The publisher keeps
its store identity, the secret-publisher leaf. The host hands the leaf and the
store's CA into the container as nspawn system credentials, and the unit loads
them; where this host mints the leaf, the container waits for swarm-bao-pki.
Authelia does not wait for the publisher, which retries every 30 seconds while
the store is down.

The container also runs the journal forwarder the other service containers
run, pushing to a receiver of its own on the swarm collector (serviceProducers
swarm-authelia) as a new machine OIDC client (swarm.authelia.otel.clientId)
that authelia registers for itself and the publisher carries to the store. The
fetch unit reads it back under the publisher's leaf, also handed to it as a
system credential, so the swarm-secret-publisher policy gains read on that one
leaf; its stanza keeps create and update, which the exact path would otherwise
take from the services/* prefix. Without the leaf the forwarder is off.

The container drops --link-journal=host, and the host collector stops reading
it. The services' "nothing at <path>" errors name the publisher in authelia's
container.
This commit is contained in:
atlas 2026-10-16 15:05:29 +02:00 • committed by mara
commit b68a8ef290
36 changed files with 753 additions and 363 deletions
Download patch file Download diff file Expand all files Collapse all files

205
nix/host-modules/swarm-authelia.nix
View file

@ -181,9 +181,82 @@ let
caBundleModule = caTrust.trustBundle {
inherit pkgs;
name = cfg.machine;
consumers = [ "swarm-authelia-bridge" ];
consumers = [
"swarm-authelia-bridge"
]
# Go, for the swarm collector's name and authelia's own. Only where the
# forwarder runs: naming it would otherwise define an empty unit.
++ lib.optional publishes "opentelemetry-collector";
};
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.swarm-authelia` in
# ./swarm-otel-service.nix declares, as a client of its own. The port suffix
# is elided on 443, as in ./swarm-bao.nix.
forwarderClientId = cfg.otel.clientId;
gatewayPortSuffix = lib.optionalString (
gatewayCfg.httpsPort != 443
) ":${toString gatewayCfg.httpsPort}";
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}${gatewayPortSuffix}/${cfg.machine}";
# In the host netns: distinct from every other collector's there, which the
# assertion below checks.
forwarderTelemetryPort = 8901;
# A client this container registers and mints for itself. Its secret goes
# through the store like every other forwarder's: the publisher carries it
# there and the forwarder reads it back under the same identity.
forwarderClient = {
id = forwarderClientId;
description = "HyperHive authelia journal forwarder";
kind = "machine";
redirectUris = [ ];
# Its own id as its own permitted audience, the one the swarm collector's
# receiver for this container checks.
audience = [ forwarderClientId ];
# The receiver verifies the token offline against `/jwks.json`;
# authelia's default opaque handle fails there.
accessTokenSignedResponseAlg = "RS256";
};
# The secret publisher runs in this container, beside the mint whose secrets
# it copies. It needs a store identity, and a reader is defined by holding a
# certificate the store accepts, never by standing next to it — the rule
# ./glue-queue-agent-credential.nix states in full. `deploy.bao.enable` here
# would be the co-location assumption itself.
publisherCfg = deployCfg.swarm-secret-publisher;
publishes =
publisherCfg.enable
&& publisherCfg.baoClientCertFile != null
&& publisherCfg.baoClientKeyFile != null;
publisherUnit = "swarm-secret-publish";
forwarderFetchUnit = "swarm-journal-forwarder-oidc";
# The publisher's store identity, handed into the container as system
# credentials (credential id → host source). The forwarder's fetch presents
# it too: one identity for the whole container.
storeFiles = lib.optionalAttrs publishes (
lib.filterAttrs (_: source: source != null) {
publisher-bao-cert = publisherCfg.baoClientCertFile;
publisher-bao-key = publisherCfg.baoClientKeyFile;
publisher-bao-server-ca = deployCfg.bao.serverCaFile;
}
);
# nixos-container expands `extraFlags` unquoted, so a source path must be
# one word.
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
# The store options both units take, with the credentials as each loads them.
storeIdentityIn =
unit:
let
credential = id: "/run/credentials/${unit}.service/${id}";
in
{
package = deployCfg.bao.package;
addr = "https://${hyperhiveCfg.swarm.bao.domain}:${toString hyperhiveCfg.swarm.bao.port}";
clientCertFile = credential "publisher-bao-cert";
clientKeyFile = credential "publisher-bao-key";
caFile =
if storeFiles ? publisher-bao-server-ca then credential "publisher-bao-server-ca" else null;
};
# The SWARM's domain, because that is where the protected apps now live
# (`forge.<swarm>`, `chat.<swarm>`, `auth.<swarm>`). It moves in the
# same commit as `domain` (./swarm-authelia-service.nix) and cannot lag
@ -529,7 +602,7 @@ in
# register every service checks itself against and its port, is
# `swarm.authelia` in ./swarm-authelia-service.nix. What the host running
# the container decides is here — which two builds it
# runs, and three filesystem paths that only exist on the machine running
# runs, and two filesystem paths that only exist on the machine running
# `swarm-authelia`. A hive that does not run it has nothing at any of those
# paths. `enable` already lives in ./deploy.nix, which also carries the
# renames.
@ -578,28 +651,6 @@ in
'';
};
hostClientSecretDir = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "/var/lib/nixos-containers/${cfg.machine}${clientsDir}";
description = ''
Where the minted client secrets sit **as seen from the host** —
`<id>.secret` holds a plaintext, `<id>.digest` the hash authelia
itself reads.
Published for the same reason as `hostUsersFile`: the plaintext's
other reader lives in a **different container**, and containers
that share this host's network namespace still have separate
filesystem roots. The host is the only place both trees are
addressable, so the host is where a delivery step has to run.
⚠️ Nothing here exists until authelia's **first boot** has run.
A consumer must wait for it — it cannot be a `bindMounts` source,
because nixos-container refuses to start when a bind source is
missing, and that turns a fresh hive into a boot-order deadlock.
'';
};
hostUsersFile = lib.mkOption {
type = lib.types.str;
readOnly = true;
@ -628,7 +679,10 @@ in
# — see `oidcEnabled`'s comment above.
services.hyperhive.swarm.authelia.oidc.clients =
lib.optionals cfg.oidc.hiveIdentities (hiveClients ++ agentClients)
++ [ bridgeClient ]
++ [
bridgeClient
forwarderClient
]
++ lib.optional (swarmDomain != null) swarmctlClient;
# A redirect URI on a machine client is not harmless-but-unused: it
@ -772,6 +826,39 @@ in
+ "evaluating and deploying cleanly. A browser client has a user "
+ "to authorise and does not need it.";
}
{
assertion = lib.all oneWord (lib.attrValues storeFiles);
message = ''
The secret publisher's store identity paths must not contain
whitespace: nspawn's --load-credential receives them through
nixos-container's unquoted extra flags.
'';
}
{
assertion =
(hyperhiveCfg.swarm.otel.serviceProducers.${cfg.machine}.clientId or null) == forwarderClientId;
message = ''
services.hyperhive.swarm.otel.serviceProducers has no entry
"${cfg.machine}" with clientId "${forwarderClientId}", so the swarm
collector has no receiver for authelia's journal forwarder. Declare
it in swarm-otel-service.nix.
'';
}
{
assertion =
!(lib.elem forwarderTelemetryPort [
hyperhiveCfg.otel.telemetryPort
hyperhiveCfg.swarm.otel.telemetryPort
hyperhiveCfg.swarm.bao.otel.telemetryPort
hyperhiveCfg.swarm.nats.otel.telemetryPort
]);
message = ''
authelia's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
collides with another collector's in the host netns: the hive
collector's, the swarm collector's, the store forwarder's or the
queue forwarder's.
'';
}
];
# Authelia's own gateway surface: the vhost that fronts it and the
@ -899,15 +986,25 @@ in
};
# Order the container after the host CA generator, so the bind source
# exists before nspawn sets the mount up.
systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;
# exists before nspawn sets the mount up. nspawn also fails to start it
# when a credential source is missing, so where this host mints the
# publisher's leaf the container pulls the minting unit in and waits for
# it, as ./swarm-grafana.nix's does. The minting is local: the store need
# not be up.
systemd.services."container@${cfg.machine}" = lib.mkMerge [
caTrust.containerOrdering
(lib.mkIf publishes {
after = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];
})
];
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# No `--link-journal=host`: the journal stays in the guest, where the
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
inherit privateNetwork;
# Public trust bundle only, read-only. Empty when the gateway is not
# self-signed, so the whole trust path drops out cleanly.
@ -918,15 +1015,61 @@ in
{
imports = [
../container-modules/swarm-container.nix
../container-modules/swarm-journal-forwarder.nix
../container-modules/swarm-secret-publisher.nix
(import ../container-modules/swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
dnsConsumers = [ cfg.unit ];
# The forwarder dials the swarm collector and authelia by name,
# and its fetch and the publisher the store.
dnsConsumers = [
cfg.unit
]
++ lib.optionals publishes [
"opentelemetry-collector.service"
"${forwarderFetchUnit}.service"
"${publisherUnit}.service"
];
})
caBundleModule
];
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# Authenticated as the forwarder's own OIDC client, its secret read
# from the store with the publisher's identity: one identity for the
# whole container. With no identity there is nothing to read it with,
# and the forwarder is off.
services.hyperhive.swarmJournalForwarder = lib.mkIf publishes {
enable = true;
serviceName = "authelia";
endpoint = forwarderEndpoint;
clientId = forwarderClientId;
tokenUrl = "${toString cfg.url}/api/oidc/token";
telemetryPort = forwarderTelemetryPort;
store = storeIdentityIn forwarderFetchUnit // {
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
};
};
systemd.services.${forwarderFetchUnit} = lib.mkIf publishes {
serviceConfig.LoadCredential = lib.attrNames storeFiles;
};
# Copies what the mint below leaves in `clientsDir` into the store,
# under the publisher's own identity, a system credential. Nothing
# orders authelia after it.
services.hyperhive.swarmSecretPublisher = lib.mkIf publishes {
enable = true;
clientSecretDir = clientsDir;
inherit (publisherCfg) secrets;
store = storeIdentityIn publisherUnit;
};
systemd.services.${publisherUnit} = lib.mkIf publishes {
# A run before the mint would only skip every secret and wait
# for the path unit.
after = [ "${unitName}-secrets.service" ];
serviceConfig.LoadCredential = lib.attrNames storeFiles;
};
# The authelia binary itself, so an operator who gets a shell
# in here can run `authelia crypto hash generate` to make a
# password for the users file. Without it the container runs