vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-ci-runner, swarm-grafana, swarm-victorialogs: forward each container's journal to the swarm collector

Browse source 
Each container now runs the journal forwarder the queue's and the builder's
containers run, pushing to a receiver of its own on the swarm collector. Each
authenticates as a new machine OIDC client of its own
(swarm.<service>.otel.clientId), registered by a glue file wherever authelia
runs, minted by authelia and published to the store by the secret publisher.
Grafana's browser login client is unchanged.

The fetch unit inside each container reads the client secret with the store
identity the host already holds for that service (the runner's, Grafana's,
and the log store responder's), now also handed into the container as an
nspawn system credential. Each of those three policies gains one read stanza
on its forwarder's client path. The host units that use those identities are
unchanged.

The three containers drop --link-journal=host, and the host collector stops
reading them.
This commit is contained in:
atlas 2026-10-16 10:14:57 +02:00 • committed by mara
commit c6ca9ab845
20 changed files with 810 additions and 107 deletions
Download patch file Download diff file Expand all files Collapse all files

3
nix/host-modules/default.nix
View file

@ -47,12 +47,15 @@
./glue-nats-wireguard.nix
./glue-queue-agent-credential.nix
./glue-builder-oidc-client.nix
./glue-ci-runner-otel-oidc-client.nix
./glue-claude-login-oidc-client.nix
./glue-grafana-otel-oidc-client.nix
./glue-review-classifier-oidc-client.nix
./glue-secret-publisher-authelia.nix
./glue-secret-publisher-bao-identity.nix
./glue-swarm-bao-otel-oidc-client.nix
./glue-swarm-otel-oidc-client.nix
./glue-victorialogs-otel-oidc-client.nix
./swarm-authelia-service.nix
./swarm-authelia.nix
./swarm-bao-service.nix

37
nix/host-modules/glue-ci-runner-otel-oidc-client.nix Normal file
View file

@ -0,0 +1,37 @@
# Glue: register the CI runner container's journal forwarder as an OIDC
# client wherever authelia runs.
#
# ONE PAIRING PER FILE: that forwarder ← authelia, and nothing else.
# Without it authelia mints no secret, the publisher has nothing to copy into
# the store, and the forwarder's export is a 401.
#
# Gated on authelia being HERE, not on this host running the runner: a
# client is a row in THIS host's provider config, the split
# ./glue-swarm-bao-otel-oidc-client.nix makes for the same reason.
{
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
clientId = hyperhiveCfg.swarm.ci-runner.otel.clientId;
in
{
config = lib.mkIf hyperhiveCfg.deploy.authelia.enable {
services.hyperhive.swarm.authelia.oidc.clients = [
{
id = clientId;
description = "HyperHive CI runner journal forwarder";
kind = "machine";
redirectUris = [ ];
# Its own id as its own permitted audience, the one the swarm
# collector's receiver for this container checks.
audience = [ clientId ];
# The receiver verifies the token offline against `/jwks.json`;
# authelia's default opaque handle fails there.
accessTokenSignedResponseAlg = "RS256";
}
];
};
}

37
nix/host-modules/glue-grafana-otel-oidc-client.nix Normal file
View file

@ -0,0 +1,37 @@
# Glue: register the Grafana container's journal forwarder as an OIDC
# client wherever authelia runs.
#
# ONE PAIRING PER FILE: that forwarder ← authelia, and nothing else.
# Without it authelia mints no secret, the publisher has nothing to copy into
# the store, and the forwarder's export is a 401.
#
# Gated on authelia being HERE, not on this host running Grafana: a
# client is a row in THIS host's provider config, the split
# ./glue-swarm-bao-otel-oidc-client.nix makes for the same reason.
{
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
clientId = hyperhiveCfg.swarm.grafana.otel.clientId;
in
{
config = lib.mkIf hyperhiveCfg.deploy.authelia.enable {
services.hyperhive.swarm.authelia.oidc.clients = [
{
id = clientId;
description = "HyperHive Grafana journal forwarder";
kind = "machine";
redirectUris = [ ];
# Its own id as its own permitted audience, the one the swarm
# collector's receiver for this container checks.
audience = [ clientId ];
# The receiver verifies the token offline against `/jwks.json`;
# authelia's default opaque handle fails there.
accessTokenSignedResponseAlg = "RS256";
}
];
};
}

37
nix/host-modules/glue-victorialogs-otel-oidc-client.nix Normal file
View file

@ -0,0 +1,37 @@
# Glue: register the log store container's journal forwarder as an OIDC
# client wherever authelia runs.
#
# ONE PAIRING PER FILE: that forwarder ← authelia, and nothing else.
# Without it authelia mints no secret, the publisher has nothing to copy into
# the store, and the forwarder's export is a 401.
#
# Gated on authelia being HERE, not on this host running the log store: a
# client is a row in THIS host's provider config, the split
# ./glue-swarm-bao-otel-oidc-client.nix makes for the same reason.
{
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
clientId = hyperhiveCfg.swarm.victorialogs.otel.clientId;
in
{
config = lib.mkIf hyperhiveCfg.deploy.authelia.enable {
services.hyperhive.swarm.authelia.oidc.clients = [
{
id = clientId;
description = "HyperHive log store journal forwarder";
kind = "machine";
redirectUris = [ ];
# Its own id as its own permitted audience, the one the swarm
# collector's receiver for this container checks.
audience = [ clientId ];
# The receiver verifies the token offline against `/jwks.json`;
# authelia's default opaque handle fails there.
accessTokenSignedResponseAlg = "RS256";
}
];
};
}

14
nix/host-modules/swarm-bao.nix
View file

@ -931,6 +931,8 @@ let
cn = baoDeploy.grafanaOidcCommonName;
policyText =
readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/oidc/client"
# The journal forwarder in its container, under this same leaf.
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.grafana.otel.clientId}/oidc/client"
+ createStanza "${credentialMountPath}/data/${grafanaAdminLeaf}"
+ createStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}"
+ ''
@ -1018,14 +1020,17 @@ let
# The log store's auth responders: the agent one checks the secret an agent
# presents against the one stored for it (the same `+` leaf as the queue
# responder's), the other checks the bearer Grafana presents against
# Grafana's metrics bearer. Nothing else.
# Grafana's metrics bearer; and the store container's journal forwarder
# reads its own client secret under the same leaf. Nothing else.
logsAuthReaders = [
{
name = "swarm-logs-auth";
cn = baoDeploy.logsAuthCommonName;
policyText =
readStanza "${credentialMountPath}/data/swarm/agents/+/queue"
+ readStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}";
+ readStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}"
# The journal forwarder in the store's container, under this same leaf.
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.victorialogs.otel.clientId}/oidc/client";
}
];
@ -1045,7 +1050,10 @@ let
{
name = "swarm-ci-runner";
cn = baoDeploy.ciRunnerCommonName;
policyText = readStanza "${credentialMountPath}/data/${ciRunnerTokenLeaf}";
policyText =
readStanza "${credentialMountPath}/data/${ciRunnerTokenLeaf}"
# The journal forwarder in the runner's container, under this same leaf.
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.ci-runner.otel.clientId}/oidc/client";
}
];

97
nix/host-modules/swarm-ci-runner.nix
View file

@ -55,9 +55,40 @@ let
consumers = [
runnerUnit
registerUnit
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
# collector's and authelia's names.
"opentelemetry-collector"
];
};
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.swarm-ci-runner` in
# ./swarm-otel-service.nix declares. The port suffix is elided on 443, as in
# ./swarm-bao.nix.
swarmOtelCfg = config.services.hyperhive.swarm.otel;
forwarderClientId = config.services.hyperhive.swarm.ci-runner.otel.clientId;
gatewayPortSuffix = lib.optionalString (
gatewayCfg.httpsPort != 443
) ":${toString gatewayCfg.httpsPort}";
forwarderEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${machine}";
forwarderTokenUrl = "${toString config.services.hyperhive.swarm.authelia.url}/api/oidc/token";
forwarderUnit = "swarm-journal-forwarder-oidc";
# Loopback in the container's own netns, and distinct from the other service
# containers' forwarders all the same.
forwarderTelemetryPort = 8895;
# The runner's store identity, handed into the container as system
# credentials (credential id → host source) for the forwarder's fetch unit.
storeFiles = lib.filterAttrs (_: source: source != null) {
ci-runner-bao-cert = baoDeploy.ciRunnerClientCertFile;
ci-runner-bao-key = baoDeploy.ciRunnerClientKeyFile;
ci-runner-bao-server-ca = baoDeploy.serverCaFile;
};
# nixos-container expands `extraFlags` unquoted, so a source path must be
# one word.
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
# The upstream unit is `DynamicUser`, so its state dir is
# `/var/lib/private/gitea-runner`; `/var/lib/gitea-runner` is a symlink to
# it, which is the spelling a step run as root (`+`) can follow.
@ -99,6 +130,21 @@ let
privateNetwork = true;
in
{
options.services.hyperhive.swarm.ci-runner.otel.clientId = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-ci-runner-collector";
description = ''
OAuth2 client id the journal forwarder inside the CI runner's container
authenticates as, and the audience it asks its token for.
Read-only because three hosts have to agree on it: authelia registers
the client (`glue-ci-runner-otel-oidc-client.nix`), the swarm collector
admits it on the container's receiver (`swarm.otel.serviceProducers`),
and the secret publisher carries its secret to the store.
'';
};
options.services.hyperhive.deploy.ci-runner = {
# `enable` is in ./deploy.nix.
@ -178,6 +224,14 @@ in
options.
'';
}
{
assertion = lib.all oneWord (lib.attrValues storeFiles);
message = ''
The CI runner's store identity paths must not contain whitespace:
nspawn's --load-credential receives them through nixos-container's
unquoted extra flags.
'';
}
];
# The container hangs off the bridge, takes its address from the
@ -274,7 +328,16 @@ in
systemd.timers.swarm-bao-ci-runner = storeRefreshTimer "re-read the CI runner's registration token from the swarm secret store";
systemd.services."container@${machine}" = caTrust.containerOrdering;
# nspawn fails to start the container when a credential source is
# missing, so where this host mints the leaf the container pulls the
# minting unit in and waits for it, as ./swarm-builder.nix's does.
systemd.services."container@${machine}" = lib.mkMerge [
caTrust.containerOrdering
{
after = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];
}
];
systemd.tmpfiles.rules = [
# The bind source must exist when the container starts.
@ -287,9 +350,9 @@ in
containers.${machine} = {
autoStart = true;
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# No `--link-journal=host`: the journal stays in the guest, where the
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
inherit privateNetwork;
hostBridge = networkCfg.bridgeName;
@ -338,9 +401,35 @@ in
{
imports = [
../container-modules/swarm-container.nix
../container-modules/swarm-journal-forwarder.nix
caBundleModule
];
# Authenticated as the forwarder's own OIDC client, with the runner's
# store identity: one identity for the whole container. It is a
# system credential, so the fetch unit loads it.
services.hyperhive.swarmJournalForwarder = {
enable = true;
serviceName = "ci-runner";
endpoint = forwarderEndpoint;
clientId = forwarderClientId;
tokenUrl = forwarderTokenUrl;
telemetryPort = forwarderTelemetryPort;
store = {
package = baoDeploy.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
clientCertFile = forwarderCredential "ci-runner-bao-cert";
clientKeyFile = forwarderCredential "ci-runner-bao-key";
caFile =
if storeFiles ? ci-runner-bao-server-ca then
forwarderCredential "ci-runner-bao-server-ca"
else
null;
};
};
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
# Its own netns, so it keeps its own firewall; resolvconf stays on
# and serves `networking.nameservers` below.
services.hyperhive.swarmContainer = {

18
nix/host-modules/swarm-grafana-service.nix
View file

@ -37,6 +37,24 @@ in
'';
};
otel.clientId = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-grafana-collector";
description = ''
OAuth2 client id the journal forwarder inside Grafana's container
authenticates as, and the audience it asks its token for. Its own,
not {option}`services.hyperhive.swarm.grafana.oidc.clientId`, which
is the browser login's.
Read-only because three hosts have to agree on it: authelia
registers the client (`glue-grafana-otel-oidc-client.nix`), the
swarm collector admits it on the container's receiver
(`swarm.otel.serviceProducers`), and the secret publisher carries
its secret to the store.
'';
};
domain = lib.mkOption {
type = lib.types.str;
default = "grafana.${domainBase}";

116
nix/host-modules/swarm-grafana.nix
View file

@ -229,6 +229,33 @@ let
autheliaUrl = toString hyperhiveCfg.swarm.authelia.url;
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.swarm-grafana` in
# ./swarm-otel-service.nix declares, as a client of its own rather than the
# browser login's. The port suffix is elided on 443, as in ./swarm-bao.nix.
forwarderClientId = cfg.otel.clientId;
gatewayPortSuffix = lib.optionalString (
gatewayCfg.httpsPort != 443
) ":${toString gatewayCfg.httpsPort}";
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}${gatewayPortSuffix}/${cfg.machine}";
forwarderUnit = "swarm-journal-forwarder-oidc";
# In the host netns: distinct from every other collector's there, which the
# assertion below checks.
forwarderTelemetryPort = 8896;
# Grafana's store identity, handed into the container as system credentials
# (credential id → host source) for the forwarder's fetch unit. The host units
# below keep reading the same files directly.
storeFiles = lib.filterAttrs (_: source: source != null) {
grafana-bao-cert = baoDeploy.grafanaOidcClientCertFile;
grafana-bao-key = baoDeploy.grafanaOidcClientKeyFile;
grafana-bao-server-ca = baoDeploy.serverCaFile;
};
# nixos-container expands `extraFlags` unquoted, so a source path must be
# one word.
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
# The three secrets below land on the host's /run (tmpfs), which the
# container bind-mounts read-only at the same path: none reaches persistent
# disk, and a reboot fetches them again. Each file is absent until its
@ -493,6 +520,39 @@ in
from that CA out of band and named here — see docs/swarm/secrets.md.
'';
}
{
assertion = lib.all oneWord (lib.attrValues storeFiles);
message = ''
Grafana's store identity paths must not contain whitespace: nspawn's
--load-credential receives them through nixos-container's unquoted
extra flags.
'';
}
{
assertion =
(hyperhiveCfg.swarm.otel.serviceProducers.${cfg.machine}.clientId or null) == forwarderClientId;
message = ''
services.hyperhive.swarm.otel.serviceProducers has no entry
"${cfg.machine}" with clientId "${forwarderClientId}", so the swarm
collector has no receiver for Grafana's journal forwarder. Declare
it in swarm-otel-service.nix.
'';
}
{
assertion =
!(lib.elem forwarderTelemetryPort [
hyperhiveCfg.otel.telemetryPort
hyperhiveCfg.swarm.otel.telemetryPort
hyperhiveCfg.swarm.bao.otel.telemetryPort
hyperhiveCfg.swarm.nats.otel.telemetryPort
]);
message = ''
Grafana's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
collides with another collector's in the host netns: the hive
collector's, the swarm collector's, the store forwarder's or the
queue forwarder's.
'';
}
];
# Websockets: Grafana Live streams panel updates over one, and without
@ -555,8 +615,17 @@ in
};
# Order the container after the host CA service so the bind source below
# exists before nspawn sets the mount up.
systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;
# exists before nspawn sets the mount up. nspawn also fails to start it
# when a credential source is missing, so where this host mints the leaf
# the container pulls the minting unit in and waits for it, as
# ./swarm-builder.nix's does.
systemd.services."container@${cfg.machine}" = lib.mkMerge [
caTrust.containerOrdering
{
after = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];
}
];
# The socket directory, created host-side before the container starts.
# nixos-container refuses to start when a bind source is missing, so this
@ -1036,9 +1105,9 @@ in
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# No `--link-journal=host`: the journal stays in the guest, where the
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
inherit privateNetwork;
# The socket directory, shared with the host so nginx can reach in.
@ -1068,9 +1137,16 @@ in
{
imports = [
../container-modules/swarm-container.nix
../container-modules/swarm-journal-forwarder.nix
(import ../container-modules/swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
dnsConsumers = [ "grafana.service" ];
# The forwarder and its secret fetch dial the swarm collector,
# authelia and the store by name too.
dnsConsumers = [
"grafana.service"
"opentelemetry-collector.service"
"${forwarderUnit}.service"
];
})
# Grafana is Go, and `SSL_CERT_FILE` *replaces* the default store
# rather than adding to it — so it needs the system CAs and the
@ -1080,12 +1156,38 @@ in
(caTrust.trustBundle {
inherit pkgs;
name = cfg.machine;
consumers = [ "grafana" ];
consumers = [
"grafana"
# Go too, for the swarm collector's and authelia's names.
"opentelemetry-collector"
];
})
];
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# Authenticated as the forwarder's own OIDC client, with Grafana's
# store identity: one identity for the whole container. It is a
# system credential, so the fetch unit loads it.
services.hyperhive.swarmJournalForwarder = {
enable = true;
serviceName = "grafana";
endpoint = forwarderEndpoint;
clientId = forwarderClientId;
tokenUrl = "${autheliaUrl}/api/oidc/token";
telemetryPort = forwarderTelemetryPort;
store = {
package = baoDeploy.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
clientCertFile = forwarderCredential "grafana-bao-cert";
clientKeyFile = forwarderCredential "grafana-bao-key";
caFile =
if storeFiles ? grafana-bao-server-ca then forwarderCredential "grafana-bao-server-ca" else null;
};
};
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
# Grafana's `secret_key` has **no default in nixpkgs** and an
# assertion refuses the build without one — which is how the first
# version of this module broke a deploy. It signs the datasource

14
nix/host-modules/swarm-otel-service.nix
View file

@ -415,6 +415,20 @@ in
clientId = hyperhiveCfg.swarm.review-classifier.clientId;
port = lib.mkDefault 4395;
};
# The forwarders in ./swarm-ci-runner.nix, ./swarm-grafana.nix and
# ./swarm-victorialogs.nix, each authenticated as a client of its own.
swarm-ci-runner = {
clientId = hyperhiveCfg.swarm.ci-runner.otel.clientId;
port = lib.mkDefault 4396;
};
swarm-grafana = {
clientId = hyperhiveCfg.swarm.grafana.otel.clientId;
port = lib.mkDefault 4397;
};
swarm-victorialogs = {
clientId = hyperhiveCfg.swarm.victorialogs.otel.clientId;
port = lib.mkDefault 4398;
};
};
# Not under any `enable`: see `publishedScrapeTargets` for why every host

3
nix/host-modules/swarm-otel.nix
View file

@ -214,7 +214,8 @@ let
# ⚠️ Drop that flag from one of THOSE containers and this receiver silently
# stops seeing it — no error, just a unit that never appears in the store.
# The `serviceProducers` containers (`swarm-bao`, `swarm-nats`,
# `swarm-builder`, `swarm-claude-login`, `swarm-review-classifier`) are the
# `swarm-builder`, `swarm-claude-login`, `swarm-review-classifier`,
# `swarm-ci-runner`, `swarm-grafana`, `swarm-victorialogs`) are the
# exceptions on purpose: their journals stay inside the container for its own
# collector instead, so this receiver never sees their units.
hostJournalDir = "/var/log/journal";

5
nix/host-modules/swarm-secret-publisher.nix
View file

@ -98,6 +98,11 @@ let
hyperhiveCfg.swarm.builder.clientId
# The claude login minter, which reads it itself inside its container.
hyperhiveCfg.swarm.claude-login.clientId
# The journal forwarders in the CI runner's, Grafana's and the log store's
# containers, each reading it inside its container.
hyperhiveCfg.swarm.ci-runner.otel.clientId
hyperhiveCfg.swarm.grafana.otel.clientId
hyperhiveCfg.swarm.victorialogs.otel.clientId
];
# The swarm controller's OIDC client, which it reads back from the store

16
nix/host-modules/swarm-victorialogs-service.nix
View file

@ -31,6 +31,22 @@ in
'';
};
otel.clientId = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-victorialogs-collector";
description = ''
OAuth2 client id the journal forwarder inside the log store's
container authenticates as, and the audience it asks its token for.
Read-only because three hosts have to agree on it: authelia
registers the client (`glue-victorialogs-otel-oidc-client.nix`),
the swarm collector admits it on the container's receiver
(`swarm.otel.serviceProducers`), and the secret publisher carries
its secret to the store.
'';
};
domain = lib.mkOption {
type = lib.types.str;
default = "logs.${domainBase}";

120
nix/host-modules/swarm-victorialogs.nix
View file

@ -80,6 +80,35 @@ let
# Shared host netns, like every sibling swarm container: Grafana reaches
# this at 127.0.0.1:<port>.
privateNetwork = false;
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.swarm-victorialogs` in
# ./swarm-otel-service.nix declares. The port suffix is elided on 443, as in
# ./swarm-bao.nix.
forwarderClientId = cfg.otel.clientId;
gatewayPortSuffix = lib.optionalString (
gatewayCfg.httpsPort != 443
) ":${toString gatewayCfg.httpsPort}";
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}${gatewayPortSuffix}/${cfg.machine}";
forwarderUnit = "swarm-journal-forwarder-oidc";
# In the host netns: distinct from every other collector's there, which the
# assertion below checks.
forwarderTelemetryPort = 8897;
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
# The responder's store identity, handed into the container as system
# credentials for the forwarder's fetch unit. The host units keep reading
# the same files.
forwarderStoreFiles = lib.filterAttrs (_: source: source != null) logsAuthStoreFiles;
# nixos-container expands `extraFlags` unquoted, so a source path must be
# one word.
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
# The forwarder reaches the swarm collector and authelia by name over TLS; in
# self-signed mode their leaves chain to the hive CA.
caTrust = import ./lib/hive-ca-trust.nix {
inherit lib gatewayCfg;
tlsCfg = deployCfg.hive-controller.tls;
};
in
{
# What the store IS from any hive's point of view, the name it answers on and
@ -211,6 +240,51 @@ in
both options.
'';
}
{
assertion = lib.all oneWord (lib.attrValues forwarderStoreFiles);
message = ''
The log store responder's store identity paths must not contain
whitespace: nspawn's --load-credential receives them through
nixos-container's unquoted extra flags.
'';
}
{
assertion =
(hyperhiveCfg.swarm.otel.serviceProducers.${cfg.machine}.clientId or null) == forwarderClientId;
message = ''
services.hyperhive.swarm.otel.serviceProducers has no entry
"${cfg.machine}" with clientId "${forwarderClientId}", so the swarm
collector has no receiver for the log store's journal forwarder.
Declare it in swarm-otel-service.nix.
'';
}
{
assertion =
!(lib.elem forwarderTelemetryPort [
hyperhiveCfg.otel.telemetryPort
hyperhiveCfg.swarm.otel.telemetryPort
hyperhiveCfg.swarm.bao.otel.telemetryPort
hyperhiveCfg.swarm.nats.otel.telemetryPort
]);
message = ''
The log store's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
collides with another collector's in the host netns: the hive
collector's, the swarm collector's, the store forwarder's or the
queue forwarder's.
'';
}
];
# nspawn fails to start the container when a credential source is
# missing, so where this host mints the leaf the container pulls the
# minting unit in and waits for it, as ./swarm-builder.nix's does. The CA
# ordering keeps the trust bundle's bind source there first.
systemd.services."container@${cfg.machine}" = lib.mkMerge [
caTrust.containerOrdering
{
after = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];
}
];
# Which check a `/select/logsql/` request gets. `auth_request` takes a
@ -506,24 +580,62 @@ in
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# No `--link-journal=host`: the journal stays in the guest, where the
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
extraFlags = lib.mapAttrsToList (
id: source: "--load-credential=${id}:${source}"
) forwarderStoreFiles;
inherit privateNetwork;
bindMounts = caTrust.bindMount;
config =
{ ... }:
{
imports = [
../container-modules/swarm-container.nix
../container-modules/swarm-journal-forwarder.nix
(import ../container-modules/swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
dnsConsumers = [ "victorialogs.service" ];
# The forwarder and its secret fetch dial the swarm collector,
# authelia and the store by name too.
dnsConsumers = [
"victorialogs.service"
"opentelemetry-collector.service"
"${forwarderUnit}.service"
];
})
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
# collector's and authelia's names.
(caTrust.trustBundle {
inherit pkgs;
name = cfg.machine;
consumers = [ "opentelemetry-collector" ];
})
];
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# Authenticated as the forwarder's own OIDC client, with the
# responder's store identity: one identity for the whole container.
# It is a system credential, so the fetch unit loads it.
services.hyperhive.swarmJournalForwarder = {
enable = true;
serviceName = "victorialogs";
endpoint = forwarderEndpoint;
clientId = forwarderClientId;
tokenUrl = "${toString hyperhiveCfg.swarm.authelia.url}/api/oidc/token";
telemetryPort = forwarderTelemetryPort;
store = {
package = deployCfg.bao.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
clientCertFile = forwarderCredential "bao-client.pem";
clientKeyFile = forwarderCredential "bao-client-key.pem";
caFile = if forwarderStoreFiles ? "bao-ca.pem" then forwarderCredential "bao-ca.pem" else null;
};
};
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames forwarderStoreFiles;
services.victorialogs = {
enable = true;
package = deployCfg.victorialogs.package;