vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

hive-forge, hive-matrix: forward each container's journal to the swarm collector

Browse source 
Each container now runs the journal forwarder the CI runner's, Grafana's and
the log store's containers run, pushing to a receiver of its own on the swarm
collector (serviceProducers hive-forge and hive-matrix). Each authenticates as
a new machine OIDC client of its own (swarm.forge.otel.clientId,
swarm.matrix.otel.clientId), registered by a glue file wherever authelia runs,
minted by authelia and published to the store by the secret publisher. The
forge's and the homeserver's browser login clients are unchanged. The client
ids are swarm-forge-collector and swarm-matrix-collector, not
hive-<x>-collector, which is the client id a hive named <x>-collector gets.

The fetch unit inside each container reads the client secret with the store
identity the host already holds for that service. The forge publisher's leaf
is handed into the forge container as an nspawn system credential, as the
grafana and log store containers do. matrix-ctl's leaf is already bound into
the homeserver's container, so its forwarder reads it at those paths, as
swarm-matrix-appservice-publish does; with no matrix-ctl identity the
homeserver's forwarder is off. Each of the two policies gains one read stanza
on its forwarder's client path. The host units that use those identities are
unchanged.

The two containers drop --link-journal=host, and the host collector stops
reading them.
This commit is contained in:
atlas 2026-10-16 11:48:22 +02:00 • committed by mara
commit ce44b6e77e
18 changed files with 511 additions and 134 deletions
Download patch file Download diff file Expand all files Collapse all files

2
nix/host-modules/default.nix
View file

@ -49,7 +49,9 @@
./glue-builder-oidc-client.nix
./glue-ci-runner-otel-oidc-client.nix
./glue-claude-login-oidc-client.nix
./glue-forge-otel-oidc-client.nix
./glue-grafana-otel-oidc-client.nix
./glue-matrix-otel-oidc-client.nix
./glue-review-classifier-oidc-client.nix
./glue-secret-publisher-authelia.nix
./glue-secret-publisher-bao-identity.nix

37
nix/host-modules/glue-forge-otel-oidc-client.nix Normal file
View file

@ -0,0 +1,37 @@
# Glue: register the forge container's journal forwarder as an OIDC
# client wherever authelia runs.
#
# ONE PAIRING PER FILE: that forwarder ← authelia, and nothing else.
# Without it authelia mints no secret, the publisher has nothing to copy into
# the store, and the forwarder's export is a 401.
#
# Gated on authelia being HERE, not on this host running the forge: a
# client is a row in THIS host's provider config, the split
# ./glue-swarm-bao-otel-oidc-client.nix makes for the same reason.
{
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
clientId = hyperhiveCfg.swarm.forge.otel.clientId;
in
{
config = lib.mkIf hyperhiveCfg.deploy.authelia.enable {
services.hyperhive.swarm.authelia.oidc.clients = [
{
id = clientId;
description = "HyperHive forge journal forwarder";
kind = "machine";
redirectUris = [ ];
# Its own id as its own permitted audience, the one the swarm
# collector's receiver for this container checks.
audience = [ clientId ];
# The receiver verifies the token offline against `/jwks.json`;
# authelia's default opaque handle fails there.
accessTokenSignedResponseAlg = "RS256";
}
];
};
}

37
nix/host-modules/glue-matrix-otel-oidc-client.nix Normal file
View file

@ -0,0 +1,37 @@
# Glue: register the homeserver container's journal forwarder as an OIDC
# client wherever authelia runs.
#
# ONE PAIRING PER FILE: that forwarder ← authelia, and nothing else.
# Without it authelia mints no secret, the publisher has nothing to copy into
# the store, and the forwarder's export is a 401.
#
# Gated on authelia being HERE, not on this host running the homeserver: a
# client is a row in THIS host's provider config, the split
# ./glue-swarm-bao-otel-oidc-client.nix makes for the same reason.
{
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
clientId = hyperhiveCfg.swarm.matrix.otel.clientId;
in
{
config = lib.mkIf hyperhiveCfg.deploy.authelia.enable {
services.hyperhive.swarm.authelia.oidc.clients = [
{
id = clientId;
description = "HyperHive homeserver journal forwarder";
kind = "machine";
redirectUris = [ ];
# Its own id as its own permitted audience, the one the swarm
# collector's receiver for this container checks.
audience = [ clientId ];
# The receiver verifies the token offline against `/jwks.json`;
# authelia's default opaque handle fails there.
accessTokenSignedResponseAlg = "RS256";
}
];
};
}

121
nix/host-modules/hive-forge/default.nix
View file

@ -41,7 +41,7 @@ let
oldOidcSecretPath = "/var/lib/nixos-containers/hive-forge/var/lib/forgejo-oidc/${cfg.sso.clientId}.secret";
# Where swarm-secret-publish on authelia's host leaves this client's secret.
# The forge publisher's store role grants read on this one path.
# The forge publisher's store role grants read on this path.
oidcStorePath = "secret/swarm/services/${cfg.sso.clientId}/oidc/client";
# The swarm-controller's forge account. Same name as
@ -78,6 +78,33 @@ let
caTrust = import ../lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.hive-forge` in
# ../swarm-otel-service.nix declares, as a client of its own rather than the
# login source's. The port suffix is elided on 443, as in ../swarm-bao.nix.
forwarderClientId = cfg.otel.clientId;
gatewayPortSuffix = lib.optionalString (
gatewayCfg.httpsPort != 443
) ":${toString gatewayCfg.httpsPort}";
forwarderEndpoint = "https://${config.services.hyperhive.swarm.otel.domain}${gatewayPortSuffix}/hive-forge";
forwarderUnit = "swarm-journal-forwarder-oidc";
# In the host netns: distinct from every other collector's there, which the
# assertion below checks.
forwarderTelemetryPort = 8898;
# The forge host's store identity, handed into the container as system
# credentials (credential id → host source) for the forwarder's fetch unit.
# The host units below keep reading the same files directly.
storeFiles = lib.filterAttrs (_: source: source != null) {
forge-bao-cert = deployCfg.bao.forgePublisherClientCertFile;
forge-bao-key = deployCfg.bao.forgePublisherClientKeyFile;
forge-bao-server-ca = deployCfg.bao.serverCaFile;
};
# nixos-container expands `extraFlags` unquoted, so a source path must be
# one word.
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
# ROOT_URL forgejo advertises in clone links + outbound URLs.
# `cfg.domain` doubles as both the forgejo `DOMAIN` setting AND the
# gateway vhost server-name, so ROOT_URL just uses it directly. The
@ -403,19 +430,62 @@ in
collide. Use a dedicated org (e.g. "actions/checkout").
'';
}
{
assertion = lib.all oneWord (lib.attrValues storeFiles);
message = ''
The forge host's store identity paths must not contain whitespace:
nspawn's --load-credential receives them through nixos-container's
unquoted extra flags.
'';
}
{
assertion =
(config.services.hyperhive.swarm.otel.serviceProducers.hive-forge.clientId or null)
== forwarderClientId;
message = ''
services.hyperhive.swarm.otel.serviceProducers has no entry
"hive-forge" with clientId "${forwarderClientId}", so the swarm
collector has no receiver for the forge's journal forwarder.
Declare it in swarm-otel-service.nix.
'';
}
{
assertion =
!(lib.elem forwarderTelemetryPort [
config.services.hyperhive.otel.telemetryPort
config.services.hyperhive.swarm.otel.telemetryPort
config.services.hyperhive.swarm.bao.otel.telemetryPort
config.services.hyperhive.swarm.nats.otel.telemetryPort
]);
message = ''
The forge's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
collides with another collector's in the host netns: the hive
collector's, the swarm collector's, the store forwarder's or the
queue forwarder's.
'';
}
];
# `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service`
# in self-signed mode (see the hive-ca-trust helper), so the CA bind
# source exists before nspawn sets the mount up.
systemd.services."container@hive-forge" = caTrust.containerOrdering;
# source exists before nspawn sets the mount up. nspawn also fails to
# start it when a credential source is missing, so where this host mints
# the leaf the container pulls the minting unit in and waits for it, as
# ../swarm-builder.nix's does.
systemd.services."container@hive-forge" = lib.mkMerge [
caTrust.containerOrdering
{
after = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];
}
];
containers.hive-forge = {
autoStart = true;
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# No `--link-journal=host`: the journal stays in the guest, where the
# container's own forwarder reads it (../../container-modules/swarm-journal-forwarder.nix).
extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
inherit privateNetwork;
# Self-signed mode: bind the hive's trust bundle read-only so forgejo
# can trust the gateway's self-signed leaf for outbound webhook
@ -458,6 +528,7 @@ in
{
imports = [
./../../container-modules/swarm-container.nix
./../../container-modules/swarm-journal-forwarder.nix
# Ask the hive's dnsmasq, not the host's resolvers. The swarm
# names — the controller's webhook endpoint, the SSO issuer —
# resolve to the bridge and have no public records, so a
@ -465,14 +536,18 @@ in
# every one of them while the same name resolves fine one
# container over.
#
# Both units that make an outbound call are consumers, for the
# same reason the trust bundle below names both: an unordered
# resolver write is a race that only shows up on a cold boot.
# Every unit that makes an outbound call is a consumer, for the
# same reason the trust bundle below names them: an unordered
# resolver write is a race that only shows up on a cold boot. The
# forwarder and its secret fetch dial the swarm collector, authelia
# and the store by name.
(import ./../../container-modules/swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
dnsConsumers = [
"forgejo.service"
"forgejo-sso-source.service"
"opentelemetry-collector.service"
"${forwarderUnit}.service"
];
})
@ -481,16 +556,18 @@ in
# store's root concatenated, not the root alone, or public mirror
# fetches lose every anchor they had.
#
# BOTH units that make an outbound HTTPS call are consumers, not
# EVERY unit that makes an outbound HTTPS call is a consumer, not
# just the obvious one: `forgejo-sso-source` fetches the issuer's
# `.well-known/openid-configuration` under the store's root, and it
# once shipped without the trust its sibling had.
# once shipped without the trust its sibling had. The collector is
# Go too, for the swarm collector's and authelia's names.
(caTrust.trustBundle {
inherit pkgs;
name = "hive-forge";
consumers = [
"forgejo"
"forgejo-sso-source"
"opentelemetry-collector"
];
})
];
@ -498,6 +575,28 @@ in
system.stateVersion = "25.11";
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# Authenticated as the forwarder's own OIDC client, with the forge
# host's store identity: one identity for the whole container. It is
# a system credential, so the fetch unit loads it.
services.hyperhive.swarmJournalForwarder = {
enable = true;
serviceName = "forgejo";
endpoint = forwarderEndpoint;
clientId = forwarderClientId;
tokenUrl = "${toString autheliaUrl}/api/oidc/token";
telemetryPort = forwarderTelemetryPort;
store = {
package = deployCfg.bao.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
clientCertFile = forwarderCredential "forge-bao-cert";
clientKeyFile = forwarderCredential "forge-bao-key";
caFile =
if storeFiles ? forge-bao-server-ca then forwarderCredential "forge-bao-server-ca" else null;
};
};
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
services.forgejo = {
enable = true;
package = deployCfg.forgejo.package;

20
nix/host-modules/hive-forge/service.nix
View file

@ -191,5 +191,25 @@ in
# The secret half is fetched from the swarm secret store on the host
# that runs the forge: `swarm-bao-forge-oidc`, in ./default.nix.
};
otel.clientId = lib.mkOption {
type = lib.types.str;
readOnly = true;
# Not `hive-forge-collector`: `hive-<name>` is a hive's own client id,
# and `forge-collector` is a name a hive may take.
default = "swarm-forge-collector";
description = ''
OAuth2 client id the journal forwarder inside the forge's container
authenticates as, and the audience it asks its token for. Its own,
not {option}`services.hyperhive.swarm.forge.sso.clientId`, which is
the browser login's.
Read-only because three hosts have to agree on it: authelia
registers the client (`glue-forge-otel-oidc-client.nix`), the
swarm collector admits it on the container's receiver
(`swarm.otel.serviceProducers`), and the secret publisher carries
its secret to the store.
'';
};
};
}

20
nix/host-modules/hive-matrix-service.nix
View file

@ -204,5 +204,25 @@ in
'';
};
};
otel.clientId = lib.mkOption {
type = lib.types.str;
readOnly = true;
# Not `hive-matrix-collector`: `hive-<name>` is a hive's own client id,
# and `matrix-collector` is a name a hive may take.
default = "swarm-matrix-collector";
description = ''
OAuth2 client id the journal forwarder inside the homeserver's
container authenticates as, and the audience it asks its token for.
Its own, not {option}`services.hyperhive.swarm.matrix.sso.clientId`,
which is the browser login's.
Read-only because three hosts have to agree on it: authelia
registers the client (`glue-matrix-otel-oidc-client.nix`), the
swarm collector admits it on the container's receiver
(`swarm.otel.serviceProducers`), and the secret publisher carries
its secret to the store.
'';
};
};
}

62
nix/host-modules/hive-matrix.nix
View file

@ -49,7 +49,7 @@ let
legacyHostSecretPath = "/var/lib/nixos-containers/hive-matrix/var/lib/tuwunel-oidc/${cfg.sso.clientId}.secret";
# Where swarm-secret-publish on authelia's host leaves this client's secret.
# matrix-ctl's store role grants read on this one path.
# matrix-ctl's store role grants read on this path.
storeSecretPath = "secret/swarm/services/${cfg.sso.clientId}/oidc/client";
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
@ -119,6 +119,21 @@ let
isReadOnly = true;
});
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.hive-matrix` in
# ./swarm-otel-service.nix declares, as a client of its own rather than the
# homeserver login's. The port suffix is elided on 443, as in
# ./swarm-bao.nix. It reads its client secret with matrix-ctl's identity,
# bound in above: one identity for the whole container.
forwarderClientId = cfg.otel.clientId;
gatewayPortSuffix = lib.optionalString (
gatewayCfg.httpsPort != 443
) ":${toString gatewayCfg.httpsPort}";
forwarderEndpoint = "https://${config.services.hyperhive.swarm.otel.domain}${gatewayPortSuffix}/hive-matrix";
# Loopback in the container's own netns, and distinct from the other service
# containers' forwarders all the same.
forwarderTelemetryPort = 8899;
# ── the swarm's own appservice ──────────────────────────────────────
#
# The homeserver's one appservice registration, and the swarm's identity on
@ -584,6 +599,17 @@ in
`gatewayHost` is. With it null the homeserver has no such name.
'';
}
{
assertion =
(config.services.hyperhive.swarm.otel.serviceProducers.hive-matrix.clientId or null)
== forwarderClientId;
message = ''
services.hyperhive.swarm.otel.serviceProducers has no entry
"hive-matrix" with clientId "${forwarderClientId}", so the swarm
collector has no receiver for the homeserver's journal forwarder.
Declare it in swarm-otel-service.nix.
'';
}
];
# Where the homeserver finds its OIDC client secret, inside the container.
@ -777,9 +803,8 @@ in
containers.hive-matrix = {
autoStart = true;
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# No `--link-journal=host`: the journal stays in the guest, where the
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
inherit privateNetwork;
hostBridge = networkCfg.bridgeName;
bindMounts = {
@ -804,6 +829,7 @@ in
{
imports = [
../container-modules/swarm-container.nix
../container-modules/swarm-journal-forwarder.nix
# tuwunel's rustls verifier resolves through `rustls-native-certs`
# → `openssl-probe`, which reads `SSL_CERT_FILE` — so the
# openssl-shaped variable is the lever despite tuwunel linking no
@ -820,10 +846,36 @@ in
(caTrust.trustBundle {
inherit pkgs;
name = "hive-matrix";
consumers = [ "tuwunel" ];
consumers = [
"tuwunel"
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
# collector's and authelia's names.
"opentelemetry-collector"
];
})
];
# Authenticated as the forwarder's own OIDC client, with matrix-ctl's
# store identity at the paths bound in from the host. With no such
# identity there is nothing to fetch the secret with, as for the
# appservice units below.
services.hyperhive.swarmJournalForwarder = lib.mkIf ctlActive {
enable = true;
serviceName = "tuwunel";
endpoint = forwarderEndpoint;
clientId = forwarderClientId;
tokenUrl = "${toString autheliaUrl}/api/oidc/token";
telemetryPort = forwarderTelemetryPort;
store = {
package = deployCfg.bao.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
clientCertFile = deployCfg.matrix.ctlBaoClientCertFile;
clientKeyFile = deployCfg.matrix.ctlBaoClientKeyFile;
caFile = deployCfg.bao.serverCaFile;
};
};
# Its own netns, so it keeps its own firewall; resolvconf stays on
# and serves `networking.nameservers` below.
services.hyperhive.swarmContainer = {

33
nix/host-modules/swarm-bao.nix
View file

@ -547,7 +547,7 @@ let
matrixCtlPolicyName = "swarm-matrix-ctl";
matrixCtlCn = baoDeploy.matrixCtlCommonName;
# Two paths, each spelled to the leaf, and every narrowing in them is
# Three paths, each spelled to the leaf, and every narrowing in them is
# load-bearing.
#
# `secret/data/` is KV v2's ACL prefix, inserted by the engine rather than
@ -563,12 +563,15 @@ let
#
# The second is the homeserver's own OIDC client secret, read-only:
# `swarm-bao-matrix-oidc` on the homeserver's host reads it under this leaf.
# The third, also read-only, is the client secret of the journal forwarder in
# the homeserver's container.
matrixCtlPolicyText = ''
path "${credentialMountPath}/data/${swarmAppserviceTokenLeaf}" {
capabilities = ["create", "update", "read"]
}
''
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.matrix.sso.clientId}/oidc/client";
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.matrix.sso.clientId}/oidc/client"
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.matrix.otel.clientId}/oidc/client";
# The swarm appservice token's leaf, the nix half of
# `swarm_secret_client::matrix::swarm_appservice_token_path`. Under
@ -1103,8 +1106,9 @@ let
# The forge host's one store identity. `swarm-bao-forge-publisher` writes
# the controller's forge token under it, the one writer of that path and
# write-only: it copies a file in and never reads one back.
# `swarm-bao-forge-oidc` reads the forge's own OIDC client secret, that one
# leaf and no other service's.
# `swarm-bao-forge-oidc` reads the forge's own OIDC client secret, and the
# journal forwarder in its container its own, those two leaves and no other
# service's.
forgePublisherWriters = [
{
name = "swarm-forge-publisher";
@ -1114,7 +1118,9 @@ let
capabilities = ["create", "update"]
}
''
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.forge.sso.clientId}/oidc/client";
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.forge.sso.clientId}/oidc/client"
# The journal forwarder in the forge's container, under this same leaf.
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.forge.otel.clientId}/oidc/client";
}
];
@ -1754,9 +1760,11 @@ in
identity the matrix container's `swarm-matrix-appservice-publish`
unit presents when it publishes the swarm appservice's token.
A **third** identity rather than reuse of either sibling above, and
the narrowest of the three: its grant is one path, that
credential itself. The point of the whole arrangement is that the
A **third** identity rather than reuse of either sibling above. Its
grant is that credential itself, plus read on two OIDC client
secrets: the homeserver's, fetched by `swarm-bao-matrix-oidc`, and
that of the journal forwarder in the homeserver's container. The
point of the whole arrangement is that the
process holding the appservice token is not the hive, so handing it
the hive's own leaf — which reads every agent credential in the swarm
and this hive's own secrets — would give the separation away in one
@ -2003,10 +2011,11 @@ in
Subject the store's `swarm-forge-publisher` cert-auth role accepts: the
forge host's one store identity, presented by `swarm-bao-forge-publisher`,
which publishes the swarm-controller's forge token, and by
`swarm-bao-forge-oidc`, which fetches the forge's OIDC client secret.
Its grant is two paths: `swarm/controller/swarm-controller/forge/token`,
write only, and `swarm/services/<forge client id>/oidc/client`, read
only.
`swarm-bao-forge-oidc`, which fetches the forge's OIDC client secret,
and by the journal forwarder in the forge's container. Its grant is
three paths: `swarm/controller/swarm-controller/forge/token`, write
only, and `swarm/services/<forge client id>/oidc/client` and
`swarm/services/<forge forwarder client id>/oidc/client`, read only.
⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
'';

10
nix/host-modules/swarm-otel-service.nix
View file

@ -429,6 +429,16 @@ in
clientId = hyperhiveCfg.swarm.victorialogs.otel.clientId;
port = lib.mkDefault 4398;
};
# The forwarders in ./hive-forge/default.nix and ./hive-matrix.nix, each
# authenticated as a client of its own.
hive-forge = {
clientId = hyperhiveCfg.swarm.forge.otel.clientId;
port = lib.mkDefault 4399;
};
hive-matrix = {
clientId = hyperhiveCfg.swarm.matrix.otel.clientId;
port = lib.mkDefault 4400;
};
};
# Not under any `enable`: see `publishedScrapeTargets` for why every host

11
nix/host-modules/swarm-otel.nix
View file

@ -50,10 +50,10 @@ let
#
# Named after the service's container, so `otlp/swarm-bao` reads as the
# thing that pushes into it. No `reserved-names.nix` entry is needed the way
# `swarm` has one: each name CONTAINS `swarm`, which
# `swarm` has one: each name CONTAINS `swarm` or `hive`, which
# `nix/reserved-hive-fragments.nix` forbids as a substring of any hive name,
# so no hive can ever own these components. The assertion below checks that
# fragment is still listed rather than trusting it.
# a fragment is still listed rather than trusting it.
serviceProducers = cfg.serviceProducers;
serviceProducerNames = lib.attrNames serviceProducers;
reservedHiveFragments = import ../reserved-hive-fragments.nix;
@ -215,9 +215,10 @@ let
# stops seeing it — no error, just a unit that never appears in the store.
# The `serviceProducers` containers (`swarm-bao`, `swarm-nats`,
# `swarm-builder`, `swarm-claude-login`, `swarm-review-classifier`,
# `swarm-ci-runner`, `swarm-grafana`, `swarm-victorialogs`) are the
# exceptions on purpose: their journals stay inside the container for its own
# collector instead, so this receiver never sees their units.
# `swarm-ci-runner`, `swarm-grafana`, `swarm-victorialogs`, `hive-forge`,
# `hive-matrix`) are the exceptions on purpose: their journals stay inside
# the container for its own collector instead, so this receiver never sees
# their units.
hostJournalDir = "/var/log/journal";
# Each store's OTLP route, by domain. One binding because the same string is

7
nix/host-modules/swarm-secret-publisher.nix
View file

@ -98,11 +98,14 @@ let
hyperhiveCfg.swarm.builder.clientId
# The claude login minter, which reads it itself inside its container.
hyperhiveCfg.swarm.claude-login.clientId
# The journal forwarders in the CI runner's, Grafana's and the log store's
# containers, each reading it inside its container.
# The journal forwarders in the CI runner's, Grafana's, the log store's,
# the forge's and the homeserver's containers, each reading it inside its
# container.
hyperhiveCfg.swarm.ci-runner.otel.clientId
hyperhiveCfg.swarm.grafana.otel.clientId
hyperhiveCfg.swarm.victorialogs.otel.clientId
hyperhiveCfg.swarm.forge.otel.clientId
hyperhiveCfg.swarm.matrix.otel.clientId
];
# The swarm controller's OIDC client, which it reads back from the store

58
nix/module-eval/bao-grants.nix
View file

@ -517,17 +517,33 @@ let
# read as tidier.
#
# The first leaf is the swarm appservice token, which matrix-ctl mints
# and publishes for the controller; the second is the homeserver's own
# OIDC client secret, read-only. Counted, so a third stanza fails rather
# than riding along beside correct ones.
name = "matrix-ctl's grant is the swarm appservice token and the homeserver's OIDC secret, nothing else";
# and publishes for the controller; the second and third are the
# homeserver's own OIDC client secret and its container forwarder's,
# read-only. The whole list, so a fourth stanza fails rather than riding
# along beside correct ones.
name = "matrix-ctl's grant is the swarm appservice token and the homeserver's and its forwarder's OIDC secrets, nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-matrix-ctl-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/controller/swarm-controller/matrix/appservice-token\" {\n capabilities = [\"create\", \"update\", \"read\"]\n}" s
&& lib.hasInfix "path \"secret/data/swarm/services/tuwunel/oidc/client\" {\n capabilities = [\"read\"]\n}" s
&& lib.length (lib.splitString "path \"" s) == 3
grantsIn s == [
{
path = "secret/data/swarm/controller/swarm-controller/matrix/appservice-token";
caps = [
"create"
"update"
"read"
];
}
{
path = "secret/data/swarm/services/tuwunel/oidc/client";
caps = [ "read" ];
}
{
path = "secret/data/swarm/services/swarm-matrix-collector/oidc/client";
caps = [ "read" ];
}
]
&& !(lib.hasInfix "secret/data/swarm/services/*" s)
&& !(lib.hasInfix "swarm-grafana" s)
&& !(lib.hasInfix "secret/data/swarm/agents" s)
@ -1715,17 +1731,31 @@ let
}
{
# The forge's host is the one writer of the controller's forge token,
# and reads the forge's own OIDC client secret under the same leaf.
# Pinned as two whole stanzas, so an added `read` on the token or a
# write or wildcard on the secret fails.
name = "the forge publisher's grant is writing the controller's forge token and reading the forge's OIDC secret, and nothing else";
# and reads the forge's own OIDC client secret and its container
# forwarder's under the same leaf. Pinned as the whole list, so an added
# `read` on the token or a write or wildcard on a secret fails.
name = "the forge publisher's grant is writing the controller's forge token and reading the forge's and its forwarder's OIDC secrets, and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-forge-publisher-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/controller/swarm-controller/forge/token\" {\n capabilities = [\"create\", \"update\"]\n}" s
&& lib.hasInfix "path \"secret/data/swarm/services/forgejo/oidc/client\" {\n capabilities = [\"read\"]\n}" s
&& lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 2
grantsIn s == [
{
path = "secret/data/swarm/controller/swarm-controller/forge/token";
caps = [
"create"
"update"
];
}
{
path = "secret/data/swarm/services/forgejo/oidc/client";
caps = [ "read" ];
}
{
path = "secret/data/swarm/services/swarm-forge-collector/oidc/client";
caps = [ "read" ];
}
]
&& !(lib.hasInfix "secret/data/swarm/services/*" s)
&& lib.hasInfix "auth/cert/certs/swarm-forge-publisher" s
&& lib.hasInfix "allowed_common_names=swarm-forge-publisher" s

2
nix/module-eval/name-guards.nix
View file

@ -53,7 +53,7 @@ let
};
# The THIRD element of the same list, colliding on its own so neither of the
# two above can carry it. matrix-ctl's grant is one path rather than a whole
# two above can carry it. matrix-ctl's grant names its leaves, not a whole
# prefix, which is exactly why a dead entry here would be easy to miss: a
# hive that inherited it would not obviously break anything, it would
# silently gain the ability to overwrite the swarm's matrix credential.

61
nix/module-eval/nats-authelia.nix
View file

@ -182,7 +182,8 @@ let
# The forwarders that authenticate as a client of their own rather than the
# service's, each with the store identity the host already holds. The CI
# runner's container cases are in ./forge-placement.nix, beside its fixture.
# runner's container cases are in ./forge-placement.nix, beside its fixture;
# the homeserver's, which reads a bound-in identity, are below.
ownClientForwarders = [
{
machine = "swarm-ci-runner";
@ -196,10 +197,18 @@ let
machine = "swarm-victorialogs";
option = "victorialogs";
}
{
machine = "hive-forge";
option = "forge";
}
{
machine = "hive-matrix";
option = "matrix";
}
];
ownClientIdOf = h: svc: h.services.hyperhive.swarm.${svc.option}.otel.clientId;
# The two of them that share the host netns, on the shared services host.
# Those that share the host netns, on the shared services host.
hostNetnsForwarders = [
{
machine = "swarm-grafana";
@ -217,6 +226,14 @@ let
key = "bao-client-key.pem";
port = 8897;
}
{
machine = "hive-forge";
option = "forge";
name = "forgejo";
cert = "forge-bao-cert";
key = "forge-bao-key";
port = 8898;
}
];
hostNetnsForwarderCases = lib.concatMap (
@ -856,7 +873,7 @@ let
lib.all ok1 (map (svc: svc.machine) forwarderServices);
}
{
name = "the swarm collector admits the CI runner, Grafana and log store forwarders on receivers of their own";
name = "the swarm collector admits the CI runner, Grafana, log store, forge and homeserver forwarders on receivers of their own";
ok =
let
h = shared.baoWithCollector;
@ -877,7 +894,7 @@ let
{
# Grafana's forwarder is a second client beside its browser login, which
# stays an interactive client.
name = "the CI runner, Grafana and log store forwarder clients are registered as machine clients with their own audience and a signed access token";
name = "the CI runner, Grafana, log store, forge and homeserver forwarder clients are registered as machine clients with their own audience and a signed access token";
ok =
let
h = shared.autheliaOnly;
@ -901,6 +918,42 @@ let
&& browser != null
&& browser.kind != "machine";
}
{
# The homeserver's container already holds matrix-ctl's identity, bound
# in read-only, and its forwarder fetches with those paths rather than a
# system credential.
name = "the homeserver container forwards its journal to its own route as its own client";
ok =
let
h = servicesHost;
container = h.containers.hive-matrix;
c = container.config;
s = c.services.opentelemetry-collector.settings;
otel = h.services.hyperhive.swarm.otel;
fetch = c.systemd.services.swarm-journal-forwarder-oidc;
clientId = h.services.hyperhive.swarm.matrix.otel.clientId;
ctlCert = h.services.hyperhive.deploy.matrix.ctlBaoClientCertFile;
in
c.services.opentelemetry-collector.enable
&& s.exporters.otlphttp.endpoint == "https://${otel.domain}/hive-matrix"
&& s.extensions."oauth2client/swarm".client_id == clientId
&& s.extensions."oauth2client/swarm".endpoint_params.audience == clientId
&& otel.serviceProducers.hive-matrix.clientId == clientId
&&
(lib.findFirst (a: a.key == "service.name") null s.processors.resource.attributes).value
== "tuwunel"
&& fetch.environment.BAO_CLIENT_CERT == ctlCert
&& fetch.environment.BAO_CLIENT_KEY == h.services.hyperhive.deploy.matrix.ctlBaoClientKeyFile
&& container.bindMounts ? ${builtins.dirOf ctlCert}
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script
&& !(lib.elem "--link-journal=host" container.extraFlags);
}
{
# With no matrix-ctl identity there is nothing to fetch the secret with.
name = "a homeserver container with no store identity runs no forwarder";
ok =
!shared.matrixNoBaoIdentity.containers.hive-matrix.config.services.opentelemetry-collector.enable;
}
]
++ forwarderCases
++ hostNetnsForwarderCases;

9
nix/module-eval/secret-publisher.nix
View file

@ -217,9 +217,10 @@ let
);
}
{
# The forwarders in the CI runner's, Grafana's and the log store's
# containers read their secrets from these paths.
name = "the publisher carries the CI runner, Grafana and log store forwarders' secrets";
# The forwarders in the CI runner's, Grafana's, the log store's, the
# forge's and the homeserver's containers read their secrets from these
# paths.
name = "the publisher carries the CI runner, Grafana, log store, forge and homeserver forwarders' secrets";
ok =
lib.all
(
@ -232,6 +233,8 @@ let
"swarm-ci-runner-collector"
"swarm-grafana-collector"
"swarm-victorialogs-collector"
"swarm-forge-collector"
"swarm-matrix-collector"
];
}
{