vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

hive-forge, hive-matrix: forward each container's journal to the swarm collector

Browse source 
Each container now runs the journal forwarder the CI runner's, Grafana's and
the log store's containers run, pushing to a receiver of its own on the swarm
collector (serviceProducers hive-forge and hive-matrix). Each authenticates as
a new machine OIDC client of its own (swarm.forge.otel.clientId,
swarm.matrix.otel.clientId), registered by a glue file wherever authelia runs,
minted by authelia and published to the store by the secret publisher. The
forge's and the homeserver's browser login clients are unchanged. The client
ids are swarm-forge-collector and swarm-matrix-collector, not
hive-<x>-collector, which is the client id a hive named <x>-collector gets.

The fetch unit inside each container reads the client secret with the store
identity the host already holds for that service. The forge publisher's leaf
is handed into the forge container as an nspawn system credential, as the
grafana and log store containers do. matrix-ctl's leaf is already bound into
the homeserver's container, so its forwarder reads it at those paths, as
swarm-matrix-appservice-publish does; with no matrix-ctl identity the
homeserver's forwarder is off. Each of the two policies gains one read stanza
on its forwarder's client path. The host units that use those identities are
unchanged.

The two containers drop --link-journal=host, and the host collector stops
reading them.
This commit is contained in:
atlas 2026-10-16 11:48:22 +02:00 • committed by mara
commit ce44b6e77e
18 changed files with 511 additions and 134 deletions
Download patch file Download diff file Expand all files Collapse all files

62
nix/host-modules/hive-matrix.nix
View file

@ -49,7 +49,7 @@ let
legacyHostSecretPath = "/var/lib/nixos-containers/hive-matrix/var/lib/tuwunel-oidc/${cfg.sso.clientId}.secret";
# Where swarm-secret-publish on authelia's host leaves this client's secret.
# matrix-ctl's store role grants read on this one path.
# matrix-ctl's store role grants read on this path.
storeSecretPath = "secret/swarm/services/${cfg.sso.clientId}/oidc/client";
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
@ -119,6 +119,21 @@ let
isReadOnly = true;
});
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.hive-matrix` in
# ./swarm-otel-service.nix declares, as a client of its own rather than the
# homeserver login's. The port suffix is elided on 443, as in
# ./swarm-bao.nix. It reads its client secret with matrix-ctl's identity,
# bound in above: one identity for the whole container.
forwarderClientId = cfg.otel.clientId;
gatewayPortSuffix = lib.optionalString (
gatewayCfg.httpsPort != 443
) ":${toString gatewayCfg.httpsPort}";
forwarderEndpoint = "https://${config.services.hyperhive.swarm.otel.domain}${gatewayPortSuffix}/hive-matrix";
# Loopback in the container's own netns, and distinct from the other service
# containers' forwarders all the same.
forwarderTelemetryPort = 8899;
# ── the swarm's own appservice ──────────────────────────────────────
#
# The homeserver's one appservice registration, and the swarm's identity on
@ -584,6 +599,17 @@ in
`gatewayHost` is. With it null the homeserver has no such name.
'';
}
{
assertion =
(config.services.hyperhive.swarm.otel.serviceProducers.hive-matrix.clientId or null)
== forwarderClientId;
message = ''
services.hyperhive.swarm.otel.serviceProducers has no entry
"hive-matrix" with clientId "${forwarderClientId}", so the swarm
collector has no receiver for the homeserver's journal forwarder.
Declare it in swarm-otel-service.nix.
'';
}
];
# Where the homeserver finds its OIDC client secret, inside the container.
@ -777,9 +803,8 @@ in
containers.hive-matrix = {
autoStart = true;
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# No `--link-journal=host`: the journal stays in the guest, where the
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
inherit privateNetwork;
hostBridge = networkCfg.bridgeName;
bindMounts = {
@ -804,6 +829,7 @@ in
{
imports = [
../container-modules/swarm-container.nix
../container-modules/swarm-journal-forwarder.nix
# tuwunel's rustls verifier resolves through `rustls-native-certs`
# → `openssl-probe`, which reads `SSL_CERT_FILE` — so the
# openssl-shaped variable is the lever despite tuwunel linking no
@ -820,10 +846,36 @@ in
(caTrust.trustBundle {
inherit pkgs;
name = "hive-matrix";
consumers = [ "tuwunel" ];
consumers = [
"tuwunel"
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
# collector's and authelia's names.
"opentelemetry-collector"
];
})
];
# Authenticated as the forwarder's own OIDC client, with matrix-ctl's
# store identity at the paths bound in from the host. With no such
# identity there is nothing to fetch the secret with, as for the
# appservice units below.
services.hyperhive.swarmJournalForwarder = lib.mkIf ctlActive {
enable = true;
serviceName = "tuwunel";
endpoint = forwarderEndpoint;
clientId = forwarderClientId;
tokenUrl = "${toString autheliaUrl}/api/oidc/token";
telemetryPort = forwarderTelemetryPort;
store = {
package = deployCfg.bao.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
clientCertFile = deployCfg.matrix.ctlBaoClientCertFile;
clientKeyFile = deployCfg.matrix.ctlBaoClientKeyFile;
caFile = deployCfg.bao.serverCaFile;
};
};
# Its own netns, so it keeps its own firewall; resolvconf stays on
# and serves `networking.nameservers` below.
services.hyperhive.swarmContainer = {