vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

hive-forge, hive-matrix: forward each container's journal to the swarm collector

Browse source 
Each container now runs the journal forwarder the CI runner's, Grafana's and
the log store's containers run, pushing to a receiver of its own on the swarm
collector (serviceProducers hive-forge and hive-matrix). Each authenticates as
a new machine OIDC client of its own (swarm.forge.otel.clientId,
swarm.matrix.otel.clientId), registered by a glue file wherever authelia runs,
minted by authelia and published to the store by the secret publisher. The
forge's and the homeserver's browser login clients are unchanged. The client
ids are swarm-forge-collector and swarm-matrix-collector, not
hive-<x>-collector, which is the client id a hive named <x>-collector gets.

The fetch unit inside each container reads the client secret with the store
identity the host already holds for that service. The forge publisher's leaf
is handed into the forge container as an nspawn system credential, as the
grafana and log store containers do. matrix-ctl's leaf is already bound into
the homeserver's container, so its forwarder reads it at those paths, as
swarm-matrix-appservice-publish does; with no matrix-ctl identity the
homeserver's forwarder is off. Each of the two policies gains one read stanza
on its forwarder's client path. The host units that use those identities are
unchanged.

The two containers drop --link-journal=host, and the host collector stops
reading them.
This commit is contained in:
atlas 2026-10-16 11:48:22 +02:00 • committed by mara
commit ce44b6e77e
18 changed files with 511 additions and 134 deletions
Download patch file Download diff file Expand all files Collapse all files

121
nix/host-modules/hive-forge/default.nix
View file

@ -41,7 +41,7 @@ let
oldOidcSecretPath = "/var/lib/nixos-containers/hive-forge/var/lib/forgejo-oidc/${cfg.sso.clientId}.secret";
# Where swarm-secret-publish on authelia's host leaves this client's secret.
# The forge publisher's store role grants read on this one path.
# The forge publisher's store role grants read on this path.
oidcStorePath = "secret/swarm/services/${cfg.sso.clientId}/oidc/client";
# The swarm-controller's forge account. Same name as
@ -78,6 +78,33 @@ let
caTrust = import ../lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.hive-forge` in
# ../swarm-otel-service.nix declares, as a client of its own rather than the
# login source's. The port suffix is elided on 443, as in ../swarm-bao.nix.
forwarderClientId = cfg.otel.clientId;
gatewayPortSuffix = lib.optionalString (
gatewayCfg.httpsPort != 443
) ":${toString gatewayCfg.httpsPort}";
forwarderEndpoint = "https://${config.services.hyperhive.swarm.otel.domain}${gatewayPortSuffix}/hive-forge";
forwarderUnit = "swarm-journal-forwarder-oidc";
# In the host netns: distinct from every other collector's there, which the
# assertion below checks.
forwarderTelemetryPort = 8898;
# The forge host's store identity, handed into the container as system
# credentials (credential id → host source) for the forwarder's fetch unit.
# The host units below keep reading the same files directly.
storeFiles = lib.filterAttrs (_: source: source != null) {
forge-bao-cert = deployCfg.bao.forgePublisherClientCertFile;
forge-bao-key = deployCfg.bao.forgePublisherClientKeyFile;
forge-bao-server-ca = deployCfg.bao.serverCaFile;
};
# nixos-container expands `extraFlags` unquoted, so a source path must be
# one word.
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
# ROOT_URL forgejo advertises in clone links + outbound URLs.
# `cfg.domain` doubles as both the forgejo `DOMAIN` setting AND the
# gateway vhost server-name, so ROOT_URL just uses it directly. The
@ -403,19 +430,62 @@ in
collide. Use a dedicated org (e.g. "actions/checkout").
'';
}
{
assertion = lib.all oneWord (lib.attrValues storeFiles);
message = ''
The forge host's store identity paths must not contain whitespace:
nspawn's --load-credential receives them through nixos-container's
unquoted extra flags.
'';
}
{
assertion =
(config.services.hyperhive.swarm.otel.serviceProducers.hive-forge.clientId or null)
== forwarderClientId;
message = ''
services.hyperhive.swarm.otel.serviceProducers has no entry
"hive-forge" with clientId "${forwarderClientId}", so the swarm
collector has no receiver for the forge's journal forwarder.
Declare it in swarm-otel-service.nix.
'';
}
{
assertion =
!(lib.elem forwarderTelemetryPort [
config.services.hyperhive.otel.telemetryPort
config.services.hyperhive.swarm.otel.telemetryPort
config.services.hyperhive.swarm.bao.otel.telemetryPort
config.services.hyperhive.swarm.nats.otel.telemetryPort
]);
message = ''
The forge's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
collides with another collector's in the host netns: the hive
collector's, the swarm collector's, the store forwarder's or the
queue forwarder's.
'';
}
];
# `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service`
# in self-signed mode (see the hive-ca-trust helper), so the CA bind
# source exists before nspawn sets the mount up.
systemd.services."container@hive-forge" = caTrust.containerOrdering;
# source exists before nspawn sets the mount up. nspawn also fails to
# start it when a credential source is missing, so where this host mints
# the leaf the container pulls the minting unit in and waits for it, as
# ../swarm-builder.nix's does.
systemd.services."container@hive-forge" = lib.mkMerge [
caTrust.containerOrdering
{
after = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];
}
];
containers.hive-forge = {
autoStart = true;
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# No `--link-journal=host`: the journal stays in the guest, where the
# container's own forwarder reads it (../../container-modules/swarm-journal-forwarder.nix).
extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
inherit privateNetwork;
# Self-signed mode: bind the hive's trust bundle read-only so forgejo
# can trust the gateway's self-signed leaf for outbound webhook
@ -458,6 +528,7 @@ in
{
imports = [
./../../container-modules/swarm-container.nix
./../../container-modules/swarm-journal-forwarder.nix
# Ask the hive's dnsmasq, not the host's resolvers. The swarm
# names — the controller's webhook endpoint, the SSO issuer —
# resolve to the bridge and have no public records, so a
@ -465,14 +536,18 @@ in
# every one of them while the same name resolves fine one
# container over.
#
# Both units that make an outbound call are consumers, for the
# same reason the trust bundle below names both: an unordered
# resolver write is a race that only shows up on a cold boot.
# Every unit that makes an outbound call is a consumer, for the
# same reason the trust bundle below names them: an unordered
# resolver write is a race that only shows up on a cold boot. The
# forwarder and its secret fetch dial the swarm collector, authelia
# and the store by name.
(import ./../../container-modules/swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
dnsConsumers = [
"forgejo.service"
"forgejo-sso-source.service"
"opentelemetry-collector.service"
"${forwarderUnit}.service"
];
})
@ -481,16 +556,18 @@ in
# store's root concatenated, not the root alone, or public mirror
# fetches lose every anchor they had.
#
# BOTH units that make an outbound HTTPS call are consumers, not
# EVERY unit that makes an outbound HTTPS call is a consumer, not
# just the obvious one: `forgejo-sso-source` fetches the issuer's
# `.well-known/openid-configuration` under the store's root, and it
# once shipped without the trust its sibling had.
# once shipped without the trust its sibling had. The collector is
# Go too, for the swarm collector's and authelia's names.
(caTrust.trustBundle {
inherit pkgs;
name = "hive-forge";
consumers = [
"forgejo"
"forgejo-sso-source"
"opentelemetry-collector"
];
})
];
@ -498,6 +575,28 @@ in
system.stateVersion = "25.11";
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# Authenticated as the forwarder's own OIDC client, with the forge
# host's store identity: one identity for the whole container. It is
# a system credential, so the fetch unit loads it.
services.hyperhive.swarmJournalForwarder = {
enable = true;
serviceName = "forgejo";
endpoint = forwarderEndpoint;
clientId = forwarderClientId;
tokenUrl = "${toString autheliaUrl}/api/oidc/token";
telemetryPort = forwarderTelemetryPort;
store = {
package = deployCfg.bao.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
clientCertFile = forwarderCredential "forge-bao-cert";
clientKeyFile = forwarderCredential "forge-bao-key";
caFile =
if storeFiles ? forge-bao-server-ca then forwarderCredential "forge-bao-server-ca" else null;
};
};
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
services.forgejo = {
enable = true;
package = deployCfg.forgejo.package;