vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-builder, swarm-claude-login, swarm-review-classifier: forward each container's journal to the swarm collector

Browse source 
Each container now runs the journal forwarder the queue's container runs,
pushing to a receiver of its own on the swarm collector as the service's
existing OIDC client, with the client's secret read inside the container
by the fetch unit. Their store identities are system credentials, so the
fetch unit loads them too. The three containers drop --link-journal=host,
which hid their journals from the forwarder, and the host collector stops
reading them.
This commit is contained in:
atlas 2026-10-16 03:28:23 +02:00
commit a2a94f63c8
10 changed files with 321 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

4
nix/host-modules/glue-builder-oidc-client.nix
View file

@ -28,6 +28,10 @@ in
# `client_credentials`: a daemon authenticating as itself.
kind = "machine";
audience = [ clientId ];
# The container's journal forwarder mints its token as this client, for
# the swarm collector's receiver, which verifies it offline against
# `/jwks.json`; authelia's default opaque handle fails there.
accessTokenSignedResponseAlg = "RS256";
}
];
};

4
nix/host-modules/glue-claude-login-oidc-client.nix
View file

@ -29,6 +29,10 @@ in
# `client_credentials`: a daemon authenticating as itself.
kind = "machine";
audience = [ clientId ];
# The container's journal forwarder mints its token as this client, for
# the swarm collector's receiver, which verifies it offline against
# `/jwks.json`; authelia's default opaque handle fails there.
accessTokenSignedResponseAlg = "RS256";
}
];
};

4
nix/host-modules/glue-review-classifier-oidc-client.nix
View file

@ -29,6 +29,10 @@ in
# `client_credentials`: a daemon authenticating as itself.
kind = "machine";
audience = [ clientId ];
# The container's journal forwarder mints its token as this client, for
# the swarm collector's receiver, which verifies it offline against
# `/jwks.json`; authelia's default opaque handle fails there.
accessTokenSignedResponseAlg = "RS256";
}
];
};

55
nix/host-modules/swarm-builder.nix
View file

@ -41,6 +41,20 @@ let
unit = "swarm-builder";
clientId = hyperhiveCfg.swarm.builder.clientId;
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.swarm-builder` in
# ./swarm-otel-service.nix declares, by the collector's gateway name. The
# port suffix is elided on 443, as in ./swarm-bao.nix.
swarmOtelCfg = hyperhiveCfg.swarm.otel;
gatewayPortSuffix = lib.optionalString (
gatewayCfg.httpsPort != 443
) ":${toString gatewayCfg.httpsPort}";
forwarderEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${machine}";
forwarderUnit = "swarm-journal-forwarder-oidc";
# Loopback in the container's own netns, and distinct from the other service
# containers' forwarders all the same.
forwarderTelemetryPort = 8892;
# The daemon reaches authelia, the store and the queue by name over TLS, and
# `git` reaches the forge; in self-signed mode their leaves chain to the hive
# CA.
@ -48,7 +62,12 @@ let
caBundleModule = caTrust.trustBundle {
inherit pkgs;
name = machine;
consumers = [ unit ];
consumers = [
unit
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
# collector's and authelia's names.
"opentelemetry-collector"
];
};
haveClientIdentity =
@ -65,6 +84,7 @@ let
# one word.
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
credential = id: "/run/credentials/${unit}.service/${id}";
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
# The `nix` the host daemon runs, so client and daemon speak one protocol.
nix = config.nix.package;
@ -149,12 +169,9 @@ in
# The daemon keeps nothing but `git`'s fetch cache, which a restart may
# drop.
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [
"--link-journal=host"
]
++ lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
# No `--link-journal=host`: the journal stays in the guest, where the
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
inherit privateNetwork;
hostBridge = networkCfg.bridgeName;
@ -173,6 +190,7 @@ in
{
imports = [
../container-modules/swarm-container.nix
../container-modules/swarm-journal-forwarder.nix
caBundleModule
];
@ -189,6 +207,29 @@ in
systemd.services.nix-daemon.enable = false;
systemd.sockets.nix-daemon.enable = false;
# Authenticated as the daemon's own OIDC client, whose secret its
# store identity already reads: one identity for the whole container.
# That identity is a system credential, so the fetch unit loads it
# too; the daemon's `LoadCredential` below is that unit's alone.
services.hyperhive.swarmJournalForwarder = {
enable = true;
serviceName = "builder";
endpoint = forwarderEndpoint;
inherit clientId;
tokenUrl = "${autheliaCfg.url}/api/oidc/token";
telemetryPort = forwarderTelemetryPort;
store = {
package = baoDeploy.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
secretPath = "secret/swarm/services/${clientId}/oidc/client";
clientCertFile = forwarderCredential "builder-bao-cert";
clientKeyFile = forwarderCredential "builder-bao-key";
caFile =
if storeFiles ? builder-bao-server-ca then forwarderCredential "builder-bao-server-ca" else null;
};
};
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
systemd.services.${unit} = {
description = "build agent configs for swarm-controller";
wantedBy = [ "multi-user.target" ];

58
nix/host-modules/swarm-claude-login.nix
View file

@ -44,13 +44,32 @@ let
unit = "swarm-claude-login";
clientId = hyperhiveCfg.swarm.claude-login.clientId;
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.swarm-claude-login` in
# ./swarm-otel-service.nix declares, by the collector's gateway name. The
# port suffix is elided on 443, as in ./swarm-bao.nix.
swarmOtelCfg = hyperhiveCfg.swarm.otel;
gatewayPortSuffix = lib.optionalString (
gatewayCfg.httpsPort != 443
) ":${toString gatewayCfg.httpsPort}";
forwarderEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${machine}";
forwarderUnit = "swarm-journal-forwarder-oidc";
# Loopback in the container's own netns, and distinct from the other service
# containers' forwarders all the same.
forwarderTelemetryPort = 8893;
# The daemon reaches authelia, the store and the queue by name over TLS, and
# in self-signed mode their leaves chain to the hive CA.
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
caBundleModule = caTrust.trustBundle {
inherit pkgs;
name = machine;
consumers = [ unit ];
consumers = [
unit
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
# collector's and authelia's names.
"opentelemetry-collector"
];
};
haveClientIdentity =
@ -67,6 +86,7 @@ let
# one word.
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
credential = id: "/run/credentials/${unit}.service/${id}";
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
privateNetwork = true;
in
@ -138,12 +158,9 @@ in
# The daemon keeps no state; each login's scratch directory is under
# PrivateTmp.
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [
"--link-journal=host"
]
++ lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
# No `--link-journal=host`: the journal stays in the guest, where the
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
inherit privateNetwork;
hostBridge = networkCfg.bridgeName;
bindMounts = caTrust.bindMount;
@ -153,6 +170,7 @@ in
{
imports = [
../container-modules/swarm-container.nix
../container-modules/swarm-journal-forwarder.nix
caBundleModule
];
@ -173,6 +191,32 @@ in
};
users.groups.${unit} = { };
# Authenticated as the daemon's own OIDC client, whose secret its
# store identity already reads: one identity for the whole container.
# That identity is a system credential, so the fetch unit loads it
# too; the daemon's `LoadCredential` below is that unit's alone.
services.hyperhive.swarmJournalForwarder = {
enable = true;
serviceName = "claude-login";
endpoint = forwarderEndpoint;
inherit clientId;
tokenUrl = "${autheliaCfg.url}/api/oidc/token";
telemetryPort = forwarderTelemetryPort;
store = {
package = baoDeploy.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
secretPath = "secret/swarm/services/${clientId}/oidc/client";
clientCertFile = forwarderCredential "claude-login-bao-cert";
clientKeyFile = forwarderCredential "claude-login-bao-key";
caFile =
if storeFiles ? claude-login-bao-server-ca then
forwarderCredential "claude-login-bao-server-ca"
else
null;
};
};
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
systemd.services.${unit} = {
description = "log agents in to claude for the swarm UI";
wantedBy = [ "multi-user.target" ];

15
nix/host-modules/swarm-otel-service.nix
View file

@ -400,6 +400,21 @@ in
clientId = hyperhiveCfg.swarm.nats.clientId;
port = lib.mkDefault 4392;
};
# The forwarders in ./swarm-builder.nix, ./swarm-claude-login.nix and
# ./swarm-review-classifier.nix, which address their routes by container
# name.
swarm-builder = {
clientId = hyperhiveCfg.swarm.builder.clientId;
port = lib.mkDefault 4393;
};
swarm-claude-login = {
clientId = hyperhiveCfg.swarm.claude-login.clientId;
port = lib.mkDefault 4394;
};
swarm-review-classifier = {
clientId = hyperhiveCfg.swarm.review-classifier.clientId;
port = lib.mkDefault 4395;
};
};
# Not under any `enable`: see `publishedScrapeTargets` for why every host

9
nix/host-modules/swarm-otel.nix
View file

@ -206,16 +206,17 @@ let
# `--link-journal=try-guest`, which puts the journal inside the container and
# leaves the host with a symlink into that container's transient root: a
# reader here cannot follow it, and it dangles the moment the container
# stops. Every container block except `swarm-bao` and `swarm-nats` therefore sets
# stops. Every container block except the `serviceProducers` ones therefore sets
# `extraFlags = [ "--link-journal=host" ]`, which the invocation expands
# AFTER the hardcoded flag, so the files land here under their own
# machine-id subdirectory and are bind-mounted into the guest instead.
#
# ⚠️ Drop that flag from one of THOSE containers and this receiver silently
# stops seeing it — no error, just a unit that never appears in the store.
# `swarm-bao` and `swarm-nats` are the exceptions on purpose: their journals
# stay inside the container for its own collector instead (see
# `serviceProducers`), so this receiver never sees their units.
# The `serviceProducers` containers (`swarm-bao`, `swarm-nats`,
# `swarm-builder`, `swarm-claude-login`, `swarm-review-classifier`) are the
# exceptions on purpose: their journals stay inside the container for its own
# collector instead, so this receiver never sees their units.
hostJournalDir = "/var/log/journal";
# Each store's OTLP route, by domain. One binding because the same string is

58
nix/host-modules/swarm-review-classifier.nix
View file

@ -46,13 +46,32 @@ let
unit = "swarm-review-classifier";
clientId = hyperhiveCfg.swarm.review-classifier.clientId;
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.swarm-review-classifier` in
# ./swarm-otel-service.nix declares, by the collector's gateway name. The
# port suffix is elided on 443, as in ./swarm-bao.nix.
swarmOtelCfg = hyperhiveCfg.swarm.otel;
gatewayPortSuffix = lib.optionalString (
gatewayCfg.httpsPort != 443
) ":${toString gatewayCfg.httpsPort}";
forwarderEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${machine}";
forwarderUnit = "swarm-journal-forwarder-oidc";
# Loopback in the container's own netns, and distinct from the other service
# containers' forwarders all the same.
forwarderTelemetryPort = 8894;
# The daemon reaches authelia, the store and the queue by name over TLS, and
# in self-signed mode their leaves chain to the hive CA.
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
caBundleModule = caTrust.trustBundle {
inherit pkgs;
name = machine;
consumers = [ unit ];
consumers = [
unit
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
# collector's and authelia's names.
"opentelemetry-collector"
];
};
haveClientIdentity =
@ -72,6 +91,7 @@ let
# one word.
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
credential = id: "/run/credentials/${unit}.service/${id}";
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
# What `claude` exports its metrics with, and the daemon its ACP runs' cost,
# where this host runs a hive-tier collector on the bridge. The daemon adds
@ -164,12 +184,9 @@ in
autoStart = true;
# The daemon keeps no state; its scratch directories are under PrivateTmp.
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [
"--link-journal=host"
]
++ lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
# No `--link-journal=host`: the journal stays in the guest, where the
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
inherit privateNetwork;
hostBridge = networkCfg.bridgeName;
bindMounts = caTrust.bindMount;
@ -179,6 +196,7 @@ in
{
imports = [
../container-modules/swarm-container.nix
../container-modules/swarm-journal-forwarder.nix
caBundleModule
];
@ -193,6 +211,32 @@ in
# nixpkgs, so the host's allowance does not reach here.
nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "claude-code" ];
# Authenticated as the daemon's own OIDC client, whose secret its
# store identity already reads: one identity for the whole container.
# That identity is a system credential, so the fetch unit loads it
# too; the daemon's `LoadCredential` below is that unit's alone.
services.hyperhive.swarmJournalForwarder = {
enable = true;
serviceName = "review-classifier";
endpoint = forwarderEndpoint;
inherit clientId;
tokenUrl = "${autheliaCfg.url}/api/oidc/token";
telemetryPort = forwarderTelemetryPort;
store = {
package = baoDeploy.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
secretPath = "secret/swarm/services/${clientId}/oidc/client";
clientCertFile = forwarderCredential "review-classifier-bao-cert";
clientKeyFile = forwarderCredential "review-classifier-bao-key";
caFile =
if storeFiles ? review-classifier-bao-server-ca then
forwarderCredential "review-classifier-bao-server-ca"
else
null;
};
};
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
systemd.services.${unit} = {
description = "answer the swarm UI's review try-outs";
wantedBy = [ "multi-user.target" ];