vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-nats: forward the queue container's journal to the swarm collector

Browse source 
The swarm-nats container now runs its own otel collector, which reads the
container's journal and pushes it to the swarm collector at
https://<swarm.otel.domain>/swarm-nats, authenticated as the queue's
existing OIDC client (swarm.nats.clientId). Records carry
service.name=nats and no hive label.

The collector lives in a reusable container module,
nix/container-modules/swarm-journal-forwarder.nix, which follows
swarm-bao's forwarder: a journald receiver over /var/log/journal with
merge and a file_storage cursor, an otlphttp exporter with an
oauth2client authenticator that requests its own client id as audience,
self-metrics on a loopback telemetry port, and the client secret handed
in through LoadCredential. Unlike swarm-bao, the secret is fetched inside
the container by swarm-journal-forwarder-oidc.service, under the
container's own store identity (the swarm-nats-auth leaf, which already
holds a read grant on secret/swarm/services/swarm-nats/oidc/client).
The fetch retries until the store answers, and the collector restarts
until the file exists. A 10-minute timer re-reads the secret and restarts
the collector when it changed.

The swarm collector's authenticated producer route was hard-wired to
swarm-bao. It is now rendered from swarm.otel.serviceProducers
(internal), one receiver, oidc authenticator, gateway location and
pipeline entry per service: swarm-bao (port 4391, unchanged names) and
swarm-nats (port 4392). The reserved-fragment and port-collision
assertions cover every entry.

The queue's OIDC client registration gains its own id as audience and
RS256-signed access tokens, which the receiver verifies offline.

swarm-nats drops --link-journal=host, so its journal stays in the guest
where the forwarder reads it, as swarm-bao did in 44009dc51. The new
swarm.nats.otel.telemetryPort (8891) keeps the forwarder off the
collectors' 8888-8890 in the shared host netns, with an assertion.

module-eval-nats-authelia gains four cases: the container's forwarder
and fetch unit on the store host and on a queue-only host, the
responder role's read grant, the swarm collector's swarm-nats receiver
on a host without the queue, and the client registration.

Part of #4499
Part of #5342
This commit is contained in:
atlas 2026-10-16 02:04:02 +02:00 • committed by mara
commit 40a2bd047b
9 changed files with 660 additions and 105 deletions
Download patch file Download diff file Expand all files Collapse all files

88
nix/host-modules/swarm-nats.nix
View file

@ -25,6 +25,16 @@ let
machine = "swarm-nats";
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.swarm-nats` in
# ./swarm-otel-service.nix declares, by the collector's gateway name. The
# port suffix is elided on 443, as in ./swarm-bao.nix.
swarmOtelCfg = config.services.hyperhive.swarm.otel;
gatewayPortSuffix = lib.optionalString (
config.services.hyperhive.gateway.httpsPort != 443
) ":${toString config.services.hyperhive.gateway.httpsPort}";
forwarderEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${machine}";
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
gatewayCfg = config.services.hyperhive.gateway;
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
@ -38,7 +48,12 @@ let
caBundleModule = caTrust.trustBundle {
inherit pkgs;
name = machine;
consumers = [ "swarm-nats-auth" ];
consumers = [
"swarm-nats-auth"
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
# collector's and authelia's names.
"opentelemetry-collector"
];
};
# Where the responder's credentials live *inside* the container, and the
# host path that resolves to. Two names for one location, because the
@ -575,6 +590,40 @@ in
both options.
'';
}
{
# The forwarder's route and the receiver behind it are two
# declarations of one fact; a mismatch is a 404 or a 401 on an export
# that looks healthy.
assertion = (swarmOtelCfg.serviceProducers.${machine}.clientId or null) == cfg.clientId;
message = ''
services.hyperhive.swarm.otel.serviceProducers has no entry
"${machine}" with clientId "${cfg.clientId}", so the swarm
collector has no receiver for the queue container's journal
forwarder. Declare it in swarm-otel-service.nix.
'';
}
{
assertion =
let
ports = [
cfg.port
cfg.monitorPort
cfg.metricsPort
cfg.otel.telemetryPort
config.services.hyperhive.otel.telemetryPort
swarmOtelCfg.telemetryPort
config.services.hyperhive.swarm.bao.otel.telemetryPort
];
in
lib.length (lib.unique ports) == lib.length ports;
message = ''
services.hyperhive.swarm.nats.otel.telemetryPort
(${toString cfg.otel.telemetryPort}) collides with another port in
the host netns: the queue's own ports, or the telemetry port of the
hive collector, the swarm collector or the store's forwarder. Pick
a free port.
'';
}
];
# Open the client port on the bridge, so agent containers can reach
@ -601,9 +650,8 @@ in
containers.swarm-nats = {
autoStart = true;
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# No `--link-journal=host`: the journal stays in the guest, where the
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
inherit privateNetwork;
# The public trust bundle (empty when the gateway is not self-signed)
# and the queue's own TLS leaf, both read-only.
@ -618,19 +666,49 @@ in
{
imports = [
../container-modules/swarm-container.nix
../container-modules/swarm-journal-forwarder.nix
(import ../container-modules/swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
# The responder introspects authelia BY NAME on every auth
# request, and a responder that cannot resolve it denies
# every client — so it must not start before the resolver
# file exists.
dnsConsumers = [ "swarm-nats-auth.service" ];
#
# The forwarder and its secret fetch dial the swarm collector,
# authelia and the store by name too.
dnsConsumers = [
"swarm-nats-auth.service"
"opentelemetry-collector.service"
"swarm-journal-forwarder-oidc.service"
];
})
caBundleModule
];
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# Authenticated as the queue's own OIDC client, whose secret the
# responder's store identity already reads: one identity for the
# whole container.
services.hyperhive.swarmJournalForwarder = {
enable = true;
# The scrape job's name, so the queue's logs and metrics carry one
# `service.name`.
serviceName = "nats";
endpoint = forwarderEndpoint;
inherit (cfg) clientId;
tokenUrl = "${toString autheliaUrl}/api/oidc/token";
telemetryPort = cfg.otel.telemetryPort;
store = {
package = baoDeploy.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
secretPath = "secret/swarm/services/${cfg.clientId}/oidc/client";
clientCertFile = inContainer "bao-client.pem";
clientKeyFile = inContainer "bao-client-key.pem";
caFile = if authStoreFiles ? "bao-ca.pem" then inContainer "bao-ca.pem" else null;
};
};
services.nats = {
enable = true;