vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-nats: forward the queue container's journal to the swarm collector

Browse source 
The swarm-nats container now runs its own otel collector, which reads the
container's journal and pushes it to the swarm collector at
https://<swarm.otel.domain>/swarm-nats, authenticated as the queue's
existing OIDC client (swarm.nats.clientId). Records carry
service.name=nats and no hive label.

The collector lives in a reusable container module,
nix/container-modules/swarm-journal-forwarder.nix, which follows
swarm-bao's forwarder: a journald receiver over /var/log/journal with
merge and a file_storage cursor, an otlphttp exporter with an
oauth2client authenticator that requests its own client id as audience,
self-metrics on a loopback telemetry port, and the client secret handed
in through LoadCredential. Unlike swarm-bao, the secret is fetched inside
the container by swarm-journal-forwarder-oidc.service, under the
container's own store identity (the swarm-nats-auth leaf, which already
holds a read grant on secret/swarm/services/swarm-nats/oidc/client).
The fetch retries until the store answers, and the collector restarts
until the file exists. A 10-minute timer re-reads the secret and restarts
the collector when it changed.

The swarm collector's authenticated producer route was hard-wired to
swarm-bao. It is now rendered from swarm.otel.serviceProducers
(internal), one receiver, oidc authenticator, gateway location and
pipeline entry per service: swarm-bao (port 4391, unchanged names) and
swarm-nats (port 4392). The reserved-fragment and port-collision
assertions cover every entry.

The queue's OIDC client registration gains its own id as audience and
RS256-signed access tokens, which the receiver verifies offline.

swarm-nats drops --link-journal=host, so its journal stays in the guest
where the forwarder reads it, as swarm-bao did in 44009dc51. The new
swarm.nats.otel.telemetryPort (8891) keeps the forwarder off the
collectors' 8888-8890 in the shared host netns, with an assertion.

module-eval-nats-authelia gains four cases: the container's forwarder
and fetch unit on the store host and on a queue-only host, the
responder role's read grant, the swarm collector's swarm-nats receiver
on a host without the queue, and the client registration.

Part of #4499
Part of #5342
This commit is contained in:
atlas 2026-10-16 02:04:02 +02:00 • committed by mara
commit 40a2bd047b
9 changed files with 660 additions and 105 deletions
Download patch file Download diff file Expand all files Collapse all files

8
nix/host-modules/glue-nats-oidc-client.nix
View file

@ -39,6 +39,14 @@ in
# responder calls, works either way, so the token endpoint's
# `unauthorized_client` is the only place a wrong kind shows.
kind = "machine";
# The queue container's journal forwarder mints its token as this
# client, for the swarm collector's `oidc/swarm-nats` receiver, which
# accepts this audience alone (`serviceProducers` in
# ./swarm-otel-service.nix).
audience = [ natsCfg.clientId ];
# That receiver verifies the token offline against `/jwks.json`;
# authelia's default opaque handle fails there.
accessTokenSignedResponseAlg = "RS256";
}
];
};

18
nix/host-modules/swarm-nats-service.nix
View file

@ -107,6 +107,24 @@ in
'';
};
otel.telemetryPort = lib.mkOption {
type = lib.types.port;
default = 8891;
description = ''
Loopback port the journal forwarder inside the queue's container
serves its own metrics on (refused and dropped records, exporter
failures).
⚠️ Not 8888, 8889 or 8890: the hive collector, the swarm collector
and the store's forwarder bind those
({option}`services.hyperhive.otel.telemetryPort`,
{option}`services.hyperhive.swarm.otel.telemetryPort`,
{option}`services.hyperhive.swarm.bao.otel.telemetryPort`), and this
container shares the host netns with them when co-located. The second
collector to bind a port dies with `bind: address already in use`.
'';
};
clientId = lib.mkOption {
type = lib.types.str;
default = "swarm-nats";

88
nix/host-modules/swarm-nats.nix
View file

@ -25,6 +25,16 @@ let
machine = "swarm-nats";
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.swarm-nats` in
# ./swarm-otel-service.nix declares, by the collector's gateway name. The
# port suffix is elided on 443, as in ./swarm-bao.nix.
swarmOtelCfg = config.services.hyperhive.swarm.otel;
gatewayPortSuffix = lib.optionalString (
config.services.hyperhive.gateway.httpsPort != 443
) ":${toString config.services.hyperhive.gateway.httpsPort}";
forwarderEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${machine}";
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
gatewayCfg = config.services.hyperhive.gateway;
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
@ -38,7 +48,12 @@ let
caBundleModule = caTrust.trustBundle {
inherit pkgs;
name = machine;
consumers = [ "swarm-nats-auth" ];
consumers = [
"swarm-nats-auth"
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
# collector's and authelia's names.
"opentelemetry-collector"
];
};
# Where the responder's credentials live *inside* the container, and the
# host path that resolves to. Two names for one location, because the
@ -575,6 +590,40 @@ in
both options.
'';
}
{
# The forwarder's route and the receiver behind it are two
# declarations of one fact; a mismatch is a 404 or a 401 on an export
# that looks healthy.
assertion = (swarmOtelCfg.serviceProducers.${machine}.clientId or null) == cfg.clientId;
message = ''
services.hyperhive.swarm.otel.serviceProducers has no entry
"${machine}" with clientId "${cfg.clientId}", so the swarm
collector has no receiver for the queue container's journal
forwarder. Declare it in swarm-otel-service.nix.
'';
}
{
assertion =
let
ports = [
cfg.port
cfg.monitorPort
cfg.metricsPort
cfg.otel.telemetryPort
config.services.hyperhive.otel.telemetryPort
swarmOtelCfg.telemetryPort
config.services.hyperhive.swarm.bao.otel.telemetryPort
];
in
lib.length (lib.unique ports) == lib.length ports;
message = ''
services.hyperhive.swarm.nats.otel.telemetryPort
(${toString cfg.otel.telemetryPort}) collides with another port in
the host netns: the queue's own ports, or the telemetry port of the
hive collector, the swarm collector or the store's forwarder. Pick
a free port.
'';
}
];
# Open the client port on the bridge, so agent containers can reach
@ -601,9 +650,8 @@ in
containers.swarm-nats = {
autoStart = true;
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# No `--link-journal=host`: the journal stays in the guest, where the
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
inherit privateNetwork;
# The public trust bundle (empty when the gateway is not self-signed)
# and the queue's own TLS leaf, both read-only.
@ -618,19 +666,49 @@ in
{
imports = [
../container-modules/swarm-container.nix
../container-modules/swarm-journal-forwarder.nix
(import ../container-modules/swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
# The responder introspects authelia BY NAME on every auth
# request, and a responder that cannot resolve it denies
# every client — so it must not start before the resolver
# file exists.
dnsConsumers = [ "swarm-nats-auth.service" ];
#
# The forwarder and its secret fetch dial the swarm collector,
# authelia and the store by name too.
dnsConsumers = [
"swarm-nats-auth.service"
"opentelemetry-collector.service"
"swarm-journal-forwarder-oidc.service"
];
})
caBundleModule
];
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# Authenticated as the queue's own OIDC client, whose secret the
# responder's store identity already reads: one identity for the
# whole container.
services.hyperhive.swarmJournalForwarder = {
enable = true;
# The scrape job's name, so the queue's logs and metrics carry one
# `service.name`.
serviceName = "nats";
endpoint = forwarderEndpoint;
inherit (cfg) clientId;
tokenUrl = "${toString autheliaUrl}/api/oidc/token";
telemetryPort = cfg.otel.telemetryPort;
store = {
package = baoDeploy.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
secretPath = "secret/swarm/services/${cfg.clientId}/oidc/client";
clientCertFile = inContainer "bao-client.pem";
clientKeyFile = inContainer "bao-client-key.pem";
caFile = if authStoreFiles ? "bao-ca.pem" then inContainer "bao-ca.pem" else null;
};
};
services.nats = {
enable = true;

56
nix/host-modules/swarm-otel-service.nix
View file

@ -166,6 +166,48 @@ in
'';
};
serviceProducers = lib.mkOption {
internal = true;
type = lib.types.attrsOf (
lib.types.submodule {
options = {
clientId = lib.mkOption {
type = lib.types.str;
description = ''
The producer's OAuth2 client id: the one audience its
receiver's `oidc/<name>` authenticator accepts, and the one
the forwarder requests.
'';
};
port = lib.mkOption {
type = lib.types.port;
description = ''
Loopback port of the producer's receiver on the collector's
host. In the port-collision assertion with every other
receiver there.
'';
};
};
}
);
default = { };
description = ''
Swarm services whose own container forwards to this collector, keyed
by the route name: each gets a receiver `otlp/<name>`, an
authenticator `oidc/<name>` and a gateway location
`https://''${domain}/<name>/`, and its records land in the swarm
tier's pipelines with no `hive` label.
A receiver per producer because an `oidc` authenticator checks one
audience, and each service mints its token as its own client.
Declared below for every host, under no `enable`: the collector's
host renders the receiver whether or not the service runs there.
Every name must contain a fragment of
`nix/reserved-hive-fragments.nix`, so no hive can own its components.
'';
};
storeProducerName = lib.mkOption {
type = lib.types.str;
readOnly = true;
@ -346,6 +388,20 @@ in
};
};
config.services.hyperhive.swarm.otel.serviceProducers = {
# The forwarder in ./swarm-bao.nix.
${cfg.storeProducerName} = {
clientId = baoCfg.otel.clientId;
port = cfg.storeProducerPort;
};
# The forwarder in ./swarm-nats.nix, which addresses this route by the
# queue container's name.
swarm-nats = {
clientId = hyperhiveCfg.swarm.nats.clientId;
port = lib.mkDefault 4392;
};
};
# Not under any `enable`: see `publishedScrapeTargets` for why every host
# declares every entry.
#

145
nix/host-modules/swarm-otel.nix
View file

@ -41,21 +41,21 @@ let
# both sites) is the only way the two can't drift apart.
swarmTierName = cfg.producerName;
# The SECOND swarm-tier producer: the collector inside the secret store's
# container (`swarm-bao.nix`). It gets an owner of its own rather than
# pushing into `swarmTierName`'s route, because the audience that route
# checks is `swarm-controller`'s own client id — one identity per principal,
# so a second principal brings its own client, its own audience and
# therefore its own authenticator and receiver.
# The other swarm-tier producers: the forwarder inside each swarm service's
# own container (`serviceProducers` in ./swarm-otel-service.nix). Each gets
# an owner of its own rather than pushing into `swarmTierName`'s route,
# because the audience that route checks is `swarm-controller`'s own client
# id — one identity per principal, so each principal brings its own client,
# its own audience and therefore its own authenticator and receiver.
#
# Named after the store's container, so `otlp/swarm-bao` reads as the thing
# that pushes into it. No `reserved-names.nix` entry is needed for it the
# way `swarm` has one: the name CONTAINS `swarm`, which
# Named after the service's container, so `otlp/swarm-bao` reads as the
# thing that pushes into it. No `reserved-names.nix` entry is needed the way
# `swarm` has one: each name CONTAINS `swarm`, which
# `nix/reserved-hive-fragments.nix` forbids as a substring of any hive name,
# so no hive can ever own these components. The assertion below checks that
# fragment is still listed rather than trusting it.
storeProducerName = baoCfg.machine;
storeClientId = baoCfg.otel.clientId;
serviceProducers = cfg.serviceProducers;
serviceProducerNames = lib.attrNames serviceProducers;
reservedHiveFragments = import ../reserved-hive-fragments.nix;
# Every `<owner>` no hive may take. Read from `nix/reserved-names.nix`, the
@ -206,16 +206,16 @@ let
# `--link-journal=try-guest`, which puts the journal inside the container and
# leaves the host with a symlink into that container's transient root: a
# reader here cannot follow it, and it dangles the moment the container
# stops. Every container block except `swarm-bao` therefore sets
# stops. Every container block except `swarm-bao` and `swarm-nats` therefore sets
# `extraFlags = [ "--link-journal=host" ]`, which the invocation expands
# AFTER the hardcoded flag, so the files land here under their own
# machine-id subdirectory and are bind-mounted into the guest instead.
#
# ⚠️ Drop that flag from one of THOSE containers and this receiver silently
# stops seeing it — no error, just a unit that never appears in the store.
# `swarm-bao` is the one exception on purpose: its journal stays inside the
# container for its own collector instead (see `swarm-bao.nix`), so it was
# never one of the units this receiver could see either.
# `swarm-bao` and `swarm-nats` are the exceptions on purpose: their journals
# stay inside the container for its own collector instead (see
# `serviceProducers`), so this receiver never sees their units.
hostJournalDir = "/var/log/journal";
# Each store's OTLP route, by domain. One binding because the same string is
@ -380,14 +380,16 @@ in
# request reaches the receiver, which knows nothing about the
# path it was found at.
"/${swarmTierName}/".proxyPass = "http://127.0.0.1:${toString cfg.producerPort}/";
# The secret store's forwarder, on its own route for its own
# audience — same trailing slashes, same reason.
"/${storeProducerName}/".proxyPass = "http://127.0.0.1:${toString cfg.storeProducerPort}/";
# There is no swarm-wide inbox, and a closed door is the honest
# description of that. Every other route into this collector
# belongs to exactly one hive or the swarm-tier producer above.
# belongs to exactly one hive or one swarm-tier producer.
"/".return = "404";
};
}
# Each service forwarder, on its own route for its own audience —
# same trailing slashes, same reason.
// lib.mapAttrs' (
name: p: lib.nameValuePair "/${name}/" { proxyPass = "http://127.0.0.1:${toString p.port}/"; }
) serviceProducers;
};
# The identities this tier authenticates against. Stated rather than
@ -673,21 +675,21 @@ in
'';
}
{
# The store forwarder's components are named after a container whose
# name merely CONTAINS the reserved word, so what protects them is
# the substring list rather than the equality one. Same reason the
# The service forwarders' components are named after containers
# whose names merely CONTAIN the reserved word, so what protects them
# is the substring list rather than the equality one. Same reason the
# assertion above exists: the list lives in another file, and a guard
# whose subject can be edited elsewhere has to assert its own case is
# still covered.
assertion = lib.any (f: lib.hasInfix f storeProducerName) reservedHiveFragments;
assertion = lib.all (n: lib.any (f: lib.hasInfix f n) reservedHiveFragments) serviceProducerNames;
message = ''
nix/reserved-hive-fragments.nix forbids no substring of
'${storeProducerName}', which the swarm collector needs kept out of
the hive namespace: it names components `<kind>/<owner>`, so a hive
called '${storeProducerName}' would replace the secret store
nix/reserved-hive-fragments.nix forbids no substring of one of
${lib.concatStringsSep ", " serviceProducerNames}, which the swarm
collector needs kept out of the hive namespace: it names components
`<kind>/<owner>`, so a hive with that name would replace a service
forwarder's own receiver and pipeline entry.
Put the fragment back, or give the store's container a name one of
Put the fragment back, or give the service's route a name one of
them covers.
'';
}
@ -776,9 +778,9 @@ in
others = [
cfg.telemetryPort
cfg.producerPort
cfg.storeProducerPort
otelCfg.collector.port
];
]
++ map (p: p.port) (lib.attrValues serviceProducers);
all = derived ++ others;
in
lib.length (lib.unique all) == lib.length all;
@ -922,15 +924,18 @@ in
auth.authenticator = "oidc/${swarmTierName}";
};
}
# The secret store's forwarder, which is a swarm-tier
# producer with a principal of its own — a receiver per
# audience, because an `oidc` authenticator checks one.
// {
"otlp/${storeProducerName}".protocols.http = {
endpoint = "127.0.0.1:${toString cfg.storeProducerPort}";
auth.authenticator = "oidc/${storeProducerName}";
};
}
# Each service forwarder, a swarm-tier producer with a
# principal of its own — a receiver per audience, because an
# `oidc` authenticator checks one.
// lib.mapAttrs' (
name: p:
lib.nameValuePair "otlp/${name}" {
protocols.http = {
endpoint = "127.0.0.1:${toString p.port}";
auth.authenticator = "oidc/${name}";
};
}
) serviceProducers
# MERGED with the per-hive receivers, never assigned over
# them. A plain assignment here would drop every hive's
# receiver and still render a valid config that starts
@ -1128,13 +1133,10 @@ in
# same reasoning as its receiver above: `otlp/${swarmTierName}`
# always exists, so the authenticator it names must too, or an
# extension-not-listed startup failure follows every deploy.
++ [
"oidc/${swarmTierName}"
# Unconditional for the same reason, and one level sharper:
# the store runs in every swarm, so its forwarder's
# receiver is as permanent as the swarm tier's own.
"oidc/${storeProducerName}"
]
++ [ "oidc/${swarmTierName}" ]
# Unconditional for the same reason: a service forwarder's
# receiver exists whether or not the service runs on this host.
++ map (n: "oidc/${n}") serviceProducerNames
# The push side's authenticators. Same rule as above: one an
# exporter names but this list omits is INERT — the collector
# starts clean and pushes unauthenticated.
@ -1178,17 +1180,16 @@ in
"metrics/${swarmTierName}" = {
receivers = [
"otlp/${swarmTierName}"
# The store forwarder's receiver joins the swarm
# tier's pipeline rather than bringing one of its own:
# the stamp it needs is `resource/${swarmTierName}`'s
# exact stamp — the swarm's name and deliberately no
# `hive` — and a second pipeline would be that same
# processor under a second name to keep in step. What
# a producer is separated FOR is the credential it
# proves, and that is the receiver's job, not the
# pipeline's.
"otlp/${storeProducerName}"
]
# The service forwarders' receivers join the swarm
# tier's pipeline rather than bringing their own: the
# stamp they need is `resource/${swarmTierName}`'s exact
# stamp — the swarm's name and deliberately no `hive` —
# and a second pipeline would be that same processor
# under a second name to keep in step. What a producer is
# separated FOR is the credential it proves, and that is
# the receiver's job, not the pipeline's.
++ map (n: "otlp/${n}") serviceProducerNames
++ lib.optional (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) "prometheus";
processors = [ "resource/${swarmTierName}" ];
exporters = exporterNames;
@ -1227,14 +1228,11 @@ in
# `hive` value to put on them.
// lib.optionalAttrs collectLogs {
"logs/${swarmTierName}" = {
# This host's own journal, and the store container's,
# which arrives over the receiver above rather than off
# a disk — the store's forwarder reads a journal no
# reader on another host can see.
receivers = [
"journald"
"otlp/${storeProducerName}"
];
# This host's own journal, and each service container's,
# which arrives over its forwarder's receiver above
# rather than off a disk — each forwarder reads a journal
# no reader on another host can see.
receivers = [ "journald" ] ++ map (n: "otlp/${n}") serviceProducerNames;
processors = [
"filter/exclude-user-sessions"
"resource/${swarmTierName}"
@ -1323,18 +1321,19 @@ in
audience = config.services.hyperhive.swarm.controller.queueClientId;
};
}
# The secret store forwarder's own authenticator. Same shape
# Each service forwarder's own authenticator. Same shape
# again, and the audience is that principal's own client id
# — the self-referential form `swarm-authelia.nix` registers
# a hive client under and `swarm-controller` uses above,
# read from the option `glue-swarm-bao-otel-oidc-client.nix`
# registers so the two ends cannot spell it differently.
// {
"oidc/${storeProducerName}" = {
# read from the option the client's registration reads so
# the two ends cannot spell it differently.
// lib.mapAttrs' (
name: p:
lib.nameValuePair "oidc/${name}" {
issuer_url = hyperhiveCfg.swarm.authelia.url;
audience = storeClientId;
};
}
audience = p.clientId;
}
) serviceProducers
# The push side. Opposite direction to every `oidc/*` above —
# those VALIDATE a token arriving; these OBTAIN one to send.
# Hence `oauth2client` rather than `oidc`, and hence a client