swarm-otel: collect only the units the swarm's services declare

The journald receiver was configured with a directory and no filter, so
the swarm's log store received every unit on the host that runs the
collector. On a hive whose services live on a workstation that includes
the operator's desktop session, in a store every swarm operator can read.

The receiver has no system-only switch and its `matches` field is an
allowlist too, so what the swarm collects has to be stated rather than
excluded. Each service module names its own units: a service that is not
running contributes nothing, and one added later arrives declared.

An empty list is fail-open — the receiver renders no filter at all and
reads everything — so it is asserted against.
This commit is contained in:
atlas 2026-08-24 21:54:45 +02:00 committed by mara
commit 4336436457
11 changed files with 145 additions and 14 deletions

View file

@ -136,6 +136,14 @@ in
];
config = lib.mkIf cfg.enable {
# The daemon that owns every container on this hive, and the helper it
# delegates its root operations to. An agent asking why a container did
# not come up is asking about one of these two.
services.hyperhive.swarm.otel.journaldUnits = [
"hive-c0re"
"hive-priv"
];
assertions = [
{
# The pinned claude reaches agents as a bare path, so this

View file

@ -420,6 +420,14 @@ in
};
config = lib.mkIf config.services.hyperhive.enable {
# Same principle as the vhost below — this service's own surface lives
# with the service. The SSO source is named alongside forgejo because
# its failure mode is a login that silently falls back, not an error.
services.hyperhive.swarm.otel.journaldUnits = [
"forgejo"
"forgejo-sso-source"
];
# This service's own gateway surface: the vhost that fronts it and
# the name the hive resolver answers for. Declared here rather than
# in the gateway so the forge's public face lives with the forge —

View file

@ -115,6 +115,18 @@ in
# file's `let` and are not option surface.
services.hyperhive.gateway.lib = vhostLib;
# Every request to every hyperhive service passes through here, so this
# is the one unit that can say a service was unreachable rather than
# merely quiet. Named even on hives that run no swarm collector: the
# option is inert unless one is collecting on this host.
#
# dnsmasq alongside it for the same reason one level down: a name that
# stops resolving presents as every client timing out at once.
services.hyperhive.swarm.otel.journaldUnits = [
"nginx"
"dnsmasq"
];
assertions = [
{
assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null);

View file

@ -983,6 +983,13 @@ in
# bridge at that wrong answer.
services.hyperhive.gateway.localNames = [ cfg.domain ];
# The bridge as well as authelia: it is the half that writes the identity
# store, and its refusals are returned to callers as a bare 401.
services.hyperhive.swarm.otel.journaldUnits = [
"authelia"
"swarm-authelia-bridge"
];
# Declared here rather than in the collector's module, per the option's
# own rule: an entry exists only where the service that named it runs.
#

View file

@ -138,6 +138,13 @@ in
};
config = lib.mkIf (hyperhiveCfg.enable && cfg.autoConfigure) {
# A CA that fails to issue is invisible until something makes a TLS call
# hours later, so these two oneshots are worth more than most services.
services.hyperhive.swarm.otel.journaldUnits = [
"swarm-ca"
"swarm-services-ca"
];
systemd.services.swarm-ca = {
description = "Generate the swarm root CA when absent";
wantedBy = [ "multi-user.target" ];

View file

@ -443,6 +443,13 @@ in
};
config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
# The daemon and the oneshot that mints its credential — the second one
# failing leaves the first running and unable to authenticate anywhere.
services.hyperhive.swarm.otel.journaldUnits = [
"swarm-controller"
"swarm-controller-credential"
];
users.users.swarm-controller = {
isSystemUser = true;
group = "swarm-controller";

View file

@ -315,6 +315,14 @@ in
# hive declaring the vhost would answer for a service it does not have.
services.hyperhive.gateway.localNames = [ cfg.domain ];
# The secret oneshots as well as grafana itself: each runs before it and
# fails in ways grafana then reports only as a login that does not work.
services.hyperhive.swarm.otel.journaldUnits = [
"grafana"
"swarm-grafana-oidc-secret"
"swarm-grafana-secret-key"
];
services.hyperhive.swarm.controller.links = [
{
label = "Grafana";

View file

@ -408,6 +408,13 @@ in
};
config = lib.mkIf cfg.enable {
# The responder as well as the server: a denial reaches the client as a
# timeout, so the server's own log is the only place it is an error.
services.hyperhive.swarm.otel.journaldUnits = [
"nats"
"swarm-nats-auth"
];
assertions = [
{
# Fail at EVAL, not at boot: a queue that comes up unable to

View file

@ -336,6 +336,32 @@ in
'';
};
journaldUnits = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
example = [ "nginx" ];
description = ''
systemd units whose journal this collector ships to the swarm's
log store. Nothing outside this list is collected.
Every module that defines a unit worth reading swarm-wide adds
its own names here, rather than one list naming them all: a
service that is not running contributes nothing, and a service
added later arrives with its units already declared. A central
list would be a second place that has to know which services
exist, and it would go stale in the direction that hides a
service's logs rather than the one that shows too many.
Names are matched as journald `_SYSTEMD_UNIT` values, so an
in-container unit is named exactly as it is inside its container
`authelia`, not `container@swarm-authelia`.
A name that matches nothing is not an error anywhere. The unit
simply never appears in the store, which looks the same as a
service that had nothing to say.
'';
};
clientId = lib.mkOption {
type = lib.types.str;
readOnly = true;
@ -362,6 +388,12 @@ in
# vhost would answer for a service it does not have.
services.hyperhive.gateway.localNames = [ cfg.domain ];
# This collector reads its own journal, so a pipeline that stops
# delivering says so in the store it stopped delivering to. That is
# less circular than it sounds: the failure that matters here is a
# single hive's receiver refusing pushes, not the process dying.
services.hyperhive.swarm.otel.journaldUnits = [ "opentelemetry-collector" ];
# OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as
# every sibling swarm service: TLS terminates here, then plain http to
# the co-located container over loopback (shared netns, like the store
@ -518,6 +550,24 @@ in
metrics store.
'';
}
{
# An empty allowlist is not an empty collection: the receiver
# renders no unit filter at all and reads the host's entire
# journal, an operator's desktop session included. Fail-open, and
# silent — so the one configuration that must not be reachable is
# "logs are being shipped and nothing said which".
assertion = !collectLogs || cfg.journaldUnits != [ ];
message = ''
The swarm collector has a log destination but
services.hyperhive.swarm.otel.journaldUnits is empty. An empty
list is not "collect nothing" it is no filter at all, and this
collector would ship every unit on the host, including any
operator user session, to a store the whole swarm can read.
Name the units worth collecting, or turn off log collection by
leaving both log destinations unset.
'';
}
{
# Without a roster there are no receivers at all, so this
# collector would listen on nothing while looking configured.
@ -825,23 +875,32 @@ in
}
) cfg.publishedScrapeTargets;
}
# The whole journal directory, deliberately unfiltered.
# The host's whole journal directory, read through a unit
# allowlist.
#
# A filter restricted to the swarm's own containers was the
# obvious shape and is the wrong one: the host units are where
# the incidents live — the gateway's nginx, the core daemon,
# dnsmasq — and none of them is a swarm container. Filtering to
# `swarm-*` would exclude the single most-needed source.
# The directory is the host's rather than the swarm
# containers', because the host units are where the incidents
# live — the gateway's nginx, the core daemon, dnsmasq — and
# none of them is a swarm container. A source restricted to
# `swarm-*` would exclude the most-needed one.
#
# Nor would a list of container names stay true: it is a second
# place that has to know which services exist, and it goes stale
# silently the next time one is added. Attribution does not need
# it — journald's `_HOSTNAME` / `_SYSTEMD_UNIT` / `_MACHINE_ID`
# are written by journald rather than by the logging process, so
# a reader can tell the origins apart without this collector
# deciding for them.
# But that directory also holds every other unit on the host,
# including an operator's `user@.service` session on a hive
# that runs its services on a workstation. The allowlist is
# what separates "this host's services" from "this host", and
# only a per-unit one can: the receiver has no system-only
# switch, and its `matches` field is an allowlist too, so
# "everything except user sessions" is not expressible.
#
# Attribution is journald's either way — `_HOSTNAME`,
# `_SYSTEMD_UNIT` and `_MACHINE_ID` are written by journald
# rather than by the logging process, so a reader can still
# tell the origins apart.
// lib.optionalAttrs collectLogs {
journald.directory = hostJournalDir;
journald = {
directory = hostJournalDir;
units = cfg.journaldUnits;
};
};
exporters =

View file

@ -147,6 +147,10 @@ in
# `swarm-victoriametrics.nix`).
services.hyperhive.gateway.localNames = [ cfg.domain ];
# Declared here rather than in the collector, so this store's logs are
# collected because it runs, not because a list elsewhere remembered it.
services.hyperhive.swarm.otel.journaldUnits = [ "victorialogs" ];
services.hyperhive.swarm.controller.links = [
{
label = "Logs";

View file

@ -105,6 +105,10 @@ in
# have.
services.hyperhive.gateway.localNames = [ cfg.domain ];
# Declared here rather than in the collector, so this store's logs are
# collected because it runs, not because a list elsewhere remembered it.
services.hyperhive.swarm.otel.journaldUnits = [ "victoriametrics" ];
services.hyperhive.swarm.controller.links = [
{
label = "Metrics";