From 4336436457239c8910d47a6e682d40ab3925b3fd Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 24 Aug 2026 21:54:45 +0200 Subject: [PATCH] swarm-otel: collect only the units the swarm's services declare MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The journald receiver was configured with a directory and no filter, so the swarm's log store received every unit on the host that runs the collector. On a hive whose services live on a workstation that includes the operator's desktop session, in a store every swarm operator can read. The receiver has no system-only switch and its `matches` field is an allowlist too, so what the swarm collects has to be stated rather than excluded. Each service module names its own units: a service that is not running contributes nothing, and one added later arrives declared. An empty list is fail-open — the receiver renders no filter at all and reads everything — so it is asserted against. --- nix/host-modules/hive-c0re/default.nix | 8 ++ nix/host-modules/hive-forge/default.nix | 8 ++ nix/host-modules/hive-gateway/default.nix | 12 +++ nix/host-modules/swarm-authelia.nix | 7 ++ nix/host-modules/swarm-ca.nix | 7 ++ nix/host-modules/swarm-controller.nix | 7 ++ nix/host-modules/swarm-grafana.nix | 8 ++ nix/host-modules/swarm-nats.nix | 7 ++ nix/host-modules/swarm-otel.nix | 87 ++++++++++++++++++---- nix/host-modules/swarm-victorialogs.nix | 4 + nix/host-modules/swarm-victoriametrics.nix | 4 + 11 files changed, 145 insertions(+), 14 deletions(-) diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index 129063d7..ed3c398d 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -136,6 +136,14 @@ in ]; config = lib.mkIf cfg.enable { + # The daemon that owns every container on this hive, and the helper it + # delegates its root operations to. An agent asking why a container did + # not come up is asking about one of these two. + services.hyperhive.swarm.otel.journaldUnits = [ + "hive-c0re" + "hive-priv" + ]; + assertions = [ { # The pinned claude reaches agents as a bare path, so this diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index 678dfcdc..fc0499dd 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -420,6 +420,14 @@ in }; config = lib.mkIf config.services.hyperhive.enable { + # Same principle as the vhost below — this service's own surface lives + # with the service. The SSO source is named alongside forgejo because + # its failure mode is a login that silently falls back, not an error. + services.hyperhive.swarm.otel.journaldUnits = [ + "forgejo" + "forgejo-sso-source" + ]; + # This service's own gateway surface: the vhost that fronts it and # the name the hive resolver answers for. Declared here rather than # in the gateway so the forge's public face lives with the forge — diff --git a/nix/host-modules/hive-gateway/default.nix b/nix/host-modules/hive-gateway/default.nix index 7e64ff24..e91d732c 100644 --- a/nix/host-modules/hive-gateway/default.nix +++ b/nix/host-modules/hive-gateway/default.nix @@ -115,6 +115,18 @@ in # file's `let` and are not option surface. services.hyperhive.gateway.lib = vhostLib; + # Every request to every hyperhive service passes through here, so this + # is the one unit that can say a service was unreachable rather than + # merely quiet. Named even on hives that run no swarm collector: the + # option is inert unless one is collecting on this host. + # + # dnsmasq alongside it for the same reason one level down: a name that + # stops resolving presents as every client timing out at once. + services.hyperhive.swarm.otel.journaldUnits = [ + "nginx" + "dnsmasq" + ]; + assertions = [ { assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null); diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index 5fff0a71..dc0cb3e0 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -983,6 +983,13 @@ in # bridge at that wrong answer. services.hyperhive.gateway.localNames = [ cfg.domain ]; + # The bridge as well as authelia: it is the half that writes the identity + # store, and its refusals are returned to callers as a bare 401. + services.hyperhive.swarm.otel.journaldUnits = [ + "authelia" + "swarm-authelia-bridge" + ]; + # Declared here rather than in the collector's module, per the option's # own rule: an entry exists only where the service that named it runs. # diff --git a/nix/host-modules/swarm-ca.nix b/nix/host-modules/swarm-ca.nix index 684b5de6..d522ae57 100644 --- a/nix/host-modules/swarm-ca.nix +++ b/nix/host-modules/swarm-ca.nix @@ -138,6 +138,13 @@ in }; config = lib.mkIf (hyperhiveCfg.enable && cfg.autoConfigure) { + # A CA that fails to issue is invisible until something makes a TLS call + # hours later, so these two oneshots are worth more than most services. + services.hyperhive.swarm.otel.journaldUnits = [ + "swarm-ca" + "swarm-services-ca" + ]; + systemd.services.swarm-ca = { description = "Generate the swarm root CA when absent"; wantedBy = [ "multi-user.target" ]; diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index f2708b5f..8a94855f 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -443,6 +443,13 @@ in }; config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) { + # The daemon and the oneshot that mints its credential — the second one + # failing leaves the first running and unable to authenticate anywhere. + services.hyperhive.swarm.otel.journaldUnits = [ + "swarm-controller" + "swarm-controller-credential" + ]; + users.users.swarm-controller = { isSystemUser = true; group = "swarm-controller"; diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 5de45e0f..a1dfb6ee 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -315,6 +315,14 @@ in # hive declaring the vhost would answer for a service it does not have. services.hyperhive.gateway.localNames = [ cfg.domain ]; + # The secret oneshots as well as grafana itself: each runs before it and + # fails in ways grafana then reports only as a login that does not work. + services.hyperhive.swarm.otel.journaldUnits = [ + "grafana" + "swarm-grafana-oidc-secret" + "swarm-grafana-secret-key" + ]; + services.hyperhive.swarm.controller.links = [ { label = "Grafana"; diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 5c97dccc..d187872c 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -408,6 +408,13 @@ in }; config = lib.mkIf cfg.enable { + # The responder as well as the server: a denial reaches the client as a + # timeout, so the server's own log is the only place it is an error. + services.hyperhive.swarm.otel.journaldUnits = [ + "nats" + "swarm-nats-auth" + ]; + assertions = [ { # Fail at EVAL, not at boot: a queue that comes up unable to diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index f87d9c40..60a8faa3 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -336,6 +336,32 @@ in ''; }; + journaldUnits = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ "nginx" ]; + description = '' + systemd units whose journal this collector ships to the swarm's + log store. Nothing outside this list is collected. + + Every module that defines a unit worth reading swarm-wide adds + its own names here, rather than one list naming them all: a + service that is not running contributes nothing, and a service + added later arrives with its units already declared. A central + list would be a second place that has to know which services + exist, and it would go stale in the direction that hides a + service's logs rather than the one that shows too many. + + Names are matched as journald `_SYSTEMD_UNIT` values, so an + in-container unit is named exactly as it is inside its container + — `authelia`, not `container@swarm-authelia`. + + ⚠️ A name that matches nothing is not an error anywhere. The unit + simply never appears in the store, which looks the same as a + service that had nothing to say. + ''; + }; + clientId = lib.mkOption { type = lib.types.str; readOnly = true; @@ -362,6 +388,12 @@ in # vhost would answer for a service it does not have. services.hyperhive.gateway.localNames = [ cfg.domain ]; + # This collector reads its own journal, so a pipeline that stops + # delivering says so in the store it stopped delivering to. That is + # less circular than it sounds: the failure that matters here is a + # single hive's receiver refusing pushes, not the process dying. + services.hyperhive.swarm.otel.journaldUnits = [ "opentelemetry-collector" ]; + # OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as # every sibling swarm service: TLS terminates here, then plain http to # the co-located container over loopback (shared netns, like the store @@ -518,6 +550,24 @@ in metrics store. ''; } + { + # An empty allowlist is not an empty collection: the receiver + # renders no unit filter at all and reads the host's entire + # journal, an operator's desktop session included. Fail-open, and + # silent — so the one configuration that must not be reachable is + # "logs are being shipped and nothing said which". + assertion = !collectLogs || cfg.journaldUnits != [ ]; + message = '' + The swarm collector has a log destination but + services.hyperhive.swarm.otel.journaldUnits is empty. An empty + list is not "collect nothing" — it is no filter at all, and this + collector would ship every unit on the host, including any + operator user session, to a store the whole swarm can read. + + Name the units worth collecting, or turn off log collection by + leaving both log destinations unset. + ''; + } { # Without a roster there are no receivers at all, so this # collector would listen on nothing while looking configured. @@ -825,23 +875,32 @@ in } ) cfg.publishedScrapeTargets; } - # The whole journal directory, deliberately unfiltered. + # The host's whole journal directory, read through a unit + # allowlist. # - # A filter restricted to the swarm's own containers was the - # obvious shape and is the wrong one: the host units are where - # the incidents live — the gateway's nginx, the core daemon, - # dnsmasq — and none of them is a swarm container. Filtering to - # `swarm-*` would exclude the single most-needed source. + # The directory is the host's rather than the swarm + # containers', because the host units are where the incidents + # live — the gateway's nginx, the core daemon, dnsmasq — and + # none of them is a swarm container. A source restricted to + # `swarm-*` would exclude the most-needed one. # - # Nor would a list of container names stay true: it is a second - # place that has to know which services exist, and it goes stale - # silently the next time one is added. Attribution does not need - # it — journald's `_HOSTNAME` / `_SYSTEMD_UNIT` / `_MACHINE_ID` - # are written by journald rather than by the logging process, so - # a reader can tell the origins apart without this collector - # deciding for them. + # But that directory also holds every other unit on the host, + # including an operator's `user@.service` session on a hive + # that runs its services on a workstation. The allowlist is + # what separates "this host's services" from "this host", and + # only a per-unit one can: the receiver has no system-only + # switch, and its `matches` field is an allowlist too, so + # "everything except user sessions" is not expressible. + # + # Attribution is journald's either way — `_HOSTNAME`, + # `_SYSTEMD_UNIT` and `_MACHINE_ID` are written by journald + # rather than by the logging process, so a reader can still + # tell the origins apart. // lib.optionalAttrs collectLogs { - journald.directory = hostJournalDir; + journald = { + directory = hostJournalDir; + units = cfg.journaldUnits; + }; }; exporters = diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix index 4787b43e..01dc5377 100644 --- a/nix/host-modules/swarm-victorialogs.nix +++ b/nix/host-modules/swarm-victorialogs.nix @@ -147,6 +147,10 @@ in # `swarm-victoriametrics.nix`). services.hyperhive.gateway.localNames = [ cfg.domain ]; + # Declared here rather than in the collector, so this store's logs are + # collected because it runs, not because a list elsewhere remembered it. + services.hyperhive.swarm.otel.journaldUnits = [ "victorialogs" ]; + services.hyperhive.swarm.controller.links = [ { label = "Logs"; diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index 25c1f328..a991f989 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -105,6 +105,10 @@ in # have. services.hyperhive.gateway.localNames = [ cfg.domain ]; + # Declared here rather than in the collector, so this store's logs are + # collected because it runs, not because a list elsewhere remembered it. + services.hyperhive.swarm.otel.journaldUnits = [ "victoriametrics" ]; + services.hyperhive.swarm.controller.links = [ { label = "Metrics";