From 790e242e8c9ebad15c5461461eb2047bbfcff033 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 1 Nov 2026 16:00:44 +0100 Subject: [PATCH] leave squid's and hive-c0re's journals to the swarm tier when both run on one host --- docs/scheduler/observability.md | 4 +++- nix/host-modules/hive-squid.nix | 5 ++++- nix/host-modules/otel.nix | 21 ++++++++++++++------- nix/module-eval/hive-otel.nix | 23 +++++++++++++++++++++++ 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 4b069f18..62bc0ea2 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -376,10 +376,12 @@ this change removed them in favour of the log line. The hive tier's own collector runs a `journald` receiver scoped to the units in `services.hyperhive.otel.journalUnits`: `hive-c0re.service`, the `container@*.service` -units (the start-time rootfs check lines; a host with the swarm collector reads them already), plus +units (the start-time rootfs check lines), plus `squid.service` and `squid-selfcheck.service` (see [Egress proxy](../networking/network.md#egress-proxy-squid)). A host that runs the cache without the swarm collector adds `swarm-bao-attic-secret.service` and `swarm-bao-attic-tokens.service`. +A host that also runs the swarm collector names none of these: its whole-host journal receiver +ships them already, and the hive collector then has no `journald` receiver. No bind mount is needed. Unlike an agent's journal, this collector already shares a filesystem with the units it reads, because it's a host systemd service rather than a container. It forwards the crash log line alongside the diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix index c7dbc73c..db122586 100644 --- a/nix/host-modules/hive-squid.nix +++ b/nix/host-modules/hive-squid.nix @@ -449,7 +449,10 @@ in services.hyperhive.network.enable = lib.mkDefault true; services.hyperhive.otel.scrapeTargets.squid = exporterListen; - services.hyperhive.otel.journalUnits = [ + # A host that also runs the swarm collector ships its whole journal + # already (./swarm-otel.nix), so naming the units there would send each + # line twice. + services.hyperhive.otel.journalUnits = lib.optionals (!hyperhiveCfg.deploy.swarm-otel.enable) [ "squid.service" "squid-selfcheck.service" ]; diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index edcc615d..06441808 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -348,11 +348,13 @@ in # and strict refusals (`hive-priv check-rootfs`, an `ExecStartPre=` of # the unit) and nspawn's own start failures. A host that also runs the # swarm collector ships its whole journal already (./swarm-otel.nix), so - # naming the containers there would send each line twice. - services.hyperhive.otel.journalUnits = [ - "hive-c0re.service" - ] - ++ lib.optionals (!config.services.hyperhive.deploy.swarm-otel.enable) [ "container@*.service" ]; + # naming either there would send each line twice. + services.hyperhive.otel.journalUnits = + lib.optionals (!config.services.hyperhive.deploy.swarm-otel.enable) + [ + "hive-c0re.service" + "container@*.service" + ]; services.opentelemetry-collector = { enable = true; @@ -492,6 +494,11 @@ in # and a hive host can be someone's workstation too, so an # allowlist is load-bearing here rather than a tidiness # preference. + # + # Absent when no unit is named: a journald receiver with an + # empty `units` list reads every unit, not none. + } + // lib.optionalAttrs (otel.journalUnits != [ ]) { journald = { directory = "/var/log/journal"; storage = "file_storage"; @@ -667,8 +674,8 @@ in service.pipelines.logs = { receivers = [ "otlp" - "journald" - ]; + ] + ++ lib.optional (otel.journalUnits != [ ]) "journald"; # `resourcedetection` and NOT `deltatocumulative`: that one is # a metrics-only processor, and naming it here is a startup # failure rather than a no-op. Host identity applies to both diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 2ad614bf..0a0133e8 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -59,6 +59,13 @@ let # credential for. hiveOtelBare = hiveWithController { }; + # The swarm collector on the same host: its whole-host journal receiver + # already ships what the hive tier would name. + hiveOtelWithSwarm = hiveWithController { + otel.clientSecretFile = "/var/lib/hive-otel-oidc/hive-h1.secret"; + deploy.swarm-otel.enable = true; + }; + hiveOtelFetch = machine: machine.systemd.services.swarm-bao-hive-otel-oidc; hiveOtelPipelines = hiveOtel.services.opentelemetry-collector.settings.service.pipelines; @@ -469,6 +476,22 @@ let "squid.service" ]; } + { + # Control: `hiveOtel` above names the units, so the absences here are not + # a misspelled lookup. With no unit named the receiver is dropped, since + # a journald receiver with an empty `units` list reads every unit. + name = "a host that also runs the swarm collector leaves the journal of squid, hive-c0re and the containers to it"; + ok = + let + s = hiveOtelWithSwarm.services.opentelemetry-collector.settings; + in + hiveOtelWithSwarm.services.hyperhive.otel.journalUnits == [ ] + && hiveOtelWithSwarm.services.squid.enable + && !(s.receivers ? journald) + && !(lib.elem "journald" s.service.pipelines.logs.receivers) + && lib.elem "otlp" s.service.pipelines.logs.receivers + && hiveOtelSettings.receivers ? journald; + } { name = "the hive collector scrapes the squid exporter on loopback"; ok =