From e316e683c3f41f1babbb4e465b68e1ff3051cc51 Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 29 Oct 2026 12:49:03 +0100 Subject: [PATCH] ship container unit journals through the hive collector --- docs/scheduler/observability.md | 3 ++- nix/host-modules/otel.nix | 10 ++++++++++ nix/module-eval/builder.nix | 15 +++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 6409b04a..0e7468b3 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -375,7 +375,8 @@ no broker event, no per-hive dashboard banner. Both of those used to exist; this change removed them in favour of the log line. The hive tier's own collector runs a `journald` receiver scoped to the units -in `services.hyperhive.otel.journalUnits`: `hive-c0re.service`, plus +in `services.hyperhive.otel.journalUnits`: `hive-c0re.service`, the `container@*.service` +units (the start-time rootfs check lines; a host with the swarm collector reads them already), plus `squid.service` and `squid-selfcheck.service` (see [Egress proxy](../networking/network.md#egress-proxy-squid)). A host that runs the cache without the swarm collector adds `swarm-bao-attic-secret.service` and `swarm-bao-attic-tokens.service`. diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 7d89639a..9b1df939 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -346,6 +346,16 @@ in # The unit whose crash-watch warning a swarm-level alert reads. services.hyperhive.otel.journalUnits = [ "hive-c0re.service" ]; + # A container's start-time output: the rootfs check's warn-only lines + # and strict refusals (`hive-priv check-rootfs`, an `ExecStartPre=` of + # the unit) and nspawn's own start failures. `journalctl -u` takes the + # glob. A host that also runs the swarm collector ships its whole + # journal already (./swarm-otel.nix), so naming it there would send + # each line twice. + services.hyperhive.otel.journalUnits = lib.optionals ( + !config.services.hyperhive.deploy.swarm-otel.enable + ) [ "container@*.service" ]; + services.opentelemetry-collector = { enable = true; # Contrib, matching the swarm tier (./swarm-otel.nix). The upstream diff --git a/nix/module-eval/builder.nix b/nix/module-eval/builder.nix index de64b63a..6237fcaa 100644 --- a/nix/module-eval/builder.nix +++ b/nix/module-eval/builder.nix @@ -432,6 +432,21 @@ let atticExternal ]; } + { + # The hive collector ships the container units' start output (the rootfs + # check's lines) unless the swarm collector already reads the whole + # journal. `hive-c0re.service` is the control that the lookup finds the + # list. + name = "the hive collector ships the container units' journal only where nothing else reads the host journal"; + ok = + let + units = m: m.services.opentelemetry-collector.settings.receivers.journald.units; + glob = "container@*.service"; + in + lib.elem glob (units atticByHand) + && !(lib.elem glob (units atticHere)) + && lib.elem "hive-c0re.service" (units atticHere); + } { # Both host units log in as the cache's principal, not the hive's, and # the container reaches the key through a read-only directory. The