diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 6409b04a7..0e7468b3a 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -375,7 +375,8 @@ no broker event, no per-hive dashboard banner. Both of those used to exist; this change removed them in favour of the log line. The hive tier's own collector runs a `journald` receiver scoped to the units -in `services.hyperhive.otel.journalUnits`: `hive-c0re.service`, plus +in `services.hyperhive.otel.journalUnits`: `hive-c0re.service`, the `container@*.service` +units (the start-time rootfs check lines; a host with the swarm collector reads them already), plus `squid.service` and `squid-selfcheck.service` (see [Egress proxy](../networking/network.md#egress-proxy-squid)). A host that runs the cache without the swarm collector adds `swarm-bao-attic-secret.service` and `swarm-bao-attic-tokens.service`. diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 7d89639a2..fa39ce436 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -343,8 +343,16 @@ in # not for agent containers. services.hyperhive.otel.scrapeTargets.collector = "127.0.0.1:${toString otel.telemetryPort}"; - # The unit whose crash-watch warning a swarm-level alert reads. - services.hyperhive.otel.journalUnits = [ "hive-c0re.service" ]; + # The unit whose crash-watch warning a swarm-level alert reads, and the + # container units' start-time output: the rootfs check's warn-only lines + # and strict refusals (`hive-priv check-rootfs`, an `ExecStartPre=` of + # the unit) and nspawn's own start failures. A host that also runs the + # swarm collector ships its whole journal already (./swarm-otel.nix), so + # naming the containers there would send each line twice. + services.hyperhive.otel.journalUnits = [ + "hive-c0re.service" + ] + ++ lib.optionals (!config.services.hyperhive.deploy.swarm-otel.enable) [ "container@*.service" ]; services.opentelemetry-collector = { enable = true; diff --git a/nix/module-eval/builder.nix b/nix/module-eval/builder.nix index de64b63a1..6237fcaaf 100644 --- a/nix/module-eval/builder.nix +++ b/nix/module-eval/builder.nix @@ -432,6 +432,21 @@ let atticExternal ]; } + { + # The hive collector ships the container units' start output (the rootfs + # check's lines) unless the swarm collector already reads the whole + # journal. `hive-c0re.service` is the control that the lookup finds the + # list. + name = "the hive collector ships the container units' journal only where nothing else reads the host journal"; + ok = + let + units = m: m.services.opentelemetry-collector.settings.receivers.journald.units; + glob = "container@*.service"; + in + lib.elem glob (units atticByHand) + && !(lib.elem glob (units atticHere)) + && lib.elem "hive-c0re.service" (units atticHere); + } { # Both host units log in as the cache's principal, not the hive's, and # the container reaches the key through a read-only directory. The diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index d420f730b..ec0398c70 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -455,11 +455,12 @@ let { # Squid rides this fixture because its journal and metrics reach the # swarm through this collector. The allow-list of journal units must stay - # exactly the three named: hive-c0re's (control: the list is not just - # squid's), squid's and its self-check's. - name = "the hive collector forwards the journal of squid, its self-check and hive-c0re and no other unit"; + # exactly the four named: hive-c0re's (control: the list is not just + # squid's), the container units', squid's and its self-check's. + name = "the hive collector forwards the journal of squid, its self-check, hive-c0re and the container units and no other unit"; ok = lib.sort builtins.lessThan hiveOtelSettings.receivers.journald.units == [ + "container@*.service" "hive-c0re.service" "squid-selfcheck.service" "squid.service"