leave squid's and hive-c0re's journals to the swarm tier when both run on one host

docs/scheduler/observability.md

@@ -376,10 +376,12 @@ this change removed them in favour of the log line.
 
 The hive tier's own collector runs a `journald` receiver scoped to the units
 in `services.hyperhive.otel.journalUnits`: `hive-c0re.service`, the `container@*.service`
-units (the start-time rootfs check lines; a host with the swarm collector reads them already), plus
+units (the start-time rootfs check lines), plus
 `squid.service` and `squid-selfcheck.service` (see [Egress proxy](../networking/network.md#egress-proxy-squid)).
 A host that runs the cache without the swarm collector adds `swarm-bao-attic-secret.service`
 and `swarm-bao-attic-tokens.service`.
+A host that also runs the swarm collector names none of these: its whole-host journal receiver
+ships them already, and the hive collector then has no `journald` receiver.
 No bind mount is needed. Unlike an agent's journal, this collector already
 shares a filesystem with the units it reads, because it's a host systemd
 service rather than a container. It forwards the crash log line alongside the

nix/host-modules/hive-squid.nix

@@ -449,7 +449,10 @@ in
     services.hyperhive.network.enable = lib.mkDefault true;
 
     services.hyperhive.otel.scrapeTargets.squid = exporterListen;
-    services.hyperhive.otel.journalUnits = [
+    # A host that also runs the swarm collector ships its whole journal
+    # already (./swarm-otel.nix), so naming the units there would send each
+    # line twice.
+    services.hyperhive.otel.journalUnits = lib.optionals (!hyperhiveCfg.deploy.swarm-otel.enable) [
       "squid.service"
       "squid-selfcheck.service"
     ];

nix/host-modules/otel.nix

@@ -348,11 +348,13 @@ in
       # and strict refusals (`hive-priv check-rootfs`, an `ExecStartPre=` of
       # the unit) and nspawn's own start failures. A host that also runs the
       # swarm collector ships its whole journal already (./swarm-otel.nix), so
-      # naming the containers there would send each line twice.
+      # naming either there would send each line twice.
-      services.hyperhive.otel.journalUnits = [
+      services.hyperhive.otel.journalUnits =
-        "hive-c0re.service"
+        lib.optionals (!config.services.hyperhive.deploy.swarm-otel.enable)
-      ]
+          [
-      ++ lib.optionals (!config.services.hyperhive.deploy.swarm-otel.enable) [ "container@*.service" ];
+            "hive-c0re.service"
+            "container@*.service"
+          ];
 
       services.opentelemetry-collector = {
         enable = true;
@@ -492,6 +494,11 @@ in
             # and a hive host can be someone's workstation too, so an
             # allowlist is load-bearing here rather than a tidiness
             # preference.
+            #
+            # Absent when no unit is named: a journald receiver with an
+            # empty `units` list reads every unit, not none.
+          }
+          // lib.optionalAttrs (otel.journalUnits != [ ]) {
             journald = {
               directory = "/var/log/journal";
               storage = "file_storage";
@@ -667,8 +674,8 @@ in
           service.pipelines.logs = {
             receivers = [
               "otlp"
-              "journald"
+            ]
-            ];
+            ++ lib.optional (otel.journalUnits != [ ]) "journald";
             # `resourcedetection` and NOT `deltatocumulative`: that one is
             # a metrics-only processor, and naming it here is a startup
             # failure rather than a no-op. Host identity applies to both

nix/module-eval/hive-otel.nix

@@ -59,6 +59,13 @@ let
   # credential for.
   hiveOtelBare = hiveWithController { };
 
+  # The swarm collector on the same host: its whole-host journal receiver
+  # already ships what the hive tier would name.
+  hiveOtelWithSwarm = hiveWithController {
+    otel.clientSecretFile = "/var/lib/hive-otel-oidc/hive-h1.secret";
+    deploy.swarm-otel.enable = true;
+  };
+
   hiveOtelFetch = machine: machine.systemd.services.swarm-bao-hive-otel-oidc;
 
   hiveOtelPipelines = hiveOtel.services.opentelemetry-collector.settings.service.pipelines;
@@ -469,6 +476,22 @@ let
           "squid.service"
         ];
     }
+    {
+      # Control: `hiveOtel` above names the units, so the absences here are not
+      # a misspelled lookup. With no unit named the receiver is dropped, since
+      # a journald receiver with an empty `units` list reads every unit.
+      name = "a host that also runs the swarm collector leaves the journal of squid, hive-c0re and the containers to it";
+      ok =
+        let
+          s = hiveOtelWithSwarm.services.opentelemetry-collector.settings;
+        in
+        hiveOtelWithSwarm.services.hyperhive.otel.journalUnits == [ ]
+        && hiveOtelWithSwarm.services.squid.enable
+        && !(s.receivers ? journald)
+        && !(lib.elem "journald" s.service.pipelines.logs.receivers)
+        && lib.elem "otlp" s.service.pipelines.logs.receivers
+        && hiveOtelSettings.receivers ? journald;
+    }
     {
       name = "the hive collector scrapes the squid exporter on loopback";
       ok =