feat(swarm-otel): collect the host journal into the swarm's log store

A journald receiver reading the host's journal directory, a logs pipeline
stamped with the swarm tier's own resource processor, and an otlphttp
exporter pointed at VictoriaLogs. All four parts are conditional on the log
store being enabled, so a swarm without one renders exactly as before.

The host's directory is enough to see every container: nspawn links a
non-ephemeral container's journal guest-side, so the files live on the host
under the container's machine-id, and journalctl descends into those
subdirectories. Measured, along with the fixed systemd-journal gid that makes
the group grant meaningful across the bind mount.

nix/host-modules/swarm-otel.nix

@@ -32,6 +32,7 @@ let
   swarmCfg = config.services.hyperhive.swarm;
   otelCfg = config.services.hyperhive.otel;
   vmCfg = config.services.hyperhive.swarm.victoriametrics;
+  vlCfg = config.services.hyperhive.swarm.victorialogs;
   hyperhiveCfg = config.services.hyperhive;
   gatewayCfg = hyperhiveCfg.gateway;
   swarmDomain = hyperhiveCfg.swarm.domain;
@@ -127,6 +128,18 @@ let
   # both "the label is absent" and "the label says unknown".
   swarmDisplayName = if hyperhiveCfg.swarm.name == null then "unknown" else hyperhiveCfg.swarm.name;
 
+  # Where the journal lives on BOTH sides of the bind mount below — one string
+  # because the receiver reads the path it is mounted at, and two spellings of
+  # one path is a mount that succeeds and a receiver that finds nothing.
+  #
+  # 🔑 Why the host's directory is enough to see every container: nspawn is
+  # invoked with `--link-journal=try-guest` for every non-ephemeral container,
+  # so a container's journal FILES live here, under its own machine-id
+  # subdirectory, and are bind-mounted into the guest rather than the other way
+  # round. Measured: `journalctl -D` on this parent directory descends into
+  # those subdirectories, so one reader covers the host and every container.
+  hostJournalDir = "/var/log/journal";
+
   # One list, read by every pipeline: the per-hive pipelines fan out to
   # exactly the same destinations as the single pipeline they replace.
   # Written once because "which exporters" is a property of this tier, not
@@ -654,7 +667,17 @@ in
         }
         # The public hive CA, read-only — only when something in here
         # actually verifies a swarm-service name.
-        // caTrust.bindMount;
+        // caTrust.bindMount
+        # The host's journal, read-only, and only when there is a log store to
+        # write to. Read-only is the whole security posture of this mount: the
+        # collector has no business writing to a journal, and a collector that
+        # cannot write cannot corrupt the record it is reporting on.
+        // lib.optionalAttrs vlCfg.enable {
+          ${hostJournalDir} = {
+            hostPath = hostJournalDir;
+            isReadOnly = true;
+          };
+        };
 
       config =
         { ... }:
@@ -772,6 +795,24 @@ in
                         };
                       }
                     ) cfg.publishedScrapeTargets;
+                }
+                # The whole journal directory, deliberately unfiltered.
+                #
+                # A filter restricted to the swarm's own containers was the
+                # obvious shape and is the wrong one: the host units are where
+                # the incidents live — the gateway's nginx, the core daemon,
+                # dnsmasq — and none of them is a swarm container. Filtering to
+                # `swarm-*` would exclude the single most-needed source.
+                #
+                # Nor would a list of container names stay true: it is a second
+                # place that has to know which services exist, and it goes stale
+                # silently the next time one is added. Attribution does not need
+                # it — journald's `_HOSTNAME` / `_SYSTEMD_UNIT` / `_MACHINE_ID`
+                # are written by journald rather than by the logging process, so
+                # a reader can tell the origins apart without this collector
+                # deciding for them.
+                // lib.optionalAttrs vlCfg.enable {
+                  journald.directory = hostJournalDir;
                 };
 
               exporters =
@@ -797,6 +838,22 @@ in
                     headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}";
                   }
                   // lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; };
+                }
+                // lib.optionalAttrs vlCfg.enable {
+                  # `logs_endpoint`, NOT `endpoint`, for exactly the reason the
+                  # metrics exporter above spells out — and the trap is worse
+                  # here, because the two stores' OTLP routes differ. `endpoint`
+                  # is a base the exporter appends `/v1/logs` to; VictoriaLogs
+                  # serves `/insert/opentelemetry/v1/logs`, which is also not
+                  # the metrics store's `/opentelemetry/api/v1/push`. Both
+                  # spellings pass `otelcol validate`.
+                  #
+                  # ⚠️ And the store cannot tell you either: a right path, a
+                  # wrong path and a nonsense path all answer 400. Only its log
+                  # distinguishes them — the real route complains about the
+                  # encoding, everything else says "unsupported path requested".
+                  "otlphttp/victorialogs".logs_endpoint =
+                    "http://127.0.0.1:${toString vlCfg.port}/insert/opentelemetry/v1/logs";
                 };
 
               # Moves this collector's self-metrics off the built-in
@@ -850,6 +907,21 @@ in
                     processors = [ "resource/${swarmTierName}" ];
                     exporters = exporterNames;
                   };
+                }
+                # Logs go to the swarm's store and NOT to `exporterNames`, so
+                # they do not follow the metrics upstream. That asymmetry is
+                # deliberate: an operator who configured an upstream endpoint
+                # agreed to ship metrics there, and quietly adding their logs to
+                # the same hop is a disclosure decision, not a symmetry fix.
+                # Same stamp as the scraped swarm services, for the same reason:
+                # a host's logs belong to the swarm, and there is no honest
+                # `hive` value to put on them.
+                // lib.optionalAttrs vlCfg.enable {
+                  "logs/${swarmTierName}" = {
+                    receivers = [ "journald" ];
+                    processors = [ "resource/${swarmTierName}" ];
+                    exporters = [ "otlphttp/victorialogs" ];
+                  };
                 };
             }
             // {
@@ -934,7 +1006,13 @@ in
                 # any one hive, so there is no honest value to put there — and
                 # an invented one (a sentinel, the local hive's name) would be
                 # queried as though it meant something.
-                // lib.optionalAttrs (cfg.scrapeTargets != { }) {
+                #
+                # ⚠️ Emitted for the LOG pipeline too, not just the scraped one:
+                # a processor a pipeline names but the config does not define is
+                # a collector that refuses to start, and enabling the log store
+                # without declaring a scrape target is a perfectly ordinary
+                # config.
+                // lib.optionalAttrs (cfg.scrapeTargets != { } || vlCfg.enable) {
                   "resource/${swarmTierName}".attributes = [
                     {
                       # The metric LABEL, a different namespace from the
@@ -966,6 +1044,18 @@ in
             # scrapes nothing published — the empty case is the shipped one.
             // lib.optionalAttrs (cfg.publishedScrapeTargets != { }) {
               LoadCredential = [ "${collectorCredentialId}:${collectorSecretInContainer}" ];
+            }
+            # Journal files are `0640 root:systemd-journal`, and this unit runs
+            # with upstream's `DynamicUser = true` — so there is no stable uid to
+            # grant, and the group is the only way in.
+            #
+            # 🔑 That works across the bind mount because `systemd-journal` is
+            # gid **62**, FIXED in nixpkgs' `ids.nix` rather than allocated per
+            # host. A dynamically allocated gid would mean the host's ownership
+            # named a different group inside the container, and the failure
+            # would be a receiver that starts cleanly and reads nothing.
+            // lib.optionalAttrs vlCfg.enable {
+              SupplementaryGroups = [ "systemd-journal" ];
             };
         };
     };