fix(swarm-otel): stop re-granting the journal group upstream already grants

nixpkgs' collector unit already sets SupplementaryGroups to systemd-journal
unconditionally, with a comment saying why. Systemd list options concatenate,
so this module's copy rendered ["systemd-journal" "systemd-journal"] and made
this a second owner of a fact upstream may later change.

The bind mount stays, since that half is genuinely ours.

nix/host-modules/default.nix

@@ -30,6 +30,7 @@
     ./swarm-otel.nix
     ./swarm-snapshot-store.nix
     ./swarm-ui.nix
+    ./swarm-victorialogs.nix
     ./swarm-victoriametrics.nix
     ./swarm-wireguard.nix
     ./swarm.nix

nix/host-modules/swarm-otel.nix

@@ -32,6 +32,7 @@ let
   swarmCfg = config.services.hyperhive.swarm;
   otelCfg = config.services.hyperhive.otel;
   vmCfg = config.services.hyperhive.swarm.victoriametrics;
+  vlCfg = config.services.hyperhive.swarm.victorialogs;
   hyperhiveCfg = config.services.hyperhive;
   gatewayCfg = hyperhiveCfg.gateway;
   swarmDomain = hyperhiveCfg.swarm.domain;
@@ -127,13 +128,41 @@ let
   # both "the label is absent" and "the label says unknown".
   swarmDisplayName = if hyperhiveCfg.swarm.name == null then "unknown" else hyperhiveCfg.swarm.name;
 
+  # Where the journal lives on BOTH sides of the bind mount below — one string
+  # because the receiver reads the path it is mounted at, and two spellings of
+  # one path is a mount that succeeds and a receiver that finds nothing.
+  #
+  # 🔑 Why the host's directory is enough to see every container: nspawn is
+  # invoked with `--link-journal=try-guest` for every non-ephemeral container,
+  # so a container's journal FILES live here, under its own machine-id
+  # subdirectory, and are bind-mounted into the guest rather than the other way
+  # round. Measured: `journalctl -D` on this parent directory descends into
+  # those subdirectories, so one reader covers the host and every container.
+  hostJournalDir = "/var/log/journal";
+
+  # The operator-configured upstream, named once: the same exporter carries
+  # every signal, so metrics and logs both reach it without a second
+  # definition.
+  upstreamExporterName = if otelCfg.protocol == "grpc" then "otlp" else "otlphttp";
+  upstreamExporters = lib.optional (otelCfg.endpoint != "") upstreamExporterName;
+
   # One list, read by every pipeline: the per-hive pipelines fan out to
   # exactly the same destinations as the single pipeline they replace.
   # Written once because "which exporters" is a property of this tier, not
   # of which hive a sample came from.
-  exporterNames =
-    lib.optional (otelCfg.endpoint != "") (if otelCfg.protocol == "grpc" then "otlp" else "otlphttp")
-    ++ lib.optional vmCfg.enable "otlphttp/victoriametrics";
+  exporterNames = upstreamExporters ++ lib.optional vmCfg.enable "otlphttp/victoriametrics";
+
+  # The same fan-out for logs, and the local store is only ONE of its
+  # destinations. A deployment that turns the swarm's log store off and keeps
+  # an upstream endpoint still collects — the store is where logs may be kept,
+  # not the reason to read the journal at all.
+  logExporterNames = upstreamExporters ++ lib.optional vlCfg.enable "otlphttp/victorialogs";
+
+  # Collect when there is anywhere to send it, and only then. A pipeline with
+  # an empty exporter list is not a quiet no-op — the collector rejects it —
+  # and reading the journal to drop it on the floor would be worse than not
+  # reading it.
+  collectLogs = logExporterNames != [ ];
 in
 {
   options.services.hyperhive.swarm.otel = {
@@ -654,7 +683,30 @@ in
         }
         # The public hive CA, read-only — only when something in here
         # actually verifies a swarm-service name.
-        // caTrust.bindMount;
+        // caTrust.bindMount
+        # The host's journal, read-only, and only when something will read it.
+        # Read-only is the whole security posture of this mount: the collector
+        # has no business writing to a journal, and one that cannot write
+        # cannot corrupt the record it is reporting on.
+        #
+        # The mount is the only half we supply. Journal files are
+        # `0640 root:systemd-journal` and the unit runs `DynamicUser`, so
+        # reading them needs that group — which upstream's own collector unit
+        # already grants unconditionally. Adding it here again is not
+        # harmless: systemd list options CONCATENATE, so a second copy renders
+        # `[ "systemd-journal" "systemd-journal" ]` and quietly becomes a
+        # second owner of a fact upstream may later change.
+        #
+        # It works across the mount because that gid is FIXED at 62 in
+        # nixpkgs' `ids.nix`. A per-host allocation would leave the host's
+        # ownership naming a different group inside the container, and the
+        # failure would be a receiver that starts cleanly and reads nothing.
+        // lib.optionalAttrs collectLogs {
+          ${hostJournalDir} = {
+            hostPath = hostJournalDir;
+            isReadOnly = true;
+          };
+        };
 
       config =
         { ... }:
@@ -772,6 +824,24 @@ in
                         };
                       }
                     ) cfg.publishedScrapeTargets;
+                }
+                # The whole journal directory, deliberately unfiltered.
+                #
+                # A filter restricted to the swarm's own containers was the
+                # obvious shape and is the wrong one: the host units are where
+                # the incidents live — the gateway's nginx, the core daemon,
+                # dnsmasq — and none of them is a swarm container. Filtering to
+                # `swarm-*` would exclude the single most-needed source.
+                #
+                # Nor would a list of container names stay true: it is a second
+                # place that has to know which services exist, and it goes stale
+                # silently the next time one is added. Attribution does not need
+                # it — journald's `_HOSTNAME` / `_SYSTEMD_UNIT` / `_MACHINE_ID`
+                # are written by journald rather than by the logging process, so
+                # a reader can tell the origins apart without this collector
+                # deciding for them.
+                // lib.optionalAttrs collectLogs {
+                  journald.directory = hostJournalDir;
                 };
 
               exporters =
@@ -797,6 +867,40 @@ in
                     headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}";
                   }
                   // lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; };
+                }
+                // lib.optionalAttrs vlCfg.enable {
+                  # `logs_endpoint`, NOT `endpoint`, for exactly the reason the
+                  # metrics exporter above spells out — and the trap is worse
+                  # here, because the two stores' OTLP routes differ. `endpoint`
+                  # is a base the exporter appends `/v1/logs` to; VictoriaLogs
+                  # serves `/insert/opentelemetry/v1/logs`, which is also not
+                  # the metrics store's `/opentelemetry/api/v1/push`. Both
+                  # spellings pass `otelcol validate`.
+                  #
+                  # ⚠️ And the store cannot tell you either: a right path, a
+                  # wrong path and a nonsense path all answer 400. Only its log
+                  # distinguishes them — the real route complains about the
+                  # encoding, everything else says "unsupported path requested".
+                  # 🔑 THE TWO QUERY PARAMETERS ARE NOT TUNING — without the
+                  # first one this pipeline fills the store with records that
+                  # cannot be searched by message text, which is the only
+                  # reason to collect logs at all.
+                  #
+                  # The journald receiver leaves the OTLP *body* empty and
+                  # carries the entry as a map of journal fields, so the store
+                  # has no message to index and writes the literal placeholder
+                  # `missing _msg field` into `_msg` on EVERY record. Nothing
+                  # errors: ingest returns 200, the data is all present, and a
+                  # plain search for a line that is sitting right there returns
+                  # nothing. `_msg_field` tells the store which field carries
+                  # the message. Measured, both with and without.
+                  #
+                  # `_stream_fields` is the difference between one enormous
+                  # stream for the whole host and one per unit per machine —
+                  # both low-cardinality, and both fields journald sets itself.
+                  "otlphttp/victorialogs".logs_endpoint =
+                    "http://127.0.0.1:${toString vlCfg.port}/insert/opentelemetry/v1/logs"
+                    + "?_msg_field=MESSAGE&_stream_fields=_HOSTNAME,_SYSTEMD_UNIT";
                 };
 
               # Moves this collector's self-metrics off the built-in
@@ -850,6 +954,22 @@ in
                     processors = [ "resource/${swarmTierName}" ];
                     exporters = exporterNames;
                   };
+                }
+                # Logs fan out exactly as metrics do: the swarm's store when it
+                # runs, the operator's upstream when one is configured, both
+                # when both. The local store is a destination rather than the
+                # reason to collect, so turning it off leaves a hive that still
+                # ships its journal to whoever aggregates across swarms.
+                #
+                # Same stamp as the scraped swarm services, for the same reason:
+                # a host's logs belong to the swarm, and there is no honest
+                # `hive` value to put on them.
+                // lib.optionalAttrs collectLogs {
+                  "logs/${swarmTierName}" = {
+                    receivers = [ "journald" ];
+                    processors = [ "resource/${swarmTierName}" ];
+                    exporters = logExporterNames;
+                  };
                 };
             }
             // {
@@ -934,7 +1054,13 @@ in
                 # any one hive, so there is no honest value to put there — and
                 # an invented one (a sentinel, the local hive's name) would be
                 # queried as though it meant something.
-                // lib.optionalAttrs (cfg.scrapeTargets != { }) {
+                #
+                # ⚠️ Emitted for the LOG pipeline too, not just the scraped one:
+                # a processor a pipeline names but the config does not define is
+                # a collector that refuses to start, and enabling the log store
+                # without declaring a scrape target is a perfectly ordinary
+                # config.
+                // lib.optionalAttrs (cfg.scrapeTargets != { } || collectLogs) {
                   "resource/${swarmTierName}".attributes = [
                     {
                       # The metric LABEL, a different namespace from the

nix/host-modules/swarm-required-services.nix

@@ -71,6 +71,13 @@ in
     victoriametrics.enable = lib.mkDefault swarmCfg.enableRequiredServices;
     grafana.enable = lib.mkDefault swarmCfg.enableRequiredServices;
 
+    # The log store, deriving from the same switch for the same reason —
+    # and deliberately in the same commit as the collector pipeline that
+    # writes to it, never before it. A store nothing writes to is worse
+    # than no store: it starts, answers queries, and returns nothing, so
+    # the first person to look concludes there were no logs.
+    victorialogs.enable = lib.mkDefault swarmCfg.enableRequiredServices;
+
     # The collector that feeds the pair above, and the only tier holding
     # the upstream credential. Same rule as the rest: once per swarm,
     # optional, and a hive that is not the service host is a *client* of

nix/host-modules/swarm-victorialogs.nix

@@ -0,0 +1,168 @@
+# The swarm's log store: one VictoriaLogs for the whole swarm, in a
+# `swarm-victorialogs` nixos-container beside the metrics store it mirrors.
+#
+# Why a store at all rather than reading journals directly: an agent can
+# verify that a unit was *launched* and never that it is *working*. The
+# container journals are not reachable from an agent, so a diagnosis stops at
+# the first component that is broken — which is precisely the component whose
+# own instrument is least likely to be legible. Collecting logs centrally
+# makes the question answerable without host access.
+#
+# It is the collector that feeds this, not the services directly: one ingest
+# point per swarm, same shape as the metrics path.
+#
+# ⚠️ NO GATEWAY VHOST, and that omission is deliberate rather than unfinished.
+# VictoriaLogs' ingest and query endpoints are unauthenticated, exactly like
+# the metrics store's — and the metrics store *is* published under a
+# resolvable name, which is an open security question rather than a settled
+# design. Publishing this one the same way would repeat that before the first
+# instance is decided. The reader is Grafana, which is on this host.
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  cfg = config.services.hyperhive.swarm.victorialogs;
+  networkCfg = config.services.hyperhive.network;
+  hyperhiveCfg = config.services.hyperhive;
+in
+{
+  options.services.hyperhive.swarm.victorialogs = {
+    enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Run the swarm's log store on this host. Off by default and not
+        derived from {option}`services.hyperhive.enable`: a swarm has one
+        log store, so enabling it is a decision about swarm topology
+        rather than about whether hyperhive is installed.
+      '';
+    };
+
+    package = lib.mkOption {
+      type = lib.types.package;
+      default = pkgs.victorialogs;
+      defaultText = lib.literalExpression "pkgs.victorialogs";
+      description = "VictoriaLogs package to run.";
+    };
+
+    machine = lib.mkOption {
+      type = lib.types.str;
+      readOnly = true;
+      default = "swarm-victorialogs";
+      description = ''
+        Container name. Read-only: the name appears in host paths and in
+        `machinectl`, so it is a fact other modules may read rather than a
+        knob.
+      '';
+    };
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      default = 9428;
+      description = ''
+        Loopback port the store listens on. Upstream's default, kept
+        because there is no reason to move it and a familiar number is
+        one less thing an operator has to look up.
+
+        ⚠️ Every swarm container shares the host's network namespace, so
+        this is a swarm-wide claim rather than a per-container one — two
+        modules picking the same number collide at runtime with no bind
+        error and nothing in any log. `state/eval-port-collisions.sh`
+        checks the class.
+      '';
+    };
+
+    retentionPeriod = lib.mkOption {
+      type = lib.types.str;
+      default = "30d";
+      example = "90d";
+      description = ''
+        How long log data is kept.
+
+        Deliberately far shorter than the metrics store's retention: logs
+        are orders of magnitude larger per unit of time, and their value
+        decays much faster. A log line answers "what happened during that
+        incident"; a metric answers "is this worse than last quarter".
+      '';
+    };
+  };
+
+  config = lib.mkIf (hyperhiveCfg.enable && cfg.enable) {
+    # This store publishes its own health as prometheus metrics on the same
+    # listener it serves queries on, so the swarm's collector scrapes it with
+    # no exporter and no extra port — same arrangement as the metrics store.
+    #
+    # Declared here rather than in the collector's module because that is the
+    # rule the option carries: an entry exists only where the service that
+    # named it runs. ⚠️ That constrains the TARGET, not the scraper — a
+    # deployment that splits this container away from the collector's host
+    # silently drops the entry, and no assertion can see it, because separate
+    # hosts are separate evaluations.
+    services.hyperhive.swarm.otel.scrapeTargets.victorialogs = "127.0.0.1:${toString cfg.port}";
+
+    containers.${cfg.machine} = {
+      autoStart = true;
+      ephemeral = false;
+      # Shared host netns, like every sibling swarm container: the collector
+      # and Grafana reach this at 127.0.0.1:<port>.
+      privateNetwork = false;
+
+      config =
+        { ... }:
+        {
+          imports = [
+            (import ./swarm-container-resolver.nix {
+              inherit (networkCfg) bridgeIp;
+              dnsConsumers = [ "victorialogs.service" ];
+            })
+          ];
+
+          system.stateVersion = "26.05";
+
+          # This container shares the host netns, so its own firewall.service
+          # would rewrite the HOST ruleset at every boot. The host firewall
+          # owns all filtering.
+          networking.firewall.enable = false;
+          # resolvconf stays off because the resolver unit imported above
+          # owns /etc/resolv.conf. Leaving it on would let host-tracking
+          # regenerate the file empty, since the host's copy doesn't cross
+          # the boundary after start.
+          networking.resolvconf.enable = lib.mkForce false;
+
+          services.victorialogs = {
+            enable = true;
+            package = cfg.package;
+
+            # ⚠️ PINNED TO LOOPBACK for the same reason the metrics store is,
+            # and it matters more here: upstream's default listens on every
+            # interface, and this endpoint accepts writes as well as reads
+            # with no authentication of its own. The bind address is the
+            # boundary.
+            listenAddress = "127.0.0.1:${toString cfg.port}";
+
+            extraOptions = [ "-retentionPeriod=${cfg.retentionPeriod}" ];
+          };
+        };
+    };
+  };
+
+  # 🔑 OTLP ingest is served at `/insert/opentelemetry/v1/logs`, measured
+  # against the pinned build rather than read from docs. The path differs
+  # from the metrics store's `/opentelemetry/api/v1/push`, so an exporter
+  # configured by analogy with the metrics one is silently wrong.
+  #
+  # ⚠️ And the status code cannot tell you which is which: a wrong path, a
+  # right path with a wrong body, and a nonsense path ALL answer 400. Only
+  # the server log distinguishes them — the real route complains about the
+  # encoding ("json encoding isn't supported ... use protobuf"), while
+  # anything else logs "unsupported path requested". Verified with a
+  # nonsense-path control, because a probe where every arm returns the same
+  # code is not evidence.
+  #
+  # ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is
+  # why `listenAddress` above is loopback, why the collector is the only
+  # writer, and why this module declares no gateway vhost.
+}