swarm-otel ships its own journal, so its logs reach the store

The collector's container ran with --link-journal=host and the host's
journal bound over /var/log/journal, so its own journal was never
written where anything read it. It now keeps its journal in the guest,
reads it with a second journald receiver, and gets the host's journal
at /run/host-journal.

On deploy swarm-otel rows start arriving in the store, stamped
service.name swarm-otel.

docs/swarm/services.md

@@ -348,6 +348,13 @@ authelia, forge and homeserver have a forwarder client of their own
 `tuwunel`) and no `hive` label. The forwarder of all but the store's is
 `nix/container-modules/swarm-journal-forwarder.nix`.
 
+The swarm collector's own container, `swarm-otel`, has no forwarder: its
+collector reads the container's journal itself (receiver `journald/self`) and
+exports it to the log store like the host's journal, stamped `service.name`
+`swarm-otel` and with no `hive` label. The container mounts the host's journal
+read-only at `/run/host-journal`, not over its own
+`/var/log/journal`.
+
 **Both store exporters are unconditional**, and `deploy.victoriametrics.enable`
 doesn't gate them: that option says this host _runs_ the store, while the swarm
 has one either way, reached by its swarm name through the gateway.

nix/host-modules/swarm-otel.nix

@@ -197,29 +197,20 @@ let
   # both "the label is absent" and "the label says unknown".
   swarmDisplayName = if hyperhiveCfg.swarm.name == null then "unknown" else hyperhiveCfg.swarm.name;
 
-  # Where the journal lives on BOTH sides of the bind mount below — one string
+  # The host's journal directory, and where it is bind-mounted in the guest.
-  # because the receiver reads the path it is mounted at, and two spellings of
+  # The mount point is NOT the guest's own /var/log/journal: that is where this
-  # one path is a mount that succeeds and a receiver that finds nothing.
+  # container's journald writes and `journald/self` reads, and a mount over it
+  # hides the guest's journal from its own reader.
   #
-  # 🔑 Why the host's directory is enough to see a container — and why it is
+  # nixpkgs hardcodes `--link-journal=try-guest`, which keeps this container's
-  # only enough because that container asks for it. nixpkgs hardcodes
+  # journal in the guest, so the host directory holds the host's units only.
-  # `--link-journal=try-guest`, which puts the journal inside the container and
+  # Every container with a `serviceProducers` entry (`swarm-bao`, `swarm-nats`,
-  # leaves the host with a symlink into that container's transient root: a
-  # reader here cannot follow it, and it dangles the moment the container
-  # stops. Every container block except the `serviceProducers` ones therefore sets
-  # `extraFlags = [ "--link-journal=host" ]`, which the invocation expands
-  # AFTER the hardcoded flag, so the files land here under their own
-  # machine-id subdirectory and are bind-mounted into the guest instead.
-  #
-  # ⚠️ Drop that flag from one of THOSE containers and this receiver silently
-  # stops seeing it — no error, just a unit that never appears in the store.
-  # The `serviceProducers` containers (`swarm-bao`, `swarm-nats`,
   # `swarm-builder`, `swarm-claude-login`, `swarm-review-classifier`,
   # `swarm-ci-runner`, `swarm-grafana`, `swarm-victorialogs`, `hive-forge`,
-  # `hive-matrix`, `swarm-victoriametrics`, `swarm-authelia`) are the exceptions
+  # `hive-matrix`, `swarm-victoriametrics`, `swarm-authelia`) forwards its own
-  # on purpose: their journals stay inside the container for its own collector
+  # journal from its own collector, so this receiver never sees their units.
-  # instead, so this receiver never sees their units.
   hostJournalDir = "/var/log/journal";
+  hostJournalMount = "/run/host-journal";
 
   # Each store's OTLP route, by domain. One binding because the same string is
   # both the address requested and the audience the token is minted for — two
@@ -802,9 +793,8 @@ in
     containers.${cfg.machine} = {
       autoStart = true;
       ephemeral = false;
-      # Journal files on the host, not inside the container: nixpkgs hardcodes
+      # No `--link-journal=host`: this container's journal stays in the guest,
-      # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
+      # where `journald/self` reads it.
-      extraFlags = [ "--link-journal=host" ];
       inherit privateNetwork;
 
       # The upstream credential is operator-provided and lives on the host.
@@ -846,7 +836,7 @@ in
         # ownership naming a different group inside the container, and the
         # failure would be a receiver that starts cleanly and reads nothing.
         // lib.optionalAttrs collectLogs {
-          ${hostJournalDir} = {
+          ${hostJournalMount} = {
             hostPath = hostJournalDir;
             isReadOnly = true;
           };
@@ -1021,7 +1011,7 @@ in
                 # machine id is per-machine by construction.
                 // lib.optionalAttrs collectLogs {
                   journald = {
-                    directory = hostJournalDir;
+                    directory = hostJournalMount;
                     # The SAME operator list the agent tier attaches to its own
                     # receiver (nix/agent-modules/otel.nix), and it transfers
                     # without adjustment: the entry shape is the receiver's,
@@ -1047,6 +1037,21 @@ in
                     # the journal it indexes does.
                     storage = "file_storage";
                   };
+                  # This container's own journal, read in the guest. The
+                  # collector's export failures land here and ship like any
+                  # other record; the read cursor is the receiver's own, so
+                  # a collector that is down at boot loses nothing and waits
+                  # on nothing.
+                  #
+                  # ⚠️ `merge` is stated: without `--merge`, `journalctl
+                  # --follow` over a journal with no entry for the current
+                  # boot crash-loops the receiver.
+                  "journald/self" = {
+                    directory = "/var/log/journal";
+                    merge = true;
+                    operators = import ../journald-severity.nix;
+                    storage = "file_storage";
+                  };
                 };
 
               exporters = {
@@ -1240,6 +1245,17 @@ in
                     ];
                     exporters = logExporterNames;
                   };
+                  # This container's own journal goes straight to the stores,
+                  # not through `otlp/<name>`: a collector forwarding to
+                  # itself would need its own receiver up first.
+                  "logs/self" = {
+                    receivers = [ "journald/self" ];
+                    processors = [
+                      "resource/self-journal"
+                      "resource/${swarmTierName}"
+                    ];
+                    exporters = logExporterNames;
+                  };
                 };
             }
             // {
@@ -1426,6 +1442,16 @@ in
                 # `ignore`: a record whose body is not a map cannot be indexed,
                 # and is kept (with a warning) rather than dropped.
                 // lib.optionalAttrs collectLogs {
+                  # `service.name` on this container's own journal, as the
+                  # other containers' forwarders stamp theirs.
+                  "resource/self-journal".attributes = [
+                    {
+                      key = "service.name";
+                      value = cfg.machine;
+                      action = "upsert";
+                    }
+                  ];
+
                   "filter/exclude-user-sessions" = {
                     error_mode = "ignore";
                     logs.log_record = [

nix/module-eval/swarm-otel-core.nix

@@ -259,6 +259,32 @@ let
         # The window has to outlast every attempt, or the burst is unreachable.
         && u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
     }
+    {
+      # 🩸 The container's own journal reaches the stores from the collector
+      # itself. Mounting the host's journal over the guest's /var/log/journal
+      # (or `--link-journal=host`) leaves `journald/self` reading nothing while
+      # the collector looks healthy. Presence controls: the host receiver and
+      # its mount are asserted too, so a misspelt path cannot pass.
+      name = "the swarm collector ships its own journal, and the host's from a mount that leaves it alone";
+      ok =
+        let
+          c = otelNoStores.containers.swarm-otel;
+          s = otelSettings otelNoStores;
+          self = lib.filter (p: builtins.elem "journald/self" p.receivers) (
+            lib.attrValues s.service.pipelines
+          );
+        in
+        !(lib.elem "--link-journal=host" (c.extraFlags or [ ]))
+        && s.receivers."journald/self".directory == "/var/log/journal"
+        && s.receivers."journald/self".merge
+        && s.receivers.journald.directory == "/run/host-journal"
+        && c.bindMounts."/run/host-journal".hostPath == "/var/log/journal"
+        && !(c.bindMounts ? "/var/log/journal")
+        && lib.length self == 1
+        && lib.all (p: p.exporters == [ "otlphttp/victorialogs" ]) self
+        && lib.elem "resource/self-journal" (lib.head self).processors
+        && builtins.elem "resource/self-journal" (lib.attrNames s.processors);
+    }
   ];
 in
 runGroup "swarm-otel-core" cases