From 549156e55f71b7b49dffe49e4a268ec08fb02e52 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Wed, 23 Sep 2026 09:59:24 +0200
Subject: [PATCH] swarm-otel: persist the journald cursor across collector
 restarts
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The swarm-tier collector's journald receiver had no storage extension, so
it started each run with no cursor: journalctl --follow --lines=0 ships
only what arrives after the receiver starts. Every collector restart
therefore dropped whatever was written to the journal while it was down,
silently — no error and no replay.

Wire the receiver to a file_storage extension, matching the agent-tier
collector in nix/agent-modules/otel.nix, so a restart resumes from the
persisted cursor instead.

Refs #4527.
---
 nix/host-modules/swarm-otel.nix | 43 +++++++++++++++++++++++++++++++--
 1 file changed, 41 insertions(+), 2 deletions(-)

diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix
index c53bfd21..9ee4ee0b 100644
--- a/nix/host-modules/swarm-otel.nix
+++ b/nix/host-modules/swarm-otel.nix
@@ -1357,6 +1357,20 @@ in
                     # the two receivers is which journal and which units —
                     # neither of which the parser reads.
                     operators = import ../journald-severity.nix;
+                    # Persists the read cursor, so a restart resumes where
+                    # the last run stopped. Without it the receiver starts
+                    # from the journal's tail and ships only what is written
+                    # AFTER it starts: every restart silently drops whatever
+                    # landed while the collector was down — no error, no
+                    # replay, just a hole.
+                    #
+                    # Cursor and journal share a lifetime here, which is what
+                    # makes the pointer meaningful: the journal is the host's,
+                    # bind-mounted in above, and this container is
+                    # `ephemeral = false`, so the cursor under the unit's
+                    # `StateDirectory` outlives a container restart exactly as
+                    # the journal it indexes does.
+                    storage = "file_storage";
                   };
                 };
 
@@ -1437,7 +1451,10 @@ in
               # naming it authenticates nothing. Derived from the same
               # attrset as the receivers so the two cannot disagree.
               service.extensions =
-                map (h: "oidc/${h}") (lib.attrNames hivePorts)
+                # Same rule, and the journald receiver's `storage:` above
+                # would name a component the collector never starts.
+                [ "file_storage" ]
+                ++ map (h: "oidc/${h}") (lib.attrNames hivePorts)
                 # The swarm-tier producer's own authenticator — unconditional,
                 # same reasoning as its receiver above: `otlp/${swarmTierName}`
                 # always exists, so the authenticator it names must too, or an
@@ -1556,7 +1573,29 @@ in
             }
             // {
               extensions =
-                lib.mapAttrs' (
+                # The journald receiver's cursor store. No `StateDirectory`
+                # is set alongside it because nixpkgs' own
+                # `opentelemetry-collector` module already declares
+                # `StateDirectory = "opentelemetry-collector"` (and points
+                # `WorkingDirectory` at the same `%S` path) — systemd
+                # therefore creates this directory, owned by the unit's
+                # `DynamicUser`, before every start. A second declaration
+                # here would only be a second owner of a fact upstream
+                # states.
+                #
+                # `create_directory` is still load-bearing, and for a
+                # build-time reason rather than a runtime one:
+                # `validateConfigFile` above runs `otelcol validate` in the
+                # nix build sandbox, where systemd has never run and the
+                # path does not exist. Without it the validator refuses
+                # with "directory must exist" and the build fails.
+                {
+                  file_storage = {
+                    directory = "/var/lib/opentelemetry-collector";
+                    create_directory = true;
+                  };
+                }
+                // lib.mapAttrs' (
                   h: _:
                   lib.nameValuePair "oidc/${h}" {
                     issuer_url = hyperhiveCfg.swarm.authelia.url;