otel: persist journald cursor across collector restarts

The journald receiver runs with --lines=0, so every collector start
only ships what's written after it starts, and a restart silently
loses whatever landed while it was down. Point it at a file_storage
extension so the read cursor survives a restart; start_at stays at
its 'end' default since the cursor now covers everything after the
first run.

Refs #3818

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
atlas 2026-09-13 19:47:36 +02:00 committed by mara
commit fbd9afa7fa

View file

@ -221,6 +221,21 @@ in
# tiers; see nix/host-modules/otel.nix for the measurement. # tiers; see nix/host-modules/otel.nix for the measurement.
validateConfigFile = true; validateConfigFile = true;
settings = { settings = {
# `journalctl --follow --lines=0` (what this receiver runs under
# the hood) ships only what's written *after* it starts — every
# collector start has a silent gap at the front. `storage:` below
# persists the read cursor, so a restart resumes from where it
# left off instead of re-opening that gap; `start_at` stays `end`
# (its default) because a cursor already covers every run after
# the first, and `beginning` without one would re-ship the whole
# journal on every restart. The cursor lives in the unit's own
# `StateDirectory` (nixpkgs' module already sets one, `%S`, below
# as `WorkingDirectory` too) — container-lifetime, not the
# host-persisted state bind mount: it indexes /var/log/journal,
# itself fresh on every container recreate, so the two must share
# a lifetime or the cursor outlives the journal it points into.
extensions.file_storage.directory = "/var/lib/opentelemetry-collector";
receivers.journald = { receivers.journald = {
# ⚠️ STATED, and it must stay stated: the receiver's own default # ⚠️ STATED, and it must stay stated: the receiver's own default
# is the RUNTIME journal (`/run/log/journal`), which in an agent # is the RUNTIME journal (`/run/log/journal`), which in an agent
@ -229,6 +244,7 @@ in
# leaves a collector that validates, starts, reports healthy and # leaves a collector that validates, starts, reports healthy and
# forwards nothing. # forwards nothing.
directory = "/var/log/journal"; directory = "/var/log/journal";
storage = "file_storage";
# No `units` allowlist, unlike the swarm tier's receiver. That # No `units` allowlist, unlike the swarm tier's receiver. That
# one needs one because the host's journal also holds an # one needs one because the host's journal also holds an
# operator's own session; a container's journal is the harness # operator's own session; a container's journal is the harness
@ -270,6 +286,11 @@ in
processors = [ "resource" ]; processors = [ "resource" ];
exporters = [ logExporter ]; exporters = [ logExporter ];
}; };
# An extension configured but not listed here is INERT — the
# journald receiver's `storage: file_storage` above would name a
# component the collector never starts.
service.extensions = [ "file_storage" ];
}; };
}; };
}; };