diff --git a/nix/agent-modules/otel.nix b/nix/agent-modules/otel.nix index f06cd8e8..2405f35c 100644 --- a/nix/agent-modules/otel.nix +++ b/nix/agent-modules/otel.nix @@ -265,6 +265,14 @@ in # and what the harness spawns, so there is no foreign traffic to # filter out and an allowlist would only be a list to forget to # update. + + # journald's PRIORITY carries the level every line already has; + # without this the record reaches VictoriaLogs with + # `severity_text: Unspecified` and the store cannot tell an error + # from a debug line. The table — and the inverted direction that + # makes it worth a file of its own — lives in + # ../journald-severity.nix, shared with the swarm tier's receiver. + operators = import ../journald-severity.nix; }; # Identity the hop above cannot supply. A host-side reader can say diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 66d35098..305f8bad 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -1208,6 +1208,16 @@ in journald = { directory = hostJournalDir; units = cfg.journaldUnits; + # The SAME operator list the agent tier attaches to its own + # receiver (nix/agent-modules/otel.nix), and it transfers + # without adjustment: the entry shape is the receiver's, + # not the journal's. Both are this one component running + # `journalctl -o json`, so `PRIORITY` is spelled and typed + # identically whether the directory it was pointed at holds + # a host's journal or a container's. What differs between + # the two receivers is which journal and which units — + # neither of which the parser reads. + operators = import ../journald-severity.nix; }; }; diff --git a/nix/journald-severity.nix b/nix/journald-severity.nix new file mode 100644 index 00000000..793a6762 --- /dev/null +++ b/nix/journald-severity.nix @@ -0,0 +1,67 @@ +# journald `PRIORITY` → OpenTelemetry severity, as the stanza operator list +# every journald receiver in this tree attaches. The table it produces and +# what a consumer does with it: docs/scheduler/observability.md, "Log severity". +# +# ONE file, imported by both tiers (`agent-modules/otel.nix` reads the +# container's own journal, `host-modules/swarm-otel.nix` the host's), because +# the mapping is the part that must not drift. The two receivers are otherwise +# unrelated — one a fixed stanza, one parameterised inside a container config — +# so a copy in each is two tables to keep in step, and getting one wrong is +# silent: every line still arrives, it just arrives labelled as the wrong thing. +# +# ⚠️ **The direction is inverted.** syslog PRIORITY counts DOWN in urgency +# (0 = emerg, 7 = debug) where OpenTelemetry's severity number counts UP +# (1 = trace, 21 = fatal). Passed through as a severity number, debug renders +# as critical and emerg as unspecified, and nothing reports an error. +[ + { + type = "severity_parser"; + + # The journald input unmarshals each `journalctl -o json` line into the + # entry BODY as a map, so every journal field is addressed under `body.`. + # `attributes.PRIORITY` is the spelling that looks equally plausible and + # finds nothing — the parser then reports a missing field per entry rather + # than failing to start. + parse_from = "body.PRIORITY"; + + # 🔑 NOT optional, and the single easiest thing to leave off. By default the + # severity parser sets the severity NUMBER from the mapping and leaves the + # severity TEXT as the original value it matched on — so `severity_text` + # would arrive in VictoriaLogs as the literal string `"6"`. That is a + # populated field, it passes every check that asks whether severity is set, + # and it is not a level any consumer renders. With this on, the text is the + # recommended short name for the mapped number: `FATAL`, `ERROR`, `WARN`, + # `INFO`, `DEBUG`. The TEXT is the whole interface here — VictoriaLogs has + # no ingest parameter naming a level field, it auto-detects one by trying + # `severity_text`, `SeverityText`, `severity`, `Severity` in that order, so + # the value has to arrive already spelled as a word it knows. + overwrite_text = true; + + # A record whose PRIORITY the parser cannot read must still REACH the + # store. Explicit rather than inherited from the operator default, because + # the alternative (`drop`) would make the "lines with no severity" panel on + # the logs dashboard read zero by deleting its own evidence — the exact + # empty-result-as-success failure this mapping exists to end. + on_error = "send"; + + # Strings, because that is what arrives: `journalctl -o json` renders every + # journal field as a JSON string, so `PRIORITY` is `"6"` and not `6`. The + # parser stringifies its mapping values too, so an integer spelling here + # would also match — the strings are for the reader, to keep the config + # shaped like the data. + mapping = { + fatal = [ + "0" + "1" + "2" + ]; + error = "3"; + warn = "4"; + info = [ + "5" + "6" + ]; + debug = "7"; + }; + } +]