From 72a375b0db9f47ec61817255bb3403d0cf17f90c Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 20 Sep 2026 04:57:17 +0200 Subject: [PATCH 1/5] otel: map journald PRIORITY onto a severity at every journald receiver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records reached VictoriaLogs carrying the journal's raw PRIORITY and severity_text "Unspecified" — every line in the store, at every tier, with no level a query or a dashboard could read. VictoriaLogs has no ingest parameter naming a level field; it auto-detects one by field name, so the mapping has to happen in the collector. A stanza severity_parser on each journald receiver, from one shared file rather than a copy per tier: the two receivers are unrelated config (a fixed stanza in the agent container, a parameterised block inside the swarm-otel container) and a drifted copy fails silently — every line still arrives, labelled as the wrong thing. Two details that are easy to get wrong and quiet when wrong. PRIORITY counts down in urgency where the OTEL severity counts up, so the table is written as a table. And overwrite_text is required: without it the parser sets the severity number and leaves the text as the raw digit, so severity_text arrives as the literal "6" — populated, and not a level anything renders. --- nix/agent-modules/otel.nix | 8 ++++ nix/host-modules/swarm-otel.nix | 10 +++++ nix/journald-severity.nix | 67 +++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 nix/journald-severity.nix diff --git a/nix/agent-modules/otel.nix b/nix/agent-modules/otel.nix index f06cd8e8..2405f35c 100644 --- a/nix/agent-modules/otel.nix +++ b/nix/agent-modules/otel.nix @@ -265,6 +265,14 @@ in # and what the harness spawns, so there is no foreign traffic to # filter out and an allowlist would only be a list to forget to # update. + + # journald's PRIORITY carries the level every line already has; + # without this the record reaches VictoriaLogs with + # `severity_text: Unspecified` and the store cannot tell an error + # from a debug line. The table — and the inverted direction that + # makes it worth a file of its own — lives in + # ../journald-severity.nix, shared with the swarm tier's receiver. + operators = import ../journald-severity.nix; }; # Identity the hop above cannot supply. A host-side reader can say diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 66d35098..305f8bad 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -1208,6 +1208,16 @@ in journald = { directory = hostJournalDir; units = cfg.journaldUnits; + # The SAME operator list the agent tier attaches to its own + # receiver (nix/agent-modules/otel.nix), and it transfers + # without adjustment: the entry shape is the receiver's, + # not the journal's. Both are this one component running + # `journalctl -o json`, so `PRIORITY` is spelled and typed + # identically whether the directory it was pointed at holds + # a host's journal or a container's. What differs between + # the two receivers is which journal and which units — + # neither of which the parser reads. + operators = import ../journald-severity.nix; }; }; diff --git a/nix/journald-severity.nix b/nix/journald-severity.nix new file mode 100644 index 00000000..793a6762 --- /dev/null +++ b/nix/journald-severity.nix @@ -0,0 +1,67 @@ +# journald `PRIORITY` → OpenTelemetry severity, as the stanza operator list +# every journald receiver in this tree attaches. The table it produces and +# what a consumer does with it: docs/scheduler/observability.md, "Log severity". +# +# ONE file, imported by both tiers (`agent-modules/otel.nix` reads the +# container's own journal, `host-modules/swarm-otel.nix` the host's), because +# the mapping is the part that must not drift. The two receivers are otherwise +# unrelated — one a fixed stanza, one parameterised inside a container config — +# so a copy in each is two tables to keep in step, and getting one wrong is +# silent: every line still arrives, it just arrives labelled as the wrong thing. +# +# ⚠️ **The direction is inverted.** syslog PRIORITY counts DOWN in urgency +# (0 = emerg, 7 = debug) where OpenTelemetry's severity number counts UP +# (1 = trace, 21 = fatal). Passed through as a severity number, debug renders +# as critical and emerg as unspecified, and nothing reports an error. +[ + { + type = "severity_parser"; + + # The journald input unmarshals each `journalctl -o json` line into the + # entry BODY as a map, so every journal field is addressed under `body.`. + # `attributes.PRIORITY` is the spelling that looks equally plausible and + # finds nothing — the parser then reports a missing field per entry rather + # than failing to start. + parse_from = "body.PRIORITY"; + + # 🔑 NOT optional, and the single easiest thing to leave off. By default the + # severity parser sets the severity NUMBER from the mapping and leaves the + # severity TEXT as the original value it matched on — so `severity_text` + # would arrive in VictoriaLogs as the literal string `"6"`. That is a + # populated field, it passes every check that asks whether severity is set, + # and it is not a level any consumer renders. With this on, the text is the + # recommended short name for the mapped number: `FATAL`, `ERROR`, `WARN`, + # `INFO`, `DEBUG`. The TEXT is the whole interface here — VictoriaLogs has + # no ingest parameter naming a level field, it auto-detects one by trying + # `severity_text`, `SeverityText`, `severity`, `Severity` in that order, so + # the value has to arrive already spelled as a word it knows. + overwrite_text = true; + + # A record whose PRIORITY the parser cannot read must still REACH the + # store. Explicit rather than inherited from the operator default, because + # the alternative (`drop`) would make the "lines with no severity" panel on + # the logs dashboard read zero by deleting its own evidence — the exact + # empty-result-as-success failure this mapping exists to end. + on_error = "send"; + + # Strings, because that is what arrives: `journalctl -o json` renders every + # journal field as a JSON string, so `PRIORITY` is `"6"` and not `6`. The + # parser stringifies its mapping values too, so an integer spelling here + # would also match — the strings are for the reader, to keep the config + # shaped like the data. + mapping = { + fatal = [ + "0" + "1" + "2" + ]; + error = "3"; + warn = "4"; + info = [ + "5" + "6" + ]; + debug = "7"; + }; + } +] From fc870c459aa58916bfb4b73b8a0a4d5decfc7f7e Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 20 Sep 2026 04:57:18 +0200 Subject: [PATCH 2/5] grafana: count log lines that arrive with no severity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mapping above needs something that says whether it is still working after whoever wrote it has gone. A timeseries rather than a stat, so a regression is a line lifting off zero rather than a number nobody reads. Two series, and the split is the point: rows that carried a PRIORITY and arrived with no severity anyway (a broken mapping — this one must reach zero and stay there), against rows that never had a priority to map. The latter is Claude Code's own OTLP telemetry, which emits log records with no severity set at the source; mapping cannot reach it, so it is named rather than folded into one number that never goes to zero. Needs no provisioning change — logstore.json is already in the shipped dashboard list. --- .../swarm-grafana/dashboards/logstore.json | 103 +++++++++++++++++- 1 file changed, 100 insertions(+), 3 deletions(-) diff --git a/nix/host-modules/swarm-grafana/dashboards/logstore.json b/nix/host-modules/swarm-grafana/dashboards/logstore.json index ce5dd7d4..8e92ff70 100644 --- a/nix/host-modules/swarm-grafana/dashboards/logstore.json +++ b/nix/host-modules/swarm-grafana/dashboards/logstore.json @@ -339,7 +339,7 @@ "h": 12, "w": 12, "x": 0, - "y": 24 + "y": 33 }, "targets": [ { @@ -391,7 +391,7 @@ "h": 12, "w": 12, "x": 12, - "y": 24 + "y": 33 }, "targets": [ { @@ -443,7 +443,7 @@ "h": 9, "w": 24, "x": 0, - "y": 36 + "y": 45 }, "targets": [ { @@ -487,6 +487,103 @@ "sort": "desc" } } + }, + { + "id": 9, + "type": "timeseries", + "title": "Log rows with no severity (must be zero)", + "description": "This panel is a check rather than a view: the collector maps journald's PRIORITY onto an OpenTelemetry severity at every tier, and this is what says out loud whether a line got one. Both series are rows whose `severity_text` is absent or the literal `Unspecified`, which is what VictoriaLogs stores when nothing set a severity — split by whether the row had a PRIORITY to map in the first place.\n\n**`journald_priority_unmapped`** is a row that carries PRIORITY and still arrived without a severity. It is the regression line: nonzero means a journald receiver somewhere lost its severity_parser operator, so read it against `nix/journald-severity.nix` and the two receivers that import it.\n\n**`no_priority_field`** is a row that never had a priority — today that is Claude Code's own OTLP telemetry (`scope.name: com.anthropic.claude_code.events`), which emits log records with no severity set at the source. Mapping cannot reach it; it needs a fix where it is produced, so this series is expected nonzero until that lands and is NOT evidence the mapping broke.\n\nRange-scoped over time on purpose. An instantaneous zero is also what a broken query returns — a line you can watch go to zero and stay there is the part that survives whoever wrote it. To sanity-check the query itself, read it against `Rows in range · all sources (control)` above: that panel nonzero while both of these are flat zero means these queries stopped matching, not that every line grew a severity.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 24 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "queryType": "statsRange", + "expr": "(severity_text:\"\" OR severity_text:\"Unspecified\") PRIORITY:* | stats count() as journald_priority_unmapped" + }, + { + "refId": "B", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "queryType": "statsRange", + "expr": "(severity_text:\"\" OR severity_text:\"Unspecified\") PRIORITY:\"\" | stats count() as no_priority_field" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "journald_priority_unmapped" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "no_priority_field" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["max", "lastNotNull"] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } } ] } From 83a604ae23a74892f5919ded25e8d9c4d0e7fd4e Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 20 Sep 2026 04:57:18 +0200 Subject: [PATCH 3/5] docs: document the PRIORITY to severity mapping The table, the inverted direction, and the one source that has no priority to map. --- docs/scheduler/observability.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index d4b7d66e..b7d17749 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -216,6 +216,38 @@ Append additional labels via `extraResourceAttributes` (see option reference above); pass custom per-data-point labels with `hive-metric --labels` (see below). +## Log severity + +Every journald receiver in the pipeline maps the journal's `PRIORITY` onto an +OpenTelemetry severity, so a line arrives in the log store already carrying its +level and `severity_text:ERROR` is a query you can write: + +| `PRIORITY` | syslog | severity | +| ---------- | ------------------ | -------- | +| 0, 1, 2 | emerg, alert, crit | `FATAL` | +| 3 | err | `ERROR` | +| 4 | warning | `WARN` | +| 5, 6 | notice, info | `INFO` | +| 7 | debug | `DEBUG` | + +Both tiers that read a journal do this — the agent container's forwarder over +its own journal, and the swarm collector over the host's — and both import the +same mapping from `nix/journald-severity.nix`, which holds the table once. + +Note the direction. Syslog's priority counts **down** in urgency, where +OpenTelemetry's severity number counts **up**. Neither is a passthrough of the +other, and a pipeline that hands one straight to the other renders debug lines +as critical without erroring anywhere. + +A record that reaches the store from somewhere other than a journal has no +priority to map. Claude Code's own telemetry is the live example: it exports +OTLP log records directly (`OTEL_LOGS_EXPORTER`, see +`nix/agent-modules/claude-settings.nix`) and sets no severity on them, so they +store as `Unspecified`. The **Log rows with no severity** panel on the +`hyperhive · logs` dashboard counts both cases and keeps them apart: lines that +had a priority and arrived without a severity anyway signal a broken mapping, +while lines that never had one signal nothing of the sort. + ## Host-emitted container-resource metrics (hive-c0re) When OTEL is enabled, **hive-c0re itself** also exports each agent From 7fa13b592f5d05ff0d8b1bd5cfe681a1bd47f480 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 20 Sep 2026 05:19:06 +0200 Subject: [PATCH 4/5] module-eval: pin the severity mapping's direction and the panel The direction is the part a reviewer cannot check by looking, so it is asserted at both ends of the table and in both tiers' groups: an inverted mapping still maps every value to something, and a case that only asks whether a severity parser exists passes on the exact defect. The reader that turns a rendered operator list back into a PRIORITY -> name function lives in lib.nix, since both tiers need it. The panel is asserted on its query rather than its title, because a panel that keeps the title and loses the expression renders an empty graph that looks exactly like zero prioless lines. --- nix/module-eval/agent-otel.nix | 35 +++++++++++++++++++++++++++++ nix/module-eval/grafana.nix | 26 +++++++++++++++++++++ nix/module-eval/lib.nix | 30 +++++++++++++++++++++++++ nix/module-eval/swarm-otel-core.nix | 30 +++++++++++++++++++++++++ 4 files changed, 121 insertions(+) diff --git a/nix/module-eval/agent-otel.nix b/nix/module-eval/agent-otel.nix index a843362f..180d4c5f 100644 --- a/nix/module-eval/agent-otel.nix +++ b/nix/module-eval/agent-otel.nix @@ -18,6 +18,8 @@ let ; }) agent + journaldSeverityOf + journaldSeverityOverwritesText runGroup ; @@ -56,6 +58,39 @@ let name = "the agent forwarder reads the persistent journal, not the runtime one"; ok = ((agentSettings agentOtel).receivers.journald.directory or null) == "/var/log/journal"; } + { + # ⚠️ The direction, and the only part of this mapping a reviewer cannot + # check by looking: syslog's PRIORITY counts DOWN in urgency (0 = emerg, + # 7 = debug) where the OTEL severity counts UP, so a copy wired across + # renders debug as critical while every line still arrives and every + # field is populated. Asserted at BOTH ENDS of the table — an inverted + # mapping still maps every value to something, so "a severity parser is + # configured" passes on the exact defect. The swarm tier's receiver gets + # the same case over in ./swarm-otel-core.nix; the two import one file, + # and these are what say they still do. + name = "the agent forwarder maps PRIORITY to severity the right way up"; + ok = + let + sev = journaldSeverityOf (agentSettings agentOtel).receivers.journald; + in + sev "0" == "fatal" + && sev "3" == "error" + && sev "4" == "warn" + && sev "6" == "info" + && sev "7" == "debug"; + } + { + # The other half of the same operator, and the half that looks already + # handled. Without `overwrite_text` the parser sets the severity NUMBER + # from the mapping and leaves the severity TEXT as the raw value it + # matched — so `severity_text` reaches the log store as the literal "6". + # That is a populated field which passes any check asking merely whether + # severity is set, and which nothing renders as a level. VictoriaLogs has + # no ingest parameter naming a level field, so the text is the whole + # interface. + name = "the agent forwarder writes a level name, not the raw priority digit"; + ok = journaldSeverityOverwritesText (agentSettings agentOtel).receivers.journald; + } { # The hop's two ends: what it reads, and where what it reads goes. # The endpoint is compared against the value the fixture handed the diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 25442c56..9ade378d 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -217,6 +217,32 @@ let grafanaRemoteAuthelia.services.hyperhive.swarm.statusPublish.tokenEndpoint == "https://auth.example.invalid/api/oidc/token"; } + { + # The standing check on the collector's PRIORITY mapping — whether a line + # ever reaches the log store carrying no severity. It is shipped config + # rather than something an operator imports by hand, which also means it + # can be deleted by hand. Read out of the dashboard the provisioner + # actually ships (`dashboardDir` in ../host-modules/swarm-grafana.nix + # names this file), and asserted on the QUERY rather than the title, + # because a panel that keeps its title and loses its expression renders + # an empty graph that looks exactly like zero prioless lines. + name = "the logs dashboard still counts lines that arrive with no severity"; + ok = + let + board = builtins.fromJSON ( + builtins.readFile ../host-modules/swarm-grafana/dashboards/logstore.json + ); + exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels; + counts = lib.filter (e: lib.hasInfix "severity_text" e && lib.hasInfix "stats count()" e) exprs; + in + # Both arms of the split: lines that HAD a priority and arrived without + # a severity anyway (the regression line, which must reach zero), and + # lines that never had one to map — Claude Code's own OTLP telemetry, + # which no mapping can reach. Folded into one number the second would + # keep the panel off zero forever and hide the first. + lib.any (e: lib.hasInfix "PRIORITY:*" e) counts + && lib.any (e: lib.hasInfix "PRIORITY:\"\"" e) counts; + } ]; in runGroup "grafana" cases diff --git a/nix/module-eval/lib.nix b/nix/module-eval/lib.nix index 016bfdd7..79f3f4fa 100644 --- a/nix/module-eval/lib.nix +++ b/nix/module-eval/lib.nix @@ -140,6 +140,34 @@ let agentHarness = machine: machine.systemd.services.hive-agent; + # Reads a journald receiver's rendered operator list back as a plain + # `PRIORITY -> severity-name` function, so a case can ask what a given + # priority actually maps to rather than matching on the shape of the config + # that produces it. Shared because two tiers run that receiver — the agent + # container over its own journal, the swarm collector over the host's — from + # one imported mapping, and the case that says so has to reach both. + # `null` for a receiver carrying no severity parser at all, which is the + # state that mapping replaced. + journaldSeverityOf = + receiver: priority: + let + parser = lib.findFirst (o: o.type or null == "severity_parser") null (receiver.operators or [ ]); + # The mapping is `severity-name -> value or list of values`; invert it + # into `value -> severity-name` so a lookup is by priority. + hits = lib.attrNames (lib.filterAttrs (_: v: lib.elem priority (lib.toList v)) parser.mapping); + in + if parser == null || hits == [ ] then null else lib.head hits; + + # Whether that same parser rewrites the severity TEXT. Its own reader, + # because it is a separate way for the mapping to be there and useless: see + # the case that asserts it. + journaldSeverityOverwritesText = + receiver: + let + parser = lib.findFirst (o: o.type or null == "severity_parser") { } (receiver.operators or [ ]); + in + parser.overwrite_text or false; + # The enables that switch owns. ⚠️ `otel` is the per-hive collector's own # option and is NOT under `deploy` — spelled at the wrong path it would be # undeclared rather than false, and a roster that quietly loses a member is @@ -197,6 +225,8 @@ in inherit baoSettings; inherit otelSettings; inherit agentHarness; + inherit journaldSeverityOf; + inherit journaldSeverityOverwritesText; inherit swarmServiceEnables; inherit runGroup; } diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix index c4433f39..8673b464 100644 --- a/nix/module-eval/swarm-otel-core.nix +++ b/nix/module-eval/swarm-otel-core.nix @@ -18,6 +18,8 @@ let ; }) hive + journaldSeverityOf + journaldSeverityOverwritesText runGroup otelSettings ; @@ -155,6 +157,34 @@ let in named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named; } + { + # The host-journal half of the mapping the agent tier's receiver carries + # (./agent-otel.nix holds the sibling case and the full reasoning). Both + # import one file, so this is what says the swarm collector still does — + # the receivers are otherwise unrelated config, and a drifted copy is + # silent: every line still arrives, labelled as the wrong thing. + # + # Asserted at both ends of the table because the direction is inverted + # and an inverted copy still maps every value to something. + name = "the swarm collector maps the host journal's PRIORITY the right way up"; + ok = + let + sev = journaldSeverityOf (otelSettings otelNoStores).receivers.journald; + in + sev "0" == "fatal" + && sev "3" == "error" + && sev "4" == "warn" + && sev "6" == "info" + && sev "7" == "debug"; + } + { + # Same reasoning as the agent tier's copy: without this the parser + # populates `severity_text` with the raw digit it matched, which passes + # any check asking whether severity is set and is not a level anything + # renders. + name = "the swarm collector writes a level name, not the raw priority digit"; + ok = journaldSeverityOverwritesText (otelSettings otelNoStores).receivers.journald; + } ]; in runGroup "swarm-otel-core" cases From 33a7ca6118439202defc6fc6b9a4c380125697f0 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 20 Sep 2026 13:32:00 +0200 Subject: [PATCH 5/5] module-eval: assert the severity table once, not once per tier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both journald receivers import one PRIORITY mapping, and each tier's suite was asserting the whole table against its own receiver. That checks one file twice: invert the table and two cases fail saying the same thing, which tells you nothing about which of the two possible defects you have. Split by subject instead. The table's contents — the inverted direction, overwrite_text, parse_from/on_error — belong to the file that holds them, so they get a suite of their own reading that file directly, with no fixture at all. Each tier keeps a case, reduced to the question only it can answer: does MY receiver carry the shared mapping. Both tier cases stay. They cover different receivers over different journals — the agent container's own and the swarm collector's host journal — and one tier quietly losing its parser while the other keeps one is exactly the half-fixed state worth catching. Membership rather than equality of the whole operator list, so a tier that later grows an unrelated operator of its own still passes. Checked against seven defect scenarios: each fails exactly one case, and names the right one. --- nix/checks.nix | 9 ++ nix/module-eval/agent-otel.nix | 56 ++++++------- nix/module-eval/journald-severity.nix | 113 ++++++++++++++++++++++++++ nix/module-eval/lib.nix | 46 ++++------- nix/module-eval/swarm-otel-core.nix | 38 +++------ 5 files changed, 174 insertions(+), 88 deletions(-) create mode 100644 nix/module-eval/journald-severity.nix diff --git a/nix/checks.nix b/nix/checks.nix index b8513be0..29f0ee56 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -91,6 +91,15 @@ in inherit pkgs self nixosSystem; inherit (pkgs) lib; }; + # Not a subsystem cluster like its siblings but a single shared file, and + # its own derivation for that reason: both journald receivers import one + # PRIORITY → severity mapping, so asserting the table inside each tier's + # suite would check one file twice and report one defect as two failures. + # Holds no fixture, so it costs nothing next to the clusters. + module-eval-journald-severity = import ./module-eval/journald-severity.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; module-eval-agent-queue-bao = import ./module-eval/agent-queue-bao.nix { inherit pkgs self nixosSystem; inherit (pkgs) lib; diff --git a/nix/module-eval/agent-otel.nix b/nix/module-eval/agent-otel.nix index 180d4c5f..8c596e1d 100644 --- a/nix/module-eval/agent-otel.nix +++ b/nix/module-eval/agent-otel.nix @@ -18,8 +18,7 @@ let ; }) agent - journaldSeverityOf - journaldSeverityOverwritesText + carriesJournaldSeverity runGroup ; @@ -59,37 +58,28 @@ let ok = ((agentSettings agentOtel).receivers.journald.directory or null) == "/var/log/journal"; } { - # ⚠️ The direction, and the only part of this mapping a reviewer cannot - # check by looking: syslog's PRIORITY counts DOWN in urgency (0 = emerg, - # 7 = debug) where the OTEL severity counts UP, so a copy wired across - # renders debug as critical while every line still arrives and every - # field is populated. Asserted at BOTH ENDS of the table — an inverted - # mapping still maps every value to something, so "a severity parser is - # configured" passes on the exact defect. The swarm tier's receiver gets - # the same case over in ./swarm-otel-core.nix; the two import one file, - # and these are what say they still do. - name = "the agent forwarder maps PRIORITY to severity the right way up"; - ok = - let - sev = journaldSeverityOf (agentSettings agentOtel).receivers.journald; - in - sev "0" == "fatal" - && sev "3" == "error" - && sev "4" == "warn" - && sev "6" == "info" - && sev "7" == "debug"; - } - { - # The other half of the same operator, and the half that looks already - # handled. Without `overwrite_text` the parser sets the severity NUMBER - # from the mapping and leaves the severity TEXT as the raw value it - # matched — so `severity_text` reaches the log store as the literal "6". - # That is a populated field which passes any check asking merely whether - # severity is set, and which nothing renders as a level. VictoriaLogs has - # no ingest parameter naming a level field, so the text is the whole - # interface. - name = "the agent forwarder writes a level name, not the raw priority digit"; - ok = journaldSeverityOverwritesText (agentSettings agentOtel).receivers.journald; + # Wiring, not contents. A journald record arrives with its level in + # `PRIORITY` and nothing downstream reads that field, so a receiver with + # no severity parser ships every line to the log store labelled + # `Unspecified` — healthy, complete, and unqueryable by level. + # + # What this asks is only whether THIS receiver carries the shared + # mapping. ⚠️ Whether that mapping is RIGHT — the inverted direction, + # which is the part of it a reviewer cannot check by looking — is + # asserted once, in ./journald-severity.nix, against the file both tiers + # import. Re-asserting the table here would test that file twice and + # report one defect as two failures. + # + # The swarm collector's own receiver gets the sibling of this case in + # ./swarm-otel-core.nix. Deliberately NOT folded together with it: they + # cover different journals — this container's own, and the host's — and + # one tier quietly losing its parser while the other keeps one is + # precisely the half-fixed state worth catching. + # + # Membership rather than equality of the whole operator list, so a tier + # that later grows an unrelated operator of its own still passes. + name = "the agent forwarder's journald receiver carries the shared PRIORITY mapping"; + ok = carriesJournaldSeverity (agentSettings agentOtel).receivers.journald; } { # The hop's two ends: what it reads, and where what it reads goes. diff --git a/nix/module-eval/journald-severity.nix b/nix/module-eval/journald-severity.nix new file mode 100644 index 00000000..265eb224 --- /dev/null +++ b/nix/module-eval/journald-severity.nix @@ -0,0 +1,113 @@ +# `checks.module-eval-journald-severity` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +# +# The odd one out in this directory: every sibling suite evaluates a host or an +# agent and reads the config that falls out, where this one reads a single file +# that is already a value. It is a suite of its own for the reason the operator +# gave when she saw the mapping asserted in two tiers at once — **test the +# thing only once**. The table lives in one file, so its contents are one +# subject and belong to one case. Each tier's suite then asks a different +# question, about wiring rather than contents: does MY receiver carry this. +# +# What that buys is a readable failure. Invert the table and exactly this suite +# goes red, naming the table. Drop the parser from one tier's receiver and +# exactly that tier goes red, naming the tier. Neither failure makes you read +# the other one to work out which of the two happened. +# +# Needs no `nixosSystem` fixture at all — the thing under test is a file — so +# this suite costs nothing to evaluate next to its siblings. +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + journaldSeverityParser + runGroup + ; + + # The parser's own mapping read back as a plain `PRIORITY -> severity-name` + # function, so the case below can ask what a priority MAPS TO instead of + # matching on the shape of the attrset that produces it. `null` for a + # priority the table does not name, which the case uses as its control. + severityOf = + priority: + let + hits = lib.attrNames ( + lib.filterAttrs (_: v: lib.elem priority (lib.toList v)) journaldSeverityParser.mapping + ); + in + if journaldSeverityParser == null || hits == [ ] then null else lib.head hits; + + cases = [ + { + # ⚠️ The direction, and the only part of this mapping a reviewer cannot + # check by looking. syslog's PRIORITY counts DOWN in urgency (0 = emerg, + # 7 = debug) where the OpenTelemetry severity counts UP (1 = trace, + # 21 = fatal). Wired across, debug renders as critical and emerg as + # unspecified — and nothing reports an error, because every line still + # arrives and every field is still populated. + # + # Every priority, not a sample. An inverted copy still maps each value + # to SOMETHING, so a case that checks one end, or checks that a mapping + # merely exists, passes on the exact defect it was written for. The + # `"8"` arm is the control: it says this reader looks values up rather + # than answering the same way whatever it is handed, so the eight + # arms above it mean what they appear to mean. + name = "journald PRIORITY maps to severity the right way up"; + ok = + severityOf "0" == "fatal" + && severityOf "1" == "fatal" + && severityOf "2" == "fatal" + && severityOf "3" == "error" + && severityOf "4" == "warn" + && severityOf "5" == "info" + && severityOf "6" == "info" + && severityOf "7" == "debug" + && severityOf "8" == null; + } + { + # The second way this mapping can be present and useless, and the one + # that looks already handled. Without `overwrite_text` the parser sets + # the severity NUMBER from the table above and leaves the severity TEXT + # as the raw value it matched — so `severity_text` reaches the log store + # as the literal "6". That is a populated field: it passes any check + # asking merely whether severity is set, and it is not a level anything + # renders. VictoriaLogs has no ingest parameter naming a level field, so + # the text is the whole interface. + name = "the mapping writes a level name, not the raw priority digit"; + ok = journaldSeverityParser.overwrite_text or false; + } + { + # Two settings that are load-bearing in opposite directions. + # + # `parse_from` names where the value is. The journald input unmarshals + # each `journalctl -o json` line into the entry BODY as a map, so the + # field lives under `body.`; `attributes.PRIORITY` is the spelling that + # looks equally plausible and finds nothing, leaving a parser that + # reports a missing field per entry rather than failing to start. + # + # `on_error` decides what happens to a record it cannot read. It must + # still REACH the store: `drop` would make the logs dashboard's + # "lines with no severity" panel read zero by deleting its own evidence, + # which is the empty-result-as-success failure this whole mapping exists + # to end. + name = "the mapping reads the field the receiver fills, and keeps what it cannot read"; + ok = + journaldSeverityParser.parse_from or null == "body.PRIORITY" + && journaldSeverityParser.on_error or null == "send"; + } + ]; +in +runGroup "journald-severity" cases diff --git a/nix/module-eval/lib.nix b/nix/module-eval/lib.nix index 79f3f4fa..cc25daf6 100644 --- a/nix/module-eval/lib.nix +++ b/nix/module-eval/lib.nix @@ -140,33 +140,23 @@ let agentHarness = machine: machine.systemd.services.hive-agent; - # Reads a journald receiver's rendered operator list back as a plain - # `PRIORITY -> severity-name` function, so a case can ask what a given - # priority actually maps to rather than matching on the shape of the config - # that produces it. Shared because two tiers run that receiver — the agent - # container over its own journal, the swarm collector over the host's — from - # one imported mapping, and the case that says so has to reach both. - # `null` for a receiver carrying no severity parser at all, which is the - # state that mapping replaced. - journaldSeverityOf = - receiver: priority: - let - parser = lib.findFirst (o: o.type or null == "severity_parser") null (receiver.operators or [ ]); - # The mapping is `severity-name -> value or list of values`; invert it - # into `value -> severity-name` so a lookup is by priority. - hits = lib.attrNames (lib.filterAttrs (_: v: lib.elem priority (lib.toList v)) parser.mapping); - in - if parser == null || hits == [ ] then null else lib.head hits; + # The PRIORITY → severity operator both journald receivers attach, read + # straight off the file that defines it. What each TIER's case then asks is + # only whether its own receiver carries this exact value — the contents are + # somebody else's case (./journald-severity.nix), asserted once. + # + # Here rather than in one of the suites because three of them want it: the + # two tiers, to say they are wired to it, and the suite that reads what it + # says. A parser rather than the whole list, so a tier that later grows an + # unrelated operator of its own still passes the wiring case. + journaldSeverityParser = lib.findFirst (o: o.type or null == "severity_parser") null ( + import ../journald-severity.nix + ); - # Whether that same parser rewrites the severity TEXT. Its own reader, - # because it is a separate way for the mapping to be there and useless: see - # the case that asserts it. - journaldSeverityOverwritesText = - receiver: - let - parser = lib.findFirst (o: o.type or null == "severity_parser") { } (receiver.operators or [ ]); - in - parser.overwrite_text or false; + # Whether a given receiver is wired to it. One expression, used by both + # tiers, so "this receiver carries the shared mapping" is spelled once and + # the two tier cases differ only in which receiver they hand it. + carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]); # The enables that switch owns. ⚠️ `otel` is the per-hive collector's own # option and is NOT under `deploy` — spelled at the wrong path it would be @@ -225,8 +215,8 @@ in inherit baoSettings; inherit otelSettings; inherit agentHarness; - inherit journaldSeverityOf; - inherit journaldSeverityOverwritesText; + inherit journaldSeverityParser; + inherit carriesJournaldSeverity; inherit swarmServiceEnables; inherit runGroup; } diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix index 8673b464..7428919c 100644 --- a/nix/module-eval/swarm-otel-core.nix +++ b/nix/module-eval/swarm-otel-core.nix @@ -18,8 +18,7 @@ let ; }) hive - journaldSeverityOf - journaldSeverityOverwritesText + carriesJournaldSeverity runGroup otelSettings ; @@ -158,32 +157,17 @@ let named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named; } { - # The host-journal half of the mapping the agent tier's receiver carries - # (./agent-otel.nix holds the sibling case and the full reasoning). Both - # import one file, so this is what says the swarm collector still does — - # the receivers are otherwise unrelated config, and a drifted copy is - # silent: every line still arrives, labelled as the wrong thing. + # The host-journal sibling of ./agent-otel.nix's wiring case, which + # carries the full reasoning. Same question, different receiver: this + # one reads the HOST's journal rather than a container's, and the two + # are unrelated config — a fixed stanza there, a parameterised block + # inside `containers.swarm-otel` here — so one losing its parser while + # the other keeps one is a real and silent state. # - # Asserted at both ends of the table because the direction is inverted - # and an inverted copy still maps every value to something. - name = "the swarm collector maps the host journal's PRIORITY the right way up"; - ok = - let - sev = journaldSeverityOf (otelSettings otelNoStores).receivers.journald; - in - sev "0" == "fatal" - && sev "3" == "error" - && sev "4" == "warn" - && sev "6" == "info" - && sev "7" == "debug"; - } - { - # Same reasoning as the agent tier's copy: without this the parser - # populates `severity_text` with the raw digit it matched, which passes - # any check asking whether severity is set and is not a level anything - # renders. - name = "the swarm collector writes a level name, not the raw priority digit"; - ok = journaldSeverityOverwritesText (otelSettings otelNoStores).receivers.journald; + # Contents are not this case's business. The table both receivers import + # is asserted once, in ./journald-severity.nix. + name = "the swarm collector's journald receiver carries the shared PRIORITY mapping"; + ok = carriesJournaldSeverity (otelSettings otelNoStores).receivers.journald; } ]; in