diff --git a/nix/module-eval/agent-otel.nix b/nix/module-eval/agent-otel.nix index a843362f..180d4c5f 100644 --- a/nix/module-eval/agent-otel.nix +++ b/nix/module-eval/agent-otel.nix @@ -18,6 +18,8 @@ let ; }) agent + journaldSeverityOf + journaldSeverityOverwritesText runGroup ; @@ -56,6 +58,39 @@ let name = "the agent forwarder reads the persistent journal, not the runtime one"; ok = ((agentSettings agentOtel).receivers.journald.directory or null) == "/var/log/journal"; } + { + # ⚠️ The direction, and the only part of this mapping a reviewer cannot + # check by looking: syslog's PRIORITY counts DOWN in urgency (0 = emerg, + # 7 = debug) where the OTEL severity counts UP, so a copy wired across + # renders debug as critical while every line still arrives and every + # field is populated. Asserted at BOTH ENDS of the table — an inverted + # mapping still maps every value to something, so "a severity parser is + # configured" passes on the exact defect. The swarm tier's receiver gets + # the same case over in ./swarm-otel-core.nix; the two import one file, + # and these are what say they still do. + name = "the agent forwarder maps PRIORITY to severity the right way up"; + ok = + let + sev = journaldSeverityOf (agentSettings agentOtel).receivers.journald; + in + sev "0" == "fatal" + && sev "3" == "error" + && sev "4" == "warn" + && sev "6" == "info" + && sev "7" == "debug"; + } + { + # The other half of the same operator, and the half that looks already + # handled. Without `overwrite_text` the parser sets the severity NUMBER + # from the mapping and leaves the severity TEXT as the raw value it + # matched — so `severity_text` reaches the log store as the literal "6". + # That is a populated field which passes any check asking merely whether + # severity is set, and which nothing renders as a level. VictoriaLogs has + # no ingest parameter naming a level field, so the text is the whole + # interface. + name = "the agent forwarder writes a level name, not the raw priority digit"; + ok = journaldSeverityOverwritesText (agentSettings agentOtel).receivers.journald; + } { # The hop's two ends: what it reads, and where what it reads goes. # The endpoint is compared against the value the fixture handed the diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 25442c56..9ade378d 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -217,6 +217,32 @@ let grafanaRemoteAuthelia.services.hyperhive.swarm.statusPublish.tokenEndpoint == "https://auth.example.invalid/api/oidc/token"; } + { + # The standing check on the collector's PRIORITY mapping — whether a line + # ever reaches the log store carrying no severity. It is shipped config + # rather than something an operator imports by hand, which also means it + # can be deleted by hand. Read out of the dashboard the provisioner + # actually ships (`dashboardDir` in ../host-modules/swarm-grafana.nix + # names this file), and asserted on the QUERY rather than the title, + # because a panel that keeps its title and loses its expression renders + # an empty graph that looks exactly like zero prioless lines. + name = "the logs dashboard still counts lines that arrive with no severity"; + ok = + let + board = builtins.fromJSON ( + builtins.readFile ../host-modules/swarm-grafana/dashboards/logstore.json + ); + exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels; + counts = lib.filter (e: lib.hasInfix "severity_text" e && lib.hasInfix "stats count()" e) exprs; + in + # Both arms of the split: lines that HAD a priority and arrived without + # a severity anyway (the regression line, which must reach zero), and + # lines that never had one to map — Claude Code's own OTLP telemetry, + # which no mapping can reach. Folded into one number the second would + # keep the panel off zero forever and hide the first. + lib.any (e: lib.hasInfix "PRIORITY:*" e) counts + && lib.any (e: lib.hasInfix "PRIORITY:\"\"" e) counts; + } ]; in runGroup "grafana" cases diff --git a/nix/module-eval/lib.nix b/nix/module-eval/lib.nix index 016bfdd7..79f3f4fa 100644 --- a/nix/module-eval/lib.nix +++ b/nix/module-eval/lib.nix @@ -140,6 +140,34 @@ let agentHarness = machine: machine.systemd.services.hive-agent; + # Reads a journald receiver's rendered operator list back as a plain + # `PRIORITY -> severity-name` function, so a case can ask what a given + # priority actually maps to rather than matching on the shape of the config + # that produces it. Shared because two tiers run that receiver — the agent + # container over its own journal, the swarm collector over the host's — from + # one imported mapping, and the case that says so has to reach both. + # `null` for a receiver carrying no severity parser at all, which is the + # state that mapping replaced. + journaldSeverityOf = + receiver: priority: + let + parser = lib.findFirst (o: o.type or null == "severity_parser") null (receiver.operators or [ ]); + # The mapping is `severity-name -> value or list of values`; invert it + # into `value -> severity-name` so a lookup is by priority. + hits = lib.attrNames (lib.filterAttrs (_: v: lib.elem priority (lib.toList v)) parser.mapping); + in + if parser == null || hits == [ ] then null else lib.head hits; + + # Whether that same parser rewrites the severity TEXT. Its own reader, + # because it is a separate way for the mapping to be there and useless: see + # the case that asserts it. + journaldSeverityOverwritesText = + receiver: + let + parser = lib.findFirst (o: o.type or null == "severity_parser") { } (receiver.operators or [ ]); + in + parser.overwrite_text or false; + # The enables that switch owns. ⚠️ `otel` is the per-hive collector's own # option and is NOT under `deploy` — spelled at the wrong path it would be # undeclared rather than false, and a roster that quietly loses a member is @@ -197,6 +225,8 @@ in inherit baoSettings; inherit otelSettings; inherit agentHarness; + inherit journaldSeverityOf; + inherit journaldSeverityOverwritesText; inherit swarmServiceEnables; inherit runGroup; } diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix index c4433f39..8673b464 100644 --- a/nix/module-eval/swarm-otel-core.nix +++ b/nix/module-eval/swarm-otel-core.nix @@ -18,6 +18,8 @@ let ; }) hive + journaldSeverityOf + journaldSeverityOverwritesText runGroup otelSettings ; @@ -155,6 +157,34 @@ let in named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named; } + { + # The host-journal half of the mapping the agent tier's receiver carries + # (./agent-otel.nix holds the sibling case and the full reasoning). Both + # import one file, so this is what says the swarm collector still does — + # the receivers are otherwise unrelated config, and a drifted copy is + # silent: every line still arrives, labelled as the wrong thing. + # + # Asserted at both ends of the table because the direction is inverted + # and an inverted copy still maps every value to something. + name = "the swarm collector maps the host journal's PRIORITY the right way up"; + ok = + let + sev = journaldSeverityOf (otelSettings otelNoStores).receivers.journald; + in + sev "0" == "fatal" + && sev "3" == "error" + && sev "4" == "warn" + && sev "6" == "info" + && sev "7" == "debug"; + } + { + # Same reasoning as the agent tier's copy: without this the parser + # populates `severity_text` with the raw digit it matched, which passes + # any check asking whether severity is set and is not a level anything + # renders. + name = "the swarm collector writes a level name, not the raw priority digit"; + ok = journaldSeverityOverwritesText (otelSettings otelNoStores).receivers.journald; + } ]; in runGroup "swarm-otel-core" cases