diff --git a/nix/checks.nix b/nix/checks.nix index b8513be0..29f0ee56 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -91,6 +91,15 @@ in inherit pkgs self nixosSystem; inherit (pkgs) lib; }; + # Not a subsystem cluster like its siblings but a single shared file, and + # its own derivation for that reason: both journald receivers import one + # PRIORITY → severity mapping, so asserting the table inside each tier's + # suite would check one file twice and report one defect as two failures. + # Holds no fixture, so it costs nothing next to the clusters. + module-eval-journald-severity = import ./module-eval/journald-severity.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; module-eval-agent-queue-bao = import ./module-eval/agent-queue-bao.nix { inherit pkgs self nixosSystem; inherit (pkgs) lib; diff --git a/nix/module-eval/agent-otel.nix b/nix/module-eval/agent-otel.nix index 180d4c5f..8c596e1d 100644 --- a/nix/module-eval/agent-otel.nix +++ b/nix/module-eval/agent-otel.nix @@ -18,8 +18,7 @@ let ; }) agent - journaldSeverityOf - journaldSeverityOverwritesText + carriesJournaldSeverity runGroup ; @@ -59,37 +58,28 @@ let ok = ((agentSettings agentOtel).receivers.journald.directory or null) == "/var/log/journal"; } { - # ⚠️ The direction, and the only part of this mapping a reviewer cannot - # check by looking: syslog's PRIORITY counts DOWN in urgency (0 = emerg, - # 7 = debug) where the OTEL severity counts UP, so a copy wired across - # renders debug as critical while every line still arrives and every - # field is populated. Asserted at BOTH ENDS of the table — an inverted - # mapping still maps every value to something, so "a severity parser is - # configured" passes on the exact defect. The swarm tier's receiver gets - # the same case over in ./swarm-otel-core.nix; the two import one file, - # and these are what say they still do. - name = "the agent forwarder maps PRIORITY to severity the right way up"; - ok = - let - sev = journaldSeverityOf (agentSettings agentOtel).receivers.journald; - in - sev "0" == "fatal" - && sev "3" == "error" - && sev "4" == "warn" - && sev "6" == "info" - && sev "7" == "debug"; - } - { - # The other half of the same operator, and the half that looks already - # handled. Without `overwrite_text` the parser sets the severity NUMBER - # from the mapping and leaves the severity TEXT as the raw value it - # matched — so `severity_text` reaches the log store as the literal "6". - # That is a populated field which passes any check asking merely whether - # severity is set, and which nothing renders as a level. VictoriaLogs has - # no ingest parameter naming a level field, so the text is the whole - # interface. - name = "the agent forwarder writes a level name, not the raw priority digit"; - ok = journaldSeverityOverwritesText (agentSettings agentOtel).receivers.journald; + # Wiring, not contents. A journald record arrives with its level in + # `PRIORITY` and nothing downstream reads that field, so a receiver with + # no severity parser ships every line to the log store labelled + # `Unspecified` — healthy, complete, and unqueryable by level. + # + # What this asks is only whether THIS receiver carries the shared + # mapping. ⚠️ Whether that mapping is RIGHT — the inverted direction, + # which is the part of it a reviewer cannot check by looking — is + # asserted once, in ./journald-severity.nix, against the file both tiers + # import. Re-asserting the table here would test that file twice and + # report one defect as two failures. + # + # The swarm collector's own receiver gets the sibling of this case in + # ./swarm-otel-core.nix. Deliberately NOT folded together with it: they + # cover different journals — this container's own, and the host's — and + # one tier quietly losing its parser while the other keeps one is + # precisely the half-fixed state worth catching. + # + # Membership rather than equality of the whole operator list, so a tier + # that later grows an unrelated operator of its own still passes. + name = "the agent forwarder's journald receiver carries the shared PRIORITY mapping"; + ok = carriesJournaldSeverity (agentSettings agentOtel).receivers.journald; } { # The hop's two ends: what it reads, and where what it reads goes. diff --git a/nix/module-eval/journald-severity.nix b/nix/module-eval/journald-severity.nix new file mode 100644 index 00000000..265eb224 --- /dev/null +++ b/nix/module-eval/journald-severity.nix @@ -0,0 +1,113 @@ +# `checks.module-eval-journald-severity` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +# +# The odd one out in this directory: every sibling suite evaluates a host or an +# agent and reads the config that falls out, where this one reads a single file +# that is already a value. It is a suite of its own for the reason the operator +# gave when she saw the mapping asserted in two tiers at once — **test the +# thing only once**. The table lives in one file, so its contents are one +# subject and belong to one case. Each tier's suite then asks a different +# question, about wiring rather than contents: does MY receiver carry this. +# +# What that buys is a readable failure. Invert the table and exactly this suite +# goes red, naming the table. Drop the parser from one tier's receiver and +# exactly that tier goes red, naming the tier. Neither failure makes you read +# the other one to work out which of the two happened. +# +# Needs no `nixosSystem` fixture at all — the thing under test is a file — so +# this suite costs nothing to evaluate next to its siblings. +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + journaldSeverityParser + runGroup + ; + + # The parser's own mapping read back as a plain `PRIORITY -> severity-name` + # function, so the case below can ask what a priority MAPS TO instead of + # matching on the shape of the attrset that produces it. `null` for a + # priority the table does not name, which the case uses as its control. + severityOf = + priority: + let + hits = lib.attrNames ( + lib.filterAttrs (_: v: lib.elem priority (lib.toList v)) journaldSeverityParser.mapping + ); + in + if journaldSeverityParser == null || hits == [ ] then null else lib.head hits; + + cases = [ + { + # ⚠️ The direction, and the only part of this mapping a reviewer cannot + # check by looking. syslog's PRIORITY counts DOWN in urgency (0 = emerg, + # 7 = debug) where the OpenTelemetry severity counts UP (1 = trace, + # 21 = fatal). Wired across, debug renders as critical and emerg as + # unspecified — and nothing reports an error, because every line still + # arrives and every field is still populated. + # + # Every priority, not a sample. An inverted copy still maps each value + # to SOMETHING, so a case that checks one end, or checks that a mapping + # merely exists, passes on the exact defect it was written for. The + # `"8"` arm is the control: it says this reader looks values up rather + # than answering the same way whatever it is handed, so the eight + # arms above it mean what they appear to mean. + name = "journald PRIORITY maps to severity the right way up"; + ok = + severityOf "0" == "fatal" + && severityOf "1" == "fatal" + && severityOf "2" == "fatal" + && severityOf "3" == "error" + && severityOf "4" == "warn" + && severityOf "5" == "info" + && severityOf "6" == "info" + && severityOf "7" == "debug" + && severityOf "8" == null; + } + { + # The second way this mapping can be present and useless, and the one + # that looks already handled. Without `overwrite_text` the parser sets + # the severity NUMBER from the table above and leaves the severity TEXT + # as the raw value it matched — so `severity_text` reaches the log store + # as the literal "6". That is a populated field: it passes any check + # asking merely whether severity is set, and it is not a level anything + # renders. VictoriaLogs has no ingest parameter naming a level field, so + # the text is the whole interface. + name = "the mapping writes a level name, not the raw priority digit"; + ok = journaldSeverityParser.overwrite_text or false; + } + { + # Two settings that are load-bearing in opposite directions. + # + # `parse_from` names where the value is. The journald input unmarshals + # each `journalctl -o json` line into the entry BODY as a map, so the + # field lives under `body.`; `attributes.PRIORITY` is the spelling that + # looks equally plausible and finds nothing, leaving a parser that + # reports a missing field per entry rather than failing to start. + # + # `on_error` decides what happens to a record it cannot read. It must + # still REACH the store: `drop` would make the logs dashboard's + # "lines with no severity" panel read zero by deleting its own evidence, + # which is the empty-result-as-success failure this whole mapping exists + # to end. + name = "the mapping reads the field the receiver fills, and keeps what it cannot read"; + ok = + journaldSeverityParser.parse_from or null == "body.PRIORITY" + && journaldSeverityParser.on_error or null == "send"; + } + ]; +in +runGroup "journald-severity" cases diff --git a/nix/module-eval/lib.nix b/nix/module-eval/lib.nix index 79f3f4fa..cc25daf6 100644 --- a/nix/module-eval/lib.nix +++ b/nix/module-eval/lib.nix @@ -140,33 +140,23 @@ let agentHarness = machine: machine.systemd.services.hive-agent; - # Reads a journald receiver's rendered operator list back as a plain - # `PRIORITY -> severity-name` function, so a case can ask what a given - # priority actually maps to rather than matching on the shape of the config - # that produces it. Shared because two tiers run that receiver — the agent - # container over its own journal, the swarm collector over the host's — from - # one imported mapping, and the case that says so has to reach both. - # `null` for a receiver carrying no severity parser at all, which is the - # state that mapping replaced. - journaldSeverityOf = - receiver: priority: - let - parser = lib.findFirst (o: o.type or null == "severity_parser") null (receiver.operators or [ ]); - # The mapping is `severity-name -> value or list of values`; invert it - # into `value -> severity-name` so a lookup is by priority. - hits = lib.attrNames (lib.filterAttrs (_: v: lib.elem priority (lib.toList v)) parser.mapping); - in - if parser == null || hits == [ ] then null else lib.head hits; + # The PRIORITY → severity operator both journald receivers attach, read + # straight off the file that defines it. What each TIER's case then asks is + # only whether its own receiver carries this exact value — the contents are + # somebody else's case (./journald-severity.nix), asserted once. + # + # Here rather than in one of the suites because three of them want it: the + # two tiers, to say they are wired to it, and the suite that reads what it + # says. A parser rather than the whole list, so a tier that later grows an + # unrelated operator of its own still passes the wiring case. + journaldSeverityParser = lib.findFirst (o: o.type or null == "severity_parser") null ( + import ../journald-severity.nix + ); - # Whether that same parser rewrites the severity TEXT. Its own reader, - # because it is a separate way for the mapping to be there and useless: see - # the case that asserts it. - journaldSeverityOverwritesText = - receiver: - let - parser = lib.findFirst (o: o.type or null == "severity_parser") { } (receiver.operators or [ ]); - in - parser.overwrite_text or false; + # Whether a given receiver is wired to it. One expression, used by both + # tiers, so "this receiver carries the shared mapping" is spelled once and + # the two tier cases differ only in which receiver they hand it. + carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]); # The enables that switch owns. ⚠️ `otel` is the per-hive collector's own # option and is NOT under `deploy` — spelled at the wrong path it would be @@ -225,8 +215,8 @@ in inherit baoSettings; inherit otelSettings; inherit agentHarness; - inherit journaldSeverityOf; - inherit journaldSeverityOverwritesText; + inherit journaldSeverityParser; + inherit carriesJournaldSeverity; inherit swarmServiceEnables; inherit runGroup; } diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix index 8673b464..7428919c 100644 --- a/nix/module-eval/swarm-otel-core.nix +++ b/nix/module-eval/swarm-otel-core.nix @@ -18,8 +18,7 @@ let ; }) hive - journaldSeverityOf - journaldSeverityOverwritesText + carriesJournaldSeverity runGroup otelSettings ; @@ -158,32 +157,17 @@ let named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named; } { - # The host-journal half of the mapping the agent tier's receiver carries - # (./agent-otel.nix holds the sibling case and the full reasoning). Both - # import one file, so this is what says the swarm collector still does — - # the receivers are otherwise unrelated config, and a drifted copy is - # silent: every line still arrives, labelled as the wrong thing. + # The host-journal sibling of ./agent-otel.nix's wiring case, which + # carries the full reasoning. Same question, different receiver: this + # one reads the HOST's journal rather than a container's, and the two + # are unrelated config — a fixed stanza there, a parameterised block + # inside `containers.swarm-otel` here — so one losing its parser while + # the other keeps one is a real and silent state. # - # Asserted at both ends of the table because the direction is inverted - # and an inverted copy still maps every value to something. - name = "the swarm collector maps the host journal's PRIORITY the right way up"; - ok = - let - sev = journaldSeverityOf (otelSettings otelNoStores).receivers.journald; - in - sev "0" == "fatal" - && sev "3" == "error" - && sev "4" == "warn" - && sev "6" == "info" - && sev "7" == "debug"; - } - { - # Same reasoning as the agent tier's copy: without this the parser - # populates `severity_text` with the raw digit it matched, which passes - # any check asking whether severity is set and is not a level anything - # renders. - name = "the swarm collector writes a level name, not the raw priority digit"; - ok = journaldSeverityOverwritesText (otelSettings otelNoStores).receivers.journald; + # Contents are not this case's business. The table both receivers import + # is asserted once, in ./journald-severity.nix. + name = "the swarm collector's journald receiver carries the shared PRIORITY mapping"; + ok = carriesJournaldSeverity (otelSettings otelNoStores).receivers.journald; } ]; in