module-eval: pin the severity mapping's direction and the panel
The direction is the part a reviewer cannot check by looking, so it is asserted at both ends of the table and in both tiers' groups: an inverted mapping still maps every value to something, and a case that only asks whether a severity parser exists passes on the exact defect. The reader that turns a rendered operator list back into a PRIORITY -> name function lives in lib.nix, since both tiers need it. The panel is asserted on its query rather than its title, because a panel that keeps the title and loses the expression renders an empty graph that looks exactly like zero prioless lines.
This commit is contained in:
parent
83a604ae23
commit
7fa13b592f
4 changed files with 121 additions and 0 deletions
|
|
@ -18,6 +18,8 @@ let
|
|||
;
|
||||
})
|
||||
agent
|
||||
journaldSeverityOf
|
||||
journaldSeverityOverwritesText
|
||||
runGroup
|
||||
;
|
||||
|
||||
|
|
@ -56,6 +58,39 @@ let
|
|||
name = "the agent forwarder reads the persistent journal, not the runtime one";
|
||||
ok = ((agentSettings agentOtel).receivers.journald.directory or null) == "/var/log/journal";
|
||||
}
|
||||
{
|
||||
# ⚠️ The direction, and the only part of this mapping a reviewer cannot
|
||||
# check by looking: syslog's PRIORITY counts DOWN in urgency (0 = emerg,
|
||||
# 7 = debug) where the OTEL severity counts UP, so a copy wired across
|
||||
# renders debug as critical while every line still arrives and every
|
||||
# field is populated. Asserted at BOTH ENDS of the table — an inverted
|
||||
# mapping still maps every value to something, so "a severity parser is
|
||||
# configured" passes on the exact defect. The swarm tier's receiver gets
|
||||
# the same case over in ./swarm-otel-core.nix; the two import one file,
|
||||
# and these are what say they still do.
|
||||
name = "the agent forwarder maps PRIORITY to severity the right way up";
|
||||
ok =
|
||||
let
|
||||
sev = journaldSeverityOf (agentSettings agentOtel).receivers.journald;
|
||||
in
|
||||
sev "0" == "fatal"
|
||||
&& sev "3" == "error"
|
||||
&& sev "4" == "warn"
|
||||
&& sev "6" == "info"
|
||||
&& sev "7" == "debug";
|
||||
}
|
||||
{
|
||||
# The other half of the same operator, and the half that looks already
|
||||
# handled. Without `overwrite_text` the parser sets the severity NUMBER
|
||||
# from the mapping and leaves the severity TEXT as the raw value it
|
||||
# matched — so `severity_text` reaches the log store as the literal "6".
|
||||
# That is a populated field which passes any check asking merely whether
|
||||
# severity is set, and which nothing renders as a level. VictoriaLogs has
|
||||
# no ingest parameter naming a level field, so the text is the whole
|
||||
# interface.
|
||||
name = "the agent forwarder writes a level name, not the raw priority digit";
|
||||
ok = journaldSeverityOverwritesText (agentSettings agentOtel).receivers.journald;
|
||||
}
|
||||
{
|
||||
# The hop's two ends: what it reads, and where what it reads goes.
|
||||
# The endpoint is compared against the value the fixture handed the
|
||||
|
|
|
|||
|
|
@ -217,6 +217,32 @@ let
|
|||
grafanaRemoteAuthelia.services.hyperhive.swarm.statusPublish.tokenEndpoint
|
||||
== "https://auth.example.invalid/api/oidc/token";
|
||||
}
|
||||
{
|
||||
# The standing check on the collector's PRIORITY mapping — whether a line
|
||||
# ever reaches the log store carrying no severity. It is shipped config
|
||||
# rather than something an operator imports by hand, which also means it
|
||||
# can be deleted by hand. Read out of the dashboard the provisioner
|
||||
# actually ships (`dashboardDir` in ../host-modules/swarm-grafana.nix
|
||||
# names this file), and asserted on the QUERY rather than the title,
|
||||
# because a panel that keeps its title and loses its expression renders
|
||||
# an empty graph that looks exactly like zero prioless lines.
|
||||
name = "the logs dashboard still counts lines that arrive with no severity";
|
||||
ok =
|
||||
let
|
||||
board = builtins.fromJSON (
|
||||
builtins.readFile ../host-modules/swarm-grafana/dashboards/logstore.json
|
||||
);
|
||||
exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels;
|
||||
counts = lib.filter (e: lib.hasInfix "severity_text" e && lib.hasInfix "stats count()" e) exprs;
|
||||
in
|
||||
# Both arms of the split: lines that HAD a priority and arrived without
|
||||
# a severity anyway (the regression line, which must reach zero), and
|
||||
# lines that never had one to map — Claude Code's own OTLP telemetry,
|
||||
# which no mapping can reach. Folded into one number the second would
|
||||
# keep the panel off zero forever and hide the first.
|
||||
lib.any (e: lib.hasInfix "PRIORITY:*" e) counts
|
||||
&& lib.any (e: lib.hasInfix "PRIORITY:\"\"" e) counts;
|
||||
}
|
||||
];
|
||||
in
|
||||
runGroup "grafana" cases
|
||||
|
|
|
|||
|
|
@ -140,6 +140,34 @@ let
|
|||
|
||||
agentHarness = machine: machine.systemd.services.hive-agent;
|
||||
|
||||
# Reads a journald receiver's rendered operator list back as a plain
|
||||
# `PRIORITY -> severity-name` function, so a case can ask what a given
|
||||
# priority actually maps to rather than matching on the shape of the config
|
||||
# that produces it. Shared because two tiers run that receiver — the agent
|
||||
# container over its own journal, the swarm collector over the host's — from
|
||||
# one imported mapping, and the case that says so has to reach both.
|
||||
# `null` for a receiver carrying no severity parser at all, which is the
|
||||
# state that mapping replaced.
|
||||
journaldSeverityOf =
|
||||
receiver: priority:
|
||||
let
|
||||
parser = lib.findFirst (o: o.type or null == "severity_parser") null (receiver.operators or [ ]);
|
||||
# The mapping is `severity-name -> value or list of values`; invert it
|
||||
# into `value -> severity-name` so a lookup is by priority.
|
||||
hits = lib.attrNames (lib.filterAttrs (_: v: lib.elem priority (lib.toList v)) parser.mapping);
|
||||
in
|
||||
if parser == null || hits == [ ] then null else lib.head hits;
|
||||
|
||||
# Whether that same parser rewrites the severity TEXT. Its own reader,
|
||||
# because it is a separate way for the mapping to be there and useless: see
|
||||
# the case that asserts it.
|
||||
journaldSeverityOverwritesText =
|
||||
receiver:
|
||||
let
|
||||
parser = lib.findFirst (o: o.type or null == "severity_parser") { } (receiver.operators or [ ]);
|
||||
in
|
||||
parser.overwrite_text or false;
|
||||
|
||||
# The enables that switch owns. ⚠️ `otel` is the per-hive collector's own
|
||||
# option and is NOT under `deploy` — spelled at the wrong path it would be
|
||||
# undeclared rather than false, and a roster that quietly loses a member is
|
||||
|
|
@ -197,6 +225,8 @@ in
|
|||
inherit baoSettings;
|
||||
inherit otelSettings;
|
||||
inherit agentHarness;
|
||||
inherit journaldSeverityOf;
|
||||
inherit journaldSeverityOverwritesText;
|
||||
inherit swarmServiceEnables;
|
||||
inherit runGroup;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@ let
|
|||
;
|
||||
})
|
||||
hive
|
||||
journaldSeverityOf
|
||||
journaldSeverityOverwritesText
|
||||
runGroup
|
||||
otelSettings
|
||||
;
|
||||
|
|
@ -155,6 +157,34 @@ let
|
|||
in
|
||||
named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named;
|
||||
}
|
||||
{
|
||||
# The host-journal half of the mapping the agent tier's receiver carries
|
||||
# (./agent-otel.nix holds the sibling case and the full reasoning). Both
|
||||
# import one file, so this is what says the swarm collector still does —
|
||||
# the receivers are otherwise unrelated config, and a drifted copy is
|
||||
# silent: every line still arrives, labelled as the wrong thing.
|
||||
#
|
||||
# Asserted at both ends of the table because the direction is inverted
|
||||
# and an inverted copy still maps every value to something.
|
||||
name = "the swarm collector maps the host journal's PRIORITY the right way up";
|
||||
ok =
|
||||
let
|
||||
sev = journaldSeverityOf (otelSettings otelNoStores).receivers.journald;
|
||||
in
|
||||
sev "0" == "fatal"
|
||||
&& sev "3" == "error"
|
||||
&& sev "4" == "warn"
|
||||
&& sev "6" == "info"
|
||||
&& sev "7" == "debug";
|
||||
}
|
||||
{
|
||||
# Same reasoning as the agent tier's copy: without this the parser
|
||||
# populates `severity_text` with the raw digit it matched, which passes
|
||||
# any check asking whether severity is set and is not a level anything
|
||||
# renders.
|
||||
name = "the swarm collector writes a level name, not the raw priority digit";
|
||||
ok = journaldSeverityOverwritesText (otelSettings otelNoStores).receivers.journald;
|
||||
}
|
||||
];
|
||||
in
|
||||
runGroup "swarm-otel-core" cases
|
||||
|
|
|
|||
Loading…
Reference in a new issue