diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index d4b7d66e..b7d17749 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -216,6 +216,38 @@ Append additional labels via `extraResourceAttributes` (see option reference above); pass custom per-data-point labels with `hive-metric --labels` (see below). +## Log severity + +Every journald receiver in the pipeline maps the journal's `PRIORITY` onto an +OpenTelemetry severity, so a line arrives in the log store already carrying its +level and `severity_text:ERROR` is a query you can write: + +| `PRIORITY` | syslog | severity | +| ---------- | ------------------ | -------- | +| 0, 1, 2 | emerg, alert, crit | `FATAL` | +| 3 | err | `ERROR` | +| 4 | warning | `WARN` | +| 5, 6 | notice, info | `INFO` | +| 7 | debug | `DEBUG` | + +Both tiers that read a journal do this — the agent container's forwarder over +its own journal, and the swarm collector over the host's — and both import the +same mapping from `nix/journald-severity.nix`, which holds the table once. + +Note the direction. Syslog's priority counts **down** in urgency, where +OpenTelemetry's severity number counts **up**. Neither is a passthrough of the +other, and a pipeline that hands one straight to the other renders debug lines +as critical without erroring anywhere. + +A record that reaches the store from somewhere other than a journal has no +priority to map. Claude Code's own telemetry is the live example: it exports +OTLP log records directly (`OTEL_LOGS_EXPORTER`, see +`nix/agent-modules/claude-settings.nix`) and sets no severity on them, so they +store as `Unspecified`. The **Log rows with no severity** panel on the +`hyperhive · logs` dashboard counts both cases and keeps them apart: lines that +had a priority and arrived without a severity anyway signal a broken mapping, +while lines that never had one signal nothing of the sort. + ## Host-emitted container-resource metrics (hive-c0re) When OTEL is enabled, **hive-c0re itself** also exports each agent diff --git a/nix/agent-modules/otel.nix b/nix/agent-modules/otel.nix index f06cd8e8..2405f35c 100644 --- a/nix/agent-modules/otel.nix +++ b/nix/agent-modules/otel.nix @@ -265,6 +265,14 @@ in # and what the harness spawns, so there is no foreign traffic to # filter out and an allowlist would only be a list to forget to # update. + + # journald's PRIORITY carries the level every line already has; + # without this the record reaches VictoriaLogs with + # `severity_text: Unspecified` and the store cannot tell an error + # from a debug line. The table — and the inverted direction that + # makes it worth a file of its own — lives in + # ../journald-severity.nix, shared with the swarm tier's receiver. + operators = import ../journald-severity.nix; }; # Identity the hop above cannot supply. A host-side reader can say diff --git a/nix/checks.nix b/nix/checks.nix index b8513be0..29f0ee56 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -91,6 +91,15 @@ in inherit pkgs self nixosSystem; inherit (pkgs) lib; }; + # Not a subsystem cluster like its siblings but a single shared file, and + # its own derivation for that reason: both journald receivers import one + # PRIORITY → severity mapping, so asserting the table inside each tier's + # suite would check one file twice and report one defect as two failures. + # Holds no fixture, so it costs nothing next to the clusters. + module-eval-journald-severity = import ./module-eval/journald-severity.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; module-eval-agent-queue-bao = import ./module-eval/agent-queue-bao.nix { inherit pkgs self nixosSystem; inherit (pkgs) lib; diff --git a/nix/host-modules/swarm-grafana/dashboards/logstore.json b/nix/host-modules/swarm-grafana/dashboards/logstore.json index ce5dd7d4..8e92ff70 100644 --- a/nix/host-modules/swarm-grafana/dashboards/logstore.json +++ b/nix/host-modules/swarm-grafana/dashboards/logstore.json @@ -339,7 +339,7 @@ "h": 12, "w": 12, "x": 0, - "y": 24 + "y": 33 }, "targets": [ { @@ -391,7 +391,7 @@ "h": 12, "w": 12, "x": 12, - "y": 24 + "y": 33 }, "targets": [ { @@ -443,7 +443,7 @@ "h": 9, "w": 24, "x": 0, - "y": 36 + "y": 45 }, "targets": [ { @@ -487,6 +487,103 @@ "sort": "desc" } } + }, + { + "id": 9, + "type": "timeseries", + "title": "Log rows with no severity (must be zero)", + "description": "This panel is a check rather than a view: the collector maps journald's PRIORITY onto an OpenTelemetry severity at every tier, and this is what says out loud whether a line got one. Both series are rows whose `severity_text` is absent or the literal `Unspecified`, which is what VictoriaLogs stores when nothing set a severity — split by whether the row had a PRIORITY to map in the first place.\n\n**`journald_priority_unmapped`** is a row that carries PRIORITY and still arrived without a severity. It is the regression line: nonzero means a journald receiver somewhere lost its severity_parser operator, so read it against `nix/journald-severity.nix` and the two receivers that import it.\n\n**`no_priority_field`** is a row that never had a priority — today that is Claude Code's own OTLP telemetry (`scope.name: com.anthropic.claude_code.events`), which emits log records with no severity set at the source. Mapping cannot reach it; it needs a fix where it is produced, so this series is expected nonzero until that lands and is NOT evidence the mapping broke.\n\nRange-scoped over time on purpose. An instantaneous zero is also what a broken query returns — a line you can watch go to zero and stay there is the part that survives whoever wrote it. To sanity-check the query itself, read it against `Rows in range · all sources (control)` above: that panel nonzero while both of these are flat zero means these queries stopped matching, not that every line grew a severity.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 24 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "queryType": "statsRange", + "expr": "(severity_text:\"\" OR severity_text:\"Unspecified\") PRIORITY:* | stats count() as journald_priority_unmapped" + }, + { + "refId": "B", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "queryType": "statsRange", + "expr": "(severity_text:\"\" OR severity_text:\"Unspecified\") PRIORITY:\"\" | stats count() as no_priority_field" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "journald_priority_unmapped" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "no_priority_field" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["max", "lastNotNull"] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } } ] } diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 66d35098..305f8bad 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -1208,6 +1208,16 @@ in journald = { directory = hostJournalDir; units = cfg.journaldUnits; + # The SAME operator list the agent tier attaches to its own + # receiver (nix/agent-modules/otel.nix), and it transfers + # without adjustment: the entry shape is the receiver's, + # not the journal's. Both are this one component running + # `journalctl -o json`, so `PRIORITY` is spelled and typed + # identically whether the directory it was pointed at holds + # a host's journal or a container's. What differs between + # the two receivers is which journal and which units — + # neither of which the parser reads. + operators = import ../journald-severity.nix; }; }; diff --git a/nix/journald-severity.nix b/nix/journald-severity.nix new file mode 100644 index 00000000..793a6762 --- /dev/null +++ b/nix/journald-severity.nix @@ -0,0 +1,67 @@ +# journald `PRIORITY` → OpenTelemetry severity, as the stanza operator list +# every journald receiver in this tree attaches. The table it produces and +# what a consumer does with it: docs/scheduler/observability.md, "Log severity". +# +# ONE file, imported by both tiers (`agent-modules/otel.nix` reads the +# container's own journal, `host-modules/swarm-otel.nix` the host's), because +# the mapping is the part that must not drift. The two receivers are otherwise +# unrelated — one a fixed stanza, one parameterised inside a container config — +# so a copy in each is two tables to keep in step, and getting one wrong is +# silent: every line still arrives, it just arrives labelled as the wrong thing. +# +# ⚠️ **The direction is inverted.** syslog PRIORITY counts DOWN in urgency +# (0 = emerg, 7 = debug) where OpenTelemetry's severity number counts UP +# (1 = trace, 21 = fatal). Passed through as a severity number, debug renders +# as critical and emerg as unspecified, and nothing reports an error. +[ + { + type = "severity_parser"; + + # The journald input unmarshals each `journalctl -o json` line into the + # entry BODY as a map, so every journal field is addressed under `body.`. + # `attributes.PRIORITY` is the spelling that looks equally plausible and + # finds nothing — the parser then reports a missing field per entry rather + # than failing to start. + parse_from = "body.PRIORITY"; + + # 🔑 NOT optional, and the single easiest thing to leave off. By default the + # severity parser sets the severity NUMBER from the mapping and leaves the + # severity TEXT as the original value it matched on — so `severity_text` + # would arrive in VictoriaLogs as the literal string `"6"`. That is a + # populated field, it passes every check that asks whether severity is set, + # and it is not a level any consumer renders. With this on, the text is the + # recommended short name for the mapped number: `FATAL`, `ERROR`, `WARN`, + # `INFO`, `DEBUG`. The TEXT is the whole interface here — VictoriaLogs has + # no ingest parameter naming a level field, it auto-detects one by trying + # `severity_text`, `SeverityText`, `severity`, `Severity` in that order, so + # the value has to arrive already spelled as a word it knows. + overwrite_text = true; + + # A record whose PRIORITY the parser cannot read must still REACH the + # store. Explicit rather than inherited from the operator default, because + # the alternative (`drop`) would make the "lines with no severity" panel on + # the logs dashboard read zero by deleting its own evidence — the exact + # empty-result-as-success failure this mapping exists to end. + on_error = "send"; + + # Strings, because that is what arrives: `journalctl -o json` renders every + # journal field as a JSON string, so `PRIORITY` is `"6"` and not `6`. The + # parser stringifies its mapping values too, so an integer spelling here + # would also match — the strings are for the reader, to keep the config + # shaped like the data. + mapping = { + fatal = [ + "0" + "1" + "2" + ]; + error = "3"; + warn = "4"; + info = [ + "5" + "6" + ]; + debug = "7"; + }; + } +] diff --git a/nix/module-eval/agent-otel.nix b/nix/module-eval/agent-otel.nix index a843362f..8c596e1d 100644 --- a/nix/module-eval/agent-otel.nix +++ b/nix/module-eval/agent-otel.nix @@ -18,6 +18,7 @@ let ; }) agent + carriesJournaldSeverity runGroup ; @@ -56,6 +57,30 @@ let name = "the agent forwarder reads the persistent journal, not the runtime one"; ok = ((agentSettings agentOtel).receivers.journald.directory or null) == "/var/log/journal"; } + { + # Wiring, not contents. A journald record arrives with its level in + # `PRIORITY` and nothing downstream reads that field, so a receiver with + # no severity parser ships every line to the log store labelled + # `Unspecified` — healthy, complete, and unqueryable by level. + # + # What this asks is only whether THIS receiver carries the shared + # mapping. ⚠️ Whether that mapping is RIGHT — the inverted direction, + # which is the part of it a reviewer cannot check by looking — is + # asserted once, in ./journald-severity.nix, against the file both tiers + # import. Re-asserting the table here would test that file twice and + # report one defect as two failures. + # + # The swarm collector's own receiver gets the sibling of this case in + # ./swarm-otel-core.nix. Deliberately NOT folded together with it: they + # cover different journals — this container's own, and the host's — and + # one tier quietly losing its parser while the other keeps one is + # precisely the half-fixed state worth catching. + # + # Membership rather than equality of the whole operator list, so a tier + # that later grows an unrelated operator of its own still passes. + name = "the agent forwarder's journald receiver carries the shared PRIORITY mapping"; + ok = carriesJournaldSeverity (agentSettings agentOtel).receivers.journald; + } { # The hop's two ends: what it reads, and where what it reads goes. # The endpoint is compared against the value the fixture handed the diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 25442c56..9ade378d 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -217,6 +217,32 @@ let grafanaRemoteAuthelia.services.hyperhive.swarm.statusPublish.tokenEndpoint == "https://auth.example.invalid/api/oidc/token"; } + { + # The standing check on the collector's PRIORITY mapping — whether a line + # ever reaches the log store carrying no severity. It is shipped config + # rather than something an operator imports by hand, which also means it + # can be deleted by hand. Read out of the dashboard the provisioner + # actually ships (`dashboardDir` in ../host-modules/swarm-grafana.nix + # names this file), and asserted on the QUERY rather than the title, + # because a panel that keeps its title and loses its expression renders + # an empty graph that looks exactly like zero prioless lines. + name = "the logs dashboard still counts lines that arrive with no severity"; + ok = + let + board = builtins.fromJSON ( + builtins.readFile ../host-modules/swarm-grafana/dashboards/logstore.json + ); + exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels; + counts = lib.filter (e: lib.hasInfix "severity_text" e && lib.hasInfix "stats count()" e) exprs; + in + # Both arms of the split: lines that HAD a priority and arrived without + # a severity anyway (the regression line, which must reach zero), and + # lines that never had one to map — Claude Code's own OTLP telemetry, + # which no mapping can reach. Folded into one number the second would + # keep the panel off zero forever and hide the first. + lib.any (e: lib.hasInfix "PRIORITY:*" e) counts + && lib.any (e: lib.hasInfix "PRIORITY:\"\"" e) counts; + } ]; in runGroup "grafana" cases diff --git a/nix/module-eval/journald-severity.nix b/nix/module-eval/journald-severity.nix new file mode 100644 index 00000000..265eb224 --- /dev/null +++ b/nix/module-eval/journald-severity.nix @@ -0,0 +1,113 @@ +# `checks.module-eval-journald-severity` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +# +# The odd one out in this directory: every sibling suite evaluates a host or an +# agent and reads the config that falls out, where this one reads a single file +# that is already a value. It is a suite of its own for the reason the operator +# gave when she saw the mapping asserted in two tiers at once — **test the +# thing only once**. The table lives in one file, so its contents are one +# subject and belong to one case. Each tier's suite then asks a different +# question, about wiring rather than contents: does MY receiver carry this. +# +# What that buys is a readable failure. Invert the table and exactly this suite +# goes red, naming the table. Drop the parser from one tier's receiver and +# exactly that tier goes red, naming the tier. Neither failure makes you read +# the other one to work out which of the two happened. +# +# Needs no `nixosSystem` fixture at all — the thing under test is a file — so +# this suite costs nothing to evaluate next to its siblings. +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + journaldSeverityParser + runGroup + ; + + # The parser's own mapping read back as a plain `PRIORITY -> severity-name` + # function, so the case below can ask what a priority MAPS TO instead of + # matching on the shape of the attrset that produces it. `null` for a + # priority the table does not name, which the case uses as its control. + severityOf = + priority: + let + hits = lib.attrNames ( + lib.filterAttrs (_: v: lib.elem priority (lib.toList v)) journaldSeverityParser.mapping + ); + in + if journaldSeverityParser == null || hits == [ ] then null else lib.head hits; + + cases = [ + { + # ⚠️ The direction, and the only part of this mapping a reviewer cannot + # check by looking. syslog's PRIORITY counts DOWN in urgency (0 = emerg, + # 7 = debug) where the OpenTelemetry severity counts UP (1 = trace, + # 21 = fatal). Wired across, debug renders as critical and emerg as + # unspecified — and nothing reports an error, because every line still + # arrives and every field is still populated. + # + # Every priority, not a sample. An inverted copy still maps each value + # to SOMETHING, so a case that checks one end, or checks that a mapping + # merely exists, passes on the exact defect it was written for. The + # `"8"` arm is the control: it says this reader looks values up rather + # than answering the same way whatever it is handed, so the eight + # arms above it mean what they appear to mean. + name = "journald PRIORITY maps to severity the right way up"; + ok = + severityOf "0" == "fatal" + && severityOf "1" == "fatal" + && severityOf "2" == "fatal" + && severityOf "3" == "error" + && severityOf "4" == "warn" + && severityOf "5" == "info" + && severityOf "6" == "info" + && severityOf "7" == "debug" + && severityOf "8" == null; + } + { + # The second way this mapping can be present and useless, and the one + # that looks already handled. Without `overwrite_text` the parser sets + # the severity NUMBER from the table above and leaves the severity TEXT + # as the raw value it matched — so `severity_text` reaches the log store + # as the literal "6". That is a populated field: it passes any check + # asking merely whether severity is set, and it is not a level anything + # renders. VictoriaLogs has no ingest parameter naming a level field, so + # the text is the whole interface. + name = "the mapping writes a level name, not the raw priority digit"; + ok = journaldSeverityParser.overwrite_text or false; + } + { + # Two settings that are load-bearing in opposite directions. + # + # `parse_from` names where the value is. The journald input unmarshals + # each `journalctl -o json` line into the entry BODY as a map, so the + # field lives under `body.`; `attributes.PRIORITY` is the spelling that + # looks equally plausible and finds nothing, leaving a parser that + # reports a missing field per entry rather than failing to start. + # + # `on_error` decides what happens to a record it cannot read. It must + # still REACH the store: `drop` would make the logs dashboard's + # "lines with no severity" panel read zero by deleting its own evidence, + # which is the empty-result-as-success failure this whole mapping exists + # to end. + name = "the mapping reads the field the receiver fills, and keeps what it cannot read"; + ok = + journaldSeverityParser.parse_from or null == "body.PRIORITY" + && journaldSeverityParser.on_error or null == "send"; + } + ]; +in +runGroup "journald-severity" cases diff --git a/nix/module-eval/lib.nix b/nix/module-eval/lib.nix index 016bfdd7..cc25daf6 100644 --- a/nix/module-eval/lib.nix +++ b/nix/module-eval/lib.nix @@ -140,6 +140,24 @@ let agentHarness = machine: machine.systemd.services.hive-agent; + # The PRIORITY → severity operator both journald receivers attach, read + # straight off the file that defines it. What each TIER's case then asks is + # only whether its own receiver carries this exact value — the contents are + # somebody else's case (./journald-severity.nix), asserted once. + # + # Here rather than in one of the suites because three of them want it: the + # two tiers, to say they are wired to it, and the suite that reads what it + # says. A parser rather than the whole list, so a tier that later grows an + # unrelated operator of its own still passes the wiring case. + journaldSeverityParser = lib.findFirst (o: o.type or null == "severity_parser") null ( + import ../journald-severity.nix + ); + + # Whether a given receiver is wired to it. One expression, used by both + # tiers, so "this receiver carries the shared mapping" is spelled once and + # the two tier cases differ only in which receiver they hand it. + carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]); + # The enables that switch owns. ⚠️ `otel` is the per-hive collector's own # option and is NOT under `deploy` — spelled at the wrong path it would be # undeclared rather than false, and a roster that quietly loses a member is @@ -197,6 +215,8 @@ in inherit baoSettings; inherit otelSettings; inherit agentHarness; + inherit journaldSeverityParser; + inherit carriesJournaldSeverity; inherit swarmServiceEnables; inherit runGroup; } diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix index c4433f39..7428919c 100644 --- a/nix/module-eval/swarm-otel-core.nix +++ b/nix/module-eval/swarm-otel-core.nix @@ -18,6 +18,7 @@ let ; }) hive + carriesJournaldSeverity runGroup otelSettings ; @@ -155,6 +156,19 @@ let in named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named; } + { + # The host-journal sibling of ./agent-otel.nix's wiring case, which + # carries the full reasoning. Same question, different receiver: this + # one reads the HOST's journal rather than a container's, and the two + # are unrelated config — a fixed stanza there, a parameterised block + # inside `containers.swarm-otel` here — so one losing its parser while + # the other keeps one is a real and silent state. + # + # Contents are not this case's business. The table both receivers import + # is asserted once, in ./journald-severity.nix. + name = "the swarm collector's journald receiver carries the shared PRIORITY mapping"; + ok = carriesJournaldSeverity (otelSettings otelNoStores).receivers.journald; + } ]; in runGroup "swarm-otel-core" cases