diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index b7d17749..d4b7d66e 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -216,38 +216,6 @@ Append additional labels via `extraResourceAttributes` (see option reference above); pass custom per-data-point labels with `hive-metric --labels` (see below). -## Log severity - -Every journald receiver in the pipeline maps the journal's `PRIORITY` onto an -OpenTelemetry severity, so a line arrives in the log store already carrying its -level and `severity_text:ERROR` is a query you can write: - -| `PRIORITY` | syslog | severity | -| ---------- | ------------------ | -------- | -| 0, 1, 2 | emerg, alert, crit | `FATAL` | -| 3 | err | `ERROR` | -| 4 | warning | `WARN` | -| 5, 6 | notice, info | `INFO` | -| 7 | debug | `DEBUG` | - -Both tiers that read a journal do this — the agent container's forwarder over -its own journal, and the swarm collector over the host's — and both import the -same mapping from `nix/journald-severity.nix`, which holds the table once. - -Note the direction. Syslog's priority counts **down** in urgency, where -OpenTelemetry's severity number counts **up**. Neither is a passthrough of the -other, and a pipeline that hands one straight to the other renders debug lines -as critical without erroring anywhere. - -A record that reaches the store from somewhere other than a journal has no -priority to map. Claude Code's own telemetry is the live example: it exports -OTLP log records directly (`OTEL_LOGS_EXPORTER`, see -`nix/agent-modules/claude-settings.nix`) and sets no severity on them, so they -store as `Unspecified`. The **Log rows with no severity** panel on the -`hyperhive · logs` dashboard counts both cases and keeps them apart: lines that -had a priority and arrived without a severity anyway signal a broken mapping, -while lines that never had one signal nothing of the sort. - ## Host-emitted container-resource metrics (hive-c0re) When OTEL is enabled, **hive-c0re itself** also exports each agent diff --git a/nix/agent-modules/otel.nix b/nix/agent-modules/otel.nix index 2405f35c..f06cd8e8 100644 --- a/nix/agent-modules/otel.nix +++ b/nix/agent-modules/otel.nix @@ -265,14 +265,6 @@ in # and what the harness spawns, so there is no foreign traffic to # filter out and an allowlist would only be a list to forget to # update. - - # journald's PRIORITY carries the level every line already has; - # without this the record reaches VictoriaLogs with - # `severity_text: Unspecified` and the store cannot tell an error - # from a debug line. The table — and the inverted direction that - # makes it worth a file of its own — lives in - # ../journald-severity.nix, shared with the swarm tier's receiver. - operators = import ../journald-severity.nix; }; # Identity the hop above cannot supply. A host-side reader can say diff --git a/nix/checks.nix b/nix/checks.nix index 29f0ee56..b8513be0 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -91,15 +91,6 @@ in inherit pkgs self nixosSystem; inherit (pkgs) lib; }; - # Not a subsystem cluster like its siblings but a single shared file, and - # its own derivation for that reason: both journald receivers import one - # PRIORITY → severity mapping, so asserting the table inside each tier's - # suite would check one file twice and report one defect as two failures. - # Holds no fixture, so it costs nothing next to the clusters. - module-eval-journald-severity = import ./module-eval/journald-severity.nix { - inherit pkgs self nixosSystem; - inherit (pkgs) lib; - }; module-eval-agent-queue-bao = import ./module-eval/agent-queue-bao.nix { inherit pkgs self nixosSystem; inherit (pkgs) lib; diff --git a/nix/host-modules/swarm-grafana/dashboards/logstore.json b/nix/host-modules/swarm-grafana/dashboards/logstore.json index 8e92ff70..ce5dd7d4 100644 --- a/nix/host-modules/swarm-grafana/dashboards/logstore.json +++ b/nix/host-modules/swarm-grafana/dashboards/logstore.json @@ -339,7 +339,7 @@ "h": 12, "w": 12, "x": 0, - "y": 33 + "y": 24 }, "targets": [ { @@ -391,7 +391,7 @@ "h": 12, "w": 12, "x": 12, - "y": 33 + "y": 24 }, "targets": [ { @@ -443,7 +443,7 @@ "h": 9, "w": 24, "x": 0, - "y": 45 + "y": 36 }, "targets": [ { @@ -487,103 +487,6 @@ "sort": "desc" } } - }, - { - "id": 9, - "type": "timeseries", - "title": "Log rows with no severity (must be zero)", - "description": "This panel is a check rather than a view: the collector maps journald's PRIORITY onto an OpenTelemetry severity at every tier, and this is what says out loud whether a line got one. Both series are rows whose `severity_text` is absent or the literal `Unspecified`, which is what VictoriaLogs stores when nothing set a severity — split by whether the row had a PRIORITY to map in the first place.\n\n**`journald_priority_unmapped`** is a row that carries PRIORITY and still arrived without a severity. It is the regression line: nonzero means a journald receiver somewhere lost its severity_parser operator, so read it against `nix/journald-severity.nix` and the two receivers that import it.\n\n**`no_priority_field`** is a row that never had a priority — today that is Claude Code's own OTLP telemetry (`scope.name: com.anthropic.claude_code.events`), which emits log records with no severity set at the source. Mapping cannot reach it; it needs a fix where it is produced, so this series is expected nonzero until that lands and is NOT evidence the mapping broke.\n\nRange-scoped over time on purpose. An instantaneous zero is also what a broken query returns — a line you can watch go to zero and stay there is the part that survives whoever wrote it. To sanity-check the query itself, read it against `Rows in range · all sources (control)` above: that panel nonzero while both of these are flat zero means these queries stopped matching, not that every line grew a severity.", - "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "@logsDatasourceUid@" - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 24 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "@logsDatasourceUid@" - }, - "queryType": "statsRange", - "expr": "(severity_text:\"\" OR severity_text:\"Unspecified\") PRIORITY:* | stats count() as journald_priority_unmapped" - }, - { - "refId": "B", - "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "@logsDatasourceUid@" - }, - "queryType": "statsRange", - "expr": "(severity_text:\"\" OR severity_text:\"Unspecified\") PRIORITY:\"\" | stats count() as no_priority_field" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "journald_priority_unmapped" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "red" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "no_priority_field" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "orange" - } - } - ] - } - ] - }, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "calcs": ["max", "lastNotNull"] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } } ] } diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 305f8bad..66d35098 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -1208,16 +1208,6 @@ in journald = { directory = hostJournalDir; units = cfg.journaldUnits; - # The SAME operator list the agent tier attaches to its own - # receiver (nix/agent-modules/otel.nix), and it transfers - # without adjustment: the entry shape is the receiver's, - # not the journal's. Both are this one component running - # `journalctl -o json`, so `PRIORITY` is spelled and typed - # identically whether the directory it was pointed at holds - # a host's journal or a container's. What differs between - # the two receivers is which journal and which units — - # neither of which the parser reads. - operators = import ../journald-severity.nix; }; }; diff --git a/nix/journald-severity.nix b/nix/journald-severity.nix deleted file mode 100644 index 793a6762..00000000 --- a/nix/journald-severity.nix +++ /dev/null @@ -1,67 +0,0 @@ -# journald `PRIORITY` → OpenTelemetry severity, as the stanza operator list -# every journald receiver in this tree attaches. The table it produces and -# what a consumer does with it: docs/scheduler/observability.md, "Log severity". -# -# ONE file, imported by both tiers (`agent-modules/otel.nix` reads the -# container's own journal, `host-modules/swarm-otel.nix` the host's), because -# the mapping is the part that must not drift. The two receivers are otherwise -# unrelated — one a fixed stanza, one parameterised inside a container config — -# so a copy in each is two tables to keep in step, and getting one wrong is -# silent: every line still arrives, it just arrives labelled as the wrong thing. -# -# ⚠️ **The direction is inverted.** syslog PRIORITY counts DOWN in urgency -# (0 = emerg, 7 = debug) where OpenTelemetry's severity number counts UP -# (1 = trace, 21 = fatal). Passed through as a severity number, debug renders -# as critical and emerg as unspecified, and nothing reports an error. -[ - { - type = "severity_parser"; - - # The journald input unmarshals each `journalctl -o json` line into the - # entry BODY as a map, so every journal field is addressed under `body.`. - # `attributes.PRIORITY` is the spelling that looks equally plausible and - # finds nothing — the parser then reports a missing field per entry rather - # than failing to start. - parse_from = "body.PRIORITY"; - - # 🔑 NOT optional, and the single easiest thing to leave off. By default the - # severity parser sets the severity NUMBER from the mapping and leaves the - # severity TEXT as the original value it matched on — so `severity_text` - # would arrive in VictoriaLogs as the literal string `"6"`. That is a - # populated field, it passes every check that asks whether severity is set, - # and it is not a level any consumer renders. With this on, the text is the - # recommended short name for the mapped number: `FATAL`, `ERROR`, `WARN`, - # `INFO`, `DEBUG`. The TEXT is the whole interface here — VictoriaLogs has - # no ingest parameter naming a level field, it auto-detects one by trying - # `severity_text`, `SeverityText`, `severity`, `Severity` in that order, so - # the value has to arrive already spelled as a word it knows. - overwrite_text = true; - - # A record whose PRIORITY the parser cannot read must still REACH the - # store. Explicit rather than inherited from the operator default, because - # the alternative (`drop`) would make the "lines with no severity" panel on - # the logs dashboard read zero by deleting its own evidence — the exact - # empty-result-as-success failure this mapping exists to end. - on_error = "send"; - - # Strings, because that is what arrives: `journalctl -o json` renders every - # journal field as a JSON string, so `PRIORITY` is `"6"` and not `6`. The - # parser stringifies its mapping values too, so an integer spelling here - # would also match — the strings are for the reader, to keep the config - # shaped like the data. - mapping = { - fatal = [ - "0" - "1" - "2" - ]; - error = "3"; - warn = "4"; - info = [ - "5" - "6" - ]; - debug = "7"; - }; - } -] diff --git a/nix/module-eval/agent-otel.nix b/nix/module-eval/agent-otel.nix index 8c596e1d..a843362f 100644 --- a/nix/module-eval/agent-otel.nix +++ b/nix/module-eval/agent-otel.nix @@ -18,7 +18,6 @@ let ; }) agent - carriesJournaldSeverity runGroup ; @@ -57,30 +56,6 @@ let name = "the agent forwarder reads the persistent journal, not the runtime one"; ok = ((agentSettings agentOtel).receivers.journald.directory or null) == "/var/log/journal"; } - { - # Wiring, not contents. A journald record arrives with its level in - # `PRIORITY` and nothing downstream reads that field, so a receiver with - # no severity parser ships every line to the log store labelled - # `Unspecified` — healthy, complete, and unqueryable by level. - # - # What this asks is only whether THIS receiver carries the shared - # mapping. ⚠️ Whether that mapping is RIGHT — the inverted direction, - # which is the part of it a reviewer cannot check by looking — is - # asserted once, in ./journald-severity.nix, against the file both tiers - # import. Re-asserting the table here would test that file twice and - # report one defect as two failures. - # - # The swarm collector's own receiver gets the sibling of this case in - # ./swarm-otel-core.nix. Deliberately NOT folded together with it: they - # cover different journals — this container's own, and the host's — and - # one tier quietly losing its parser while the other keeps one is - # precisely the half-fixed state worth catching. - # - # Membership rather than equality of the whole operator list, so a tier - # that later grows an unrelated operator of its own still passes. - name = "the agent forwarder's journald receiver carries the shared PRIORITY mapping"; - ok = carriesJournaldSeverity (agentSettings agentOtel).receivers.journald; - } { # The hop's two ends: what it reads, and where what it reads goes. # The endpoint is compared against the value the fixture handed the diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 9ade378d..25442c56 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -217,32 +217,6 @@ let grafanaRemoteAuthelia.services.hyperhive.swarm.statusPublish.tokenEndpoint == "https://auth.example.invalid/api/oidc/token"; } - { - # The standing check on the collector's PRIORITY mapping — whether a line - # ever reaches the log store carrying no severity. It is shipped config - # rather than something an operator imports by hand, which also means it - # can be deleted by hand. Read out of the dashboard the provisioner - # actually ships (`dashboardDir` in ../host-modules/swarm-grafana.nix - # names this file), and asserted on the QUERY rather than the title, - # because a panel that keeps its title and loses its expression renders - # an empty graph that looks exactly like zero prioless lines. - name = "the logs dashboard still counts lines that arrive with no severity"; - ok = - let - board = builtins.fromJSON ( - builtins.readFile ../host-modules/swarm-grafana/dashboards/logstore.json - ); - exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels; - counts = lib.filter (e: lib.hasInfix "severity_text" e && lib.hasInfix "stats count()" e) exprs; - in - # Both arms of the split: lines that HAD a priority and arrived without - # a severity anyway (the regression line, which must reach zero), and - # lines that never had one to map — Claude Code's own OTLP telemetry, - # which no mapping can reach. Folded into one number the second would - # keep the panel off zero forever and hide the first. - lib.any (e: lib.hasInfix "PRIORITY:*" e) counts - && lib.any (e: lib.hasInfix "PRIORITY:\"\"" e) counts; - } ]; in runGroup "grafana" cases diff --git a/nix/module-eval/journald-severity.nix b/nix/module-eval/journald-severity.nix deleted file mode 100644 index 265eb224..00000000 --- a/nix/module-eval/journald-severity.nix +++ /dev/null @@ -1,113 +0,0 @@ -# `checks.module-eval-journald-severity` — see ./lib.nix for the shared -# rationale (why this suite exists, naming convention, "evaluates -# not executes"). -# -# The odd one out in this directory: every sibling suite evaluates a host or an -# agent and reads the config that falls out, where this one reads a single file -# that is already a value. It is a suite of its own for the reason the operator -# gave when she saw the mapping asserted in two tiers at once — **test the -# thing only once**. The table lives in one file, so its contents are one -# subject and belong to one case. Each tier's suite then asks a different -# question, about wiring rather than contents: does MY receiver carry this. -# -# What that buys is a readable failure. Invert the table and exactly this suite -# goes red, naming the table. Drop the parser from one tier's receiver and -# exactly that tier goes red, naming the tier. Neither failure makes you read -# the other one to work out which of the two happened. -# -# Needs no `nixosSystem` fixture at all — the thing under test is a file — so -# this suite costs nothing to evaluate next to its siblings. -{ - pkgs, - lib, - self, - nixosSystem, -}: -let - inherit - (import ./lib.nix { - inherit - pkgs - lib - self - nixosSystem - ; - }) - journaldSeverityParser - runGroup - ; - - # The parser's own mapping read back as a plain `PRIORITY -> severity-name` - # function, so the case below can ask what a priority MAPS TO instead of - # matching on the shape of the attrset that produces it. `null` for a - # priority the table does not name, which the case uses as its control. - severityOf = - priority: - let - hits = lib.attrNames ( - lib.filterAttrs (_: v: lib.elem priority (lib.toList v)) journaldSeverityParser.mapping - ); - in - if journaldSeverityParser == null || hits == [ ] then null else lib.head hits; - - cases = [ - { - # ⚠️ The direction, and the only part of this mapping a reviewer cannot - # check by looking. syslog's PRIORITY counts DOWN in urgency (0 = emerg, - # 7 = debug) where the OpenTelemetry severity counts UP (1 = trace, - # 21 = fatal). Wired across, debug renders as critical and emerg as - # unspecified — and nothing reports an error, because every line still - # arrives and every field is still populated. - # - # Every priority, not a sample. An inverted copy still maps each value - # to SOMETHING, so a case that checks one end, or checks that a mapping - # merely exists, passes on the exact defect it was written for. The - # `"8"` arm is the control: it says this reader looks values up rather - # than answering the same way whatever it is handed, so the eight - # arms above it mean what they appear to mean. - name = "journald PRIORITY maps to severity the right way up"; - ok = - severityOf "0" == "fatal" - && severityOf "1" == "fatal" - && severityOf "2" == "fatal" - && severityOf "3" == "error" - && severityOf "4" == "warn" - && severityOf "5" == "info" - && severityOf "6" == "info" - && severityOf "7" == "debug" - && severityOf "8" == null; - } - { - # The second way this mapping can be present and useless, and the one - # that looks already handled. Without `overwrite_text` the parser sets - # the severity NUMBER from the table above and leaves the severity TEXT - # as the raw value it matched — so `severity_text` reaches the log store - # as the literal "6". That is a populated field: it passes any check - # asking merely whether severity is set, and it is not a level anything - # renders. VictoriaLogs has no ingest parameter naming a level field, so - # the text is the whole interface. - name = "the mapping writes a level name, not the raw priority digit"; - ok = journaldSeverityParser.overwrite_text or false; - } - { - # Two settings that are load-bearing in opposite directions. - # - # `parse_from` names where the value is. The journald input unmarshals - # each `journalctl -o json` line into the entry BODY as a map, so the - # field lives under `body.`; `attributes.PRIORITY` is the spelling that - # looks equally plausible and finds nothing, leaving a parser that - # reports a missing field per entry rather than failing to start. - # - # `on_error` decides what happens to a record it cannot read. It must - # still REACH the store: `drop` would make the logs dashboard's - # "lines with no severity" panel read zero by deleting its own evidence, - # which is the empty-result-as-success failure this whole mapping exists - # to end. - name = "the mapping reads the field the receiver fills, and keeps what it cannot read"; - ok = - journaldSeverityParser.parse_from or null == "body.PRIORITY" - && journaldSeverityParser.on_error or null == "send"; - } - ]; -in -runGroup "journald-severity" cases diff --git a/nix/module-eval/lib.nix b/nix/module-eval/lib.nix index cc25daf6..016bfdd7 100644 --- a/nix/module-eval/lib.nix +++ b/nix/module-eval/lib.nix @@ -140,24 +140,6 @@ let agentHarness = machine: machine.systemd.services.hive-agent; - # The PRIORITY → severity operator both journald receivers attach, read - # straight off the file that defines it. What each TIER's case then asks is - # only whether its own receiver carries this exact value — the contents are - # somebody else's case (./journald-severity.nix), asserted once. - # - # Here rather than in one of the suites because three of them want it: the - # two tiers, to say they are wired to it, and the suite that reads what it - # says. A parser rather than the whole list, so a tier that later grows an - # unrelated operator of its own still passes the wiring case. - journaldSeverityParser = lib.findFirst (o: o.type or null == "severity_parser") null ( - import ../journald-severity.nix - ); - - # Whether a given receiver is wired to it. One expression, used by both - # tiers, so "this receiver carries the shared mapping" is spelled once and - # the two tier cases differ only in which receiver they hand it. - carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]); - # The enables that switch owns. ⚠️ `otel` is the per-hive collector's own # option and is NOT under `deploy` — spelled at the wrong path it would be # undeclared rather than false, and a roster that quietly loses a member is @@ -215,8 +197,6 @@ in inherit baoSettings; inherit otelSettings; inherit agentHarness; - inherit journaldSeverityParser; - inherit carriesJournaldSeverity; inherit swarmServiceEnables; inherit runGroup; } diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix index 7428919c..c4433f39 100644 --- a/nix/module-eval/swarm-otel-core.nix +++ b/nix/module-eval/swarm-otel-core.nix @@ -18,7 +18,6 @@ let ; }) hive - carriesJournaldSeverity runGroup otelSettings ; @@ -156,19 +155,6 @@ let in named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named; } - { - # The host-journal sibling of ./agent-otel.nix's wiring case, which - # carries the full reasoning. Same question, different receiver: this - # one reads the HOST's journal rather than a container's, and the two - # are unrelated config — a fixed stanza there, a parameterised block - # inside `containers.swarm-otel` here — so one losing its parser while - # the other keeps one is a real and silent state. - # - # Contents are not this case's business. The table both receivers import - # is asserted once, in ./journald-severity.nix. - name = "the swarm collector's journald receiver carries the shared PRIORITY mapping"; - ok = carriesJournaldSeverity (otelSettings otelNoStores).receivers.journald; - } ]; in runGroup "swarm-otel-core" cases