diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 6172a469..d56a5e44 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -256,6 +256,24 @@ store as `Unspecified`. The **Log rows with no severity** panel on the had a priority and arrived without a severity anyway signal a broken mapping, while lines that never had one signal nothing of the sort. +### Why Explore's level buttons need the datasource told + +The stored field is `severity_text`, and Grafana's log-level buttons filter on +a field called `level` — a name no row here carries. That is not something the +pipeline can fix: VictoriaLogs' OTLP ingester chooses the stored name itself, +and its ingest parameters have no option to rename it. + +So the datasource is told instead. `nix/host-modules/swarm-grafana.nix` +provisions the VictoriaLogs datasource with a `logLevelRules` entry per +severity in the table above, each matching `severity_text` against one of +those values — which is the plugin's way of saying "the level lives in this +field". A level button then filters on `level` **or** the matching +`severity_text`, and the store answers the second half. + +`Unspecified` is deliberately left out of those rules, so rows with no +severity stay unfiltered by every level button and keep showing up in the +panel that counts them. + ## Host-emitted container-resource metrics (hive-c0re) diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index f6e9d209..aaa301bd 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -40,6 +40,65 @@ let # free to drift into a panel that renders empty rather than erroring. logsDatasourceUid = "swarm-victorialogs"; + # Which field carries the log level, told to the READER because the writer + # cannot be told. Grafana's log-level buttons filter on a field called + # `level`, and no row in this store has one: VictoriaLogs' OTLP ingester + # names the stored field `severity_text` unconditionally (v1.52.0, + # `app/vlinsert/opentelemetry/pb.go`) and its ingest parameters have no + # `_level_field` sibling to rename it with. So the mapping is made here. + # + # `logLevelRules` is the datasource plugin's own jsonData key and the only + # level-related one it has — there is no "the level lives in field X" + # string to set. A rule names its own field, so saying it takes one rule + # per level. + # + # 🔑 Each enabled rule appends an `OR :""` term to the + # query a level button emits, beside the `level:…` term that matches + # nothing here. Clicking "info" goes from + # + # level:contains_common_case("info","information","informational","notice") + # + # to that OR `severity_text:="INFO"`, which our rows do match. + # + # ⚠️ Three ways to get a rule wrong, all of them SILENT — Grafana accepts + # any jsonData it does not recognise, so a bad rule provisions cleanly and + # the buttons go on returning zero: + # + # - `enabled` must be literally `true`, not merely not-false. The query + # builder keeps rules on `rule.enabled` being truthy while the row + # colouring path keeps them on `!== false`, so an omitted flag colours + # rows correctly and leaves the buttons broken — working in the half + # nobody is looking at. + # - `level` must be a CANONICAL Grafana `LogLevel` value. The builder + # groups rules by it and only ever looks up `critical`, `error`, + # `warning`, `info`, `debug`, `trace`; `warn` and `fatal` are enum + # aliases that resolve to other spellings and match no group. + # - `value` is compared with `===`, so it is the severity text exactly as + # STORED: the uppercase OpenTelemetry short names `overwrite_text` + # writes in ../journald-severity.nix, not that table's lowercase keys. + # + # `Unspecified` is deliberately unmapped. It is VictoriaLogs' own rendering + # of an absent severity, and it is what the logs dashboard's "no severity" + # panel counts — giving it a level would dress the missing data up as a + # colour and retire the instrument that measures it. + logLevelRules = + let + fromSeverityText = value: level: { + field = "severity_text"; + operator = "equals"; + enabled = true; + inherit value level; + }; + in + [ + (fromSeverityText "FATAL" "critical") + (fromSeverityText "ERROR" "error") + (fromSeverityText "WARN" "warning") + (fromSeverityText "INFO" "info") + (fromSeverityText "DEBUG" "debug") + (fromSeverityText "TRACE" "trace") + ]; + # The shipped dashboards carry `@datasourceUid@` / `@logsDatasourceUid@` where # a real deployment needs the uids above. They are substituted here rather # than committed with the literals so the single binding stays single. @@ -944,6 +1003,11 @@ in # two defaults is a coin toss over which one an untyped panel # gets. isDefault = false; + # See `logLevelRules` above for why the level lives here and + # not in the pipeline. + jsonData = { + inherit logLevelRules; + }; } ]; }; diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 9ade378d..c1008194 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -243,6 +243,43 @@ let lib.any (e: lib.hasInfix "PRIORITY:*" e) counts && lib.any (e: lib.hasInfix "PRIORITY:\"\"" e) counts; } + { + # The reader's half of the same mapping. The collector writes a severity + # TEXT; nothing downstream reads it as a level unless the datasource is + # told which field holds it, and that telling is a list of rules rather + # than a field name — so it goes stale one severity at a time. Asserted + # against ../journald-severity.nix rather than a literal list here, + # because the way this regresses is a severity added to the parser and + # not to Grafana: every line still arrives, the new one is just + # unfilterable, and no query errors to say so. + # + # `enabled` and the canonical `level` spelling are pinned alongside + # because both fail silently in the direction of "provisions fine, + # returns nothing" — see the comment on `logLevelRules` in + # ../host-modules/swarm-grafana.nix. + name = "the logs datasource maps every severity the collector can emit to a log level"; + ok = + let + sources = + grafanaOldPath.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources; + logs = lib.head (lib.filter (d: d.uid == "swarm-victorialogs") sources); + rules = logs.jsonData.logLevelRules or [ ]; + # `overwrite_text` makes the stored text the OpenTelemetry short name + # for each mapped severity, which is the parser's own key uppercased. + emitted = map lib.toUpper (lib.attrNames (lib.head (import ../journald-severity.nix)).mapping); + canonical = [ + "critical" + "error" + "warning" + "info" + "debug" + "trace" + ]; + in + rules != [ ] + && lib.all (r: r.field == "severity_text" && r.enabled == true && lib.elem r.level canonical) rules + && lib.all (text: lib.any (r: r.value == text) rules) emitted; + } ]; in runGroup "grafana" cases