diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 3bd06be8..6172a469 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -256,54 +256,6 @@ store as `Unspecified`. The **Log rows with no severity** panel on the had a priority and arrived without a severity anyway signal a broken mapping, while lines that never had one signal nothing of the sort. -### Why Explore's level buttons need the datasource told - -The stored field is `severity_text`, and Grafana's log-level buttons filter on -a field called `level` — a name no row here carries. That's not something the -pipeline can fix: VictoriaLogs' OTLP ingester names the stored field -`severity_text` unconditionally (v1.52.0, `app/vlinsert/opentelemetry/pb.go`), -and its ingest parameters have no `_level_field` sibling to rename it with. - -The datasource is told instead. `nix/host-modules/swarm-grafana.nix` -provisions the VictoriaLogs datasource with a `logLevelRules` entry per -severity in the table above, each matching `severity_text` against one of -those values — which is the plugin's way of saying "the level lives in this -field." `logLevelRules` is the datasource plugin's own jsonData key and the -only level-related one it has — there is no single "the level lives in field -X" string to set, so mapping every level takes one rule each. A level button -then filters on `level` **or** the matching `severity_text`, and the store -answers the second half. Each enabled rule appends an `OR -:""` term to the query a level button emits, beside the -`level:…` term that matches nothing here. Clicking "info" goes from - -``` -level:contains_common_case("info","information","informational","notice") -``` - -to that OR `severity_text:="INFO"`, which our rows do match. - -`Unspecified` is deliberately left out of those rules. It's VictoriaLogs' own -rendering of an absent severity, and it's what the **Log rows with no -severity** panel counts — giving it a level would dress the missing data up -as a colour and retire the instrument that measures it. - -#### Three ways to get a `logLevelRules` entry wrong, all of them silent - -Grafana accepts any jsonData it doesn't recognise, so a bad rule provisions -cleanly and the affected level button goes on returning zero rows: - -- `enabled` must be literally `true`, not merely not-false. The query - builder keeps rules on `rule.enabled` being truthy while the row colouring - path keeps them on `!== false`, so an omitted flag colours rows correctly - and leaves the buttons broken — working in the half nobody is looking at. -- `level` must be a canonical Grafana `LogLevel` value. The builder groups - rules by it and only ever looks up `critical`, `error`, `warning`, `info`, - `debug`, `trace`; `warn` and `fatal` are enum aliases that resolve to - other spellings and match no group. -- `value` is compared with `===`, so it must be the severity text exactly as - stored: the uppercase OpenTelemetry short names `overwrite_text` writes in - `nix/journald-severity.nix`, not that table's lowercase keys. - ## Host-emitted container-resource metrics (hive-c0re) diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 5e6ac674..f6e9d209 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -40,38 +40,6 @@ let # free to drift into a panel that renders empty rather than erroring. logsDatasourceUid = "swarm-victorialogs"; - # Which field carries the log level, told to the READER because the writer - # cannot be told: VictoriaLogs' OTLP ingester names the stored field - # `severity_text` unconditionally, with no option to rename it, but - # Grafana's log-level buttons filter on a field called `level`. So the - # mapping is made here, one `logLevelRules` entry per level below. - # - # 🔑 A rule wrong in any of three ways provisions cleanly and its level - # button silently returns zero rows — `enabled` must be literally `true`, - # `level` must be a canonical Grafana `LogLevel` (not an alias like `warn` - # or `fatal`), and `value` must match ../journald-severity.nix's uppercase - # STORED text, not its lowercase keys. `Unspecified` is deliberately left - # unmapped so the "no severity" panel keeps counting it. Full walkthrough - # (why each rule shape is what it is, the query a level button actually - # emits): docs/scheduler/observability.md#why-explores-level-buttons-need-the-datasource-told - logLevelRules = - let - fromSeverityText = value: level: { - field = "severity_text"; - operator = "equals"; - enabled = true; - inherit value level; - }; - in - [ - (fromSeverityText "FATAL" "critical") - (fromSeverityText "ERROR" "error") - (fromSeverityText "WARN" "warning") - (fromSeverityText "INFO" "info") - (fromSeverityText "DEBUG" "debug") - (fromSeverityText "TRACE" "trace") - ]; - # The shipped dashboards carry `@datasourceUid@` / `@logsDatasourceUid@` where # a real deployment needs the uids above. They are substituted here rather # than committed with the literals so the single binding stays single. @@ -976,11 +944,6 @@ in # two defaults is a coin toss over which one an untyped panel # gets. isDefault = false; - # See `logLevelRules` above for why the level lives here and - # not in the pipeline. - jsonData = { - inherit logLevelRules; - }; } ]; }; diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index c1008194..9ade378d 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -243,43 +243,6 @@ let lib.any (e: lib.hasInfix "PRIORITY:*" e) counts && lib.any (e: lib.hasInfix "PRIORITY:\"\"" e) counts; } - { - # The reader's half of the same mapping. The collector writes a severity - # TEXT; nothing downstream reads it as a level unless the datasource is - # told which field holds it, and that telling is a list of rules rather - # than a field name — so it goes stale one severity at a time. Asserted - # against ../journald-severity.nix rather than a literal list here, - # because the way this regresses is a severity added to the parser and - # not to Grafana: every line still arrives, the new one is just - # unfilterable, and no query errors to say so. - # - # `enabled` and the canonical `level` spelling are pinned alongside - # because both fail silently in the direction of "provisions fine, - # returns nothing" — see the comment on `logLevelRules` in - # ../host-modules/swarm-grafana.nix. - name = "the logs datasource maps every severity the collector can emit to a log level"; - ok = - let - sources = - grafanaOldPath.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources; - logs = lib.head (lib.filter (d: d.uid == "swarm-victorialogs") sources); - rules = logs.jsonData.logLevelRules or [ ]; - # `overwrite_text` makes the stored text the OpenTelemetry short name - # for each mapped severity, which is the parser's own key uppercased. - emitted = map lib.toUpper (lib.attrNames (lib.head (import ../journald-severity.nix)).mapping); - canonical = [ - "critical" - "error" - "warning" - "info" - "debug" - "trace" - ]; - in - rules != [ ] - && lib.all (r: r.field == "severity_text" && r.enabled == true && lib.elem r.level canonical) rules - && lib.all (text: lib.any (r: r.value == text) rules) emitted; - } ]; in runGroup "grafana" cases