diff --git a/nix/host-modules/swarm-grafana/dashboards/logstore.json b/nix/host-modules/swarm-grafana/dashboards/logstore.json index ce5dd7d4..8e92ff70 100644 --- a/nix/host-modules/swarm-grafana/dashboards/logstore.json +++ b/nix/host-modules/swarm-grafana/dashboards/logstore.json @@ -339,7 +339,7 @@ "h": 12, "w": 12, "x": 0, - "y": 24 + "y": 33 }, "targets": [ { @@ -391,7 +391,7 @@ "h": 12, "w": 12, "x": 12, - "y": 24 + "y": 33 }, "targets": [ { @@ -443,7 +443,7 @@ "h": 9, "w": 24, "x": 0, - "y": 36 + "y": 45 }, "targets": [ { @@ -487,6 +487,103 @@ "sort": "desc" } } + }, + { + "id": 9, + "type": "timeseries", + "title": "Log rows with no severity (must be zero)", + "description": "This panel is a check rather than a view: the collector maps journald's PRIORITY onto an OpenTelemetry severity at every tier, and this is what says out loud whether a line got one. Both series are rows whose `severity_text` is absent or the literal `Unspecified`, which is what VictoriaLogs stores when nothing set a severity โ€” split by whether the row had a PRIORITY to map in the first place.\n\n**`journald_priority_unmapped`** is a row that carries PRIORITY and still arrived without a severity. It is the regression line: nonzero means a journald receiver somewhere lost its severity_parser operator, so read it against `nix/journald-severity.nix` and the two receivers that import it.\n\n**`no_priority_field`** is a row that never had a priority โ€” today that is Claude Code's own OTLP telemetry (`scope.name: com.anthropic.claude_code.events`), which emits log records with no severity set at the source. Mapping cannot reach it; it needs a fix where it is produced, so this series is expected nonzero until that lands and is NOT evidence the mapping broke.\n\nRange-scoped over time on purpose. An instantaneous zero is also what a broken query returns โ€” a line you can watch go to zero and stay there is the part that survives whoever wrote it. To sanity-check the query itself, read it against `Rows in range ยท all sources (control)` above: that panel nonzero while both of these are flat zero means these queries stopped matching, not that every line grew a severity.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 24 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "queryType": "statsRange", + "expr": "(severity_text:\"\" OR severity_text:\"Unspecified\") PRIORITY:* | stats count() as journald_priority_unmapped" + }, + { + "refId": "B", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "queryType": "statsRange", + "expr": "(severity_text:\"\" OR severity_text:\"Unspecified\") PRIORITY:\"\" | stats count() as no_priority_field" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "journald_priority_unmapped" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "no_priority_field" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": ["max", "lastNotNull"] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } } ] }