From a204b5f45795cba62beefdcc7a8064aa95621549 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 7 Sep 2026 23:15:09 +0200 Subject: [PATCH] grafana: show which sources are shipping, not just that the store is up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The log-store board reports rows ingested, disk size, free disk and errors — every one of them a fact about VictoriaLogs itself. None of them can say whether a particular unit or host is contributing, so aggregate ingestion reads healthy on host-tier units while a whole tier ships nothing, and there was no way to tell those apart from a dashboard. Add four panels to that board rather than a second one. Every other board here is per service — agents, authelia, openbao, forge, queue, the two stores — so a second board about the same service would have made an operator guess which of the two answers their question. Nothing in the new panels names a unit: both breakdowns discover their rows from the data, so a source that starts shipping appears without an edit, and one that never existed is simply not there. The ungrouped total is a control, not a summary. An empty breakdown renders the same whether the query is malformed or the source genuinely never shipped; with the total beside it, nonzero-and-empty is a broken query and zero-and-empty is an empty store. Being on one board buys a second reading it could not have alone: that total and `Log rows ingested` are the same quantity measured by querying and from the store's own metrics, so the two disagreeing means rows arrived that no query can reach — which is the failure this pipeline actually had once. The two breakdowns are bargauges rather than tables. A `stats` query returns one frame per group, and a table panel renders one frame at a time behind a series picker; bargauge consumes multi-series natively, for the same reason the timeseries beside it always did. The alternative was a table plus labelsToFields plus merge plus organize — three assumptions where this needs none, in a spot I cannot render to check. Series are named by their label rather than `rows{_SYSTEMD_UNIT="x.service"}`, which also cleans up the timeseries legend. The datasource supports legendFormat; its query editor's own placeholder is `{{label}}`. These are the first panels to query the logs datasource at all; the other eight boards are prometheus, including this one until now, which reads VictoriaLogs' self-metrics out of VictoriaMetrics. So `renderDashboard` grows a `@logsDatasourceUid@` substitution alongside the metrics one. The binding it points at already existed, and its comment claiming a dashboard panel named it only becomes true with this commit. The uid is unchanged, so existing links and bookmarks still resolve; only the title widens to match what the board now covers. The query model was read out of the plugin in the store rather than guessed: `queryType` is one of hits/instant/logs/stats/statsRange, and Stats/StatsRange are the two that consume `expr`. The stream field names come from the module that builds them — swarm-otel.nix's `_stream_fields=_HOSTNAME,_SYSTEMD_UNIT`. The queries themselves are confirmed against the live store: mara ran the by-unit one over seven days and it returned rows. Refs #4084 --- nix/host-modules/swarm-grafana.nix | 13 +- .../swarm-grafana/dashboards/logstore.json | 228 +++++++++++++++++- 2 files changed, 233 insertions(+), 8 deletions(-) diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 08ebc799..bbc68473 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -39,15 +39,16 @@ let # free to drift into a panel that renders empty rather than erroring. logsDatasourceUid = "swarm-victorialogs"; - # The shipped dashboards carry `@datasourceUid@` where a real deployment needs - # the uid above. They are substituted here rather than committed with the - # literal so the single binding stays single. + # The shipped dashboards carry `@datasourceUid@` / `@logsDatasourceUid@` where + # a real deployment needs the uids above. They are substituted here rather + # than committed with the literals so the single binding stays single. renderDashboard = name: pkgs.writeText name ( - builtins.replaceStrings [ "@datasourceUid@" ] [ datasourceUid ] ( - builtins.readFile (./swarm-grafana/dashboards + "/${name}") - ) + builtins.replaceStrings + [ "@datasourceUid@" "@logsDatasourceUid@" ] + [ datasourceUid logsDatasourceUid ] + (builtins.readFile (./swarm-grafana/dashboards + "/${name}")) ); # Grafana's file provider wants a DIRECTORY to scan, so the rendered files are diff --git a/nix/host-modules/swarm-grafana/dashboards/logstore.json b/nix/host-modules/swarm-grafana/dashboards/logstore.json index e2d0a4c6..ce5dd7d4 100644 --- a/nix/host-modules/swarm-grafana/dashboards/logstore.json +++ b/nix/host-modules/swarm-grafana/dashboards/logstore.json @@ -1,7 +1,7 @@ { - "title": "hyperhive · log store (victorialogs)", + "title": "hyperhive · logs (victorialogs)", "uid": "hyperhive-swarm-logstore", - "description": "VictoriaLogs health for the swarm log store. One of the per-service boards split out of the combined swarm-services page.", + "description": "The swarm's log pipeline: whether the store is healthy (top) and which sources are actually feeding it (bottom). One of the per-service boards split out of the combined swarm-services page. The two halves answer different questions and are on one board on purpose — aggregate ingestion stays normal on host-tier units while a whole tier ships nothing, so a healthy top half is not evidence about the bottom one.", "editable": true, "refresh": "1m", "schemaVersion": 39, @@ -263,6 +263,230 @@ } }, "x-zero-is-healthy": true + }, + { + "id": 5, + "type": "stat", + "title": "Rows in range · all sources (control)", + "description": "Deliberately ungrouped, and it is what makes the two tables below readable. An empty table renders identically whether the query is malformed or the source genuinely never shipped: nonzero here with an empty table means the QUERY is broken; zero here means the store really is empty for this range. Read it against `Log rows ingested` above too — that counter is the same quantity measured from the store's own metrics rather than by querying, so the two disagreeing means rows arrived that a query cannot reach.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 20 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "queryType": "stats", + "expr": "* | stats count() as rows" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 6, + "type": "bargauge", + "title": "Rows by unit", + "description": "Every `_SYSTEMD_UNIT` present in the range, discovered rather than enumerated — nothing here names a unit, so a source that starts shipping appears on its own. ⚠️ Absence means NOT COLLECTED, which is not the same as not running: the collector's journald receiver only reads the units listed in `services.hyperhive.swarm.otel.journaldUnits`, so a unit missing from that list can be running and logging and still never reach this board.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 24 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "queryType": "stats", + "expr": "* | stats by (_SYSTEMD_UNIT) count() as rows", + "legendFormat": "{{_SYSTEMD_UNIT}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "continuous-BlPu" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "minVizWidth": 8, + "minVizHeight": 12, + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 7, + "type": "bargauge", + "title": "Rows by host", + "description": "The same count grouped by `_HOSTNAME`. Read together with the unit table this separates two very different failures: one host contributing nothing is a collector or journal-linkage problem on that host, while a unit missing across every host is a configuration problem in what gets collected.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 24 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "queryType": "stats", + "expr": "* | stats by (_HOSTNAME) count() as rows", + "legendFormat": "{{_HOSTNAME}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "continuous-BlPu" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "minVizWidth": 8, + "minVizHeight": 12, + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Rows over time, by unit", + "description": "The tables say who shipped over the whole range; this says when. A line that stops is a source that died, and it is the only view here that distinguishes that from one which never existed — a source absent for the entire range looks the same as an unknown one in a table, but shows up here as a line that ends.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 36 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "queryType": "statsRange", + "expr": "* | stats by (_SYSTEMD_UNIT) count() as rows", + "legendFormat": "{{_SYSTEMD_UNIT}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } } ] }