diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 08ebc799..bbc68473 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -39,15 +39,16 @@ let # free to drift into a panel that renders empty rather than erroring. logsDatasourceUid = "swarm-victorialogs"; - # The shipped dashboards carry `@datasourceUid@` where a real deployment needs - # the uid above. They are substituted here rather than committed with the - # literal so the single binding stays single. + # The shipped dashboards carry `@datasourceUid@` / `@logsDatasourceUid@` where + # a real deployment needs the uids above. They are substituted here rather + # than committed with the literals so the single binding stays single. renderDashboard = name: pkgs.writeText name ( - builtins.replaceStrings [ "@datasourceUid@" ] [ datasourceUid ] ( - builtins.readFile (./swarm-grafana/dashboards + "/${name}") - ) + builtins.replaceStrings + [ "@datasourceUid@" "@logsDatasourceUid@" ] + [ datasourceUid logsDatasourceUid ] + (builtins.readFile (./swarm-grafana/dashboards + "/${name}")) ); # Grafana's file provider wants a DIRECTORY to scan, so the rendered files are diff --git a/nix/host-modules/swarm-grafana/dashboards/logstore.json b/nix/host-modules/swarm-grafana/dashboards/logstore.json index e2d0a4c6..ce5dd7d4 100644 --- a/nix/host-modules/swarm-grafana/dashboards/logstore.json +++ b/nix/host-modules/swarm-grafana/dashboards/logstore.json @@ -1,7 +1,7 @@ { - "title": "hyperhive · log store (victorialogs)", + "title": "hyperhive · logs (victorialogs)", "uid": "hyperhive-swarm-logstore", - "description": "VictoriaLogs health for the swarm log store. One of the per-service boards split out of the combined swarm-services page.", + "description": "The swarm's log pipeline: whether the store is healthy (top) and which sources are actually feeding it (bottom). One of the per-service boards split out of the combined swarm-services page. The two halves answer different questions and are on one board on purpose — aggregate ingestion stays normal on host-tier units while a whole tier ships nothing, so a healthy top half is not evidence about the bottom one.", "editable": true, "refresh": "1m", "schemaVersion": 39, @@ -263,6 +263,230 @@ } }, "x-zero-is-healthy": true + }, + { + "id": 5, + "type": "stat", + "title": "Rows in range · all sources (control)", + "description": "Deliberately ungrouped, and it is what makes the two tables below readable. An empty table renders identically whether the query is malformed or the source genuinely never shipped: nonzero here with an empty table means the QUERY is broken; zero here means the store really is empty for this range. Read it against `Log rows ingested` above too — that counter is the same quantity measured from the store's own metrics rather than by querying, so the two disagreeing means rows arrived that a query cannot reach.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 20 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "queryType": "stats", + "expr": "* | stats count() as rows" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 6, + "type": "bargauge", + "title": "Rows by unit", + "description": "Every `_SYSTEMD_UNIT` present in the range, discovered rather than enumerated — nothing here names a unit, so a source that starts shipping appears on its own. ⚠️ Absence means NOT COLLECTED, which is not the same as not running: the collector's journald receiver only reads the units listed in `services.hyperhive.swarm.otel.journaldUnits`, so a unit missing from that list can be running and logging and still never reach this board.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 24 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "queryType": "stats", + "expr": "* | stats by (_SYSTEMD_UNIT) count() as rows", + "legendFormat": "{{_SYSTEMD_UNIT}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "continuous-BlPu" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "minVizWidth": 8, + "minVizHeight": 12, + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 7, + "type": "bargauge", + "title": "Rows by host", + "description": "The same count grouped by `_HOSTNAME`. Read together with the unit table this separates two very different failures: one host contributing nothing is a collector or journal-linkage problem on that host, while a unit missing across every host is a configuration problem in what gets collected.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 24 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "queryType": "stats", + "expr": "* | stats by (_HOSTNAME) count() as rows", + "legendFormat": "{{_HOSTNAME}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "continuous-BlPu" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "minVizWidth": 8, + "minVizHeight": 12, + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Rows over time, by unit", + "description": "The tables say who shipped over the whole range; this says when. A line that stops is a source that died, and it is the only view here that distinguishes that from one which never existed — a source absent for the entire range looks the same as an unknown one in a table, but shows up here as a line that ends.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 36 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "queryType": "statsRange", + "expr": "* | stats by (_SYSTEMD_UNIT) count() as rows", + "legendFormat": "{{_SYSTEMD_UNIT}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } } ] }