From 8ed4b8c9c3154c9f3958d45748d1aa608c5474a5 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 26 Aug 2026 19:05:57 +0200 Subject: [PATCH] swarm-grafana: provision log store and metrics store dashboards Two more per-service boards, cut from the withdrawn combined services board: the log store (4 panels) and the metrics store (5). The combined board was one page carrying panels for four different services; the seams were already there, so this is a re-cut rather than new authoring. Each board's subject is one service, and the services page that remains will carry only panels whose subject is the *set*. Both were imported by the operator before being committed here, so the queries are known to draw against the live stores rather than only parsing. --- nix/host-modules/swarm-grafana.nix | 2 + .../swarm-grafana/dashboards/logstore.json | 277 +++++++++++++++ .../swarm-grafana/dashboards/metricstore.json | 317 ++++++++++++++++++ 3 files changed, 596 insertions(+) create mode 100644 nix/host-modules/swarm-grafana/dashboards/logstore.json create mode 100644 nix/host-modules/swarm-grafana/dashboards/metricstore.json diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 7325de0b..0ba6b51c 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -61,6 +61,8 @@ let "agents.json" "authelia.json" "claude-usage.json" + "logstore.json" + "metricstore.json" "queue.json" ] ); diff --git a/nix/host-modules/swarm-grafana/dashboards/logstore.json b/nix/host-modules/swarm-grafana/dashboards/logstore.json new file mode 100644 index 00000000..b7d9ca18 --- /dev/null +++ b/nix/host-modules/swarm-grafana/dashboards/logstore.json @@ -0,0 +1,277 @@ +{ + "title": "hyperhive · log store (victorialogs)", + "uid": "hyperhive-swarm-logstore", + "description": "VictoriaLogs health for the swarm log store. One of the per-service boards split out of the combined swarm-services page.", + "editable": true, + "refresh": "1m", + "schemaVersion": 39, + "tags": [ + "hyperhive", + "swarm", + "victorialogs", + "logs" + ], + "time": { + "from": "now-24h", + "to": "now" + }, + "timezone": "utc", + "version": 1, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Log rows ingested (range)", + "description": "Rows accepted by the log store over the range. ⚠️ This counts what ARRIVED, never what became findable — the pipeline shipped for days with every record carrying a placeholder in place of its message, while this counter climbed normally. A healthy line here is necessary and not sufficient; the check that closes it is a query returning readable lines.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase(vl_rows_ingested_total[$__range])) or vector(0)", + "legendFormat": "rows" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 2, + "type": "timeseries", + "title": "Log store size on disk", + "description": "Compressed on-disk size, split by the store's own `type` label (storage vs indexdb). Retention is 30 days, so this is expected to climb to a plateau rather than forever; a straight line past that is the signal.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 12 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "vl_data_size_bytes", + "legendFormat": "{{type}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "decimals": 1, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 3, + "type": "stat", + "title": "Log store free disk", + "description": "Free space on the volume holding the log store. It shares a filesystem with the metrics store, so the two panels move together — a drop here that the metrics one does not show means something outside this swarm is filling the disk.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "vl_free_disk_space_bytes", + "instant": true, + "legendFormat": "free" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "decimals": 1, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 5000000000 + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 4, + "type": "stat", + "title": "Log store errors", + "description": "Internal errors plus rejected HTTP requests. Healthy value is zero. ⚠️ A wrong OTLP path answers 400 and lands here, and so does a right path with a bad payload — the status code alone does not separate them, only the store's own log does.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(vl_errors_total) or vector(0)", + "instant": true, + "legendFormat": "internal" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(vl_http_errors_total) or vector(0)", + "instant": true, + "legendFormat": "http" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "x-zero-is-healthy": true + } + ] +} diff --git a/nix/host-modules/swarm-grafana/dashboards/metricstore.json b/nix/host-modules/swarm-grafana/dashboards/metricstore.json new file mode 100644 index 00000000..3e2daec0 --- /dev/null +++ b/nix/host-modules/swarm-grafana/dashboards/metricstore.json @@ -0,0 +1,317 @@ +{ + "title": "hyperhive · metrics store (victoriametrics)", + "uid": "hyperhive-swarm-metricstore", + "description": "VictoriaMetrics health for the swarm metrics store. One of the per-service boards split out of the combined swarm-services page.", + "editable": true, + "refresh": "1m", + "schemaVersion": 39, + "tags": [ + "hyperhive", + "swarm", + "victoriametrics", + "metrics" + ], + "time": { + "from": "now-24h", + "to": "now" + }, + "timezone": "utc", + "version": 1, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Samples ingested/s", + "description": "vm_rows_inserted_total across all insert paths. This is the whole swarm's telemetry arriving — a drop to zero here means the collector stopped, not that the services went quiet.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(rate(vm_rows_inserted_total[$__rate_interval]))", + "legendFormat": "rows/s" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 1, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 2, + "type": "timeseries", + "title": "New time series created (range)", + "description": "Cardinality growth over the range. Bursty by nature — a deploy creates a batch and then nothing — so a per-second rate reads zero between bursts and hides the batches entirely. Sustained growth on a swarm whose shape is not changing is the early warning for a label carrying something unbounded, the failure mode that kills a metrics store slowly rather than loudly.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase(vm_new_timeseries_created_total[$__range])) or vector(0)", + "legendFormat": "series/s" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 3, + "type": "timeseries", + "title": "Store HTTP errors/s", + "description": "vm_http_request_errors_total — the store refusing or failing requests, which covers both a broken collector push and a broken grafana query. Non-zero here while 'Samples ingested/s' looks fine means the read side, not the write side.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(rate(vm_http_request_errors_total[$__rate_interval]))", + "legendFormat": "errors/s" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 3, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "x-zero-is-healthy": true + }, + { + "id": 4, + "type": "timeseries", + "title": "Store size on disk", + "description": "vm_data_size_bytes summed over the storage components. Paired with free disk below rather than shown alone: the absolute size is uninteresting, the ratio is the operational question.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(vm_data_size_bytes)", + "legendFormat": "on disk" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "decimals": 1, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Free disk space", + "description": "vm_free_disk_space_bytes. VictoriaMetrics stops accepting writes when this falls under vm_free_disk_space_limit_bytes, so this is the panel that predicts a silent ingestion stop rather than reporting one.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "vm_free_disk_space_bytes", + "legendFormat": "free" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "vm_free_disk_space_limit_bytes", + "legendFormat": "write-stop limit" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "decimals": 1, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] +}