vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

swarm-grafana: provision log store and metrics store dashboards

Browse source 
Two more per-service boards, cut from the withdrawn combined services
board: the log store (4 panels) and the metrics store (5).

The combined board was one page carrying panels for four different
services; the seams were already there, so this is a re-cut rather than
new authoring. Each board's subject is one service, and the services
page that remains will carry only panels whose subject is the *set*.

Both were imported by the operator before being committed here, so the
queries are known to draw against the live stores rather than only
parsing.
This commit is contained in:
atlas 2026-08-26 19:05:57 +02:00 • committed by mara
commit 8ed4b8c9c3
3 changed files with 596 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
nix/host-modules/swarm-grafana.nix
View file

@ -61,6 +61,8 @@ let
"agents.json"
"authelia.json"
"claude-usage.json"
"logstore.json"
"metricstore.json"
"queue.json"
]
);

277
nix/host-modules/swarm-grafana/dashboards/logstore.json Normal file
View file

@ -0,0 +1,277 @@
{
"title": "hyperhive · log store (victorialogs)",
"uid": "hyperhive-swarm-logstore",
"description": "VictoriaLogs health for the swarm log store. One of the per-service boards split out of the combined swarm-services page.",
"editable": true,
"refresh": "1m",
"schemaVersion": 39,
"tags": [
"hyperhive",
"swarm",
"victorialogs",
"logs"
],
"time": {
"from": "now-24h",
"to": "now"
},
"timezone": "utc",
"version": 1,
"panels": [
{
"id": 1,
"type": "timeseries",
"title": "Log rows ingested (range)",
"description": "Rows accepted by the log store over the range. ⚠️ This counts what ARRIVED, never what became findable — the pipeline shipped for days with every record carrying a placeholder in place of its message, while this counter climbed normally. A healthy line here is necessary and not sufficient; the check that closes it is a query returning readable lines.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 4
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(increase(vl_rows_ingested_total[$__range])) or vector(0)",
"legendFormat": "rows"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 2,
"type": "timeseries",
"title": "Log store size on disk",
"description": "Compressed on-disk size, split by the store's own `type` label (storage vs indexdb). Retention is 30 days, so this is expected to climb to a plateau rather than forever; a straight line past that is the signal.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 12
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "vl_data_size_bytes",
"legendFormat": "{{type}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"decimals": 1,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 3,
"type": "stat",
"title": "Log store free disk",
"description": "Free space on the volume holding the log store. It shares a filesystem with the metrics store, so the two panels move together — a drop here that the metrics one does not show means something outside this swarm is filling the disk.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 12,
"x": 0,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "vl_free_disk_space_bytes",
"instant": true,
"legendFormat": "free"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"decimals": 1,
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 5000000000
}
]
}
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
}
}
},
{
"id": 4,
"type": "stat",
"title": "Log store errors",
"description": "Internal errors plus rejected HTTP requests. Healthy value is zero. ⚠️ A wrong OTLP path answers 400 and lands here, and so does a right path with a bad payload — the status code alone does not separate them, only the store's own log does.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 12,
"x": 12,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(vl_errors_total) or vector(0)",
"instant": true,
"legendFormat": "internal"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(vl_http_errors_total) or vector(0)",
"instant": true,
"legendFormat": "http"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
}
},
"x-zero-is-healthy": true
}
]
}

317
nix/host-modules/swarm-grafana/dashboards/metricstore.json Normal file
View file

@ -0,0 +1,317 @@
{
"title": "hyperhive · metrics store (victoriametrics)",
"uid": "hyperhive-swarm-metricstore",
"description": "VictoriaMetrics health for the swarm metrics store. One of the per-service boards split out of the combined swarm-services page.",
"editable": true,
"refresh": "1m",
"schemaVersion": 39,
"tags": [
"hyperhive",
"swarm",
"victoriametrics",
"metrics"
],
"time": {
"from": "now-24h",
"to": "now"
},
"timezone": "utc",
"version": 1,
"panels": [
{
"id": 1,
"type": "timeseries",
"title": "Samples ingested/s",
"description": "vm_rows_inserted_total across all insert paths. This is the whole swarm's telemetry arriving — a drop to zero here means the collector stopped, not that the services went quiet.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(rate(vm_rows_inserted_total[$__rate_interval]))",
"legendFormat": "rows/s"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 1,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 2,
"type": "timeseries",
"title": "New time series created (range)",
"description": "Cardinality growth over the range. Bursty by nature — a deploy creates a batch and then nothing — so a per-second rate reads zero between bursts and hides the batches entirely. Sustained growth on a swarm whose shape is not changing is the early warning for a label carrying something unbounded, the failure mode that kills a metrics store slowly rather than loudly.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(increase(vm_new_timeseries_created_total[$__range])) or vector(0)",
"legendFormat": "series/s"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 3,
"type": "timeseries",
"title": "Store HTTP errors/s",
"description": "vm_http_request_errors_total — the store refusing or failing requests, which covers both a broken collector push and a broken grafana query. Non-zero here while 'Samples ingested/s' looks fine means the read side, not the write side.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(rate(vm_http_request_errors_total[$__rate_interval]))",
"legendFormat": "errors/s"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 3,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"x-zero-is-healthy": true
},
{
"id": 4,
"type": "timeseries",
"title": "Store size on disk",
"description": "vm_data_size_bytes summed over the storage components. Paired with free disk below rather than shown alone: the absolute size is uninteresting, the ratio is the operational question.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(vm_data_size_bytes)",
"legendFormat": "on disk"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"decimals": 1,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 5,
"type": "timeseries",
"title": "Free disk space",
"description": "vm_free_disk_space_bytes. VictoriaMetrics stops accepting writes when this falls under vm_free_disk_space_limit_bytes, so this is the panel that predicts a silent ingestion stop rather than reporting one.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 16
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "vm_free_disk_space_bytes",
"legendFormat": "free"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "vm_free_disk_space_limit_bytes",
"legendFormat": "write-stop limit"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"decimals": 1,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 0,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
}
]
}