From 8796f4edac7fb336108deacea3c1d445911b1d00 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 26 Aug 2026 18:41:12 +0200 Subject: [PATCH] swarm-grafana: provision a queue (nats) dashboard Split out of the combined swarm-services board, which mara declined as multiple dashboards in one trench coat. Its 17 panels partition exactly by subject -- 3 services / 5 metrics store / 5 queue / 4 log store, with no panel belonging to two -- so this is a re-cut of verified panels, not new authoring. Seven panels: connections, subscriptions, slow consumers and JetStream API errors as stats, messages over the range, plus pending bytes (the leading indicator for a slow consumer) and JetStream store size against its configured limit (the only capacity question the queue has). Connections is red at zero, unlike every other count here: three things hold a connection at all times, so an empty count means the queue is refusing everyone -- and a callout denial reaches a client as a timeout rather than an error, which makes this often the first visible sign. --- nix/host-modules/swarm-grafana.nix | 1 + .../swarm-grafana/dashboards/queue.json | 462 ++++++++++++++++++ 2 files changed, 463 insertions(+) create mode 100644 nix/host-modules/swarm-grafana/dashboards/queue.json diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index a1dfb6ee..7325de0b 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -61,6 +61,7 @@ let "agents.json" "authelia.json" "claude-usage.json" + "queue.json" ] ); diff --git a/nix/host-modules/swarm-grafana/dashboards/queue.json b/nix/host-modules/swarm-grafana/dashboards/queue.json new file mode 100644 index 00000000..6a788132 --- /dev/null +++ b/nix/host-modules/swarm-grafana/dashboards/queue.json @@ -0,0 +1,462 @@ +{ + "title": "hyperhive · queue (nats)", + "uid": "hyperhive-swarm-queue", + "description": "NATS / JetStream health for the swarm queue. Split out of the combined swarm-services board: a page whose content is really N services should be N pages.", + "editable": true, + "refresh": "1m", + "schemaVersion": 39, + "tags": [ + "hyperhive", + "swarm", + "nats", + "queue" + ], + "time": { + "from": "now-24h", + "to": "now" + }, + "timezone": "utc", + "version": 1, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Queue connections", + "description": "Clients currently attached to the swarm's NATS server. Zero is NOT healthy here: hive-c0re's status publisher, swarm-controller and the notices stream all hold a connection, so an empty count means the queue is refusing everyone rather than being idle. A callout denial reaches a client as a timeout, so this panel is often the first place that failure is visible at all. No `or vector(0)`, deliberately: that fallback belongs on counts-of-failures, and here it would render a missing exporter as zero clients — conflating a dead queue with a dead scrape, which the scrape panels above already answer.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "gnatsd_varz_connections", + "instant": true, + "legendFormat": "clients" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 2, + "type": "stat", + "title": "Queue subscriptions", + "description": "Active subscriptions across all connected clients. Read alongside the connection count: connections steady while subscriptions collapse means clients are attached but no longer listening, which a liveness check cannot see.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "gnatsd_varz_subscriptions", + "instant": true, + "legendFormat": "subs" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 3, + "type": "stat", + "title": "Slow consumers", + "description": "Clients the server has disconnected for failing to keep up. Healthy value is zero, so this panel reads empty when everything is fine — see the queue-connections panel for whether the server is being talked to at all.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "gnatsd_varz_slow_consumers or vector(0)", + "instant": true, + "legendFormat": "slow" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "x-zero-is-healthy": true + }, + { + "id": 4, + "type": "stat", + "title": "JetStream API errors", + "description": "Errors answering JetStream API requests. The KV bucket the hive status publisher writes to lives behind this API, so a non-zero value here explains a status page that has gone stale without anything else looking wrong.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "gnatsd_varz_jetstream_stats_api_errors or vector(0)", + "instant": true, + "legendFormat": "errors" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "x-zero-is-healthy": true + }, + { + "id": 5, + "type": "timeseries", + "title": "Queue messages (range)", + "description": "Messages in and out over the selected range. Deliberately `increase()` rather than a per-second rate: this queue carries a status publish per hive per interval and little else, so a rate divides a handful of messages across an hour and draws a flat zero.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase(gnatsd_varz_in_msgs[$__range])) or vector(0)", + "legendFormat": "in" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase(gnatsd_varz_out_msgs[$__range])) or vector(0)", + "legendFormat": "out" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "type": "stat", + "title": "Pending bytes to clients", + "description": "Bytes queued for delivery to clients. Rises BEFORE a slow consumer is counted, so this is the leading indicator for the panel beside it. Zero is the healthy value.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 12 + }, + "targets": [ + { + "expr": "gnatsd_connz_pending_bytes or vector(0)", + "legendFormat": "pending" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "x-zero-is-healthy": true + }, + { + "id": 7, + "type": "timeseries", + "title": "JetStream store size vs limit", + "description": "The only capacity question the queue has. Used and configured limit on one axis so the headroom is legible without arithmetic; a stream that grows unboundedly shows up here first.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "targets": [ + { + "expr": "gnatsd_varz_jetstream_stats_storage", + "legendFormat": "used" + }, + { + "expr": "gnatsd_varz_jetstream_config_max_storage", + "legendFormat": "limit" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] +}