diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 7325de0b..a1dfb6ee 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -61,7 +61,6 @@ let "agents.json" "authelia.json" "claude-usage.json" - "queue.json" ] ); diff --git a/nix/host-modules/swarm-grafana/dashboards/queue.json b/nix/host-modules/swarm-grafana/dashboards/queue.json deleted file mode 100644 index ee9d30cd..00000000 --- a/nix/host-modules/swarm-grafana/dashboards/queue.json +++ /dev/null @@ -1,462 +0,0 @@ -{ - "title": "hyperhive · queue (nats)", - "uid": "hyperhive-swarm-queue", - "description": "NATS / JetStream health for the swarm queue. Split out of the combined swarm-services board: a page whose content is really N services should be N pages.", - "editable": true, - "refresh": "1m", - "schemaVersion": 39, - "tags": [ - "hyperhive", - "swarm", - "nats", - "queue" - ], - "time": { - "from": "now-24h", - "to": "now" - }, - "timezone": "utc", - "version": 1, - "panels": [ - { - "id": 1, - "type": "stat", - "title": "Queue connections", - "description": "Clients currently attached to the swarm's NATS server. Zero is NOT healthy here: hive-c0re's status publisher, swarm-controller and the notices stream all hold a connection, so an empty count means the queue is refusing everyone rather than being idle. A callout denial reaches a client as a timeout, so this panel is often the first place that failure is visible at all. No `or vector(0)`, deliberately: that fallback belongs on counts-of-failures, and here it would render a missing exporter as zero clients — conflating a dead queue with a dead scrape, which the scrape panels above already answer.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "gnatsd_varz_connections", - "instant": true, - "legendFormat": "clients" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - } - }, - { - "id": 2, - "type": "stat", - "title": "Queue subscriptions", - "description": "Active subscriptions across all connected clients. Read alongside the connection count: connections steady while subscriptions collapse means clients are attached but no longer listening, which a liveness check cannot see.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "gnatsd_varz_subscriptions", - "instant": true, - "legendFormat": "subs" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - } - }, - { - "id": 3, - "type": "stat", - "title": "Slow consumers", - "description": "Clients the server has disconnected for failing to keep up. Healthy value is zero, so this panel reads empty when everything is fine — see the queue-connections panel for whether the server is being talked to at all.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "gnatsd_varz_slow_consumers or vector(0)", - "instant": true, - "legendFormat": "slow" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "x-zero-is-healthy": true - }, - { - "id": 4, - "type": "stat", - "title": "JetStream API errors", - "description": "Errors answering JetStream API requests. The KV bucket the hive status publisher writes to lives behind this API, so a non-zero value here explains a status page that has gone stale without anything else looking wrong.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 4 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "gnatsd_varz_jetstream_stats_api_errors or vector(0)", - "instant": true, - "legendFormat": "errors" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "x-zero-is-healthy": true - }, - { - "id": 5, - "type": "timeseries", - "title": "Queue messages (range)", - "description": "Messages in and out over the selected range. Deliberately `increase()` rather than a per-second rate: this queue carries a status publish per hive per interval and little else, so a rate divides a handful of messages across an hour and draws a flat zero.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 8 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum(increase(gnatsd_varz_in_msgs[$__range])) or vector(0)", - "legendFormat": "in" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum(increase(gnatsd_varz_out_msgs[$__range])) or vector(0)", - "legendFormat": "out" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 6, - "type": "stat", - "title": "Pending bytes to clients", - "description": "Bytes queued for delivery to clients. Rises BEFORE a slow consumer is counted, so this is the leading indicator for the panel beside it. Zero is the healthy value.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 0 - }, - "targets": [ - { - "expr": "gnatsd_connz_pending_bytes or vector(0)", - "legendFormat": "pending" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "x-zero-is-healthy": true - }, - { - "id": 7, - "type": "timeseries", - "title": "JetStream store size vs limit", - "description": "The only capacity question the queue has. Used and configured limit on one axis so the headroom is legible without arithmetic; a stream that grows unboundedly shows up here first.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 18, - "x": 6, - "y": 4 - }, - "targets": [ - { - "expr": "gnatsd_varz_jetstream_stats_storage", - "legendFormat": "used" - }, - { - "expr": "gnatsd_varz_jetstream_config_max_storage", - "legendFormat": "limit" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - } - ] -}