From 8796f4edac7fb336108deacea3c1d445911b1d00 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Wed, 26 Aug 2026 18:41:12 +0200
Subject: [PATCH 1/2] swarm-grafana: provision a queue (nats) dashboard

Split out of the combined swarm-services board, which mara declined as
multiple dashboards in one trench coat. Its 17 panels partition exactly
by subject -- 3 services / 5 metrics store / 5 queue / 4 log store, with
no panel belonging to two -- so this is a re-cut of verified panels, not
new authoring.

Seven panels: connections, subscriptions, slow consumers and JetStream
API errors as stats, messages over the range, plus pending bytes (the
leading indicator for a slow consumer) and JetStream store size against
its configured limit (the only capacity question the queue has).

Connections is red at zero, unlike every other count here: three things
hold a connection at all times, so an empty count means the queue is
refusing everyone -- and a callout denial reaches a client as a timeout
rather than an error, which makes this often the first visible sign.
---
 nix/host-modules/swarm-grafana.nix            |   1 +
 .../swarm-grafana/dashboards/queue.json       | 462 ++++++++++++++++++
 2 files changed, 463 insertions(+)
 create mode 100644 nix/host-modules/swarm-grafana/dashboards/queue.json

diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix
index a1dfb6ee..7325de0b 100644
--- a/nix/host-modules/swarm-grafana.nix
+++ b/nix/host-modules/swarm-grafana.nix
@@ -61,6 +61,7 @@ let
         "agents.json"
         "authelia.json"
         "claude-usage.json"
+        "queue.json"
       ]
   );
 
diff --git a/nix/host-modules/swarm-grafana/dashboards/queue.json b/nix/host-modules/swarm-grafana/dashboards/queue.json
new file mode 100644
index 00000000..6a788132
--- /dev/null
+++ b/nix/host-modules/swarm-grafana/dashboards/queue.json
@@ -0,0 +1,462 @@
+{
+  "title": "hyperhive · queue (nats)",
+  "uid": "hyperhive-swarm-queue",
+  "description": "NATS / JetStream health for the swarm queue. Split out of the combined swarm-services board: a page whose content is really N services should be N pages.",
+  "editable": true,
+  "refresh": "1m",
+  "schemaVersion": 39,
+  "tags": [
+    "hyperhive",
+    "swarm",
+    "nats",
+    "queue"
+  ],
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timezone": "utc",
+  "version": 1,
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "Queue connections",
+      "description": "Clients currently attached to the swarm's NATS server. Zero is NOT healthy here: hive-c0re's status publisher, swarm-controller and the notices stream all hold a connection, so an empty count means the queue is refusing everyone rather than being idle. A callout denial reaches a client as a timeout, so this panel is often the first place that failure is visible at all. No `or vector(0)`, deliberately: that fallback belongs on counts-of-failures, and here it would render a missing exporter as zero clients — conflating a dead queue with a dead scrape, which the scrape panels above already answer.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "gnatsd_varz_connections",
+          "instant": true,
+          "legendFormat": "clients"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "textMode": "value",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        }
+      }
+    },
+    {
+      "id": 2,
+      "type": "stat",
+      "title": "Queue subscriptions",
+      "description": "Active subscriptions across all connected clients. Read alongside the connection count: connections steady while subscriptions collapse means clients are attached but no longer listening, which a liveness check cannot see.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "gnatsd_varz_subscriptions",
+          "instant": true,
+          "legendFormat": "subs"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "text",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "textMode": "value",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        }
+      }
+    },
+    {
+      "id": 3,
+      "type": "stat",
+      "title": "Slow consumers",
+      "description": "Clients the server has disconnected for failing to keep up. Healthy value is zero, so this panel reads empty when everything is fine — see the queue-connections panel for whether the server is being talked to at all.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "gnatsd_varz_slow_consumers or vector(0)",
+          "instant": true,
+          "legendFormat": "slow"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "textMode": "value",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        }
+      },
+      "x-zero-is-healthy": true
+    },
+    {
+      "id": 4,
+      "type": "stat",
+      "title": "JetStream API errors",
+      "description": "Errors answering JetStream API requests. The KV bucket the hive status publisher writes to lives behind this API, so a non-zero value here explains a status page that has gone stale without anything else looking wrong.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "gnatsd_varz_jetstream_stats_api_errors or vector(0)",
+          "instant": true,
+          "legendFormat": "errors"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "textMode": "value",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        }
+      },
+      "x-zero-is-healthy": true
+    },
+    {
+      "id": 5,
+      "type": "timeseries",
+      "title": "Queue messages (range)",
+      "description": "Messages in and out over the selected range. Deliberately `increase()` rather than a per-second rate: this queue carries a status publish per hive per interval and little else, so a rate divides a handful of messages across an hour and draws a flat zero.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 4
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum(increase(gnatsd_varz_in_msgs[$__range])) or vector(0)",
+          "legendFormat": "in"
+        },
+        {
+          "refId": "B",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum(increase(gnatsd_varz_out_msgs[$__range])) or vector(0)",
+          "legendFormat": "out"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false,
+            "axisSoftMin": 0
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true,
+          "calcs": []
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 6,
+      "type": "stat",
+      "title": "Pending bytes to clients",
+      "description": "Bytes queued for delivery to clients. Rises BEFORE a slow consumer is counted, so this is the leading indicator for the panel beside it. Zero is the healthy value.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 12
+      },
+      "targets": [
+        {
+          "expr": "gnatsd_connz_pending_bytes or vector(0)",
+          "legendFormat": "pending"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "textMode": "value",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        }
+      },
+      "x-zero-is-healthy": true
+    },
+    {
+      "id": 7,
+      "type": "timeseries",
+      "title": "JetStream store size vs limit",
+      "description": "The only capacity question the queue has. Used and configured limit on one axis so the headroom is legible without arithmetic; a stream that grows unboundedly shows up here first.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 16
+      },
+      "targets": [
+        {
+          "expr": "gnatsd_varz_jetstream_stats_storage",
+          "legendFormat": "used"
+        },
+        {
+          "expr": "gnatsd_varz_jetstream_config_max_storage",
+          "legendFormat": "limit"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false,
+            "axisSoftMin": 0
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true,
+          "calcs": []
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    }
+  ]
+}

From c2b6bbd23b95edf1e8a369d1cf77e73fd819b886 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Wed, 26 Aug 2026 18:51:25 +0200
Subject: [PATCH 2/2] swarm-grafana: put the queue panels that pair next to
 each other

The pending-bytes panel described itself as the leading indicator for
the slow-consumer stat 'beside it', and then sat alone in its own row
with eighteen grid units of empty space. A description that tells the
reader to look beside it, where nothing is, is worse than no
description.

Slow consumers and pending bytes are now adjacent in the top row, and
the two JetStream panels share the second. No gaps.
---
 .../swarm-grafana/dashboards/queue.json        | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/nix/host-modules/swarm-grafana/dashboards/queue.json b/nix/host-modules/swarm-grafana/dashboards/queue.json
index 6a788132..ee9d30cd 100644
--- a/nix/host-modules/swarm-grafana/dashboards/queue.json
+++ b/nix/host-modules/swarm-grafana/dashboards/queue.json
@@ -222,8 +222,8 @@
       "gridPos": {
         "h": 4,
         "w": 6,
-        "x": 18,
-        "y": 0
+        "x": 0,
+        "y": 4
       },
       "targets": [
         {
@@ -289,7 +289,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 4
+        "y": 8
       },
       "targets": [
         {
@@ -354,8 +354,8 @@
       "gridPos": {
         "h": 4,
         "w": 6,
-        "x": 0,
-        "y": 12
+        "x": 18,
+        "y": 0
       },
       "targets": [
         {
@@ -412,10 +412,10 @@
         "uid": "@datasourceUid@"
       },
       "gridPos": {
-        "h": 8,
-        "w": 24,
-        "x": 0,
-        "y": 16
+        "h": 4,
+        "w": 18,
+        "x": 6,
+        "y": 4
       },
       "targets": [
         {