8796f4edac
Split out of the combined swarm-services board, which mara declined as multiple dashboards in one trench coat. Its 17 panels partition exactly by subject -- 3 services / 5 metrics store / 5 queue / 4 log store, with no panel belonging to two -- so this is a re-cut of verified panels, not new authoring. Seven panels: connections, subscriptions, slow consumers and JetStream API errors as stats, messages over the range, plus pending bytes (the leading indicator for a slow consumer) and JetStream store size against its configured limit (the only capacity question the queue has). Connections is red at zero, unlike every other count here: three things hold a connection at all times, so an empty count means the queue is refusing everyone -- and a callout denial reaches a client as a timeout rather than an error, which makes this often the first visible sign.
462 lines
12 KiB
JSON
462 lines
12 KiB
JSON
{
|
|
"title": "hyperhive · queue (nats)",
|
|
"uid": "hyperhive-swarm-queue",
|
|
"description": "NATS / JetStream health for the swarm queue. Split out of the combined swarm-services board: a page whose content is really N services should be N pages.",
|
|
"editable": true,
|
|
"refresh": "1m",
|
|
"schemaVersion": 39,
|
|
"tags": [
|
|
"hyperhive",
|
|
"swarm",
|
|
"nats",
|
|
"queue"
|
|
],
|
|
"time": {
|
|
"from": "now-24h",
|
|
"to": "now"
|
|
},
|
|
"timezone": "utc",
|
|
"version": 1,
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "stat",
|
|
"title": "Queue connections",
|
|
"description": "Clients currently attached to the swarm's NATS server. Zero is NOT healthy here: hive-c0re's status publisher, swarm-controller and the notices stream all hold a connection, so an empty count means the queue is refusing everyone rather than being idle. A callout denial reaches a client as a timeout, so this panel is often the first place that failure is visible at all. No `or vector(0)`, deliberately: that fallback belongs on counts-of-failures, and here it would render a missing exporter as zero clients — conflating a dead queue with a dead scrape, which the scrape panels above already answer.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "gnatsd_varz_connections",
|
|
"instant": true,
|
|
"legendFormat": "clients"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "red",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "green",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "value",
|
|
"textMode": "value",
|
|
"justifyMode": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "stat",
|
|
"title": "Queue subscriptions",
|
|
"description": "Active subscriptions across all connected clients. Read alongside the connection count: connections steady while subscriptions collapse means clients are attached but no longer listening, which a liveness check cannot see.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "gnatsd_varz_subscriptions",
|
|
"instant": true,
|
|
"legendFormat": "subs"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "text",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "value",
|
|
"textMode": "value",
|
|
"justifyMode": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "stat",
|
|
"title": "Slow consumers",
|
|
"description": "Clients the server has disconnected for failing to keep up. Healthy value is zero, so this panel reads empty when everything is fine — see the queue-connections panel for whether the server is being talked to at all.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "gnatsd_varz_slow_consumers or vector(0)",
|
|
"instant": true,
|
|
"legendFormat": "slow"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "value",
|
|
"textMode": "value",
|
|
"justifyMode": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
},
|
|
"x-zero-is-healthy": true
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "stat",
|
|
"title": "JetStream API errors",
|
|
"description": "Errors answering JetStream API requests. The KV bucket the hive status publisher writes to lives behind this API, so a non-zero value here explains a status page that has gone stale without anything else looking wrong.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "gnatsd_varz_jetstream_stats_api_errors or vector(0)",
|
|
"instant": true,
|
|
"legendFormat": "errors"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "value",
|
|
"textMode": "value",
|
|
"justifyMode": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
},
|
|
"x-zero-is-healthy": true
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "timeseries",
|
|
"title": "Queue messages (range)",
|
|
"description": "Messages in and out over the selected range. Deliberately `increase()` rather than a per-second rate: this queue carries a status publish per hive per interval and little else, so a rate divides a handful of messages across an hour and draws a flat zero.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 4
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "sum(increase(gnatsd_varz_in_msgs[$__range])) or vector(0)",
|
|
"legendFormat": "in"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "sum(increase(gnatsd_varz_out_msgs[$__range])) or vector(0)",
|
|
"legendFormat": "out"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"axisSoftMin": 0
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true,
|
|
"calcs": []
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 6,
|
|
"type": "stat",
|
|
"title": "Pending bytes to clients",
|
|
"description": "Bytes queued for delivery to clients. Rises BEFORE a slow consumer is counted, so this is the leading indicator for the panel beside it. Zero is the healthy value.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 12
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "gnatsd_connz_pending_bytes or vector(0)",
|
|
"legendFormat": "pending"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "value",
|
|
"textMode": "value",
|
|
"justifyMode": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
},
|
|
"x-zero-is-healthy": true
|
|
},
|
|
{
|
|
"id": 7,
|
|
"type": "timeseries",
|
|
"title": "JetStream store size vs limit",
|
|
"description": "The only capacity question the queue has. Used and configured limit on one axis so the headroom is legible without arithmetic; a stream that grows unboundedly shows up here first.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "gnatsd_varz_jetstream_stats_storage",
|
|
"legendFormat": "used"
|
|
},
|
|
{
|
|
"expr": "gnatsd_varz_jetstream_config_max_storage",
|
|
"legendFormat": "limit"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"axisSoftMin": 0
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true,
|
|
"calcs": []
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|