From b2694b0a92fda25a910c9b717d4cf529dd01b565 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 1 Nov 2026 17:43:59 +0100 Subject: [PATCH] add a grafana dashboard for the swarm binary cache --- docs/swarm/services.md | 1 + nix/host-modules/swarm-grafana.nix | 1 + .../swarm-grafana/dashboards/attic.json | 811 ++++++++++++++++++ 3 files changed, 813 insertions(+) create mode 100644 nix/host-modules/swarm-grafana/dashboards/attic.json diff --git a/docs/swarm/services.md b/docs/swarm/services.md index d0b67762..c0ee2320 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -201,6 +201,7 @@ Grafana start — see [`credentials.md`](credentials.md). kept. - `agents`: per-agent resource usage and Claude Code activity. The **activity** tab ends with a **Busiest agents** table: turns, input, output and cache-read tokens, and cost for each agent in the range, busiest first. +- `attic`: the swarm binary cache. Agents whose latest closure this hive can pull, and why not; the builder's pushes and push failures; closures hive-c0re pulled and deploy steps it refused. It reads the closure-pullable gauge and the `swarm-builder` and `hive-c0re` logs. atticd itself exports no metrics and its logs don't reach the log store, so the board has no cache request, hit or storage panel. Filter by hive and agent. - `authelia`, `bao`, `queue`: the identity provider, the secret store and the NATS queue. - `claude-usage`: Claude use by model, effort, token type and source. Filter by hive, service (agents or the review classifier) and agent. The service filter separates the classifier's API-key spend from the agents. - `egress`: what agents send through the egress proxy, in three tabs. **overview** has the totals (requests, agents active, refused, credential challenges, data transferred), requests over time by status, and the busiest agents and destinations. **refused** lists the hosts the proxy refused for each agent. **would-deny** lists what agents in `log` mode reached that restricted mode would refuse. The host lists are tables, one row per host (and agent), with a count column you can sort. Filter by agent. diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 35e50e95..facb846a 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -269,6 +269,7 @@ let }) [ "agents.json" + "attic.json" "authelia.json" "bao.json" "builds.json" diff --git a/nix/host-modules/swarm-grafana/dashboards/attic.json b/nix/host-modules/swarm-grafana/dashboards/attic.json new file mode 100644 index 00000000..06b095ae --- /dev/null +++ b/nix/host-modules/swarm-grafana/dashboards/attic.json @@ -0,0 +1,811 @@ +{ + "title": "hyperhive · binary cache", + "uid": "hyperhive-attic", + "description": "The swarm's binary cache: what the builder pushes, what each hive can pull, and the refusals.", + "editable": true, + "refresh": "1m", + "schemaVersion": 39, + "tags": ["hyperhive", "swarm", "attic", "cache"], + "time": { + "from": "now-24h", + "to": "now" + }, + "timezone": "utc", + "version": 1, + "templating": { + "list": [ + { + "name": "hive", + "label": "Hive", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "query": "label_values({__name__=\"hyperhive.agent.closure_pullable\"}, hive)", + "refresh": 2, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1 + }, + { + "name": "agent", + "label": "Agent", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "query": "label_values({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\"}, agent)", + "refresh": 2, + "includeAll": true, + "allValue": ".*", + "multi": false, + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1 + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Pullable closures", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Agents pullable", + "description": "Agents whose latest closure this hive can pull and verify. Source: closure_pullable gauge.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(max by (hive, agent) (last_over_time({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"}[3m]))) or vector(0)", + "instant": true, + "legendFormat": "pullable" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "green" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 3, + "type": "stat", + "title": "Agents not pullable", + "description": "Agents whose latest closure this hive cannot pull. Source: closure_pullable gauge.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "count(max by (hive, agent) (last_over_time({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"}[3m])) == 0) or vector(0)", + "instant": true, + "legendFormat": "not pullable" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "red" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 4, + "type": "stat", + "title": "Closures pushed (range)", + "description": "Closures the builder pushed to the cache in the range. Source: swarm-builder logs.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure pushed to the cache\" | stats count_uniq(__CURSOR) as n", + "queryType": "stats", + "legendFormat": "pushed" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "green" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 5, + "type": "stat", + "title": "Push failures (range)", + "description": "Closures the builder failed to push in the range. Source: swarm-builder logs.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure not pushed to the cache\" | stats count_uniq(__CURSOR) as n", + "queryType": "stats", + "legendFormat": "failed" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "red" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 6, + "type": "table", + "title": "Pull verdict per agent", + "description": "Per agent: whether the latest closure is pullable, and why not. Source: closure_pullable gauge.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 5 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max by (agent, reason) (last_over_time({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"}[3m]))", + "instant": true, + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "renameByName": { + "agent": "Agent", + "reason": "Reason", + "Value": "Pullable" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Pullable" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "0": { + "text": "no", + "color": "red", + "index": 0 + }, + "1": { + "text": "yes", + "color": "green", + "index": 1 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [ + { + "displayName": "Pullable", + "desc": false + } + ] + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Pullable agents over time", + "description": "Agents with a pullable closure against agents without. Source: closure_pullable gauge.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(max by (hive, agent) ({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"})) or vector(0)", + "legendFormat": "pullable" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "count(max by (hive, agent) ({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"}) == 0) or vector(0)", + "legendFormat": "not pullable" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 10, + "type": "row", + "title": "Builder pushes", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "panels": [] + }, + { + "id": 11, + "type": "timeseries", + "title": "Pushes over time", + "description": "Closures pushed and not pushed to the cache, per interval. Source: swarm-builder logs.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure pushed to the cache\" | stats count_uniq(__CURSOR) as n", + "queryType": "statsRange", + "legendFormat": "pushed" + }, + { + "refId": "B", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure not pushed to the cache\" | stats count_uniq(__CURSOR) as n", + "queryType": "statsRange", + "legendFormat": "not pushed" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 60, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 12, + "type": "timeseries", + "title": "Agent meta builds", + "description": "Agent builds from the meta flake by outcome, per interval. Source: swarm-builder logs.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"agent meta build built\" | stats count_uniq(__CURSOR) as n", + "queryType": "statsRange", + "legendFormat": "built" + }, + { + "refId": "B", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" (\"agent meta build failed\" OR \"agent meta build not run\" OR \"meta build refused\") | stats count_uniq(__CURSOR) as n", + "queryType": "statsRange", + "legendFormat": "failed or refused" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 60, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 13, + "type": "logs", + "title": "Recent push failures", + "description": "The builder's push-failure lines with the reason, newest first. Source: swarm-builder logs.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 23 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure not pushed to the cache\"", + "queryType": "instant" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + } + }, + { + "id": 20, + "type": "row", + "title": "Hive pulls", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "panels": [] + }, + { + "id": 21, + "type": "stat", + "title": "Closures substituted (range)", + "description": "Closures hive-c0re pulled from the cache for a prebuild or create. Source: hive-c0re logs.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 33 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" F_NAME:~\"$agent\" (\"prebuild: substituted\" OR \"create: substituted\") | stats count_uniq(__CURSOR) as n", + "queryType": "stats", + "legendFormat": "substituted" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "green" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 23, + "type": "timeseries", + "title": "Substitutes and refusals", + "description": "Closures pulled and deploy steps refused, per interval. Source: hive-c0re logs.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 37 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" F_NAME:~\"$agent\" (\"prebuild: substituted\" OR \"create: substituted\") | stats count_uniq(__CURSOR) as n", + "queryType": "statsRange", + "legendFormat": "substituted" + }, + { + "refId": "B", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" F_NAME:~\"$agent\" \"nothing was changed\" (\"the swarm has recorded no build\" OR \"closure cannot be pulled\") | stats count_uniq(__CURSOR) as n", + "queryType": "statsRange", + "legendFormat": "refused" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 60, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 24, + "type": "timeseries", + "title": "Probe verdict changes by reason", + "description": "Times the pull probe's verdict for an agent changed, by reason. Source: hive-c0re logs.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 45 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" F_AGENT:~\"$agent\" \"attic: can this hive pull\" F_REASON:* | stats by (F_REASON) count_uniq(__CURSOR) as n", + "queryType": "statsRange", + "legendFormat": "{{F_REASON}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 60, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] +}