diff --git a/nix/host-modules/swarm-grafana/dashboards/agents.json b/nix/host-modules/swarm-grafana/dashboards/agents.json index 85bffbc2..ab58ce60 100644 --- a/nix/host-modules/swarm-grafana/dashboards/agents.json +++ b/nix/host-modules/swarm-grafana/dashboards/agents.json @@ -1,11 +1,11 @@ { "uid": "hyperhive-agents", "title": "hyperhive · agents", - "description": "Per-agent view: resource usage from hive-c0re's container sampler, activity from Claude Code's own export. Renamed from 'overview' — every producer we have today is per-agent, so a genuine swarm overview has nothing cross-cutting to summarise yet and should be built last, not first. Metric names are the OTLP DOTTED form, so every query uses the {__name__=\"...\"} selector — the underscore spelling matches nothing in this store. Cost/token totals use increase(), which is correct here: the claude_code.* counters are cumulative (forced in nix/agent-modules/otel.nix) but reset once per turn, since each turn is a new claude process.", + "description": "Per-agent view: resource usage from hive-c0re's container sampler, activity from Claude Code's own export. Renamed from 'overview' — every producer we have today is per-agent, so a genuine swarm overview has nothing cross-cutting to summarise yet and should be built last, not first. Metric names are the OTLP DOTTED form, so every query uses the {__name__=\"...\"} selector — the underscore spelling matches nothing in this store. Cost/token totals use increase(), which is correct here: the claude_code.* counters are cumulative (forced in nix/agent-modules/otel.nix) but reset once per turn, since each turn is a new claude process.\n\nFour rows (mara/#4387: split cost/health/subagents apart) rather than tabs — Grafana's tab layout is a Scenes-API (v2 schema) feature, and this Grafana's file-based provisioner (allowUiUpdates=false, this JSON is authoritative) doesn't reliably load v2-schema dashboards. Collapsible rows are the v1-schema equivalent: one open, the rest collapsed, same as a tab strip behaviourally. 'activity' is a fourth row for the five panels that are neither cost, resource health, nor subagent-specific (agent count, turn count, code/config-size trends) — left expanded so the dashboard opens on something besides a wall of accordion headers.", "tags": ["hyperhive", "agents"], "timezone": "utc", "schemaVersion": 39, - "version": 2, + "version": 3, "editable": true, "refresh": "1m", "time": { @@ -55,6 +55,14 @@ ] }, "panels": [ + { + "id": 200, + "type": "row", + "title": "activity", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "panels": [] + }, { "id": 1, "type": "stat", @@ -64,12 +72,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 0 - }, + "gridPos": { "h": 4, "w": 12, "x": 0, "y": 1 }, "targets": [ { "refId": "A", @@ -106,106 +109,6 @@ } } }, - { - "id": 2, - "type": "stat", - "title": "Cost", - "description": "sum(increase(claude_code.cost.usage)). Agent-reported, so it covers claude's own token spend only — nothing else in the swarm reports cost.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum(increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", - "instant": true, - "legendFormat": "cost" - } - ], - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "decimals": 2, - "color": { - "mode": "fixed", - "fixedColor": "text" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 3, - "type": "stat", - "title": "Tokens (excl. cache reads)", - "description": "Excludes type=cacheRead, which would otherwise BE this number: measured over 6h on this store, cacheRead was 273.5M of 275.8M total — 99.2%. Counting it makes the tile a cache-hit-rate readout wearing a token label, and it drowns the input/output volume that actually tracks work done. Remaining types: input, output, cacheCreation.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum(increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\", type!=\"cacheRead\"}[$__range]))", - "instant": true, - "legendFormat": "tokens" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 1, - "color": { - "mode": "fixed", - "fixedColor": "text" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, { "id": 4, "type": "stat", @@ -215,12 +118,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 0 - }, + "gridPos": { "h": 4, "w": 12, "x": 12, "y": 1 }, "targets": [ { "refId": "A", @@ -256,292 +154,6 @@ } } }, - { - "id": 10, - "type": "timeseries", - "title": "Cost rate by agent", - "description": "USD per hour, from rate() over the cumulative counter — reset-safe. A flat line at zero is an idle agent, not a broken exporter; check the CPU panel to tell them apart.\n\nThe 10m minimum step is load-bearing, not a style choice: agents export every 60s, and measured against this store a rate window of 1m or 2m resolves ONE agent instead of five. Any shorter and the panel quietly under-reports.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "interval": "10m", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 4 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum by (agent) (rate({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])) * 3600", - "legendFormat": "{{agent}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "decimals": 2, - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 0, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 11, - "type": "timeseries", - "title": "Container CPU by agent", - "description": "hyperhive.container.cpu.percent, sampled by hive-c0re. Percent of one core.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 12 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum by (agent) ({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\"})", - "legendFormat": "{{agent}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "percent", - "decimals": 1, - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 0, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 20, - "type": "timeseries", - "title": "Container memory by agent", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 12 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum by (agent) ({__name__=\"container.memory.usage\", hive=~\"$hive\", agent=~\"$agent\"})", - "legendFormat": "{{agent}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "bytes", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 0, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 21, - "type": "timeseries", - "title": "Tokens by model (excl. cache reads)", - "description": "Split by model rather than by agent on purpose — 'which model is the spend on' is the question this answers. Excludes type=cacheRead for the same reason as the tile above: at ~99% of volume it flattens every model into one line shaped like its cache. 10m minimum step for the same reason as the cost-rate panel.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "interval": "10m", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 4 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum by (model) (rate({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\", type!=\"cacheRead\"}[$__rate_interval])) * 60", - "legendFormat": "{{model}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 0, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 30, - "type": "bargauge", - "title": "Container storage by agent", - "description": "Magnitude comparison, so a bar rather than seven lines. One fixed colour on purpose: the agents are nominal categories, and colouring them by size would encode the bar length twice.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 20 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum by (agent) ({__name__=\"hyperhive.container.storage.usage\", hive=~\"$hive\", agent=~\"$agent\"})", - "instant": true, - "legendFormat": "{{agent}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "bytes", - "color": { - "mode": "fixed", - "fixedColor": "blue" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "displayMode": "basic", - "orientation": "horizontal", - "showUnfilled": true, - "valueMode": "text", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, { "id": 31, "type": "timeseries", @@ -552,12 +164,7 @@ "uid": "@datasourceUid@" }, "interval": "10m", - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 20 - }, + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 5 }, "targets": [ { "refId": "A", @@ -609,12 +216,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 20 - }, + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 5 }, "targets": [ { "refId": "A", @@ -663,17 +265,12 @@ "id": 33, "type": "timeseries", "title": "CLAUDE.md size by agent", - "description": "hyperhive.agent.claude_md.lines over time — mara: \"i want to see it over time\" (a bargauge snapshot in an earlier round of this panel only showed the current count). A continuously-live gauge, not a per-turn counter, so this is the raw value plotted across the range, same treatment as the CPU/memory-by-agent panels above rather than a rate()/increase(). The harness itself raises a loose-ends todo past 500 lines suggesting a trim, not a hard limit — this panel is visibility, not an alarm.", + "description": "hyperhive.agent.claude_md.lines over time — mara: \"i want to see it over time\" (a bargauge snapshot in an earlier round of this panel only showed the current count). A continuously-live gauge, not a per-turn counter, so this is the raw value plotted across the range, same treatment as the CPU/memory-by-agent panels rather than a rate()/increase(). The harness itself raises a loose-ends todo past 500 lines suggesting a trim, not a hard limit — this panel is visibility, not an alarm.", "datasource": { "type": "prometheus", "uid": "@datasourceUid@" }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 27 - }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 12 }, "targets": [ { "refId": "A", @@ -717,272 +314,625 @@ } }, { - "id": 40, - "type": "stat", - "title": "Subagent cost", - "description": "sum(increase(claude_code.cost.usage{subagent!=\"\"})). `hive-subagent-mcp` stamps a `subagent=` resource attribute (alongside the parent's own `agent=`) on every claude_code.* sample a spawned subagent turn emits — this label already existed, it just wasn't plotted anywhere.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 8, - "x": 0, - "y": 35 - }, - "targets": [ + "id": 201, + "type": "row", + "title": "cost", + "collapsed": true, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 }, + "panels": [ { - "refId": "A", + "id": 2, + "type": "stat", + "title": "Cost", + "description": "sum(increase(claude_code.cost.usage)). Agent-reported, so it covers claude's own token spend only — nothing else in the swarm reports cost.", "datasource": { "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sum(increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\", subagent!=\"\"}[$__range]))", - "instant": true, - "legendFormat": "subagent cost" - } - ], - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "decimals": 2, - "color": { - "mode": "fixed", - "fixedColor": "text" + "gridPos": { "h": 4, "w": 12, "x": 0, "y": 21 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "instant": true, + "legendFormat": "cost" + } + ], + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "decimals": 2, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [] + }, + "overrides": [] }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 41, - "type": "stat", - "title": "Subagent turns", - "description": "claude_code.session.count where subagent!=\"\" — a proxy for subagent turns, same PROXY caveat as the plain 'CLI sessions started' tile above.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 8, - "x": 8, - "y": 35 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum(increase({__name__=\"claude_code.session.count\", hive=~\"$hive\", agent=~\"$agent\", subagent!=\"\"}[$__range]))", - "instant": true, - "legendFormat": "subagent turns" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "fixed", - "fixedColor": "text" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 42, - "type": "stat", - "title": "Subagent cost share", - "description": "Subagent cost as a fraction of all cost in the selected range — 'basically free' is a claim this tile can actually check.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 8, - "x": 16, - "y": 35 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum(increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\", subagent!=\"\"}[$__range])) / sum(increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", - "instant": true, - "legendFormat": "subagent share" - } - ], - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "decimals": 1, - "color": { - "mode": "fixed", - "fixedColor": "text" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 43, - "type": "bargauge", - "title": "Cost by subagent name", - "description": "claude_code.cost.usage summed over the selected range, split by the caller-chosen `subagent` name (e.g. an exploration/relabeling task's session name) rather than which agent spawned it.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 39 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum by (subagent) (increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\", subagent!=\"\"}[$__range]))", - "instant": true, - "legendFormat": "{{subagent}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "decimals": 2, - "color": { - "mode": "fixed", - "fixedColor": "purple" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": null - } - ] + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } } }, - "overrides": [] - }, - "options": { - "displayMode": "basic", - "orientation": "horizontal", - "showUnfilled": true, - "valueMode": "text", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 44, - "type": "bargauge", - "title": "Subagent cost by parent agent", - "description": "Same claude_code.cost.usage{subagent!=\"\"}, grouped by the spawning agent's own `agent` label instead — which agent's subagent habit is actually costing something.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 39 - }, - "targets": [ { - "refId": "A", + "id": 3, + "type": "stat", + "title": "Tokens (excl. cache reads)", + "description": "Excludes type=cacheRead, which would otherwise BE this number: measured over 6h on this store, cacheRead was 273.5M of 275.8M total — 99.2%. Counting it makes the tile a cache-hit-rate readout wearing a token label, and it drowns the input/output volume that actually tracks work done. Remaining types: input, output, cacheCreation.", "datasource": { "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sum by (agent) (increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\", subagent!=\"\"}[$__range]))", - "instant": true, - "legendFormat": "{{agent}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "decimals": 2, - "color": { - "mode": "fixed", - "fixedColor": "blue" + "gridPos": { "h": 4, "w": 12, "x": 12, "y": 21 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\", type!=\"cacheRead\"}[$__range]))", + "instant": true, + "legendFormat": "tokens" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 1, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [] + }, + "overrides": [] }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } } }, - "overrides": [] - }, - "options": { - "displayMode": "basic", - "orientation": "horizontal", - "showUnfilled": true, - "valueMode": "text", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false + { + "id": 10, + "type": "timeseries", + "title": "Cost rate by agent", + "description": "USD per hour, from rate() over the cumulative counter — reset-safe. A flat line at zero is an idle agent, not a broken exporter; check the CPU panel to tell them apart.\n\nThe 10m minimum step is load-bearing, not a style choice: agents export every 60s, and measured against this store a rate window of 1m or 2m resolves ONE agent instead of five. Any shorter and the panel quietly under-reports.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "interval": "10m", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (agent) (rate({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])) * 3600", + "legendFormat": "{{agent}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "decimals": 2, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 21, + "type": "timeseries", + "title": "Tokens by model (excl. cache reads)", + "description": "Split by model rather than by agent on purpose — 'which model is the spend on' is the question this answers. Excludes type=cacheRead for the same reason as the tile above: at ~99% of volume it flattens every model into one line shaped like its cache. 10m minimum step for the same reason as the cost-rate panel.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "interval": "10m", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (model) (rate({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\", type!=\"cacheRead\"}[$__rate_interval])) * 60", + "legendFormat": "{{model}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } } - } + ] + }, + { + "id": 202, + "type": "row", + "title": "health", + "collapsed": true, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 33 }, + "panels": [ + { + "id": 11, + "type": "timeseries", + "title": "Container CPU by agent", + "description": "hyperhive.container.cpu.percent, sampled by hive-c0re. Percent of one core.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (agent) ({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\"})", + "legendFormat": "{{agent}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 1, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 20, + "type": "timeseries", + "title": "Container memory by agent", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 34 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (agent) ({__name__=\"container.memory.usage\", hive=~\"$hive\", agent=~\"$agent\"})", + "legendFormat": "{{agent}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 30, + "type": "bargauge", + "title": "Container storage by agent", + "description": "Magnitude comparison, so a bar rather than seven lines. One fixed colour on purpose: the agents are nominal categories, and colouring them by size would encode the bar length twice.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { "h": 7, "w": 24, "x": 0, "y": 42 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (agent) ({__name__=\"hyperhive.container.storage.usage\", hive=~\"$hive\", agent=~\"$agent\"})", + "instant": true, + "legendFormat": "{{agent}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + } + ] + }, + { + "id": 203, + "type": "row", + "title": "subagents", + "collapsed": true, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 49 }, + "panels": [ + { + "id": 40, + "type": "stat", + "title": "Subagent cost", + "description": "sum(increase(claude_code.cost.usage{subagent!=\"\"})). `hive-subagent-mcp` stamps a `subagent=` resource attribute (alongside the parent's own `agent=`) on every claude_code.* sample a spawned subagent turn emits — this label already existed, it just wasn't plotted anywhere.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { "h": 4, "w": 8, "x": 0, "y": 50 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\", subagent!=\"\"}[$__range]))", + "instant": true, + "legendFormat": "subagent cost" + } + ], + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "decimals": 2, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 41, + "type": "stat", + "title": "Subagent turns", + "description": "claude_code.session.count where subagent!=\"\" — a proxy for subagent turns, same PROXY caveat as the plain 'CLI sessions started' tile above.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { "h": 4, "w": 8, "x": 8, "y": 50 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"claude_code.session.count\", hive=~\"$hive\", agent=~\"$agent\", subagent!=\"\"}[$__range]))", + "instant": true, + "legendFormat": "subagent turns" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 42, + "type": "stat", + "title": "Subagent cost share", + "description": "Subagent cost as a fraction of all cost in the selected range — 'basically free' is a claim this tile can actually check.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 50 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\", subagent!=\"\"}[$__range])) / sum(increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "instant": true, + "legendFormat": "subagent share" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 1, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 43, + "type": "bargauge", + "title": "Cost by subagent name", + "description": "claude_code.cost.usage summed over the selected range, split by the caller-chosen `subagent` name (e.g. an exploration/relabeling task's session name) rather than which agent spawned it.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 54 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (subagent) (increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\", subagent!=\"\"}[$__range]))", + "instant": true, + "legendFormat": "{{subagent}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "decimals": 2, + "color": { + "mode": "fixed", + "fixedColor": "purple" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 44, + "type": "bargauge", + "title": "Subagent cost by parent agent", + "description": "Same claude_code.cost.usage{subagent!=\"\"}, grouped by the spawning agent's own `agent` label instead — which agent's subagent habit is actually costing something.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 54 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (agent) (increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\", subagent!=\"\"}[$__range]))", + "instant": true, + "legendFormat": "{{agent}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "decimals": 2, + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + } + ] } ] }