diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 62a9a58d..640d8f32 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -46,10 +46,15 @@ let # Grafana's file provider wants a DIRECTORY to scan, so the rendered files are # collected into one. dashboardDir = pkgs.linkFarm "hyperhive-grafana-dashboards" ( - map (name: { - inherit name; - path = renderDashboard name; - }) [ "agents.json" ] + map + (name: { + inherit name; + path = renderDashboard name; + }) + [ + "agents.json" + "claude-usage.json" + ] ); # Total on a null swarm domain for the same reason every sibling module is: diff --git a/nix/host-modules/swarm-grafana/dashboards/claude-usage.json b/nix/host-modules/swarm-grafana/dashboards/claude-usage.json new file mode 100644 index 00000000..834cedcd --- /dev/null +++ b/nix/host-modules/swarm-grafana/dashboards/claude-usage.json @@ -0,0 +1,906 @@ +{ + "uid": "hyperhive-claude-usage", + "title": "hyperhive · claude usage", + "description": "How Claude is being used, rather than which agent is using it: the axes here are model, effort, token type and query source. Per-agent resource usage lives on the 'agents' dashboard; the only overlap is the cost/token headline. Metric names are the OTLP DOTTED form, so every query uses the {__name__=\"...\"} selector — the underscore spelling matches nothing in this store. Every total uses increase(): the claude_code.* counters are cumulative (forced in nix/agent-modules/otel.nix) but reset once per turn, because each turn is a new claude process, so sum_over_time would be wrong. Rate panels pin a 10m minimum interval — agents export every 60s, and a shorter window silently drops most series while still drawing a plausible line.", + "tags": [ + "hyperhive", + "claude" + ], + "timezone": "utc", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "1m", + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "hive", + "label": "Hive", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "query": "label_values({__name__=\"claude_code.cost.usage\"}, hive)", + "refresh": 2, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1 + }, + { + "name": "agent", + "label": "Agent", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "query": "label_values({__name__=\"claude_code.cost.usage\", hive=~\"$hive\"}, agent)", + "refresh": 2, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1 + } + ] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Cost", + "description": "claude_code.cost.usage, summed over the selected range.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "decimals": 2, + "color": { + "mode": "fixed", + "fixedColor": "purple" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 2, + "type": "stat", + "title": "Tokens", + "description": "claude_code.token.usage across every type (input, output, cacheRead, cacheCreation).", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 3, + "type": "stat", + "title": "Cache read share", + "description": "cacheRead as a fraction of all tokens. High is good and cheap: cached input is billed at a fraction of fresh input. A sudden drop usually means prompts stopped being stable, not that more work happened.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\", type=\"cacheRead\"}[$__range])) / sum(increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 1, + "color": { + "mode": "fixed", + "fixedColor": "green" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 4, + "type": "stat", + "title": "Active time", + "description": "claude_code.active_time.total — seconds actually spent working, not wall-clock uptime.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"claude_code.active_time.total\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "orange" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Token rate by type", + "description": "Tokens per minute, split by type. cacheRead usually dominates by an order of magnitude — that is the cache working, not a bug. 10m minimum step: agents export every 60s, so a shorter window drops series silently.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "interval": "10m", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (type) (rate({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])) * 60", + "legendFormat": "{{type}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0, + "scaleDistribution": { + "type": "log", + "log": 10 + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 11, + "type": "bargauge", + "title": "Tokens by model", + "description": "Which model the tokens went to, over the selected range.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (model) (increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "instant": true, + "legendFormat": "{{model}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 20, + "type": "timeseries", + "title": "Cost rate by model", + "description": "USD per minute, split by model. Same 10m minimum step as the token panel.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "interval": "10m", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 11 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (model) (rate({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])) * 60", + "legendFormat": "{{model}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "decimals": 3, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 21, + "type": "bargauge", + "title": "Cost by effort", + "description": "claude_code.cost.usage split by the effort label (high / medium / xhigh).", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 11 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (effort) (increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "instant": true, + "legendFormat": "{{effort}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "decimals": 2, + "color": { + "mode": "fixed", + "fixedColor": "purple" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 22, + "type": "bargauge", + "title": "Cost by query source", + "description": "'main' is the turn you asked for; 'auxiliary' is everything Claude runs on the side (titles, summaries, small helper calls). Worth watching as a ratio — auxiliary creeping up means spend that no turn asked for.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 11 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (query_source) (increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "instant": true, + "legendFormat": "{{query_source}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "decimals": 2, + "color": { + "mode": "fixed", + "fixedColor": "orange" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 30, + "type": "bargauge", + "title": "Edit tool decisions by language", + "description": "claude_code.code_edit_tool.decision. The decision is in the legend rather than assumed: today every recorded decision is 'accept', so a title saying 'accepted' would be silently wrong the first time one is rejected.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 18 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (language, decision) (increase({__name__=\"claude_code.code_edit_tool.decision\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "instant": true, + "legendFormat": "{{language}} · {{decision}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "green" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 31, + "type": "bargauge", + "title": "Edit tool decisions by tool", + "description": "Same series split by tool_name (Edit / Write) instead of language.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 18 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (tool_name, decision) (increase({__name__=\"claude_code.code_edit_tool.decision\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "instant": true, + "legendFormat": "{{tool_name}} · {{decision}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "yellow" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 5, + "type": "stat", + "title": "Turns", + "description": "claude_code.session.count. Each turn runs a new claude process, so a session record is emitted per turn — this is a PROXY for turn count, not a turn metric. Nothing exports turn stats to OTEL today; they live in each agent turn-stats.sqlite and are aggregated only for the hive dashboard.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"claude_code.session.count\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 6, + "type": "stat", + "title": "Active time per turn", + "description": "Mean of claude_code.active_time.total divided by the session count above. This is ACTIVE seconds per turn, not wall-clock turn duration — time spent waiting on a tool or on the operator is not in it. A counter gives a mean only; there is no p50/p95 without the harness exporting a histogram.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"claude_code.active_time.total\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])) / sum(increase({__name__=\"claude_code.session.count\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "orange" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 40, + "type": "timeseries", + "title": "Turns per hour by agent", + "description": "Turns per hour, as a rolling 1h count. PROXY: each turn runs a new claude process, so a session record stands in for a turn.\n\n⚠️ The window is a fixed [1h] and deliberately does NOT follow the zoom: \"per hour\" is the definition of the number, not a resolution choice. rate() cannot be used here at all — a session series is only 1-3 samples long, so rate() returns a series whose value is zero, which draws a flat line rather than an error.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "interval": "10m", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (agent) (increase({__name__=\"claude_code.session.count\", hive=~\"$hive\", agent=~\"$agent\"}[1h]))", + "legendFormat": "{{agent}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 1, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 41, + "type": "bargauge", + "title": "Active time per turn by agent", + "description": "Mean active seconds per turn, per agent, over the selected range. A high value is a long working turn, not a slow one — waiting is not counted.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 25 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (agent) (increase({__name__=\"claude_code.active_time.total\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])) / sum by (agent) (increase({__name__=\"claude_code.session.count\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "instant": true, + "legendFormat": "{{agent}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "orange" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + } + ] +}