diff --git a/docs/swarm/services.md b/docs/swarm/services.md index ed4a2d4e..65b55758 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -173,7 +173,7 @@ kept. - `agents`: per-agent resource usage and Claude Code activity. - `authelia`, `bao`, `queue`: the identity provider, the secret store and the NATS queue. -- `claude-usage`: Claude use by model, effort, token type and source. +- `claude-usage`: Claude use by model, effort, token type and source. Filter by hive, service (agents or the review classifier) and agent. The service filter separates the classifier's API-key spend from the agents. - `controller`: the swarm controller's job graph by state, and the push and commit rates it sees on the forge. - `forge`: what the forge holds, and the forge process's CPU, memory and descriptors. - `gateway`: the gateway nginx on each hive's host: requests per second, connections by state, and accepted against handled connections. diff --git a/nix/host-modules/swarm-grafana/dashboards/claude-usage.json b/nix/host-modules/swarm-grafana/dashboards/claude-usage.json index 2198f185..ab96b404 100644 --- a/nix/host-modules/swarm-grafana/dashboards/claude-usage.json +++ b/nix/host-modules/swarm-grafana/dashboards/claude-usage.json @@ -33,6 +33,20 @@ }, "sort": 1 }, + { + "name": "service", + "label": "Service", + "description": "Which process emitted the usage. Agents run on subscription or API key per agent; the review classifier is billed by API key.", + "type": "custom", + "query": "agents : hyperhive-agent, review classifier : hyperhive-review-classifier", + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + }, { "name": "agent", "label": "Agent", @@ -41,7 +55,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "query": "label_values({__name__=\"claude_code.cost.usage\", hive=~\"$hive\"}, agent)", + "query": "label_values({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", \"service.name\"=~\"$service\"}, agent)", "refresh": 2, "includeAll": true, "allValue": ".*", @@ -77,7 +91,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sum(increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "expr": "sum(increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__range]))", "instant": true } ], @@ -125,7 +139,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sum(increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "expr": "sum(increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__range]))", "instant": true } ], @@ -173,7 +187,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sum(increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\", type=\"cacheRead\"}[$__range])) / sum(increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "expr": "sum(increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\", type=\"cacheRead\"}[$__range])) / sum(increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__range]))", "instant": true } ], @@ -221,7 +235,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sum(increase({__name__=\"claude_code.active_time.total\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "expr": "sum(increase({__name__=\"claude_code.active_time.total\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__range]))", "instant": true } ], @@ -270,7 +284,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sum by (type) (rate({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])) * 60", + "expr": "sum by (type) (rate({__name__=\"claude_code.token.usage\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__rate_interval])) * 60", "legendFormat": "{{type}}" } ], @@ -331,7 +345,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sort_desc(sum by (model) (increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])))", + "expr": "sort_desc(sum by (model) (increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__range])))", "instant": true, "legendFormat": "{{model}}" } @@ -391,7 +405,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sum by (model) (rate({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])) * 60", + "expr": "sum by (model) (rate({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__rate_interval])) * 60", "legendFormat": "{{model}}" } ], @@ -448,7 +462,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sort_desc(sum by (effort) (increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])))", + "expr": "sort_desc(sum by (effort) (increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__range])))", "instant": true, "legendFormat": "{{effort}}" } @@ -507,7 +521,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sort_desc(sum by (query_source) (increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])))", + "expr": "sort_desc(sum by (query_source) (increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__range])))", "instant": true, "legendFormat": "{{query_source}}" } @@ -566,7 +580,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sort_desc(sum by (language, decision) (increase({__name__=\"claude_code.code_edit_tool.decision\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])))", + "expr": "sort_desc(sum by (language, decision) (increase({__name__=\"claude_code.code_edit_tool.decision\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__range])))", "instant": true, "legendFormat": "{{language}} · {{decision}}" } @@ -625,7 +639,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sort_desc(sum by (tool_name, decision) (increase({__name__=\"claude_code.code_edit_tool.decision\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])))", + "expr": "sort_desc(sum by (tool_name, decision) (increase({__name__=\"claude_code.code_edit_tool.decision\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__range])))", "instant": true, "legendFormat": "{{tool_name}} · {{decision}}" } @@ -684,7 +698,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sum(increase({__name__=\"claude_code.session.count\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "expr": "sum(increase({__name__=\"claude_code.session.count\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__range]))", "instant": true } ], @@ -732,7 +746,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sum(increase({__name__=\"claude_code.active_time.total\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])) / sum(increase({__name__=\"claude_code.session.count\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "expr": "sum(increase({__name__=\"claude_code.active_time.total\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__range])) / sum(increase({__name__=\"claude_code.session.count\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__range]))", "instant": true } ], @@ -781,7 +795,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sum by (agent) (increase({__name__=\"claude_code.session.count\", hive=~\"$hive\", agent=~\"$agent\"}[1h]))", + "expr": "sum by (agent) (increase({__name__=\"claude_code.session.count\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[1h]))", "legendFormat": "{{agent}}" } ], @@ -838,7 +852,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sort_desc(sum by (agent) (increase({__name__=\"claude_code.active_time.total\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])) / sum by (agent) (increase({__name__=\"claude_code.session.count\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])))", + "expr": "sort_desc(sum by (agent) (increase({__name__=\"claude_code.active_time.total\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__range])) / sum by (agent) (increase({__name__=\"claude_code.session.count\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__range])))", "instant": true, "legendFormat": "{{agent}}" } @@ -897,7 +911,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sort_desc(sum by (agent) (increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])))", + "expr": "sort_desc(sum by (agent) (increase({__name__=\"claude_code.cost.usage\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__range])))", "instant": true, "legendFormat": "{{agent}}" } @@ -956,7 +970,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sort_desc(sum by (agent) (increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])))", + "expr": "sort_desc(sum by (agent) (increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"}[$__range])))", "instant": true, "legendFormat": "{{agent}}" } @@ -1016,7 +1030,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "max({__name__=\"hyperhive.agent.claude_usage.percent\", hive=~\"$hive\", agent=~\"$agent\"} unless on(agent, hive, window) ({__name__=\"hyperhive.agent.claude_usage.resets_at\", hive=~\"$hive\", agent=~\"$agent\"} < time()))", + "expr": "max({__name__=\"hyperhive.agent.claude_usage.percent\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"} unless on(agent, hive, window) ({__name__=\"hyperhive.agent.claude_usage.resets_at\", hive=~\"$hive\", \"service.name\"=~\"$service\", agent=~\"$agent\"} < time()))", "legendFormat": "highest window" } ],