From aecde1dc75e81873df6e62b063d421f0161648f5 Mon Sep 17 00:00:00 2001 From: iris Date: Tue, 3 Nov 2026 00:16:47 +0100 Subject: [PATCH] agents board: a turns tab, and turn duration bounds past 10 s The per-agent /stats page shows what the swarm agents board lacked about turns. The new tab reads the turn count and duration the harness already records: average duration, cache hit rate, tokens per turn, turns per hour by agent and by model, duration percentiles, and the wake source and result mixes. The duration histogram used the SDK default buckets, which stop at 10 s while a turn averages 24 s, so every p95 read a flat 10 s; it now has explicit bounds up to an hour. --- docs/swarm/services.md | 2 +- hive-agent/src/otel_turn_metrics.rs | 38 + .../swarm-grafana/dashboards/agents.json | 825 ++++++++++++++++++ nix/module-eval/grafana.nix | 34 + 4 files changed, 898 insertions(+), 1 deletion(-) diff --git a/docs/swarm/services.md b/docs/swarm/services.md index a39412a4..2a0a5fa5 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -200,7 +200,7 @@ Grafana start — see [`credentials.md`](credentials.md). `nix/host-modules/swarm-grafana/dashboards/`; edits made in the UI aren't kept. -- `agents`: per-agent resource usage and Claude Code activity. The **activity** tab ends with a **Busiest agents** table: turns, input, output and cache-read tokens, and cost for each agent in the range, busiest first. +- `agents`: per-agent resource usage and Claude Code activity. The **activity** tab ends with a **Busiest agents** table: turns, input, output and cache-read tokens, and cost for each agent in the range, busiest first. The **turns** tab holds what each agent's own `/stats` page shows about turns: average turn duration, cache hit rate and tokens per turn, turns per hour by agent and by model, the turn duration over time (median, 95th percentile and mean), and the mix of wake sources and results. The duration percentiles are right from the first deploy that includes the histogram bounds in `hive-agent`; before it, they read a flat 10 seconds. - `attic`: the swarm binary cache. Agents whose latest closure this hive can pull, and why not; the builder's pushes and push failures; closures hive-c0re pulled and deploy steps it refused; the cache's size and store-path count. It reads the closure-pullable gauge, the `swarm-builder` and `hive-c0re` logs, and the `hyperhive.attic.cache_bytes` and `hyperhive.attic.cache_nars` gauges, which a timer in the `swarm-attic` container (`swarm-attic-cache-size`) reads from atticd's database every five minutes and pushes through the container's forwarder. atticd itself exports no metrics and its logs don't reach the log store, so the board has no cache request or hit panel. Filter by hive and agent. - `authelia`, `bao`, `queue`: the identity provider, the secret store and the NATS queue. - `claude-usage`: Claude use by model, effort, token type and source. Filter by hive, service (agents or the review classifier) and agent. The service filter separates the classifier's API-key spend from the agents. diff --git a/hive-agent/src/otel_turn_metrics.rs b/hive-agent/src/otel_turn_metrics.rs index efd90d4b..bb182be6 100644 --- a/hive-agent/src/otel_turn_metrics.rs +++ b/hive-agent/src/otel_turn_metrics.rs @@ -43,6 +43,26 @@ use crate::turn_stats::TurnStatRow; /// two batch on the same rhythm. const EXPORT_INTERVAL: Duration = Duration::from_mins(1); +/// Bucket upper bounds of the turn-duration histogram, in milliseconds. The +/// SDK's default bounds stop at 10 s, but a turn takes about 24 s on average, +/// so with them every p95 read a flat 10 s. These run from half a second to an +/// hour. +const TURN_DURATION_BOUNDS_MS: [f64; 13] = [ + 500.0, + 1_000.0, + 2_500.0, + 5_000.0, + 10_000.0, + 20_000.0, + 30_000.0, + 60_000.0, + 120_000.0, + 300_000.0, + 600_000.0, + 1_800_000.0, + 3_600_000.0, +]; + struct Instruments { /// Kept alive for the process lifetime — the `PeriodicReader` only /// exports while its provider lives. @@ -198,6 +218,7 @@ fn build() -> Option { turn_duration_ms: meter .u64_histogram("hyperhive.agent.turn.duration") .with_unit("ms") + .with_boundaries(TURN_DURATION_BOUNDS_MS.to_vec()) .build(), turn_count: meter.u64_counter("hyperhive.agent.turn.count").build(), session_count: meter.u64_counter("hyperhive.agent.session.count").build(), @@ -267,3 +288,20 @@ fn build_provider(interval: Duration) -> anyhow::Result { pub fn enabled() -> bool { std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").is_ok_and(|s| !s.trim().is_empty()) } + +#[cfg(test)] +mod tests { + use super::TURN_DURATION_BOUNDS_MS; + + #[test] + fn the_duration_bounds_ascend_and_reach_past_the_old_ten_second_ceiling() { + // The SDK drops bounds that are not strictly ascending, which would + // silently bring the default buckets (and the flat p95) back. + assert!(TURN_DURATION_BOUNDS_MS.windows(2).all(|w| w[0] < w[1])); + assert!( + TURN_DURATION_BOUNDS_MS + .last() + .is_some_and(|&b| b > 10_000.0) + ); + } +} diff --git a/nix/host-modules/swarm-grafana/dashboards/agents.json b/nix/host-modules/swarm-grafana/dashboards/agents.json index 6d7e0dd9..433a8c83 100644 --- a/nix/host-modules/swarm-grafana/dashboards/agents.json +++ b/nix/host-modules/swarm-grafana/dashboards/agents.json @@ -2623,6 +2623,701 @@ } } } + }, + "panel-69": { + "kind": "Panel", + "spec": { + "id": 69, + "title": "Average turn duration", + "description": "Mean wall-clock time of a turn in the selected range, as the harness measures it.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "sum(increase({__name__=\"hyperhive.agent.turn.duration_sum\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])) / sum(increase({__name__=\"hyperhive.agent.turn.duration_count\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "instant": true, + "legendFormat": "Average turn duration" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-70": { + "kind": "Panel", + "spec": { + "id": 70, + "title": "Cache hit rate", + "description": "Cached input over all input-side tokens (input, cache reads and cache writes) in the selected range.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "sum(increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\", type=\"cacheRead\"}[$__range])) / sum(increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\", type=~\"input|cacheRead|cacheCreation\"}[$__range]))", + "instant": true, + "legendFormat": "Cache hit rate" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 1, + "color": { + "mode": "fixed", + "fixedColor": "text" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-71": { + "kind": "Panel", + "spec": { + "id": 71, + "title": "Tokens per turn", + "description": "Tokens of every type, subagents included, divided by turns in the selected range.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "sum(increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])) / sum(increase({__name__=\"hyperhive.agent.turn.count\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))", + "instant": true, + "legendFormat": "Tokens per turn" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-72": { + "kind": "Panel", + "spec": { + "id": 72, + "title": "Turns per hour by agent", + "description": "How many turns each agent runs per hour.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "sum by (agent) (rate({__name__=\"hyperhive.agent.turn.count\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])) * 3600", + "legendFormat": "{{agent}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": { + "interval": "10m" + } + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 1, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisSoftMin": 0, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-73": { + "kind": "Panel", + "spec": { + "id": 73, + "title": "Turns per hour by model", + "description": "Turns per hour by the model that ran them; the model drives the token cost.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "sum by (model) (rate({__name__=\"hyperhive.agent.turn.count\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])) * 3600", + "legendFormat": "{{model}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": { + "interval": "10m" + } + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 1, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisSoftMin": 0, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-74": { + "kind": "Panel", + "spec": { + "id": 74, + "title": "Turn duration", + "description": "Median, 95th percentile and mean turn duration.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "histogram_quantile(0.5, sum by (le) (rate({__name__=\"hyperhive.agent.turn.duration_bucket\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])))", + "legendFormat": "p50" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=\"hyperhive.agent.turn.duration_bucket\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])))", + "legendFormat": "p95" + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "sum(rate({__name__=\"hyperhive.agent.turn.duration_sum\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])) / sum(rate({__name__=\"hyperhive.agent.turn.duration_count\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval]))", + "legendFormat": "mean" + } + }, + "refId": "C", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": { + "interval": "10m" + } + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisSoftMin": 0, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-75": { + "kind": "Panel", + "spec": { + "id": 75, + "title": "Wake source mix", + "description": "What woke the turns in the selected range: a peer, the operator, a reminder, a todo or the system.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "sort_desc(sum by (wake_from) (increase({__name__=\"hyperhive.agent.turn.count\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])))", + "instant": true, + "legendFormat": "{{wake_from}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "bargauge", + "version": "", + "spec": { + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "text" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + } + ] + }, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-76": { + "kind": "Panel", + "spec": { + "id": 76, + "title": "Result mix", + "description": "How the turns in the selected range ended, as the harness classifies them.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "sort_desc(sum by (result_kind) (increase({__name__=\"hyperhive.agent.turn.count\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])))", + "instant": true, + "legendFormat": "{{result_kind}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "bargauge", + "version": "", + "spec": { + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "text" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + } + ] + }, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-77": { + "kind": "Panel", + "spec": { + "id": 77, + "title": "Turns that did not end ok", + "description": "Turns per hour that ended in anything but ok: compacted, failed or an API stall.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "sum by (result_kind) (rate({__name__=\"hyperhive.agent.turn.count\", hive=~\"$hive\", agent=~\"$agent\", result_kind!=\"ok\"}[$__rate_interval])) * 3600", + "legendFormat": "{{result_kind}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": { + "interval": "10m" + } + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 2, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisSoftMin": 0, + "drawStyle": "bars", + "fillOpacity": 60, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } } }, "layout": { @@ -2733,6 +3428,136 @@ } } }, + { + "kind": "TabsLayoutTab", + "spec": { + "title": "turns", + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 8, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-69" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 8, + "y": 0, + "width": 8, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-70" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 0, + "width": 8, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-71" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 4, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-72" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 4, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-73" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 12, + "width": 24, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-74" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 20, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-75" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 8, + "y": 20, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-76" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 20, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-77" + } + } + } + ] + } + } + } + }, { "kind": "TabsLayoutTab", "spec": { diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 99a66366..61907f75 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -502,6 +502,40 @@ let ] && (lib.head xf).spec.options.byField == "agent"; } + { + # The agents board's turns tab reads the turn count and duration the harness + # records, grouped by the labels it sets on them. A renamed instrument or + # label there is a tab of empty panels, so each must still be spelled in the + # harness; and the duration percentiles are only right while the histogram + # has bounds past the SDK's 10 s default ceiling. + name = "the agents dashboard's turns tab reads the turn metrics and labels the harness records"; + ok = + let + board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/agents.json); + tab = lib.findFirst (t: t.spec.title == "turns") null board.spec.layout.spec.tabs; + names = map (i: i.spec.element.name) tab.spec.layout.spec.items; + exprs = lib.concatMap ( + n: map (q: q.spec.query.spec.expr) board.spec.elements.${n}.spec.data.spec.queries + ) names; + recorded = builtins.readFile ../../hive-agent/src/otel_turn_metrics.rs; + in + tab != null + && lib.length names == 9 + && lib.all (m: lib.hasInfix "\"${m}\"" recorded) [ + "hyperhive.agent.turn.count" + "hyperhive.agent.turn.duration" + "wake_from" + "result_kind" + "model" + ] + && lib.all (l: lib.any (e: lib.hasInfix l e) exprs) [ + "wake_from" + "result_kind" + "by (model)" + "turn.duration_bucket" + ] + && lib.hasInfix "with_boundaries(TURN_DURATION_BOUNDS_MS" recorded; + } { # The builds board reads the fields hive-c0re puts on every container-build # log line: the build id, agent and kind on each, the event on the start and