diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 70437f48..85a920e7 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -403,28 +403,33 @@ of metrics per claude turn, recorded the moment the turn ends (not polled). These are deliberately the fields Claude Code's own built-in export (see above) can't know about — the harness's own wall-clock timing, what woke the turn, its own outcome classification, the loose-ends backlog, session -boundaries, and how full the context window is. Token usage and tool-call -counts are **not** duplicated here; that's already covered by Claude's own -export. Cost is here only for agents on an ACP runtime, which have no export +boundaries, how full the context window is, the tools a turn called, the shell +commands bash tasks ran, and reminders. Token usage is **not** duplicated here; +that's already covered by Claude's own export, which has no tool-call counter. +Cost is here only for agents on an ACP runtime, which have no export of their own — see [ACP-reported cost](#acp-reported-cost). -| Metric | Unit | Kind | Attributes | -| ---------------------------------------- | --------- | --------- | ------------------------------------------------------------------------------------------------------------------ | -| `hyperhive.agent.turn.duration` | `ms` | histogram | `wake_from`, `result_kind`, `model` | -| `hyperhive.agent.turn.count` | — | counter | `wake_from`, `result_kind`, `model` | -| `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) | -| `hyperhive.agent.loose_ends.threads` | — | gauge | none | -| `hyperhive.agent.loose_ends.reminders` | — | gauge | none | -| `hyperhive.agent.todos.open` | — | gauge | none — un-acked todos in the harness-local todo store (reminders excluded), recorded every 30 seconds | -| `hyperhive.agent.claude_md.lines` | — | gauge | none — recorded from the `AGENTS.md`-size watch's own ~15-minute tick, **not** per turn like the rows above | -| `hyperhive.agent.subagent_role.lines` | — | gauge | `role` (the file stem) — one series per role file in `subagent_roles/`, on the same ~15-minute tick | -| `hyperhive.agent.claude_usage.percent` | `%` | gauge | `window` (`five_hour`, `seven_day`, … as the usage endpoint names them) — polled every 5 minutes, **not** per turn | -| `hyperhive.agent.claude_usage.resets_at` | `s` | gauge | `window` — unix seconds at which that window resets; same 5-minute poll | -| `hyperhive.agent.context.used` | `{token}` | gauge | none — tokens in the context window at turn end, the numerator of the web UI's ctx% | -| `hyperhive.agent.context.size` | `{token}` | gauge | none — the context window those tokens fill, ctx%'s denominator | -| `hyperhive.agent.cost.usage` | — | counter | `model`, `currency` — ACP agents only, see below | +| Metric | Unit | Kind | Attributes | +| ---------------------------------------- | --------- | --------- | --------------------------------------------------------------------------------------------------------------------- | +| `hyperhive.agent.turn.duration` | `ms` | histogram | `wake_from`, `result_kind`, `model` | +| `hyperhive.agent.turn.count` | — | counter | `wake_from`, `result_kind`, `model` | +| `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) | +| `hyperhive.agent.loose_ends.threads` | — | gauge | none | +| `hyperhive.agent.loose_ends.reminders` | — | gauge | none | +| `hyperhive.agent.todos.open` | — | gauge | none — un-acked todos in the harness-local todo store (reminders excluded), recorded every 30 seconds | +| `hyperhive.agent.claude_md.lines` | — | gauge | none — recorded from the `AGENTS.md`-size watch's own ~15-minute tick, **not** per turn like the rows above | +| `hyperhive.agent.subagent_role.lines` | — | gauge | `role` (the file stem) — one series per role file in `subagent_roles/`, on the same ~15-minute tick | +| `hyperhive.agent.claude_usage.percent` | `%` | gauge | `window` (`five_hour`, `seven_day`, … as the usage endpoint names them) — polled every 5 minutes, **not** per turn | +| `hyperhive.agent.claude_usage.resets_at` | `s` | gauge | `window` — unix seconds at which that window resets; same 5-minute poll | +| `hyperhive.agent.context.used` | `{token}` | gauge | none — tokens in the context window at turn end, the numerator of the web UI's ctx% | +| `hyperhive.agent.context.size` | `{token}` | gauge | none — the context window those tokens fill, ctx%'s denominator | +| `hyperhive.agent.cost.usage` | — | counter | `model`, `currency` — ACP agents only, see below | +| `hyperhive.agent.tool.calls` | — | counter | `tool` — the tool's name, or `Skill:` for a skill; counted when a turn ends | +| `hyperhive.agent.bash.commands` | — | counter | `command` — the command name a bash task ran (the first real command, past `cd` and `sudo`), counted when a turn ends | +| `hyperhive.agent.reminders` | — | counter | `event` (`scheduled` or `delivered`) | +| `hyperhive.agent.first_turn.context` | `{token}` | gauge | none — input tokens of the first turn of a fresh session, the cold cost of the system prompt and `AGENTS.md` | An agent with no `subagent_roles/` directory exports no `subagent_role.lines` series. A role file deleted while the harness runs diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 91f480d6..76524c38 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -200,7 +200,7 @@ Grafana start — see [`credentials.md`](credentials.md). `nix/host-modules/swarm-grafana/dashboards/`; edits made in the UI aren't kept. -- `agents`: per-agent resource usage and Claude Code activity. The **activity** tab ends with a **Busiest agents** table: turns, input, output and cache-read tokens, and cost for each agent in the range, busiest first. The **turns** tab holds what each agent's own `/stats` page shows about turns: average turn duration, cache hit rate and tokens per turn, turns per hour by agent and by model, the turn duration over time (median, 95th percentile and mean), and the mix of wake sources and results. The duration percentiles are right from the first deploy that includes the histogram bounds in `hive-agent`; before it, they read a flat 10 seconds. +- `agents`: per-agent resource usage and Claude Code activity. The **activity** tab ends with a **Busiest agents** table: turns, input, output and cache-read tokens, and cost for each agent in the range, busiest first. The **turns** tab holds what each agent's own `/stats` page shows about turns: average turn duration, cache hit rate and tokens per turn, turns per hour by agent and by model, the turn duration over time (median, 95th percentile and mean), and the mix of wake sources and results. It also has the tools called, the commands bash tasks ran, reminders per hour, the first-turn context size and fresh sessions per agent: everything the `/stats` page shows, except the per-bucket token bars the **cost** tab covers. The duration percentiles are right from the first deploy that includes the histogram bounds in `hive-agent`; before it, they read a flat 10 seconds. - `attic`: the swarm binary cache. Agents whose latest closure this hive can pull, and why not; the builder's pushes and push failures; closures hive-c0re pulled and deploy steps it refused; the cache's size and store-path count. It reads the closure-pullable gauge, the `swarm-builder` and `hive-c0re` logs, and the `hyperhive.attic.cache_bytes` and `hyperhive.attic.cache_nars` gauges, which a timer in the `swarm-attic` container (`swarm-attic-cache-size`) reads from atticd's database every five minutes and pushes through the container's forwarder. atticd itself exports no metrics and its logs don't reach the log store, so the board has no cache request or hit panel. Filter by hive and agent. - `authelia`, `bao`, `queue`: the identity provider, the secret store and the NATS queue. - `claude-usage`: Claude use by model, effort, token type and source. Filter by hive, service (agents or the review classifier) and agent. The service filter separates the classifier's API-key spend from the agents. diff --git a/hive-agent/src/otel_turn_metrics.rs b/hive-agent/src/otel_turn_metrics.rs index bb182be6..1a5ae88f 100644 --- a/hive-agent/src/otel_turn_metrics.rs +++ b/hive-agent/src/otel_turn_metrics.rs @@ -1,13 +1,13 @@ //! Per-turn OTEL metric export for the fields Claude Code's own built-in //! OTEL integration cannot know about. Claude's own export already covers -//! token usage, cost, and tool-call counts (see `docs/scheduler/observability.md`) — +//! token usage and cost (see `docs/scheduler/observability.md`) — //! duplicating those here under different metric names would just give a //! collector two disagreeing series for the same number. This module emits //! only the harness-only concepts: wall-clock turn duration as *this harness* -//! measures it (not Claude's own per-request latency), what woke the turn, -//! the harness's own outcome classification, the loose-ends backlog at turn -//! end, session boundaries (fresh vs. resumed by title), context-window fill, -//! and the cost an ACP agent reports — claude's runtime reports none. +//! measures it, what woke the turn, its outcome classification, the loose-ends +//! backlog, sessions and their first turn's cost, context-window fill, the +//! tools a turn called (claude exports no tool-call counter), bash commands, +//! reminders, and the cost an ACP agent reports — claude's runtime reports none. //! //! Synchronous instruments, not observable ones: unlike hive-c0re's //! container-resource gauges (`hive-c0re/src/stats/otel_metrics.rs`), which @@ -29,7 +29,9 @@ //! up automatically by the SDK from the container-wide `OTEL_RESOURCE_ATTRIBUTES` //! env var (same mechanism `hive-metric` relies on) — nothing to set here. +use std::collections::HashMap; use std::sync::OnceLock; +use std::sync::atomic::{AtomicI64, Ordering}; use std::time::Duration; use opentelemetry::KeyValue; @@ -92,6 +94,16 @@ struct Instruments { /// Keyed by `model` and `currency`: an ACP agent reports cost in a /// currency of its choosing. reported_cost: Counter, + /// Tool calls made, keyed by `tool` (`Skill:` for a skill). + tool_calls: Counter, + /// Shell commands run by bash tasks, keyed by `command`: the normalised + /// head the bash daemon records, not the full command line. + bash_commands: Counter, + /// Reminders scheduled and delivered, keyed by `event`. + reminders: Counter, + /// Input tokens of the first turn of a fresh claude session: what the + /// system prompt and `AGENTS.md` cost before anything is cached. + first_turn_context: Gauge, } /// Lazily built on the first call to [`record`]. `None` when OTEL isn't @@ -130,7 +142,48 @@ pub fn record(row: &TurnStatRow, fresh_session: bool) { if fresh_session { inst.session_count .add(1, &[KeyValue::new("model", row.model.clone())]); + inst.first_turn_context.record(row.input_tokens, &[]); } + for (tool, calls) in tool_call_counts(row.tool_call_breakdown_json.as_deref()) { + inst.tool_calls.add(calls, &[KeyValue::new("tool", tool)]); + } + record_new_bash_commands(inst, row.started_at); +} + +/// The per-tool call counts a turn stored in `tool_call_breakdown_json`; a +/// turn with no tool calls, or a value that does not parse, has none. +fn tool_call_counts(json: Option<&str>) -> Vec<(String, u64)> { + json.and_then(|j| serde_json::from_str::>(j).ok()) + .map(|m| m.into_iter().collect()) + .unwrap_or_default() +} + +/// Unix seconds up to which bash commands are already counted; `0` before the +/// first turn of this run. +static BASH_COUNTED_UNTIL: AtomicI64 = AtomicI64::new(0); + +/// Count the bash commands recorded since the last call. The first call after +/// the harness starts counts only commands newer than `turn_started_at`, so a +/// restart does not count the whole history again. +fn record_new_bash_commands(inst: &Instruments, turn_started_at: i64) { + let now = chrono::Utc::now().timestamp(); + let mut since = BASH_COUNTED_UNTIL.swap(now, Ordering::Relaxed); + if since == 0 { + since = turn_started_at.saturating_sub(1); + } + for (command, n) in crate::stats::bash_heads_since(since) { + inst.bash_commands + .add(n, &[KeyValue::new("command", command)]); + } +} + +/// Count one reminder event, `scheduled` or `delivered`. No-op when OTEL isn't +/// configured, same as [`record`]. +pub fn record_reminder(event: &'static str) { + let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else { + return; + }; + inst.reminders.add(1, &[KeyValue::new("event", event)]); } /// Record the current `AGENTS.md` line count. Called from @@ -250,6 +303,13 @@ fn build() -> Option { .with_unit("{token}") .build(), reported_cost: meter.f64_counter("hyperhive.agent.cost.usage").build(), + tool_calls: meter.u64_counter("hyperhive.agent.tool.calls").build(), + bash_commands: meter.u64_counter("hyperhive.agent.bash.commands").build(), + reminders: meter.u64_counter("hyperhive.agent.reminders").build(), + first_turn_context: meter + .u64_gauge("hyperhive.agent.first_turn.context") + .with_unit("{token}") + .build(), _provider: provider, }) } @@ -291,7 +351,23 @@ pub fn enabled() -> bool { #[cfg(test)] mod tests { - use super::TURN_DURATION_BOUNDS_MS; + use super::{TURN_DURATION_BOUNDS_MS, tool_call_counts}; + + #[test] + fn a_turns_stored_tool_breakdown_becomes_per_tool_counts() { + let mut got = tool_call_counts(Some(r#"{"Bash":3,"Skill:base:pr-review":1}"#)); + got.sort(); + assert_eq!( + got, + vec![ + ("Bash".to_owned(), 3), + ("Skill:base:pr-review".to_owned(), 1) + ] + ); + // No tool calls, and a value that is not a count map, both count nothing. + assert!(tool_call_counts(None).is_empty()); + assert!(tool_call_counts(Some("not json")).is_empty()); + } #[test] fn the_duration_bounds_ascend_and_reach_past_the_old_ten_second_ceiling() { diff --git a/hive-agent/src/reminders.rs b/hive-agent/src/reminders.rs index c18efea3..08e1a323 100644 --- a/hive-agent/src/reminders.rs +++ b/hive-agent/src/reminders.rs @@ -88,6 +88,7 @@ impl Reminders { VALUES (?1, ?2, ?3, ?4, NULL)", params![message, file_path, due_at, now], )?; + crate::otel_turn_metrics::record_reminder("scheduled"); Ok(conn.last_insert_rowid()) } @@ -173,6 +174,7 @@ impl Reminders { "UPDATE reminders SET sent_at = ?1 WHERE id = ?2", params![Utc::now().timestamp(), id], )?; + crate::otel_turn_metrics::record_reminder("delivered"); Ok(()) } diff --git a/hive-agent/src/stats.rs b/hive-agent/src/stats.rs index cf8e1df5..d9627b15 100644 --- a/hive-agent/src/stats.rs +++ b/hive-agent/src/stats.rs @@ -449,6 +449,32 @@ fn read_bash_breakdown(conn: &Connection, from: i64) -> rusqlite::Result Vec<(String, u64)> { + let Ok(conn) = Connection::open_with_flags(default_path(), OpenFlags::SQLITE_OPEN_READ_ONLY) + else { + return Vec::new(); + }; + let _ = conn.busy_timeout(std::time::Duration::from_millis(500)); + read_bash_heads_since(&conn, since).unwrap_or_default() +} + +fn read_bash_heads_since(conn: &Connection, since: i64) -> rusqlite::Result> { + let mut stmt = + conn.prepare("SELECT head, COUNT(*) FROM bash_commands WHERE ts > ?1 GROUP BY head")?; + let rows = stmt.query_map([since], |row| { + Ok(( + row.get::<_, String>(0)?, + u64_from_i64(row.get::<_, i64>(1)?), + )) + })?; + rows.collect() +} + struct Row { started_at: i64, duration_ms: i64, @@ -782,6 +808,29 @@ mod tests { assert!(s.bash_breakdown.is_empty()); } + /// The OTEL bash-command counter reads only the heads recorded after its + /// watermark, and a database without the table yields nothing. + #[test] + fn bash_heads_since_counts_only_newer_rows() { + let db = tmp_db(); + let _ = std::fs::remove_file(&db); + let conn = Connection::open(&db).unwrap(); + assert!(read_bash_heads_since(&conn, 0).is_err(), "no table yet"); + conn.execute_batch("CREATE TABLE bash_commands (ts INTEGER NOT NULL, head TEXT NOT NULL);") + .unwrap(); + for (ts, head) in [(10, "cargo"), (20, "cargo"), (30, "git"), (30, "cargo")] { + conn.execute( + "INSERT INTO bash_commands (ts, head) VALUES (?1, ?2)", + params![ts, head], + ) + .unwrap(); + } + let mut got = read_bash_heads_since(&conn, 10).unwrap(); + got.sort(); + assert_eq!(got, vec![("cargo".to_owned(), 2), ("git".to_owned(), 1)]); + assert!(read_bash_heads_since(&conn, 30).unwrap().is_empty()); + } + /// With a populated `bash_commands` table, `bash_breakdown` rolls up /// per-head counts (busiest first) and respects the window cutoff. #[test] diff --git a/nix/host-modules/swarm-grafana/dashboards/agents.json b/nix/host-modules/swarm-grafana/dashboards/agents.json index 433a8c83..d468fec1 100644 --- a/nix/host-modules/swarm-grafana/dashboards/agents.json +++ b/nix/host-modules/swarm-grafana/dashboards/agents.json @@ -3318,6 +3318,386 @@ } } } + }, + "panel-78": { + "kind": "Panel", + "spec": { + "id": 78, + "title": "Tool calls", + "description": "The ten tools agents called most in the selected range.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "sort_desc(topk(10, sum by (tool) (increase({__name__=\"hyperhive.agent.tool.calls\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))))", + "instant": true, + "legendFormat": "{{tool}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "bargauge", + "version": "", + "spec": { + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "text" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + } + ] + }, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-79": { + "kind": "Panel", + "spec": { + "id": 79, + "title": "Bash commands", + "description": "The ten commands bash tasks ran most in the selected range, by command name.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "sort_desc(topk(10, sum by (command) (increase({__name__=\"hyperhive.agent.bash.commands\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))))", + "instant": true, + "legendFormat": "{{command}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "bargauge", + "version": "", + "spec": { + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "text" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + } + ] + }, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-80": { + "kind": "Panel", + "spec": { + "id": 80, + "title": "Reminders per hour", + "description": "Reminders agents scheduled and the harness delivered, per hour.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "sum by (event) (rate({__name__=\"hyperhive.agent.reminders\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])) * 3600", + "legendFormat": "{{event}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": { + "interval": "10m" + } + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 1, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisSoftMin": 0, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-81": { + "kind": "Panel", + "spec": { + "id": 81, + "title": "First-turn context", + "description": "Input tokens of the first turn of the latest fresh session, by agent.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "max by (agent) ({__name__=\"hyperhive.agent.first_turn.context\", hive=~\"$hive\", agent=~\"$agent\"})", + "legendFormat": "{{agent}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": { + "interval": "10m" + } + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisSoftMin": 0, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-82": { + "kind": "Panel", + "spec": { + "id": 82, + "title": "Fresh sessions", + "description": "Fresh claude sessions started in the selected range, by agent.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "sort_desc(sum by (agent) (increase({__name__=\"hyperhive.agent.session.count\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])))", + "instant": true, + "legendFormat": "{{agent}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "bargauge", + "version": "", + "spec": { + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "text" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + } + ] + }, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, + "overrides": [] + } + } + } + } } }, "layout": { @@ -3552,6 +3932,71 @@ "name": "panel-77" } } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 28, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-78" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 28, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-79" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 36, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-80" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 8, + "y": 36, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-81" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 36, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-82" + } + } } ] } diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 61907f75..b58ce329 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -503,8 +503,8 @@ let && (lib.head xf).spec.options.byField == "agent"; } { - # The agents board's turns tab reads the turn count and duration the harness - # records, grouped by the labels it sets on them. A renamed instrument or + # The agents board's turns tab reads the turn, tool, bash command, reminder + # and session metrics the harness records, grouped by the labels it sets on them. A renamed instrument or # label there is a tab of empty panels, so each must still be spelled in the # harness; and the duration percentiles are only right while the histogram # has bounds past the SDK's 10 s default ceiling. @@ -520,19 +520,32 @@ let recorded = builtins.readFile ../../hive-agent/src/otel_turn_metrics.rs; in tab != null - && lib.length names == 9 + && lib.length names == 14 && lib.all (m: lib.hasInfix "\"${m}\"" recorded) [ "hyperhive.agent.turn.count" "hyperhive.agent.turn.duration" + "hyperhive.agent.session.count" + "hyperhive.agent.tool.calls" + "hyperhive.agent.bash.commands" + "hyperhive.agent.reminders" + "hyperhive.agent.first_turn.context" "wake_from" "result_kind" "model" + "tool" + "command" + "event" ] && lib.all (l: lib.any (e: lib.hasInfix l e) exprs) [ "wake_from" "result_kind" "by (model)" "turn.duration_bucket" + "by (tool)" + "by (command)" + "by (event)" + "first_turn.context" + "session.count" ] && lib.hasInfix "with_boundaries(TURN_DURATION_BOUNDS_MS" recorded; }