From 08b8eb3f73295270876dfe6c5a2e960ea05e1dbe Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 1 Nov 2026 15:08:27 +0100 Subject: [PATCH] export an open-todos gauge and chart it on the agent dashboard --- docs/scheduler/observability.md | 1 + hive-agent/src/main.rs | 3 + hive-agent/src/otel_turn_metrics.rs | 13 +++ hive-agent/src/todos.rs | 33 +++++++ hive-agent/src/todos_gauge.rs | 25 ++++++ .../swarm-grafana/dashboards/agents.json | 87 +++++++++++++++++++ 6 files changed, 162 insertions(+) create mode 100644 hive-agent/src/todos_gauge.rs diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 705b7b45..4b069f18 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -415,6 +415,7 @@ of their own — see [ACP-reported cost](#acp-reported-cost). | `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) | | `hyperhive.agent.loose_ends.threads` | — | gauge | none | | `hyperhive.agent.loose_ends.reminders` | — | gauge | none | +| `hyperhive.agent.todos.open` | — | gauge | none — un-acked todos in the harness-local todo store (reminders excluded), recorded every 30 seconds | | `hyperhive.agent.claude_md.lines` | — | gauge | none — recorded from the `AGENTS.md`-size watch's own ~15-minute tick, **not** per turn like the rows above | | `hyperhive.agent.subagent_role.lines` | — | gauge | `role` (the file stem) — one series per role file in `subagent_roles/`, on the same ~15-minute tick | | `hyperhive.agent.claude_usage.percent` | `%` | gauge | `window` (`five_hour`, `seven_day`, … as the usage endpoint names them) — polled every 5 minutes, **not** per turn | diff --git a/hive-agent/src/main.rs b/hive-agent/src/main.rs index 8fd80b56..00536983 100644 --- a/hive-agent/src/main.rs +++ b/hive-agent/src/main.rs @@ -46,6 +46,7 @@ mod swarm_term; mod term_msg; mod todo_server; mod todos; +mod todos_gauge; mod turn; mod turn_stats; mod vacuum; @@ -458,6 +459,8 @@ fn spawn_todo_socket( tokio::spawn(claude_md_watch::run(store.clone(), todo_wake.clone())); // Swarm-scheduled prompts fire as todos too, from the list the // swarm keeps for this agent. + // Open-todo count for the OTEL gauge. + tokio::spawn(todos_gauge::run(store.clone())); tokio::spawn(schedule_timer::run(store.clone(), todo_wake.clone())); // And so do the todos the swarm notification router queues for // this agent. diff --git a/hive-agent/src/otel_turn_metrics.rs b/hive-agent/src/otel_turn_metrics.rs index 592c6c27..efd90d4b 100644 --- a/hive-agent/src/otel_turn_metrics.rs +++ b/hive-agent/src/otel_turn_metrics.rs @@ -52,6 +52,9 @@ struct Instruments { session_count: Counter, open_threads: Gauge, open_reminders: Gauge, + /// Recorded from [`crate::todos_gauge`]'s own tick: todos change between + /// turns. + open_todos: Gauge, /// Recorded from [`crate::claude_md_watch`]'s own tick, not from /// [`record`] — `AGENTS.md` size is a continuously-live value (like /// hive-c0re's container gauges), not a once-per-turn one, so it has @@ -133,6 +136,15 @@ pub fn record_subagent_role_lines(role: String, lines: u64) { .record(lines, &[KeyValue::new("role", role)]); } +/// Record the number of open todos. No-op when OTEL isn't configured, same +/// as [`record`]. +pub fn record_open_todos(count: u64) { + let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else { + return; + }; + inst.open_todos.record(count, &[]); +} + /// Record one subscription usage window (`five_hour`, `seven_day`, …): /// `percent` is 0-100, `resets_at` unix seconds. No-op when OTEL isn't /// configured, same as [`record`]. @@ -195,6 +207,7 @@ fn build() -> Option { open_reminders: meter .u64_gauge("hyperhive.agent.loose_ends.reminders") .build(), + open_todos: meter.u64_gauge("hyperhive.agent.todos.open").build(), claude_md_lines: meter.u64_gauge("hyperhive.agent.claude_md.lines").build(), subagent_role_lines: meter .u64_gauge("hyperhive.agent.subagent_role.lines") diff --git a/hive-agent/src/todos.rs b/hive-agent/src/todos.rs index a0d8dbfa..4d94e817 100644 --- a/hive-agent/src/todos.rs +++ b/hive-agent/src/todos.rs @@ -376,6 +376,24 @@ impl Todos { Ok(any) } + /// Number of un-acked todo rows across every subsystem: the length of + /// [`Todos::list`]`(None)`, without materialising the rows. + /// + /// # Errors + /// + /// Propagates the sqlite query failure. + /// + /// # Panics + /// + /// Panics if the connection mutex is poisoned. + pub fn open_count(&self) -> Result { + let conn = self.conn.lock().unwrap(); + let n: u64 = conn.query_row("SELECT COUNT(*) FROM todos WHERE acked = 0", [], |row| { + row.get(0) + })?; + Ok(n) + } + /// Reap acked rows whose `acked_at` is older than `cutoff` (a unix /// timestamp). Called by the harness's hourly vacuum sweep /// (`vacuum.rs`), not on any hot path — bounds the table against acked @@ -492,6 +510,21 @@ mod tests { ); } + #[test] + fn open_count_follows_adds_and_acks() { + let (_dir, s) = store(); + assert_eq!(s.open_count().unwrap(), 0); + let (a, _) = s.upsert("bash", None, "one", None, false).unwrap(); + s.upsert("bash", None, "two", None, false).unwrap(); + s.upsert("matrix", Some("!r:x"), "three", None, false) + .unwrap(); + assert_eq!(s.open_count().unwrap(), 3); + s.mark_done(a).unwrap(); + assert_eq!(s.open_count().unwrap(), 2, "acked rows are not open"); + assert_eq!(s.clear("matrix", Some("!r:x")).unwrap(), 1); + assert_eq!(s.open_count().unwrap(), 1); + } + #[test] fn has_any_reflects_emptiness() { let (_dir, s) = store(); diff --git a/hive-agent/src/todos_gauge.rs b/hive-agent/src/todos_gauge.rs new file mode 100644 index 00000000..8a56e731 --- /dev/null +++ b/hive-agent/src/todos_gauge.rs @@ -0,0 +1,25 @@ +//! Feeds the count of open (un-acked) todos into an OTEL gauge on a timer. +//! Todos arrive and leave between turns, so the per-turn record would +//! leave the series stale while the agent sleeps. + +use std::sync::Arc; +use std::time::Duration; + +use crate::todos::Todos; + +/// Half the exporter's one-minute interval, so every export carries a +/// value at most one tick old. +const TICK: Duration = Duration::from_secs(30); + +/// Background loop: record the open-todo count every [`TICK`]. Detached +/// task — runs for the harness's lifetime; read errors are logged, never +/// fatal. +pub async fn run(todos: Arc) { + loop { + match todos.open_count() { + Ok(n) => crate::otel_turn_metrics::record_open_todos(n), + Err(e) => tracing::warn!(error = %e, "open todo count failed"), + } + tokio::time::sleep(TICK).await; + } +} diff --git a/nix/host-modules/swarm-grafana/dashboards/agents.json b/nix/host-modules/swarm-grafana/dashboards/agents.json index 53631f4a..6d7e0dd9 100644 --- a/nix/host-modules/swarm-grafana/dashboards/agents.json +++ b/nix/host-modules/swarm-grafana/dashboards/agents.json @@ -923,6 +923,80 @@ } } }, + "panel-68": { + "kind": "Panel", + "spec": { + "id": 68, + "title": "Open todos by agent", + "description": "Open todos in each agent's todo list over time; reminders excluded.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "sum by (agent) ({__name__=\"hyperhive.agent.todos.open\", hive=~\"$hive\", agent=~\"$agent\"})", + "legendFormat": "{{agent}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisSoftMin": 0, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, "panel-4": { "kind": "Panel", "spec": { @@ -2913,6 +2987,19 @@ "name": "panel-34" } } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 31, + "width": 24, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-68" + } + } } ] }