vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

export an open-todos gauge and chart it on the agent dashboard

Browse source
This commit is contained in:
atlas 2026-11-01 15:08:27 +01:00 • committed by mara
commit 08b8eb3f73
6 changed files with 162 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
docs/scheduler/observability.md
View file

@ -415,6 +415,7 @@ of their own — see [ACP-reported cost](#acp-reported-cost).
| `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) |
| `hyperhive.agent.loose_ends.threads` | — | gauge | none |
| `hyperhive.agent.loose_ends.reminders` | — | gauge | none |
| `hyperhive.agent.todos.open` | — | gauge | none — un-acked todos in the harness-local todo store (reminders excluded), recorded every 30 seconds |
| `hyperhive.agent.claude_md.lines` | — | gauge | none — recorded from the `AGENTS.md`-size watch's own ~15-minute tick, **not** per turn like the rows above |
| `hyperhive.agent.subagent_role.lines` | — | gauge | `role` (the file stem) — one series per role file in `subagent_roles/`, on the same ~15-minute tick |
| `hyperhive.agent.claude_usage.percent` | `%` | gauge | `window` (`five_hour`, `seven_day`, … as the usage endpoint names them) — polled every 5 minutes, **not** per turn |

3
hive-agent/src/main.rs
View file

@ -46,6 +46,7 @@ mod swarm_term;
mod term_msg;
mod todo_server;
mod todos;
mod todos_gauge;
mod turn;
mod turn_stats;
mod vacuum;
@ -458,6 +459,8 @@ fn spawn_todo_socket(
tokio::spawn(claude_md_watch::run(store.clone(), todo_wake.clone()));
// Swarm-scheduled prompts fire as todos too, from the list the
// swarm keeps for this agent.
// Open-todo count for the OTEL gauge.
tokio::spawn(todos_gauge::run(store.clone()));
tokio::spawn(schedule_timer::run(store.clone(), todo_wake.clone()));
// And so do the todos the swarm notification router queues for
// this agent.

13
hive-agent/src/otel_turn_metrics.rs
View file

@ -52,6 +52,9 @@ struct Instruments {
session_count: Counter<u64>,
open_threads: Gauge<u64>,
open_reminders: Gauge<u64>,
/// Recorded from [`crate::todos_gauge`]'s own tick: todos change between
/// turns.
open_todos: Gauge<u64>,
/// Recorded from [`crate::claude_md_watch`]'s own tick, not from
/// [`record`] — `AGENTS.md` size is a continuously-live value (like
/// hive-c0re's container gauges), not a once-per-turn one, so it has
@ -133,6 +136,15 @@ pub fn record_subagent_role_lines(role: String, lines: u64) {
.record(lines, &[KeyValue::new("role", role)]);
}
/// Record the number of open todos. No-op when OTEL isn't configured, same
/// as [`record`].
pub fn record_open_todos(count: u64) {
let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else {
return;
};
inst.open_todos.record(count, &[]);
}
/// Record one subscription usage window (`five_hour`, `seven_day`, …):
/// `percent` is 0-100, `resets_at` unix seconds. No-op when OTEL isn't
/// configured, same as [`record`].
@ -195,6 +207,7 @@ fn build() -> Option<Instruments> {
open_reminders: meter
.u64_gauge("hyperhive.agent.loose_ends.reminders")
.build(),
open_todos: meter.u64_gauge("hyperhive.agent.todos.open").build(),
claude_md_lines: meter.u64_gauge("hyperhive.agent.claude_md.lines").build(),
subagent_role_lines: meter
.u64_gauge("hyperhive.agent.subagent_role.lines")

33
hive-agent/src/todos.rs
View file

@ -376,6 +376,24 @@ impl Todos {
Ok(any)
}
/// Number of un-acked todo rows across every subsystem: the length of
/// [`Todos::list`]`(None)`, without materialising the rows.
///
/// # Errors
///
/// Propagates the sqlite query failure.
///
/// # Panics
///
/// Panics if the connection mutex is poisoned.
pub fn open_count(&self) -> Result<u64> {
let conn = self.conn.lock().unwrap();
let n: u64 = conn.query_row("SELECT COUNT(*) FROM todos WHERE acked = 0", [], |row| {
row.get(0)
})?;
Ok(n)
}
/// Reap acked rows whose `acked_at` is older than `cutoff` (a unix
/// timestamp). Called by the harness's hourly vacuum sweep
/// (`vacuum.rs`), not on any hot path — bounds the table against acked
@ -492,6 +510,21 @@ mod tests {
);
}
#[test]
fn open_count_follows_adds_and_acks() {
let (_dir, s) = store();
assert_eq!(s.open_count().unwrap(), 0);
let (a, _) = s.upsert("bash", None, "one", None, false).unwrap();
s.upsert("bash", None, "two", None, false).unwrap();
s.upsert("matrix", Some("!r:x"), "three", None, false)
.unwrap();
assert_eq!(s.open_count().unwrap(), 3);
s.mark_done(a).unwrap();
assert_eq!(s.open_count().unwrap(), 2, "acked rows are not open");
assert_eq!(s.clear("matrix", Some("!r:x")).unwrap(), 1);
assert_eq!(s.open_count().unwrap(), 1);
}
#[test]
fn has_any_reflects_emptiness() {
let (_dir, s) = store();

25
hive-agent/src/todos_gauge.rs Normal file
View file

@ -0,0 +1,25 @@
//! Feeds the count of open (un-acked) todos into an OTEL gauge on a timer.
//! Todos arrive and leave between turns, so the per-turn record would
//! leave the series stale while the agent sleeps.
use std::sync::Arc;
use std::time::Duration;
use crate::todos::Todos;
/// Half the exporter's one-minute interval, so every export carries a
/// value at most one tick old.
const TICK: Duration = Duration::from_secs(30);
/// Background loop: record the open-todo count every [`TICK`]. Detached
/// task — runs for the harness's lifetime; read errors are logged, never
/// fatal.
pub async fn run(todos: Arc<Todos>) {
loop {
match todos.open_count() {
Ok(n) => crate::otel_turn_metrics::record_open_todos(n),
Err(e) => tracing::warn!(error = %e, "open todo count failed"),
}
tokio::time::sleep(TICK).await;
}
}

87
nix/host-modules/swarm-grafana/dashboards/agents.json
View file

@ -923,6 +923,80 @@
}
}
},
"panel-68": {
"kind": "Panel",
"spec": {
"id": 68,
"title": "Open todos by agent",
"description": "Open todos in each agent's todo list over time; reminders excluded.",
"links": [],
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"query": {
"kind": "DataQuery",
"group": "prometheus",
"version": "v0",
"datasource": {
"name": "@datasourceUid@"
},
"spec": {
"expr": "sum by (agent) ({__name__=\"hyperhive.agent.todos.open\", hive=~\"$hive\", agent=~\"$agent\"})",
"legendFormat": "{{agent}}"
}
},
"refId": "A",
"hidden": false
}
}
],
"transformations": [],
"queryOptions": {}
}
},
"vizConfig": {
"kind": "VizConfig",
"group": "timeseries",
"version": "",
"spec": {
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"axisSoftMin": 0,
"drawStyle": "line",
"fillOpacity": 0,
"lineWidth": 1,
"showPoints": "never",
"spanNulls": false
}
},
"overrides": []
}
}
}
}
},
"panel-4": {
"kind": "Panel",
"spec": {
@ -2913,6 +2987,19 @@
"name": "panel-34"
}
}
},
{
"kind": "GridLayoutItem",
"spec": {
"x": 0,
"y": 31,
"width": 24,
"height": 8,
"element": {
"kind": "ElementReference",
"name": "panel-68"
}
}
}
]
}