export an open-todos gauge and chart it on the agent dashboard
This commit is contained in:
parent
c273527219
commit
08b8eb3f73
4 changed files with 162 additions and 0 deletions
|
|
@ -46,6 +46,7 @@ mod swarm_term;
|
|||
mod term_msg;
|
||||
mod todo_server;
|
||||
mod todos;
|
||||
mod todos_gauge;
|
||||
mod turn;
|
||||
mod turn_stats;
|
||||
mod vacuum;
|
||||
|
|
@ -458,6 +459,8 @@ fn spawn_todo_socket(
|
|||
tokio::spawn(claude_md_watch::run(store.clone(), todo_wake.clone()));
|
||||
// Swarm-scheduled prompts fire as todos too, from the list the
|
||||
// swarm keeps for this agent.
|
||||
// Open-todo count for the OTEL gauge.
|
||||
tokio::spawn(todos_gauge::run(store.clone()));
|
||||
tokio::spawn(schedule_timer::run(store.clone(), todo_wake.clone()));
|
||||
// And so do the todos the swarm notification router queues for
|
||||
// this agent.
|
||||
|
|
|
|||
|
|
@ -52,6 +52,9 @@ struct Instruments {
|
|||
session_count: Counter<u64>,
|
||||
open_threads: Gauge<u64>,
|
||||
open_reminders: Gauge<u64>,
|
||||
/// Recorded from [`crate::todos_gauge`]'s own tick: todos change between
|
||||
/// turns.
|
||||
open_todos: Gauge<u64>,
|
||||
/// Recorded from [`crate::claude_md_watch`]'s own tick, not from
|
||||
/// [`record`] — `AGENTS.md` size is a continuously-live value (like
|
||||
/// hive-c0re's container gauges), not a once-per-turn one, so it has
|
||||
|
|
@ -133,6 +136,15 @@ pub fn record_subagent_role_lines(role: String, lines: u64) {
|
|||
.record(lines, &[KeyValue::new("role", role)]);
|
||||
}
|
||||
|
||||
/// Record the number of open todos. No-op when OTEL isn't configured, same
|
||||
/// as [`record`].
|
||||
pub fn record_open_todos(count: u64) {
|
||||
let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else {
|
||||
return;
|
||||
};
|
||||
inst.open_todos.record(count, &[]);
|
||||
}
|
||||
|
||||
/// Record one subscription usage window (`five_hour`, `seven_day`, …):
|
||||
/// `percent` is 0-100, `resets_at` unix seconds. No-op when OTEL isn't
|
||||
/// configured, same as [`record`].
|
||||
|
|
@ -195,6 +207,7 @@ fn build() -> Option<Instruments> {
|
|||
open_reminders: meter
|
||||
.u64_gauge("hyperhive.agent.loose_ends.reminders")
|
||||
.build(),
|
||||
open_todos: meter.u64_gauge("hyperhive.agent.todos.open").build(),
|
||||
claude_md_lines: meter.u64_gauge("hyperhive.agent.claude_md.lines").build(),
|
||||
subagent_role_lines: meter
|
||||
.u64_gauge("hyperhive.agent.subagent_role.lines")
|
||||
|
|
|
|||
|
|
@ -376,6 +376,24 @@ impl Todos {
|
|||
Ok(any)
|
||||
}
|
||||
|
||||
/// Number of un-acked todo rows across every subsystem: the length of
|
||||
/// [`Todos::list`]`(None)`, without materialising the rows.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Propagates the sqlite query failure.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if the connection mutex is poisoned.
|
||||
pub fn open_count(&self) -> Result<u64> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let n: u64 = conn.query_row("SELECT COUNT(*) FROM todos WHERE acked = 0", [], |row| {
|
||||
row.get(0)
|
||||
})?;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Reap acked rows whose `acked_at` is older than `cutoff` (a unix
|
||||
/// timestamp). Called by the harness's hourly vacuum sweep
|
||||
/// (`vacuum.rs`), not on any hot path — bounds the table against acked
|
||||
|
|
@ -492,6 +510,21 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_count_follows_adds_and_acks() {
|
||||
let (_dir, s) = store();
|
||||
assert_eq!(s.open_count().unwrap(), 0);
|
||||
let (a, _) = s.upsert("bash", None, "one", None, false).unwrap();
|
||||
s.upsert("bash", None, "two", None, false).unwrap();
|
||||
s.upsert("matrix", Some("!r:x"), "three", None, false)
|
||||
.unwrap();
|
||||
assert_eq!(s.open_count().unwrap(), 3);
|
||||
s.mark_done(a).unwrap();
|
||||
assert_eq!(s.open_count().unwrap(), 2, "acked rows are not open");
|
||||
assert_eq!(s.clear("matrix", Some("!r:x")).unwrap(), 1);
|
||||
assert_eq!(s.open_count().unwrap(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_any_reflects_emptiness() {
|
||||
let (_dir, s) = store();
|
||||
|
|
|
|||
25
hive-agent/src/todos_gauge.rs
Normal file
25
hive-agent/src/todos_gauge.rs
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
//! Feeds the count of open (un-acked) todos into an OTEL gauge on a timer.
|
||||
//! Todos arrive and leave between turns, so the per-turn record would
|
||||
//! leave the series stale while the agent sleeps.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::todos::Todos;
|
||||
|
||||
/// Half the exporter's one-minute interval, so every export carries a
|
||||
/// value at most one tick old.
|
||||
const TICK: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Background loop: record the open-todo count every [`TICK`]. Detached
|
||||
/// task — runs for the harness's lifetime; read errors are logged, never
|
||||
/// fatal.
|
||||
pub async fn run(todos: Arc<Todos>) {
|
||||
loop {
|
||||
match todos.open_count() {
|
||||
Ok(n) => crate::otel_turn_metrics::record_open_todos(n),
|
||||
Err(e) => tracing::warn!(error = %e, "open todo count failed"),
|
||||
}
|
||||
tokio::time::sleep(TICK).await;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue