export an open-todos gauge and chart it on the agent dashboard
This commit is contained in:
parent
c273527219
commit
08b8eb3f73
6 changed files with 162 additions and 0 deletions
|
|
@ -415,6 +415,7 @@ of their own — see [ACP-reported cost](#acp-reported-cost).
|
|||
| `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) |
|
||||
| `hyperhive.agent.loose_ends.threads` | — | gauge | none |
|
||||
| `hyperhive.agent.loose_ends.reminders` | — | gauge | none |
|
||||
| `hyperhive.agent.todos.open` | — | gauge | none — un-acked todos in the harness-local todo store (reminders excluded), recorded every 30 seconds |
|
||||
| `hyperhive.agent.claude_md.lines` | — | gauge | none — recorded from the `AGENTS.md`-size watch's own ~15-minute tick, **not** per turn like the rows above |
|
||||
| `hyperhive.agent.subagent_role.lines` | — | gauge | `role` (the file stem) — one series per role file in `subagent_roles/`, on the same ~15-minute tick |
|
||||
| `hyperhive.agent.claude_usage.percent` | `%` | gauge | `window` (`five_hour`, `seven_day`, … as the usage endpoint names them) — polled every 5 minutes, **not** per turn |
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ mod swarm_term;
|
|||
mod term_msg;
|
||||
mod todo_server;
|
||||
mod todos;
|
||||
mod todos_gauge;
|
||||
mod turn;
|
||||
mod turn_stats;
|
||||
mod vacuum;
|
||||
|
|
@ -458,6 +459,8 @@ fn spawn_todo_socket(
|
|||
tokio::spawn(claude_md_watch::run(store.clone(), todo_wake.clone()));
|
||||
// Swarm-scheduled prompts fire as todos too, from the list the
|
||||
// swarm keeps for this agent.
|
||||
// Open-todo count for the OTEL gauge.
|
||||
tokio::spawn(todos_gauge::run(store.clone()));
|
||||
tokio::spawn(schedule_timer::run(store.clone(), todo_wake.clone()));
|
||||
// And so do the todos the swarm notification router queues for
|
||||
// this agent.
|
||||
|
|
|
|||
|
|
@ -52,6 +52,9 @@ struct Instruments {
|
|||
session_count: Counter<u64>,
|
||||
open_threads: Gauge<u64>,
|
||||
open_reminders: Gauge<u64>,
|
||||
/// Recorded from [`crate::todos_gauge`]'s own tick: todos change between
|
||||
/// turns.
|
||||
open_todos: Gauge<u64>,
|
||||
/// Recorded from [`crate::claude_md_watch`]'s own tick, not from
|
||||
/// [`record`] — `AGENTS.md` size is a continuously-live value (like
|
||||
/// hive-c0re's container gauges), not a once-per-turn one, so it has
|
||||
|
|
@ -133,6 +136,15 @@ pub fn record_subagent_role_lines(role: String, lines: u64) {
|
|||
.record(lines, &[KeyValue::new("role", role)]);
|
||||
}
|
||||
|
||||
/// Record the number of open todos. No-op when OTEL isn't configured, same
|
||||
/// as [`record`].
|
||||
pub fn record_open_todos(count: u64) {
|
||||
let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else {
|
||||
return;
|
||||
};
|
||||
inst.open_todos.record(count, &[]);
|
||||
}
|
||||
|
||||
/// Record one subscription usage window (`five_hour`, `seven_day`, …):
|
||||
/// `percent` is 0-100, `resets_at` unix seconds. No-op when OTEL isn't
|
||||
/// configured, same as [`record`].
|
||||
|
|
@ -195,6 +207,7 @@ fn build() -> Option<Instruments> {
|
|||
open_reminders: meter
|
||||
.u64_gauge("hyperhive.agent.loose_ends.reminders")
|
||||
.build(),
|
||||
open_todos: meter.u64_gauge("hyperhive.agent.todos.open").build(),
|
||||
claude_md_lines: meter.u64_gauge("hyperhive.agent.claude_md.lines").build(),
|
||||
subagent_role_lines: meter
|
||||
.u64_gauge("hyperhive.agent.subagent_role.lines")
|
||||
|
|
|
|||
|
|
@ -376,6 +376,24 @@ impl Todos {
|
|||
Ok(any)
|
||||
}
|
||||
|
||||
/// Number of un-acked todo rows across every subsystem: the length of
|
||||
/// [`Todos::list`]`(None)`, without materialising the rows.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Propagates the sqlite query failure.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if the connection mutex is poisoned.
|
||||
pub fn open_count(&self) -> Result<u64> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let n: u64 = conn.query_row("SELECT COUNT(*) FROM todos WHERE acked = 0", [], |row| {
|
||||
row.get(0)
|
||||
})?;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Reap acked rows whose `acked_at` is older than `cutoff` (a unix
|
||||
/// timestamp). Called by the harness's hourly vacuum sweep
|
||||
/// (`vacuum.rs`), not on any hot path — bounds the table against acked
|
||||
|
|
@ -492,6 +510,21 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_count_follows_adds_and_acks() {
|
||||
let (_dir, s) = store();
|
||||
assert_eq!(s.open_count().unwrap(), 0);
|
||||
let (a, _) = s.upsert("bash", None, "one", None, false).unwrap();
|
||||
s.upsert("bash", None, "two", None, false).unwrap();
|
||||
s.upsert("matrix", Some("!r:x"), "three", None, false)
|
||||
.unwrap();
|
||||
assert_eq!(s.open_count().unwrap(), 3);
|
||||
s.mark_done(a).unwrap();
|
||||
assert_eq!(s.open_count().unwrap(), 2, "acked rows are not open");
|
||||
assert_eq!(s.clear("matrix", Some("!r:x")).unwrap(), 1);
|
||||
assert_eq!(s.open_count().unwrap(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_any_reflects_emptiness() {
|
||||
let (_dir, s) = store();
|
||||
|
|
|
|||
25
hive-agent/src/todos_gauge.rs
Normal file
25
hive-agent/src/todos_gauge.rs
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
//! Feeds the count of open (un-acked) todos into an OTEL gauge on a timer.
|
||||
//! Todos arrive and leave between turns, so the per-turn record would
|
||||
//! leave the series stale while the agent sleeps.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::todos::Todos;
|
||||
|
||||
/// Half the exporter's one-minute interval, so every export carries a
|
||||
/// value at most one tick old.
|
||||
const TICK: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Background loop: record the open-todo count every [`TICK`]. Detached
|
||||
/// task — runs for the harness's lifetime; read errors are logged, never
|
||||
/// fatal.
|
||||
pub async fn run(todos: Arc<Todos>) {
|
||||
loop {
|
||||
match todos.open_count() {
|
||||
Ok(n) => crate::otel_turn_metrics::record_open_todos(n),
|
||||
Err(e) => tracing::warn!(error = %e, "open todo count failed"),
|
||||
}
|
||||
tokio::time::sleep(TICK).await;
|
||||
}
|
||||
}
|
||||
|
|
@ -923,6 +923,80 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"panel-68": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"id": 68,
|
||||
"title": "Open todos by agent",
|
||||
"description": "Open todos in each agent's todo list over time; reminders excluded.",
|
||||
"links": [],
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"query": {
|
||||
"kind": "DataQuery",
|
||||
"group": "prometheus",
|
||||
"version": "v0",
|
||||
"datasource": {
|
||||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "sum by (agent) ({__name__=\"hyperhive.agent.todos.open\", hive=~\"$hive\", agent=~\"$agent\"})",
|
||||
"legendFormat": "{{agent}}"
|
||||
}
|
||||
},
|
||||
"refId": "A",
|
||||
"hidden": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"transformations": [],
|
||||
"queryOptions": {}
|
||||
}
|
||||
},
|
||||
"vizConfig": {
|
||||
"kind": "VizConfig",
|
||||
"group": "timeseries",
|
||||
"version": "",
|
||||
"spec": {
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisSoftMin": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"lineWidth": 1,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-4": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
|
|
@ -2913,6 +2987,19 @@
|
|||
"name": "panel-34"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"x": 0,
|
||||
"y": 31,
|
||||
"width": 24,
|
||||
"height": 8,
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-68"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue