export an open-todos gauge and chart it on the agent dashboard
This commit is contained in:
parent
c273527219
commit
08b8eb3f73
6 changed files with 162 additions and 0 deletions
|
|
@ -415,6 +415,7 @@ of their own — see [ACP-reported cost](#acp-reported-cost).
|
||||||
| `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) |
|
| `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) |
|
||||||
| `hyperhive.agent.loose_ends.threads` | — | gauge | none |
|
| `hyperhive.agent.loose_ends.threads` | — | gauge | none |
|
||||||
| `hyperhive.agent.loose_ends.reminders` | — | gauge | none |
|
| `hyperhive.agent.loose_ends.reminders` | — | gauge | none |
|
||||||
|
| `hyperhive.agent.todos.open` | — | gauge | none — un-acked todos in the harness-local todo store (reminders excluded), recorded every 30 seconds |
|
||||||
| `hyperhive.agent.claude_md.lines` | — | gauge | none — recorded from the `AGENTS.md`-size watch's own ~15-minute tick, **not** per turn like the rows above |
|
| `hyperhive.agent.claude_md.lines` | — | gauge | none — recorded from the `AGENTS.md`-size watch's own ~15-minute tick, **not** per turn like the rows above |
|
||||||
| `hyperhive.agent.subagent_role.lines` | — | gauge | `role` (the file stem) — one series per role file in `subagent_roles/`, on the same ~15-minute tick |
|
| `hyperhive.agent.subagent_role.lines` | — | gauge | `role` (the file stem) — one series per role file in `subagent_roles/`, on the same ~15-minute tick |
|
||||||
| `hyperhive.agent.claude_usage.percent` | `%` | gauge | `window` (`five_hour`, `seven_day`, … as the usage endpoint names them) — polled every 5 minutes, **not** per turn |
|
| `hyperhive.agent.claude_usage.percent` | `%` | gauge | `window` (`five_hour`, `seven_day`, … as the usage endpoint names them) — polled every 5 minutes, **not** per turn |
|
||||||
|
|
|
||||||
|
|
@ -46,6 +46,7 @@ mod swarm_term;
|
||||||
mod term_msg;
|
mod term_msg;
|
||||||
mod todo_server;
|
mod todo_server;
|
||||||
mod todos;
|
mod todos;
|
||||||
|
mod todos_gauge;
|
||||||
mod turn;
|
mod turn;
|
||||||
mod turn_stats;
|
mod turn_stats;
|
||||||
mod vacuum;
|
mod vacuum;
|
||||||
|
|
@ -458,6 +459,8 @@ fn spawn_todo_socket(
|
||||||
tokio::spawn(claude_md_watch::run(store.clone(), todo_wake.clone()));
|
tokio::spawn(claude_md_watch::run(store.clone(), todo_wake.clone()));
|
||||||
// Swarm-scheduled prompts fire as todos too, from the list the
|
// Swarm-scheduled prompts fire as todos too, from the list the
|
||||||
// swarm keeps for this agent.
|
// swarm keeps for this agent.
|
||||||
|
// Open-todo count for the OTEL gauge.
|
||||||
|
tokio::spawn(todos_gauge::run(store.clone()));
|
||||||
tokio::spawn(schedule_timer::run(store.clone(), todo_wake.clone()));
|
tokio::spawn(schedule_timer::run(store.clone(), todo_wake.clone()));
|
||||||
// And so do the todos the swarm notification router queues for
|
// And so do the todos the swarm notification router queues for
|
||||||
// this agent.
|
// this agent.
|
||||||
|
|
|
||||||
|
|
@ -52,6 +52,9 @@ struct Instruments {
|
||||||
session_count: Counter<u64>,
|
session_count: Counter<u64>,
|
||||||
open_threads: Gauge<u64>,
|
open_threads: Gauge<u64>,
|
||||||
open_reminders: Gauge<u64>,
|
open_reminders: Gauge<u64>,
|
||||||
|
/// Recorded from [`crate::todos_gauge`]'s own tick: todos change between
|
||||||
|
/// turns.
|
||||||
|
open_todos: Gauge<u64>,
|
||||||
/// Recorded from [`crate::claude_md_watch`]'s own tick, not from
|
/// Recorded from [`crate::claude_md_watch`]'s own tick, not from
|
||||||
/// [`record`] — `AGENTS.md` size is a continuously-live value (like
|
/// [`record`] — `AGENTS.md` size is a continuously-live value (like
|
||||||
/// hive-c0re's container gauges), not a once-per-turn one, so it has
|
/// hive-c0re's container gauges), not a once-per-turn one, so it has
|
||||||
|
|
@ -133,6 +136,15 @@ pub fn record_subagent_role_lines(role: String, lines: u64) {
|
||||||
.record(lines, &[KeyValue::new("role", role)]);
|
.record(lines, &[KeyValue::new("role", role)]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Record the number of open todos. No-op when OTEL isn't configured, same
|
||||||
|
/// as [`record`].
|
||||||
|
pub fn record_open_todos(count: u64) {
|
||||||
|
let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
inst.open_todos.record(count, &[]);
|
||||||
|
}
|
||||||
|
|
||||||
/// Record one subscription usage window (`five_hour`, `seven_day`, …):
|
/// Record one subscription usage window (`five_hour`, `seven_day`, …):
|
||||||
/// `percent` is 0-100, `resets_at` unix seconds. No-op when OTEL isn't
|
/// `percent` is 0-100, `resets_at` unix seconds. No-op when OTEL isn't
|
||||||
/// configured, same as [`record`].
|
/// configured, same as [`record`].
|
||||||
|
|
@ -195,6 +207,7 @@ fn build() -> Option<Instruments> {
|
||||||
open_reminders: meter
|
open_reminders: meter
|
||||||
.u64_gauge("hyperhive.agent.loose_ends.reminders")
|
.u64_gauge("hyperhive.agent.loose_ends.reminders")
|
||||||
.build(),
|
.build(),
|
||||||
|
open_todos: meter.u64_gauge("hyperhive.agent.todos.open").build(),
|
||||||
claude_md_lines: meter.u64_gauge("hyperhive.agent.claude_md.lines").build(),
|
claude_md_lines: meter.u64_gauge("hyperhive.agent.claude_md.lines").build(),
|
||||||
subagent_role_lines: meter
|
subagent_role_lines: meter
|
||||||
.u64_gauge("hyperhive.agent.subagent_role.lines")
|
.u64_gauge("hyperhive.agent.subagent_role.lines")
|
||||||
|
|
|
||||||
|
|
@ -376,6 +376,24 @@ impl Todos {
|
||||||
Ok(any)
|
Ok(any)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Number of un-acked todo rows across every subsystem: the length of
|
||||||
|
/// [`Todos::list`]`(None)`, without materialising the rows.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
///
|
||||||
|
/// Propagates the sqlite query failure.
|
||||||
|
///
|
||||||
|
/// # Panics
|
||||||
|
///
|
||||||
|
/// Panics if the connection mutex is poisoned.
|
||||||
|
pub fn open_count(&self) -> Result<u64> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let n: u64 = conn.query_row("SELECT COUNT(*) FROM todos WHERE acked = 0", [], |row| {
|
||||||
|
row.get(0)
|
||||||
|
})?;
|
||||||
|
Ok(n)
|
||||||
|
}
|
||||||
|
|
||||||
/// Reap acked rows whose `acked_at` is older than `cutoff` (a unix
|
/// Reap acked rows whose `acked_at` is older than `cutoff` (a unix
|
||||||
/// timestamp). Called by the harness's hourly vacuum sweep
|
/// timestamp). Called by the harness's hourly vacuum sweep
|
||||||
/// (`vacuum.rs`), not on any hot path — bounds the table against acked
|
/// (`vacuum.rs`), not on any hot path — bounds the table against acked
|
||||||
|
|
@ -492,6 +510,21 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn open_count_follows_adds_and_acks() {
|
||||||
|
let (_dir, s) = store();
|
||||||
|
assert_eq!(s.open_count().unwrap(), 0);
|
||||||
|
let (a, _) = s.upsert("bash", None, "one", None, false).unwrap();
|
||||||
|
s.upsert("bash", None, "two", None, false).unwrap();
|
||||||
|
s.upsert("matrix", Some("!r:x"), "three", None, false)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(s.open_count().unwrap(), 3);
|
||||||
|
s.mark_done(a).unwrap();
|
||||||
|
assert_eq!(s.open_count().unwrap(), 2, "acked rows are not open");
|
||||||
|
assert_eq!(s.clear("matrix", Some("!r:x")).unwrap(), 1);
|
||||||
|
assert_eq!(s.open_count().unwrap(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn has_any_reflects_emptiness() {
|
fn has_any_reflects_emptiness() {
|
||||||
let (_dir, s) = store();
|
let (_dir, s) = store();
|
||||||
|
|
|
||||||
25
hive-agent/src/todos_gauge.rs
Normal file
25
hive-agent/src/todos_gauge.rs
Normal file
|
|
@ -0,0 +1,25 @@
|
||||||
|
//! Feeds the count of open (un-acked) todos into an OTEL gauge on a timer.
|
||||||
|
//! Todos arrive and leave between turns, so the per-turn record would
|
||||||
|
//! leave the series stale while the agent sleeps.
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use crate::todos::Todos;
|
||||||
|
|
||||||
|
/// Half the exporter's one-minute interval, so every export carries a
|
||||||
|
/// value at most one tick old.
|
||||||
|
const TICK: Duration = Duration::from_secs(30);
|
||||||
|
|
||||||
|
/// Background loop: record the open-todo count every [`TICK`]. Detached
|
||||||
|
/// task — runs for the harness's lifetime; read errors are logged, never
|
||||||
|
/// fatal.
|
||||||
|
pub async fn run(todos: Arc<Todos>) {
|
||||||
|
loop {
|
||||||
|
match todos.open_count() {
|
||||||
|
Ok(n) => crate::otel_turn_metrics::record_open_todos(n),
|
||||||
|
Err(e) => tracing::warn!(error = %e, "open todo count failed"),
|
||||||
|
}
|
||||||
|
tokio::time::sleep(TICK).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -923,6 +923,80 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"panel-68": {
|
||||||
|
"kind": "Panel",
|
||||||
|
"spec": {
|
||||||
|
"id": 68,
|
||||||
|
"title": "Open todos by agent",
|
||||||
|
"description": "Open todos in each agent's todo list over time; reminders excluded.",
|
||||||
|
"links": [],
|
||||||
|
"data": {
|
||||||
|
"kind": "QueryGroup",
|
||||||
|
"spec": {
|
||||||
|
"queries": [
|
||||||
|
{
|
||||||
|
"kind": "PanelQuery",
|
||||||
|
"spec": {
|
||||||
|
"query": {
|
||||||
|
"kind": "DataQuery",
|
||||||
|
"group": "prometheus",
|
||||||
|
"version": "v0",
|
||||||
|
"datasource": {
|
||||||
|
"name": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"spec": {
|
||||||
|
"expr": "sum by (agent) ({__name__=\"hyperhive.agent.todos.open\", hive=~\"$hive\", agent=~\"$agent\"})",
|
||||||
|
"legendFormat": "{{agent}}"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"refId": "A",
|
||||||
|
"hidden": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"transformations": [],
|
||||||
|
"queryOptions": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"vizConfig": {
|
||||||
|
"kind": "VizConfig",
|
||||||
|
"group": "timeseries",
|
||||||
|
"version": "",
|
||||||
|
"spec": {
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"calcs": [],
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom",
|
||||||
|
"showLegend": true
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "desc"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": {
|
||||||
|
"mode": "palette-classic"
|
||||||
|
},
|
||||||
|
"custom": {
|
||||||
|
"axisSoftMin": 0,
|
||||||
|
"drawStyle": "line",
|
||||||
|
"fillOpacity": 0,
|
||||||
|
"lineWidth": 1,
|
||||||
|
"showPoints": "never",
|
||||||
|
"spanNulls": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"panel-4": {
|
"panel-4": {
|
||||||
"kind": "Panel",
|
"kind": "Panel",
|
||||||
"spec": {
|
"spec": {
|
||||||
|
|
@ -2913,6 +2987,19 @@
|
||||||
"name": "panel-34"
|
"name": "panel-34"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"kind": "GridLayoutItem",
|
||||||
|
"spec": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 31,
|
||||||
|
"width": 24,
|
||||||
|
"height": 8,
|
||||||
|
"element": {
|
||||||
|
"kind": "ElementReference",
|
||||||
|
"name": "panel-68"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue