harness metrics for tool calls, bash commands, reminders and first-turn context, on the agents board
The per-agent /stats page read these from its sqlite; the swarm board had none of them. hive-agent now counts the tools a turn called, the commands bash tasks ran (from the bash daemon's own table, newer rows only), reminders scheduled and delivered, and records the first turn of a fresh session; the turns tab shows them, with fresh sessions per agent. Claude's own export has no tool-call counter, so the docs no longer say it does.
This commit is contained in:
parent
5f9629dfc7
commit
23c2527199
1 changed files with 618 additions and 28 deletions
|
|
@ -1,13 +1,13 @@
|
|||
//! Per-turn OTEL metric export for the fields Claude Code's own built-in
|
||||
//! OTEL integration cannot know about. Claude's own export already covers
|
||||
//! token usage, cost, and tool-call counts (see `docs/scheduler/observability.md`) —
|
||||
//! token usage and cost (see `docs/scheduler/observability.md`) —
|
||||
//! duplicating those here under different metric names would just give a
|
||||
//! collector two disagreeing series for the same number. This module emits
|
||||
//! only the harness-only concepts: wall-clock turn duration as *this harness*
|
||||
//! measures it (not Claude's own per-request latency), what woke the turn,
|
||||
//! the harness's own outcome classification, the loose-ends backlog at turn
|
||||
//! end, session boundaries (fresh vs. resumed by title), context-window fill,
|
||||
//! and the cost an ACP agent reports — claude's runtime reports none.
|
||||
//! measures it, what woke the turn, its outcome classification, the loose-ends
|
||||
//! backlog, sessions and their first turn's cost, context-window fill, the
|
||||
//! tools a turn called (claude exports no tool-call counter), bash commands,
|
||||
//! reminders, and the cost an ACP agent reports — claude's runtime reports none.
|
||||
//!
|
||||
//! Synchronous instruments, not observable ones: unlike hive-c0re's
|
||||
//! container-resource gauges (`hive-c0re/src/stats/otel_metrics.rs`), which
|
||||
|
|
@ -29,7 +29,9 @@
|
|||
//! up automatically by the SDK from the container-wide `OTEL_RESOURCE_ATTRIBUTES`
|
||||
//! env var (same mechanism `hive-metric` relies on) — nothing to set here.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::atomic::{AtomicI64, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use opentelemetry::KeyValue;
|
||||
|
|
@ -92,6 +94,16 @@ struct Instruments {
|
|||
/// Keyed by `model` and `currency`: an ACP agent reports cost in a
|
||||
/// currency of its choosing.
|
||||
reported_cost: Counter<f64>,
|
||||
/// Tool calls made, keyed by `tool` (`Skill:<skill>` for a skill).
|
||||
tool_calls: Counter<u64>,
|
||||
/// Shell commands run by bash tasks, keyed by `command`: the normalised
|
||||
/// head the bash daemon records, not the full command line.
|
||||
bash_commands: Counter<u64>,
|
||||
/// Reminders scheduled and delivered, keyed by `event`.
|
||||
reminders: Counter<u64>,
|
||||
/// Input tokens of the first turn of a fresh claude session: what the
|
||||
/// system prompt and `AGENTS.md` cost before anything is cached.
|
||||
first_turn_context: Gauge<u64>,
|
||||
}
|
||||
|
||||
/// Lazily built on the first call to [`record`]. `None` when OTEL isn't
|
||||
|
|
@ -130,7 +142,48 @@ pub fn record(row: &TurnStatRow, fresh_session: bool) {
|
|||
if fresh_session {
|
||||
inst.session_count
|
||||
.add(1, &[KeyValue::new("model", row.model.clone())]);
|
||||
inst.first_turn_context.record(row.input_tokens, &[]);
|
||||
}
|
||||
for (tool, calls) in tool_call_counts(row.tool_call_breakdown_json.as_deref()) {
|
||||
inst.tool_calls.add(calls, &[KeyValue::new("tool", tool)]);
|
||||
}
|
||||
record_new_bash_commands(inst, row.started_at);
|
||||
}
|
||||
|
||||
/// The per-tool call counts a turn stored in `tool_call_breakdown_json`; a
|
||||
/// turn with no tool calls, or a value that does not parse, has none.
|
||||
fn tool_call_counts(json: Option<&str>) -> Vec<(String, u64)> {
|
||||
json.and_then(|j| serde_json::from_str::<HashMap<String, u64>>(j).ok())
|
||||
.map(|m| m.into_iter().collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Unix seconds up to which bash commands are already counted; `0` before the
|
||||
/// first turn of this run.
|
||||
static BASH_COUNTED_UNTIL: AtomicI64 = AtomicI64::new(0);
|
||||
|
||||
/// Count the bash commands recorded since the last call. The first call after
|
||||
/// the harness starts counts only commands newer than `turn_started_at`, so a
|
||||
/// restart does not count the whole history again.
|
||||
fn record_new_bash_commands(inst: &Instruments, turn_started_at: i64) {
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let mut since = BASH_COUNTED_UNTIL.swap(now, Ordering::Relaxed);
|
||||
if since == 0 {
|
||||
since = turn_started_at.saturating_sub(1);
|
||||
}
|
||||
for (command, n) in crate::stats::bash_heads_since(since) {
|
||||
inst.bash_commands
|
||||
.add(n, &[KeyValue::new("command", command)]);
|
||||
}
|
||||
}
|
||||
|
||||
/// Count one reminder event, `scheduled` or `delivered`. No-op when OTEL isn't
|
||||
/// configured, same as [`record`].
|
||||
pub fn record_reminder(event: &'static str) {
|
||||
let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else {
|
||||
return;
|
||||
};
|
||||
inst.reminders.add(1, &[KeyValue::new("event", event)]);
|
||||
}
|
||||
|
||||
/// Record the current `AGENTS.md` line count. Called from
|
||||
|
|
@ -250,6 +303,13 @@ fn build() -> Option<Instruments> {
|
|||
.with_unit("{token}")
|
||||
.build(),
|
||||
reported_cost: meter.f64_counter("hyperhive.agent.cost.usage").build(),
|
||||
tool_calls: meter.u64_counter("hyperhive.agent.tool.calls").build(),
|
||||
bash_commands: meter.u64_counter("hyperhive.agent.bash.commands").build(),
|
||||
reminders: meter.u64_counter("hyperhive.agent.reminders").build(),
|
||||
first_turn_context: meter
|
||||
.u64_gauge("hyperhive.agent.first_turn.context")
|
||||
.with_unit("{token}")
|
||||
.build(),
|
||||
_provider: provider,
|
||||
})
|
||||
}
|
||||
|
|
@ -291,7 +351,23 @@ pub fn enabled() -> bool {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::TURN_DURATION_BOUNDS_MS;
|
||||
use super::{TURN_DURATION_BOUNDS_MS, tool_call_counts};
|
||||
|
||||
#[test]
|
||||
fn a_turns_stored_tool_breakdown_becomes_per_tool_counts() {
|
||||
let mut got = tool_call_counts(Some(r#"{"Bash":3,"Skill:base:pr-review":1}"#));
|
||||
got.sort();
|
||||
assert_eq!(
|
||||
got,
|
||||
vec![
|
||||
("Bash".to_owned(), 3),
|
||||
("Skill:base:pr-review".to_owned(), 1)
|
||||
]
|
||||
);
|
||||
// No tool calls, and a value that is not a count map, both count nothing.
|
||||
assert!(tool_call_counts(None).is_empty());
|
||||
assert!(tool_call_counts(Some("not json")).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_duration_bounds_ascend_and_reach_past_the_old_ten_second_ceiling() {
|
||||
|
|
|
|||
Loading…
Reference in a new issue