Watch
0
0
Fork
You've already forked hyperhive
0

hive-agent: keep the last turn's context size in the harness state file and drop the turn sqlite writes

This commit is contained in:
damocles 2026-11-18 02:24:13 +01:00
commit 5e5ff64b3c
13 changed files with 261 additions and 629 deletions

View file

@ -26,7 +26,7 @@ map of the module tree.
`hive-core-agent-sock` wire types.
- **`turn.rs`** — the turn-loop policy layer: renders the system
prompt + MCP config, runs the turn through `hive-runtime`, classifies
the outcome, and feeds the event/turn-stats sinks.
the outcome, and feeds the event sink and the OTEL turn metrics.
- **`login.rs`** — the Online or NeedsLogin state at boot, and the wait
for credentials to arrive (the swarm login stores them).
- **`mcp_config.rs`** — renders the per-turn `--mcp-config` /
@ -36,9 +36,9 @@ map of the module tree.
extra MCP daemons + `hive-agent-mcp` dial into for todo/reminder ops.
- **`vacuum.rs`** — periodic sqlite vacuum sweep for the harness-local
stores.
- **`events.rs` / `turn_stats.rs` / `stats.rs`** — append-only event
sink and per-turn telemetry recording (context usage, cost, tool
favorites) under `harness/`.
- **`events.rs` / `stats.rs`** — the event bus and sink, and the read of
the bash daemon's command heads for the OTEL bash counter under
`harness/`.
- **`prompt.rs`** — system-prompt renderer (persona + tool docs +
environment facts).
- **`web_ui/`** — the per-agent dashboard (terminal pane, status,

View file

@ -183,16 +183,11 @@ pub struct Bus {
/// by the serve loop when it builds the session; unset on claude, whose
/// turn the swarm UI's cancel stops by signalling the `claude` process.
turn_canceller: Arc<OnceLock<hive_runtime::Canceller>>,
/// Current fresh-claude-session id (FK to `sessions.id`). Set by the
/// bin loop after minting a session row on a fresh start; stamped onto
/// every `turn_stats` row until the next fresh session. `None` before
/// the first fresh turn or when the stats db is unavailable.
session_id: Arc<Mutex<Option<i64>>>,
/// One-shot: `run_claude` flips this true when it creates a fresh claude
/// session (`--name <title>` on a title miss). The bin loop takes-and-
/// clears it after the turn to decide whether to mint a new `sessions`
/// row. Session identity itself is handled entirely in `turn.rs` via the
/// constant title + archive — this flag is purely a stats signal.
/// clears it after the turn for the OTEL session-boundary counter. Session
/// identity itself is handled entirely in `turn.rs` via the constant title
/// + archive — this flag is purely a metrics signal.
fresh_session: Arc<AtomicBool>,
/// Per-turn tool-call counter. Reset by the bin loop between
/// turns via `take_tool_calls`. Populated by `observe_stream` as
@ -260,7 +255,6 @@ impl Bus {
compact_pending: Arc::new(Mutex::new(None)),
post_compact_wake: Arc::new(Mutex::new(None)),
turn_canceller: Arc::default(),
session_id: Arc::new(Mutex::new(None)),
fresh_session: Arc::new(AtomicBool::new(false)),
tool_calls: Arc::new(Mutex::new(std::collections::HashMap::new())),
last_turn_ended_unix: Arc::new(AtomicI64::new(0)),
@ -347,32 +341,12 @@ impl Bus {
}
/// Take + clear the fresh-session one-shot. The bin loop calls this
/// after the turn to decide whether to mint a new `sessions` row.
/// after the turn for the OTEL session-boundary counter.
#[must_use]
pub fn take_fresh_session(&self) -> bool {
self.fresh_session.swap(false, Ordering::SeqCst)
}
/// Currently-active session id (FK to `sessions.id`), or `None`.
///
/// # Panics
///
/// Panics if the internal lock is poisoned.
#[must_use]
pub fn current_session_id(&self) -> Option<i64> {
*self.session_id.lock().unwrap()
}
/// Set the active session id after minting a `sessions` row on a
/// fresh start.
///
/// # Panics
///
/// Panics if the internal lock is poisoned.
pub fn set_session_id(&self, id: Option<i64>) {
*self.session_id.lock().unwrap() = id;
}
/// Currently-selected claude model name. Read on every turn so a
/// swarm change takes effect on the next turn.
///
@ -445,8 +419,9 @@ impl Bus {
/// Seed `last_ctx_usage` + `last_cost_usage` at startup without
/// emitting a SSE event. Used by the bin entrypoints to backfill
/// from the most recent `turn_stats` row so the per-agent web UI's
/// ctx + cost badges paint real numbers on cold load.
/// from the context size the last turn kept in the harness state file
/// (`harness_state::read_context_tokens`), so the ctx figure starts from
/// what the last turn saw rather than from nothing.
///
/// # Panics
///
@ -642,7 +617,7 @@ impl Bus {
/// Snapshot + clear the per-turn tool-call counter. The harness
/// calls this between turns to fold the breakdown into a
/// `turn_stats` row, then start the next turn with an empty map.
/// OTEL tool counts, then start the next turn with an empty map.
///
/// # Panics
///

View file

@ -190,6 +190,28 @@ pub(crate) fn write_api_key_mode(enabled: bool) {
write_harness_json(&v);
}
/// Keep `tokens`, the context size the turn that just ended left (what warming
/// the cache costs), in the consolidated state file, so a restarted harness
/// can seed the context gauge and the auto-reset check before its first turn
/// (`Bus::seed_usage`). One value, overwritten each turn.
pub(crate) fn write_context_tokens(tokens: u64) {
let _guard = HARNESS_JSON_LOCK
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let mut v = read_harness_json();
v["context_tokens"] = tokens.into();
write_harness_json(&v);
}
/// The context size [`write_context_tokens`] kept, `None` when none was kept
/// or it is zero.
pub(crate) fn read_context_tokens() -> Option<u64> {
read_harness_json()
.get("context_tokens")
.and_then(serde_json::Value::as_u64)
.filter(|n| *n > 0)
}
/// Compiled-in fallback model used when neither `HIVE_DEFAULT_MODEL` nor a
/// `hyperhive-model` file is present.
pub const DEFAULT_MODEL: &str = "haiku";
@ -227,18 +249,14 @@ pub fn configured_effort() -> Option<&'static str> {
#[cfg(test)]
mod tests {
use super::{
DEFAULT_EFFORT, DEFAULT_MODEL, Source, effort_from, model_from, read_harness_state,
write_harness_state,
DEFAULT_EFFORT, DEFAULT_MODEL, Source, effort_from, model_from, read_context_tokens,
read_harness_state, write_context_tokens, write_harness_state,
};
/// A state file an older harness wrote, with the `active_model` nothing
/// reads any more, neither stops the agent reading its flags nor is lost
/// or turned into an error by the next write: the file is read as a JSON
/// value, not a typed struct, so a leftover field cannot fail to parse.
#[test]
fn a_leftover_active_model_is_ignored_and_survives_a_write() {
// `HYPERHIVE_STATE_DIR` is process-wide; this is the only test that
// sets it, so a lock of its own is enough.
/// Run `f` with `HYPERHIVE_STATE_DIR` pointing at an empty temp dir.
fn in_state_dir(f: impl FnOnce(&std::path::Path)) {
// `HYPERHIVE_STATE_DIR` is process-wide: one lock for every test that
// sets it.
static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
let _lock = LOCK
.lock()
@ -247,19 +265,7 @@ mod tests {
let previous = std::env::var_os("HYPERHIVE_STATE_DIR");
// SAFETY: serialised by `LOCK`, and restored below.
unsafe { std::env::set_var("HYPERHIVE_STATE_DIR", dir.path()) };
let file = dir.path().join("hyperhive-harness.json");
std::fs::write(
&file,
r#"{"rate_limited":true,"needs_login":false,"active_model":"opus"}"#,
)
.unwrap();
let read = read_harness_state();
write_harness_state(false, true);
let after = read_harness_state();
let stored: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(&file).unwrap()).unwrap();
f(dir.path());
// SAFETY: still holding `LOCK`.
unsafe {
match previous {
@ -267,11 +273,54 @@ mod tests {
None => std::env::remove_var("HYPERHIVE_STATE_DIR"),
}
}
assert_eq!(read, (true, false));
assert_eq!(after, (false, true));
assert_eq!(stored["active_model"], "opus", "the leftover is kept");
assert_eq!(stored["rate_limited"], false);
assert_eq!(stored["needs_login"], true);
}
/// A state file an older harness wrote, with the `active_model` nothing
/// reads any more, neither stops the agent reading its flags nor is lost
/// or turned into an error by the next write: the file is read as a JSON
/// value, not a typed struct, so a leftover field cannot fail to parse.
#[test]
fn a_leftover_active_model_is_ignored_and_survives_a_write() {
in_state_dir(|dir| {
let file = dir.join("hyperhive-harness.json");
std::fs::write(
&file,
r#"{"rate_limited":true,"needs_login":false,"active_model":"opus"}"#,
)
.unwrap();
let read = read_harness_state();
write_harness_state(false, true);
let after = read_harness_state();
let stored: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(&file).unwrap()).unwrap();
assert_eq!(read, (true, false));
assert_eq!(after, (false, true));
assert_eq!(stored["active_model"], "opus", "the leftover is kept");
assert_eq!(stored["rate_limited"], false);
assert_eq!(stored["needs_login"], true);
});
}
/// The context size is one field of the same file: nothing kept reads as
/// nothing, the latest turn replaces the one before, a zero is no reading,
/// and the flags beside it survive every write.
#[test]
fn the_context_size_is_one_field_that_the_latest_turn_replaces() {
in_state_dir(|_| {
assert_eq!(read_context_tokens(), None);
write_harness_state(true, false);
write_context_tokens(100_000);
assert_eq!(read_context_tokens(), Some(100_000));
write_context_tokens(180_000);
assert_eq!(read_context_tokens(), Some(180_000));
assert_eq!(read_harness_state(), (true, false));
write_context_tokens(0);
assert_eq!(read_context_tokens(), None);
});
}
#[test]

View file

@ -31,7 +31,6 @@ mod reminders;
mod router_todos;
mod schedule_timer;
mod serve_common;
mod sqlite_writer;
mod state_entry_watch;
mod stats;
mod subagent_role_watch;
@ -52,7 +51,6 @@ mod todos;
mod todos_gauge;
mod transcribe;
mod turn;
mod turn_stats;
mod vacuum;
mod web_ui;
@ -107,7 +105,6 @@ use std::time::Duration;
use crate::events::{Bus, LiveEvent, TurnState};
use crate::login::LoginState;
use crate::turn_stats::TurnStats;
use anyhow::Result;
use clap::Parser;
use hive_core_agent_sock::{Request, Response};
@ -526,12 +523,11 @@ async fn serve_main<S: Surface>(socket: &Path, poll_ms: u64) -> Result<()> {
// `hive_sh4re::inbox::INTERRUPTED_HINT`. Shared between the web server task and
// the serve loop the same way `bus`/`todo_wake` are.
let interrupted = Arc::new(std::sync::atomic::AtomicBool::new(false));
let stats = TurnStats::open_default();
if let Some(s) = &stats {
let (ctx, cost) = s.last_usage();
if ctx.is_some() || cost.is_some() {
bus.seed_usage(ctx, cost);
}
if let Some(tokens) = harness_state::read_context_tokens() {
// Only the total is kept; it is the one figure anything reads.
let mut ctx = hive_claude::TokenUsage::default();
ctx.input_tokens = tokens;
bus.seed_usage(Some(ctx), None);
}
let files = turn::TurnFiles::prepare(socket, &label).await?;
plugins::install_configured().await;
@ -612,7 +608,6 @@ async fn serve_main<S: Surface>(socket: &Path, poll_ms: u64) -> Result<()> {
login_state,
claude_dir,
bus,
stats,
&files,
todo_wake,
todos_store,
@ -729,7 +724,6 @@ async fn serve_loop<S: Surface>(
login_state: Arc<Mutex<LoginState>>,
claude_dir: std::path::PathBuf,
bus: Bus,
stats: Option<TurnStats>,
files: &turn::TurnFiles,
todo_wake: Arc<tokio::sync::Notify>,
todos_store: Option<Arc<todos::Todos>>,
@ -863,7 +857,6 @@ async fn serve_loop<S: Surface>(
let _ = handle_turn::<S>(
socket,
&bus,
stats.as_ref(),
files,
&session,
graceful_stop_message(),
@ -877,7 +870,6 @@ async fn serve_loop<S: Surface>(
let turn_ctx = TurnCtx {
socket,
bus: &bus,
stats: stats.as_ref(),
files,
session: &session,
interrupted: &interrupted,
@ -897,7 +889,6 @@ async fn serve_loop<S: Surface>(
struct TurnCtx<'a> {
socket: &'a Path,
bus: &'a Bus,
stats: Option<&'a TurnStats>,
files: &'a turn::TurnFiles,
session: &'a turn::AgentSession,
interrupted: &'a Arc<std::sync::atomic::AtomicBool>,
@ -923,7 +914,6 @@ async fn drive_turn_and_wake_chain<S: Surface>(
let ctrl = handle_turn::<S>(
ctx.socket,
ctx.bus,
ctx.stats,
ctx.files,
ctx.session,
next,
@ -1005,12 +995,11 @@ fn apply_todo_wake_checked(checked: Option<bool>, todo_miss_streak: &mut u32, bu
/// Drive a single turn: emit boot-of-turn events, run claude, ack on
/// success / requeue on rate-limit-or-401 / log `turn failed` on a hard
/// failure (the line the `hh-agent-turn-failed` Grafana rule alerts on),
/// record stats. Returns a `TurnControl` carrying the auth-failed flag and
/// keep the context size. Returns a `TurnControl` carrying the auth-failed flag and
/// the todo-wake-checked signal — the serve loop decides what to do next.
async fn handle_turn<S: Surface>(
socket: &Path,
bus: &Bus,
stats: Option<&TurnStats>,
files: &turn::TurnFiles,
session: &turn::AgentSession,
first: hive_sh4re::inbox::DeliveredMessage,
@ -1045,37 +1034,20 @@ async fn handle_turn<S: Surface>(
handle_turn_error_recovery::<S>(&outcome, bus, socket).await;
// Single read of the per-turn tool-call counter: `take_tool_calls`
// resets it, so this is the only chance to see which tools this turn
// invoked. Used both for the stats row below (if stats are configured)
// and for the todo-miss-streak signal returned to the serve loop —
// deliberately unconditional (not gated on `stats.is_some()`) so the
// counter is reset every turn regardless, and so a stats-less harness
// (tests, or a future config without a stats sink) still gets the
// miss-streak signal.
// invoked. Used both for the OTEL tool counts below and for the
// todo-miss-streak signal returned to the serve loop.
let tool_calls = bus.take_tool_calls();
let todo_wake_checked =
(from == "todo").then(|| tool_calls.contains_key("mcp__hyperhive__get_loose_ends"));
// Read-and-clear once per turn unconditionally (same rationale as
// `take_tool_calls` above — the harness gets exactly one chance to
// observe this flag) rather than only when the sqlite sink happens to be
// configured: the OTEL turn-metrics exporter's session-boundary counter
// needs it too, independent of `stats`.
// Read-and-clear once per turn (same rationale as `take_tool_calls`
// above — the harness gets exactly one chance to observe this flag): the
// OTEL turn-metrics exporter's session-boundary counter needs it.
let fresh_session = bus.take_fresh_session();
if let Some(stats) = stats {
// Fresh session this turn → mint a `sessions` row and set its id on
// the bus so this turn (and subsequent ones until the next fresh
// start) stamp `turn_stats.session_id`.
if fresh_session {
let sid = stats.start_session(started_at, &model_at_start);
bus.set_session_id(sid);
}
}
let ended_at = chrono::Utc::now().timestamp();
let duration_ms = i64::try_from(started_instant.elapsed().as_millis()).unwrap_or(i64::MAX);
let (open_threads, open_reminders) = S::post_turn_counts(socket).await;
let context_window = bus.api_context_window();
let row = serve_common::build_row(serve_common::TurnRowArgs {
started_at,
ended_at,
duration_ms,
model: model_at_start,
wake_from: from.clone(),
@ -1086,8 +1058,8 @@ async fn handle_turn<S: Surface>(
open_reminders_count: open_reminders,
});
// Harness-only OTEL metrics (duration/wake_from/result_kind/loose-ends/
// session boundaries/context fill) — independent of the sqlite sink
// below, and a cheap no-op when OTEL isn't configured. See
// session boundaries/context fill), a cheap no-op when OTEL isn't
// configured. See
// `otel_turn_metrics`'s module doc for why token/tool-count are
// deliberately not here, and why cost is only ever an ACP agent's.
otel_turn_metrics::record(&row, fresh_session);
@ -1097,8 +1069,8 @@ async fn handle_turn<S: Surface>(
for cost in hive_runtime::Runtime::take_reported_cost(session) {
otel_turn_metrics::record_reported_cost(&row.model, &cost);
}
if let Some(stats) = stats {
stats.record(&row);
if let Some(ctx) = bus.last_ctx_usage() {
harness_state::write_context_tokens(ctx.context_tokens());
}
let pending = S::inbox_unread(socket).await;
if pending > 0 {

View file

@ -39,7 +39,7 @@ use opentelemetry::metrics::{Counter, Gauge, Histogram, MeterProvider as _};
use opentelemetry_otlp::{MetricExporter, Protocol, WithExportConfig};
use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider};
use crate::turn_stats::TurnStatRow;
use crate::serve_common::TurnStatRow;
/// Export cadence. Matches `hive-c0re`'s container-metrics exporter so the
/// two batch on the same rhythm.
@ -123,7 +123,7 @@ static INSTRUMENTS: OnceLock<Option<Instruments>> = OnceLock::new();
/// `fresh_session` is the harness's own "did this turn start a brand-new
/// claude session" flag (`Bus::take_fresh_session`) — pass it through rather
/// than re-deriving it here, since the harness only gets to observe it once
/// per turn (the same flag gates minting a `turn_stats` `sessions` row).
/// per turn.
pub fn record(row: &TurnStatRow, fresh_session: bool) {
let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else {
return;

View file

@ -42,7 +42,7 @@ pub fn harness_dir() -> PathBuf {
/// Consolidated harness-local state db — todos + reminders, one table
/// each — mutable per-agent state the harness owns, kept apart
/// from the append-only turn-stats db. Each store's `open()` only
/// from the usage db. Each store's `open()` only
/// applies its own `CREATE TABLE IF NOT EXISTS` against this same path
/// (see their call sites), so distinct table names keep them from
/// colliding without needing a per-store path wrapper here.

View file

@ -6,7 +6,6 @@
use crate::events::Bus;
use crate::turn::{TurnError, TurnOutcome};
use crate::turn_stats::TurnStatRow;
/// Assemble the per-turn wake prompt string. The role/tools/etc. live in the
/// system prompt; this is just the wake signal body. `redelivered` prepends a "may already be handled" banner; `interrupted`
@ -26,15 +25,33 @@ pub fn format_wake_prompt(from: &str, body: &str, redelivered: bool, interrupted
format!("{banner}Incoming message from `{from}`:\n---\n{body}\n---")
}
/// Field-named args for [`build_row`]. Mirrors the turn-stats row
/// columns; `outcome` and `bus` borrow for the duration of the call.
/// `tool_calls` is passed in (rather than pulled from `bus` internally)
/// because the caller needs the same map to check for a `get_loose_ends`
/// call before it's consumed — `Bus::take_tool_calls` resets the counter,
/// so there's only one chance to read it per turn.
/// What one finished turn leaves for the OTEL turn metrics
/// (`otel_turn_metrics::record`).
#[derive(Debug, Clone)]
pub struct TurnStatRow {
pub started_at: i64,
pub duration_ms: i64,
pub model: String,
pub wake_from: String,
/// Prompt tokens billed at the full rate, summed over the turn's inferences.
pub input_tokens: u64,
/// Per-tool breakdown as JSON: `{"Read":12,"Bash":3,...}`; `None` when no
/// tool was called.
pub tool_call_breakdown_json: Option<String>,
/// Post-turn loose-ends snapshot; `None` when the request failed.
pub open_threads_count: Option<u64>,
pub open_reminders_count: Option<u64>,
/// `"ok" | "compacted" | "failed" | ...`, see [`build_row`].
pub result_kind: &'static str,
}
/// Field-named args for [`build_row`]; `outcome` and `bus` borrow for the
/// duration of the call. `tool_calls` is passed in (rather than pulled from
/// `bus` internally) because the caller needs the same map to check for a
/// `get_loose_ends` call before it's consumed — `Bus::take_tool_calls` resets
/// the counter, so there's only one chance to read it per turn.
pub struct TurnRowArgs<'a> {
pub started_at: i64,
pub ended_at: i64,
pub duration_ms: i64,
pub model: String,
pub wake_from: String,
@ -52,7 +69,6 @@ pub struct TurnRowArgs<'a> {
pub fn build_row(args: TurnRowArgs<'_>) -> TurnStatRow {
let TurnRowArgs {
started_at,
ended_at,
duration_ms,
model,
wake_from,
@ -64,50 +80,36 @@ pub fn build_row(args: TurnRowArgs<'_>) -> TurnStatRow {
} = args;
// Prefer the API-resolved model id (e.g. `claude-opus-4-8`) captured
// from this turn's assistant events over the requested `--model`
// name/alias, so the model-mix + cost rollup label the concrete
// version that ran. Falls back to the requested name on a degenerate
// turn that produced no assistant event.
// name/alias, so the model-mix rollup labels the concrete version that
// ran. Falls back to the requested name on a degenerate turn that
// produced no assistant event.
let model = bus.last_resolved_model().unwrap_or(model);
let cost = bus.last_cost_usage().unwrap_or_default();
let ctx = bus.last_ctx_usage().unwrap_or(cost);
let tool_call_count: u64 = tool_calls.values().copied().sum();
let tool_call_breakdown_json = if tool_calls.is_empty() {
None
} else {
serde_json::to_string(&tool_calls).ok()
};
let (result_kind, note) = match outcome {
Ok(false) => ("ok", None),
Ok(true) => ("compacted", None),
Err(TurnError::PromptTooLong) => ("prompt_too_long", None),
Err(TurnError::RateLimited) => ("rate_limited", None),
Err(TurnError::AuthFailed) => ("auth_failed", None),
Err(TurnError::SessionNotFound) => ("session_not_found", None),
Err(TurnError::ApiStall) => ("api_stall", None),
Err(TurnError::AgentStall(note)) => ("api_stall", Some(note.clone())),
Err(TurnError::Failed(e)) => ("failed", Some(format!("{e:#}"))),
let result_kind = match outcome {
Ok(false) => "ok",
Ok(true) => "compacted",
Err(TurnError::PromptTooLong) => "prompt_too_long",
Err(TurnError::RateLimited) => "rate_limited",
Err(TurnError::AuthFailed) => "auth_failed",
Err(TurnError::SessionNotFound) => "session_not_found",
Err(TurnError::ApiStall | TurnError::AgentStall(_)) => "api_stall",
Err(TurnError::Failed(_)) => "failed",
};
TurnStatRow {
started_at,
ended_at,
duration_ms,
model,
wake_from,
input_tokens: cost.input_tokens,
output_tokens: cost.output_tokens,
cache_read_input_tokens: cost.cache_read_input_tokens,
cache_creation_input_tokens: cost.cache_creation_input_tokens,
last_input_tokens: ctx.input_tokens,
last_output_tokens: ctx.output_tokens,
last_cache_read_input_tokens: ctx.cache_read_input_tokens,
last_cache_creation_input_tokens: ctx.cache_creation_input_tokens,
tool_call_count,
tool_call_breakdown_json,
open_threads_count,
open_reminders_count,
result_kind,
note,
session_id: bus.current_session_id(),
}
}
@ -135,3 +137,89 @@ mod wake_prompt_tests {
assert!(plain.starts_with("Incoming message from `a`:"));
}
}
#[cfg(test)]
mod build_row_tests {
use super::{TurnRowArgs, build_row};
use crate::events::Bus;
use crate::turn::TurnError;
fn row_for(
outcome: &crate::turn::TurnOutcome,
tool_calls: &[(&str, u64)],
) -> super::TurnStatRow {
build_row(TurnRowArgs {
started_at: 10,
duration_ms: 2_000,
model: "opus".to_owned(),
wake_from: "todo".to_owned(),
outcome,
bus: &Bus::new(),
tool_calls: tool_calls
.iter()
.map(|(k, v)| ((*k).to_owned(), *v))
.collect(),
open_threads_count: Some(2),
open_reminders_count: None,
})
}
#[test]
fn the_outcome_names_the_result_kind() {
let _dir = crate::paths::harness_dir_guard();
for (outcome, kind) in [
(Ok(false), "ok"),
(Ok(true), "compacted"),
(Err(TurnError::PromptTooLong), "prompt_too_long"),
(Err(TurnError::RateLimited), "rate_limited"),
(Err(TurnError::AuthFailed), "auth_failed"),
(Err(TurnError::SessionNotFound), "session_not_found"),
(Err(TurnError::ApiStall), "api_stall"),
(Err(TurnError::AgentStall("quiet".into())), "api_stall"),
(Err(TurnError::Failed(anyhow::anyhow!("boom"))), "failed"),
] {
assert_eq!(row_for(&outcome, &[]).result_kind, kind);
}
}
#[test]
fn the_row_carries_the_turns_identity_counts_and_tool_breakdown() {
let _dir = crate::paths::harness_dir_guard();
let row = row_for(&Ok(false), &[("Bash", 3)]);
assert_eq!((row.started_at, row.duration_ms), (10, 2_000));
assert_eq!(
(row.model.as_str(), row.wake_from.as_str()),
("opus", "todo")
);
assert_eq!(
(row.open_threads_count, row.open_reminders_count),
(Some(2), None)
);
assert_eq!(
row.tool_call_breakdown_json.as_deref(),
Some(r#"{"Bash":3}"#)
);
assert_eq!(row_for(&Ok(false), &[]).tool_call_breakdown_json, None);
}
/// The model the API resolved wins over the requested alias.
#[test]
fn the_resolved_model_replaces_the_requested_one() {
let _dir = crate::paths::harness_dir_guard();
let bus = Bus::new();
bus.set_resolved_model(Some("claude-opus-4-8".to_owned()));
let row = build_row(TurnRowArgs {
started_at: 0,
duration_ms: 0,
model: "opus".to_owned(),
wake_from: "todo".to_owned(),
outcome: &Ok(false),
bus: &bus,
tool_calls: std::collections::HashMap::new(),
open_threads_count: None,
open_reminders_count: None,
});
assert_eq!(row.model, "claude-opus-4-8");
}
}

View file

@ -1,10 +0,0 @@
//! Shared connection setup for the harness's own sqlite writers.
//!
//! A write that overlaps another connection's lock (the hourly vacuum, a
//! reader) fails at once with `SQLITE_BUSY` unless the connection has a busy
//! timeout, so writers set [`BUSY_TIMEOUT`] to wait instead of dropping the row.
use std::time::Duration;
/// How long a writer waits on a locked database before giving up.
pub const BUSY_TIMEOUT: Duration = Duration::from_secs(5);

View file

@ -1,4 +1,4 @@
//! The shell-command heads the bash daemon records in the turn-stats database,
//! The shell-command heads the bash daemon records in the harness's `hyperhive-turn-stats.sqlite`,
//! read for the OTEL bash-command counter.
use std::path::PathBuf;

View file

@ -381,8 +381,7 @@ mod tests {
use tokio::time::{Duration, Instant};
/// `TokenUsage` is `#[non_exhaustive]`, so a downstream crate cannot
/// write one as a struct expression at all — see `turn_stats`'s note on
/// the same restriction. Deserializing one is the shortest way to a
/// write one as a struct expression at all. Deserializing one is the shortest way to a
/// populated fixture here, and it keeps the numbers next to the field
/// names they belong to.
///

View file

@ -56,7 +56,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_todos_dedup
ON todos (subsystem, subsystem_key);
";
/// Additive column migrations — see `turn_stats.rs`'s identical pattern.
/// Additive column migrations.
/// Each runs unconditionally and ignores `duplicate column name` errors, so
/// re-running on an already-migrated db is a silent no-op.
const MIGRATIONS: &[&str] = &[

View file

@ -477,8 +477,8 @@ pub async fn drive_turn(
// (badges, stats, auto-reset watermark input).
apply_telemetry(bus, &progress.telemetry);
if progress.created {
// Fresh session minted this turn → flag it so the bin loop
// mints a `sessions` row + stamps its id onto this turn's stats.
// Fresh session minted this turn → flag it so the bin loop counts
// the session boundary in the OTEL metrics.
bus.mark_fresh_session();
bus.emit(LiveEvent::Note {
text: format!("created fresh session titled \"{}\"", session_title()),

View file

@ -1,432 +0,0 @@
//! Per-turn analytics sink. One sqlite row per claude turn captures:
//! identity (`model`, `wake_from`, `result_kind`), timing (`started_at`,
//! `ended_at`, `duration_ms`), cost (token counts), and behaviour (tool-call
//! count + per-tool breakdown).
//!
//! **Captured but not yet read** (written every turn, no reader in this
//! repository — kept for a future chart / backfill): `tool_call_count` (a
//! reader recomputes tool totals from `tool_call_breakdown_json` instead),
//! `open_threads_count` +
//! `open_reminders_count` (planned: a loose-ends-over-time trend), and
//! `note` (failure detail for `result_kind = "failed"`).
//!
//! Lives in the harness dir (`HYPERHIVE_HARNESS_DIR`) as
//! `hyperhive-turn-stats.sqlite`; the standalone legacy stats viewer reads it
//! and no vacuum sweeps it. Schema is
//! intentionally append-only — every column has a default so future
//! additions don't break old readers; new columns land via
//! `ALTER TABLE ... ADD COLUMN ... DEFAULT ...` in the migration
//! block.
//!
//! Writes are best-effort: a failed insert logs a warning and lets
//! the turn loop continue. The next turn either succeeds or the
//! operator sees the journal trail.
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use anyhow::{Context, Result};
use rusqlite::{Connection, params};
/// SQL bootstrap. CREATE TABLE IF NOT EXISTS so first-boot agents
/// and existing ones converge on the same shape. The base table is
/// fresh-install only; additive migrations land via `MIGRATIONS`
/// below as try-and-ignore ALTERs so existing dbs catch up.
const SCHEMA: &str = "
CREATE TABLE IF NOT EXISTS turn_stats (
id INTEGER PRIMARY KEY AUTOINCREMENT,
started_at INTEGER NOT NULL,
ended_at INTEGER NOT NULL,
duration_ms INTEGER NOT NULL,
model TEXT NOT NULL,
wake_from TEXT NOT NULL,
input_tokens INTEGER NOT NULL DEFAULT 0,
output_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_input_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_input_tokens INTEGER NOT NULL DEFAULT 0,
last_input_tokens INTEGER NOT NULL DEFAULT 0,
last_output_tokens INTEGER NOT NULL DEFAULT 0,
last_cache_read_input_tokens INTEGER NOT NULL DEFAULT 0,
last_cache_creation_input_tokens INTEGER NOT NULL DEFAULT 0,
tool_call_count INTEGER NOT NULL DEFAULT 0,
tool_call_breakdown_json TEXT,
open_threads_count INTEGER,
open_reminders_count INTEGER,
result_kind TEXT NOT NULL,
note TEXT,
session_id INTEGER
);
CREATE INDEX IF NOT EXISTS idx_turn_stats_started
ON turn_stats (started_at DESC);
-- NOTE: the index on session_id is created in MIGRATIONS, not here. On an
-- existing pre-session db the `CREATE TABLE IF NOT EXISTS` above is a no-op
-- (the old table has no session_id column), so indexing session_id in this
-- batch would fail with `no such column` and abort the whole SCHEMA apply —
-- which disables the stats sink. The column is added by MIGRATIONS first.
-- One row per fresh claude session (minted when a session is created rather
-- than resumed by title).
-- turn_stats.session_id FKs here so per-session stats (first-turn tokens,
-- per-session totals, turn count, duration) are one GROUP BY away.
CREATE TABLE IF NOT EXISTS sessions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
started_at INTEGER NOT NULL,
model TEXT NOT NULL
);
";
/// Additive column migrations. Each runs unconditionally and ignores
/// `duplicate column name` errors — sqlite < 3.35 lacks
/// `ADD COLUMN IF NOT EXISTS`, so try-and-ignore is the portable path.
/// New columns MUST carry a default so existing rows decode.
const MIGRATIONS: &[&str] = &[
"ALTER TABLE turn_stats ADD COLUMN last_input_tokens INTEGER NOT NULL DEFAULT 0",
"ALTER TABLE turn_stats ADD COLUMN last_output_tokens INTEGER NOT NULL DEFAULT 0",
"ALTER TABLE turn_stats ADD COLUMN last_cache_read_input_tokens INTEGER NOT NULL DEFAULT 0",
"ALTER TABLE turn_stats ADD COLUMN last_cache_creation_input_tokens INTEGER NOT NULL DEFAULT 0",
// Nullable FK to sessions.id — no default; pre-migration rows stay NULL
// (the surface treats NULL as "no session", inert until capture lands).
"ALTER TABLE turn_stats ADD COLUMN session_id INTEGER",
// Index on session_id — must run AFTER the column is added, so it lives
// here rather than in SCHEMA (see the note there). Idempotent.
"CREATE INDEX IF NOT EXISTS idx_turn_stats_session ON turn_stats (session_id)",
];
/// One row to be inserted. `Option`-wrapped fields default to NULL
/// when the harness couldn't gather them (e.g. socket roundtrip for
/// `open_threads` failed) so a partial row beats no row.
#[derive(Debug, Clone)]
pub struct TurnStatRow {
pub started_at: i64,
pub ended_at: i64,
pub duration_ms: i64,
pub model: String,
pub wake_from: String,
/// Cumulative across every inference in the turn (cost signal).
pub input_tokens: u64,
pub output_tokens: u64,
pub cache_read_input_tokens: u64,
pub cache_creation_input_tokens: u64,
/// Last inference's usage — the actual context size at turn end.
pub last_input_tokens: u64,
pub last_output_tokens: u64,
pub last_cache_read_input_tokens: u64,
pub last_cache_creation_input_tokens: u64,
/// Captured, not yet read — the snapshot recomputes tool totals from
/// `tool_call_breakdown_json` (see the module doc).
pub tool_call_count: u64,
/// Per-tool breakdown as JSON: `{"Read":12,"Bash":3,...}`. None
/// when no tools were called (saves a sqlite write of `"{}"`).
pub tool_call_breakdown_json: Option<String>,
/// Post-turn loose-ends snapshot. Captured, not yet read — planned to
/// feed a loose-ends-over-time trend.
pub open_threads_count: Option<u64>,
pub open_reminders_count: Option<u64>,
/// `"ok" | "failed" | "prompt_too_long"`.
pub result_kind: &'static str,
/// Failure detail for `result_kind = "failed"`. Captured, not yet read.
pub note: Option<String>,
/// FK to `sessions.id` for the fresh claude session this turn belongs
/// to. `None` on pre-capture rows (and when the stats db couldn't mint
/// a session) so the read side degrades to empty.
pub session_id: Option<i64>,
}
/// Thin sqlite wrapper. Cloning is cheap (Arc-shared connection).
#[derive(Clone)]
pub struct TurnStats {
inner: std::sync::Arc<Mutex<Connection>>,
}
impl TurnStats {
/// Open the per-agent stats db, creating the file + schema if
/// missing. Returns `None` when the db can't be opened (read-only
/// fs in tests, missing state dir) — the harness logs and
/// continues without a sink rather than failing the turn loop.
#[must_use]
pub fn open_default() -> Option<Self> {
let path = default_path();
match Self::open(&path) {
Ok(s) => Some(s),
Err(e) => {
tracing::warn!(
error = ?e,
path = %path.display(),
"turn_stats: open failed; per-turn analytics disabled"
);
None
}
}
}
fn open(path: &Path) -> Result<Self> {
if let Some(parent) = path.parent() {
let _ = std::fs::create_dir_all(parent);
}
let conn = Connection::open(path)
.with_context(|| format!("open turn_stats db {}", path.display()))?;
// Rollback journal, not WAL: the default, and what every existing
// database already uses.
conn.busy_timeout(crate::sqlite_writer::BUSY_TIMEOUT)
.context("set turn_stats busy_timeout")?;
conn.execute_batch(SCHEMA)
.context("apply turn_stats schema")?;
for stmt in MIGRATIONS {
// Ignore "duplicate column name" — the migration already ran.
// Any other error is logged but doesn't fail open() because the
// base schema works and we'd rather keep the harness alive than
// crash on an upgrade hiccup.
if let Err(e) = conn.execute(stmt, []) {
let msg = e.to_string();
if !msg.contains("duplicate column name") {
tracing::warn!(error = %msg, stmt, "turn_stats migration failed");
}
}
}
Ok(Self {
inner: std::sync::Arc::new(Mutex::new(conn)),
})
}
/// Insert a row. Best-effort — logs + swallows errors so a sqlite
/// hiccup (locked db, full disk) doesn't crash the harness.
///
/// # Panics
///
/// Panics if the internal lock is poisoned.
pub fn record(&self, row: &TurnStatRow) {
let conn = self.inner.lock().unwrap();
let res = conn.execute(
"INSERT INTO turn_stats (
started_at, ended_at, duration_ms, model, wake_from,
input_tokens, output_tokens,
cache_read_input_tokens, cache_creation_input_tokens,
last_input_tokens, last_output_tokens,
last_cache_read_input_tokens, last_cache_creation_input_tokens,
tool_call_count, tool_call_breakdown_json,
open_threads_count, open_reminders_count,
result_kind, note, session_id
) VALUES (
?1, ?2, ?3, ?4, ?5,
?6, ?7,
?8, ?9,
?10, ?11,
?12, ?13,
?14, ?15,
?16, ?17,
?18, ?19, ?20
)",
params![
row.started_at,
row.ended_at,
row.duration_ms,
row.model,
row.wake_from,
i64::try_from(row.input_tokens).unwrap_or(i64::MAX),
i64::try_from(row.output_tokens).unwrap_or(i64::MAX),
i64::try_from(row.cache_read_input_tokens).unwrap_or(i64::MAX),
i64::try_from(row.cache_creation_input_tokens).unwrap_or(i64::MAX),
i64::try_from(row.last_input_tokens).unwrap_or(i64::MAX),
i64::try_from(row.last_output_tokens).unwrap_or(i64::MAX),
i64::try_from(row.last_cache_read_input_tokens).unwrap_or(i64::MAX),
i64::try_from(row.last_cache_creation_input_tokens).unwrap_or(i64::MAX),
i64::try_from(row.tool_call_count).unwrap_or(i64::MAX),
row.tool_call_breakdown_json,
row.open_threads_count
.map(|n| i64::try_from(n).unwrap_or(i64::MAX)),
row.open_reminders_count
.map(|n| i64::try_from(n).unwrap_or(i64::MAX)),
row.result_kind,
row.note,
row.session_id,
],
);
if let Err(e) = res {
tracing::warn!(error = ?e, "turn_stats: insert failed");
}
}
/// Mint a new session row at fresh-session start, returning its `id`
/// for stamping onto this session's `turn_stats` rows. Best-effort —
/// returns `None` (and logs) on any sqlite error, so a hiccup degrades
/// to NULL `session_id` rows rather than crashing the turn loop.
///
/// # Panics
///
/// Panics if the internal lock is poisoned.
#[must_use]
pub fn start_session(&self, started_at: i64, model: &str) -> Option<i64> {
let conn = self.inner.lock().unwrap();
match conn.execute(
"INSERT INTO sessions (started_at, model) VALUES (?1, ?2)",
params![started_at, model],
) {
Ok(_) => Some(conn.last_insert_rowid()),
Err(e) => {
tracing::warn!(error = ?e, "turn_stats: start_session insert failed");
None
}
}
}
/// Token counts from the most recently inserted row, if any.
/// Returns `(ctx, cost)` — both backfill `Bus` on startup so the
/// per-agent web UI's ctx + cost badges paint with real numbers on
/// cold load instead of waiting for the next `TokenUsageChanged`
/// SSE event. Best-effort: any sqlite error returns `(None, None)`.
///
/// Pre-migration rows (before the `last_*_tokens` columns existed)
/// have last-inference zeros — those rows yield `ctx = None` so the
/// badge stays empty until the next real turn rather than showing a
/// misleading 0.
/// # Panics
///
/// Panics if the internal lock is poisoned.
#[must_use]
pub fn last_usage(
&self,
) -> (
Option<hive_claude::TokenUsage>,
Option<hive_claude::TokenUsage>,
) {
let conn = self.inner.lock().unwrap();
conn.query_row(
"SELECT input_tokens, output_tokens,
cache_read_input_tokens, cache_creation_input_tokens,
last_input_tokens, last_output_tokens,
last_cache_read_input_tokens, last_cache_creation_input_tokens
FROM turn_stats
-- `id` (AUTOINCREMENT) is monotonic with insertion, so this is
-- the most-recently-inserted row even among same-second turns
-- (which `started_at DESC` would order arbitrarily).
ORDER BY id DESC
LIMIT 1",
[],
|row| {
let g = |i: usize| -> rusqlite::Result<u64> {
Ok(u64::try_from(row.get::<_, i64>(i)?).unwrap_or(0))
};
// `TokenUsage` is `#[non_exhaustive]` (hive-claude 0.1.0+) —
// struct-expression construction is blocked for downstream
// crates entirely, even with a `..base` (functional update
// doesn't carve out an exception, per the non_exhaustive
// reference semantics). Build off `Default` and assign the
// (all-`pub`) fields instead.
let mut cost = hive_claude::TokenUsage::default();
cost.input_tokens = g(0)?;
cost.output_tokens = g(1)?;
cost.cache_read_input_tokens = g(2)?;
cost.cache_creation_input_tokens = g(3)?;
let mut last = hive_claude::TokenUsage::default();
last.input_tokens = g(4)?;
last.output_tokens = g(5)?;
last.cache_read_input_tokens = g(6)?;
last.cache_creation_input_tokens = g(7)?;
let ctx = if last == hive_claude::TokenUsage::default() {
None
} else {
Some(last)
};
Ok((ctx, Some(cost)))
},
)
.unwrap_or((None, None))
}
}
fn default_path() -> PathBuf {
crate::paths::harness_dir().join("hyperhive-turn-stats.sqlite")
}
#[cfg(test)]
mod tests {
use super::*;
/// A `turn_stats` db in the *pre-session* shape: the original table with no
/// `session_id` column, plus the `started_at` index. This is what every
/// agent created before the sessions feature has on disk.
fn seed_pre_session_db(path: &Path) {
let conn = Connection::open(path).unwrap();
conn.execute_batch(
"CREATE TABLE turn_stats (
id INTEGER PRIMARY KEY AUTOINCREMENT,
started_at INTEGER NOT NULL,
ended_at INTEGER NOT NULL,
duration_ms INTEGER NOT NULL,
model TEXT NOT NULL,
wake_from TEXT NOT NULL,
input_tokens INTEGER NOT NULL DEFAULT 0,
output_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_input_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_input_tokens INTEGER NOT NULL DEFAULT 0,
last_input_tokens INTEGER NOT NULL DEFAULT 0,
last_output_tokens INTEGER NOT NULL DEFAULT 0,
last_cache_read_input_tokens INTEGER NOT NULL DEFAULT 0,
last_cache_creation_input_tokens INTEGER NOT NULL DEFAULT 0,
tool_call_count INTEGER NOT NULL DEFAULT 0,
tool_call_breakdown_json TEXT,
open_threads_count INTEGER,
open_reminders_count INTEGER,
result_kind TEXT NOT NULL,
note TEXT
);
CREATE INDEX idx_turn_stats_started ON turn_stats (started_at DESC);",
)
.unwrap();
}
fn sample_row() -> TurnStatRow {
TurnStatRow {
started_at: 100,
ended_at: 101,
duration_ms: 1_000,
model: "opus".to_owned(),
wake_from: "recv".to_owned(),
input_tokens: 10,
output_tokens: 5,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
last_input_tokens: 10,
last_output_tokens: 5,
last_cache_read_input_tokens: 0,
last_cache_creation_input_tokens: 0,
tool_call_count: 1,
tool_call_breakdown_json: None,
open_threads_count: None,
open_reminders_count: None,
result_kind: "ok",
note: None,
session_id: None,
}
}
/// Regression: a pre-session db must `open()` cleanly (the `session_id`
/// index used to live in `SCHEMA` and aborted the apply with
/// `no such column`, silently disabling the stats sink), get the column
/// added by `MIGRATIONS`, and then accept writes.
#[test]
fn open_upgrades_pre_session_db_and_writes() {
let path = std::env::temp_dir().join("hyperhive-turnstats-pre-session-regression.sqlite");
let _ = std::fs::remove_file(&path);
seed_pre_session_db(&path);
let stats = TurnStats::open(&path).expect("open() must succeed on a pre-session db");
stats.record(&sample_row());
let conn = Connection::open(&path).unwrap();
let rows: i64 = conn
.query_row("SELECT COUNT(*) FROM turn_stats", [], |r| r.get(0))
.unwrap();
assert_eq!(rows, 1, "the row must be written once the column is added");
let has_session_id: i64 = conn
.query_row(
"SELECT COUNT(*) FROM pragma_table_info('turn_stats') \
WHERE name = 'session_id'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(has_session_id, 1, "MIGRATIONS must add session_id");
let _ = std::fs::remove_file(&path);
}
}