From 4e0f2ac9ca07b21e7884193fb23df349dbf6b000 Mon Sep 17 00:00:00 2001 From: damocles Date: Mon, 8 Jun 2026 20:00:24 +0200 Subject: [PATCH] feat(stats): record api-resolved model id in turn-stats --- hive-ag3nt/src/events.rs | 84 ++++++++++++++++++++++++++++++++++ hive-ag3nt/src/serve_common.rs | 6 +++ hive-ag3nt/src/turn.rs | 12 +++++ 3 files changed, 102 insertions(+) diff --git a/hive-ag3nt/src/events.rs b/hive-ag3nt/src/events.rs index db2afb70..7bd9a955 100644 --- a/hive-ag3nt/src/events.rs +++ b/hive-ag3nt/src/events.rs @@ -388,6 +388,26 @@ impl TokenUsage { } None } + + /// Extract the *resolved* model id from an `assistant` stream-json + /// event (`message.model`). Unlike the requested `--model` name + /// (which may be a short alias like `opus` or a default), the API + /// echoes the concrete version it actually ran on (e.g. + /// `claude-opus-4-8`). Recording the resolved id (not the requested + /// name) is what lets the ST4TS model-mix + cost rollup label the + /// exact version that ran. Returns `None` for non-assistant events + /// or ones missing `message.model`. + #[must_use] + pub fn model_from_assistant_event(v: &serde_json::Value) -> Option { + if v.get("type").and_then(|t| t.as_str()) != Some("assistant") { + return None; + } + v.get("message") + .and_then(|m| m.get("model")) + .and_then(serde_json::Value::as_str) + .filter(|s| !s.is_empty()) + .map(ToOwned::to_owned) + } } /// Authoritative turn-loop state. The harness owns it; the web UI @@ -529,6 +549,14 @@ pub struct Bus { /// limit the model enforces, which may differ from what the operator /// configured (e.g. 200 k active window on a 1 M cache-enabled model). api_context_window: Arc>>, + /// Resolved model id from the most recent turn's `assistant` events + /// (the API-echoed `message.model`, e.g. `claude-opus-4-8`), as + /// opposed to the requested `--model` name which may be a short + /// alias. Set once per turn by the stdout pump at result-time; + /// `None` until the first assistant event of a turn is seen. The + /// per-turn stats sink prefers this over the requested name so the + /// model-mix + cost rollup reflect the concrete version that ran. + last_resolved_model: Arc>>, } impl Bus { @@ -570,6 +598,7 @@ impl Bus { tool_calls: Arc::new(Mutex::new(std::collections::HashMap::new())), last_turn_ended_unix: Arc::new(AtomicI64::new(0)), api_context_window: Arc::new(Mutex::new(None)), + last_resolved_model: Arc::new(Mutex::new(None)), } } @@ -671,6 +700,30 @@ impl Bus { self.last_turn_ended_unix.load(Ordering::Relaxed) } + /// Record the resolved model id observed for the just-ended turn + /// (from `assistant` events' `message.model`). `None` clears it so a + /// degenerate turn that produced no assistant event doesn't inherit a + /// stale id — the stats sink then falls back to the requested name. + /// + /// # Panics + /// + /// Panics if the internal lock is poisoned. + pub fn set_resolved_model(&self, model: Option) { + *self.last_resolved_model.lock().unwrap() = model; + } + + /// Resolved model id from the most recent turn, if an `assistant` + /// event reported one. The per-turn stats sink prefers this over the + /// requested `--model` name. + /// + /// # Panics + /// + /// Panics if the internal lock is poisoned. + #[must_use] + pub fn last_resolved_model(&self) -> Option { + self.last_resolved_model.lock().unwrap().clone() + } + /// Update the API-reported context-window size from the stream-json /// `result` event's `modelUsage.*.contextWindow` field. Called by the /// stdout pump once per completed turn. `0` is ignored (sentinel for @@ -896,3 +949,34 @@ impl Default for Bus { Self::new() } } + +#[cfg(test)] +mod tests { + use super::TokenUsage; + use serde_json::json; + + #[test] + fn resolved_model_from_assistant_event() { + let v = json!({ + "type": "assistant", + "message": { "model": "claude-opus-4-8", "role": "assistant" } + }); + assert_eq!( + TokenUsage::model_from_assistant_event(&v), + Some("claude-opus-4-8".to_owned()) + ); + } + + #[test] + fn resolved_model_ignores_non_assistant_and_missing() { + // Wrong event type. + let result = json!({ "type": "result", "message": { "model": "claude-opus-4-8" } }); + assert_eq!(TokenUsage::model_from_assistant_event(&result), None); + // Assistant event missing message.model. + let no_model = json!({ "type": "assistant", "message": { "role": "assistant" } }); + assert_eq!(TokenUsage::model_from_assistant_event(&no_model), None); + // Empty model string is treated as absent. + let empty = json!({ "type": "assistant", "message": { "model": "" } }); + assert_eq!(TokenUsage::model_from_assistant_event(&empty), None); + } +} diff --git a/hive-ag3nt/src/serve_common.rs b/hive-ag3nt/src/serve_common.rs index 67bd7820..d44adb3f 100644 --- a/hive-ag3nt/src/serve_common.rs +++ b/hive-ag3nt/src/serve_common.rs @@ -52,6 +52,12 @@ pub fn build_row( open_threads_count: Option, open_reminders_count: Option, ) -> TurnStatRow { + // Prefer the API-resolved model id (e.g. `claude-opus-4-8`) captured + // from this turn's assistant events over the requested `--model` + // name/alias, so the model-mix + cost rollup label the concrete + // version that ran. Falls back to the requested name on a degenerate + // turn that produced no assistant event. + let model = bus.last_resolved_model().unwrap_or(model); let cost = bus.last_cost_usage().unwrap_or_default(); let ctx = bus.last_ctx_usage().unwrap_or(cost); let tool_calls = bus.take_tool_calls(); diff --git a/hive-ag3nt/src/turn.rs b/hive-ag3nt/src/turn.rs index d418e990..ca3129a0 100644 --- a/hive-ag3nt/src/turn.rs +++ b/hive-ag3nt/src/turn.rs @@ -712,6 +712,11 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool, // get handed to `record_turn_usage` together so a single SSE // event updates both badges. let mut last_inference: Option = None; + // Resolved model id (API-echoed `message.model`) from this turn's + // assistant events; recorded onto the bus at result-time so the + // per-turn stats label the concrete version that ran, not the + // requested `--model` alias. + let mut last_model: Option = None; while let Ok(Some(line)) = reader.next_line().await { if line.contains(PROMPT_TOO_LONG_MARKER) { flag_out.store(true, Ordering::Relaxed); @@ -738,12 +743,19 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool, if let Some(u) = crate::events::TokenUsage::from_assistant_event(&v) { last_inference = Some(u); } + if let Some(m) = crate::events::TokenUsage::model_from_assistant_event(&v) { + last_model = Some(m); + } if let Some(cost) = crate::events::TokenUsage::from_stream_event(&v) { // Fallback to `cost` if the turn somehow produced // a result without any assistant event — keeps the // ctx badge from going stale on a degenerate turn. let ctx = last_inference.unwrap_or(cost); bus_out.record_turn_usage(ctx, cost); + // Pin the resolved model for this turn's stats row + // (cleared to None if no assistant event reported one + // → stats sink falls back to the requested name). + bus_out.set_resolved_model(last_model.clone()); } // Seed the API-reported context-window from the result // event's `modelUsage.*.contextWindow` field. This is