feat(stats): record api-resolved model id in turn-stats

This commit is contained in:
damocles 2026-06-08 20:00:24 +02:00 committed by mara
commit 4e0f2ac9ca
3 changed files with 102 additions and 0 deletions

View file

@ -388,6 +388,26 @@ impl TokenUsage {
}
None
}
/// Extract the *resolved* model id from an `assistant` stream-json
/// event (`message.model`). Unlike the requested `--model` name
/// (which may be a short alias like `opus` or a default), the API
/// echoes the concrete version it actually ran on (e.g.
/// `claude-opus-4-8`). Recording the resolved id (not the requested
/// name) is what lets the ST4TS model-mix + cost rollup label the
/// exact version that ran. Returns `None` for non-assistant events
/// or ones missing `message.model`.
#[must_use]
pub fn model_from_assistant_event(v: &serde_json::Value) -> Option<String> {
if v.get("type").and_then(|t| t.as_str()) != Some("assistant") {
return None;
}
v.get("message")
.and_then(|m| m.get("model"))
.and_then(serde_json::Value::as_str)
.filter(|s| !s.is_empty())
.map(ToOwned::to_owned)
}
}
/// Authoritative turn-loop state. The harness owns it; the web UI
@ -529,6 +549,14 @@ pub struct Bus {
/// limit the model enforces, which may differ from what the operator
/// configured (e.g. 200 k active window on a 1 M cache-enabled model).
api_context_window: Arc<Mutex<Option<u64>>>,
/// Resolved model id from the most recent turn's `assistant` events
/// (the API-echoed `message.model`, e.g. `claude-opus-4-8`), as
/// opposed to the requested `--model` name which may be a short
/// alias. Set once per turn by the stdout pump at result-time;
/// `None` until the first assistant event of a turn is seen. The
/// per-turn stats sink prefers this over the requested name so the
/// model-mix + cost rollup reflect the concrete version that ran.
last_resolved_model: Arc<Mutex<Option<String>>>,
}
impl Bus {
@ -570,6 +598,7 @@ impl Bus {
tool_calls: Arc::new(Mutex::new(std::collections::HashMap::new())),
last_turn_ended_unix: Arc::new(AtomicI64::new(0)),
api_context_window: Arc::new(Mutex::new(None)),
last_resolved_model: Arc::new(Mutex::new(None)),
}
}
@ -671,6 +700,30 @@ impl Bus {
self.last_turn_ended_unix.load(Ordering::Relaxed)
}
/// Record the resolved model id observed for the just-ended turn
/// (from `assistant` events' `message.model`). `None` clears it so a
/// degenerate turn that produced no assistant event doesn't inherit a
/// stale id — the stats sink then falls back to the requested name.
///
/// # Panics
///
/// Panics if the internal lock is poisoned.
pub fn set_resolved_model(&self, model: Option<String>) {
*self.last_resolved_model.lock().unwrap() = model;
}
/// Resolved model id from the most recent turn, if an `assistant`
/// event reported one. The per-turn stats sink prefers this over the
/// requested `--model` name.
///
/// # Panics
///
/// Panics if the internal lock is poisoned.
#[must_use]
pub fn last_resolved_model(&self) -> Option<String> {
self.last_resolved_model.lock().unwrap().clone()
}
/// Update the API-reported context-window size from the stream-json
/// `result` event's `modelUsage.*.contextWindow` field. Called by the
/// stdout pump once per completed turn. `0` is ignored (sentinel for
@ -896,3 +949,34 @@ impl Default for Bus {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::TokenUsage;
use serde_json::json;
#[test]
fn resolved_model_from_assistant_event() {
let v = json!({
"type": "assistant",
"message": { "model": "claude-opus-4-8", "role": "assistant" }
});
assert_eq!(
TokenUsage::model_from_assistant_event(&v),
Some("claude-opus-4-8".to_owned())
);
}
#[test]
fn resolved_model_ignores_non_assistant_and_missing() {
// Wrong event type.
let result = json!({ "type": "result", "message": { "model": "claude-opus-4-8" } });
assert_eq!(TokenUsage::model_from_assistant_event(&result), None);
// Assistant event missing message.model.
let no_model = json!({ "type": "assistant", "message": { "role": "assistant" } });
assert_eq!(TokenUsage::model_from_assistant_event(&no_model), None);
// Empty model string is treated as absent.
let empty = json!({ "type": "assistant", "message": { "model": "" } });
assert_eq!(TokenUsage::model_from_assistant_event(&empty), None);
}
}

View file

@ -52,6 +52,12 @@ pub fn build_row(
open_threads_count: Option<u64>,
open_reminders_count: Option<u64>,
) -> TurnStatRow {
// Prefer the API-resolved model id (e.g. `claude-opus-4-8`) captured
// from this turn's assistant events over the requested `--model`
// name/alias, so the model-mix + cost rollup label the concrete
// version that ran. Falls back to the requested name on a degenerate
// turn that produced no assistant event.
let model = bus.last_resolved_model().unwrap_or(model);
let cost = bus.last_cost_usage().unwrap_or_default();
let ctx = bus.last_ctx_usage().unwrap_or(cost);
let tool_calls = bus.take_tool_calls();

View file

@ -712,6 +712,11 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool,
// get handed to `record_turn_usage` together so a single SSE
// event updates both badges.
let mut last_inference: Option<crate::events::TokenUsage> = None;
// Resolved model id (API-echoed `message.model`) from this turn's
// assistant events; recorded onto the bus at result-time so the
// per-turn stats label the concrete version that ran, not the
// requested `--model` alias.
let mut last_model: Option<String> = None;
while let Ok(Some(line)) = reader.next_line().await {
if line.contains(PROMPT_TOO_LONG_MARKER) {
flag_out.store(true, Ordering::Relaxed);
@ -738,12 +743,19 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool,
if let Some(u) = crate::events::TokenUsage::from_assistant_event(&v) {
last_inference = Some(u);
}
if let Some(m) = crate::events::TokenUsage::model_from_assistant_event(&v) {
last_model = Some(m);
}
if let Some(cost) = crate::events::TokenUsage::from_stream_event(&v) {
// Fallback to `cost` if the turn somehow produced
// a result without any assistant event — keeps the
// ctx badge from going stale on a degenerate turn.
let ctx = last_inference.unwrap_or(cost);
bus_out.record_turn_usage(ctx, cost);
// Pin the resolved model for this turn's stats row
// (cleared to None if no assistant event reported one
// → stats sink falls back to the requested name).
bus_out.set_resolved_model(last_model.clone());
}
// Seed the API-reported context-window from the result
// event's `modelUsage.*.contextWindow` field. This is