feat(stats): record api-resolved model id in turn-stats
This commit is contained in:
parent
8b66bf78df
commit
4e0f2ac9ca
3 changed files with 102 additions and 0 deletions
|
|
@ -388,6 +388,26 @@ impl TokenUsage {
|
|||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Extract the *resolved* model id from an `assistant` stream-json
|
||||
/// event (`message.model`). Unlike the requested `--model` name
|
||||
/// (which may be a short alias like `opus` or a default), the API
|
||||
/// echoes the concrete version it actually ran on (e.g.
|
||||
/// `claude-opus-4-8`). Recording the resolved id (not the requested
|
||||
/// name) is what lets the ST4TS model-mix + cost rollup label the
|
||||
/// exact version that ran. Returns `None` for non-assistant events
|
||||
/// or ones missing `message.model`.
|
||||
#[must_use]
|
||||
pub fn model_from_assistant_event(v: &serde_json::Value) -> Option<String> {
|
||||
if v.get("type").and_then(|t| t.as_str()) != Some("assistant") {
|
||||
return None;
|
||||
}
|
||||
v.get("message")
|
||||
.and_then(|m| m.get("model"))
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
}
|
||||
|
||||
/// Authoritative turn-loop state. The harness owns it; the web UI
|
||||
|
|
@ -529,6 +549,14 @@ pub struct Bus {
|
|||
/// limit the model enforces, which may differ from what the operator
|
||||
/// configured (e.g. 200 k active window on a 1 M cache-enabled model).
|
||||
api_context_window: Arc<Mutex<Option<u64>>>,
|
||||
/// Resolved model id from the most recent turn's `assistant` events
|
||||
/// (the API-echoed `message.model`, e.g. `claude-opus-4-8`), as
|
||||
/// opposed to the requested `--model` name which may be a short
|
||||
/// alias. Set once per turn by the stdout pump at result-time;
|
||||
/// `None` until the first assistant event of a turn is seen. The
|
||||
/// per-turn stats sink prefers this over the requested name so the
|
||||
/// model-mix + cost rollup reflect the concrete version that ran.
|
||||
last_resolved_model: Arc<Mutex<Option<String>>>,
|
||||
}
|
||||
|
||||
impl Bus {
|
||||
|
|
@ -570,6 +598,7 @@ impl Bus {
|
|||
tool_calls: Arc::new(Mutex::new(std::collections::HashMap::new())),
|
||||
last_turn_ended_unix: Arc::new(AtomicI64::new(0)),
|
||||
api_context_window: Arc::new(Mutex::new(None)),
|
||||
last_resolved_model: Arc::new(Mutex::new(None)),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -671,6 +700,30 @@ impl Bus {
|
|||
self.last_turn_ended_unix.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Record the resolved model id observed for the just-ended turn
|
||||
/// (from `assistant` events' `message.model`). `None` clears it so a
|
||||
/// degenerate turn that produced no assistant event doesn't inherit a
|
||||
/// stale id — the stats sink then falls back to the requested name.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if the internal lock is poisoned.
|
||||
pub fn set_resolved_model(&self, model: Option<String>) {
|
||||
*self.last_resolved_model.lock().unwrap() = model;
|
||||
}
|
||||
|
||||
/// Resolved model id from the most recent turn, if an `assistant`
|
||||
/// event reported one. The per-turn stats sink prefers this over the
|
||||
/// requested `--model` name.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if the internal lock is poisoned.
|
||||
#[must_use]
|
||||
pub fn last_resolved_model(&self) -> Option<String> {
|
||||
self.last_resolved_model.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
/// Update the API-reported context-window size from the stream-json
|
||||
/// `result` event's `modelUsage.*.contextWindow` field. Called by the
|
||||
/// stdout pump once per completed turn. `0` is ignored (sentinel for
|
||||
|
|
@ -896,3 +949,34 @@ impl Default for Bus {
|
|||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::TokenUsage;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn resolved_model_from_assistant_event() {
|
||||
let v = json!({
|
||||
"type": "assistant",
|
||||
"message": { "model": "claude-opus-4-8", "role": "assistant" }
|
||||
});
|
||||
assert_eq!(
|
||||
TokenUsage::model_from_assistant_event(&v),
|
||||
Some("claude-opus-4-8".to_owned())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolved_model_ignores_non_assistant_and_missing() {
|
||||
// Wrong event type.
|
||||
let result = json!({ "type": "result", "message": { "model": "claude-opus-4-8" } });
|
||||
assert_eq!(TokenUsage::model_from_assistant_event(&result), None);
|
||||
// Assistant event missing message.model.
|
||||
let no_model = json!({ "type": "assistant", "message": { "role": "assistant" } });
|
||||
assert_eq!(TokenUsage::model_from_assistant_event(&no_model), None);
|
||||
// Empty model string is treated as absent.
|
||||
let empty = json!({ "type": "assistant", "message": { "model": "" } });
|
||||
assert_eq!(TokenUsage::model_from_assistant_event(&empty), None);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,6 +52,12 @@ pub fn build_row(
|
|||
open_threads_count: Option<u64>,
|
||||
open_reminders_count: Option<u64>,
|
||||
) -> TurnStatRow {
|
||||
// Prefer the API-resolved model id (e.g. `claude-opus-4-8`) captured
|
||||
// from this turn's assistant events over the requested `--model`
|
||||
// name/alias, so the model-mix + cost rollup label the concrete
|
||||
// version that ran. Falls back to the requested name on a degenerate
|
||||
// turn that produced no assistant event.
|
||||
let model = bus.last_resolved_model().unwrap_or(model);
|
||||
let cost = bus.last_cost_usage().unwrap_or_default();
|
||||
let ctx = bus.last_ctx_usage().unwrap_or(cost);
|
||||
let tool_calls = bus.take_tool_calls();
|
||||
|
|
|
|||
|
|
@ -712,6 +712,11 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool,
|
|||
// get handed to `record_turn_usage` together so a single SSE
|
||||
// event updates both badges.
|
||||
let mut last_inference: Option<crate::events::TokenUsage> = None;
|
||||
// Resolved model id (API-echoed `message.model`) from this turn's
|
||||
// assistant events; recorded onto the bus at result-time so the
|
||||
// per-turn stats label the concrete version that ran, not the
|
||||
// requested `--model` alias.
|
||||
let mut last_model: Option<String> = None;
|
||||
while let Ok(Some(line)) = reader.next_line().await {
|
||||
if line.contains(PROMPT_TOO_LONG_MARKER) {
|
||||
flag_out.store(true, Ordering::Relaxed);
|
||||
|
|
@ -738,12 +743,19 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool,
|
|||
if let Some(u) = crate::events::TokenUsage::from_assistant_event(&v) {
|
||||
last_inference = Some(u);
|
||||
}
|
||||
if let Some(m) = crate::events::TokenUsage::model_from_assistant_event(&v) {
|
||||
last_model = Some(m);
|
||||
}
|
||||
if let Some(cost) = crate::events::TokenUsage::from_stream_event(&v) {
|
||||
// Fallback to `cost` if the turn somehow produced
|
||||
// a result without any assistant event — keeps the
|
||||
// ctx badge from going stale on a degenerate turn.
|
||||
let ctx = last_inference.unwrap_or(cost);
|
||||
bus_out.record_turn_usage(ctx, cost);
|
||||
// Pin the resolved model for this turn's stats row
|
||||
// (cleared to None if no assistant event reported one
|
||||
// → stats sink falls back to the requested name).
|
||||
bus_out.set_resolved_model(last_model.clone());
|
||||
}
|
||||
// Seed the API-reported context-window from the result
|
||||
// event's `modelUsage.*.contextWindow` field. This is
|
||||
|
|
|
|||
Loading…
Reference in a new issue