feat(stats): record api-resolved model id in turn-stats
This commit is contained in:
parent
8b66bf78df
commit
4e0f2ac9ca
3 changed files with 102 additions and 0 deletions
|
|
@ -388,6 +388,26 @@ impl TokenUsage {
|
||||||
}
|
}
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Extract the *resolved* model id from an `assistant` stream-json
|
||||||
|
/// event (`message.model`). Unlike the requested `--model` name
|
||||||
|
/// (which may be a short alias like `opus` or a default), the API
|
||||||
|
/// echoes the concrete version it actually ran on (e.g.
|
||||||
|
/// `claude-opus-4-8`). Recording the resolved id (not the requested
|
||||||
|
/// name) is what lets the ST4TS model-mix + cost rollup label the
|
||||||
|
/// exact version that ran. Returns `None` for non-assistant events
|
||||||
|
/// or ones missing `message.model`.
|
||||||
|
#[must_use]
|
||||||
|
pub fn model_from_assistant_event(v: &serde_json::Value) -> Option<String> {
|
||||||
|
if v.get("type").and_then(|t| t.as_str()) != Some("assistant") {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
v.get("message")
|
||||||
|
.and_then(|m| m.get("model"))
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Authoritative turn-loop state. The harness owns it; the web UI
|
/// Authoritative turn-loop state. The harness owns it; the web UI
|
||||||
|
|
@ -529,6 +549,14 @@ pub struct Bus {
|
||||||
/// limit the model enforces, which may differ from what the operator
|
/// limit the model enforces, which may differ from what the operator
|
||||||
/// configured (e.g. 200 k active window on a 1 M cache-enabled model).
|
/// configured (e.g. 200 k active window on a 1 M cache-enabled model).
|
||||||
api_context_window: Arc<Mutex<Option<u64>>>,
|
api_context_window: Arc<Mutex<Option<u64>>>,
|
||||||
|
/// Resolved model id from the most recent turn's `assistant` events
|
||||||
|
/// (the API-echoed `message.model`, e.g. `claude-opus-4-8`), as
|
||||||
|
/// opposed to the requested `--model` name which may be a short
|
||||||
|
/// alias. Set once per turn by the stdout pump at result-time;
|
||||||
|
/// `None` until the first assistant event of a turn is seen. The
|
||||||
|
/// per-turn stats sink prefers this over the requested name so the
|
||||||
|
/// model-mix + cost rollup reflect the concrete version that ran.
|
||||||
|
last_resolved_model: Arc<Mutex<Option<String>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Bus {
|
impl Bus {
|
||||||
|
|
@ -570,6 +598,7 @@ impl Bus {
|
||||||
tool_calls: Arc::new(Mutex::new(std::collections::HashMap::new())),
|
tool_calls: Arc::new(Mutex::new(std::collections::HashMap::new())),
|
||||||
last_turn_ended_unix: Arc::new(AtomicI64::new(0)),
|
last_turn_ended_unix: Arc::new(AtomicI64::new(0)),
|
||||||
api_context_window: Arc::new(Mutex::new(None)),
|
api_context_window: Arc::new(Mutex::new(None)),
|
||||||
|
last_resolved_model: Arc::new(Mutex::new(None)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -671,6 +700,30 @@ impl Bus {
|
||||||
self.last_turn_ended_unix.load(Ordering::Relaxed)
|
self.last_turn_ended_unix.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Record the resolved model id observed for the just-ended turn
|
||||||
|
/// (from `assistant` events' `message.model`). `None` clears it so a
|
||||||
|
/// degenerate turn that produced no assistant event doesn't inherit a
|
||||||
|
/// stale id — the stats sink then falls back to the requested name.
|
||||||
|
///
|
||||||
|
/// # Panics
|
||||||
|
///
|
||||||
|
/// Panics if the internal lock is poisoned.
|
||||||
|
pub fn set_resolved_model(&self, model: Option<String>) {
|
||||||
|
*self.last_resolved_model.lock().unwrap() = model;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolved model id from the most recent turn, if an `assistant`
|
||||||
|
/// event reported one. The per-turn stats sink prefers this over the
|
||||||
|
/// requested `--model` name.
|
||||||
|
///
|
||||||
|
/// # Panics
|
||||||
|
///
|
||||||
|
/// Panics if the internal lock is poisoned.
|
||||||
|
#[must_use]
|
||||||
|
pub fn last_resolved_model(&self) -> Option<String> {
|
||||||
|
self.last_resolved_model.lock().unwrap().clone()
|
||||||
|
}
|
||||||
|
|
||||||
/// Update the API-reported context-window size from the stream-json
|
/// Update the API-reported context-window size from the stream-json
|
||||||
/// `result` event's `modelUsage.*.contextWindow` field. Called by the
|
/// `result` event's `modelUsage.*.contextWindow` field. Called by the
|
||||||
/// stdout pump once per completed turn. `0` is ignored (sentinel for
|
/// stdout pump once per completed turn. `0` is ignored (sentinel for
|
||||||
|
|
@ -896,3 +949,34 @@ impl Default for Bus {
|
||||||
Self::new()
|
Self::new()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::TokenUsage;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolved_model_from_assistant_event() {
|
||||||
|
let v = json!({
|
||||||
|
"type": "assistant",
|
||||||
|
"message": { "model": "claude-opus-4-8", "role": "assistant" }
|
||||||
|
});
|
||||||
|
assert_eq!(
|
||||||
|
TokenUsage::model_from_assistant_event(&v),
|
||||||
|
Some("claude-opus-4-8".to_owned())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolved_model_ignores_non_assistant_and_missing() {
|
||||||
|
// Wrong event type.
|
||||||
|
let result = json!({ "type": "result", "message": { "model": "claude-opus-4-8" } });
|
||||||
|
assert_eq!(TokenUsage::model_from_assistant_event(&result), None);
|
||||||
|
// Assistant event missing message.model.
|
||||||
|
let no_model = json!({ "type": "assistant", "message": { "role": "assistant" } });
|
||||||
|
assert_eq!(TokenUsage::model_from_assistant_event(&no_model), None);
|
||||||
|
// Empty model string is treated as absent.
|
||||||
|
let empty = json!({ "type": "assistant", "message": { "model": "" } });
|
||||||
|
assert_eq!(TokenUsage::model_from_assistant_event(&empty), None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -52,6 +52,12 @@ pub fn build_row(
|
||||||
open_threads_count: Option<u64>,
|
open_threads_count: Option<u64>,
|
||||||
open_reminders_count: Option<u64>,
|
open_reminders_count: Option<u64>,
|
||||||
) -> TurnStatRow {
|
) -> TurnStatRow {
|
||||||
|
// Prefer the API-resolved model id (e.g. `claude-opus-4-8`) captured
|
||||||
|
// from this turn's assistant events over the requested `--model`
|
||||||
|
// name/alias, so the model-mix + cost rollup label the concrete
|
||||||
|
// version that ran. Falls back to the requested name on a degenerate
|
||||||
|
// turn that produced no assistant event.
|
||||||
|
let model = bus.last_resolved_model().unwrap_or(model);
|
||||||
let cost = bus.last_cost_usage().unwrap_or_default();
|
let cost = bus.last_cost_usage().unwrap_or_default();
|
||||||
let ctx = bus.last_ctx_usage().unwrap_or(cost);
|
let ctx = bus.last_ctx_usage().unwrap_or(cost);
|
||||||
let tool_calls = bus.take_tool_calls();
|
let tool_calls = bus.take_tool_calls();
|
||||||
|
|
|
||||||
|
|
@ -712,6 +712,11 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool,
|
||||||
// get handed to `record_turn_usage` together so a single SSE
|
// get handed to `record_turn_usage` together so a single SSE
|
||||||
// event updates both badges.
|
// event updates both badges.
|
||||||
let mut last_inference: Option<crate::events::TokenUsage> = None;
|
let mut last_inference: Option<crate::events::TokenUsage> = None;
|
||||||
|
// Resolved model id (API-echoed `message.model`) from this turn's
|
||||||
|
// assistant events; recorded onto the bus at result-time so the
|
||||||
|
// per-turn stats label the concrete version that ran, not the
|
||||||
|
// requested `--model` alias.
|
||||||
|
let mut last_model: Option<String> = None;
|
||||||
while let Ok(Some(line)) = reader.next_line().await {
|
while let Ok(Some(line)) = reader.next_line().await {
|
||||||
if line.contains(PROMPT_TOO_LONG_MARKER) {
|
if line.contains(PROMPT_TOO_LONG_MARKER) {
|
||||||
flag_out.store(true, Ordering::Relaxed);
|
flag_out.store(true, Ordering::Relaxed);
|
||||||
|
|
@ -738,12 +743,19 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool,
|
||||||
if let Some(u) = crate::events::TokenUsage::from_assistant_event(&v) {
|
if let Some(u) = crate::events::TokenUsage::from_assistant_event(&v) {
|
||||||
last_inference = Some(u);
|
last_inference = Some(u);
|
||||||
}
|
}
|
||||||
|
if let Some(m) = crate::events::TokenUsage::model_from_assistant_event(&v) {
|
||||||
|
last_model = Some(m);
|
||||||
|
}
|
||||||
if let Some(cost) = crate::events::TokenUsage::from_stream_event(&v) {
|
if let Some(cost) = crate::events::TokenUsage::from_stream_event(&v) {
|
||||||
// Fallback to `cost` if the turn somehow produced
|
// Fallback to `cost` if the turn somehow produced
|
||||||
// a result without any assistant event — keeps the
|
// a result without any assistant event — keeps the
|
||||||
// ctx badge from going stale on a degenerate turn.
|
// ctx badge from going stale on a degenerate turn.
|
||||||
let ctx = last_inference.unwrap_or(cost);
|
let ctx = last_inference.unwrap_or(cost);
|
||||||
bus_out.record_turn_usage(ctx, cost);
|
bus_out.record_turn_usage(ctx, cost);
|
||||||
|
// Pin the resolved model for this turn's stats row
|
||||||
|
// (cleared to None if no assistant event reported one
|
||||||
|
// → stats sink falls back to the requested name).
|
||||||
|
bus_out.set_resolved_model(last_model.clone());
|
||||||
}
|
}
|
||||||
// Seed the API-reported context-window from the result
|
// Seed the API-reported context-window from the result
|
||||||
// event's `modelUsage.*.contextWindow` field. This is
|
// event's `modelUsage.*.contextWindow` field. This is
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue