feat(stats): cache hit-rate, tokens/turn, and result trend over time
First slice of #1424 (per-agent /stats enrichments): - Backend: add per-bucket result_counts to the stats Snapshot (mirrors model_counts), so result outcomes can be charted over time, not just as a window total. - Frontend: two new summary chips — cache hit-rate % (cached input vs all input-side tokens) and avg tokens/turn — both derived from the existing per-bucket token sums. Plus a stacked result-trend chart so error / rate-limit / compaction spikes are visible across the window. Hive-wide aggregate, cost estimate, and container resource load land in follow-up PRs.
This commit is contained in:
parent
86f0de751f
commit
ded7474b2b
3 changed files with 63 additions and 0 deletions
|
|
@ -35,6 +35,7 @@
|
|||
<div class="stats-card"><h3>top tools</h3><div class="chart-wrap"><canvas id="chart-tools"></canvas></div></div>
|
||||
<div class="stats-card"><h3>wake source mix</h3><div class="chart-wrap"><canvas id="chart-wake"></canvas></div></div>
|
||||
<div class="stats-card"><h3>result mix</h3><div class="chart-wrap"><canvas id="chart-result"></canvas></div></div>
|
||||
<div class="stats-card wide"><h3>result trend per bucket — errors / rate-limits / compactions over time</h3><div class="chart-wrap"><canvas id="chart-result-trend"></canvas></div></div>
|
||||
</div>
|
||||
|
||||
<!-- Chart.js is now bundled into stats.js by esbuild (npm dep
|
||||
|
|
|
|||
|
|
@ -82,6 +82,18 @@ window.Chart = Chart;
|
|||
ctx.fillText(msg, cv.width / 2, cv.height / 2);
|
||||
}
|
||||
|
||||
// Sum the four token streams across every bucket in the window.
|
||||
function tokenTotals(s) {
|
||||
let input = 0, output = 0, cacheRead = 0, cacheCreation = 0;
|
||||
for (const b of s.buckets || []) {
|
||||
input += b.input_tokens || 0;
|
||||
output += b.output_tokens || 0;
|
||||
cacheRead += b.cache_read_input_tokens || 0;
|
||||
cacheCreation += b.cache_creation_input_tokens || 0;
|
||||
}
|
||||
return { input, output, cacheRead, cacheCreation };
|
||||
}
|
||||
|
||||
function renderSummary(s) {
|
||||
const root = document.getElementById('summary');
|
||||
root.replaceChildren();
|
||||
|
|
@ -92,6 +104,17 @@ window.Chart = Chart;
|
|||
['p95 duration', fmtMs(s.duration_summary.p95_ms)],
|
||||
['window', s.window],
|
||||
];
|
||||
// Token-efficiency chips: cache hit-rate (cached input vs all
|
||||
// input-side tokens) and average tokens billed per turn.
|
||||
const t = tokenTotals(s);
|
||||
const inputSide = t.input + t.cacheRead + t.cacheCreation;
|
||||
if (inputSide > 0) {
|
||||
chips.push(['cache hit-rate', (100 * t.cacheRead / inputSide).toFixed(1) + '%']);
|
||||
}
|
||||
if (s.turn_count > 0) {
|
||||
const perTurn = (t.input + t.output + t.cacheRead + t.cacheCreation) / s.turn_count;
|
||||
chips.push(['tokens/turn', fmtInt(perTurn)]);
|
||||
}
|
||||
if (s.reminder_stats) {
|
||||
chips.push(
|
||||
['reminders scheduled', fmtInt(s.reminder_stats.scheduled)],
|
||||
|
|
@ -269,6 +292,35 @@ window.Chart = Chart;
|
|||
});
|
||||
}
|
||||
|
||||
function renderResultTrendChart(s) {
|
||||
const id = 'chart-result-trend';
|
||||
destroy(id);
|
||||
// Series = the result kinds seen in the window (same order +
|
||||
// colours as the result-mix doughnut). One stacked bar series per
|
||||
// kind, so error / rate-limit / compaction spikes line up in time.
|
||||
const kinds = (s.result_mix || []).map((kc) => kc.key);
|
||||
if (!kinds.length) { paintEmpty(id, 'no results'); return; }
|
||||
const labels = s.buckets.map((b) => bucketLabel(b.ts, s.bucket_seconds));
|
||||
const datasets = kinds.map((k, i) => ({
|
||||
label: k,
|
||||
data: s.buckets.map((b) => (b.result_counts && b.result_counts[k]) || 0),
|
||||
backgroundColor: wheel[i % wheel.length],
|
||||
}));
|
||||
charts[id] = new Chart(document.getElementById(id), {
|
||||
type: 'bar',
|
||||
data: { labels, datasets },
|
||||
options: {
|
||||
responsive: true, maintainAspectRatio: false,
|
||||
plugins: { legend: { position: 'top', labels: { boxWidth: 12 } } },
|
||||
scales: {
|
||||
x: { stacked: true, grid: { color: palette.border } },
|
||||
y: { stacked: true, beginAtZero: true,
|
||||
grid: { color: palette.border }, ticks: { precision: 0 } },
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function renderKeyCount(canvasId, items, emptyMsg) {
|
||||
destroy(canvasId);
|
||||
if (!items || items.length === 0) {
|
||||
|
|
@ -299,6 +351,7 @@ window.Chart = Chart;
|
|||
paintEmpty('chart-tools', 'no tool calls');
|
||||
paintEmpty('chart-wake', 'no wakes');
|
||||
paintEmpty('chart-result', 'no results');
|
||||
paintEmpty('chart-result-trend', 'no results');
|
||||
return;
|
||||
}
|
||||
renderTurnsChart(s);
|
||||
|
|
@ -309,6 +362,7 @@ window.Chart = Chart;
|
|||
renderKeyCount('chart-tools', s.tool_breakdown, 'no tool calls');
|
||||
renderKeyCount('chart-wake', s.wake_mix, 'no wakes');
|
||||
renderKeyCount('chart-result', s.result_mix, 'no results');
|
||||
renderResultTrendChart(s);
|
||||
}
|
||||
|
||||
async function loadStats() {
|
||||
|
|
|
|||
|
|
@ -129,6 +129,10 @@ pub struct Bucket {
|
|||
/// affects token cost, so this lets the operator line model usage
|
||||
/// up against the cost series over time.
|
||||
pub model_counts: HashMap<String, u64>,
|
||||
/// Turn count per `result_kind` in this bucket. Lets the stats
|
||||
/// page chart error / rate-limit / compaction outcomes *over time*
|
||||
/// (the window-total lives in `Snapshot::result_mix`).
|
||||
pub result_counts: HashMap<String, u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
|
|
@ -243,6 +247,7 @@ fn snapshot(path: &Path, window: Window) -> Result<Snapshot> {
|
|||
acc.ctx_sum = acc.ctx_sum.saturating_add(r.last_input_tokens);
|
||||
acc.ctx_max = acc.ctx_max.max(r.last_input_tokens);
|
||||
*acc.model_counts.entry(r.model.clone()).or_insert(0) += 1;
|
||||
*acc.result_counts.entry(r.result_kind.clone()).or_insert(0) += 1;
|
||||
|
||||
all_durations.push(r.duration_ms.max(0));
|
||||
*wake_totals.entry(r.wake_from).or_insert(0) += 1;
|
||||
|
|
@ -304,6 +309,7 @@ struct BucketAcc {
|
|||
ctx_sum: u64,
|
||||
ctx_max: u64,
|
||||
model_counts: HashMap<String, u64>,
|
||||
result_counts: HashMap<String, u64>,
|
||||
}
|
||||
|
||||
fn fill_buckets(
|
||||
|
|
@ -352,6 +358,7 @@ fn fill_buckets(
|
|||
avg_ctx_tokens: avg_ctx,
|
||||
max_ctx_tokens: acc.ctx_max,
|
||||
model_counts: acc.model_counts.clone(),
|
||||
result_counts: acc.result_counts.clone(),
|
||||
}
|
||||
} else {
|
||||
Bucket {
|
||||
|
|
@ -367,6 +374,7 @@ fn fill_buckets(
|
|||
avg_ctx_tokens: 0.0,
|
||||
max_ctx_tokens: 0,
|
||||
model_counts: HashMap::new(),
|
||||
result_counts: HashMap::new(),
|
||||
}
|
||||
};
|
||||
out.push(bucket);
|
||||
|
|
|
|||
Loading…
Reference in a new issue