feat(stats): cache hit-rate, tokens/turn, and result trend over time
First slice of #1424 (per-agent /stats enrichments): - Backend: add per-bucket result_counts to the stats Snapshot (mirrors model_counts), so result outcomes can be charted over time, not just as a window total. - Frontend: two new summary chips — cache hit-rate % (cached input vs all input-side tokens) and avg tokens/turn — both derived from the existing per-bucket token sums. Plus a stacked result-trend chart so error / rate-limit / compaction spikes are visible across the window. Hive-wide aggregate, cost estimate, and container resource load land in follow-up PRs.
This commit is contained in:
parent
86f0de751f
commit
ded7474b2b
3 changed files with 63 additions and 0 deletions
|
|
@ -35,6 +35,7 @@
|
||||||
<div class="stats-card"><h3>top tools</h3><div class="chart-wrap"><canvas id="chart-tools"></canvas></div></div>
|
<div class="stats-card"><h3>top tools</h3><div class="chart-wrap"><canvas id="chart-tools"></canvas></div></div>
|
||||||
<div class="stats-card"><h3>wake source mix</h3><div class="chart-wrap"><canvas id="chart-wake"></canvas></div></div>
|
<div class="stats-card"><h3>wake source mix</h3><div class="chart-wrap"><canvas id="chart-wake"></canvas></div></div>
|
||||||
<div class="stats-card"><h3>result mix</h3><div class="chart-wrap"><canvas id="chart-result"></canvas></div></div>
|
<div class="stats-card"><h3>result mix</h3><div class="chart-wrap"><canvas id="chart-result"></canvas></div></div>
|
||||||
|
<div class="stats-card wide"><h3>result trend per bucket — errors / rate-limits / compactions over time</h3><div class="chart-wrap"><canvas id="chart-result-trend"></canvas></div></div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- Chart.js is now bundled into stats.js by esbuild (npm dep
|
<!-- Chart.js is now bundled into stats.js by esbuild (npm dep
|
||||||
|
|
|
||||||
|
|
@ -82,6 +82,18 @@ window.Chart = Chart;
|
||||||
ctx.fillText(msg, cv.width / 2, cv.height / 2);
|
ctx.fillText(msg, cv.width / 2, cv.height / 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Sum the four token streams across every bucket in the window.
|
||||||
|
function tokenTotals(s) {
|
||||||
|
let input = 0, output = 0, cacheRead = 0, cacheCreation = 0;
|
||||||
|
for (const b of s.buckets || []) {
|
||||||
|
input += b.input_tokens || 0;
|
||||||
|
output += b.output_tokens || 0;
|
||||||
|
cacheRead += b.cache_read_input_tokens || 0;
|
||||||
|
cacheCreation += b.cache_creation_input_tokens || 0;
|
||||||
|
}
|
||||||
|
return { input, output, cacheRead, cacheCreation };
|
||||||
|
}
|
||||||
|
|
||||||
function renderSummary(s) {
|
function renderSummary(s) {
|
||||||
const root = document.getElementById('summary');
|
const root = document.getElementById('summary');
|
||||||
root.replaceChildren();
|
root.replaceChildren();
|
||||||
|
|
@ -92,6 +104,17 @@ window.Chart = Chart;
|
||||||
['p95 duration', fmtMs(s.duration_summary.p95_ms)],
|
['p95 duration', fmtMs(s.duration_summary.p95_ms)],
|
||||||
['window', s.window],
|
['window', s.window],
|
||||||
];
|
];
|
||||||
|
// Token-efficiency chips: cache hit-rate (cached input vs all
|
||||||
|
// input-side tokens) and average tokens billed per turn.
|
||||||
|
const t = tokenTotals(s);
|
||||||
|
const inputSide = t.input + t.cacheRead + t.cacheCreation;
|
||||||
|
if (inputSide > 0) {
|
||||||
|
chips.push(['cache hit-rate', (100 * t.cacheRead / inputSide).toFixed(1) + '%']);
|
||||||
|
}
|
||||||
|
if (s.turn_count > 0) {
|
||||||
|
const perTurn = (t.input + t.output + t.cacheRead + t.cacheCreation) / s.turn_count;
|
||||||
|
chips.push(['tokens/turn', fmtInt(perTurn)]);
|
||||||
|
}
|
||||||
if (s.reminder_stats) {
|
if (s.reminder_stats) {
|
||||||
chips.push(
|
chips.push(
|
||||||
['reminders scheduled', fmtInt(s.reminder_stats.scheduled)],
|
['reminders scheduled', fmtInt(s.reminder_stats.scheduled)],
|
||||||
|
|
@ -269,6 +292,35 @@ window.Chart = Chart;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function renderResultTrendChart(s) {
|
||||||
|
const id = 'chart-result-trend';
|
||||||
|
destroy(id);
|
||||||
|
// Series = the result kinds seen in the window (same order +
|
||||||
|
// colours as the result-mix doughnut). One stacked bar series per
|
||||||
|
// kind, so error / rate-limit / compaction spikes line up in time.
|
||||||
|
const kinds = (s.result_mix || []).map((kc) => kc.key);
|
||||||
|
if (!kinds.length) { paintEmpty(id, 'no results'); return; }
|
||||||
|
const labels = s.buckets.map((b) => bucketLabel(b.ts, s.bucket_seconds));
|
||||||
|
const datasets = kinds.map((k, i) => ({
|
||||||
|
label: k,
|
||||||
|
data: s.buckets.map((b) => (b.result_counts && b.result_counts[k]) || 0),
|
||||||
|
backgroundColor: wheel[i % wheel.length],
|
||||||
|
}));
|
||||||
|
charts[id] = new Chart(document.getElementById(id), {
|
||||||
|
type: 'bar',
|
||||||
|
data: { labels, datasets },
|
||||||
|
options: {
|
||||||
|
responsive: true, maintainAspectRatio: false,
|
||||||
|
plugins: { legend: { position: 'top', labels: { boxWidth: 12 } } },
|
||||||
|
scales: {
|
||||||
|
x: { stacked: true, grid: { color: palette.border } },
|
||||||
|
y: { stacked: true, beginAtZero: true,
|
||||||
|
grid: { color: palette.border }, ticks: { precision: 0 } },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
function renderKeyCount(canvasId, items, emptyMsg) {
|
function renderKeyCount(canvasId, items, emptyMsg) {
|
||||||
destroy(canvasId);
|
destroy(canvasId);
|
||||||
if (!items || items.length === 0) {
|
if (!items || items.length === 0) {
|
||||||
|
|
@ -299,6 +351,7 @@ window.Chart = Chart;
|
||||||
paintEmpty('chart-tools', 'no tool calls');
|
paintEmpty('chart-tools', 'no tool calls');
|
||||||
paintEmpty('chart-wake', 'no wakes');
|
paintEmpty('chart-wake', 'no wakes');
|
||||||
paintEmpty('chart-result', 'no results');
|
paintEmpty('chart-result', 'no results');
|
||||||
|
paintEmpty('chart-result-trend', 'no results');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
renderTurnsChart(s);
|
renderTurnsChart(s);
|
||||||
|
|
@ -309,6 +362,7 @@ window.Chart = Chart;
|
||||||
renderKeyCount('chart-tools', s.tool_breakdown, 'no tool calls');
|
renderKeyCount('chart-tools', s.tool_breakdown, 'no tool calls');
|
||||||
renderKeyCount('chart-wake', s.wake_mix, 'no wakes');
|
renderKeyCount('chart-wake', s.wake_mix, 'no wakes');
|
||||||
renderKeyCount('chart-result', s.result_mix, 'no results');
|
renderKeyCount('chart-result', s.result_mix, 'no results');
|
||||||
|
renderResultTrendChart(s);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function loadStats() {
|
async function loadStats() {
|
||||||
|
|
|
||||||
|
|
@ -129,6 +129,10 @@ pub struct Bucket {
|
||||||
/// affects token cost, so this lets the operator line model usage
|
/// affects token cost, so this lets the operator line model usage
|
||||||
/// up against the cost series over time.
|
/// up against the cost series over time.
|
||||||
pub model_counts: HashMap<String, u64>,
|
pub model_counts: HashMap<String, u64>,
|
||||||
|
/// Turn count per `result_kind` in this bucket. Lets the stats
|
||||||
|
/// page chart error / rate-limit / compaction outcomes *over time*
|
||||||
|
/// (the window-total lives in `Snapshot::result_mix`).
|
||||||
|
pub result_counts: HashMap<String, u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
|
|
@ -243,6 +247,7 @@ fn snapshot(path: &Path, window: Window) -> Result<Snapshot> {
|
||||||
acc.ctx_sum = acc.ctx_sum.saturating_add(r.last_input_tokens);
|
acc.ctx_sum = acc.ctx_sum.saturating_add(r.last_input_tokens);
|
||||||
acc.ctx_max = acc.ctx_max.max(r.last_input_tokens);
|
acc.ctx_max = acc.ctx_max.max(r.last_input_tokens);
|
||||||
*acc.model_counts.entry(r.model.clone()).or_insert(0) += 1;
|
*acc.model_counts.entry(r.model.clone()).or_insert(0) += 1;
|
||||||
|
*acc.result_counts.entry(r.result_kind.clone()).or_insert(0) += 1;
|
||||||
|
|
||||||
all_durations.push(r.duration_ms.max(0));
|
all_durations.push(r.duration_ms.max(0));
|
||||||
*wake_totals.entry(r.wake_from).or_insert(0) += 1;
|
*wake_totals.entry(r.wake_from).or_insert(0) += 1;
|
||||||
|
|
@ -304,6 +309,7 @@ struct BucketAcc {
|
||||||
ctx_sum: u64,
|
ctx_sum: u64,
|
||||||
ctx_max: u64,
|
ctx_max: u64,
|
||||||
model_counts: HashMap<String, u64>,
|
model_counts: HashMap<String, u64>,
|
||||||
|
result_counts: HashMap<String, u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn fill_buckets(
|
fn fill_buckets(
|
||||||
|
|
@ -352,6 +358,7 @@ fn fill_buckets(
|
||||||
avg_ctx_tokens: avg_ctx,
|
avg_ctx_tokens: avg_ctx,
|
||||||
max_ctx_tokens: acc.ctx_max,
|
max_ctx_tokens: acc.ctx_max,
|
||||||
model_counts: acc.model_counts.clone(),
|
model_counts: acc.model_counts.clone(),
|
||||||
|
result_counts: acc.result_counts.clone(),
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
Bucket {
|
Bucket {
|
||||||
|
|
@ -367,6 +374,7 @@ fn fill_buckets(
|
||||||
avg_ctx_tokens: 0.0,
|
avg_ctx_tokens: 0.0,
|
||||||
max_ctx_tokens: 0,
|
max_ctx_tokens: 0,
|
||||||
model_counts: HashMap::new(),
|
model_counts: HashMap::new(),
|
||||||
|
result_counts: HashMap::new(),
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
out.push(bucket);
|
out.push(bucket);
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue