add per-agent and hive-wide skill invocation stats

This commit is contained in:
damocles 2026-07-27 23:52:04 +02:00
commit ef1554a1b2
4 changed files with 163 additions and 51 deletions

View file

@ -44,6 +44,10 @@
recorded data, so the section never shows an empty block. -->
<h3 id="hive-stats-bash-h" hidden>◇ favorite tools (bash commands across the swarm)</h3>
<div id="hive-stats-bash" hidden></div>
<!-- most-triggered skills across the swarm. Header + list hidden until
at least one agent has actually invoked a skill. -->
<h3 id="hive-stats-skills-h" hidden>◇ skill mix (invocations across the swarm)</h3>
<div id="hive-stats-skills" hidden></div>
</main>
<script type="module" src="/static/stats.js" defer></script>

View file

@ -41,6 +41,39 @@ function hsMeta(parent, text) {
p.textContent = text;
parent.append(p);
}
// Shared bar-list renderer for any `KeyCount[]` mix (model/bash/skill).
function renderKeyCountBars(container, mix) {
container.replaceChildren();
const max = mix[0].count || 1;
for (const kc of mix) {
const row = document.createElement('div'); row.className = 'hive-stats-bar';
const lbl = document.createElement('span'); lbl.className = 'lbl'; lbl.textContent = kc.key;
const track = document.createElement('span'); track.className = 'track';
const fill = document.createElement('span'); fill.className = 'fill';
fill.style.width = Math.max(2, Math.round(100 * kc.count / max)) + '%';
track.append(fill);
const cnt = document.createElement('span'); cnt.className = 'cnt'; cnt.textContent = hsFmtInt(kc.count);
row.append(lbl, track, cnt);
container.append(row);
}
}
// A mix section that hides itself (+ its header) until the backing
// capture has recorded data, so the section never shows an empty block
// while data is pre-capture. Used by bash_mix and skill_mix.
function renderOptionalMix(containerId, headerId, mix) {
const el = $(containerId);
const h = $(headerId);
if (!el) return;
if (!mix.length) {
el.replaceChildren();
el.hidden = true;
if (h) h.hidden = true;
return;
}
el.hidden = false;
if (h) h.hidden = false;
renderKeyCountBars(el, mix);
}
function renderHiveStats(s) {
const sum = $('hive-stats-summary');
@ -93,54 +126,14 @@ function renderHiveStats(s) {
const mm = $('hive-stats-models');
if (mm) {
const mix = s.model_mix || [];
if (!mix.length) {
hsMeta(mm, 'no turns in window');
} else {
mm.replaceChildren();
const max = mix[0].count || 1;
for (const kc of mix) {
const row = document.createElement('div'); row.className = 'hive-stats-bar';
const lbl = document.createElement('span'); lbl.className = 'lbl'; lbl.textContent = kc.key;
const track = document.createElement('span'); track.className = 'track';
const fill = document.createElement('span'); fill.className = 'fill';
fill.style.width = Math.max(2, Math.round(100 * kc.count / max)) + '%';
track.append(fill);
const cnt = document.createElement('span'); cnt.className = 'cnt'; cnt.textContent = hsFmtInt(kc.count);
row.append(lbl, track, cnt);
mm.append(row);
}
}
if (!mix.length) hsMeta(mm, 'no turns in window');
else renderKeyCountBars(mm, mix);
}
// "favorite tools": most-run bash commands across the swarm. Hidden
// (header + list) until the capture has recorded data, so the
// section never shows an empty block while capture is pre-data.
const bh = $('hive-stats-bash');
const bhH = $('hive-stats-bash-h');
if (bh) {
const bmix = s.bash_mix || [];
if (!bmix.length) {
bh.replaceChildren();
bh.hidden = true;
if (bhH) bhH.hidden = true;
} else {
bh.hidden = false;
if (bhH) bhH.hidden = false;
bh.replaceChildren();
const max = bmix[0].count || 1;
for (const kc of bmix) {
const row = document.createElement('div'); row.className = 'hive-stats-bar';
const lbl = document.createElement('span'); lbl.className = 'lbl'; lbl.textContent = kc.key;
const track = document.createElement('span'); track.className = 'track';
const fill = document.createElement('span'); fill.className = 'fill';
fill.style.width = Math.max(2, Math.round(100 * kc.count / max)) + '%';
track.append(fill);
const cnt = document.createElement('span'); cnt.className = 'cnt'; cnt.textContent = hsFmtInt(kc.count);
row.append(lbl, track, cnt);
bh.append(row);
}
}
}
// "favorite tools": most-run bash commands across the swarm.
renderOptionalMix('hive-stats-bash', 'hive-stats-bash-h', s.bash_mix || []);
// Most-triggered skills across the swarm.
renderOptionalMix('hive-stats-skills', 'hive-stats-skills-h', s.skill_mix || []);
}
async function refreshHiveStats() {

View file

@ -643,6 +643,11 @@ impl Bus {
/// pump on every parsed line. Cheap when the line isn't an
/// assistant message — the field-check short-circuits.
///
/// A `Skill` invocation is one meta-tool (`name == "Skill"`) dispatching
/// to whichever skill matched, so the bare tool name collapses every
/// skill into one undifferentiated count. Special-cased (via
/// [`breakdown_key`]) to key by `Skill:<skill>` instead.
///
/// # Panics
///
/// Panics if the internal lock is poisoned.
@ -665,9 +670,9 @@ impl Bus {
let name = block
.get("name")
.and_then(|n| n.as_str())
.unwrap_or("<unnamed>")
.to_owned();
*counts.entry(name).or_insert(0) += 1;
.unwrap_or("<unnamed>");
let key = breakdown_key(name, block.get("input"));
*counts.entry(key).or_insert(0) += 1;
}
}
@ -914,6 +919,24 @@ fn degraded_mcp_servers(
.collect()
}
/// The `tool_call_breakdown_json` key for one `tool_use` block: the bare
/// tool `name`, except a `Skill` invocation (one meta-tool dispatching to
/// whichever skill matched) is keyed `Skill:<skill>` using the invocation's
/// own `input.skill` field — the fully-qualified `plugin:skill-name` — so
/// distinct skills don't collapse into one undifferentiated `"Skill"`
/// count. Falls back to the bare `"Skill"` key if `input.skill` is ever
/// missing/non-string, so a schema change degrades safely instead of losing
/// the count. Pure so it's unit-testable without a live `Bus`.
fn breakdown_key(name: &str, input: Option<&serde_json::Value>) -> String {
if name != "Skill" {
return name.to_owned();
}
input
.and_then(|i| i.get("skill"))
.and_then(|s| s.as_str())
.map_or_else(|| name.to_owned(), |skill| format!("Skill:{skill}"))
}
#[cfg(test)]
mod tests {
use super::{BusEvent, LiveEvent, StoredEvent};
@ -976,6 +999,30 @@ mod tests {
assert!(super::degraded_mcp_servers(&configured, &reported).is_empty());
}
#[test]
fn breakdown_key_keys_skill_by_input_field() {
let input = serde_json::json!({"skill": "base:async-task-hygiene"});
assert_eq!(
super::breakdown_key("Skill", Some(&input)),
"Skill:base:async-task-hygiene"
);
}
#[test]
fn breakdown_key_non_skill_tool_is_bare_name() {
assert_eq!(super::breakdown_key("Read", None), "Read");
}
#[test]
fn breakdown_key_skill_missing_input_field_degrades_to_bare_name() {
// Schema-drift fallback: an unexpected/missing `input.skill` still
// counts the invocation, just undifferentiated, rather than losing
// it entirely.
assert_eq!(super::breakdown_key("Skill", None), "Skill");
let input = serde_json::json!({"unexpected": "field"});
assert_eq!(super::breakdown_key("Skill", Some(&input)), "Skill");
}
#[test]
fn degraded_mcp_servers_ignores_pending() {
// `pending` is the CLI's normal init-event race for a stdio server

View file

@ -209,6 +209,11 @@ pub struct HiveStats {
/// hive-bash-daemon capture has recorded `bash_commands` rows on at
/// least one active agent.
pub bash_mix: Vec<KeyCount>,
/// Most-triggered skills (fully-qualified `plugin:skill-name`) across
/// the whole swarm, busiest first, capped to 10. Sourced from each
/// agent's `tool_call_breakdown_json` — see [`read_skill_breakdown`].
/// Empty until at least one agent has actually invoked a skill.
pub skill_mix: Vec<KeyCount>,
}
#[derive(Default)]
@ -224,6 +229,10 @@ struct AgentAgg {
/// `bash_commands` table (written by hive-bash-daemon). Empty when that
/// capture hasn't run for this agent (table absent).
bash: HashMap<String, u64>,
/// Skill (fully-qualified `plugin:skill-name`) → invocation count,
/// unpacked from `turn_stats.tool_call_breakdown_json`'s `Skill:*`
/// entries. Empty when no skill was invoked in the window.
skills: HashMap<String, u64>,
}
#[allow(
@ -246,7 +255,8 @@ fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<A
conn.busy_timeout(Duration::from_millis(500))?;
let mut stmt = conn.prepare(
"SELECT model, input_tokens, output_tokens,
cache_read_input_tokens, cache_creation_input_tokens
cache_read_input_tokens, cache_creation_input_tokens,
tool_call_breakdown_json
FROM turn_stats
WHERE started_at >= ?1",
)?;
@ -258,10 +268,11 @@ fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<A
u64_from_i64(row.get::<_, i64>(2)?),
u64_from_i64(row.get::<_, i64>(3)?),
u64_from_i64(row.get::<_, i64>(4)?),
row.get::<_, Option<String>>(5)?,
))
})?;
for r in rows {
let (model, input, output, cache_read, cache_creation) = r?;
let (model, input, output, cache_read, cache_creation, breakdown_json) = r?;
agg.turns += 1;
agg.input = agg.input.saturating_add(input);
agg.output = agg.output.saturating_add(output);
@ -280,11 +291,31 @@ fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<A
/ 1_000_000.0;
}
*agg.models.entry(model).or_insert(0) += 1;
if let Some(json) = breakdown_json {
for (skill, count) in skills_from_breakdown(&json) {
*agg.skills.entry(skill).or_insert(0) += count;
}
}
}
agg.bash = read_bash_heads(&conn, from);
Ok(agg)
}
/// Pull the `Skill:<skill>` entries out of one turn's
/// `tool_call_breakdown_json` blob (see `hive_agent::events::observe_stream`
/// for how they're written), stripping the `Skill:` marker so the rollup
/// keys by the bare (fully-qualified `plugin:skill-name`) skill identifier.
/// Malformed/absent JSON yields nothing rather than erroring — one bad row
/// shouldn't drop a whole agent from the rollup.
fn skills_from_breakdown(json: &str) -> Vec<(String, u64)> {
let Ok(map) = serde_json::from_str::<HashMap<String, u64>>(json) else {
return Vec::new();
};
map.into_iter()
.filter_map(|(k, v)| k.strip_prefix("Skill:").map(|s| (s.to_owned(), v)))
.collect()
}
/// Tally normalised bash-command heads from the agent's `bash_commands`
/// table (`ts INTEGER, head TEXT`, written by hive-bash-daemon) over
/// `[from, now]`. Best-effort + isolated from `read_agent`'s error path:
@ -329,6 +360,7 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
let mut agents: Vec<AgentRollup> = Vec::new();
let mut model_mix: HashMap<String, u64> = HashMap::new();
let mut bash_mix: HashMap<String, u64> = HashMap::new();
let mut skill_mix: HashMap<String, u64> = HashMap::new();
let mut total_turns = 0u64;
let mut total_input = 0u64;
let mut total_output = 0u64;
@ -365,6 +397,9 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
for (h, c) in &agg.bash {
*bash_mix.entry(h.clone()).or_insert(0) += c;
}
for (s, c) in &agg.skills {
*skill_mix.entry(s.clone()).or_insert(0) += c;
}
agents.push(AgentRollup {
name: name.into_string(),
turns: agg.turns,
@ -393,6 +428,14 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
bash_mix.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.key.cmp(&b.key)));
bash_mix.truncate(10);
// Busiest skills first, capped to a top-10 list — same shape as bash_mix.
let mut skill_mix: Vec<KeyCount> = skill_mix
.into_iter()
.map(|(key, count)| KeyCount { key, count })
.collect();
skill_mix.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.key.cmp(&b.key)));
skill_mix.truncate(10);
HiveStats {
window: window.label(),
from,
@ -407,6 +450,7 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
agents,
model_mix,
bash_mix,
skill_mix,
}
}
@ -447,4 +491,28 @@ mod tests {
let conn = Connection::open_in_memory().unwrap();
assert!(read_bash_heads(&conn, 0).is_empty());
}
/// `Skill:<skill>` entries are unpacked and the marker stripped; a
/// non-skill tool entry in the same blob is ignored.
#[test]
fn skills_from_breakdown_strips_marker_and_ignores_other_tools() {
let json = r#"{"Skill:base:async-task-hygiene":3,"Read":7}"#;
let mut got = skills_from_breakdown(json);
got.sort();
assert_eq!(got, vec![("base:async-task-hygiene".to_owned(), 3)]);
}
/// Malformed JSON degrades to empty rather than erroring — one bad row
/// shouldn't drop a whole agent from the rollup.
#[test]
fn skills_from_breakdown_malformed_json_is_empty() {
assert!(skills_from_breakdown("not json").is_empty());
}
/// No `Skill:*` keys at all (an agent that hasn't triggered one yet)
/// is also empty, not an error.
#[test]
fn skills_from_breakdown_no_skills_is_empty() {
assert!(skills_from_breakdown(r#"{"Read":1,"Edit":2}"#).is_empty());
}
}