add per-agent and hive-wide skill invocation stats

This commit is contained in:
damocles 2026-07-27 23:52:04 +02:00
commit ef1554a1b2
4 changed files with 163 additions and 51 deletions

View file

@ -209,6 +209,11 @@ pub struct HiveStats {
/// hive-bash-daemon capture has recorded `bash_commands` rows on at
/// least one active agent.
pub bash_mix: Vec<KeyCount>,
/// Most-triggered skills (fully-qualified `plugin:skill-name`) across
/// the whole swarm, busiest first, capped to 10. Sourced from each
/// agent's `tool_call_breakdown_json` — see [`read_skill_breakdown`].
/// Empty until at least one agent has actually invoked a skill.
pub skill_mix: Vec<KeyCount>,
}
#[derive(Default)]
@ -224,6 +229,10 @@ struct AgentAgg {
/// `bash_commands` table (written by hive-bash-daemon). Empty when that
/// capture hasn't run for this agent (table absent).
bash: HashMap<String, u64>,
/// Skill (fully-qualified `plugin:skill-name`) → invocation count,
/// unpacked from `turn_stats.tool_call_breakdown_json`'s `Skill:*`
/// entries. Empty when no skill was invoked in the window.
skills: HashMap<String, u64>,
}
#[allow(
@ -246,7 +255,8 @@ fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<A
conn.busy_timeout(Duration::from_millis(500))?;
let mut stmt = conn.prepare(
"SELECT model, input_tokens, output_tokens,
cache_read_input_tokens, cache_creation_input_tokens
cache_read_input_tokens, cache_creation_input_tokens,
tool_call_breakdown_json
FROM turn_stats
WHERE started_at >= ?1",
)?;
@ -258,10 +268,11 @@ fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<A
u64_from_i64(row.get::<_, i64>(2)?),
u64_from_i64(row.get::<_, i64>(3)?),
u64_from_i64(row.get::<_, i64>(4)?),
row.get::<_, Option<String>>(5)?,
))
})?;
for r in rows {
let (model, input, output, cache_read, cache_creation) = r?;
let (model, input, output, cache_read, cache_creation, breakdown_json) = r?;
agg.turns += 1;
agg.input = agg.input.saturating_add(input);
agg.output = agg.output.saturating_add(output);
@ -280,11 +291,31 @@ fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<A
/ 1_000_000.0;
}
*agg.models.entry(model).or_insert(0) += 1;
if let Some(json) = breakdown_json {
for (skill, count) in skills_from_breakdown(&json) {
*agg.skills.entry(skill).or_insert(0) += count;
}
}
}
agg.bash = read_bash_heads(&conn, from);
Ok(agg)
}
/// Pull the `Skill:<skill>` entries out of one turn's
/// `tool_call_breakdown_json` blob (see `hive_agent::events::observe_stream`
/// for how they're written), stripping the `Skill:` marker so the rollup
/// keys by the bare (fully-qualified `plugin:skill-name`) skill identifier.
/// Malformed/absent JSON yields nothing rather than erroring — one bad row
/// shouldn't drop a whole agent from the rollup.
fn skills_from_breakdown(json: &str) -> Vec<(String, u64)> {
let Ok(map) = serde_json::from_str::<HashMap<String, u64>>(json) else {
return Vec::new();
};
map.into_iter()
.filter_map(|(k, v)| k.strip_prefix("Skill:").map(|s| (s.to_owned(), v)))
.collect()
}
/// Tally normalised bash-command heads from the agent's `bash_commands`
/// table (`ts INTEGER, head TEXT`, written by hive-bash-daemon) over
/// `[from, now]`. Best-effort + isolated from `read_agent`'s error path:
@ -329,6 +360,7 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
let mut agents: Vec<AgentRollup> = Vec::new();
let mut model_mix: HashMap<String, u64> = HashMap::new();
let mut bash_mix: HashMap<String, u64> = HashMap::new();
let mut skill_mix: HashMap<String, u64> = HashMap::new();
let mut total_turns = 0u64;
let mut total_input = 0u64;
let mut total_output = 0u64;
@ -365,6 +397,9 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
for (h, c) in &agg.bash {
*bash_mix.entry(h.clone()).or_insert(0) += c;
}
for (s, c) in &agg.skills {
*skill_mix.entry(s.clone()).or_insert(0) += c;
}
agents.push(AgentRollup {
name: name.into_string(),
turns: agg.turns,
@ -393,6 +428,14 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
bash_mix.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.key.cmp(&b.key)));
bash_mix.truncate(10);
// Busiest skills first, capped to a top-10 list — same shape as bash_mix.
let mut skill_mix: Vec<KeyCount> = skill_mix
.into_iter()
.map(|(key, count)| KeyCount { key, count })
.collect();
skill_mix.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.key.cmp(&b.key)));
skill_mix.truncate(10);
HiveStats {
window: window.label(),
from,
@ -407,6 +450,7 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
agents,
model_mix,
bash_mix,
skill_mix,
}
}
@ -447,4 +491,28 @@ mod tests {
let conn = Connection::open_in_memory().unwrap();
assert!(read_bash_heads(&conn, 0).is_empty());
}
/// `Skill:<skill>` entries are unpacked and the marker stripped; a
/// non-skill tool entry in the same blob is ignored.
#[test]
fn skills_from_breakdown_strips_marker_and_ignores_other_tools() {
let json = r#"{"Skill:base:async-task-hygiene":3,"Read":7}"#;
let mut got = skills_from_breakdown(json);
got.sort();
assert_eq!(got, vec![("base:async-task-hygiene".to_owned(), 3)]);
}
/// Malformed JSON degrades to empty rather than erroring — one bad row
/// shouldn't drop a whole agent from the rollup.
#[test]
fn skills_from_breakdown_malformed_json_is_empty() {
assert!(skills_from_breakdown("not json").is_empty());
}
/// No `Skill:*` keys at all (an agent that hasn't triggered one yet)
/// is also empty, not an error.
#[test]
fn skills_from_breakdown_no_skills_is_empty() {
assert!(skills_from_breakdown(r#"{"Read":1,"Edit":2}"#).is_empty());
}
}