add per-agent and hive-wide skill invocation stats
This commit is contained in:
parent
2c11a437b4
commit
ef1554a1b2
4 changed files with 163 additions and 51 deletions
|
|
@ -209,6 +209,11 @@ pub struct HiveStats {
|
|||
/// hive-bash-daemon capture has recorded `bash_commands` rows on at
|
||||
/// least one active agent.
|
||||
pub bash_mix: Vec<KeyCount>,
|
||||
/// Most-triggered skills (fully-qualified `plugin:skill-name`) across
|
||||
/// the whole swarm, busiest first, capped to 10. Sourced from each
|
||||
/// agent's `tool_call_breakdown_json` — see [`read_skill_breakdown`].
|
||||
/// Empty until at least one agent has actually invoked a skill.
|
||||
pub skill_mix: Vec<KeyCount>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
|
|
@ -224,6 +229,10 @@ struct AgentAgg {
|
|||
/// `bash_commands` table (written by hive-bash-daemon). Empty when that
|
||||
/// capture hasn't run for this agent (table absent).
|
||||
bash: HashMap<String, u64>,
|
||||
/// Skill (fully-qualified `plugin:skill-name`) → invocation count,
|
||||
/// unpacked from `turn_stats.tool_call_breakdown_json`'s `Skill:*`
|
||||
/// entries. Empty when no skill was invoked in the window.
|
||||
skills: HashMap<String, u64>,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
|
|
@ -246,7 +255,8 @@ fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<A
|
|||
conn.busy_timeout(Duration::from_millis(500))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT model, input_tokens, output_tokens,
|
||||
cache_read_input_tokens, cache_creation_input_tokens
|
||||
cache_read_input_tokens, cache_creation_input_tokens,
|
||||
tool_call_breakdown_json
|
||||
FROM turn_stats
|
||||
WHERE started_at >= ?1",
|
||||
)?;
|
||||
|
|
@ -258,10 +268,11 @@ fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<A
|
|||
u64_from_i64(row.get::<_, i64>(2)?),
|
||||
u64_from_i64(row.get::<_, i64>(3)?),
|
||||
u64_from_i64(row.get::<_, i64>(4)?),
|
||||
row.get::<_, Option<String>>(5)?,
|
||||
))
|
||||
})?;
|
||||
for r in rows {
|
||||
let (model, input, output, cache_read, cache_creation) = r?;
|
||||
let (model, input, output, cache_read, cache_creation, breakdown_json) = r?;
|
||||
agg.turns += 1;
|
||||
agg.input = agg.input.saturating_add(input);
|
||||
agg.output = agg.output.saturating_add(output);
|
||||
|
|
@ -280,11 +291,31 @@ fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<A
|
|||
/ 1_000_000.0;
|
||||
}
|
||||
*agg.models.entry(model).or_insert(0) += 1;
|
||||
if let Some(json) = breakdown_json {
|
||||
for (skill, count) in skills_from_breakdown(&json) {
|
||||
*agg.skills.entry(skill).or_insert(0) += count;
|
||||
}
|
||||
}
|
||||
}
|
||||
agg.bash = read_bash_heads(&conn, from);
|
||||
Ok(agg)
|
||||
}
|
||||
|
||||
/// Pull the `Skill:<skill>` entries out of one turn's
|
||||
/// `tool_call_breakdown_json` blob (see `hive_agent::events::observe_stream`
|
||||
/// for how they're written), stripping the `Skill:` marker so the rollup
|
||||
/// keys by the bare (fully-qualified `plugin:skill-name`) skill identifier.
|
||||
/// Malformed/absent JSON yields nothing rather than erroring — one bad row
|
||||
/// shouldn't drop a whole agent from the rollup.
|
||||
fn skills_from_breakdown(json: &str) -> Vec<(String, u64)> {
|
||||
let Ok(map) = serde_json::from_str::<HashMap<String, u64>>(json) else {
|
||||
return Vec::new();
|
||||
};
|
||||
map.into_iter()
|
||||
.filter_map(|(k, v)| k.strip_prefix("Skill:").map(|s| (s.to_owned(), v)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Tally normalised bash-command heads from the agent's `bash_commands`
|
||||
/// table (`ts INTEGER, head TEXT`, written by hive-bash-daemon) over
|
||||
/// `[from, now]`. Best-effort + isolated from `read_agent`'s error path:
|
||||
|
|
@ -329,6 +360,7 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
|
|||
let mut agents: Vec<AgentRollup> = Vec::new();
|
||||
let mut model_mix: HashMap<String, u64> = HashMap::new();
|
||||
let mut bash_mix: HashMap<String, u64> = HashMap::new();
|
||||
let mut skill_mix: HashMap<String, u64> = HashMap::new();
|
||||
let mut total_turns = 0u64;
|
||||
let mut total_input = 0u64;
|
||||
let mut total_output = 0u64;
|
||||
|
|
@ -365,6 +397,9 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
|
|||
for (h, c) in &agg.bash {
|
||||
*bash_mix.entry(h.clone()).or_insert(0) += c;
|
||||
}
|
||||
for (s, c) in &agg.skills {
|
||||
*skill_mix.entry(s.clone()).or_insert(0) += c;
|
||||
}
|
||||
agents.push(AgentRollup {
|
||||
name: name.into_string(),
|
||||
turns: agg.turns,
|
||||
|
|
@ -393,6 +428,14 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
|
|||
bash_mix.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.key.cmp(&b.key)));
|
||||
bash_mix.truncate(10);
|
||||
|
||||
// Busiest skills first, capped to a top-10 list — same shape as bash_mix.
|
||||
let mut skill_mix: Vec<KeyCount> = skill_mix
|
||||
.into_iter()
|
||||
.map(|(key, count)| KeyCount { key, count })
|
||||
.collect();
|
||||
skill_mix.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.key.cmp(&b.key)));
|
||||
skill_mix.truncate(10);
|
||||
|
||||
HiveStats {
|
||||
window: window.label(),
|
||||
from,
|
||||
|
|
@ -407,6 +450,7 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
|
|||
agents,
|
||||
model_mix,
|
||||
bash_mix,
|
||||
skill_mix,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -447,4 +491,28 @@ mod tests {
|
|||
let conn = Connection::open_in_memory().unwrap();
|
||||
assert!(read_bash_heads(&conn, 0).is_empty());
|
||||
}
|
||||
|
||||
/// `Skill:<skill>` entries are unpacked and the marker stripped; a
|
||||
/// non-skill tool entry in the same blob is ignored.
|
||||
#[test]
|
||||
fn skills_from_breakdown_strips_marker_and_ignores_other_tools() {
|
||||
let json = r#"{"Skill:base:async-task-hygiene":3,"Read":7}"#;
|
||||
let mut got = skills_from_breakdown(json);
|
||||
got.sort();
|
||||
assert_eq!(got, vec![("base:async-task-hygiene".to_owned(), 3)]);
|
||||
}
|
||||
|
||||
/// Malformed JSON degrades to empty rather than erroring — one bad row
|
||||
/// shouldn't drop a whole agent from the rollup.
|
||||
#[test]
|
||||
fn skills_from_breakdown_malformed_json_is_empty() {
|
||||
assert!(skills_from_breakdown("not json").is_empty());
|
||||
}
|
||||
|
||||
/// No `Skill:*` keys at all (an agent that hasn't triggered one yet)
|
||||
/// is also empty, not an error.
|
||||
#[test]
|
||||
fn skills_from_breakdown_no_skills_is_empty() {
|
||||
assert!(skills_from_breakdown(r#"{"Read":1,"Edit":2}"#).is_empty());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue