add per-agent and hive-wide skill invocation stats
This commit is contained in:
parent
2c11a437b4
commit
ef1554a1b2
4 changed files with 163 additions and 51 deletions
|
|
@ -44,6 +44,10 @@
|
|||
recorded data, so the section never shows an empty block. -->
|
||||
<h3 id="hive-stats-bash-h" hidden>◇ favorite tools (bash commands across the swarm)</h3>
|
||||
<div id="hive-stats-bash" hidden></div>
|
||||
<!-- most-triggered skills across the swarm. Header + list hidden until
|
||||
at least one agent has actually invoked a skill. -->
|
||||
<h3 id="hive-stats-skills-h" hidden>◇ skill mix (invocations across the swarm)</h3>
|
||||
<div id="hive-stats-skills" hidden></div>
|
||||
</main>
|
||||
|
||||
<script type="module" src="/static/stats.js" defer></script>
|
||||
|
|
|
|||
|
|
@ -41,6 +41,39 @@ function hsMeta(parent, text) {
|
|||
p.textContent = text;
|
||||
parent.append(p);
|
||||
}
|
||||
// Shared bar-list renderer for any `KeyCount[]` mix (model/bash/skill).
|
||||
function renderKeyCountBars(container, mix) {
|
||||
container.replaceChildren();
|
||||
const max = mix[0].count || 1;
|
||||
for (const kc of mix) {
|
||||
const row = document.createElement('div'); row.className = 'hive-stats-bar';
|
||||
const lbl = document.createElement('span'); lbl.className = 'lbl'; lbl.textContent = kc.key;
|
||||
const track = document.createElement('span'); track.className = 'track';
|
||||
const fill = document.createElement('span'); fill.className = 'fill';
|
||||
fill.style.width = Math.max(2, Math.round(100 * kc.count / max)) + '%';
|
||||
track.append(fill);
|
||||
const cnt = document.createElement('span'); cnt.className = 'cnt'; cnt.textContent = hsFmtInt(kc.count);
|
||||
row.append(lbl, track, cnt);
|
||||
container.append(row);
|
||||
}
|
||||
}
|
||||
// A mix section that hides itself (+ its header) until the backing
|
||||
// capture has recorded data, so the section never shows an empty block
|
||||
// while data is pre-capture. Used by bash_mix and skill_mix.
|
||||
function renderOptionalMix(containerId, headerId, mix) {
|
||||
const el = $(containerId);
|
||||
const h = $(headerId);
|
||||
if (!el) return;
|
||||
if (!mix.length) {
|
||||
el.replaceChildren();
|
||||
el.hidden = true;
|
||||
if (h) h.hidden = true;
|
||||
return;
|
||||
}
|
||||
el.hidden = false;
|
||||
if (h) h.hidden = false;
|
||||
renderKeyCountBars(el, mix);
|
||||
}
|
||||
|
||||
function renderHiveStats(s) {
|
||||
const sum = $('hive-stats-summary');
|
||||
|
|
@ -93,54 +126,14 @@ function renderHiveStats(s) {
|
|||
const mm = $('hive-stats-models');
|
||||
if (mm) {
|
||||
const mix = s.model_mix || [];
|
||||
if (!mix.length) {
|
||||
hsMeta(mm, 'no turns in window');
|
||||
} else {
|
||||
mm.replaceChildren();
|
||||
const max = mix[0].count || 1;
|
||||
for (const kc of mix) {
|
||||
const row = document.createElement('div'); row.className = 'hive-stats-bar';
|
||||
const lbl = document.createElement('span'); lbl.className = 'lbl'; lbl.textContent = kc.key;
|
||||
const track = document.createElement('span'); track.className = 'track';
|
||||
const fill = document.createElement('span'); fill.className = 'fill';
|
||||
fill.style.width = Math.max(2, Math.round(100 * kc.count / max)) + '%';
|
||||
track.append(fill);
|
||||
const cnt = document.createElement('span'); cnt.className = 'cnt'; cnt.textContent = hsFmtInt(kc.count);
|
||||
row.append(lbl, track, cnt);
|
||||
mm.append(row);
|
||||
}
|
||||
}
|
||||
if (!mix.length) hsMeta(mm, 'no turns in window');
|
||||
else renderKeyCountBars(mm, mix);
|
||||
}
|
||||
|
||||
// "favorite tools": most-run bash commands across the swarm. Hidden
|
||||
// (header + list) until the capture has recorded data, so the
|
||||
// section never shows an empty block while capture is pre-data.
|
||||
const bh = $('hive-stats-bash');
|
||||
const bhH = $('hive-stats-bash-h');
|
||||
if (bh) {
|
||||
const bmix = s.bash_mix || [];
|
||||
if (!bmix.length) {
|
||||
bh.replaceChildren();
|
||||
bh.hidden = true;
|
||||
if (bhH) bhH.hidden = true;
|
||||
} else {
|
||||
bh.hidden = false;
|
||||
if (bhH) bhH.hidden = false;
|
||||
bh.replaceChildren();
|
||||
const max = bmix[0].count || 1;
|
||||
for (const kc of bmix) {
|
||||
const row = document.createElement('div'); row.className = 'hive-stats-bar';
|
||||
const lbl = document.createElement('span'); lbl.className = 'lbl'; lbl.textContent = kc.key;
|
||||
const track = document.createElement('span'); track.className = 'track';
|
||||
const fill = document.createElement('span'); fill.className = 'fill';
|
||||
fill.style.width = Math.max(2, Math.round(100 * kc.count / max)) + '%';
|
||||
track.append(fill);
|
||||
const cnt = document.createElement('span'); cnt.className = 'cnt'; cnt.textContent = hsFmtInt(kc.count);
|
||||
row.append(lbl, track, cnt);
|
||||
bh.append(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
// "favorite tools": most-run bash commands across the swarm.
|
||||
renderOptionalMix('hive-stats-bash', 'hive-stats-bash-h', s.bash_mix || []);
|
||||
// Most-triggered skills across the swarm.
|
||||
renderOptionalMix('hive-stats-skills', 'hive-stats-skills-h', s.skill_mix || []);
|
||||
}
|
||||
|
||||
async function refreshHiveStats() {
|
||||
|
|
|
|||
|
|
@ -643,6 +643,11 @@ impl Bus {
|
|||
/// pump on every parsed line. Cheap when the line isn't an
|
||||
/// assistant message — the field-check short-circuits.
|
||||
///
|
||||
/// A `Skill` invocation is one meta-tool (`name == "Skill"`) dispatching
|
||||
/// to whichever skill matched, so the bare tool name collapses every
|
||||
/// skill into one undifferentiated count. Special-cased (via
|
||||
/// [`breakdown_key`]) to key by `Skill:<skill>` instead.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if the internal lock is poisoned.
|
||||
|
|
@ -665,9 +670,9 @@ impl Bus {
|
|||
let name = block
|
||||
.get("name")
|
||||
.and_then(|n| n.as_str())
|
||||
.unwrap_or("<unnamed>")
|
||||
.to_owned();
|
||||
*counts.entry(name).or_insert(0) += 1;
|
||||
.unwrap_or("<unnamed>");
|
||||
let key = breakdown_key(name, block.get("input"));
|
||||
*counts.entry(key).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -914,6 +919,24 @@ fn degraded_mcp_servers(
|
|||
.collect()
|
||||
}
|
||||
|
||||
/// The `tool_call_breakdown_json` key for one `tool_use` block: the bare
|
||||
/// tool `name`, except a `Skill` invocation (one meta-tool dispatching to
|
||||
/// whichever skill matched) is keyed `Skill:<skill>` using the invocation's
|
||||
/// own `input.skill` field — the fully-qualified `plugin:skill-name` — so
|
||||
/// distinct skills don't collapse into one undifferentiated `"Skill"`
|
||||
/// count. Falls back to the bare `"Skill"` key if `input.skill` is ever
|
||||
/// missing/non-string, so a schema change degrades safely instead of losing
|
||||
/// the count. Pure so it's unit-testable without a live `Bus`.
|
||||
fn breakdown_key(name: &str, input: Option<&serde_json::Value>) -> String {
|
||||
if name != "Skill" {
|
||||
return name.to_owned();
|
||||
}
|
||||
input
|
||||
.and_then(|i| i.get("skill"))
|
||||
.and_then(|s| s.as_str())
|
||||
.map_or_else(|| name.to_owned(), |skill| format!("Skill:{skill}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{BusEvent, LiveEvent, StoredEvent};
|
||||
|
|
@ -976,6 +999,30 @@ mod tests {
|
|||
assert!(super::degraded_mcp_servers(&configured, &reported).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn breakdown_key_keys_skill_by_input_field() {
|
||||
let input = serde_json::json!({"skill": "base:async-task-hygiene"});
|
||||
assert_eq!(
|
||||
super::breakdown_key("Skill", Some(&input)),
|
||||
"Skill:base:async-task-hygiene"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn breakdown_key_non_skill_tool_is_bare_name() {
|
||||
assert_eq!(super::breakdown_key("Read", None), "Read");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn breakdown_key_skill_missing_input_field_degrades_to_bare_name() {
|
||||
// Schema-drift fallback: an unexpected/missing `input.skill` still
|
||||
// counts the invocation, just undifferentiated, rather than losing
|
||||
// it entirely.
|
||||
assert_eq!(super::breakdown_key("Skill", None), "Skill");
|
||||
let input = serde_json::json!({"unexpected": "field"});
|
||||
assert_eq!(super::breakdown_key("Skill", Some(&input)), "Skill");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn degraded_mcp_servers_ignores_pending() {
|
||||
// `pending` is the CLI's normal init-event race for a stdio server
|
||||
|
|
|
|||
|
|
@ -209,6 +209,11 @@ pub struct HiveStats {
|
|||
/// hive-bash-daemon capture has recorded `bash_commands` rows on at
|
||||
/// least one active agent.
|
||||
pub bash_mix: Vec<KeyCount>,
|
||||
/// Most-triggered skills (fully-qualified `plugin:skill-name`) across
|
||||
/// the whole swarm, busiest first, capped to 10. Sourced from each
|
||||
/// agent's `tool_call_breakdown_json` — see [`read_skill_breakdown`].
|
||||
/// Empty until at least one agent has actually invoked a skill.
|
||||
pub skill_mix: Vec<KeyCount>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
|
|
@ -224,6 +229,10 @@ struct AgentAgg {
|
|||
/// `bash_commands` table (written by hive-bash-daemon). Empty when that
|
||||
/// capture hasn't run for this agent (table absent).
|
||||
bash: HashMap<String, u64>,
|
||||
/// Skill (fully-qualified `plugin:skill-name`) → invocation count,
|
||||
/// unpacked from `turn_stats.tool_call_breakdown_json`'s `Skill:*`
|
||||
/// entries. Empty when no skill was invoked in the window.
|
||||
skills: HashMap<String, u64>,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
|
|
@ -246,7 +255,8 @@ fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<A
|
|||
conn.busy_timeout(Duration::from_millis(500))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT model, input_tokens, output_tokens,
|
||||
cache_read_input_tokens, cache_creation_input_tokens
|
||||
cache_read_input_tokens, cache_creation_input_tokens,
|
||||
tool_call_breakdown_json
|
||||
FROM turn_stats
|
||||
WHERE started_at >= ?1",
|
||||
)?;
|
||||
|
|
@ -258,10 +268,11 @@ fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<A
|
|||
u64_from_i64(row.get::<_, i64>(2)?),
|
||||
u64_from_i64(row.get::<_, i64>(3)?),
|
||||
u64_from_i64(row.get::<_, i64>(4)?),
|
||||
row.get::<_, Option<String>>(5)?,
|
||||
))
|
||||
})?;
|
||||
for r in rows {
|
||||
let (model, input, output, cache_read, cache_creation) = r?;
|
||||
let (model, input, output, cache_read, cache_creation, breakdown_json) = r?;
|
||||
agg.turns += 1;
|
||||
agg.input = agg.input.saturating_add(input);
|
||||
agg.output = agg.output.saturating_add(output);
|
||||
|
|
@ -280,11 +291,31 @@ fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<A
|
|||
/ 1_000_000.0;
|
||||
}
|
||||
*agg.models.entry(model).or_insert(0) += 1;
|
||||
if let Some(json) = breakdown_json {
|
||||
for (skill, count) in skills_from_breakdown(&json) {
|
||||
*agg.skills.entry(skill).or_insert(0) += count;
|
||||
}
|
||||
}
|
||||
}
|
||||
agg.bash = read_bash_heads(&conn, from);
|
||||
Ok(agg)
|
||||
}
|
||||
|
||||
/// Pull the `Skill:<skill>` entries out of one turn's
|
||||
/// `tool_call_breakdown_json` blob (see `hive_agent::events::observe_stream`
|
||||
/// for how they're written), stripping the `Skill:` marker so the rollup
|
||||
/// keys by the bare (fully-qualified `plugin:skill-name`) skill identifier.
|
||||
/// Malformed/absent JSON yields nothing rather than erroring — one bad row
|
||||
/// shouldn't drop a whole agent from the rollup.
|
||||
fn skills_from_breakdown(json: &str) -> Vec<(String, u64)> {
|
||||
let Ok(map) = serde_json::from_str::<HashMap<String, u64>>(json) else {
|
||||
return Vec::new();
|
||||
};
|
||||
map.into_iter()
|
||||
.filter_map(|(k, v)| k.strip_prefix("Skill:").map(|s| (s.to_owned(), v)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Tally normalised bash-command heads from the agent's `bash_commands`
|
||||
/// table (`ts INTEGER, head TEXT`, written by hive-bash-daemon) over
|
||||
/// `[from, now]`. Best-effort + isolated from `read_agent`'s error path:
|
||||
|
|
@ -329,6 +360,7 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
|
|||
let mut agents: Vec<AgentRollup> = Vec::new();
|
||||
let mut model_mix: HashMap<String, u64> = HashMap::new();
|
||||
let mut bash_mix: HashMap<String, u64> = HashMap::new();
|
||||
let mut skill_mix: HashMap<String, u64> = HashMap::new();
|
||||
let mut total_turns = 0u64;
|
||||
let mut total_input = 0u64;
|
||||
let mut total_output = 0u64;
|
||||
|
|
@ -365,6 +397,9 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
|
|||
for (h, c) in &agg.bash {
|
||||
*bash_mix.entry(h.clone()).or_insert(0) += c;
|
||||
}
|
||||
for (s, c) in &agg.skills {
|
||||
*skill_mix.entry(s.clone()).or_insert(0) += c;
|
||||
}
|
||||
agents.push(AgentRollup {
|
||||
name: name.into_string(),
|
||||
turns: agg.turns,
|
||||
|
|
@ -393,6 +428,14 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
|
|||
bash_mix.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.key.cmp(&b.key)));
|
||||
bash_mix.truncate(10);
|
||||
|
||||
// Busiest skills first, capped to a top-10 list — same shape as bash_mix.
|
||||
let mut skill_mix: Vec<KeyCount> = skill_mix
|
||||
.into_iter()
|
||||
.map(|(key, count)| KeyCount { key, count })
|
||||
.collect();
|
||||
skill_mix.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.key.cmp(&b.key)));
|
||||
skill_mix.truncate(10);
|
||||
|
||||
HiveStats {
|
||||
window: window.label(),
|
||||
from,
|
||||
|
|
@ -407,6 +450,7 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
|
|||
agents,
|
||||
model_mix,
|
||||
bash_mix,
|
||||
skill_mix,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -447,4 +491,28 @@ mod tests {
|
|||
let conn = Connection::open_in_memory().unwrap();
|
||||
assert!(read_bash_heads(&conn, 0).is_empty());
|
||||
}
|
||||
|
||||
/// `Skill:<skill>` entries are unpacked and the marker stripped; a
|
||||
/// non-skill tool entry in the same blob is ignored.
|
||||
#[test]
|
||||
fn skills_from_breakdown_strips_marker_and_ignores_other_tools() {
|
||||
let json = r#"{"Skill:base:async-task-hygiene":3,"Read":7}"#;
|
||||
let mut got = skills_from_breakdown(json);
|
||||
got.sort();
|
||||
assert_eq!(got, vec![("base:async-task-hygiene".to_owned(), 3)]);
|
||||
}
|
||||
|
||||
/// Malformed JSON degrades to empty rather than erroring — one bad row
|
||||
/// shouldn't drop a whole agent from the rollup.
|
||||
#[test]
|
||||
fn skills_from_breakdown_malformed_json_is_empty() {
|
||||
assert!(skills_from_breakdown("not json").is_empty());
|
||||
}
|
||||
|
||||
/// No `Skill:*` keys at all (an agent that hasn't triggered one yet)
|
||||
/// is also empty, not an error.
|
||||
#[test]
|
||||
fn skills_from_breakdown_no_skills_is_empty() {
|
||||
assert!(skills_from_breakdown(r#"{"Read":1,"Edit":2}"#).is_empty());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue