add per-agent and hive-wide skill invocation stats
This commit is contained in:
parent
2c11a437b4
commit
ef1554a1b2
4 changed files with 163 additions and 51 deletions
|
|
@ -44,6 +44,10 @@
|
||||||
recorded data, so the section never shows an empty block. -->
|
recorded data, so the section never shows an empty block. -->
|
||||||
<h3 id="hive-stats-bash-h" hidden>◇ favorite tools (bash commands across the swarm)</h3>
|
<h3 id="hive-stats-bash-h" hidden>◇ favorite tools (bash commands across the swarm)</h3>
|
||||||
<div id="hive-stats-bash" hidden></div>
|
<div id="hive-stats-bash" hidden></div>
|
||||||
|
<!-- most-triggered skills across the swarm. Header + list hidden until
|
||||||
|
at least one agent has actually invoked a skill. -->
|
||||||
|
<h3 id="hive-stats-skills-h" hidden>◇ skill mix (invocations across the swarm)</h3>
|
||||||
|
<div id="hive-stats-skills" hidden></div>
|
||||||
</main>
|
</main>
|
||||||
|
|
||||||
<script type="module" src="/static/stats.js" defer></script>
|
<script type="module" src="/static/stats.js" defer></script>
|
||||||
|
|
|
||||||
|
|
@ -41,6 +41,39 @@ function hsMeta(parent, text) {
|
||||||
p.textContent = text;
|
p.textContent = text;
|
||||||
parent.append(p);
|
parent.append(p);
|
||||||
}
|
}
|
||||||
|
// Shared bar-list renderer for any `KeyCount[]` mix (model/bash/skill).
|
||||||
|
function renderKeyCountBars(container, mix) {
|
||||||
|
container.replaceChildren();
|
||||||
|
const max = mix[0].count || 1;
|
||||||
|
for (const kc of mix) {
|
||||||
|
const row = document.createElement('div'); row.className = 'hive-stats-bar';
|
||||||
|
const lbl = document.createElement('span'); lbl.className = 'lbl'; lbl.textContent = kc.key;
|
||||||
|
const track = document.createElement('span'); track.className = 'track';
|
||||||
|
const fill = document.createElement('span'); fill.className = 'fill';
|
||||||
|
fill.style.width = Math.max(2, Math.round(100 * kc.count / max)) + '%';
|
||||||
|
track.append(fill);
|
||||||
|
const cnt = document.createElement('span'); cnt.className = 'cnt'; cnt.textContent = hsFmtInt(kc.count);
|
||||||
|
row.append(lbl, track, cnt);
|
||||||
|
container.append(row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A mix section that hides itself (+ its header) until the backing
|
||||||
|
// capture has recorded data, so the section never shows an empty block
|
||||||
|
// while data is pre-capture. Used by bash_mix and skill_mix.
|
||||||
|
function renderOptionalMix(containerId, headerId, mix) {
|
||||||
|
const el = $(containerId);
|
||||||
|
const h = $(headerId);
|
||||||
|
if (!el) return;
|
||||||
|
if (!mix.length) {
|
||||||
|
el.replaceChildren();
|
||||||
|
el.hidden = true;
|
||||||
|
if (h) h.hidden = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
el.hidden = false;
|
||||||
|
if (h) h.hidden = false;
|
||||||
|
renderKeyCountBars(el, mix);
|
||||||
|
}
|
||||||
|
|
||||||
function renderHiveStats(s) {
|
function renderHiveStats(s) {
|
||||||
const sum = $('hive-stats-summary');
|
const sum = $('hive-stats-summary');
|
||||||
|
|
@ -93,54 +126,14 @@ function renderHiveStats(s) {
|
||||||
const mm = $('hive-stats-models');
|
const mm = $('hive-stats-models');
|
||||||
if (mm) {
|
if (mm) {
|
||||||
const mix = s.model_mix || [];
|
const mix = s.model_mix || [];
|
||||||
if (!mix.length) {
|
if (!mix.length) hsMeta(mm, 'no turns in window');
|
||||||
hsMeta(mm, 'no turns in window');
|
else renderKeyCountBars(mm, mix);
|
||||||
} else {
|
|
||||||
mm.replaceChildren();
|
|
||||||
const max = mix[0].count || 1;
|
|
||||||
for (const kc of mix) {
|
|
||||||
const row = document.createElement('div'); row.className = 'hive-stats-bar';
|
|
||||||
const lbl = document.createElement('span'); lbl.className = 'lbl'; lbl.textContent = kc.key;
|
|
||||||
const track = document.createElement('span'); track.className = 'track';
|
|
||||||
const fill = document.createElement('span'); fill.className = 'fill';
|
|
||||||
fill.style.width = Math.max(2, Math.round(100 * kc.count / max)) + '%';
|
|
||||||
track.append(fill);
|
|
||||||
const cnt = document.createElement('span'); cnt.className = 'cnt'; cnt.textContent = hsFmtInt(kc.count);
|
|
||||||
row.append(lbl, track, cnt);
|
|
||||||
mm.append(row);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// "favorite tools": most-run bash commands across the swarm. Hidden
|
// "favorite tools": most-run bash commands across the swarm.
|
||||||
// (header + list) until the capture has recorded data, so the
|
renderOptionalMix('hive-stats-bash', 'hive-stats-bash-h', s.bash_mix || []);
|
||||||
// section never shows an empty block while capture is pre-data.
|
// Most-triggered skills across the swarm.
|
||||||
const bh = $('hive-stats-bash');
|
renderOptionalMix('hive-stats-skills', 'hive-stats-skills-h', s.skill_mix || []);
|
||||||
const bhH = $('hive-stats-bash-h');
|
|
||||||
if (bh) {
|
|
||||||
const bmix = s.bash_mix || [];
|
|
||||||
if (!bmix.length) {
|
|
||||||
bh.replaceChildren();
|
|
||||||
bh.hidden = true;
|
|
||||||
if (bhH) bhH.hidden = true;
|
|
||||||
} else {
|
|
||||||
bh.hidden = false;
|
|
||||||
if (bhH) bhH.hidden = false;
|
|
||||||
bh.replaceChildren();
|
|
||||||
const max = bmix[0].count || 1;
|
|
||||||
for (const kc of bmix) {
|
|
||||||
const row = document.createElement('div'); row.className = 'hive-stats-bar';
|
|
||||||
const lbl = document.createElement('span'); lbl.className = 'lbl'; lbl.textContent = kc.key;
|
|
||||||
const track = document.createElement('span'); track.className = 'track';
|
|
||||||
const fill = document.createElement('span'); fill.className = 'fill';
|
|
||||||
fill.style.width = Math.max(2, Math.round(100 * kc.count / max)) + '%';
|
|
||||||
track.append(fill);
|
|
||||||
const cnt = document.createElement('span'); cnt.className = 'cnt'; cnt.textContent = hsFmtInt(kc.count);
|
|
||||||
row.append(lbl, track, cnt);
|
|
||||||
bh.append(row);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function refreshHiveStats() {
|
async function refreshHiveStats() {
|
||||||
|
|
|
||||||
|
|
@ -643,6 +643,11 @@ impl Bus {
|
||||||
/// pump on every parsed line. Cheap when the line isn't an
|
/// pump on every parsed line. Cheap when the line isn't an
|
||||||
/// assistant message — the field-check short-circuits.
|
/// assistant message — the field-check short-circuits.
|
||||||
///
|
///
|
||||||
|
/// A `Skill` invocation is one meta-tool (`name == "Skill"`) dispatching
|
||||||
|
/// to whichever skill matched, so the bare tool name collapses every
|
||||||
|
/// skill into one undifferentiated count. Special-cased (via
|
||||||
|
/// [`breakdown_key`]) to key by `Skill:<skill>` instead.
|
||||||
|
///
|
||||||
/// # Panics
|
/// # Panics
|
||||||
///
|
///
|
||||||
/// Panics if the internal lock is poisoned.
|
/// Panics if the internal lock is poisoned.
|
||||||
|
|
@ -665,9 +670,9 @@ impl Bus {
|
||||||
let name = block
|
let name = block
|
||||||
.get("name")
|
.get("name")
|
||||||
.and_then(|n| n.as_str())
|
.and_then(|n| n.as_str())
|
||||||
.unwrap_or("<unnamed>")
|
.unwrap_or("<unnamed>");
|
||||||
.to_owned();
|
let key = breakdown_key(name, block.get("input"));
|
||||||
*counts.entry(name).or_insert(0) += 1;
|
*counts.entry(key).or_insert(0) += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -914,6 +919,24 @@ fn degraded_mcp_servers(
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The `tool_call_breakdown_json` key for one `tool_use` block: the bare
|
||||||
|
/// tool `name`, except a `Skill` invocation (one meta-tool dispatching to
|
||||||
|
/// whichever skill matched) is keyed `Skill:<skill>` using the invocation's
|
||||||
|
/// own `input.skill` field — the fully-qualified `plugin:skill-name` — so
|
||||||
|
/// distinct skills don't collapse into one undifferentiated `"Skill"`
|
||||||
|
/// count. Falls back to the bare `"Skill"` key if `input.skill` is ever
|
||||||
|
/// missing/non-string, so a schema change degrades safely instead of losing
|
||||||
|
/// the count. Pure so it's unit-testable without a live `Bus`.
|
||||||
|
fn breakdown_key(name: &str, input: Option<&serde_json::Value>) -> String {
|
||||||
|
if name != "Skill" {
|
||||||
|
return name.to_owned();
|
||||||
|
}
|
||||||
|
input
|
||||||
|
.and_then(|i| i.get("skill"))
|
||||||
|
.and_then(|s| s.as_str())
|
||||||
|
.map_or_else(|| name.to_owned(), |skill| format!("Skill:{skill}"))
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{BusEvent, LiveEvent, StoredEvent};
|
use super::{BusEvent, LiveEvent, StoredEvent};
|
||||||
|
|
@ -976,6 +999,30 @@ mod tests {
|
||||||
assert!(super::degraded_mcp_servers(&configured, &reported).is_empty());
|
assert!(super::degraded_mcp_servers(&configured, &reported).is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn breakdown_key_keys_skill_by_input_field() {
|
||||||
|
let input = serde_json::json!({"skill": "base:async-task-hygiene"});
|
||||||
|
assert_eq!(
|
||||||
|
super::breakdown_key("Skill", Some(&input)),
|
||||||
|
"Skill:base:async-task-hygiene"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn breakdown_key_non_skill_tool_is_bare_name() {
|
||||||
|
assert_eq!(super::breakdown_key("Read", None), "Read");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn breakdown_key_skill_missing_input_field_degrades_to_bare_name() {
|
||||||
|
// Schema-drift fallback: an unexpected/missing `input.skill` still
|
||||||
|
// counts the invocation, just undifferentiated, rather than losing
|
||||||
|
// it entirely.
|
||||||
|
assert_eq!(super::breakdown_key("Skill", None), "Skill");
|
||||||
|
let input = serde_json::json!({"unexpected": "field"});
|
||||||
|
assert_eq!(super::breakdown_key("Skill", Some(&input)), "Skill");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn degraded_mcp_servers_ignores_pending() {
|
fn degraded_mcp_servers_ignores_pending() {
|
||||||
// `pending` is the CLI's normal init-event race for a stdio server
|
// `pending` is the CLI's normal init-event race for a stdio server
|
||||||
|
|
|
||||||
|
|
@ -209,6 +209,11 @@ pub struct HiveStats {
|
||||||
/// hive-bash-daemon capture has recorded `bash_commands` rows on at
|
/// hive-bash-daemon capture has recorded `bash_commands` rows on at
|
||||||
/// least one active agent.
|
/// least one active agent.
|
||||||
pub bash_mix: Vec<KeyCount>,
|
pub bash_mix: Vec<KeyCount>,
|
||||||
|
/// Most-triggered skills (fully-qualified `plugin:skill-name`) across
|
||||||
|
/// the whole swarm, busiest first, capped to 10. Sourced from each
|
||||||
|
/// agent's `tool_call_breakdown_json` — see [`read_skill_breakdown`].
|
||||||
|
/// Empty until at least one agent has actually invoked a skill.
|
||||||
|
pub skill_mix: Vec<KeyCount>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
|
|
@ -224,6 +229,10 @@ struct AgentAgg {
|
||||||
/// `bash_commands` table (written by hive-bash-daemon). Empty when that
|
/// `bash_commands` table (written by hive-bash-daemon). Empty when that
|
||||||
/// capture hasn't run for this agent (table absent).
|
/// capture hasn't run for this agent (table absent).
|
||||||
bash: HashMap<String, u64>,
|
bash: HashMap<String, u64>,
|
||||||
|
/// Skill (fully-qualified `plugin:skill-name`) → invocation count,
|
||||||
|
/// unpacked from `turn_stats.tool_call_breakdown_json`'s `Skill:*`
|
||||||
|
/// entries. Empty when no skill was invoked in the window.
|
||||||
|
skills: HashMap<String, u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
#[allow(
|
||||||
|
|
@ -246,7 +255,8 @@ fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<A
|
||||||
conn.busy_timeout(Duration::from_millis(500))?;
|
conn.busy_timeout(Duration::from_millis(500))?;
|
||||||
let mut stmt = conn.prepare(
|
let mut stmt = conn.prepare(
|
||||||
"SELECT model, input_tokens, output_tokens,
|
"SELECT model, input_tokens, output_tokens,
|
||||||
cache_read_input_tokens, cache_creation_input_tokens
|
cache_read_input_tokens, cache_creation_input_tokens,
|
||||||
|
tool_call_breakdown_json
|
||||||
FROM turn_stats
|
FROM turn_stats
|
||||||
WHERE started_at >= ?1",
|
WHERE started_at >= ?1",
|
||||||
)?;
|
)?;
|
||||||
|
|
@ -258,10 +268,11 @@ fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<A
|
||||||
u64_from_i64(row.get::<_, i64>(2)?),
|
u64_from_i64(row.get::<_, i64>(2)?),
|
||||||
u64_from_i64(row.get::<_, i64>(3)?),
|
u64_from_i64(row.get::<_, i64>(3)?),
|
||||||
u64_from_i64(row.get::<_, i64>(4)?),
|
u64_from_i64(row.get::<_, i64>(4)?),
|
||||||
|
row.get::<_, Option<String>>(5)?,
|
||||||
))
|
))
|
||||||
})?;
|
})?;
|
||||||
for r in rows {
|
for r in rows {
|
||||||
let (model, input, output, cache_read, cache_creation) = r?;
|
let (model, input, output, cache_read, cache_creation, breakdown_json) = r?;
|
||||||
agg.turns += 1;
|
agg.turns += 1;
|
||||||
agg.input = agg.input.saturating_add(input);
|
agg.input = agg.input.saturating_add(input);
|
||||||
agg.output = agg.output.saturating_add(output);
|
agg.output = agg.output.saturating_add(output);
|
||||||
|
|
@ -280,11 +291,31 @@ fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<A
|
||||||
/ 1_000_000.0;
|
/ 1_000_000.0;
|
||||||
}
|
}
|
||||||
*agg.models.entry(model).or_insert(0) += 1;
|
*agg.models.entry(model).or_insert(0) += 1;
|
||||||
|
if let Some(json) = breakdown_json {
|
||||||
|
for (skill, count) in skills_from_breakdown(&json) {
|
||||||
|
*agg.skills.entry(skill).or_insert(0) += count;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
agg.bash = read_bash_heads(&conn, from);
|
agg.bash = read_bash_heads(&conn, from);
|
||||||
Ok(agg)
|
Ok(agg)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Pull the `Skill:<skill>` entries out of one turn's
|
||||||
|
/// `tool_call_breakdown_json` blob (see `hive_agent::events::observe_stream`
|
||||||
|
/// for how they're written), stripping the `Skill:` marker so the rollup
|
||||||
|
/// keys by the bare (fully-qualified `plugin:skill-name`) skill identifier.
|
||||||
|
/// Malformed/absent JSON yields nothing rather than erroring — one bad row
|
||||||
|
/// shouldn't drop a whole agent from the rollup.
|
||||||
|
fn skills_from_breakdown(json: &str) -> Vec<(String, u64)> {
|
||||||
|
let Ok(map) = serde_json::from_str::<HashMap<String, u64>>(json) else {
|
||||||
|
return Vec::new();
|
||||||
|
};
|
||||||
|
map.into_iter()
|
||||||
|
.filter_map(|(k, v)| k.strip_prefix("Skill:").map(|s| (s.to_owned(), v)))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
/// Tally normalised bash-command heads from the agent's `bash_commands`
|
/// Tally normalised bash-command heads from the agent's `bash_commands`
|
||||||
/// table (`ts INTEGER, head TEXT`, written by hive-bash-daemon) over
|
/// table (`ts INTEGER, head TEXT`, written by hive-bash-daemon) over
|
||||||
/// `[from, now]`. Best-effort + isolated from `read_agent`'s error path:
|
/// `[from, now]`. Best-effort + isolated from `read_agent`'s error path:
|
||||||
|
|
@ -329,6 +360,7 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
|
||||||
let mut agents: Vec<AgentRollup> = Vec::new();
|
let mut agents: Vec<AgentRollup> = Vec::new();
|
||||||
let mut model_mix: HashMap<String, u64> = HashMap::new();
|
let mut model_mix: HashMap<String, u64> = HashMap::new();
|
||||||
let mut bash_mix: HashMap<String, u64> = HashMap::new();
|
let mut bash_mix: HashMap<String, u64> = HashMap::new();
|
||||||
|
let mut skill_mix: HashMap<String, u64> = HashMap::new();
|
||||||
let mut total_turns = 0u64;
|
let mut total_turns = 0u64;
|
||||||
let mut total_input = 0u64;
|
let mut total_input = 0u64;
|
||||||
let mut total_output = 0u64;
|
let mut total_output = 0u64;
|
||||||
|
|
@ -365,6 +397,9 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
|
||||||
for (h, c) in &agg.bash {
|
for (h, c) in &agg.bash {
|
||||||
*bash_mix.entry(h.clone()).or_insert(0) += c;
|
*bash_mix.entry(h.clone()).or_insert(0) += c;
|
||||||
}
|
}
|
||||||
|
for (s, c) in &agg.skills {
|
||||||
|
*skill_mix.entry(s.clone()).or_insert(0) += c;
|
||||||
|
}
|
||||||
agents.push(AgentRollup {
|
agents.push(AgentRollup {
|
||||||
name: name.into_string(),
|
name: name.into_string(),
|
||||||
turns: agg.turns,
|
turns: agg.turns,
|
||||||
|
|
@ -393,6 +428,14 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
|
||||||
bash_mix.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.key.cmp(&b.key)));
|
bash_mix.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.key.cmp(&b.key)));
|
||||||
bash_mix.truncate(10);
|
bash_mix.truncate(10);
|
||||||
|
|
||||||
|
// Busiest skills first, capped to a top-10 list — same shape as bash_mix.
|
||||||
|
let mut skill_mix: Vec<KeyCount> = skill_mix
|
||||||
|
.into_iter()
|
||||||
|
.map(|(key, count)| KeyCount { key, count })
|
||||||
|
.collect();
|
||||||
|
skill_mix.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.key.cmp(&b.key)));
|
||||||
|
skill_mix.truncate(10);
|
||||||
|
|
||||||
HiveStats {
|
HiveStats {
|
||||||
window: window.label(),
|
window: window.label(),
|
||||||
from,
|
from,
|
||||||
|
|
@ -407,6 +450,7 @@ pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
|
||||||
agents,
|
agents,
|
||||||
model_mix,
|
model_mix,
|
||||||
bash_mix,
|
bash_mix,
|
||||||
|
skill_mix,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -447,4 +491,28 @@ mod tests {
|
||||||
let conn = Connection::open_in_memory().unwrap();
|
let conn = Connection::open_in_memory().unwrap();
|
||||||
assert!(read_bash_heads(&conn, 0).is_empty());
|
assert!(read_bash_heads(&conn, 0).is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `Skill:<skill>` entries are unpacked and the marker stripped; a
|
||||||
|
/// non-skill tool entry in the same blob is ignored.
|
||||||
|
#[test]
|
||||||
|
fn skills_from_breakdown_strips_marker_and_ignores_other_tools() {
|
||||||
|
let json = r#"{"Skill:base:async-task-hygiene":3,"Read":7}"#;
|
||||||
|
let mut got = skills_from_breakdown(json);
|
||||||
|
got.sort();
|
||||||
|
assert_eq!(got, vec![("base:async-task-hygiene".to_owned(), 3)]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Malformed JSON degrades to empty rather than erroring — one bad row
|
||||||
|
/// shouldn't drop a whole agent from the rollup.
|
||||||
|
#[test]
|
||||||
|
fn skills_from_breakdown_malformed_json_is_empty() {
|
||||||
|
assert!(skills_from_breakdown("not json").is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// No `Skill:*` keys at all (an agent that hasn't triggered one yet)
|
||||||
|
/// is also empty, not an error.
|
||||||
|
#[test]
|
||||||
|
fn skills_from_breakdown_no_skills_is_empty() {
|
||||||
|
assert!(skills_from_breakdown(r#"{"Read":1,"Edit":2}"#).is_empty());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue