diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 73d3b60a..8749d221 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -377,12 +377,17 @@ of their own — see [ACP-reported cost](#acp-reported-cost). | `hyperhive.agent.loose_ends.threads` | — | gauge | none | | `hyperhive.agent.loose_ends.reminders` | — | gauge | none | | `hyperhive.agent.claude_md.lines` | — | gauge | none — recorded from the `CLAUDE.md`-size watch's own ~15-minute tick, **not** per turn like the rows above | +| `hyperhive.agent.subagent_role.lines` | — | gauge | `role` (the file stem) — one series per role file in `subagent_roles/`, on the same ~15-minute tick | | `hyperhive.agent.claude_usage.percent` | `%` | gauge | `window` (`five_hour`, `seven_day`, … as the usage endpoint names them) — polled every 5 minutes, **not** per turn | | `hyperhive.agent.claude_usage.resets_at` | `s` | gauge | `window` — unix seconds at which that window resets; same 5-minute poll | | `hyperhive.agent.context.used` | `{token}` | gauge | none — tokens in the context window at turn end, the numerator of the web UI's ctx% | | `hyperhive.agent.context.size` | `{token}` | gauge | none — the context window those tokens fill, ctx%'s denominator | | `hyperhive.agent.cost.usage` | — | counter | `model`, `currency` — ACP agents only, see below | +An agent with no `subagent_roles/` directory exports no +`subagent_role.lines` series. A role file deleted while the harness runs +keeps its last value until the harness restarts. + For the two `claude_usage` gauges the harness polls the Claude subscription usage endpoint (`GET /api/oauth/usage`, the one behind claude's own `/usage`) with the OAuth access token from the agent's own diff --git a/hive-agent/src/main.rs b/hive-agent/src/main.rs index 84577fe2..fed0a357 100644 --- a/hive-agent/src/main.rs +++ b/hive-agent/src/main.rs @@ -30,6 +30,7 @@ mod schedule_timer; mod serve_common; mod state_entry_watch; mod stats; +mod subagent_role_watch; mod swarm_agent_icon; mod swarm_agent_state; mod swarm_inference; @@ -519,6 +520,7 @@ async fn serve_main(socket: &Path, poll_ms: u64) -> Result<()> { // hive-c0re, because the files are agent-owned — see `vacuum` module docs. tokio::spawn(crate::vacuum::run()); tokio::spawn(claude_usage_watch::run()); + tokio::spawn(subagent_role_watch::run()); // Log web_ui::serve's error instead of dropping it. A bare // `tokio::spawn(web_ui::serve(...))` discards the JoinHandle, so // any Err (e.g. EACCES from `bind_unix` when HIVE_WEB_SOCKET points diff --git a/hive-agent/src/otel_turn_metrics.rs b/hive-agent/src/otel_turn_metrics.rs index 2039f725..6cf96ff6 100644 --- a/hive-agent/src/otel_turn_metrics.rs +++ b/hive-agent/src/otel_turn_metrics.rs @@ -18,12 +18,12 @@ //! from `handle_turn`. The `PeriodicReader` still batches + exports on its //! own interval; only the recording is event-driven, not the export. //! -//! [`record_claude_md_lines`] is the one exception, sharing this module's -//! exporter setup rather than [`record`]'s per-turn call site — `CLAUDE.md` -//! size *is* continuously live (like hive-c0re's cgroup values), so -//! [`crate::claude_md_watch`] records it from its own periodic tick instead. -//! [`record_claude_usage`] is the same kind of exception, fed from -//! [`crate::claude_usage_watch`]'s poll of the subscription usage endpoint. +//! Three exceptions share this module's exporter but not [`record`]'s +//! per-turn call site: their values are continuously live (like hive-c0re's +//! cgroup values), so each is recorded from its own watch's periodic tick — +//! [`record_claude_md_lines`] from [`crate::claude_md_watch`], +//! [`record_subagent_role_lines`] from [`crate::subagent_role_watch`], and +//! [`record_claude_usage`] from [`crate::claude_usage_watch`]. //! //! Resource attributes (`service.name`, `agent`, `hive`, `swarm`) are picked //! up automatically by the SDK from the container-wide `OTEL_RESOURCE_ATTRIBUTES` @@ -58,6 +58,9 @@ struct Instruments { /// hive-c0re's container gauges), not a once-per-turn one, so it has /// its own entry point rather than riding `record`'s per-turn call. claude_md_lines: Gauge, + /// Recorded from [`crate::subagent_role_watch`]'s own tick, keyed by + /// `role`. + subagent_role_lines: Gauge, /// Recorded from [`crate::claude_usage_watch`]'s own tick, keyed by /// `window`. claude_usage_percent: Gauge, @@ -120,6 +123,17 @@ pub fn record_claude_md_lines(lines: u64) { inst.claude_md_lines.record(lines, &[]); } +/// Record one subagent role file's line count, keyed by `role` (the file +/// stem). Called from [`crate::subagent_role_watch`]'s periodic tick. +/// No-op when OTEL isn't configured, same as [`record`]. +pub fn record_subagent_role_lines(role: String, lines: u64) { + let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else { + return; + }; + inst.subagent_role_lines + .record(lines, &[KeyValue::new("role", role)]); +} + /// Record one subscription usage window (`five_hour`, `seven_day`, …): /// `percent` is 0-100, `resets_at` unix seconds. No-op when OTEL isn't /// configured, same as [`record`]. @@ -182,6 +196,9 @@ fn build() -> Option { .u64_gauge("hyperhive.agent.loose_ends.reminders") .build(), claude_md_lines: meter.u64_gauge("hyperhive.agent.claude_md.lines").build(), + subagent_role_lines: meter + .u64_gauge("hyperhive.agent.subagent_role.lines") + .build(), claude_usage_percent: meter .f64_gauge("hyperhive.agent.claude_usage.percent") .with_unit("%") diff --git a/hive-agent/src/subagent_role_watch.rs b/hive-agent/src/subagent_role_watch.rs new file mode 100644 index 00000000..8901665a --- /dev/null +++ b/hive-agent/src/subagent_role_watch.rs @@ -0,0 +1,93 @@ +//! Subagent role-file size watch: feeds each role file's line count into +//! an OTEL gauge, one series per role, on the same ~15-minute tick as +//! [`crate::claude_md_watch`]. Metric only — no todo, no threshold. +//! +//! A role is read into every subagent dispatch that names it, so one +//! role's growth is the signal; a per-file series shows which role grew, +//! where a total would not. +//! +//! Role files are `.md` directly under `subagent_roles/` in the +//! agent's state dir — the directory `hive-subagent-mcp`'s +//! `paths::subagent_roles_dir` loads them from. An absent directory is +//! the ordinary state for an agent with no roles and exports nothing. +//! +//! A role file deleted while the harness runs keeps its last value until +//! the harness restarts: a synchronous gauge has no way to retract a +//! series. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +/// Same cadence as `claude_md_watch`. +const CHECK_INTERVAL: Duration = Duration::from_mins(15); +/// Extension every role file carries; the role name is the file stem. +const ROLE_EXT: &str = "md"; + +/// Background loop: periodically record every role file's line count. +/// Detached task — runs for the harness's lifetime; errors are logged, +/// never fatal. +pub async fn run() { + loop { + tokio::time::sleep(CHECK_INTERVAL).await; + match tokio::task::spawn_blocking(|| role_line_counts(&roles_dir())).await { + Ok(counts) => { + for (role, lines) in counts { + crate::otel_turn_metrics::record_subagent_role_lines(role, lines); + } + } + Err(e) => tracing::warn!(error = %e, "subagent role watch probe panicked"), + } + } +} + +/// The agent's role directory, resolved the same way as +/// `hive-subagent-mcp`'s `paths::subagent_roles_dir`. +fn roles_dir() -> PathBuf { + crate::paths::state_dir().join("subagent_roles") +} + +/// `(role, line count)` for every readable `.md` in `dir`. Empty when +/// `dir` is absent or unreadable — no roles yet is not a warning. +fn role_line_counts(dir: &Path) -> Vec<(String, u64)> { + std::fs::read_dir(dir) + .into_iter() + .flatten() + .flatten() + .filter_map(|entry| { + let path = entry.path(); + if path.extension()?.to_str()? != ROLE_EXT { + return None; + } + let role = path.file_stem()?.to_str()?.to_owned(); + let contents = std::fs::read_to_string(&path).ok()?; + Some((role, contents.lines().count() as u64)) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn one_series_per_role_file_labelled_by_stem() { + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::write(dir.path().join("reviewer.md"), "one\ntwo\nthree\n").expect("write"); + std::fs::write(dir.path().join("scout.md"), "a\nb\nc\nd\ne").expect("write"); + // Not a role file: wrong extension. + std::fs::write(dir.path().join("notes.txt"), "x\n").expect("write"); + + let mut counts = role_line_counts(dir.path()); + counts.sort(); + assert_eq!( + counts, + vec![("reviewer".to_owned(), 3), ("scout".to_owned(), 5)] + ); + } + + #[test] + fn absent_dir_gives_no_series() { + let dir = tempfile::tempdir().expect("tempdir"); + assert!(role_line_counts(&dir.path().join("subagent_roles")).is_empty()); + } +}