//! Per-turn OTEL metric export for the fields Claude Code's own built-in //! OTEL integration cannot know about. Claude's own export already covers //! token usage and cost (see `docs/4-implementation/scheduler/observability.md`) — //! duplicating those here under different metric names would just give a //! collector two disagreeing series for the same number. This module emits //! only the harness-only concepts: wall-clock turn duration as *this harness* //! measures it, what woke the turn, its outcome classification, the loose-ends //! backlog, sessions and their first turn's cost, context-window fill, the //! tools a turn called (claude exports no tool-call counter), bash commands, //! reminders, and the cost an ACP agent reports — claude's runtime reports none. //! //! Synchronous instruments, not observable ones: unlike hive-c0re's //! container-resource gauges (`hive-c0re/src/stats/otel_metrics.rs`), which //! poll a periodically-refreshed snapshot because the underlying cgroup value //! is continuously live, every value here only exists at one instant — the //! moment a turn ends — so it's recorded directly (`counter.add`, //! `histogram.record`, `gauge.record`) from [`record`], called once per turn //! from `handle_turn`. The `PeriodicReader` still batches + exports on its //! own interval; only the recording is event-driven, not the export. //! //! Four exceptions share this module's exporter but not [`record`]'s //! per-turn call site: their values are continuously live (like hive-c0re's //! cgroup values), so each is recorded from its own watch's tick: //! [`record_claude_md_lines`] ([`crate::claude_md_watch`]), [`record_subagent_role_lines`] //! ([`crate::subagent_role_watch`]), [`record_claude_usage`] ([`crate::claude_usage_watch`]), //! and [`record_swarm_queue_connected`] ([`crate::swarm_queue_gauge`]). //! //! Resource attributes (`service.name`, `agent`, `hive`, `swarm`) are picked //! up automatically by the SDK from the container-wide `OTEL_RESOURCE_ATTRIBUTES` //! env var (same mechanism `hive-metric` relies on) — nothing to set here. use std::collections::HashMap; use std::sync::OnceLock; use std::sync::atomic::{AtomicI64, Ordering}; use std::time::Duration; use opentelemetry::KeyValue; use opentelemetry::metrics::{Counter, Gauge, Histogram, MeterProvider as _}; use opentelemetry_otlp::{MetricExporter, Protocol, WithExportConfig}; use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider}; use crate::turn_stats::TurnStatRow; /// Export cadence. Matches `hive-c0re`'s container-metrics exporter so the /// two batch on the same rhythm. const EXPORT_INTERVAL: Duration = Duration::from_mins(1); /// Bucket upper bounds of the turn-duration histogram, in milliseconds. The /// SDK's default bounds stop at 10 s, but a turn takes about 24 s on average, /// so with them every p95 read a flat 10 s. These run from half a second to an /// hour. const TURN_DURATION_BOUNDS_MS: [f64; 13] = [ 500.0, 1_000.0, 2_500.0, 5_000.0, 10_000.0, 20_000.0, 30_000.0, 60_000.0, 120_000.0, 300_000.0, 600_000.0, 1_800_000.0, 3_600_000.0, ]; struct Instruments { /// Kept alive for the process lifetime — the `PeriodicReader` only /// exports while its provider lives. _provider: SdkMeterProvider, turn_duration_ms: Histogram, turn_count: Counter, session_count: Counter, open_threads: Gauge, open_reminders: Gauge, /// Recorded from [`crate::todos_gauge`]'s own tick: todos change between /// turns. open_todos: Gauge, /// Recorded from [`crate::claude_md_watch`]'s own tick, not from /// [`record`] — `AGENTS.md` size is a continuously-live value (like /// hive-c0re's container gauges), not a once-per-turn one, so it has /// its own entry point rather than riding `record`'s per-turn call. claude_md_lines: Gauge, /// Recorded from [`crate::subagent_role_watch`]'s own tick, keyed by /// `role`. subagent_role_lines: Gauge, /// Recorded from [`crate::claude_usage_watch`]'s own tick, keyed by /// `window`. claude_usage_percent: Gauge, claude_usage_resets_at: Gauge, context_used: Gauge, context_size: Gauge, /// Keyed by `model` and `currency`: an ACP agent reports cost in a /// currency of its choosing. reported_cost: Counter, /// Tool calls made, keyed by `tool` (`Skill:` for a skill). tool_calls: Counter, /// Shell commands run by bash tasks, keyed by `command`: the normalised /// head the bash daemon records, not the full command line. bash_commands: Counter, /// Reminders scheduled and delivered, keyed by `event`. reminders: Counter, /// Input tokens of the first turn of a fresh claude session: what the /// system prompt and `AGENTS.md` cost before anything is cached. first_turn_context: Gauge, /// `1` while the swarm-queue client reports `Connected`, else `0`; recorded /// from [`crate::swarm_queue_gauge`]'s own tick. swarm_queue_connected: Gauge, /// Publishes dropped because the queue client was not connected, keyed by /// `site` (the publisher that dropped it). swarm_queue_publish_skipped: Counter, } /// Lazily built on the first call to [`record`]. `None` when OTEL isn't /// configured (`OTEL_EXPORTER_OTLP_ENDPOINT` unset/empty) or the exporter /// failed to build — either way `record` becomes a cheap no-op. static INSTRUMENTS: OnceLock> = OnceLock::new(); /// Record one turn's harness-only metrics. No-op (and cheap — one /// `OnceLock` read after the first call) when OTEL isn't configured. /// /// `fresh_session` is the harness's own "did this turn start a brand-new /// claude session" flag (`Bus::take_fresh_session`) — pass it through rather /// than re-deriving it here, since the harness only gets to observe it once /// per turn (the same flag gates minting a `turn_stats` `sessions` row). pub fn record(row: &TurnStatRow, fresh_session: bool) { let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else { return; }; let attrs = [ KeyValue::new("wake_from", row.wake_from.clone()), KeyValue::new("result_kind", row.result_kind), KeyValue::new("model", row.model.clone()), ]; // `try_from` rather than `as` — `duration_ms` is never negative in // practice (it's `Instant::elapsed().as_millis()`), but a fallback of 0 // beats a lossy cast or a panic if that ever stops holding. inst.turn_duration_ms .record(u64::try_from(row.duration_ms).unwrap_or(0), &attrs); inst.turn_count.add(1, &attrs); if let Some(threads) = row.open_threads_count { inst.open_threads.record(threads, &[]); } if let Some(reminders) = row.open_reminders_count { inst.open_reminders.record(reminders, &[]); } if fresh_session { inst.session_count .add(1, &[KeyValue::new("model", row.model.clone())]); inst.first_turn_context.record(row.input_tokens, &[]); } for (tool, calls) in tool_call_counts(row.tool_call_breakdown_json.as_deref()) { inst.tool_calls.add(calls, &[KeyValue::new("tool", tool)]); } record_new_bash_commands(inst, row.started_at); } /// The per-tool call counts a turn stored in `tool_call_breakdown_json`; a /// turn with no tool calls, or a value that does not parse, has none. fn tool_call_counts(json: Option<&str>) -> Vec<(String, u64)> { json.and_then(|j| serde_json::from_str::>(j).ok()) .map(|m| m.into_iter().collect()) .unwrap_or_default() } /// Unix seconds up to which bash commands are already counted; `0` before the /// first turn of this run. static BASH_COUNTED_UNTIL: AtomicI64 = AtomicI64::new(0); /// Count the bash commands recorded since the last call. The first call after /// the harness starts counts only commands newer than `turn_started_at`, so a /// restart does not count the whole history again. fn record_new_bash_commands(inst: &Instruments, turn_started_at: i64) { let now = chrono::Utc::now().timestamp(); let mut since = BASH_COUNTED_UNTIL.swap(now, Ordering::Relaxed); if since == 0 { since = turn_started_at.saturating_sub(1); } for (command, n) in crate::stats::bash_heads_since(since) { inst.bash_commands .add(n, &[KeyValue::new("command", command)]); } } /// Count one reminder event, `scheduled` or `delivered`. No-op when OTEL isn't /// configured, same as [`record`]. pub fn record_reminder(event: &'static str) { let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else { return; }; inst.reminders.add(1, &[KeyValue::new("event", event)]); } /// Record the current `AGENTS.md` line count. Called from /// [`crate::claude_md_watch`]'s periodic tick (every ~15 minutes, not /// per turn) — see the field doc on [`Instruments::claude_md_lines`] for /// why this doesn't ride [`record`]. No-op when OTEL isn't configured, /// same as `record`. pub fn record_claude_md_lines(lines: u64) { let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else { return; }; inst.claude_md_lines.record(lines, &[]); } /// Record one subagent role file's line count, keyed by `role` (the file /// stem). Called from [`crate::subagent_role_watch`]'s periodic tick. /// No-op when OTEL isn't configured, same as [`record`]. pub fn record_subagent_role_lines(role: String, lines: u64) { let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else { return; }; inst.subagent_role_lines .record(lines, &[KeyValue::new("role", role)]); } /// Record the number of open todos. No-op when OTEL isn't configured, same /// as [`record`]. pub fn record_open_todos(count: u64) { let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else { return; }; inst.open_todos.record(count, &[]); } /// Attributes of a usage series: `window`, plus `account` when the login is /// known. An unknown account omits the label rather than sending an empty one. fn claude_usage_attrs(account: Option<&str>, window: &str) -> Vec { let mut attrs = vec![KeyValue::new("window", window.to_owned())]; if let Some(account) = account.filter(|a| !a.is_empty()) { attrs.push(KeyValue::new("account", account.to_owned())); } attrs } /// Record one subscription usage window (`five_hour`, `seven_day`, …): /// `percent` is 0-100, `resets_at` unix seconds. `account` is the opaque /// id of the claude login, shared by every agent on that login. No-op when /// OTEL isn't configured, same as [`record`]. pub fn record_claude_usage( account: Option<&str>, window: &str, percent: f64, resets_at: Option, ) { let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else { return; }; let attrs = claude_usage_attrs(account, window); inst.claude_usage_percent.record(percent, &attrs); if let Some(resets_at) = resets_at { inst.claude_usage_resets_at.record(resets_at, &attrs); } } /// Record whether the swarm-queue client is connected (`1`) or not (`0`). No-op /// when OTEL isn't configured, same as [`record`]. pub fn record_swarm_queue_connected(connected: bool) { let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else { return; }; inst.swarm_queue_connected.record(u64::from(connected), &[]); } /// Count one publish dropped for want of a queue connection, keyed by `site`. /// No-op when OTEL isn't configured, same as [`record`]. pub fn record_swarm_queue_publish_skipped(site: &'static str) { let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else { return; }; inst.swarm_queue_publish_skipped .add(1, &[KeyValue::new("site", site)]); } /// Record the context window's fill at turn end: `used` tokens of `size`, /// the two numbers the web UI's ctx% divides. No-op when OTEL isn't /// configured, same as [`record`]. pub fn record_context(used: u64, size: u64) { let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else { return; }; inst.context_used.record(used, &[]); inst.context_size.record(size, &[]); } /// Add what an ACP agent reported spending since the last call. No-op when /// OTEL isn't configured, same as [`record`]. pub fn record_reported_cost(model: &str, cost: &hive_runtime::ReportedCost) { let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else { return; }; let attrs = [ KeyValue::new("model", model.to_owned()), KeyValue::new("currency", cost.currency.clone()), ]; inst.reported_cost.add(cost.amount, &attrs); } fn build() -> Option { if !enabled() { tracing::debug!("otel turn-metrics: no endpoint configured, exporter disabled"); return None; } let interval = EXPORT_INTERVAL; match build_provider(interval) { Ok(provider) => { tracing::info!(?interval, "otel turn-metrics: exporter enabled"); let meter = provider.meter("hyperhive.turn_stats"); swarm_secret_client::cert_metrics::register(&meter, "agent"); Some(Instruments { turn_duration_ms: meter .u64_histogram("hyperhive.agent.turn.duration") .with_unit("ms") .with_boundaries(TURN_DURATION_BOUNDS_MS.to_vec()) .build(), turn_count: meter.u64_counter("hyperhive.agent.turn.count").build(), session_count: meter.u64_counter("hyperhive.agent.session.count").build(), open_threads: meter .u64_gauge("hyperhive.agent.loose_ends.threads") .build(), open_reminders: meter .u64_gauge("hyperhive.agent.loose_ends.reminders") .build(), open_todos: meter.u64_gauge("hyperhive.agent.todos.open").build(), claude_md_lines: meter.u64_gauge("hyperhive.agent.claude_md.lines").build(), subagent_role_lines: meter .u64_gauge("hyperhive.agent.subagent_role.lines") .build(), claude_usage_percent: meter .f64_gauge("hyperhive.agent.claude_usage.percent") .with_unit("%") .build(), claude_usage_resets_at: meter .u64_gauge("hyperhive.agent.claude_usage.resets_at") .with_unit("s") .build(), context_used: meter .u64_gauge("hyperhive.agent.context.used") .with_unit("{token}") .build(), context_size: meter .u64_gauge("hyperhive.agent.context.size") .with_unit("{token}") .build(), reported_cost: meter.f64_counter("hyperhive.agent.cost.usage").build(), tool_calls: meter.u64_counter("hyperhive.agent.tool.calls").build(), bash_commands: meter.u64_counter("hyperhive.agent.bash.commands").build(), reminders: meter.u64_counter("hyperhive.agent.reminders").build(), first_turn_context: meter .u64_gauge("hyperhive.agent.first_turn.context") .with_unit("{token}") .build(), swarm_queue_connected: meter .u64_gauge("hyperhive.agent.swarm_queue.connected") .build(), swarm_queue_publish_skipped: meter .u64_counter("hyperhive.agent.swarm_queue.publish_skipped") .build(), _provider: provider, }) } Err(e) => { tracing::warn!(error = ?e, "otel turn-metrics: exporter init failed"); None } } } fn build_provider(interval: Duration) -> anyhow::Result { // http/json, no explicit endpoint/resource: same rationale as // `hive-c0re/src/stats/otel_metrics.rs` for the transport (the blocking // OTLP client is required — `PeriodicReader` drives export from a // background thread with no Tokio reactor), and same rationale as // `hive-metric` for the resource (the SDK reads `OTEL_RESOURCE_ATTRIBUTES` // — set container-wide, see `docs/4-implementation/scheduler/observability.md::Built-in resource // labels` — on its own; passing an explicit `Resource` here would need to // duplicate `agent`/`hive`/`swarm` this crate has no other reason to know). let exporter = MetricExporter::builder() .with_http() .with_protocol(Protocol::HttpJson) .build()?; let reader = PeriodicReader::builder(exporter) .with_interval(interval) .build(); Ok(SdkMeterProvider::builder().with_reader(reader).build()) } /// `OTEL_EXPORTER_OTLP_ENDPOINT` non-empty — the same enable signal every /// other exporter in this workspace uses (see `hive-c0re/src/stats/ /// otel_metrics.rs::endpoint` for why this exact variable and not a /// hyperhive-specific one: it's also the variable the SDK itself reads to /// build the exporter's URL, so "configured" and "where it goes" cannot /// disagree). pub fn enabled() -> bool { std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").is_ok_and(|s| !s.trim().is_empty()) } #[cfg(test)] mod tests { use opentelemetry::KeyValue; use super::{TURN_DURATION_BOUNDS_MS, claude_usage_attrs, tool_call_counts}; #[test] fn a_turns_stored_tool_breakdown_becomes_per_tool_counts() { let mut got = tool_call_counts(Some(r#"{"Bash":3,"Skill:base:pr-review":1}"#)); got.sort(); assert_eq!( got, vec![ ("Bash".to_owned(), 3), ("Skill:base:pr-review".to_owned(), 1) ] ); // No tool calls, and a value that is not a count map, both count nothing. assert!(tool_call_counts(None).is_empty()); assert!(tool_call_counts(Some("not json")).is_empty()); } #[test] fn the_duration_bounds_ascend_and_reach_past_the_old_ten_second_ceiling() { // The SDK drops bounds that are not strictly ascending, which would // silently bring the default buckets (and the flat p95) back. assert!(TURN_DURATION_BOUNDS_MS.windows(2).all(|w| w[0] < w[1])); assert!( TURN_DURATION_BOUNDS_MS .last() .is_some_and(|&b| b > 10_000.0) ); } #[test] fn usage_attrs_carry_account_only_when_known() { let has = |attrs: &[KeyValue], k: &str| attrs.iter().any(|kv| kv.key.as_str() == k); let known = claude_usage_attrs(Some("acct-1"), "seven_day"); assert!(has(&known, "window")); assert!( known .iter() .any(|kv| kv.key.as_str() == "account" && kv.value.as_str() == "acct-1") ); assert!(!has(&claude_usage_attrs(None, "seven_day"), "account")); assert!(!has(&claude_usage_attrs(Some(""), "seven_day"), "account")); } }