Compare commits
5 changed files with 18 additions and 222 deletions
3
Cargo.lock
generated
3
Cargo.lock
generated
|
|
@ -1593,9 +1593,6 @@ dependencies = [
|
||||||
"hyper",
|
"hyper",
|
||||||
"hyper-util",
|
"hyper-util",
|
||||||
"libc",
|
"libc",
|
||||||
"opentelemetry",
|
|
||||||
"opentelemetry-otlp",
|
|
||||||
"opentelemetry_sdk",
|
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"rmcp",
|
"rmcp",
|
||||||
"rusqlite",
|
"rusqlite",
|
||||||
|
|
|
||||||
|
|
@ -219,32 +219,6 @@ Cadence follows `metricIntervalMs` (default 60s). Transport is OTLP/HTTP
|
||||||
first hop is unauthenticated for every producer on this host, and the upstream
|
first hop is unauthenticated for every producer on this host, and the upstream
|
||||||
credential stays on the swarm tier.
|
credential stays on the swarm tier.
|
||||||
|
|
||||||
## Agent-emitted per-turn metrics (`hive-agent`)
|
|
||||||
|
|
||||||
When OTEL is enabled, the harness itself (`hive-agent`) exports one small set
|
|
||||||
of metrics per claude turn, recorded the moment the turn ends (not polled).
|
|
||||||
These are deliberately the fields Claude Code's own built-in export (see
|
|
||||||
above) can't know about — the harness's own wall-clock timing, what woke the
|
|
||||||
turn, its own outcome classification, the loose-ends backlog, and session
|
|
||||||
boundaries. Token usage, cost, and tool-call counts are **not** duplicated
|
|
||||||
here; that's already covered by Claude's own export.
|
|
||||||
|
|
||||||
| Metric | Unit | Kind | Attributes |
|
|
||||||
|--------|------|------|------------|
|
|
||||||
| `hyperhive.agent.turn.duration` | `ms` | histogram | `wake_from`, `result_kind`, `model` |
|
|
||||||
| `hyperhive.agent.turn.count` | — | counter | `wake_from`, `result_kind`, `model` |
|
|
||||||
| `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) |
|
|
||||||
| `hyperhive.agent.loose_ends.threads` | — | gauge | none |
|
|
||||||
| `hyperhive.agent.loose_ends.reminders` | — | gauge | none |
|
|
||||||
|
|
||||||
Resource attributes (`service.name`, `agent`, `hive`, `swarm`) come from the
|
|
||||||
same container-wide `OTEL_RESOURCE_ATTRIBUTES` as everything else in this
|
|
||||||
section — nothing extra to configure. Cadence follows
|
|
||||||
`HYPERHIVE_OTEL_METRIC_INTERVAL_MS` (default 60s, same variable + default as
|
|
||||||
`hive-c0re`'s container-resource export above) — that only controls how often
|
|
||||||
the batched points are flushed to the collector, not how often they're
|
|
||||||
recorded (every turn, always).
|
|
||||||
|
|
||||||
## Agent-emitted custom metrics (`hive-metric`)
|
## Agent-emitted custom metrics (`hive-metric`)
|
||||||
|
|
||||||
Agents can push arbitrary labeled metrics to the same OTEL collector via the
|
Agents can push arbitrary labeled metrics to the same OTEL collector via the
|
||||||
|
|
|
||||||
|
|
@ -23,9 +23,6 @@ hive-core-agent-sock.workspace = true
|
||||||
hive-sh4re.workspace = true
|
hive-sh4re.workspace = true
|
||||||
hive-sock-client.workspace = true
|
hive-sock-client.workspace = true
|
||||||
libc.workspace = true
|
libc.workspace = true
|
||||||
opentelemetry.workspace = true
|
|
||||||
opentelemetry_sdk.workspace = true
|
|
||||||
opentelemetry-otlp.workspace = true
|
|
||||||
rmcp.workspace = true
|
rmcp.workspace = true
|
||||||
rusqlite.workspace = true
|
rusqlite.workspace = true
|
||||||
schemars.workspace = true
|
schemars.workspace = true
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,6 @@ mod identity;
|
||||||
mod login;
|
mod login;
|
||||||
mod login_session;
|
mod login_session;
|
||||||
mod mcp_config;
|
mod mcp_config;
|
||||||
mod otel_turn_metrics;
|
|
||||||
mod paths;
|
mod paths;
|
||||||
mod plugins;
|
mod plugins;
|
||||||
mod prompt;
|
mod prompt;
|
||||||
|
|
@ -1007,21 +1006,15 @@ async fn handle_turn<S: Surface>(
|
||||||
let tool_calls = bus.take_tool_calls();
|
let tool_calls = bus.take_tool_calls();
|
||||||
let todo_wake_checked =
|
let todo_wake_checked =
|
||||||
(from == "todo").then(|| tool_calls.contains_key("mcp__hyperhive__get_loose_ends"));
|
(from == "todo").then(|| tool_calls.contains_key("mcp__hyperhive__get_loose_ends"));
|
||||||
// Read-and-clear once per turn unconditionally (same rationale as
|
|
||||||
// `take_tool_calls` above — the harness gets exactly one chance to
|
|
||||||
// observe this flag) rather than only when the sqlite sink happens to be
|
|
||||||
// configured: the OTEL turn-metrics exporter's session-boundary counter
|
|
||||||
// needs it too, independent of `stats`.
|
|
||||||
let fresh_session = bus.take_fresh_session();
|
|
||||||
if let Some(stats) = stats {
|
if let Some(stats) = stats {
|
||||||
// Fresh session this turn → mint a `sessions` row and set its id on
|
// Fresh session this turn → mint a `sessions` row and set its id on
|
||||||
// the bus so this turn (and subsequent ones until the next fresh
|
// the bus so this turn (and subsequent ones until the next fresh
|
||||||
// start) stamp `turn_stats.session_id`.
|
// start) stamp `turn_stats.session_id`. Takes the one-shot flag
|
||||||
if fresh_session {
|
// `run_claude` set when it suppressed `--continue`.
|
||||||
|
if bus.take_fresh_session() {
|
||||||
let sid = stats.start_session(started_at, &model_at_start);
|
let sid = stats.start_session(started_at, &model_at_start);
|
||||||
bus.set_session_id(sid);
|
bus.set_session_id(sid);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
let ended_at = chrono::Utc::now().timestamp();
|
let ended_at = chrono::Utc::now().timestamp();
|
||||||
let duration_ms = i64::try_from(started_instant.elapsed().as_millis()).unwrap_or(i64::MAX);
|
let duration_ms = i64::try_from(started_instant.elapsed().as_millis()).unwrap_or(i64::MAX);
|
||||||
let (open_threads, open_reminders) = S::post_turn_counts(socket).await;
|
let (open_threads, open_reminders) = S::post_turn_counts(socket).await;
|
||||||
|
|
@ -1037,12 +1030,6 @@ async fn handle_turn<S: Surface>(
|
||||||
open_threads_count: open_threads,
|
open_threads_count: open_threads,
|
||||||
open_reminders_count: open_reminders,
|
open_reminders_count: open_reminders,
|
||||||
});
|
});
|
||||||
// Harness-only OTEL metrics (duration/wake_from/result_kind/loose-ends/
|
|
||||||
// session boundaries) — independent of the sqlite sink below, and a
|
|
||||||
// cheap no-op when OTEL isn't configured. See `otel_turn_metrics`'s
|
|
||||||
// module doc for why token/cost/tool-count are deliberately not here.
|
|
||||||
otel_turn_metrics::record(&row, fresh_session);
|
|
||||||
if let Some(stats) = stats {
|
|
||||||
stats.record(&row);
|
stats.record(&row);
|
||||||
}
|
}
|
||||||
let pending = S::inbox_unread(socket).await;
|
let pending = S::inbox_unread(socket).await;
|
||||||
|
|
|
||||||
|
|
@ -1,159 +0,0 @@
|
||||||
//! Per-turn OTEL metric export for the fields Claude Code's own built-in
|
|
||||||
//! OTEL integration cannot know about. Claude's own export already covers
|
|
||||||
//! token usage, cost, and tool-call counts (see `docs/observability.md`) —
|
|
||||||
//! duplicating those here under different metric names would just give a
|
|
||||||
//! collector two disagreeing series for the same number. This module emits
|
|
||||||
//! only the harness-only concepts: wall-clock turn duration as *this harness*
|
|
||||||
//! measures it (not Claude's own per-request latency), what woke the turn,
|
|
||||||
//! the harness's own outcome classification, the loose-ends backlog at turn
|
|
||||||
//! end, and session boundaries (fresh vs. `--continue`'d). Scoped this way
|
|
||||||
//! per the tracker discussion on the "emit agent stats as OTEL metrics" issue.
|
|
||||||
//!
|
|
||||||
//! Synchronous instruments, not observable ones: unlike hive-c0re's
|
|
||||||
//! container-resource gauges (`hive-c0re/src/stats/otel_metrics.rs`), which
|
|
||||||
//! poll a periodically-refreshed snapshot because the underlying cgroup value
|
|
||||||
//! is continuously live, every value here only exists at one instant — the
|
|
||||||
//! moment a turn ends — so it's recorded directly (`counter.add`,
|
|
||||||
//! `histogram.record`, `gauge.record`) from [`record`], called once per turn
|
|
||||||
//! from `handle_turn`. The `PeriodicReader` still batches + exports on its
|
|
||||||
//! own interval; only the recording is event-driven, not the export.
|
|
||||||
//!
|
|
||||||
//! Resource attributes (`service.name`, `agent`, `hive`, `swarm`) are picked
|
|
||||||
//! up automatically by the SDK from the container-wide `OTEL_RESOURCE_ATTRIBUTES`
|
|
||||||
//! env var (same mechanism `hive-metric` relies on) — nothing to set here.
|
|
||||||
|
|
||||||
use std::sync::OnceLock;
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use opentelemetry::KeyValue;
|
|
||||||
use opentelemetry::metrics::{Counter, Gauge, Histogram, MeterProvider as _};
|
|
||||||
use opentelemetry_otlp::{MetricExporter, Protocol, WithExportConfig};
|
|
||||||
use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider};
|
|
||||||
|
|
||||||
use crate::turn_stats::TurnStatRow;
|
|
||||||
|
|
||||||
/// Default export cadence when `HYPERHIVE_OTEL_METRIC_INTERVAL_MS` is unset.
|
|
||||||
/// Matches `hive-c0re`'s container-metrics default so the two exporters
|
|
||||||
/// batch on the same rhythm by default.
|
|
||||||
const DEFAULT_INTERVAL: Duration = Duration::from_mins(1);
|
|
||||||
|
|
||||||
struct Instruments {
|
|
||||||
/// Kept alive for the process lifetime — the `PeriodicReader` only
|
|
||||||
/// exports while its provider lives.
|
|
||||||
_provider: SdkMeterProvider,
|
|
||||||
turn_duration_ms: Histogram<u64>,
|
|
||||||
turn_count: Counter<u64>,
|
|
||||||
session_count: Counter<u64>,
|
|
||||||
open_threads: Gauge<u64>,
|
|
||||||
open_reminders: Gauge<u64>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Lazily built on the first call to [`record`]. `None` when OTEL isn't
|
|
||||||
/// configured (`OTEL_EXPORTER_OTLP_ENDPOINT` unset/empty) or the exporter
|
|
||||||
/// failed to build — either way `record` becomes a cheap no-op.
|
|
||||||
static INSTRUMENTS: OnceLock<Option<Instruments>> = OnceLock::new();
|
|
||||||
|
|
||||||
/// Record one turn's harness-only metrics. No-op (and cheap — one
|
|
||||||
/// `OnceLock` read after the first call) when OTEL isn't configured.
|
|
||||||
///
|
|
||||||
/// `fresh_session` is the harness's own "did this turn start a brand-new
|
|
||||||
/// claude session" flag (`Bus::take_fresh_session`) — pass it through rather
|
|
||||||
/// than re-deriving it here, since the harness only gets to observe it once
|
|
||||||
/// per turn (the same flag gates minting a `turn_stats` `sessions` row).
|
|
||||||
pub fn record(row: &TurnStatRow, fresh_session: bool) {
|
|
||||||
let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
let attrs = [
|
|
||||||
KeyValue::new("wake_from", row.wake_from.clone()),
|
|
||||||
KeyValue::new("result_kind", row.result_kind),
|
|
||||||
KeyValue::new("model", row.model.clone()),
|
|
||||||
];
|
|
||||||
// `try_from` rather than `as` — `duration_ms` is never negative in
|
|
||||||
// practice (it's `Instant::elapsed().as_millis()`), but a fallback of 0
|
|
||||||
// beats a lossy cast or a panic if that ever stops holding.
|
|
||||||
inst.turn_duration_ms
|
|
||||||
.record(u64::try_from(row.duration_ms).unwrap_or(0), &attrs);
|
|
||||||
inst.turn_count.add(1, &attrs);
|
|
||||||
if let Some(threads) = row.open_threads_count {
|
|
||||||
inst.open_threads.record(threads, &[]);
|
|
||||||
}
|
|
||||||
if let Some(reminders) = row.open_reminders_count {
|
|
||||||
inst.open_reminders.record(reminders, &[]);
|
|
||||||
}
|
|
||||||
if fresh_session {
|
|
||||||
inst.session_count
|
|
||||||
.add(1, &[KeyValue::new("model", row.model.clone())]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn build() -> Option<Instruments> {
|
|
||||||
if !enabled() {
|
|
||||||
tracing::debug!("otel turn-metrics: no endpoint configured, exporter disabled");
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let interval = interval();
|
|
||||||
match build_provider(interval) {
|
|
||||||
Ok(provider) => {
|
|
||||||
tracing::info!(?interval, "otel turn-metrics: exporter enabled");
|
|
||||||
let meter = provider.meter("hyperhive.turn_stats");
|
|
||||||
Some(Instruments {
|
|
||||||
turn_duration_ms: meter
|
|
||||||
.u64_histogram("hyperhive.agent.turn.duration")
|
|
||||||
.with_unit("ms")
|
|
||||||
.build(),
|
|
||||||
turn_count: meter.u64_counter("hyperhive.agent.turn.count").build(),
|
|
||||||
session_count: meter.u64_counter("hyperhive.agent.session.count").build(),
|
|
||||||
open_threads: meter
|
|
||||||
.u64_gauge("hyperhive.agent.loose_ends.threads")
|
|
||||||
.build(),
|
|
||||||
open_reminders: meter
|
|
||||||
.u64_gauge("hyperhive.agent.loose_ends.reminders")
|
|
||||||
.build(),
|
|
||||||
_provider: provider,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
tracing::warn!(error = ?e, "otel turn-metrics: exporter init failed");
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn build_provider(interval: Duration) -> anyhow::Result<SdkMeterProvider> {
|
|
||||||
// http/json, no explicit endpoint/resource: same rationale as
|
|
||||||
// `hive-c0re/src/stats/otel_metrics.rs` for the transport (the blocking
|
|
||||||
// OTLP client is required — `PeriodicReader` drives export from a
|
|
||||||
// background thread with no Tokio reactor), and same rationale as
|
|
||||||
// `hive-metric` for the resource (the SDK reads `OTEL_RESOURCE_ATTRIBUTES`
|
|
||||||
// — set container-wide, see `docs/observability.md::Built-in resource
|
|
||||||
// labels` — on its own; passing an explicit `Resource` here would need to
|
|
||||||
// duplicate `agent`/`hive`/`swarm` this crate has no other reason to know).
|
|
||||||
let exporter = MetricExporter::builder()
|
|
||||||
.with_http()
|
|
||||||
.with_protocol(Protocol::HttpJson)
|
|
||||||
.build()?;
|
|
||||||
let reader = PeriodicReader::builder(exporter)
|
|
||||||
.with_interval(interval)
|
|
||||||
.build();
|
|
||||||
Ok(SdkMeterProvider::builder().with_reader(reader).build())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `OTEL_EXPORTER_OTLP_ENDPOINT` non-empty — the same enable signal every
|
|
||||||
/// other exporter in this workspace uses (see `hive-c0re/src/stats/
|
|
||||||
/// otel_metrics.rs::endpoint` for why this exact variable and not a
|
|
||||||
/// hyperhive-specific one: it's also the variable the SDK itself reads to
|
|
||||||
/// build the exporter's URL, so "configured" and "where it goes" cannot
|
|
||||||
/// disagree).
|
|
||||||
fn enabled() -> bool {
|
|
||||||
std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").is_ok_and(|s| !s.trim().is_empty())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Export cadence from `HYPERHIVE_OTEL_METRIC_INTERVAL_MS`, else the default.
|
|
||||||
fn interval() -> Duration {
|
|
||||||
std::env::var("HYPERHIVE_OTEL_METRIC_INTERVAL_MS")
|
|
||||||
.ok()
|
|
||||||
.and_then(|s| s.trim().parse::<u64>().ok())
|
|
||||||
.filter(|ms| *ms > 0)
|
|
||||||
.map_or(DEFAULT_INTERVAL, Duration::from_millis)
|
|
||||||
}
|
|
||||||
Loading…
Reference in a new issue