//! Per-agent container-resource OTEL export. hive-c0re already samples each //! agent container's cgroup load for the dashboard //! ([`super::container_stats`]); this rides those same gauges out to the //! hive-wide OTLP endpoint, reusing the SAME `services.hyperhive.otel` config //! (endpoint + auth header) that Claude Code's in-container SDK export uses — //! no new toggle. The auth header is loaded onto hive-c0re's own unit via //! systemd `LoadCredential` (see `nix/host-modules/hive-c0re`) and read from //! `$CREDENTIALS_DIRECTORY/otel-headers`. //! //! Emits the OTEL **semconv `container.*`** metrics with the standard //! `container.name` attribute (so off-the-shelf OTEL/Grafana container //! dashboards work), plus the hive-specific `agent` / `hive` / `swarm` labels //! for our own dashboards. The resource additionally carries semconv //! `host.arch` and `service.version` (the running hyperhive flake rev), so a //! sample can be attributed to a machine and a deploy. //! Uses the OpenTelemetry Rust SDK (same crates as //! `hive-metric`); the blocking OTLP client is deliberate — the metrics SDK's //! `PeriodicReader` runs on a background thread with no Tokio reactor. //! //! Bridging async→sync: [`container_stats::gather`] is async, but the SDK's //! observable-instrument callbacks are sync. So an async task refreshes a //! shared snapshot on an interval, and the (sync) callbacks read it. use std::collections::HashMap; use std::sync::{Arc, Mutex, OnceLock}; use std::time::Duration; use anyhow::{Context, Result}; use opentelemetry::KeyValue; use opentelemetry::metrics::MeterProvider as _; use opentelemetry_otlp::{MetricExporter, Protocol, WithExportConfig, WithHttpConfig}; use opentelemetry_sdk::Resource; use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider}; use super::container_stats::{self, ContainerResource}; /// Default export cadence when `HYPERHIVE_OTEL_METRIC_INTERVAL_MS` is unset. const DEFAULT_INTERVAL: Duration = Duration::from_mins(1); /// Latest container-stats snapshot: written by the async refresher, read by /// the sync observable-instrument callbacks. static SNAPSHOT: OnceLock>>> = OnceLock::new(); /// Keep the provider alive for the process lifetime — the `PeriodicReader` /// exports only while the provider lives. static PROVIDER: OnceLock = OnceLock::new(); /// Spawn the container-resource OTEL exporter if OTEL is configured /// (`HYPERHIVE_OTEL_ENDPOINT` non-empty — the same enable signal /// `meta::otel_config` uses). No-op otherwise. Call once at startup. pub fn spawn_exporter(hyperhive_flake: &str) { let Some(endpoint) = endpoint() else { tracing::debug!("otel container-metrics: no endpoint configured, exporter disabled"); return; }; let snapshot = SNAPSHOT .get_or_init(|| Arc::new(Mutex::new(Vec::new()))) .clone(); let interval = interval(); // Async refresher: keep the shared snapshot current for the sync callbacks. let refresh = snapshot.clone(); tokio::spawn(async move { loop { let fresh = container_stats::gather().await; if let Ok(mut g) = refresh.lock() { *g = fresh; } tokio::time::sleep(interval).await; } }); match build_provider(&endpoint, interval, snapshot, hyperhive_flake) { Ok(provider) => { let _ = PROVIDER.set(provider); tracing::info!(%endpoint, ?interval, "otel container-metrics: exporter enabled"); } Err(e) => { tracing::warn!(error = ?e, "otel container-metrics: exporter init failed"); } } } fn build_provider( endpoint: &str, interval: Duration, snapshot: Arc>>, hyperhive_flake: &str, ) -> Result { // http/json — the only OTLP transport this crate enables (matching // hive-metric). The Claude SDK path honours `HYPERHIVE_OTEL_PROTOCOL` for // its own export; this exporter is always http/json. let mut builder = MetricExporter::builder() .with_http() .with_endpoint(endpoint) .with_protocol(Protocol::HttpJson); let headers = auth_headers(); if !headers.is_empty() { builder = builder.with_headers(headers); } let exporter = builder.build().context("build OTLP metric exporter")?; // Drive the reader's export at `interval` so `HYPERHIVE_OTEL_METRIC_INTERVAL_MS` // is the real export cadence (not just the snapshot-refresh cadence). The // refresher runs at the same interval, gather-first, so the snapshot is // populated before the first export. let reader = PeriodicReader::builder(exporter) .with_interval(interval) .build(); let provider = SdkMeterProvider::builder() .with_reader(reader) .with_resource(resource(hyperhive_flake)) .build(); register_instruments(&provider, snapshot); Ok(provider) } /// Register the observable instruments. Each callback reads the shared /// snapshot and reports one data point per agent. Instruments are held by the /// meter/provider (kept alive in `PROVIDER`). fn register_instruments(provider: &SdkMeterProvider, snapshot: Arc>>) { let meter = provider.meter("hyperhive.container_stats"); // semconv `container.cpu.time` — cumulative CPU seconds (monotonic counter). let snap = snapshot.clone(); meter .f64_observable_counter("container.cpu.time") .with_unit("s") .with_callback(move |obs| { if let Ok(g) = snap.lock() { for c in g.iter() { if let Some(usec) = c.cpu_time_usec { #[allow(clippy::cast_precision_loss)] obs.observe(usec as f64 / 1_000_000.0, &attrs(c)); } } } }) .build(); // semconv `container.memory.usage` — current usage bytes. let snap = snapshot.clone(); meter .u64_observable_gauge("container.memory.usage") .with_unit("By") .with_callback(move |obs| { if let Ok(g) = snap.lock() { for c in g.iter() { obs.observe(c.mem_current_bytes, &attrs(c)); } } }) .build(); // The cgroup memory ceiling (omit when unlimited). Kept `hyperhive.` // custom — semconv defines `container.memory.usage` but not a matching // `.limit` metric, so don't claim a spec name that isn't in the spec. let snap = snapshot.clone(); meter .u64_observable_gauge("hyperhive.container.memory.limit") .with_unit("By") .with_callback(move |obs| { if let Ok(g) = snap.lock() { for c in g.iter() { if let Some(limit) = c.mem_max_bytes { obs.observe(limit, &attrs(c)); } } } }) .build(); // Custom (`hyperhive.`) metrics semconv doesn't standardise: memory peak, // on-disk footprint, and the instantaneous cpu percent the dashboard shows. let snap = snapshot.clone(); meter .u64_observable_gauge("hyperhive.container.memory.peak") .with_unit("By") .with_callback(move |obs| { if let Ok(g) = snap.lock() { for c in g.iter() { if let Some(peak) = c.mem_peak_bytes { obs.observe(peak, &attrs(c)); } } } }) .build(); let snap = snapshot.clone(); meter .u64_observable_gauge("hyperhive.container.storage.usage") .with_unit("By") .with_callback(move |obs| { if let Ok(g) = snap.lock() { for c in g.iter() { if let Some(disk) = c.disk_bytes { obs.observe(disk, &attrs(c)); } } } }) .build(); let snap = snapshot; meter .f64_observable_gauge("hyperhive.container.cpu.percent") .with_unit("%") .with_callback(move |obs| { if let Ok(g) = snap.lock() { for c in g.iter() { obs.observe(c.cpu_pct, &attrs(c)); } } }) .build(); } /// Per-data-point attributes: the spec `container.name` plus the hive `agent` /// label. (`hive` / `swarm` are constant across this c0re, so they live on the /// resource.) fn attrs(c: &ContainerResource) -> Vec { vec![ KeyValue::new("container.name", format!("h-{}", c.name)), KeyValue::new("agent", c.name.clone()), ] } /// Resource: `service.name = hyperhive-c0re`, the host architecture, the /// running hyperhive revision, plus whatever the operator set in /// `extraResourceAttributes` (where the hive `hive` / `swarm` labels live, same /// channel the agent-side export uses). /// /// These belong on the resource rather than on each data point for the same /// reason `hive` / `swarm` do: they are constant across one `hive-c0re`. Only /// facts that vary per container (see [`attrs`]) are per-data-point. /// /// `service.version` is the **flake revision**, not the crate version — the /// crate version is a workspace constant that never moves between deploys, so /// it could not answer "which build produced this sample?". `None` when the /// flake ref carries no rev (a local path ref), in which case the attribute is /// omitted rather than reported as a guess. fn resource(hyperhive_flake: &str) -> Resource { let mut builder = Resource::builder() .with_service_name("hyperhive-c0re") .with_attribute(KeyValue::new("host.arch", host_arch())); if let Some(rev) = crate::auto_update::current_flake_rev(hyperhive_flake) { builder = builder.with_attribute(KeyValue::new("service.version", rev)); } for (k, v) in resource_attributes() { builder = builder.with_attribute(KeyValue::new(k, v)); } builder.build() } /// The host architecture as OTEL semconv spells it. /// /// Rust's [`std::env::consts::ARCH`] and semconv `host.arch` disagree on the /// two architectures this actually runs on: rust says `x86_64` / `aarch64`, /// semconv says `amd64` / `arm64`. Forwarding rust's spelling would emit a /// label that off-the-shelf dashboards silently fail to match — the same class /// of problem as guessing a metric name. Anything else is passed through: a /// non-standard value is more useful than a wrong standard one. fn host_arch() -> &'static str { match std::env::consts::ARCH { "x86_64" => "amd64", "aarch64" => "arm64", other => other, } } /// `HYPERHIVE_OTEL_ENDPOINT`, non-empty. The enable signal. fn endpoint() -> Option { std::env::var("HYPERHIVE_OTEL_ENDPOINT") .ok() .map(|s| s.trim().to_owned()) .filter(|s| !s.is_empty()) } /// Export cadence from `HYPERHIVE_OTEL_METRIC_INTERVAL_MS`, else the default. fn interval() -> Duration { std::env::var("HYPERHIVE_OTEL_METRIC_INTERVAL_MS") .ok() .and_then(|s| s.trim().parse::().ok()) .filter(|ms| *ms > 0) .map_or(DEFAULT_INTERVAL, Duration::from_millis) } /// Extra resource attributes from `HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES` /// (`key=value,key=value`), same env the agent-side export reads. fn resource_attributes() -> Vec<(String, String)> { std::env::var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES") .ok() .map(|s| parse_kv(&s)) .unwrap_or_default() } /// OTLP auth headers from the systemd credential at /// `$CREDENTIALS_DIRECTORY/otel-headers` (an `OTEL_EXPORTER_OTLP_HEADERS`-style /// `Key=Value` line). Empty when the credential is absent — the collector is /// then assumed unauthenticated. fn auth_headers() -> HashMap { let Some(dir) = std::env::var("CREDENTIALS_DIRECTORY").ok() else { return HashMap::new(); }; let path = std::path::Path::new(&dir).join("otel-headers"); let Ok(raw) = std::fs::read_to_string(&path) else { tracing::warn!( "otel container-metrics: no auth header at $CREDENTIALS_DIRECTORY/otel-headers; \ exporting without Authorization" ); return HashMap::new(); }; parse_kv(&raw).into_iter().collect() } /// Parse `key=value` pairs separated by commas and/or newlines. The value /// keeps any `=` after the first (so `Authorization=Bearer x=y` → `Bearer x=y`). fn parse_kv(s: &str) -> Vec<(String, String)> { s.split([',', '\n']) .filter_map(|pair| { let pair = pair.trim(); if pair.is_empty() { return None; } let (k, v) = pair.split_once('=')?; let k = k.trim(); if k.is_empty() { None } else { Some((k.to_owned(), v.trim().to_owned())) } }) .collect() } #[cfg(test)] mod tests { use super::*; #[test] fn parse_kv_keeps_bearer_value() { assert_eq!( parse_kv("Authorization=Bearer abc=def"), vec![("Authorization".to_owned(), "Bearer abc=def".to_owned())] ); assert_eq!( parse_kv("a=1,\n b=2 ,=bad,"), vec![ ("a".to_owned(), "1".to_owned()), ("b".to_owned(), "2".to_owned()) ] ); } /// `host.arch` must be the semconv spelling, not rust's. The two disagree /// on exactly the architectures hyperhive runs on, and the failure is /// silent — a dashboard filtering `host.arch="amd64"` simply matches /// nothing if we emit `x86_64`. #[test] fn host_arch_is_semconv_not_rust_spelling() { assert!( !matches!(host_arch(), "x86_64" | "aarch64"), "host_arch() forwarded rust's ARCH spelling ({}) — semconv wants amd64 / arm64", host_arch() ); // On the architectures we actually ship, pin the exact expected value // rather than only asserting the negative above. #[cfg(target_arch = "x86_64")] assert_eq!(host_arch(), "amd64"); #[cfg(target_arch = "aarch64")] assert_eq!(host_arch(), "arm64"); } }