From 39fc088907751893f3db208f22f62242d39631f6 Mon Sep 17 00:00:00 2001 From: damocles Date: Sat, 4 Jul 2026 22:48:56 +0200 Subject: [PATCH] feat(#2100): add hive-metric OTLP CLI for agent-emitted custom metrics --- Cargo.lock | 11 +++ Cargo.toml | 1 + docs/observability.md | 50 ++++++++++++ hive-metric/Cargo.toml | 15 ++++ hive-metric/src/main.rs | 169 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 246 insertions(+) create mode 100644 hive-metric/Cargo.toml create mode 100644 hive-metric/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 03443d9a..3ba044a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1514,6 +1514,17 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "hive-metric" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "reqwest", + "serde_json", + "tokio", +] + [[package]] name = "hive-priv" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index b06e72c9..c9b9238d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "hive-claude", "hive-forge", "hive-matrix-mcp", + "hive-metric", "hive-priv", "hive-sh4re", ] diff --git a/docs/observability.md b/docs/observability.md index 25e7e358..95413e21 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -125,6 +125,56 @@ Every agent's export includes these resource attributes automatically: Additional labels can be appended via `extraResourceAttributes`. +Additional labels can be appended per-agent via `extraResourceAttributes` (see +option reference above); custom per-data-point labels can be passed with +`hive-metric --labels` (see below). + +## Agent-emitted custom metrics (`hive-metric`) + +Agents can push arbitrary labeled metrics to the same OTEL collector via the +`hive-metric` CLI tool, available in every agent container when +`services.hyperhive.otel.enable = true`. + +### Usage + +```text +hive-metric [--type gauge|counter] [--labels key=value...] +``` + +- `` — metric name (e.g. `tasks_completed`, `latency_ms`). +- `` — numeric value (f64; integers and floats both accepted). +- `--type gauge|counter` — metric kind: `gauge` (instantaneous, default) or + `counter` (monotonically increasing cumulative sum). +- `--labels key=value` — extra per-data-point labels. May be repeated. + The resource labels (agent, hive, swarm, service.name) are inherited + automatically from `OTEL_RESOURCE_ATTRIBUTES` — do not re-specify them. + +### Examples + +```text +# Gauge: current queue depth +hive-metric queue_depth 17 + +# Counter: cumulative tasks finished, with a custom label +hive-metric tasks_completed 1 --type counter --labels phase=scan + +# Float gauge with multiple labels +hive-metric api_latency_ms 142.5 --labels model=sonnet --labels tier=api +``` + +### Error when OTEL is not configured + +When `services.hyperhive.otel.enable = false` (the default), the +`OTEL_EXPORTER_OTLP_ENDPOINT` env var is not set and `hive-metric` exits +with an informative error message. No silently-dropped metrics. + +### Wire format + +`hive-metric` always uses **OTLP HTTP/JSON** (`application/json` POST to +`$OTEL_EXPORTER_OTLP_ENDPOINT/v1/metrics`), regardless of the +`OTEL_EXPORTER_OTLP_PROTOCOL` setting. Auth headers from +`OTEL_EXPORTER_OTLP_HEADERS` are forwarded verbatim. + ## Metrics temporality OTEL export is always configured with **cumulative** temporality diff --git a/hive-metric/Cargo.toml b/hive-metric/Cargo.toml new file mode 100644 index 00000000..e5790915 --- /dev/null +++ b/hive-metric/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "hive-metric" +version.workspace = true +edition.workspace = true + +[[bin]] +name = "hive-metric" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +clap.workspace = true +reqwest.workspace = true +serde_json.workspace = true +tokio = { workspace = true, features = ["rt", "macros"] } diff --git a/hive-metric/src/main.rs b/hive-metric/src/main.rs new file mode 100644 index 00000000..46f06037 --- /dev/null +++ b/hive-metric/src/main.rs @@ -0,0 +1,169 @@ +//! `hive-metric` — push a single labeled metric to the OTEL collector. +//! +//! Reads `OTEL_EXPORTER_OTLP_ENDPOINT` and `OTEL_EXPORTER_OTLP_HEADERS` +//! from the environment (set by the harness via claude's managed settings +//! when `services.hyperhive.otel.enable = true`). Resource labels (agent, +//! hive, swarm, service.name) are inherited from `OTEL_RESOURCE_ATTRIBUTES`, +//! which the harness already populates per-agent — the caller does not need +//! to re-specify them. +//! +//! # Usage +//! ```text +//! hive-metric [--type gauge|counter] [--labels key=value...] +//! ``` +//! +//! # Examples +//! ```text +//! hive-metric tasks_completed 42 +//! hive-metric errors_total 3 --type counter --labels phase=scan +//! hive-metric latency_ms 250.5 --labels model=sonnet --labels tier=api +//! ``` + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use serde_json::{Value, json}; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[derive(Parser)] +#[command( + name = "hive-metric", + about = "Push a labeled metric to the OTEL collector (requires services.hyperhive.otel.enable = true)" +)] +struct Cli { + /// Metric name (e.g. `tasks_completed`, `errors_total`). + name: String, + + /// Numeric value (f64). Floats and integers both accepted. + value: f64, + + /// Metric kind: `gauge` (instantaneous value, default) or `counter` + /// (monotonically increasing sum, cumulative temporality). + #[arg(long = "type", value_name = "TYPE", default_value = "gauge")] + metric_type: MetricKind, + + /// Extra label(s) as `key=value` pairs. May be repeated. + /// Resource labels (agent, hive, swarm) are inherited automatically + /// from `OTEL_RESOURCE_ATTRIBUTES` and must not be duplicated here. + #[arg(long, value_name = "KEY=VALUE")] + labels: Vec, +} + +#[derive(Clone, Debug, clap::ValueEnum)] +enum MetricKind { + /// Gauge: represents a value at a point in time (default). + Gauge, + /// Counter: monotonically increasing cumulative sum. + Counter, +} + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<()> { + let cli = Cli::parse(); + + let endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").context( + "OTEL_EXPORTER_OTLP_ENDPOINT not set — enable OTEL in NixOS config \ + (services.hyperhive.otel.enable = true)", + )?; + let metrics_url = format!("{}/v1/metrics", endpoint.trim_end_matches('/')); + + let now_ns = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + .to_string(); + + let resource_attrs = parse_kv_list(&std::env::var("OTEL_RESOURCE_ATTRIBUTES").unwrap_or_default(), ','); + let data_attrs = parse_label_args(&cli.labels)?; + + let data_point = json!({ + "attributes": data_attrs, + "startTimeUnixNano": now_ns, + "timeUnixNano": now_ns, + "asDouble": cli.value, + }); + + let metric = match cli.metric_type { + MetricKind::Gauge => json!({ + "name": cli.name, + "gauge": { "dataPoints": [data_point] }, + }), + MetricKind::Counter => json!({ + "name": cli.name, + "sum": { + "dataPoints": [data_point], + // AGGREGATION_TEMPORALITY_CUMULATIVE = 2 + "aggregationTemporality": 2, + "isMonotonic": true, + }, + }), + }; + + let payload = json!({ + "resourceMetrics": [{ + "resource": { "attributes": resource_attrs }, + "scopeMetrics": [{ + "scope": { "name": "hive-metric", "version": "0.1.0" }, + "metrics": [metric], + }], + }] + }); + + let mut builder = reqwest::Client::new() + .post(&metrics_url) + .header("content-type", "application/json") + .json(&payload); + + // OTEL_EXPORTER_OTLP_HEADERS: `Key=Value,Key2=Value2` (comma-separated). + if let Ok(h) = std::env::var("OTEL_EXPORTER_OTLP_HEADERS") { + for (k, v) in parse_kv_list(&h, ',') + .into_iter() + .filter_map(|kv| { + let k = kv.get("key")?.as_str()?.to_owned(); + let v = kv + .get("value")? + .get("stringValue")? + .as_str()? + .to_owned(); + Some((k, v)) + }) + { + builder = builder.header(k, v); + } + } + + let resp = builder + .send() + .await + .with_context(|| format!("POST to {metrics_url} failed"))?; + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + bail!("OTEL endpoint {metrics_url} returned {status}: {body}"); + } + + Ok(()) +} + +/// Parse a `key=value` delimited list into OTLP attribute objects. +/// Entries that lack `=` are silently skipped. +fn parse_kv_list(s: &str, sep: char) -> Vec { + s.split(sep) + .filter_map(|entry| { + let (k, v) = entry.split_once('=')?; + Some(json!({ "key": k.trim(), "value": { "stringValue": v.trim() } })) + }) + .collect() +} + +/// Parse `--labels k=v` CLI args into OTLP attribute objects. +fn parse_label_args(args: &[String]) -> Result> { + args.iter() + .map(|s| { + let (k, v) = s + .split_once('=') + .with_context(|| format!("--labels value must be key=value, got {s:?}"))?; + Ok(json!({ "key": k.trim(), "value": { "stringValue": v.trim() } })) + }) + .collect() +}