diff --git a/docs/observability.md b/docs/observability.md index 67224a0e..18d71712 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -327,13 +327,18 @@ Agents can push arbitrary labeled metrics to the same OTEL collector via the ### Usage ```text -hive-metric [--type counter|gauge] [--labels key=value...] +hive-metric [--type counter|gauge] [--temporality delta|cumulative] [--labels key=value...] ``` - `` — metric name (e.g. `tasks_completed`, `latency_ms`). - `` — numeric value (f64; integers and floats both accepted). -- `--type counter|gauge` — metric kind: `counter` (cumulative sum, default) or +- `--type counter|gauge` — metric kind: `counter` (increasing sum, default) or `gauge` (instantaneous point-in-time value). +- `--temporality delta|cumulative` — counter reporting mode (`counter` only, + ignored for `gauge`): `delta` (this call's own contribution, default — send + `1` each time and the collector accumulates) or `cumulative` (this call + reports the running total, which a stateless one-shot CLI can't track + itself). - `--labels key=value` — extra per-data-point labels. May be repeated. The resource labels (agent, hive, swarm, service.name) are inherited automatically from `OTEL_RESOURCE_ATTRIBUTES` — do not re-specify them. @@ -341,7 +346,7 @@ hive-metric [--type counter|gauge] [--labels key=value...] ### Examples ```text -# Counter: cumulative tasks finished (default type — no --type flag needed) +# Counter: one more task finished (delta is the default — no flag needed) hive-metric tasks_completed 1 --labels phase=scan # Gauge: current queue depth (absolute value — must use --type gauge) @@ -366,8 +371,17 @@ with an informative error message. No silently-dropped metrics. ## Metrics temporality -OTEL export is always configured with **cumulative** temporality +OTEL export is configured with **cumulative** temporality by default (`OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE=cumulative`), overriding Claude Code's default of DELTA. This avoids silent metric drops in Prometheus-family backends (including Grafana LGTM / Mimir) that don't ship a delta-to-cumulative processor. + +**`hive-metric` counters are the one exception**, reporting delta by default +(see above) — programmatically set on the exporter, which overrides this +container-wide env var for that tool specifically. `--type gauge` is +unaffected either way; gauges have no temporality. The hive-tier collector +runs a `deltatocumulative` processor ahead of export, so a delta +`hive-metric` counter still lands in VictoriaMetrics as a cumulative +series — the standard `rate()`/`increase()` idioms work on it exactly like +any other counter in this system, no special query needed. diff --git a/hive-metric/src/main.rs b/hive-metric/src/main.rs index 48e066eb..0e107031 100644 --- a/hive-metric/src/main.rs +++ b/hive-metric/src/main.rs @@ -12,7 +12,7 @@ //! //! # Usage //! ```text -//! hive-metric [--type counter|gauge] [--labels key=value...] +//! hive-metric [--type counter|gauge] [--temporality delta|cumulative] [--labels key=value...] //! ``` use anyhow::{Context, Result, bail}; @@ -20,6 +20,7 @@ use clap::Parser; use opentelemetry::KeyValue; use opentelemetry::metrics::MeterProvider; use opentelemetry_otlp::{Protocol, WithExportConfig}; +use opentelemetry_sdk::metrics::Temporality; #[derive(Parser)] #[command( @@ -39,10 +40,17 @@ struct Cli { /// negative; counters must be >= 0 (they only increase). value: f64, - /// Metric kind: `counter` (cumulative sum, default) or `gauge` (instantaneous value). + /// Metric kind: `counter` (increasing sum, default) or `gauge` (instantaneous value). #[arg(long = "type", value_name = "TYPE", default_value = "counter")] metric_type: MetricKind, + /// Counter reporting mode: `delta` (this call's own contribution, default — + /// send `1` each time and the collector accumulates) or `cumulative` (this + /// call reports the running total, which a stateless one-shot CLI can't + /// track itself — only meaningful for `--type counter`; ignored for `gauge`). + #[arg(long, value_name = "MODE", default_value = "delta")] + temporality: TemporalityArg, + /// Extra label(s) as `key=value` pairs. May be repeated. /// Resource labels (agent, hive, swarm) come from `OTEL_RESOURCE_ATTRIBUTES` /// automatically — do not re-specify them here. @@ -52,12 +60,29 @@ struct Cli { #[derive(Clone, Debug, clap::ValueEnum)] enum MetricKind { - /// Counter: monotonically increasing cumulative sum (default). + /// Counter: increasing sum (default). Counter, /// Gauge: point-in-time instantaneous value. Gauge, } +#[derive(Clone, Debug, clap::ValueEnum)] +enum TemporalityArg { + /// Report this call's own contribution since the last report (default). + Delta, + /// Report the running total as of this call. + Cumulative, +} + +impl From for Temporality { + fn from(arg: TemporalityArg) -> Self { + match arg { + TemporalityArg::Delta => Temporality::Delta, + TemporalityArg::Cumulative => Temporality::Cumulative, + } + } +} + fn main() -> Result<()> { let cli = Cli::parse(); @@ -86,6 +111,7 @@ fn main() -> Result<()> { let exporter = opentelemetry_otlp::MetricExporter::builder() .with_http() .with_protocol(Protocol::HttpJson) + .with_temporality(cli.temporality.into()) .build() .context("failed to build OTLP metric exporter")?;