From af10a4bfefd159a7aedd7ab5b09b97f596949908 Mon Sep 17 00:00:00 2001 From: damocles Date: Wed, 24 Jun 2026 20:28:37 +0200 Subject: [PATCH] fix(otel): cumulative metric temporality + metricIntervalMs knob (real metrics-export fix) --- hive-c0re/src/meta.rs | 11 +++++++++ nix/modules/hive-c0re.nix | 20 ++++++++++++++++ nix/templates/harness-base.nix | 43 ++++++++++++++++++++++++++-------- 3 files changed, 64 insertions(+), 10 deletions(-) diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index c5da4c40..8977f160 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -668,6 +668,7 @@ struct OtelConfig { protocol: String, extra_resource_attributes: Option, headers_credential: Option, + metric_interval_ms: Option, } /// Read the hive-wide OTEL config from env, or `None` when OTEL is off. @@ -690,11 +691,16 @@ fn otel_config() -> Option { let headers_credential = std::env::var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL") .ok() .filter(|v| !v.is_empty()); + let metric_interval_ms = std::env::var("HYPERHIVE_OTEL_METRIC_INTERVAL_MS") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|v| *v > 0); Some(OtelConfig { endpoint, protocol, extra_resource_attributes, headers_credential, + metric_interval_ms, }) } @@ -982,6 +988,11 @@ where esc(cred) ); } + if let Some(ms) = otel.metric_interval_ms { + // Int option — emit a bare numeric literal (no quotes). `ms` is a + // parsed u64, so it can't inject anything into the rendered nix. + let _ = writeln!(out, " hyperhive.otel.metricIntervalMs = {ms};"); + } } out.push_str( r#" # The harness service inside the container runs as a diff --git a/nix/modules/hive-c0re.nix b/nix/modules/hive-c0re.nix index 7f2f8fb7..fd35d38a 100644 --- a/nix/modules/hive-c0re.nix +++ b/nix/modules/hive-c0re.nix @@ -257,6 +257,23 @@ in `service.name` / `agent` / `hive` / `swarm` labels. ''; }; + + metricIntervalMs = lib.mkOption { + type = lib.types.nullOr lib.types.ints.positive; + default = null; + example = 10000; + description = '' + Metric export interval in milliseconds, set as + `OTEL_METRIC_EXPORT_INTERVAL` for every agent. Claude Code's + default is 60000 (60s). Leave `null` to use that default. + + Each agent runs claude as a short-lived per-turn process; claude + force-flushes metrics on shutdown, so this is not required for + metrics to be exported, but a lower value gives more frequent + intermediate flushes within long turns. Cosmetic, not a + correctness knob. + ''; + }; }; # Peer hives in the same swarm. Each entry declares a remote hive @@ -888,6 +905,9 @@ in // lib.optionalAttrs (otel.headersCredential != null) { HYPERHIVE_OTEL_HEADERS_CREDENTIAL = otel.headersCredential; } + // lib.optionalAttrs (otel.metricIntervalMs != null) { + HYPERHIVE_OTEL_METRIC_INTERVAL_MS = toString otel.metricIntervalMs; + } ) // { # In-cluster forge URL — the gateway vhost (`forge.`), which diff --git a/nix/templates/harness-base.nix b/nix/templates/harness-base.nix index d198a909..6af020f1 100644 --- a/nix/templates/harness-base.nix +++ b/nix/templates/harness-base.nix @@ -250,6 +250,17 @@ in Host-driven via `services.hyperhive.otel.extraResourceAttributes`. ''; }; + + metricIntervalMs = lib.mkOption { + type = lib.types.nullOr lib.types.ints.positive; + default = null; + internal = true; + description = '' + Metric export interval in milliseconds, set as + `OTEL_METRIC_EXPORT_INTERVAL`. Null leaves Claude Code's 60s + default. Host-driven via `services.hyperhive.otel.metricIntervalMs`. + ''; + }; }; options.hyperhive.allowedRecipients = lib.mkOption { @@ -1735,16 +1746,28 @@ in # Claude Code's native OpenTelemetry is env-driven; the harness # spawns `claude` as a child which inherits this unit's env, so # setting these here is all it takes to export per-agent stats. - otelEnv = lib.optionalAttrs otel.enable { - CLAUDE_CODE_ENABLE_TELEMETRY = "1"; - OTEL_METRICS_EXPORTER = "otlp"; - OTEL_LOGS_EXPORTER = "otlp"; - # Route traces to OTLP too so any spans Claude Code emits land - # at the configured collector rather than a default exporter. - OTEL_TRACES_EXPORTER = "otlp"; - OTEL_EXPORTER_OTLP_PROTOCOL = otel.protocol; - OTEL_EXPORTER_OTLP_ENDPOINT = otel.endpoint; - }; + otelEnv = lib.optionalAttrs otel.enable ( + { + CLAUDE_CODE_ENABLE_TELEMETRY = "1"; + OTEL_METRICS_EXPORTER = "otlp"; + OTEL_LOGS_EXPORTER = "otlp"; + # Route traces to OTLP too so any spans Claude Code emits land + # at the configured collector rather than a default exporter. + OTEL_TRACES_EXPORTER = "otlp"; + OTEL_EXPORTER_OTLP_PROTOCOL = otel.protocol; + OTEL_EXPORTER_OTLP_ENDPOINT = otel.endpoint; + # Force CUMULATIVE metric temporality. Claude Code defaults to + # DELTA, which Prometheus/Mimir-family backends (the common case, + # incl. grafana-lgtm) silently drop unless a deltatocumulative + # processor is wired — so delta = "logs arrive, metrics vanish". + # Cumulative is what those backends ingest natively. Verified: + # a delta export never registers the metric name; cumulative does. + OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative"; + } + // lib.optionalAttrs (otel.metricIntervalMs != null) { + OTEL_METRIC_EXPORT_INTERVAL = toString otel.metricIntervalMs; + } + ); # When OTEL is on, wrap the harness launch so (1) the bearer-token # header is read from the systemd credential at start (never in the # nix store or argv) and (2) the resource attributes are assembled