fix(otel): cumulative metric temporality + metricIntervalMs knob (real metrics-export fix)

This commit is contained in:
damocles 2026-06-24 20:28:37 +02:00
commit af10a4bfef
3 changed files with 64 additions and 10 deletions

View file

@ -668,6 +668,7 @@ struct OtelConfig {
protocol: String, protocol: String,
extra_resource_attributes: Option<String>, extra_resource_attributes: Option<String>,
headers_credential: Option<String>, headers_credential: Option<String>,
metric_interval_ms: Option<u64>,
} }
/// Read the hive-wide OTEL config from env, or `None` when OTEL is off. /// Read the hive-wide OTEL config from env, or `None` when OTEL is off.
@ -690,11 +691,16 @@ fn otel_config() -> Option<OtelConfig> {
let headers_credential = std::env::var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL") let headers_credential = std::env::var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL")
.ok() .ok()
.filter(|v| !v.is_empty()); .filter(|v| !v.is_empty());
let metric_interval_ms = std::env::var("HYPERHIVE_OTEL_METRIC_INTERVAL_MS")
.ok()
.and_then(|v| v.parse::<u64>().ok())
.filter(|v| *v > 0);
Some(OtelConfig { Some(OtelConfig {
endpoint, endpoint,
protocol, protocol,
extra_resource_attributes, extra_resource_attributes,
headers_credential, headers_credential,
metric_interval_ms,
}) })
} }
@ -982,6 +988,11 @@ where
esc(cred) esc(cred)
); );
} }
if let Some(ms) = otel.metric_interval_ms {
// Int option — emit a bare numeric literal (no quotes). `ms` is a
// parsed u64, so it can't inject anything into the rendered nix.
let _ = writeln!(out, " hyperhive.otel.metricIntervalMs = {ms};");
}
} }
out.push_str( out.push_str(
r#" # The harness service inside the container runs as a r#" # The harness service inside the container runs as a

View file

@ -257,6 +257,23 @@ in
`service.name` / `agent` / `hive` / `swarm` labels. `service.name` / `agent` / `hive` / `swarm` labels.
''; '';
}; };
metricIntervalMs = lib.mkOption {
type = lib.types.nullOr lib.types.ints.positive;
default = null;
example = 10000;
description = ''
Metric export interval in milliseconds, set as
`OTEL_METRIC_EXPORT_INTERVAL` for every agent. Claude Code's
default is 60000 (60s). Leave `null` to use that default.
Each agent runs claude as a short-lived per-turn process; claude
force-flushes metrics on shutdown, so this is not required for
metrics to be exported, but a lower value gives more frequent
intermediate flushes within long turns. Cosmetic, not a
correctness knob.
'';
};
}; };
# Peer hives in the same swarm. Each entry declares a remote hive # Peer hives in the same swarm. Each entry declares a remote hive
@ -888,6 +905,9 @@ in
// lib.optionalAttrs (otel.headersCredential != null) { // lib.optionalAttrs (otel.headersCredential != null) {
HYPERHIVE_OTEL_HEADERS_CREDENTIAL = otel.headersCredential; HYPERHIVE_OTEL_HEADERS_CREDENTIAL = otel.headersCredential;
} }
// lib.optionalAttrs (otel.metricIntervalMs != null) {
HYPERHIVE_OTEL_METRIC_INTERVAL_MS = toString otel.metricIntervalMs;
}
) )
// { // {
# In-cluster forge URL — the gateway vhost (`forge.<domain>`), which # In-cluster forge URL — the gateway vhost (`forge.<domain>`), which

View file

@ -250,6 +250,17 @@ in
Host-driven via `services.hyperhive.otel.extraResourceAttributes`. Host-driven via `services.hyperhive.otel.extraResourceAttributes`.
''; '';
}; };
metricIntervalMs = lib.mkOption {
type = lib.types.nullOr lib.types.ints.positive;
default = null;
internal = true;
description = ''
Metric export interval in milliseconds, set as
`OTEL_METRIC_EXPORT_INTERVAL`. Null leaves Claude Code's 60s
default. Host-driven via `services.hyperhive.otel.metricIntervalMs`.
'';
};
}; };
options.hyperhive.allowedRecipients = lib.mkOption { options.hyperhive.allowedRecipients = lib.mkOption {
@ -1735,16 +1746,28 @@ in
# Claude Code's native OpenTelemetry is env-driven; the harness # Claude Code's native OpenTelemetry is env-driven; the harness
# spawns `claude` as a child which inherits this unit's env, so # spawns `claude` as a child which inherits this unit's env, so
# setting these here is all it takes to export per-agent stats. # setting these here is all it takes to export per-agent stats.
otelEnv = lib.optionalAttrs otel.enable { otelEnv = lib.optionalAttrs otel.enable (
CLAUDE_CODE_ENABLE_TELEMETRY = "1"; {
OTEL_METRICS_EXPORTER = "otlp"; CLAUDE_CODE_ENABLE_TELEMETRY = "1";
OTEL_LOGS_EXPORTER = "otlp"; OTEL_METRICS_EXPORTER = "otlp";
# Route traces to OTLP too so any spans Claude Code emits land OTEL_LOGS_EXPORTER = "otlp";
# at the configured collector rather than a default exporter. # Route traces to OTLP too so any spans Claude Code emits land
OTEL_TRACES_EXPORTER = "otlp"; # at the configured collector rather than a default exporter.
OTEL_EXPORTER_OTLP_PROTOCOL = otel.protocol; OTEL_TRACES_EXPORTER = "otlp";
OTEL_EXPORTER_OTLP_ENDPOINT = otel.endpoint; OTEL_EXPORTER_OTLP_PROTOCOL = otel.protocol;
}; OTEL_EXPORTER_OTLP_ENDPOINT = otel.endpoint;
# Force CUMULATIVE metric temporality. Claude Code defaults to
# DELTA, which Prometheus/Mimir-family backends (the common case,
# incl. grafana-lgtm) silently drop unless a deltatocumulative
# processor is wired — so delta = "logs arrive, metrics vanish".
# Cumulative is what those backends ingest natively. Verified:
# a delta export never registers the metric name; cumulative does.
OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
}
// lib.optionalAttrs (otel.metricIntervalMs != null) {
OTEL_METRIC_EXPORT_INTERVAL = toString otel.metricIntervalMs;
}
);
# When OTEL is on, wrap the harness launch so (1) the bearer-token # When OTEL is on, wrap the harness launch so (1) the bearer-token
# header is read from the systemd credential at start (never in the # header is read from the systemd credential at start (never in the
# nix store or argv) and (2) the resource attributes are assembled # nix store or argv) and (2) the resource attributes are assembled