fix(otel): cumulative metric temporality + metricIntervalMs knob (real metrics-export fix)
This commit is contained in:
parent
609438a889
commit
af10a4bfef
3 changed files with 64 additions and 10 deletions
|
|
@ -668,6 +668,7 @@ struct OtelConfig {
|
||||||
protocol: String,
|
protocol: String,
|
||||||
extra_resource_attributes: Option<String>,
|
extra_resource_attributes: Option<String>,
|
||||||
headers_credential: Option<String>,
|
headers_credential: Option<String>,
|
||||||
|
metric_interval_ms: Option<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Read the hive-wide OTEL config from env, or `None` when OTEL is off.
|
/// Read the hive-wide OTEL config from env, or `None` when OTEL is off.
|
||||||
|
|
@ -690,11 +691,16 @@ fn otel_config() -> Option<OtelConfig> {
|
||||||
let headers_credential = std::env::var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL")
|
let headers_credential = std::env::var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL")
|
||||||
.ok()
|
.ok()
|
||||||
.filter(|v| !v.is_empty());
|
.filter(|v| !v.is_empty());
|
||||||
|
let metric_interval_ms = std::env::var("HYPERHIVE_OTEL_METRIC_INTERVAL_MS")
|
||||||
|
.ok()
|
||||||
|
.and_then(|v| v.parse::<u64>().ok())
|
||||||
|
.filter(|v| *v > 0);
|
||||||
Some(OtelConfig {
|
Some(OtelConfig {
|
||||||
endpoint,
|
endpoint,
|
||||||
protocol,
|
protocol,
|
||||||
extra_resource_attributes,
|
extra_resource_attributes,
|
||||||
headers_credential,
|
headers_credential,
|
||||||
|
metric_interval_ms,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -982,6 +988,11 @@ where
|
||||||
esc(cred)
|
esc(cred)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if let Some(ms) = otel.metric_interval_ms {
|
||||||
|
// Int option — emit a bare numeric literal (no quotes). `ms` is a
|
||||||
|
// parsed u64, so it can't inject anything into the rendered nix.
|
||||||
|
let _ = writeln!(out, " hyperhive.otel.metricIntervalMs = {ms};");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
out.push_str(
|
out.push_str(
|
||||||
r#" # The harness service inside the container runs as a
|
r#" # The harness service inside the container runs as a
|
||||||
|
|
|
||||||
|
|
@ -257,6 +257,23 @@ in
|
||||||
`service.name` / `agent` / `hive` / `swarm` labels.
|
`service.name` / `agent` / `hive` / `swarm` labels.
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
metricIntervalMs = lib.mkOption {
|
||||||
|
type = lib.types.nullOr lib.types.ints.positive;
|
||||||
|
default = null;
|
||||||
|
example = 10000;
|
||||||
|
description = ''
|
||||||
|
Metric export interval in milliseconds, set as
|
||||||
|
`OTEL_METRIC_EXPORT_INTERVAL` for every agent. Claude Code's
|
||||||
|
default is 60000 (60s). Leave `null` to use that default.
|
||||||
|
|
||||||
|
Each agent runs claude as a short-lived per-turn process; claude
|
||||||
|
force-flushes metrics on shutdown, so this is not required for
|
||||||
|
metrics to be exported, but a lower value gives more frequent
|
||||||
|
intermediate flushes within long turns. Cosmetic, not a
|
||||||
|
correctness knob.
|
||||||
|
'';
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
# Peer hives in the same swarm. Each entry declares a remote hive
|
# Peer hives in the same swarm. Each entry declares a remote hive
|
||||||
|
|
@ -888,6 +905,9 @@ in
|
||||||
// lib.optionalAttrs (otel.headersCredential != null) {
|
// lib.optionalAttrs (otel.headersCredential != null) {
|
||||||
HYPERHIVE_OTEL_HEADERS_CREDENTIAL = otel.headersCredential;
|
HYPERHIVE_OTEL_HEADERS_CREDENTIAL = otel.headersCredential;
|
||||||
}
|
}
|
||||||
|
// lib.optionalAttrs (otel.metricIntervalMs != null) {
|
||||||
|
HYPERHIVE_OTEL_METRIC_INTERVAL_MS = toString otel.metricIntervalMs;
|
||||||
|
}
|
||||||
)
|
)
|
||||||
// {
|
// {
|
||||||
# In-cluster forge URL — the gateway vhost (`forge.<domain>`), which
|
# In-cluster forge URL — the gateway vhost (`forge.<domain>`), which
|
||||||
|
|
|
||||||
|
|
@ -250,6 +250,17 @@ in
|
||||||
Host-driven via `services.hyperhive.otel.extraResourceAttributes`.
|
Host-driven via `services.hyperhive.otel.extraResourceAttributes`.
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
metricIntervalMs = lib.mkOption {
|
||||||
|
type = lib.types.nullOr lib.types.ints.positive;
|
||||||
|
default = null;
|
||||||
|
internal = true;
|
||||||
|
description = ''
|
||||||
|
Metric export interval in milliseconds, set as
|
||||||
|
`OTEL_METRIC_EXPORT_INTERVAL`. Null leaves Claude Code's 60s
|
||||||
|
default. Host-driven via `services.hyperhive.otel.metricIntervalMs`.
|
||||||
|
'';
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
options.hyperhive.allowedRecipients = lib.mkOption {
|
options.hyperhive.allowedRecipients = lib.mkOption {
|
||||||
|
|
@ -1735,16 +1746,28 @@ in
|
||||||
# Claude Code's native OpenTelemetry is env-driven; the harness
|
# Claude Code's native OpenTelemetry is env-driven; the harness
|
||||||
# spawns `claude` as a child which inherits this unit's env, so
|
# spawns `claude` as a child which inherits this unit's env, so
|
||||||
# setting these here is all it takes to export per-agent stats.
|
# setting these here is all it takes to export per-agent stats.
|
||||||
otelEnv = lib.optionalAttrs otel.enable {
|
otelEnv = lib.optionalAttrs otel.enable (
|
||||||
CLAUDE_CODE_ENABLE_TELEMETRY = "1";
|
{
|
||||||
OTEL_METRICS_EXPORTER = "otlp";
|
CLAUDE_CODE_ENABLE_TELEMETRY = "1";
|
||||||
OTEL_LOGS_EXPORTER = "otlp";
|
OTEL_METRICS_EXPORTER = "otlp";
|
||||||
# Route traces to OTLP too so any spans Claude Code emits land
|
OTEL_LOGS_EXPORTER = "otlp";
|
||||||
# at the configured collector rather than a default exporter.
|
# Route traces to OTLP too so any spans Claude Code emits land
|
||||||
OTEL_TRACES_EXPORTER = "otlp";
|
# at the configured collector rather than a default exporter.
|
||||||
OTEL_EXPORTER_OTLP_PROTOCOL = otel.protocol;
|
OTEL_TRACES_EXPORTER = "otlp";
|
||||||
OTEL_EXPORTER_OTLP_ENDPOINT = otel.endpoint;
|
OTEL_EXPORTER_OTLP_PROTOCOL = otel.protocol;
|
||||||
};
|
OTEL_EXPORTER_OTLP_ENDPOINT = otel.endpoint;
|
||||||
|
# Force CUMULATIVE metric temporality. Claude Code defaults to
|
||||||
|
# DELTA, which Prometheus/Mimir-family backends (the common case,
|
||||||
|
# incl. grafana-lgtm) silently drop unless a deltatocumulative
|
||||||
|
# processor is wired — so delta = "logs arrive, metrics vanish".
|
||||||
|
# Cumulative is what those backends ingest natively. Verified:
|
||||||
|
# a delta export never registers the metric name; cumulative does.
|
||||||
|
OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
|
||||||
|
}
|
||||||
|
// lib.optionalAttrs (otel.metricIntervalMs != null) {
|
||||||
|
OTEL_METRIC_EXPORT_INTERVAL = toString otel.metricIntervalMs;
|
||||||
|
}
|
||||||
|
);
|
||||||
# When OTEL is on, wrap the harness launch so (1) the bearer-token
|
# When OTEL is on, wrap the harness launch so (1) the bearer-token
|
||||||
# header is read from the systemd credential at start (never in the
|
# header is read from the systemd credential at start (never in the
|
||||||
# nix store or argv) and (2) the resource attributes are assembled
|
# nix store or argv) and (2) the resource attributes are assembled
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue