16d578e692
Review catch: this PR relaxed the `otel.endpoint` assertion and staled the canonical OTEL reference in the same stroke — `docs/observability.md` is what CLAUDE.md points readers at for "what OTEL options are available", and it still said required-full-stop while the new swarm/services.md section said a local store satisfies it. Also corrects the option's own description in otel.nix, which said the same thing and renders into the generated options doc. Grepping the reviewer's phrasing did not find that one; grepping the claim did. Records the second destination where the "endpoint is where telemetry ultimately goes" paragraph makes its claim, rather than only in the new section a reader may not reach.
291 lines
13 KiB
Nix
291 lines
13 KiB
Nix
# Hive-wide OTEL stats export. Set ONCE here at host level; the
|
||
# meta-flake renderer (`hive-c0re/src/meta.rs::otel_config`) reads the
|
||
# HYPERHIVE_OTEL_* env exported off hive-c0re's unit (see
|
||
# ./hive-c0re) and injects the matching `hyperhive.otel.*` build-time
|
||
# config into EVERY agent (mirroring the CA-cert injection), so each
|
||
# agent's harness exports its own Claude Code stats directly to the
|
||
# collector. There is no per-agent opt-in — this is the single switch
|
||
# for the whole hive.
|
||
{
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
{
|
||
options.services.hyperhive.otel = {
|
||
enable = lib.mkEnableOption ''
|
||
hive-wide export of every agent's Claude Code stats (token usage,
|
||
cost, tool calls) to an OTLP endpoint via Claude Code's built-in
|
||
OpenTelemetry. One switch for all agents.
|
||
|
||
Enabling this also runs a collector on the host: there is exactly
|
||
one way telemetry leaves this hive, and it is through that
|
||
collector. Agents export unauthenticated to a bridge address only
|
||
their own containers can reach, and the collector is the only
|
||
holder of the upstream credential — an agent never sees it.
|
||
|
||
⚠️ The collector is therefore in the path of all telemetry. It runs
|
||
on the same host as the agents and restarts on failure, and
|
||
telemetry is not the control plane, so degraded telemetry is not
|
||
degraded operation — but the export no longer survives independently
|
||
of anything host-side
|
||
'';
|
||
|
||
endpoint = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "https://collector.example.com/otel";
|
||
description = ''
|
||
Upstream OTLP endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT` for
|
||
every agent.
|
||
|
||
Required when `enable` is true, **unless** this host runs the
|
||
swarm's metrics store
|
||
({option}`services.hyperhive.swarm.victoriametrics.enable`) — that
|
||
store is a destination in its own right, and with both configured
|
||
telemetry goes to both. With neither, `enable` is refused rather
|
||
than silently exporting nowhere.
|
||
'';
|
||
};
|
||
|
||
protocol = lib.mkOption {
|
||
type = lib.types.enum [
|
||
"http/protobuf"
|
||
"http/json"
|
||
"grpc"
|
||
];
|
||
default = "http/protobuf";
|
||
description = ''
|
||
OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`.
|
||
'';
|
||
};
|
||
|
||
headersCredential = lib.mkOption {
|
||
# `str`, not `path`: a `path`-typed relative literal is hash-copied
|
||
# into the world-readable nix store at eval time, defeating the
|
||
# point. Keep it a string + require an absolute runtime path so the
|
||
# secret is only ever read from disk by systemd at start.
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/run/secrets/otel-headers";
|
||
description = ''
|
||
Absolute path to an operator-provided secret file holding the
|
||
upstream auth header as `NAME=value` (e.g.
|
||
`Authorization=Bearer <token>`).
|
||
|
||
**Only the host-side collector reads this.** It reaches the
|
||
collector as an `EnvironmentFile`, so the value is never read by
|
||
nix, never copied into the store or the generated config, and
|
||
never passed in argv — and it is never forwarded into an agent
|
||
container, which is the point of the collector existing. Must be
|
||
absolute.
|
||
|
||
Leave null if the upstream needs no auth header; the collector
|
||
then sends none rather than an empty one.
|
||
'';
|
||
};
|
||
|
||
extraResourceAttributes = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "deployment.environment=prod";
|
||
description = ''
|
||
Extra comma-separated entries appended to
|
||
`OTEL_RESOURCE_ATTRIBUTES` after the built-in
|
||
`service.name` / `agent` / `hive` / `swarm` labels.
|
||
'';
|
||
};
|
||
|
||
debug = lib.mkOption {
|
||
type = lib.types.bool;
|
||
default = false;
|
||
description = ''
|
||
Emit OTEL SDK diagnostic messages to every agent's stderr by
|
||
setting `CLAUDE_CODE_OTEL_DIAG_STDERR=1`. Useful when
|
||
troubleshooting collector connectivity or endpoint config;
|
||
leave off in normal operation to avoid noise in agent logs.
|
||
Only meaningful when `enable` is true.
|
||
'';
|
||
};
|
||
|
||
collector.upstreamHeaderName = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "Authorization";
|
||
description = ''
|
||
Name of the HTTP header the collector sends upstream, whose
|
||
*value* comes from `headersCredential`.
|
||
|
||
The name is here and the value is not, and that split is forced
|
||
rather than chosen: the collector models exporter headers as a
|
||
static map, so rendering them means nix reading the value — the
|
||
one thing `headersCredential` being a path exists to prevent.
|
||
A name is public, a value is not.
|
||
|
||
⇒ exactly one header is expressible this way. A credential
|
||
carrying several (`a=1,b=2`) would be read as a single value,
|
||
which is why the shape is a named header rather than an opaque
|
||
blob: a second header has to be *declared*, not smuggled.
|
||
'';
|
||
};
|
||
|
||
collector.port = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 4318;
|
||
description = ''
|
||
Port the collector's OTLP/HTTP receiver listens on, at
|
||
`services.hyperhive.network.bridgeIp`. 4318 is the OTLP/HTTP
|
||
default.
|
||
|
||
The port is contributed to
|
||
`services.hyperhive.network.exposeHostPorts`, which opens it on
|
||
the bridge interface only — so it is reachable from agent
|
||
containers and not from the outside world.
|
||
'';
|
||
};
|
||
|
||
metricIntervalMs = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.ints.positive;
|
||
default = null;
|
||
example = 10000;
|
||
description = ''
|
||
Metric export interval in milliseconds, set as
|
||
`OTEL_METRIC_EXPORT_INTERVAL` for every agent. Claude Code's
|
||
default is 60000 (60s). Leave `null` to use that default.
|
||
|
||
Each agent runs claude as a short-lived per-turn process; claude
|
||
force-flushes metrics on shutdown, so this is not required for
|
||
metrics to be exported, but a lower value gives more frequent
|
||
intermediate flushes within long turns. Cosmetic, not a
|
||
correctness knob.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkMerge [
|
||
(lib.mkIf config.services.hyperhive.c0re.enable {
|
||
assertions = lib.optionals config.services.hyperhive.otel.enable [
|
||
{
|
||
# Telemetry has to go SOMEWHERE, but "somewhere" stopped meaning
|
||
# "an upstream endpoint" once the swarm grew its own store: a hive
|
||
# running `swarm.victoriametrics` is a complete destination on its
|
||
# own, and requiring an external endpoint as well would make the
|
||
# all-local mode impossible to express.
|
||
#
|
||
# This only ever relaxes the old rule — every config that passed
|
||
# before still passes.
|
||
assertion =
|
||
config.services.hyperhive.otel.endpoint != ""
|
||
|| config.services.hyperhive.swarm.victoriametrics.enable;
|
||
message = ''
|
||
services.hyperhive.otel.enable is true but telemetry has nowhere
|
||
to go: services.hyperhive.otel.endpoint is empty and
|
||
services.hyperhive.swarm.victoriametrics.enable is false.
|
||
|
||
Set the endpoint to export upstream, or enable the swarm's
|
||
metrics store to keep telemetry on this host.
|
||
'';
|
||
}
|
||
];
|
||
})
|
||
|
||
(lib.mkIf config.services.hyperhive.otel.enable (
|
||
(
|
||
let
|
||
otel = config.services.hyperhive.otel;
|
||
listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}";
|
||
# `otel.protocol` describes the UPSTREAM link and always did.
|
||
# Inserting a collector splits the path in two, and the
|
||
# upstream half is the one that has to keep honouring it — so
|
||
# the exporter is chosen by it, rather than the option quietly
|
||
# becoming "how agents talk to the collector". The agent half
|
||
# is pinned to OTLP/HTTP by the receiver below (derived in
|
||
# hive-c0re/environment.nix).
|
||
vmCfg = config.services.hyperhive.swarm.victoriametrics;
|
||
# Two independent destinations, either of which may be absent: an
|
||
# upstream the operator named, and the swarm's own store when this
|
||
# host runs it. The assertion above guarantees at least one.
|
||
upstreamConfigured = otel.endpoint != "";
|
||
localStore = vmCfg.enable;
|
||
|
||
storeName = "otlphttp/victoriametrics";
|
||
store = {
|
||
# ⚠️ `metrics_endpoint`, NOT `endpoint`, and the difference is
|
||
# invisible until you read the far end: `endpoint` is a BASE that
|
||
# otlphttp appends `/v1/metrics` to, while VictoriaMetrics serves
|
||
# OTLP at `/opentelemetry/api/v1/push`. With `endpoint` the
|
||
# collector still answers 200 to its own clients and the samples
|
||
# are silently posted to a path that does not exist.
|
||
# `metrics_endpoint` is used verbatim.
|
||
#
|
||
# Measured end-to-end rather than read: a real sample crossed a
|
||
# real collector into a real store, and the same probe with
|
||
# `endpoint` never arrived — see `state/probe-3265-collector-to-vm.sh`.
|
||
metrics_endpoint = "http://127.0.0.1:${toString vmCfg.port}/opentelemetry/api/v1/push";
|
||
};
|
||
|
||
grpcUpstream = otel.protocol == "grpc";
|
||
upstreamName = if grpcUpstream then "otlp" else "otlphttp";
|
||
upstream = {
|
||
endpoint = otel.endpoint;
|
||
}
|
||
// lib.optionalAttrs (otel.headersCredential != null) {
|
||
# The value is interpolated by the collector at runtime from
|
||
# its environment, never by nix. `EnvironmentFile` below is
|
||
# what puts it there. No credential configured means no
|
||
# header at all — an upstream that needs no auth is a
|
||
# legitimate deployment, and rendering `${env:…}` for a
|
||
# variable nothing sets would send the literal.
|
||
headers.${otel.collector.upstreamHeaderName} = "\${env:${otel.collector.upstreamHeaderName}}";
|
||
}
|
||
// lib.optionalAttrs (otel.protocol == "http/json") { encoding = "json"; };
|
||
in
|
||
{
|
||
# Reachable from agent containers and nowhere else: this opens
|
||
# the port on the bridge interface only.
|
||
services.hyperhive.network.exposeHostPorts = [ otel.collector.port ];
|
||
|
||
services.opentelemetry-collector = {
|
||
enable = true;
|
||
# `validateConfigFile` defaults to `isStorePath configFile`,
|
||
# and `configFile` is null on the `settings` path — so the
|
||
# upstream default is OFF for exactly the way this module
|
||
# configures it. Turning it on runs `otelcol validate` at
|
||
# build time, which is the collector checking its own config.
|
||
# ⚠️ It is a PARSER, not a wiring check, and the gap is wider
|
||
# than "no sample was sent": measured 2026-08-15, `validate`
|
||
# ACCEPTS a receiver naming an auth extension that is absent
|
||
# from the build, and the collector then dies at startup with
|
||
# `Failed to start component`. So a green build does not
|
||
# prove this config STARTS, never mind that a sample arrives.
|
||
validateConfigFile = true;
|
||
settings = {
|
||
receivers.otlp.protocols.http.endpoint = listen;
|
||
exporters =
|
||
lib.optionalAttrs upstreamConfigured { ${upstreamName} = upstream; }
|
||
// lib.optionalAttrs localStore { ${storeName} = store; };
|
||
service.pipelines.metrics = {
|
||
receivers = [ "otlp" ];
|
||
# Fan-out, not a choice: with both configured the same
|
||
# samples go upstream AND into the swarm's store. A local
|
||
# store is for looking at this swarm; an upstream is for
|
||
# whoever aggregates across swarms, and neither replaces
|
||
# the other.
|
||
exporters = lib.optional upstreamConfigured upstreamName ++ lib.optional localStore storeName;
|
||
};
|
||
};
|
||
};
|
||
|
||
# The credential file is already `NAME=value`, which is
|
||
# systemd's EnvironmentFile format — so the secret reaches the
|
||
# process as an environment variable without ever being read by
|
||
# nix, written to the store, or passed in argv.
|
||
systemd.services.opentelemetry-collector.serviceConfig =
|
||
lib.optionalAttrs (otel.headersCredential != null)
|
||
{
|
||
EnvironmentFile = otel.headersCredential;
|
||
};
|
||
}
|
||
)
|
||
))
|
||
];
|
||
}
|