c38778b885
The comment said "It parses; it does not prove a sample arrives", which understates the gap in the direction that matters: it reads as if a green build proves the collector *starts* and only runtime delivery is unverified. Measured while probing ingest-auth options for #3283: `otelcol validate` ACCEPTS a receiver naming an auth extension that is absent from the build, and the collector then dies at startup with `Failed to start component`. So the check does not prove this config starts at all. Comment-only; no evaluated config changes. Refs #3283.
237 lines
9.7 KiB
Nix
237 lines
9.7 KiB
Nix
# Hive-wide OTEL stats export. Set ONCE here at host level; the
|
||
# meta-flake renderer (`hive-c0re/src/meta.rs::otel_config`) reads the
|
||
# HYPERHIVE_OTEL_* env exported off hive-c0re's unit (see
|
||
# ./hive-c0re) and injects the matching `hyperhive.otel.*` build-time
|
||
# config into EVERY agent (mirroring the CA-cert injection), so each
|
||
# agent's harness exports its own Claude Code stats directly to the
|
||
# collector. There is no per-agent opt-in — this is the single switch
|
||
# for the whole hive.
|
||
{
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
{
|
||
options.services.hyperhive.otel = {
|
||
enable = lib.mkEnableOption ''
|
||
hive-wide export of every agent's Claude Code stats (token usage,
|
||
cost, tool calls) to an OTLP endpoint via Claude Code's built-in
|
||
OpenTelemetry. One switch for all agents.
|
||
|
||
Enabling this also runs a collector on the host: there is exactly
|
||
one way telemetry leaves this hive, and it is through that
|
||
collector. Agents export unauthenticated to a bridge address only
|
||
their own containers can reach, and the collector is the only
|
||
holder of the upstream credential — an agent never sees it.
|
||
|
||
⚠️ The collector is therefore in the path of all telemetry. It runs
|
||
on the same host as the agents and restarts on failure, and
|
||
telemetry is not the control plane, so degraded telemetry is not
|
||
degraded operation — but the export no longer survives independently
|
||
of anything host-side
|
||
'';
|
||
|
||
endpoint = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "https://collector.example.com/otel";
|
||
description = ''
|
||
OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT`
|
||
for every agent. Required when `enable` is true.
|
||
'';
|
||
};
|
||
|
||
protocol = lib.mkOption {
|
||
type = lib.types.enum [
|
||
"http/protobuf"
|
||
"http/json"
|
||
"grpc"
|
||
];
|
||
default = "http/protobuf";
|
||
description = ''
|
||
OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`.
|
||
'';
|
||
};
|
||
|
||
headersCredential = lib.mkOption {
|
||
# `str`, not `path`: a `path`-typed relative literal is hash-copied
|
||
# into the world-readable nix store at eval time, defeating the
|
||
# point. Keep it a string + require an absolute runtime path so the
|
||
# secret is only ever read from disk by systemd at start.
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/run/secrets/otel-headers";
|
||
description = ''
|
||
Absolute path to an operator-provided secret file holding the
|
||
upstream auth header as `NAME=value` (e.g.
|
||
`Authorization=Bearer <token>`).
|
||
|
||
**Only the host-side collector reads this.** It reaches the
|
||
collector as an `EnvironmentFile`, so the value is never read by
|
||
nix, never copied into the store or the generated config, and
|
||
never passed in argv — and it is never forwarded into an agent
|
||
container, which is the point of the collector existing. Must be
|
||
absolute.
|
||
|
||
Leave null if the upstream needs no auth header; the collector
|
||
then sends none rather than an empty one.
|
||
'';
|
||
};
|
||
|
||
extraResourceAttributes = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "deployment.environment=prod";
|
||
description = ''
|
||
Extra comma-separated entries appended to
|
||
`OTEL_RESOURCE_ATTRIBUTES` after the built-in
|
||
`service.name` / `agent` / `hive` / `swarm` labels.
|
||
'';
|
||
};
|
||
|
||
debug = lib.mkOption {
|
||
type = lib.types.bool;
|
||
default = false;
|
||
description = ''
|
||
Emit OTEL SDK diagnostic messages to every agent's stderr by
|
||
setting `CLAUDE_CODE_OTEL_DIAG_STDERR=1`. Useful when
|
||
troubleshooting collector connectivity or endpoint config;
|
||
leave off in normal operation to avoid noise in agent logs.
|
||
Only meaningful when `enable` is true.
|
||
'';
|
||
};
|
||
|
||
collector.upstreamHeaderName = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "Authorization";
|
||
description = ''
|
||
Name of the HTTP header the collector sends upstream, whose
|
||
*value* comes from `headersCredential`.
|
||
|
||
The name is here and the value is not, and that split is forced
|
||
rather than chosen: the collector models exporter headers as a
|
||
static map, so rendering them means nix reading the value — the
|
||
one thing `headersCredential` being a path exists to prevent.
|
||
A name is public, a value is not.
|
||
|
||
⇒ exactly one header is expressible this way. A credential
|
||
carrying several (`a=1,b=2`) would be read as a single value,
|
||
which is why the shape is a named header rather than an opaque
|
||
blob: a second header has to be *declared*, not smuggled.
|
||
'';
|
||
};
|
||
|
||
collector.port = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 4318;
|
||
description = ''
|
||
Port the collector's OTLP/HTTP receiver listens on, at
|
||
`services.hyperhive.network.bridgeIp`. 4318 is the OTLP/HTTP
|
||
default.
|
||
|
||
The port is contributed to
|
||
`services.hyperhive.network.exposeHostPorts`, which opens it on
|
||
the bridge interface only — so it is reachable from agent
|
||
containers and not from the outside world.
|
||
'';
|
||
};
|
||
|
||
metricIntervalMs = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.ints.positive;
|
||
default = null;
|
||
example = 10000;
|
||
description = ''
|
||
Metric export interval in milliseconds, set as
|
||
`OTEL_METRIC_EXPORT_INTERVAL` for every agent. Claude Code's
|
||
default is 60000 (60s). Leave `null` to use that default.
|
||
|
||
Each agent runs claude as a short-lived per-turn process; claude
|
||
force-flushes metrics on shutdown, so this is not required for
|
||
metrics to be exported, but a lower value gives more frequent
|
||
intermediate flushes within long turns. Cosmetic, not a
|
||
correctness knob.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkMerge [
|
||
(lib.mkIf config.services.hyperhive.c0re.enable {
|
||
assertions = lib.optionals config.services.hyperhive.otel.enable [
|
||
{
|
||
assertion = config.services.hyperhive.otel.endpoint != "";
|
||
message = "services.hyperhive.otel.enable is true but services.hyperhive.otel.endpoint is empty.";
|
||
}
|
||
];
|
||
})
|
||
|
||
(lib.mkIf config.services.hyperhive.otel.enable (
|
||
(
|
||
let
|
||
otel = config.services.hyperhive.otel;
|
||
listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}";
|
||
# `otel.protocol` describes the UPSTREAM link and always did.
|
||
# Inserting a collector splits the path in two, and the
|
||
# upstream half is the one that has to keep honouring it — so
|
||
# the exporter is chosen by it, rather than the option quietly
|
||
# becoming "how agents talk to the collector". The agent half
|
||
# is pinned to OTLP/HTTP by the receiver below (derived in
|
||
# hive-c0re/environment.nix).
|
||
grpcUpstream = otel.protocol == "grpc";
|
||
upstreamName = if grpcUpstream then "otlp" else "otlphttp";
|
||
upstream = {
|
||
endpoint = otel.endpoint;
|
||
}
|
||
// lib.optionalAttrs (otel.headersCredential != null) {
|
||
# The value is interpolated by the collector at runtime from
|
||
# its environment, never by nix. `EnvironmentFile` below is
|
||
# what puts it there. No credential configured means no
|
||
# header at all — an upstream that needs no auth is a
|
||
# legitimate deployment, and rendering `${env:…}` for a
|
||
# variable nothing sets would send the literal.
|
||
headers.${otel.collector.upstreamHeaderName} = "\${env:${otel.collector.upstreamHeaderName}}";
|
||
}
|
||
// lib.optionalAttrs (otel.protocol == "http/json") { encoding = "json"; };
|
||
in
|
||
{
|
||
# Reachable from agent containers and nowhere else: this opens
|
||
# the port on the bridge interface only.
|
||
services.hyperhive.network.exposeHostPorts = [ otel.collector.port ];
|
||
|
||
services.opentelemetry-collector = {
|
||
enable = true;
|
||
# `validateConfigFile` defaults to `isStorePath configFile`,
|
||
# and `configFile` is null on the `settings` path — so the
|
||
# upstream default is OFF for exactly the way this module
|
||
# configures it. Turning it on runs `otelcol validate` at
|
||
# build time, which is the collector checking its own config.
|
||
# ⚠️ It is a PARSER, not a wiring check, and the gap is wider
|
||
# than "no sample was sent": measured 2026-08-15, `validate`
|
||
# ACCEPTS a receiver naming an auth extension that is absent
|
||
# from the build, and the collector then dies at startup with
|
||
# `Failed to start component`. So a green build does not
|
||
# prove this config STARTS, never mind that a sample arrives.
|
||
validateConfigFile = true;
|
||
settings = {
|
||
receivers.otlp.protocols.http.endpoint = listen;
|
||
exporters.${upstreamName} = upstream;
|
||
service.pipelines.metrics = {
|
||
receivers = [ "otlp" ];
|
||
exporters = [ upstreamName ];
|
||
};
|
||
};
|
||
};
|
||
|
||
# The credential file is already `NAME=value`, which is
|
||
# systemd's EnvironmentFile format — so the secret reaches the
|
||
# process as an environment variable without ever being read by
|
||
# nix, written to the store, or passed in argv.
|
||
systemd.services.opentelemetry-collector.serviceConfig =
|
||
lib.optionalAttrs (otel.headersCredential != null)
|
||
{
|
||
EnvironmentFile = otel.headersCredential;
|
||
};
|
||
}
|
||
)
|
||
))
|
||
];
|
||
}
|