513eb5729d
The headersCredential section still said hive-c0re forwards the file into each agent container - that is exactly the behaviour this change removes, so the doc contradicted the code rather than merely lagging it. Same for the collector section, which presented the two paths the change collapses. Also drops the dangling collector.enable reference in the network section.
232 lines
9.3 KiB
Nix
232 lines
9.3 KiB
Nix
# Hive-wide OTEL stats export. Set ONCE here at host level; the
|
||
# meta-flake renderer (`hive-c0re/src/meta.rs::otel_config`) reads the
|
||
# HYPERHIVE_OTEL_* env exported off hive-c0re's unit (see
|
||
# ./hive-c0re) and injects the matching `hyperhive.otel.*` build-time
|
||
# config into EVERY agent (mirroring the CA-cert injection), so each
|
||
# agent's harness exports its own Claude Code stats directly to the
|
||
# collector. There is no per-agent opt-in — this is the single switch
|
||
# for the whole hive.
|
||
{
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
{
|
||
options.services.hyperhive.otel = {
|
||
enable = lib.mkEnableOption ''
|
||
hive-wide export of every agent's Claude Code stats (token usage,
|
||
cost, tool calls) to an OTLP endpoint via Claude Code's built-in
|
||
OpenTelemetry. One switch for all agents.
|
||
|
||
Enabling this also runs a collector on the host: there is exactly
|
||
one way telemetry leaves this hive, and it is through that
|
||
collector. Agents export unauthenticated to a bridge address only
|
||
their own containers can reach, and the collector is the only
|
||
holder of the upstream credential — an agent never sees it.
|
||
|
||
⚠️ The collector is therefore in the path of all telemetry. It runs
|
||
on the same host as the agents and restarts on failure, and
|
||
telemetry is not the control plane, so degraded telemetry is not
|
||
degraded operation — but the export no longer survives independently
|
||
of anything host-side
|
||
'';
|
||
|
||
endpoint = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "https://collector.example.com/otel";
|
||
description = ''
|
||
OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT`
|
||
for every agent. Required when `enable` is true.
|
||
'';
|
||
};
|
||
|
||
protocol = lib.mkOption {
|
||
type = lib.types.enum [
|
||
"http/protobuf"
|
||
"http/json"
|
||
"grpc"
|
||
];
|
||
default = "http/protobuf";
|
||
description = ''
|
||
OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`.
|
||
'';
|
||
};
|
||
|
||
headersCredential = lib.mkOption {
|
||
# `str`, not `path`: a `path`-typed relative literal is hash-copied
|
||
# into the world-readable nix store at eval time, defeating the
|
||
# point. Keep it a string + require an absolute runtime path so the
|
||
# secret is only ever read from disk by systemd at start.
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/run/secrets/otel-headers";
|
||
description = ''
|
||
Absolute path to an operator-provided secret file holding the
|
||
upstream auth header as `NAME=value` (e.g.
|
||
`Authorization=Bearer <token>`).
|
||
|
||
**Only the host-side collector reads this.** It reaches the
|
||
collector as an `EnvironmentFile`, so the value is never read by
|
||
nix, never copied into the store or the generated config, and
|
||
never passed in argv — and it is never forwarded into an agent
|
||
container, which is the point of the collector existing. Must be
|
||
absolute.
|
||
|
||
Leave null if the upstream needs no auth header; the collector
|
||
then sends none rather than an empty one.
|
||
'';
|
||
};
|
||
|
||
extraResourceAttributes = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "deployment.environment=prod";
|
||
description = ''
|
||
Extra comma-separated entries appended to
|
||
`OTEL_RESOURCE_ATTRIBUTES` after the built-in
|
||
`service.name` / `agent` / `hive` / `swarm` labels.
|
||
'';
|
||
};
|
||
|
||
debug = lib.mkOption {
|
||
type = lib.types.bool;
|
||
default = false;
|
||
description = ''
|
||
Emit OTEL SDK diagnostic messages to every agent's stderr by
|
||
setting `CLAUDE_CODE_OTEL_DIAG_STDERR=1`. Useful when
|
||
troubleshooting collector connectivity or endpoint config;
|
||
leave off in normal operation to avoid noise in agent logs.
|
||
Only meaningful when `enable` is true.
|
||
'';
|
||
};
|
||
|
||
collector.upstreamHeaderName = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "Authorization";
|
||
description = ''
|
||
Name of the HTTP header the collector sends upstream, whose
|
||
*value* comes from `headersCredential`.
|
||
|
||
The name is here and the value is not, and that split is forced
|
||
rather than chosen: the collector models exporter headers as a
|
||
static map, so rendering them means nix reading the value — the
|
||
one thing `headersCredential` being a path exists to prevent.
|
||
A name is public, a value is not.
|
||
|
||
⇒ exactly one header is expressible this way. A credential
|
||
carrying several (`a=1,b=2`) would be read as a single value,
|
||
which is why the shape is a named header rather than an opaque
|
||
blob: a second header has to be *declared*, not smuggled.
|
||
'';
|
||
};
|
||
|
||
collector.port = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 4318;
|
||
description = ''
|
||
Port the collector's OTLP/HTTP receiver listens on, at
|
||
`services.hyperhive.network.bridgeIp`. 4318 is the OTLP/HTTP
|
||
default.
|
||
|
||
The port is contributed to
|
||
`services.hyperhive.network.exposeHostPorts`, which opens it on
|
||
the bridge interface only — so it is reachable from agent
|
||
containers and not from the outside world.
|
||
'';
|
||
};
|
||
|
||
metricIntervalMs = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.ints.positive;
|
||
default = null;
|
||
example = 10000;
|
||
description = ''
|
||
Metric export interval in milliseconds, set as
|
||
`OTEL_METRIC_EXPORT_INTERVAL` for every agent. Claude Code's
|
||
default is 60000 (60s). Leave `null` to use that default.
|
||
|
||
Each agent runs claude as a short-lived per-turn process; claude
|
||
force-flushes metrics on shutdown, so this is not required for
|
||
metrics to be exported, but a lower value gives more frequent
|
||
intermediate flushes within long turns. Cosmetic, not a
|
||
correctness knob.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkMerge [
|
||
(lib.mkIf config.services.hyperhive.c0re.enable {
|
||
assertions = lib.optionals config.services.hyperhive.otel.enable [
|
||
{
|
||
assertion = config.services.hyperhive.otel.endpoint != "";
|
||
message = "services.hyperhive.otel.enable is true but services.hyperhive.otel.endpoint is empty.";
|
||
}
|
||
];
|
||
})
|
||
|
||
(lib.mkIf config.services.hyperhive.otel.enable (
|
||
(
|
||
let
|
||
otel = config.services.hyperhive.otel;
|
||
listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}";
|
||
# `otel.protocol` describes the UPSTREAM link and always did.
|
||
# Inserting a collector splits the path in two, and the
|
||
# upstream half is the one that has to keep honouring it — so
|
||
# the exporter is chosen by it, rather than the option quietly
|
||
# becoming "how agents talk to the collector". The agent half
|
||
# is pinned to OTLP/HTTP by the receiver below (derived in
|
||
# hive-c0re/environment.nix).
|
||
grpcUpstream = otel.protocol == "grpc";
|
||
upstreamName = if grpcUpstream then "otlp" else "otlphttp";
|
||
upstream = {
|
||
endpoint = otel.endpoint;
|
||
}
|
||
// lib.optionalAttrs (otel.headersCredential != null) {
|
||
# The value is interpolated by the collector at runtime from
|
||
# its environment, never by nix. `EnvironmentFile` below is
|
||
# what puts it there. No credential configured means no
|
||
# header at all — an upstream that needs no auth is a
|
||
# legitimate deployment, and rendering `${env:…}` for a
|
||
# variable nothing sets would send the literal.
|
||
headers.${otel.collector.upstreamHeaderName} = "\${env:${otel.collector.upstreamHeaderName}}";
|
||
}
|
||
// lib.optionalAttrs (otel.protocol == "http/json") { encoding = "json"; };
|
||
in
|
||
{
|
||
# Reachable from agent containers and nowhere else: this opens
|
||
# the port on the bridge interface only.
|
||
services.hyperhive.network.exposeHostPorts = [ otel.collector.port ];
|
||
|
||
services.opentelemetry-collector = {
|
||
enable = true;
|
||
# `validateConfigFile` defaults to `isStorePath configFile`,
|
||
# and `configFile` is null on the `settings` path — so the
|
||
# upstream default is OFF for exactly the way this module
|
||
# configures it. Turning it on runs `otelcol validate` at
|
||
# build time, which is the collector checking its own config.
|
||
# ⚠️ It parses; it does not prove a sample arrives.
|
||
validateConfigFile = true;
|
||
settings = {
|
||
receivers.otlp.protocols.http.endpoint = listen;
|
||
exporters.${upstreamName} = upstream;
|
||
service.pipelines.metrics = {
|
||
receivers = [ "otlp" ];
|
||
exporters = [ upstreamName ];
|
||
};
|
||
};
|
||
};
|
||
|
||
# The credential file is already `NAME=value`, which is
|
||
# systemd's EnvironmentFile format — so the secret reaches the
|
||
# process as an environment variable without ever being read by
|
||
# nix, written to the store, or passed in argv.
|
||
systemd.services.opentelemetry-collector.serviceConfig =
|
||
lib.optionalAttrs (otel.headersCredential != null)
|
||
{
|
||
EnvironmentFile = otel.headersCredential;
|
||
};
|
||
}
|
||
)
|
||
))
|
||
];
|
||
}
|