1fc267b880
The module hardcoded otlphttp, so a hive with protocol = "grpc" would have had its agents' protocol respected end to end before the collector and silently rewritten to HTTP after it. Splitting the path in two makes the upstream half the one that has to keep honouring the option; the agent half is pinned to the receiver's protocol and derived, not configured.
252 lines
10 KiB
Nix
252 lines
10 KiB
Nix
# Hive-wide OTEL stats export. Set ONCE here at host level; the
|
||
# meta-flake renderer (`hive-c0re/src/meta.rs::otel_config`) reads the
|
||
# HYPERHIVE_OTEL_* env exported off hive-c0re's unit (see
|
||
# ./hive-c0re) and injects the matching `hyperhive.otel.*` build-time
|
||
# config into EVERY agent (mirroring the CA-cert injection), so each
|
||
# agent's harness exports its own Claude Code stats directly to the
|
||
# collector. There is no per-agent opt-in — this is the single switch
|
||
# for the whole hive.
|
||
{
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
{
|
||
options.services.hyperhive.otel = {
|
||
enable = lib.mkEnableOption ''
|
||
hive-wide export of every agent's Claude Code stats (token usage,
|
||
cost, tool calls) to an OTLP endpoint via Claude Code's built-in
|
||
OpenTelemetry. One switch for all agents; each harness exports
|
||
directly to the collector, so it keeps working even when hive-c0re
|
||
is down
|
||
'';
|
||
|
||
endpoint = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "https://collector.example.com/otel";
|
||
description = ''
|
||
OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT`
|
||
for every agent. Required when `enable` is true.
|
||
'';
|
||
};
|
||
|
||
protocol = lib.mkOption {
|
||
type = lib.types.enum [
|
||
"http/protobuf"
|
||
"http/json"
|
||
"grpc"
|
||
];
|
||
default = "http/protobuf";
|
||
description = ''
|
||
OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`.
|
||
'';
|
||
};
|
||
|
||
headersCredential = lib.mkOption {
|
||
# `str`, not `path`: a `path`-typed relative literal is hash-copied
|
||
# into the world-readable nix store at eval time, defeating the
|
||
# point. Keep it a string + require an absolute runtime path so the
|
||
# secret is only ever read from disk by systemd at start.
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/run/secrets/otel-headers";
|
||
description = ''
|
||
Absolute path to an operator-provided secret file whose contents
|
||
become `OTEL_EXPORTER_OTLP_HEADERS` (e.g.
|
||
`Authorization=Bearer <token>`). hive-c0re forwards this host
|
||
file into each agent container's credential store via
|
||
systemd-nspawn `--load-credential=otel-headers:<path>`; the inner
|
||
harness unit inherits it by name (`LoadCredential`), so the token
|
||
is never copied into the nix store, the generated config, a bind
|
||
mount, or argv. Must be absolute. Leave null if the endpoint
|
||
needs no auth header. A configured-but-missing file is skipped
|
||
with a log warning (OTEL still exports, without the auth header).
|
||
'';
|
||
};
|
||
|
||
extraResourceAttributes = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "deployment.environment=prod";
|
||
description = ''
|
||
Extra comma-separated entries appended to
|
||
`OTEL_RESOURCE_ATTRIBUTES` after the built-in
|
||
`service.name` / `agent` / `hive` / `swarm` labels.
|
||
'';
|
||
};
|
||
|
||
debug = lib.mkOption {
|
||
type = lib.types.bool;
|
||
default = false;
|
||
description = ''
|
||
Emit OTEL SDK diagnostic messages to every agent's stderr by
|
||
setting `CLAUDE_CODE_OTEL_DIAG_STDERR=1`. Useful when
|
||
troubleshooting collector connectivity or endpoint config;
|
||
leave off in normal operation to avoid noise in agent logs.
|
||
Only meaningful when `enable` is true.
|
||
'';
|
||
};
|
||
|
||
collector.enable = lib.mkOption {
|
||
type = lib.types.bool;
|
||
default = false;
|
||
description = ''
|
||
Run an OpenTelemetry collector on this host and have agents
|
||
export to it instead of straight to `endpoint`.
|
||
|
||
The point is the credential. Without this, every agent needs
|
||
`headersCredential` in order to talk to the upstream — and the
|
||
harness delivers it into the agent's own `settings.json`, where
|
||
the agent can read it. With a collector the token stops at the
|
||
host: the collector holds it, agents send unauthenticated to a
|
||
bridge address only their own containers can reach.
|
||
|
||
⚠️ It also makes the collector a dependency in the export path.
|
||
Today each harness exports directly, so telemetry survives
|
||
anything host-side being down. That property is traded for the
|
||
credential reduction; the collector is on the same host as the
|
||
agents, so the window is small, but it is not zero.
|
||
|
||
Off by default, and off means *absent*: no unit, no port, and
|
||
`endpoint` keeps its current meaning for every agent.
|
||
'';
|
||
};
|
||
|
||
collector.upstreamHeaderName = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "Authorization";
|
||
description = ''
|
||
Name of the HTTP header the collector sends upstream, whose
|
||
*value* comes from `headersCredential`.
|
||
|
||
The name is here and the value is not, and that split is forced
|
||
rather than chosen: the collector models exporter headers as a
|
||
static map, so rendering them means nix reading the value — the
|
||
one thing `headersCredential` being a path exists to prevent.
|
||
A name is public, a value is not.
|
||
|
||
⇒ exactly one header is expressible this way. A credential
|
||
carrying several (`a=1,b=2`) would be read as a single value,
|
||
which is why the shape is a named header rather than an opaque
|
||
blob: a second header has to be *declared*, not smuggled.
|
||
'';
|
||
};
|
||
|
||
collector.port = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 4318;
|
||
description = ''
|
||
Port the collector's OTLP/HTTP receiver listens on, at
|
||
`services.hyperhive.network.bridgeIp`. 4318 is the OTLP/HTTP
|
||
default.
|
||
|
||
The port is contributed to
|
||
`services.hyperhive.network.exposeHostPorts`, which opens it on
|
||
the bridge interface only — so it is reachable from agent
|
||
containers and not from the outside world.
|
||
'';
|
||
};
|
||
|
||
metricIntervalMs = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.ints.positive;
|
||
default = null;
|
||
example = 10000;
|
||
description = ''
|
||
Metric export interval in milliseconds, set as
|
||
`OTEL_METRIC_EXPORT_INTERVAL` for every agent. Claude Code's
|
||
default is 60000 (60s). Leave `null` to use that default.
|
||
|
||
Each agent runs claude as a short-lived per-turn process; claude
|
||
force-flushes metrics on shutdown, so this is not required for
|
||
metrics to be exported, but a lower value gives more frequent
|
||
intermediate flushes within long turns. Cosmetic, not a
|
||
correctness knob.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkMerge [
|
||
(lib.mkIf config.services.hyperhive.c0re.enable {
|
||
assertions = lib.optionals config.services.hyperhive.otel.enable [
|
||
{
|
||
assertion = config.services.hyperhive.otel.endpoint != "";
|
||
message = "services.hyperhive.otel.enable is true but services.hyperhive.otel.endpoint is empty.";
|
||
}
|
||
];
|
||
})
|
||
|
||
(lib.mkIf (config.services.hyperhive.otel.enable && config.services.hyperhive.otel.collector.enable)
|
||
(
|
||
let
|
||
otel = config.services.hyperhive.otel;
|
||
listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}";
|
||
# `otel.protocol` describes the UPSTREAM link and always did.
|
||
# Inserting a collector splits the path in two, and the
|
||
# upstream half is the one that has to keep honouring it — so
|
||
# the exporter is chosen by it, rather than the option quietly
|
||
# becoming "how agents talk to the collector". The agent half
|
||
# is pinned to OTLP/HTTP by the receiver below (derived in
|
||
# hive-c0re/environment.nix).
|
||
grpcUpstream = otel.protocol == "grpc";
|
||
upstreamName = if grpcUpstream then "otlp" else "otlphttp";
|
||
upstream = {
|
||
endpoint = otel.endpoint;
|
||
# The value is interpolated by the collector at runtime from
|
||
# its environment, never by nix. `EnvironmentFile` below is
|
||
# what puts it there.
|
||
headers.${otel.collector.upstreamHeaderName} = "\${env:${otel.collector.upstreamHeaderName}}";
|
||
}
|
||
// lib.optionalAttrs (otel.protocol == "http/json") { encoding = "json"; };
|
||
in
|
||
{
|
||
assertions = [
|
||
{
|
||
# The collector's whole purpose is to hold the credential so
|
||
# agents do not have to. With none configured it is pure
|
||
# indirection, and the operator has almost certainly not got
|
||
# the deployment they think they have.
|
||
assertion = otel.headersCredential != null;
|
||
message = ''
|
||
services.hyperhive.otel.collector.enable is true but
|
||
otel.headersCredential is null. The collector exists to be
|
||
the only holder of the upstream credential; with no
|
||
credential it just forwards, and every agent's exporter
|
||
would be unauthenticated end to end.
|
||
'';
|
||
}
|
||
];
|
||
|
||
# Reachable from agent containers and nowhere else: this opens
|
||
# the port on the bridge interface only.
|
||
services.hyperhive.network.exposeHostPorts = [ otel.collector.port ];
|
||
|
||
services.opentelemetry-collector = {
|
||
enable = true;
|
||
# `validateConfigFile` defaults to `isStorePath configFile`,
|
||
# and `configFile` is null on the `settings` path — so the
|
||
# upstream default is OFF for exactly the way this module
|
||
# configures it. Turning it on runs `otelcol validate` at
|
||
# build time, which is the collector checking its own config.
|
||
# ⚠️ It parses; it does not prove a sample arrives.
|
||
validateConfigFile = true;
|
||
settings = {
|
||
receivers.otlp.protocols.http.endpoint = listen;
|
||
exporters.${upstreamName} = upstream;
|
||
service.pipelines.metrics = {
|
||
receivers = [ "otlp" ];
|
||
exporters = [ upstreamName ];
|
||
};
|
||
};
|
||
};
|
||
|
||
# The credential file is already `NAME=value`, which is
|
||
# systemd's EnvironmentFile format — so the secret reaches the
|
||
# process as an environment variable without ever being read by
|
||
# nix, written to the store, or passed in argv.
|
||
systemd.services.opentelemetry-collector.serviceConfig.EnvironmentFile = otel.headersCredential;
|
||
}
|
||
)
|
||
)
|
||
];
|
||
}
|