9ad4db842a
Renders services.opentelemetry-collector on the host: OTLP/HTTP receiver on the bridge address, otlphttp exporter to otel.endpoint, and the upstream credential delivered as EnvironmentFile so the collector interpolates it at runtime and nix never sees the value. Three things worth knowing, each measured rather than assumed: - network.exposeHostPorts already exists and is wired (it opens the port on the bridge interface only), so bridge reachability costs nothing. - validateConfigFile defaults to isStorePath configFile, which is null on the settings path - so upstream's default is OFF for exactly the way this module configures it. Set true: it runs otelcol validate at build time. It parses, it does not prove delivery. - headersCredential's file is already NAME=value, i.e. EnvironmentFile format, verified against a real settings.json rather than the doc. An assertion refuses collector.enable with no headersCredential: the collector exists to be the only holder of that token, and without one it is indirection that reads as security.
241 lines
9.7 KiB
Nix
241 lines
9.7 KiB
Nix
# Hive-wide OTEL stats export. Set ONCE here at host level; the
|
||
# meta-flake renderer (`hive-c0re/src/meta.rs::otel_config`) reads the
|
||
# HYPERHIVE_OTEL_* env exported off hive-c0re's unit (see
|
||
# ./hive-c0re) and injects the matching `hyperhive.otel.*` build-time
|
||
# config into EVERY agent (mirroring the CA-cert injection), so each
|
||
# agent's harness exports its own Claude Code stats directly to the
|
||
# collector. There is no per-agent opt-in — this is the single switch
|
||
# for the whole hive.
|
||
{
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
{
|
||
options.services.hyperhive.otel = {
|
||
enable = lib.mkEnableOption ''
|
||
hive-wide export of every agent's Claude Code stats (token usage,
|
||
cost, tool calls) to an OTLP endpoint via Claude Code's built-in
|
||
OpenTelemetry. One switch for all agents; each harness exports
|
||
directly to the collector, so it keeps working even when hive-c0re
|
||
is down
|
||
'';
|
||
|
||
endpoint = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "https://collector.example.com/otel";
|
||
description = ''
|
||
OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT`
|
||
for every agent. Required when `enable` is true.
|
||
'';
|
||
};
|
||
|
||
protocol = lib.mkOption {
|
||
type = lib.types.enum [
|
||
"http/protobuf"
|
||
"http/json"
|
||
"grpc"
|
||
];
|
||
default = "http/protobuf";
|
||
description = ''
|
||
OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`.
|
||
'';
|
||
};
|
||
|
||
headersCredential = lib.mkOption {
|
||
# `str`, not `path`: a `path`-typed relative literal is hash-copied
|
||
# into the world-readable nix store at eval time, defeating the
|
||
# point. Keep it a string + require an absolute runtime path so the
|
||
# secret is only ever read from disk by systemd at start.
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/run/secrets/otel-headers";
|
||
description = ''
|
||
Absolute path to an operator-provided secret file whose contents
|
||
become `OTEL_EXPORTER_OTLP_HEADERS` (e.g.
|
||
`Authorization=Bearer <token>`). hive-c0re forwards this host
|
||
file into each agent container's credential store via
|
||
systemd-nspawn `--load-credential=otel-headers:<path>`; the inner
|
||
harness unit inherits it by name (`LoadCredential`), so the token
|
||
is never copied into the nix store, the generated config, a bind
|
||
mount, or argv. Must be absolute. Leave null if the endpoint
|
||
needs no auth header. A configured-but-missing file is skipped
|
||
with a log warning (OTEL still exports, without the auth header).
|
||
'';
|
||
};
|
||
|
||
extraResourceAttributes = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "deployment.environment=prod";
|
||
description = ''
|
||
Extra comma-separated entries appended to
|
||
`OTEL_RESOURCE_ATTRIBUTES` after the built-in
|
||
`service.name` / `agent` / `hive` / `swarm` labels.
|
||
'';
|
||
};
|
||
|
||
debug = lib.mkOption {
|
||
type = lib.types.bool;
|
||
default = false;
|
||
description = ''
|
||
Emit OTEL SDK diagnostic messages to every agent's stderr by
|
||
setting `CLAUDE_CODE_OTEL_DIAG_STDERR=1`. Useful when
|
||
troubleshooting collector connectivity or endpoint config;
|
||
leave off in normal operation to avoid noise in agent logs.
|
||
Only meaningful when `enable` is true.
|
||
'';
|
||
};
|
||
|
||
collector.enable = lib.mkOption {
|
||
type = lib.types.bool;
|
||
default = false;
|
||
description = ''
|
||
Run an OpenTelemetry collector on this host and have agents
|
||
export to it instead of straight to `endpoint`.
|
||
|
||
The point is the credential. Without this, every agent needs
|
||
`headersCredential` in order to talk to the upstream — and the
|
||
harness delivers it into the agent's own `settings.json`, where
|
||
the agent can read it. With a collector the token stops at the
|
||
host: the collector holds it, agents send unauthenticated to a
|
||
bridge address only their own containers can reach.
|
||
|
||
⚠️ It also makes the collector a dependency in the export path.
|
||
Today each harness exports directly, so telemetry survives
|
||
anything host-side being down. That property is traded for the
|
||
credential reduction; the collector is on the same host as the
|
||
agents, so the window is small, but it is not zero.
|
||
|
||
Off by default, and off means *absent*: no unit, no port, and
|
||
`endpoint` keeps its current meaning for every agent.
|
||
'';
|
||
};
|
||
|
||
collector.upstreamHeaderName = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "Authorization";
|
||
description = ''
|
||
Name of the HTTP header the collector sends upstream, whose
|
||
*value* comes from `headersCredential`.
|
||
|
||
The name is here and the value is not, and that split is forced
|
||
rather than chosen: the collector models exporter headers as a
|
||
static map, so rendering them means nix reading the value — the
|
||
one thing `headersCredential` being a path exists to prevent.
|
||
A name is public, a value is not.
|
||
|
||
⇒ exactly one header is expressible this way. A credential
|
||
carrying several (`a=1,b=2`) would be read as a single value,
|
||
which is why the shape is a named header rather than an opaque
|
||
blob: a second header has to be *declared*, not smuggled.
|
||
'';
|
||
};
|
||
|
||
collector.port = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 4318;
|
||
description = ''
|
||
Port the collector's OTLP/HTTP receiver listens on, at
|
||
`services.hyperhive.network.bridgeIp`. 4318 is the OTLP/HTTP
|
||
default.
|
||
|
||
The port is contributed to
|
||
`services.hyperhive.network.exposeHostPorts`, which opens it on
|
||
the bridge interface only — so it is reachable from agent
|
||
containers and not from the outside world.
|
||
'';
|
||
};
|
||
|
||
metricIntervalMs = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.ints.positive;
|
||
default = null;
|
||
example = 10000;
|
||
description = ''
|
||
Metric export interval in milliseconds, set as
|
||
`OTEL_METRIC_EXPORT_INTERVAL` for every agent. Claude Code's
|
||
default is 60000 (60s). Leave `null` to use that default.
|
||
|
||
Each agent runs claude as a short-lived per-turn process; claude
|
||
force-flushes metrics on shutdown, so this is not required for
|
||
metrics to be exported, but a lower value gives more frequent
|
||
intermediate flushes within long turns. Cosmetic, not a
|
||
correctness knob.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkMerge [
|
||
(lib.mkIf config.services.hyperhive.c0re.enable {
|
||
assertions = lib.optionals config.services.hyperhive.otel.enable [
|
||
{
|
||
assertion = config.services.hyperhive.otel.endpoint != "";
|
||
message = "services.hyperhive.otel.enable is true but services.hyperhive.otel.endpoint is empty.";
|
||
}
|
||
];
|
||
})
|
||
|
||
(lib.mkIf (config.services.hyperhive.otel.enable && config.services.hyperhive.otel.collector.enable)
|
||
(
|
||
let
|
||
otel = config.services.hyperhive.otel;
|
||
listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}";
|
||
in
|
||
{
|
||
assertions = [
|
||
{
|
||
# The collector's whole purpose is to hold the credential so
|
||
# agents do not have to. With none configured it is pure
|
||
# indirection, and the operator has almost certainly not got
|
||
# the deployment they think they have.
|
||
assertion = otel.headersCredential != null;
|
||
message = ''
|
||
services.hyperhive.otel.collector.enable is true but
|
||
otel.headersCredential is null. The collector exists to be
|
||
the only holder of the upstream credential; with no
|
||
credential it just forwards, and every agent's exporter
|
||
would be unauthenticated end to end.
|
||
'';
|
||
}
|
||
];
|
||
|
||
# Reachable from agent containers and nowhere else: this opens
|
||
# the port on the bridge interface only.
|
||
services.hyperhive.network.exposeHostPorts = [ otel.collector.port ];
|
||
|
||
services.opentelemetry-collector = {
|
||
enable = true;
|
||
# `validateConfigFile` defaults to `isStorePath configFile`,
|
||
# and `configFile` is null on the `settings` path — so the
|
||
# upstream default is OFF for exactly the way this module
|
||
# configures it. Turning it on runs `otelcol validate` at
|
||
# build time, which is the collector checking its own config.
|
||
# ⚠️ It parses; it does not prove a sample arrives.
|
||
validateConfigFile = true;
|
||
settings = {
|
||
receivers.otlp.protocols.http.endpoint = listen;
|
||
exporters.otlphttp = {
|
||
endpoint = otel.endpoint;
|
||
# The value is interpolated by the collector at runtime
|
||
# from its environment, never by nix. `EnvironmentFile`
|
||
# below is what puts it there.
|
||
headers.${otel.collector.upstreamHeaderName} = "\${env:${otel.collector.upstreamHeaderName}}";
|
||
};
|
||
service.pipelines.metrics = {
|
||
receivers = [ "otlp" ];
|
||
exporters = [ "otlphttp" ];
|
||
};
|
||
};
|
||
};
|
||
|
||
# The credential file is already `NAME=value`, which is
|
||
# systemd's EnvironmentFile format — so the secret reaches the
|
||
# process as an environment variable without ever being read by
|
||
# nix, written to the store, or passed in argv.
|
||
systemd.services.opentelemetry-collector.serviceConfig.EnvironmentFile = otel.headersCredential;
|
||
}
|
||
)
|
||
)
|
||
];
|
||
}
|