0f801c645f
The hive tier has to *present* a credential to the swarm tier, and the upstream collector build ships no auth extensions at all — `oauth2client` exists only in contrib. So this is what makes authenticated egress expressible, not a preference between two equivalent packages. It also matches the swarm tier, which has been on contrib since it was written. Two tiers of the same pipeline built from different component sets is a difference nobody would predict from reading either module. Not a build-farm cost: contrib is fetched, not compiled. No behaviour change on its own — nothing names an extension yet. The config that does lands separately, because that is the commit whose failure mode needs a running collector to detect: `otelcol validate` accepts a receiver naming an absent extension and the process then dies at startup, so a green build proves nothing about it.
293 lines
12 KiB
Nix
293 lines
12 KiB
Nix
# Hive-wide OTEL stats export, and the HIVE tier of the collector pair.
|
||
# Set ONCE here at host level; the meta-flake renderer
|
||
# (`hive-c0re/src/meta.rs::otel_config`) reads the HYPERHIVE_OTEL_* env
|
||
# exported off hive-c0re's unit (see ./hive-c0re) and injects the matching
|
||
# `hyperhive.otel.*` build-time config into EVERY agent (mirroring the
|
||
# CA-cert injection), so each agent's harness exports its own Claude Code
|
||
# stats directly to the collector. There is no per-agent opt-in — this is
|
||
# the single switch for the whole hive.
|
||
#
|
||
# This tier receives from this hive's agents and forwards to the swarm's
|
||
# collector (./swarm-otel.nix). It holds no credential and picks no
|
||
# destination: an agent's samples cross a hive boundary exactly once, and
|
||
# what happens after that is the swarm's decision, not a hive's. The
|
||
# upstream options declared below describe that far end and are read one
|
||
# tier up — they stay here because they mean what they have always meant.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
# This tier now reaches the swarm's collector by name through the
|
||
# gateway (`swarm-otel.nix`'s `domain`) instead of a loopback URL, so it
|
||
# needs the same hive-CA trust every other host consumer of an `https://`
|
||
# swarm-service name needs — see `swarm-controller.nix` for the sibling
|
||
# wiring this copies.
|
||
#
|
||
# `hostUnit`: `opentelemetry-collector` is a host systemd service, not a
|
||
# container, so it reads the CA from the host path and the bundle oneshot
|
||
# waits on `hive-tls-ca.service` itself. `enable`: `imports` is
|
||
# unconditional at the host's top level, so without it a hive with this
|
||
# tier off would still get a bundle oneshot and a phantom
|
||
# `opentelemetry-collector` service holding an `SSL_CERT_FILE`.
|
||
caTrust = import ./lib/hive-ca-trust.nix {
|
||
inherit lib;
|
||
tlsCfg = config.services.hyperhive.tls;
|
||
gatewayCfg = config.services.hyperhive.gateway;
|
||
};
|
||
in
|
||
{
|
||
imports = [
|
||
(caTrust.trustBundle {
|
||
inherit pkgs;
|
||
name = "hive-otel";
|
||
consumers = [ "opentelemetry-collector" ];
|
||
hostUnit = true;
|
||
enable = config.services.hyperhive.otel.enable;
|
||
})
|
||
];
|
||
|
||
options.services.hyperhive.otel = {
|
||
enable = lib.mkEnableOption ''
|
||
hive-wide export of every agent's Claude Code stats (token usage,
|
||
cost, tool calls) to an OTLP endpoint via Claude Code's built-in
|
||
OpenTelemetry. One switch for all agents.
|
||
|
||
Enabling this also runs this hive's collector on the host: there is
|
||
exactly one way telemetry leaves this hive, and it is through that
|
||
collector. Agents export unauthenticated to a bridge address only
|
||
their own containers can reach.
|
||
|
||
That collector forwards to the swarm's
|
||
({option}`services.hyperhive.swarm.otel.enable`), which holds the
|
||
upstream credential and writes the swarm's store. So an agent never
|
||
sees the credential, and neither does this tier.
|
||
|
||
⚠️ The collector is therefore in the path of all telemetry. It runs
|
||
on the same host as the agents and restarts on failure, and
|
||
telemetry is not the control plane, so degraded telemetry is not
|
||
degraded operation — but the export no longer survives independently
|
||
of anything host-side
|
||
'';
|
||
|
||
endpoint = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "https://collector.example.com/otel";
|
||
description = ''
|
||
Upstream OTLP endpoint: where telemetry ultimately goes, after it
|
||
has left the swarm.
|
||
|
||
Read by the swarm's collector
|
||
({option}`services.hyperhive.swarm.otel.enable`), which is the only
|
||
tier that holds the upstream credential. An agent is handed the
|
||
*first* hop instead — this hive's own collector — so this value is
|
||
never given to a container.
|
||
|
||
Optional. Leave it empty and the swarm's own metrics store
|
||
({option}`services.hyperhive.swarm.victoriametrics.enable`) is the
|
||
destination; that is a complete deployment, not a degraded one.
|
||
Set both and telemetry goes to both.
|
||
'';
|
||
};
|
||
|
||
protocol = lib.mkOption {
|
||
type = lib.types.enum [
|
||
"http/protobuf"
|
||
"http/json"
|
||
"grpc"
|
||
];
|
||
default = "http/protobuf";
|
||
description = ''
|
||
OTLP wire protocol for the **upstream** link, honoured by the
|
||
swarm collector's exporter.
|
||
|
||
Not what agents speak: their first hop is this hive's collector,
|
||
whose OTLP/HTTP receiver takes protobuf whatever the upstream
|
||
wants (see `hive-c0re/environment.nix`).
|
||
'';
|
||
};
|
||
|
||
headersCredential = lib.mkOption {
|
||
# `str`, not `path`: a `path`-typed relative literal is hash-copied
|
||
# into the world-readable nix store at eval time, defeating the
|
||
# point. Keep it a string + require an absolute runtime path so the
|
||
# secret is only ever read from disk by systemd at start.
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/run/secrets/otel-headers";
|
||
description = ''
|
||
Absolute path to an operator-provided secret file holding the
|
||
upstream auth header as `NAME=value` (e.g.
|
||
`Authorization=Bearer <token>`).
|
||
|
||
**Only the swarm's collector reads this** — the one tier that
|
||
talks to the upstream. It arrives as an `EnvironmentFile`, so the
|
||
value is never read by nix, never copied into the store or the
|
||
generated config, and never passed in argv; and it reaches
|
||
neither an agent container nor this hive's own collector, which
|
||
is the point of the tiers existing. Must be absolute.
|
||
|
||
Leave null if the upstream needs no auth header; the collector
|
||
then sends none rather than an empty one.
|
||
'';
|
||
};
|
||
|
||
extraResourceAttributes = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "deployment.environment=prod";
|
||
description = ''
|
||
Extra comma-separated entries appended to
|
||
`OTEL_RESOURCE_ATTRIBUTES` after the built-in
|
||
`service.name` / `agent` / `hive` / `swarm` labels.
|
||
'';
|
||
};
|
||
|
||
debug = lib.mkOption {
|
||
type = lib.types.bool;
|
||
default = false;
|
||
description = ''
|
||
Emit OTEL SDK diagnostic messages to every agent's stderr by
|
||
setting `CLAUDE_CODE_OTEL_DIAG_STDERR=1`. Useful when
|
||
troubleshooting collector connectivity or endpoint config;
|
||
leave off in normal operation to avoid noise in agent logs.
|
||
Only meaningful when `enable` is true.
|
||
'';
|
||
};
|
||
|
||
collector.upstreamHeaderName = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "Authorization";
|
||
description = ''
|
||
Name of the HTTP header the collector sends upstream, whose
|
||
*value* comes from `headersCredential`.
|
||
|
||
The name is here and the value is not, and that split is forced
|
||
rather than chosen: the collector models exporter headers as a
|
||
static map, so rendering them means nix reading the value — the
|
||
one thing `headersCredential` being a path exists to prevent.
|
||
A name is public, a value is not.
|
||
|
||
⇒ exactly one header is expressible this way. A credential
|
||
carrying several (`a=1,b=2`) would be read as a single value,
|
||
which is why the shape is a named header rather than an opaque
|
||
blob: a second header has to be *declared*, not smuggled.
|
||
'';
|
||
};
|
||
|
||
collector.port = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 4318;
|
||
description = ''
|
||
Port the collector's OTLP/HTTP receiver listens on, at
|
||
`services.hyperhive.network.bridgeIp`. 4318 is the OTLP/HTTP
|
||
default.
|
||
|
||
The port is contributed to
|
||
`services.hyperhive.network.exposeHostPorts`, which opens it on
|
||
the bridge interface only — so it is reachable from agent
|
||
containers and not from the outside world.
|
||
'';
|
||
};
|
||
|
||
metricIntervalMs = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.ints.positive;
|
||
default = null;
|
||
example = 10000;
|
||
description = ''
|
||
Metric export interval in milliseconds, set as
|
||
`OTEL_METRIC_EXPORT_INTERVAL` for every agent. Claude Code's
|
||
default is 60000 (60s). Leave `null` to use that default.
|
||
|
||
Each agent runs claude as a short-lived per-turn process; claude
|
||
force-flushes metrics on shutdown, so this is not required for
|
||
metrics to be exported, but a lower value gives more frequent
|
||
intermediate flushes within long turns. Cosmetic, not a
|
||
correctness knob.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkIf config.services.hyperhive.otel.enable (
|
||
let
|
||
otel = config.services.hyperhive.otel;
|
||
listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}";
|
||
swarmName = "otlphttp/swarm";
|
||
in
|
||
{
|
||
# Reachable from agent containers and nowhere else: this opens
|
||
# the port on the bridge interface only.
|
||
services.hyperhive.network.exposeHostPorts = [ otel.collector.port ];
|
||
|
||
services.opentelemetry-collector = {
|
||
enable = true;
|
||
# Contrib, matching the swarm tier (./swarm-otel.nix). The upstream
|
||
# default build has no auth extensions at all, and this tier has to
|
||
# *present* a credential to the swarm tier — `oauth2client` lives
|
||
# only in contrib, so the package choice is what makes authenticated
|
||
# egress expressible rather than a preference.
|
||
#
|
||
# Not a build-farm cost: contrib is fetched, not compiled.
|
||
#
|
||
# ⚠️ Read the note directly below before adding any extension here.
|
||
# It describes precisely the trap this package unlocks: naming an
|
||
# extension the build lacks passes `validate` and then kills the
|
||
# collector at startup. With contrib the extensions exist — but the
|
||
# gap it warns about (a green build proving nothing about whether
|
||
# the process starts) is exactly why this module's auth wiring is
|
||
# gated by a probe that runs both collectors, not by eval.
|
||
package = pkgs.opentelemetry-collector-contrib;
|
||
# `validateConfigFile` defaults to `isStorePath configFile`,
|
||
# and `configFile` is null on the `settings` path — so the
|
||
# upstream default is OFF for exactly the way this module
|
||
# configures it. Turning it on runs `otelcol validate` at
|
||
# build time, which is the collector checking its own config.
|
||
# ⚠️ It is a PARSER, not a wiring check, and the gap is wider
|
||
# than "no sample was sent": measured 2026-08-15, `validate`
|
||
# ACCEPTS a receiver naming an auth extension that is absent
|
||
# from the build, and the collector then dies at startup with
|
||
# `Failed to start component`. So a green build does not
|
||
# prove this config STARTS, never mind that a sample arrives.
|
||
validateConfigFile = true;
|
||
settings = {
|
||
receivers.otlp.protocols.http.endpoint = listen;
|
||
|
||
# One destination, and it is the swarm's collector. This tier
|
||
# holds no upstream credential and writes no store: it receives
|
||
# from this hive's agents and forwards, which is the whole of
|
||
# its job. Everything that decides where telemetry ultimately
|
||
# goes lives one tier up, in ./swarm-otel.nix.
|
||
exporters.${swarmName} = {
|
||
# ⚠️ Plain `endpoint`, and the exporter beside this one in
|
||
# ./swarm-otel.nix warns against exactly that spelling — read
|
||
# both before "fixing" either. The difference is the far end,
|
||
# not the exporter: `endpoint` is a BASE that otlphttp appends
|
||
# `/v1/metrics` to, which is precisely the path an OTLP/HTTP
|
||
# receiver serves. VictoriaMetrics is the odd one out, serving
|
||
# OTLP at `/opentelemetry/api/v1/push`, and that is why the
|
||
# store exporter needs `metrics_endpoint` while this one must
|
||
# not have it.
|
||
#
|
||
# By name through the gateway, not a loopback literal: a
|
||
# loopback literal is correct only while listener and caller
|
||
# share a netns, an assumption that has cost this project two
|
||
# outages, and it is exactly the split-host case a swarm
|
||
# service name exists to make a config fact rather than a code
|
||
# change. `https://` because that name resolves through the
|
||
# gateway even on a co-located host — see `caTrust` above for
|
||
# the trust half that makes this verify.
|
||
endpoint = "https://${config.services.hyperhive.swarm.otel.domain}";
|
||
};
|
||
|
||
service.pipelines.metrics = {
|
||
receivers = [ "otlp" ];
|
||
exporters = [ swarmName ];
|
||
};
|
||
};
|
||
};
|
||
}
|
||
);
|
||
}
|