d2fb4bff79
Drops swarm.otel.url (a loopback default an operator had to override on a split host) in favor of swarm.otel.domain -- the same gateway.localNames + nginx-vhost-through-the-gateway shape every other swarm service (authelia, grafana, victoriametrics, ui) already uses. The hive tier's exporter now reaches it as https://<domain> unconditionally, resolved locally by dnsmasq on a co-located host and over the real network otherwise, instead of a config knob nobody sets until they hit the silent drop. Costs CA trust on the hive tier: otel.nix wires lib/hive-ca-trust.nix's trustBundle with hostUnit = true on the opentelemetry-collector host unit, the same flag #3441/#3442 added for swarm-controller and hive-c0re. mara, #3125 comment 58363: "go c".
277 lines
12 KiB
Nix
277 lines
12 KiB
Nix
# Hive-wide OTEL stats export, and the HIVE tier of the collector pair.
|
||
# Set ONCE here at host level; the meta-flake renderer
|
||
# (`hive-c0re/src/meta.rs::otel_config`) reads the HYPERHIVE_OTEL_* env
|
||
# exported off hive-c0re's unit (see ./hive-c0re) and injects the matching
|
||
# `hyperhive.otel.*` build-time config into EVERY agent (mirroring the
|
||
# CA-cert injection), so each agent's harness exports its own Claude Code
|
||
# stats directly to the collector. There is no per-agent opt-in — this is
|
||
# the single switch for the whole hive.
|
||
#
|
||
# This tier receives from this hive's agents and forwards to the swarm's
|
||
# collector (./swarm-otel.nix). It holds no credential and picks no
|
||
# destination: an agent's samples cross a hive boundary exactly once, and
|
||
# what happens after that is the swarm's decision, not a hive's. The
|
||
# upstream options declared below describe that far end and are read one
|
||
# tier up — they stay here because they mean what they have always meant.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
# This tier now reaches the swarm's collector by name through the
|
||
# gateway (`swarm-otel.nix`'s `domain`) instead of a loopback URL, so it
|
||
# needs the same hive-CA trust every other host consumer of an `https://`
|
||
# swarm-service name needs — see `swarm-controller.nix` for the sibling
|
||
# wiring this copies.
|
||
#
|
||
# `hostUnit`: `opentelemetry-collector` is a host systemd service, not a
|
||
# container, so it reads the CA from the host path and the bundle oneshot
|
||
# waits on `hive-tls-ca.service` itself. `enable`: `imports` is
|
||
# unconditional at the host's top level, so without it a hive with this
|
||
# tier off would still get a bundle oneshot and a phantom
|
||
# `opentelemetry-collector` service holding an `SSL_CERT_FILE`.
|
||
caTrust = import ./lib/hive-ca-trust.nix {
|
||
inherit lib;
|
||
tlsCfg = config.services.hyperhive.tls;
|
||
gatewayCfg = config.services.hyperhive.gateway;
|
||
};
|
||
in
|
||
{
|
||
imports = [
|
||
(caTrust.trustBundle {
|
||
inherit pkgs;
|
||
name = "hive-otel";
|
||
consumers = [ "opentelemetry-collector" ];
|
||
hostUnit = true;
|
||
enable = config.services.hyperhive.otel.enable;
|
||
})
|
||
];
|
||
|
||
options.services.hyperhive.otel = {
|
||
enable = lib.mkEnableOption ''
|
||
hive-wide export of every agent's Claude Code stats (token usage,
|
||
cost, tool calls) to an OTLP endpoint via Claude Code's built-in
|
||
OpenTelemetry. One switch for all agents.
|
||
|
||
Enabling this also runs this hive's collector on the host: there is
|
||
exactly one way telemetry leaves this hive, and it is through that
|
||
collector. Agents export unauthenticated to a bridge address only
|
||
their own containers can reach.
|
||
|
||
That collector forwards to the swarm's
|
||
({option}`services.hyperhive.swarm.otel.enable`), which holds the
|
||
upstream credential and writes the swarm's store. So an agent never
|
||
sees the credential, and neither does this tier.
|
||
|
||
⚠️ The collector is therefore in the path of all telemetry. It runs
|
||
on the same host as the agents and restarts on failure, and
|
||
telemetry is not the control plane, so degraded telemetry is not
|
||
degraded operation — but the export no longer survives independently
|
||
of anything host-side
|
||
'';
|
||
|
||
endpoint = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "https://collector.example.com/otel";
|
||
description = ''
|
||
Upstream OTLP endpoint: where telemetry ultimately goes, after it
|
||
has left the swarm.
|
||
|
||
Read by the swarm's collector
|
||
({option}`services.hyperhive.swarm.otel.enable`), which is the only
|
||
tier that holds the upstream credential. An agent is handed the
|
||
*first* hop instead — this hive's own collector — so this value is
|
||
never given to a container.
|
||
|
||
Optional. Leave it empty and the swarm's own metrics store
|
||
({option}`services.hyperhive.swarm.victoriametrics.enable`) is the
|
||
destination; that is a complete deployment, not a degraded one.
|
||
Set both and telemetry goes to both.
|
||
'';
|
||
};
|
||
|
||
protocol = lib.mkOption {
|
||
type = lib.types.enum [
|
||
"http/protobuf"
|
||
"http/json"
|
||
"grpc"
|
||
];
|
||
default = "http/protobuf";
|
||
description = ''
|
||
OTLP wire protocol for the **upstream** link, honoured by the
|
||
swarm collector's exporter.
|
||
|
||
Not what agents speak: their first hop is this hive's collector,
|
||
whose OTLP/HTTP receiver takes protobuf whatever the upstream
|
||
wants (see `hive-c0re/environment.nix`).
|
||
'';
|
||
};
|
||
|
||
headersCredential = lib.mkOption {
|
||
# `str`, not `path`: a `path`-typed relative literal is hash-copied
|
||
# into the world-readable nix store at eval time, defeating the
|
||
# point. Keep it a string + require an absolute runtime path so the
|
||
# secret is only ever read from disk by systemd at start.
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/run/secrets/otel-headers";
|
||
description = ''
|
||
Absolute path to an operator-provided secret file holding the
|
||
upstream auth header as `NAME=value` (e.g.
|
||
`Authorization=Bearer <token>`).
|
||
|
||
**Only the swarm's collector reads this** — the one tier that
|
||
talks to the upstream. It arrives as an `EnvironmentFile`, so the
|
||
value is never read by nix, never copied into the store or the
|
||
generated config, and never passed in argv; and it reaches
|
||
neither an agent container nor this hive's own collector, which
|
||
is the point of the tiers existing. Must be absolute.
|
||
|
||
Leave null if the upstream needs no auth header; the collector
|
||
then sends none rather than an empty one.
|
||
'';
|
||
};
|
||
|
||
extraResourceAttributes = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "deployment.environment=prod";
|
||
description = ''
|
||
Extra comma-separated entries appended to
|
||
`OTEL_RESOURCE_ATTRIBUTES` after the built-in
|
||
`service.name` / `agent` / `hive` / `swarm` labels.
|
||
'';
|
||
};
|
||
|
||
debug = lib.mkOption {
|
||
type = lib.types.bool;
|
||
default = false;
|
||
description = ''
|
||
Emit OTEL SDK diagnostic messages to every agent's stderr by
|
||
setting `CLAUDE_CODE_OTEL_DIAG_STDERR=1`. Useful when
|
||
troubleshooting collector connectivity or endpoint config;
|
||
leave off in normal operation to avoid noise in agent logs.
|
||
Only meaningful when `enable` is true.
|
||
'';
|
||
};
|
||
|
||
collector.upstreamHeaderName = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "Authorization";
|
||
description = ''
|
||
Name of the HTTP header the collector sends upstream, whose
|
||
*value* comes from `headersCredential`.
|
||
|
||
The name is here and the value is not, and that split is forced
|
||
rather than chosen: the collector models exporter headers as a
|
||
static map, so rendering them means nix reading the value — the
|
||
one thing `headersCredential` being a path exists to prevent.
|
||
A name is public, a value is not.
|
||
|
||
⇒ exactly one header is expressible this way. A credential
|
||
carrying several (`a=1,b=2`) would be read as a single value,
|
||
which is why the shape is a named header rather than an opaque
|
||
blob: a second header has to be *declared*, not smuggled.
|
||
'';
|
||
};
|
||
|
||
collector.port = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 4318;
|
||
description = ''
|
||
Port the collector's OTLP/HTTP receiver listens on, at
|
||
`services.hyperhive.network.bridgeIp`. 4318 is the OTLP/HTTP
|
||
default.
|
||
|
||
The port is contributed to
|
||
`services.hyperhive.network.exposeHostPorts`, which opens it on
|
||
the bridge interface only — so it is reachable from agent
|
||
containers and not from the outside world.
|
||
'';
|
||
};
|
||
|
||
metricIntervalMs = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.ints.positive;
|
||
default = null;
|
||
example = 10000;
|
||
description = ''
|
||
Metric export interval in milliseconds, set as
|
||
`OTEL_METRIC_EXPORT_INTERVAL` for every agent. Claude Code's
|
||
default is 60000 (60s). Leave `null` to use that default.
|
||
|
||
Each agent runs claude as a short-lived per-turn process; claude
|
||
force-flushes metrics on shutdown, so this is not required for
|
||
metrics to be exported, but a lower value gives more frequent
|
||
intermediate flushes within long turns. Cosmetic, not a
|
||
correctness knob.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkIf config.services.hyperhive.otel.enable (
|
||
let
|
||
otel = config.services.hyperhive.otel;
|
||
listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}";
|
||
swarmName = "otlphttp/swarm";
|
||
in
|
||
{
|
||
# Reachable from agent containers and nowhere else: this opens
|
||
# the port on the bridge interface only.
|
||
services.hyperhive.network.exposeHostPorts = [ otel.collector.port ];
|
||
|
||
services.opentelemetry-collector = {
|
||
enable = true;
|
||
# `validateConfigFile` defaults to `isStorePath configFile`,
|
||
# and `configFile` is null on the `settings` path — so the
|
||
# upstream default is OFF for exactly the way this module
|
||
# configures it. Turning it on runs `otelcol validate` at
|
||
# build time, which is the collector checking its own config.
|
||
# ⚠️ It is a PARSER, not a wiring check, and the gap is wider
|
||
# than "no sample was sent": measured 2026-08-15, `validate`
|
||
# ACCEPTS a receiver naming an auth extension that is absent
|
||
# from the build, and the collector then dies at startup with
|
||
# `Failed to start component`. So a green build does not
|
||
# prove this config STARTS, never mind that a sample arrives.
|
||
validateConfigFile = true;
|
||
settings = {
|
||
receivers.otlp.protocols.http.endpoint = listen;
|
||
|
||
# One destination, and it is the swarm's collector. This tier
|
||
# holds no upstream credential and writes no store: it receives
|
||
# from this hive's agents and forwards, which is the whole of
|
||
# its job. Everything that decides where telemetry ultimately
|
||
# goes lives one tier up, in ./swarm-otel.nix.
|
||
exporters.${swarmName} = {
|
||
# ⚠️ Plain `endpoint`, and the exporter beside this one in
|
||
# ./swarm-otel.nix warns against exactly that spelling — read
|
||
# both before "fixing" either. The difference is the far end,
|
||
# not the exporter: `endpoint` is a BASE that otlphttp appends
|
||
# `/v1/metrics` to, which is precisely the path an OTLP/HTTP
|
||
# receiver serves. VictoriaMetrics is the odd one out, serving
|
||
# OTLP at `/opentelemetry/api/v1/push`, and that is why the
|
||
# store exporter needs `metrics_endpoint` while this one must
|
||
# not have it.
|
||
#
|
||
# By name through the gateway, not a loopback literal: a
|
||
# loopback literal is correct only while listener and caller
|
||
# share a netns, an assumption that has cost this project two
|
||
# outages, and it is exactly the split-host case a swarm
|
||
# service name exists to make a config fact rather than a code
|
||
# change. `https://` because that name resolves through the
|
||
# gateway even on a co-located host — see `caTrust` above for
|
||
# the trust half that makes this verify.
|
||
endpoint = "https://${config.services.hyperhive.swarm.otel.domain}";
|
||
};
|
||
|
||
service.pipelines.metrics = {
|
||
receivers = [ "otlp" ];
|
||
exporters = [ swarmName ];
|
||
};
|
||
};
|
||
};
|
||
}
|
||
);
|
||
}
|