7da7915150
Turning ingest auth off without clearing a hive's credential leaves that hive's collector authenticating and addressing its own path, while an unauthenticated swarm tier serves one catch-all and forwards the URI unchanged. The receiver is asked for a path it does not serve, so telemetry stops with 404s and retries — no 401, no assertion, nothing in any log naming auth. Only reachable by overriding one side without the other, since both defaults derive from the same flag. That is what makes it worth a build error rather than a caveat: an operator who flips the documented escape hatch has no reason to suspect the sending half. Found in review by argus.
444 lines
20 KiB
Nix
444 lines
20 KiB
Nix
# Hive-wide OTEL stats export, and the HIVE tier of the collector pair.
|
||
# Set ONCE here at host level; the meta-flake renderer
|
||
# (`hive-c0re/src/meta.rs::otel_config`) reads the HYPERHIVE_OTEL_* env
|
||
# exported off hive-c0re's unit (see ./hive-c0re) and injects the matching
|
||
# `hyperhive.otel.*` build-time config into EVERY agent (mirroring the
|
||
# CA-cert injection), so each agent's harness exports its own Claude Code
|
||
# stats directly to the collector. There is no per-agent opt-in — this is
|
||
# the single switch for the whole hive.
|
||
#
|
||
# This tier receives from this hive's agents and forwards to the swarm's
|
||
# collector (./swarm-otel.nix). It holds no credential and picks no
|
||
# destination: an agent's samples cross a hive boundary exactly once, and
|
||
# what happens after that is the swarm's decision, not a hive's. The
|
||
# upstream options declared below describe that far end and are read one
|
||
# tier up — they stay here because they mean what they have always meant.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
# This tier now reaches the swarm's collector by name through the
|
||
# gateway (`swarm-otel.nix`'s `domain`) instead of a loopback URL, so it
|
||
# needs the same hive-CA trust every other host consumer of an `https://`
|
||
# swarm-service name needs — see `swarm-controller.nix` for the sibling
|
||
# wiring this copies.
|
||
#
|
||
# `hostUnit`: `opentelemetry-collector` is a host systemd service, not a
|
||
# container, so it reads the CA from the host path and the bundle oneshot
|
||
# waits on `hive-tls-ca.service` itself. `enable`: `imports` is
|
||
# unconditional at the host's top level, so without it a hive with this
|
||
# tier off would still get a bundle oneshot and a phantom
|
||
# `opentelemetry-collector` service holding an `SSL_CERT_FILE`.
|
||
caTrust = import ./lib/hive-ca-trust.nix {
|
||
inherit lib;
|
||
tlsCfg = config.services.hyperhive.tls;
|
||
gatewayCfg = config.services.hyperhive.gateway;
|
||
};
|
||
in
|
||
{
|
||
imports = [
|
||
(caTrust.trustBundle {
|
||
inherit pkgs;
|
||
name = "hive-otel";
|
||
consumers = [ "opentelemetry-collector" ];
|
||
hostUnit = true;
|
||
enable = config.services.hyperhive.otel.enable;
|
||
})
|
||
];
|
||
|
||
options.services.hyperhive.otel = {
|
||
enable = lib.mkEnableOption ''
|
||
hive-wide export of every agent's Claude Code stats (token usage,
|
||
cost, tool calls) to an OTLP endpoint via Claude Code's built-in
|
||
OpenTelemetry. One switch for all agents.
|
||
|
||
Enabling this also runs this hive's collector on the host: there is
|
||
exactly one way telemetry leaves this hive, and it is through that
|
||
collector. Agents export unauthenticated to a bridge address only
|
||
their own containers can reach.
|
||
|
||
That collector forwards to the swarm's
|
||
({option}`services.hyperhive.swarm.otel.enable`), which holds the
|
||
upstream credential and writes the swarm's store. So an agent never
|
||
sees the credential, and neither does this tier.
|
||
|
||
⚠️ The collector is therefore in the path of all telemetry. It runs
|
||
on the same host as the agents and restarts on failure, and
|
||
telemetry is not the control plane, so degraded telemetry is not
|
||
degraded operation — but the export no longer survives independently
|
||
of anything host-side
|
||
'';
|
||
|
||
endpoint = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "https://collector.example.com/otel";
|
||
description = ''
|
||
Upstream OTLP endpoint: where telemetry ultimately goes, after it
|
||
has left the swarm.
|
||
|
||
Read by the swarm's collector
|
||
({option}`services.hyperhive.swarm.otel.enable`), which is the only
|
||
tier that holds the upstream credential. An agent is handed the
|
||
*first* hop instead — this hive's own collector — so this value is
|
||
never given to a container.
|
||
|
||
Optional. Leave it empty and the swarm's own metrics store
|
||
({option}`services.hyperhive.swarm.victoriametrics.enable`) is the
|
||
destination; that is a complete deployment, not a degraded one.
|
||
Set both and telemetry goes to both.
|
||
'';
|
||
};
|
||
|
||
protocol = lib.mkOption {
|
||
type = lib.types.enum [
|
||
"http/protobuf"
|
||
"http/json"
|
||
"grpc"
|
||
];
|
||
default = "http/protobuf";
|
||
description = ''
|
||
OTLP wire protocol for the **upstream** link, honoured by the
|
||
swarm collector's exporter.
|
||
|
||
Not what agents speak: their first hop is this hive's collector,
|
||
whose OTLP/HTTP receiver takes protobuf whatever the upstream
|
||
wants (see `hive-c0re/environment.nix`).
|
||
'';
|
||
};
|
||
|
||
headersCredential = lib.mkOption {
|
||
# `str`, not `path`: a `path`-typed relative literal is hash-copied
|
||
# into the world-readable nix store at eval time, defeating the
|
||
# point. Keep it a string + require an absolute runtime path so the
|
||
# secret is only ever read from disk by systemd at start.
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/run/secrets/otel-headers";
|
||
description = ''
|
||
Absolute path to an operator-provided secret file holding the
|
||
upstream auth header as `NAME=value` (e.g.
|
||
`Authorization=Bearer <token>`).
|
||
|
||
**Only the swarm's collector reads this** — the one tier that
|
||
talks to the upstream. It arrives as an `EnvironmentFile`, so the
|
||
value is never read by nix, never copied into the store or the
|
||
generated config, and never passed in argv; and it reaches
|
||
neither an agent container nor this hive's own collector, which
|
||
is the point of the tiers existing. Must be absolute.
|
||
|
||
Leave null if the upstream needs no auth header; the collector
|
||
then sends none rather than an empty one.
|
||
'';
|
||
};
|
||
|
||
extraResourceAttributes = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "deployment.environment=prod";
|
||
description = ''
|
||
Extra comma-separated entries appended to
|
||
`OTEL_RESOURCE_ATTRIBUTES` after the built-in
|
||
`service.name` / `agent` / `hive` / `swarm` labels.
|
||
'';
|
||
};
|
||
|
||
debug = lib.mkOption {
|
||
type = lib.types.bool;
|
||
default = false;
|
||
description = ''
|
||
Emit OTEL SDK diagnostic messages to every agent's stderr by
|
||
setting `CLAUDE_CODE_OTEL_DIAG_STDERR=1`. Useful when
|
||
troubleshooting collector connectivity or endpoint config;
|
||
leave off in normal operation to avoid noise in agent logs.
|
||
Only meaningful when `enable` is true.
|
||
'';
|
||
};
|
||
|
||
collector.upstreamHeaderName = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "Authorization";
|
||
description = ''
|
||
Name of the HTTP header the collector sends upstream, whose
|
||
*value* comes from `headersCredential`.
|
||
|
||
The name is here and the value is not, and that split is forced
|
||
rather than chosen: the collector models exporter headers as a
|
||
static map, so rendering them means nix reading the value — the
|
||
one thing `headersCredential` being a path exists to prevent.
|
||
A name is public, a value is not.
|
||
|
||
⇒ exactly one header is expressible this way. A credential
|
||
carrying several (`a=1,b=2`) would be read as a single value,
|
||
which is why the shape is a named header rather than an opaque
|
||
blob: a second header has to be *declared*, not smuggled.
|
||
'';
|
||
};
|
||
|
||
collector.port = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 4318;
|
||
description = ''
|
||
Port the collector's OTLP/HTTP receiver listens on, at
|
||
`services.hyperhive.network.bridgeIp`. 4318 is the OTLP/HTTP
|
||
default.
|
||
|
||
The port is contributed to
|
||
`services.hyperhive.network.exposeHostPorts`, which opens it on
|
||
the bridge interface only — so it is reachable from agent
|
||
containers and not from the outside world.
|
||
'';
|
||
};
|
||
|
||
clientSecretFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default =
|
||
if
|
||
config.services.hyperhive.swarm.authelia.enable && config.services.hyperhive.hiveName != null
|
||
then
|
||
"${config.services.hyperhive.swarm.authelia.hostClientSecretDir}/"
|
||
+ "${config.services.hyperhive.swarm.authelia.hiveClientPrefix}${config.services.hyperhive.hiveName}.secret"
|
||
else
|
||
null;
|
||
defaultText = lib.literalExpression ''"''${swarm.authelia.hostClientSecretDir}/''${swarm.authelia.hiveClientPrefix}''${hiveName}.secret" when this host runs the swarm's IdP, else null'';
|
||
example = "/var/lib/secrets/hive-telemetry.secret";
|
||
description = ''
|
||
Absolute path to this hive's OAuth2 client secret, used to
|
||
authenticate to the swarm's collector as this hive.
|
||
|
||
**Whether this is set is what decides whether this tier
|
||
authenticates at all.** A credential is the only thing that makes
|
||
authenticated egress possible, so its presence is the condition
|
||
rather than a second switch that could disagree with it.
|
||
|
||
Defaults to the secret this host's own authelia minted, which is
|
||
correct exactly when the swarm's IdP runs here. On a hive that is
|
||
not that host, the file has to arrive some other way and this
|
||
option names wherever it landed — the same manual-copy shape
|
||
`services.hyperhive.swarm.authelia.oidc.hiveIdentities` documents,
|
||
where delivering a secret to a hive that is not this host is
|
||
deliberately not solved.
|
||
|
||
Read by `LoadCredential`, so it is never evaluated by nix, never
|
||
copied into the store and never passed in argv.
|
||
'';
|
||
};
|
||
|
||
metricIntervalMs = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.ints.positive;
|
||
default = null;
|
||
example = 10000;
|
||
description = ''
|
||
Metric export interval in milliseconds, set as
|
||
`OTEL_METRIC_EXPORT_INTERVAL` for every agent. Claude Code's
|
||
default is 60000 (60s). Leave `null` to use that default.
|
||
|
||
Each agent runs claude as a short-lived per-turn process; claude
|
||
force-flushes metrics on shutdown, so this is not required for
|
||
metrics to be exported, but a lower value gives more frequent
|
||
intermediate flushes within long turns. Cosmetic, not a
|
||
correctness knob.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkIf config.services.hyperhive.otel.enable (
|
||
let
|
||
otel = config.services.hyperhive.otel;
|
||
autheliaCfg = config.services.hyperhive.swarm.authelia;
|
||
swarmOtelCfg = config.services.hyperhive.swarm.otel;
|
||
hiveName = config.services.hyperhive.hiveName;
|
||
listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}";
|
||
swarmName = "otlphttp/swarm";
|
||
authName = "oauth2client/swarm";
|
||
|
||
# Holding a credential IS the condition — see `clientSecretFile`.
|
||
senderAuth = otel.clientSecretFile != null && hiveName != null;
|
||
|
||
# This hive's client id, and also the audience it must ASK for. Both
|
||
# are `hiveClientPrefix` + the hive's name because that is the one
|
||
# name the swarm already agrees on; the receiver one tier up derives
|
||
# the same string.
|
||
hiveClient = "${autheliaCfg.hiveClientPrefix}${toString hiveName}";
|
||
|
||
# systemd exports `CREDENTIALS_DIRECTORY` to any unit with
|
||
# `LoadCredential`, and the collector expands `${env:…}` at load. So
|
||
# the secret reaches the process as a PATH resolved at runtime — nix
|
||
# renders neither the value nor the directory, and nothing has to
|
||
# hardcode `/run/credentials/<unit>`.
|
||
credName = "swarm-client.secret";
|
||
in
|
||
{
|
||
# Reachable from agent containers and nowhere else: this opens
|
||
# the port on the bridge interface only.
|
||
services.hyperhive.network.exposeHostPorts = [ otel.collector.port ];
|
||
|
||
services.opentelemetry-collector = {
|
||
enable = true;
|
||
# Contrib, matching the swarm tier (./swarm-otel.nix). The upstream
|
||
# default build has no auth extensions at all, and this tier has to
|
||
# *present* a credential to the swarm tier — `oauth2client` lives
|
||
# only in contrib, so the package choice is what makes authenticated
|
||
# egress expressible rather than a preference.
|
||
#
|
||
# Not a build-farm cost: contrib is fetched, not compiled.
|
||
#
|
||
# ⚠️ Read the note directly below before adding any extension here.
|
||
# It describes precisely the trap this package unlocks: naming an
|
||
# extension the build lacks passes `validate` and then kills the
|
||
# collector at startup. With contrib the extensions exist — but the
|
||
# gap it warns about (a green build proving nothing about whether
|
||
# the process starts) is exactly why this module's auth wiring is
|
||
# gated by a probe that runs both collectors, not by eval.
|
||
package = pkgs.opentelemetry-collector-contrib;
|
||
# `validateConfigFile` defaults to `isStorePath configFile`,
|
||
# and `configFile` is null on the `settings` path — so the
|
||
# upstream default is OFF for exactly the way this module
|
||
# configures it. Turning it on runs `otelcol validate` at
|
||
# build time, which is the collector checking its own config.
|
||
# ⚠️ It is a PARSER, not a wiring check, and the gap is wider
|
||
# than "no sample was sent": measured 2026-08-15, `validate`
|
||
# ACCEPTS a receiver naming an auth extension that is absent
|
||
# from the build, and the collector then dies at startup with
|
||
# `Failed to start component`. So a green build does not
|
||
# prove this config STARTS, never mind that a sample arrives.
|
||
validateConfigFile = true;
|
||
settings = {
|
||
receivers.otlp.protocols.http.endpoint = listen;
|
||
|
||
# One destination, and it is the swarm's collector. This tier
|
||
# holds no upstream credential and writes no store: it receives
|
||
# from this hive's agents and forwards, which is the whole of
|
||
# its job. Everything that decides where telemetry ultimately
|
||
# goes lives one tier up, in ./swarm-otel.nix.
|
||
exporters.${swarmName} = {
|
||
# ⚠️ Plain `endpoint`, and the exporter beside this one in
|
||
# ./swarm-otel.nix warns against exactly that spelling — read
|
||
# both before "fixing" either. The difference is the far end,
|
||
# not the exporter: `endpoint` is a BASE that otlphttp appends
|
||
# `/v1/metrics` to, which is precisely the path an OTLP/HTTP
|
||
# receiver serves. VictoriaMetrics is the odd one out, serving
|
||
# OTLP at `/opentelemetry/api/v1/push`, and that is why the
|
||
# store exporter needs `metrics_endpoint` while this one must
|
||
# not have it.
|
||
#
|
||
# By name through the gateway, not a loopback literal: a
|
||
# loopback literal is correct only while listener and caller
|
||
# share a netns, an assumption that has cost this project two
|
||
# outages, and it is exactly the split-host case a swarm
|
||
# service name exists to make a config fact rather than a code
|
||
# change. `https://` because that name resolves through the
|
||
# gateway even on a co-located host — see `caTrust` above for
|
||
# the trust half that makes this verify.
|
||
# The hive's own path under the collector's single name. The
|
||
# swarm tier gives each hive its own authenticated receiver and
|
||
# routes to it by this prefix, so the path is not decoration —
|
||
# it selects WHICH receiver, and therefore which hive the
|
||
# samples get labelled as.
|
||
endpoint = "https://${swarmOtelCfg.domain}" + lib.optionalString senderAuth "/${toString hiveName}";
|
||
}
|
||
// lib.optionalAttrs senderAuth { auth.authenticator = authName; };
|
||
|
||
# ⚠️ An extension not listed here is INERT: the collector starts
|
||
# clean and the exporter naming it sends nothing authenticated.
|
||
service.extensions = lib.optional senderAuth authName;
|
||
|
||
service.pipelines.metrics = {
|
||
receivers = [ "otlp" ];
|
||
exporters = [ swarmName ];
|
||
};
|
||
}
|
||
// lib.optionalAttrs senderAuth {
|
||
extensions.${authName} = {
|
||
client_id = hiveClient;
|
||
# A real key, measured against this collector version rather
|
||
# than assumed — with a deliberate typo rejected in the same
|
||
# run, so "accepted" is distinguishable from "ignores
|
||
# everything". Keeps the secret out of nix entirely: the
|
||
# collector opens the file itself.
|
||
client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${credName}";
|
||
token_url = "${toString autheliaCfg.url}/api/oidc/token";
|
||
# ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted.
|
||
# Registering it on the client only makes it permissible; a
|
||
# token minted without asking carries `aud: []` and every
|
||
# receiver refuses it — with a config that reads perfectly at
|
||
# both ends. Measured against authelia 4.39.20.
|
||
endpoint_params.audience = hiveClient;
|
||
};
|
||
};
|
||
};
|
||
|
||
# `LoadCredential` and not a copy-oneshot: this collector is a HOST
|
||
# unit, so there is no container boundary to cross and therefore no
|
||
# reason for a second on-disk copy of the secret. systemd hands it to
|
||
# the process in a private tmpfs and exports the directory, which is
|
||
# what the config above names.
|
||
systemd.services.opentelemetry-collector.serviceConfig = lib.optionalAttrs senderAuth {
|
||
LoadCredential = [ "${credName}:${otel.clientSecretFile}" ];
|
||
};
|
||
|
||
assertions = [
|
||
{
|
||
# Only checkable on a host that runs BOTH tiers — which is the
|
||
# deployment where it can actually go wrong silently. A remote
|
||
# hive cannot see the swarm tier's config at all, so its operator
|
||
# sets the path explicitly and this says nothing.
|
||
assertion = !(swarmOtelCfg.enable && swarmOtelCfg.requireHiveIdentity) || senderAuth;
|
||
message = ''
|
||
This host runs the swarm's telemetry collector with
|
||
services.hyperhive.swarm.otel.requireHiveIdentity = true, so
|
||
ingest is authenticated per hive — but this hive's own
|
||
collector has no credential to present:
|
||
|
||
services.hyperhive.otel.clientSecretFile = ${
|
||
if otel.clientSecretFile == null then "null" else otel.clientSecretFile
|
||
}
|
||
services.hyperhive.hiveName = ${if hiveName == null then "null" else hiveName}
|
||
|
||
Its samples would be refused with a 401 by the collector
|
||
running beside it. Set both, or set requireHiveIdentity =
|
||
false to accept unauthenticated ingest.
|
||
'';
|
||
}
|
||
{
|
||
# The mirror of the assertion above, and the reason it exists is
|
||
# that the failure is SILENT rather than loud. With ingest
|
||
# unauthenticated the swarm tier serves one catch-all location
|
||
# and passes the URI through unchanged — but this tier still
|
||
# appends `/<hive>` whenever it holds a credential, so the
|
||
# receiver is asked for `/<hive>/v1/metrics`, a path it does not
|
||
# serve. The result is 404s and retries: no 401, no assertion, no
|
||
# log anywhere saying telemetry stopped.
|
||
#
|
||
# Only reachable by overriding one side without the other, since
|
||
# both defaults derive from `swarm.authelia.enable` and move
|
||
# together. That is exactly why it is worth a build error — an
|
||
# operator who flips the escape hatch has no reason to suspect
|
||
# the sending half.
|
||
assertion = !(swarmOtelCfg.enable && !swarmOtelCfg.requireHiveIdentity && senderAuth);
|
||
message = ''
|
||
This host accepts unauthenticated telemetry ingest
|
||
(services.hyperhive.swarm.otel.requireHiveIdentity = false),
|
||
but its own collector still holds a credential:
|
||
|
||
services.hyperhive.otel.clientSecretFile = ${
|
||
if otel.clientSecretFile == null then "null" else otel.clientSecretFile
|
||
}
|
||
|
||
A collector that authenticates also addresses its hive's own
|
||
path, and an unauthenticated swarm tier serves no per-hive
|
||
paths — so this hive's samples would 404 rather than be
|
||
refused, which no log names as an auth problem.
|
||
|
||
Set services.hyperhive.otel.clientSecretFile = null to send
|
||
unauthenticated too, or drop the requireHiveIdentity
|
||
override.
|
||
'';
|
||
}
|
||
];
|
||
}
|
||
);
|
||
}
|