hyperhive/nix/host-modules/otel.nix

535 lines
25 KiB
Nix
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# Hive-wide OTEL stats export, and the HIVE tier of the collector pair.
# Set ONCE here at host level; the meta-flake renderer
# (`hive-c0re/src/meta.rs::otel_config`) reads the HYPERHIVE_OTEL_* env
# exported off hive-c0re's unit (see ./hive-c0re) and injects the matching
# `hyperhive.otel.*` build-time config into EVERY agent (mirroring the
# CA-cert injection), so each agent's harness exports its own Claude Code
# stats directly to the collector. There is no per-agent opt-in — this is
# the single switch for the whole hive.
#
# This tier receives from this hive's agents and forwards to the swarm's
# collector (./swarm-otel.nix). It holds no credential and picks no
# destination: an agent's samples cross a hive boundary exactly once, and
# what happens after that is the swarm's decision, not a hive's. The
# upstream options declared below describe that far end and are read one
# tier up — they stay here because they mean what they have always meant.
{
pkgs,
lib,
config,
...
}:
let
# This tier now reaches the swarm's collector by name through the
# gateway (`swarm-otel.nix`'s `domain`) instead of a loopback URL, so it
# needs the same hive-CA trust every other host consumer of an `https://`
# swarm-service name needs — see `swarm-controller.nix` for the sibling
# wiring this copies.
#
# `hostUnit`: `opentelemetry-collector` is a host systemd service, not a
# container, so it reads the CA from the host path and the bundle oneshot
# waits on `hive-tls-ca.service` itself. `enable`: `imports` is
# unconditional at the host's top level, so without it a hive with this
# tier off would still get a bundle oneshot and a phantom
# `opentelemetry-collector` service holding an `SSL_CERT_FILE`.
caTrust = import ./lib/hive-ca-trust.nix {
inherit lib;
tlsCfg = config.services.hyperhive.tls;
gatewayCfg = config.services.hyperhive.gateway;
};
in
{
imports = [
(caTrust.trustBundle {
inherit pkgs;
name = "hive-otel";
consumers = [ "opentelemetry-collector" ];
hostUnit = true;
enable = config.services.hyperhive.otel.enable;
})
];
options.services.hyperhive.otel = {
enable = lib.mkEnableOption ''
hive-wide export of every agent's Claude Code stats (token usage,
cost, tool calls) to an OTLP endpoint via Claude Code's built-in
OpenTelemetry. One switch for all agents.
Enabling this also runs this hive's collector on the host: there is
exactly one way telemetry leaves this hive, and it is through that
collector. Agents export unauthenticated to a bridge address only
their own containers can reach.
That collector forwards to the swarm's
({option}`services.hyperhive.swarm.otel.enable`), which holds the
upstream credential and writes the swarm's store. So an agent never
sees the credential, and neither does this tier.
The collector is therefore in the path of all telemetry. It runs
on the same host as the agents and restarts on failure, and
telemetry is not the control plane, so degraded telemetry is not
degraded operation but the export no longer survives independently
of anything host-side
'';
endpoint = lib.mkOption {
type = lib.types.str;
default = "";
example = "https://collector.example.com/otel";
description = ''
Upstream OTLP endpoint: where telemetry ultimately goes, after it
has left the swarm.
Read by the swarm's collector
({option}`services.hyperhive.swarm.otel.enable`), which is the only
tier that holds the upstream credential. An agent is handed the
*first* hop instead this hive's own collector so this value is
never given to a container.
Optional. Leave it empty and the swarm's own metrics store
({option}`services.hyperhive.swarm.victoriametrics.enable`) is the
destination; that is a complete deployment, not a degraded one.
Set both and telemetry goes to both.
'';
};
protocol = lib.mkOption {
type = lib.types.enum [
"http/protobuf"
"http/json"
"grpc"
];
default = "http/protobuf";
description = ''
OTLP wire protocol for the **upstream** link, honoured by the
swarm collector's exporter.
Not what agents speak: their first hop is this hive's collector,
whose OTLP/HTTP receiver takes protobuf whatever the upstream
wants (see `hive-c0re/environment.nix`).
'';
};
headersCredential = lib.mkOption {
# `str`, not `path`: a `path`-typed relative literal is hash-copied
# into the world-readable nix store at eval time, defeating the
# point. Keep it a string + require an absolute runtime path so the
# secret is only ever read from disk by systemd at start.
type = lib.types.nullOr lib.types.str;
default = null;
example = "/run/secrets/otel-headers";
description = ''
Absolute path to an operator-provided secret file holding the
upstream auth header as `NAME=value` (e.g.
`Authorization=Bearer <token>`).
**Only the swarm's collector reads this** the one tier that
talks to the upstream. It arrives as an `EnvironmentFile`, so the
value is never read by nix, never copied into the store or the
generated config, and never passed in argv; and it reaches
neither an agent container nor this hive's own collector, which
is the point of the tiers existing. Must be absolute.
Leave null if the upstream needs no auth header; the collector
then sends none rather than an empty one.
'';
};
extraResourceAttributes = lib.mkOption {
type = lib.types.str;
default = "";
example = "deployment.environment=prod";
description = ''
Extra comma-separated entries appended to
`OTEL_RESOURCE_ATTRIBUTES` after the built-in
`service.name` / `agent` / `hive` / `swarm` labels.
'';
};
debug = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Emit OTEL SDK diagnostic messages to every agent's stderr by
setting `CLAUDE_CODE_OTEL_DIAG_STDERR=1`. Useful when
troubleshooting collector connectivity or endpoint config;
leave off in normal operation to avoid noise in agent logs.
Only meaningful when `enable` is true.
'';
};
collector.upstreamHeaderName = lib.mkOption {
type = lib.types.str;
default = "Authorization";
description = ''
Name of the HTTP header the collector sends upstream, whose
*value* comes from `headersCredential`.
The name is here and the value is not, and that split is forced
rather than chosen: the collector models exporter headers as a
static map, so rendering them means nix reading the value the
one thing `headersCredential` being a path exists to prevent.
A name is public, a value is not.
exactly one header is expressible this way. A credential
carrying several (`a=1,b=2`) would be read as a single value,
which is why the shape is a named header rather than an opaque
blob: a second header has to be *declared*, not smuggled.
'';
};
collector.port = lib.mkOption {
type = lib.types.port;
default = 4318;
description = ''
Port the collector's OTLP/HTTP receiver listens on, at
`services.hyperhive.network.bridgeIp`. 4318 is the OTLP/HTTP
default.
The port is contributed to
`services.hyperhive.network.exposeHostPorts`, which opens it on
the bridge interface only so it is reachable from agent
containers and not from the outside world.
'';
};
scrapeTargets = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
default = { };
example = lib.literalExpression ''{ hive-forge = "127.0.0.1:3000"; }'';
description = ''
Prometheus exposition endpoints this hive's collector scrapes, as
`<job name> = "<host>:<port>"`.
**A service declares its own entry, from its own module, under its
own `enable`** the same rule as the swarm tier's option of the
same name, and for the same reason: an entry exists only where the
service that named it runs, so the scraper and its target are
co-located by construction rather than by luck.
**This tier, not the swarm one, is where a HIVE-owned target
belongs.** The two are not interchangeable: a swarm service does
not belong to a hive, so its samples must never acquire a `hive`
label which is why that scraper lives one tier up. Putting a
hive-owned target there would either mislabel it or leave it
unattributed.
Scraping is unauthenticated on purpose. This receiver is a
prometheus-to-OTLP converter sitting next to what it reads, in the
same trust position as the agents already pushing to this
collector; the authenticated hop is the collector's own export
onward, which is one hop for the whole hive rather than one per
target.
Empty by default, and that is the shipped case no scrape
receiver is emitted at all, because a `prometheus` receiver with
nothing to scrape is a config that renders, starts and collects
nothing.
'';
};
clientSecretFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default =
if
config.services.hyperhive.swarm.authelia.enable && config.services.hyperhive.hiveName != null
then
"${config.services.hyperhive.swarm.authelia.hostClientSecretDir}/"
+ "${config.services.hyperhive.swarm.authelia.hiveClientPrefix}${config.services.hyperhive.hiveName}.secret"
else
null;
defaultText = lib.literalExpression ''"''${swarm.authelia.hostClientSecretDir}/''${swarm.authelia.hiveClientPrefix}''${hiveName}.secret" when this host runs the swarm's IdP, else null'';
example = "/var/lib/secrets/hive-telemetry.secret";
description = ''
Absolute path to this hive's OAuth2 client secret, used to
authenticate to the swarm's collector as this hive.
**Whether this is set is what decides whether this tier
authenticates at all.** A credential is the only thing that makes
authenticated egress possible, so its presence is the condition
rather than a second switch that could disagree with it.
Defaults to the secret this host's own authelia minted, which is
correct exactly when the swarm's IdP runs here. On a hive that is
not that host, the file has to arrive some other way and this
option names wherever it landed the same manual-copy shape
`services.hyperhive.swarm.authelia.oidc.hiveIdentities` documents,
where delivering a secret to a hive that is not this host is
deliberately not solved.
Read by `LoadCredential`, so it is never evaluated by nix, never
copied into the store and never passed in argv.
'';
};
metricIntervalMs = lib.mkOption {
type = lib.types.nullOr lib.types.ints.positive;
default = null;
example = 10000;
description = ''
Metric export interval in milliseconds, set as
`OTEL_METRIC_EXPORT_INTERVAL` for every agent. Claude Code's
default is 60000 (60s). Leave `null` to use that default.
Each agent runs claude as a short-lived per-turn process; claude
force-flushes metrics on shutdown, so this is not required for
metrics to be exported, but a lower value gives more frequent
intermediate flushes within long turns. Cosmetic, not a
correctness knob.
'';
};
};
config = lib.mkIf config.services.hyperhive.otel.enable (
let
otel = config.services.hyperhive.otel;
autheliaCfg = config.services.hyperhive.swarm.authelia;
swarmOtelCfg = config.services.hyperhive.swarm.otel;
hiveName = config.services.hyperhive.hiveName;
listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}";
swarmName = "otlphttp/swarm";
authName = "oauth2client/swarm";
# A hive always authenticates to the swarm's collector as itself, so
# this is not a mode — it is a precondition, and the assertion below
# is what enforces it. Kept as a name because several places have to
# read "do we have what it takes", and an eval error from a null path
# names this file rather than the option an operator has to set.
senderAuth = otel.clientSecretFile != null && hiveName != null;
# This hive's client id, and also the audience it must ASK for. Both
# are `hiveClientPrefix` + the hive's name because that is the one
# name the swarm already agrees on; the receiver one tier up derives
# the same string.
hiveClient = "${autheliaCfg.hiveClientPrefix}${toString hiveName}";
# systemd exports `CREDENTIALS_DIRECTORY` to any unit with
# `LoadCredential`, and the collector expands `${env:…}` at load. So
# the secret reaches the process as a PATH resolved at runtime — nix
# renders neither the value nor the directory, and nothing has to
# hardcode `/run/credentials/<unit>`.
credName = "swarm-client.secret";
secretWaitUnit = "hive-otel-client-secret";
in
{
# Reachable from agent containers and nowhere else: this opens
# the port on the bridge interface only.
services.hyperhive.network.exposeHostPorts = [ otel.collector.port ];
services.opentelemetry-collector = {
enable = true;
# Contrib, matching the swarm tier (./swarm-otel.nix). The upstream
# default build has no auth extensions at all, and this tier has to
# *present* a credential to the swarm tier — `oauth2client` lives
# only in contrib, so the package choice is what makes authenticated
# egress expressible rather than a preference.
#
# Not a build-farm cost: contrib is fetched, not compiled.
#
# ⚠️ Read the note directly below before adding any extension here.
# It describes precisely the trap this package unlocks: naming an
# extension the build lacks passes `validate` and then kills the
# collector at startup. With contrib the extensions exist — but the
# gap it warns about (a green build proving nothing about whether
# the process starts) is exactly why this module's auth wiring is
# gated by a probe that runs both collectors, not by eval.
package = pkgs.opentelemetry-collector-contrib;
# `validateConfigFile` defaults to `isStorePath configFile`,
# and `configFile` is null on the `settings` path — so the
# upstream default is OFF for exactly the way this module
# configures it. Turning it on runs `otelcol validate` at
# build time, which is the collector checking its own config.
# ⚠️ It is a PARSER, not a wiring check, and the gap is wider
# than "no sample was sent": measured 2026-08-15, `validate`
# ACCEPTS a receiver naming an auth extension that is absent
# from the build, and the collector then dies at startup with
# `Failed to start component`. So a green build does not
# prove this config STARTS, never mind that a sample arrives.
validateConfigFile = true;
settings = {
receivers = {
otlp.protocols.http.endpoint = listen;
}
# Only when a service has actually declared a target. An enabled
# `prometheus` receiver with an empty `scrape_configs` renders,
# validates and starts perfectly while reading nothing — and the
# empty set is the default, so that inert shape would be what
# most hives deploy.
// lib.optionalAttrs (otel.scrapeTargets != { }) {
prometheus.config.scrape_configs = lib.mapAttrsToList (job: target: {
job_name = job;
static_configs = [ { targets = [ target ]; } ];
}) otel.scrapeTargets;
};
# One destination, and it is the swarm's collector. This tier
# holds no upstream credential and writes no store: it receives
# from this hive's agents and forwards, which is the whole of
# its job. Everything that decides where telemetry ultimately
# goes lives one tier up, in ./swarm-otel.nix.
exporters.${swarmName} = {
# ⚠️ Plain `endpoint`, and the exporter beside this one in
# ./swarm-otel.nix warns against exactly that spelling — read
# both before "fixing" either. The difference is the far end,
# not the exporter: `endpoint` is a BASE that otlphttp appends
# `/v1/metrics` to, which is precisely the path an OTLP/HTTP
# receiver serves. VictoriaMetrics is the odd one out, serving
# OTLP at `/opentelemetry/api/v1/push`, and that is why the
# store exporter needs `metrics_endpoint` while this one must
# not have it.
#
# By name through the gateway, not a loopback literal: a
# loopback literal is correct only while listener and caller
# share a netns, an assumption that has cost this project two
# outages, and it is exactly the split-host case a swarm
# service name exists to make a config fact rather than a code
# change. `https://` because that name resolves through the
# gateway even on a co-located host — see `caTrust` above for
# the trust half that makes this verify.
# The hive's own path under the collector's single name. The
# swarm tier gives each hive its own authenticated receiver and
# routes to it by this prefix, so the path is not decoration —
# it selects WHICH receiver, and therefore which hive the
# samples get labelled as.
endpoint = "https://${swarmOtelCfg.domain}" + lib.optionalString senderAuth "/${toString hiveName}";
}
// lib.optionalAttrs senderAuth { auth.authenticator = authName; };
# ⚠️ An extension not listed here is INERT: the collector starts
# clean and the exporter naming it sends nothing authenticated.
service.extensions = lib.optional senderAuth authName;
service.pipelines.metrics = {
# EXTENDED, not replaced. Assigning here instead of appending
# would drop `otlp` — the hive would stop receiving from its
# own agents while still rendering a config the collector
# starts cleanly on, so nothing would report the loss. The
# `optional` matters too: a pipeline naming a receiver that no
# longer exists is a startup failure, which `validateConfigFile`
# below turns into a build failure rather than a broken deploy.
receivers = [ "otlp" ] ++ lib.optional (otel.scrapeTargets != { }) "prometheus";
exporters = [ swarmName ];
};
}
// lib.optionalAttrs senderAuth {
extensions.${authName} = {
client_id = hiveClient;
# A real key, measured against this collector version rather
# than assumed — with a deliberate typo rejected in the same
# run, so "accepted" is distinguishable from "ignores
# everything". Keeps the secret out of nix entirely: the
# collector opens the file itself.
client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${credName}";
token_url = "${toString autheliaCfg.url}/api/oidc/token";
# ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted.
# Registering it on the client only makes it permissible; a
# token minted without asking carries `aud: []` and every
# receiver refuses it — with a config that reads perfectly at
# both ends. Measured against authelia 4.39.20.
endpoint_params.audience = hiveClient;
};
};
};
# `LoadCredential` and not a copy-oneshot: this collector is a HOST
# unit, so there is no container boundary to cross and therefore no
# reason for a second on-disk copy of the secret. systemd hands it to
# the process in a private tmpfs and exports the directory, which is
# what the config above names.
systemd.services.opentelemetry-collector = lib.optionalAttrs senderAuth {
serviceConfig.LoadCredential = [ "${credName}:${otel.clientSecretFile}" ];
# `requires`, not merely `after`: ordering without gating leaves the
# collector starting anyway and failing on its own `LoadCredential`,
# which is the failure the wait exists to prevent. Same reasoning
# `lib/hive-ca-trust.nix` records for its bundle consumers.
requires = [ "${secretWaitUnit}.service" ];
after = [ "${secretWaitUnit}.service" ];
};
# Wait for the credential rather than racing it.
#
# `LoadCredential` naming a missing path is FATAL AT UNIT START, and the
# secret is minted by authelia's first-boot generator inside its own
# container — nothing orders a host unit against that. Worse, nixpkgs
# sets `Restart = "always"` with no `RestartSec`, so the failure is
# instant and the unit burns systemd's 5-starts-in-10s allowance in well
# under a second, landing in `start-limit-hit` where it stops retrying
# entirely and needs `systemctl reset-failed` by hand.
#
# 🔑 `Restart = always` reads like it makes this self-healing and does
# the opposite: a SLOW-failing unit retries until the secret appears, a
# FAST-failing one exhausts its limit before the thing it waits for can
# exist. This oneshot converts the fast failure into a slow one, which
# is what that restart policy is actually good at.
#
# Copied from `hive-forge-oidc-secret.service`, which solves exactly
# this for the forge — see `docs/swarm/secrets.md`.
systemd.services.${secretWaitUnit} = lib.mkIf senderAuth {
description = "wait for this hive's telemetry client secret";
# Deliberately says nothing about the collector: the CONSUMER declares
# `requires` + `after` above, which both pulls this into the
# transaction and gates on it. Stating the order from both sides too
# would be one more thing to keep consistent for no added guarantee.
#
# Only when the minting container is on THIS host. Elsewhere the file
# is operator-provided and there is no local unit to order against —
# naming one that does not exist orders nothing, silently.
after = lib.optional autheliaCfg.enable "container@${autheliaCfg.machine}.service";
requires = lib.optional autheliaCfg.enable "container@${autheliaCfg.machine}.service";
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = secretWaitUnit;
# ⚠️ Longer than the bounded wait below. `DefaultTimeoutStartSec` is
# 90s, so without this systemd kills the unit at 90 — before it can
# emit the message naming the file it was waiting for.
TimeoutStartSec = "180s";
};
path = [ pkgs.coreutils ];
script = ''
set -euo pipefail
secret=${lib.escapeShellArg (toString otel.clientSecretFile)}
# Bounded, then FAIL never skip. A silent skip produces a
# collector that starts and ships nothing, which is the failure
# mode this whole path is trying not to have.
for _ in $(seq 1 60); do
[ -s "$secret" ] && exit 0
sleep 2
done
echo "telemetry client secret $secret has not appeared after 120s" >&2
exit 1
'';
};
assertions = [
{
# A hive authenticates to the swarm's collector as itself — there
# is no unauthenticated path to fall back to, so a missing
# credential is a broken deployment rather than a quieter mode.
# Caught here because the alternative is a collector that starts
# cleanly, retries forever, and reports nothing to anyone.
assertion = senderAuth;
message = ''
services.hyperhive.otel.enable is true but this hive has no
identity to present to the swarm's collector:
services.hyperhive.otel.clientSecretFile = ${
if otel.clientSecretFile == null then "null" else otel.clientSecretFile
}
services.hyperhive.hiveName = ${if hiveName == null then "null" else hiveName}
Every hive authenticates as itself that is what makes the
`hive` label on its metrics mean anything so both are
required.
On a host that runs the swarm's identity provider, the default
already points at the secret authelia minted. On a hive that
does not, copy that hive's secret across and name it here.
'';
}
];
}
);
}