every hive runs its otel collector, and agents export to it by default
This commit is contained in:
atlas
mara
parent
839efd1642
commit
13daf4c7dc
28 changed files with 245 additions and 303 deletions
|
|
@ -297,7 +297,7 @@ in
|
|||
|
||||
Named `swarm-otel` rather than `otel` because there are two
|
||||
collectors and the tier is the whole distinction:
|
||||
{option}`services.hyperhive.otel.enable` is the **hive-tier** one,
|
||||
{option}`services.hyperhive.otel.collector.enable` is the **hive-tier** one,
|
||||
which every hive runs. A bare `deploy.otel` would not say which
|
||||
it meant. A hive that does not run the swarm collector still runs
|
||||
its own, and reaches this one by name at
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ let
|
|||
# ./otel.nix: `fetches`, the collector loading the file this unit writes
|
||||
# under the hive's own leaf.
|
||||
swarm-bao-hive-otel-oidc =
|
||||
otel.enable
|
||||
otel.collector.enable
|
||||
&& hyperhiveCfg.hiveName != null
|
||||
&& havePair baoDeploy.clientCertFile baoDeploy.clientKeyFile
|
||||
&&
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ let
|
|||
waits = senderAuth && !fetches;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf (hyperhiveCfg.deploy.authelia.enable && otel.enable && waits) {
|
||||
config = lib.mkIf (hyperhiveCfg.deploy.authelia.enable && otel.collector.enable && waits) {
|
||||
systemd.services.hive-otel-client-secret = {
|
||||
after = [ "container@${autheliaMachine}.service" ];
|
||||
requires = [ "container@${autheliaMachine}.service" ];
|
||||
|
|
|
|||
|
|
@ -71,64 +71,34 @@ in
|
|||
# every agent. See services.hyperhive.swarm.github.enable.
|
||||
HYPERHIVE_GITHUB_DISABLED = "1";
|
||||
}
|
||||
// lib.optionalAttrs config.services.hyperhive.otel.enable (
|
||||
# OTEL config (`swarm.hiveOtel`, `deploy.hive-controller.otel`, gated on
|
||||
# this hive's `otel.enable`) -> read by meta.rs::otel_config and injected
|
||||
# as build-time `services.hyperhive.agent.otel.*` into every agent.
|
||||
# Endpoint presence is the enable signal on the meta side; the
|
||||
# optional fields are only emitted when set so absent values
|
||||
# don't render no-op env lines.
|
||||
let
|
||||
otel = config.services.hyperhive.otel;
|
||||
hiveOtel = config.services.hyperhive.swarm.hiveOtel;
|
||||
controllerOtel = config.services.hyperhive.deploy.hive-controller.otel;
|
||||
# The first hop, this hive's own collector, under the two spellings its
|
||||
# two kinds of caller need. Agents get the swarm-wide name, so their
|
||||
# builds carry nothing hive-specific; this daemon gets the bridge
|
||||
# address, because on the host the name may resolve to loopback
|
||||
# (`gateway.localHostsEntry`), where the receiver does not listen.
|
||||
agentFirstHop = "http://${otel.collector.host}:${toString hiveOtel.collectorPort}";
|
||||
hostFirstHop = "http://${config.services.hyperhive.network.bridgeIp}:${toString hiveOtel.collectorPort}";
|
||||
in
|
||||
{
|
||||
# `otel.endpoint` means "where telemetry ultimately goes" and keeps
|
||||
# that meaning; what agents are handed is the *first hop*, which is
|
||||
# always this hive's own collector. Deriving it rather than
|
||||
# redefining `endpoint` is what lets every existing deployment keep
|
||||
# its configured value untouched.
|
||||
HYPERHIVE_OTEL_ENDPOINT = agentFirstHop;
|
||||
# hive-c0re's OWN container-resource exporter (stats/otel_metrics.rs)
|
||||
# reads the STANDARD OTLP variable — the same one hive-metric and every
|
||||
# agent read — and lets the SDK resolve the URL, which appends the
|
||||
# signal path (`/v1/metrics`). Handing the SDK an address
|
||||
# programmatically instead takes it verbatim: it POSTs to the
|
||||
# collector's root, gets a 404 on every export, and says nothing,
|
||||
# because OTLP export failures go to an error handler no binary here
|
||||
# installs — the daemon logged "exporter enabled" and never delivered a
|
||||
# sample, for as long as it had that exporter. The variable above cannot
|
||||
# replace this one:
|
||||
# it is the agent-config transport meta.rs reads, and the SDK does not
|
||||
# know that name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT = hostFirstHop;
|
||||
# The first hop is the collector's OTLP/HTTP receiver, which speaks
|
||||
# protobuf regardless of what the upstream wants — `otel.protocol`
|
||||
# describes the *upstream* link, and the collector's own exporter is
|
||||
# what has to honour it (see nix/host-modules/otel.nix).
|
||||
HYPERHIVE_OTEL_PROTOCOL = "http/protobuf";
|
||||
}
|
||||
// lib.optionalAttrs (controllerOtel.extraResourceAttributes != "") {
|
||||
HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES = controllerOtel.extraResourceAttributes;
|
||||
}
|
||||
# HYPERHIVE_OTEL_HEADERS_CREDENTIAL is deliberately NOT emitted, and
|
||||
# its absence is the security half of this design: that variable would
|
||||
# put the upstream token into an agent's own settings.json. Nothing
|
||||
# delivers the token into the container — no nspawn credential, no
|
||||
# agent-readable header file.
|
||||
// {
|
||||
# hive-c0re's OWN container-resource exporter (stats/otel_metrics.rs) reads
|
||||
# the STANDARD OTLP variable — the same one hive-metric and every agent
|
||||
# read — and lets the SDK resolve the URL, which appends the signal path
|
||||
# (`/v1/metrics`). Handing the SDK an address programmatically instead
|
||||
# takes it verbatim: it POSTs to the collector's root, gets a 404 on every
|
||||
# export, and says nothing, because OTLP export failures go to an error
|
||||
# handler no binary here installs.
|
||||
#
|
||||
# The rule: the collector holding the credential achieves nothing while
|
||||
# anything else hands out a copy, so there is exactly one holder and it
|
||||
# is on the host.
|
||||
)
|
||||
# The bridge address, not `otel.collector.host`: on the host that name may
|
||||
# resolve to loopback (`gateway.localHostsEntry`), where the receiver does
|
||||
# not listen. Agents are not told an address at all; their module defaults
|
||||
# to the collector's name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT = "http://${config.services.hyperhive.network.bridgeIp}:${toString config.services.hyperhive.swarm.hiveOtel.collectorPort}";
|
||||
}
|
||||
//
|
||||
lib.optionalAttrs
|
||||
(config.services.hyperhive.deploy.hive-controller.otel.extraResourceAttributes != "")
|
||||
{
|
||||
HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES =
|
||||
config.services.hyperhive.deploy.hive-controller.otel.extraResourceAttributes;
|
||||
}
|
||||
# No upstream credential is emitted here, and its absence is the security half
|
||||
# of this design: it would put the upstream token where an agent could read
|
||||
# it. Nothing delivers the token into a container — no nspawn credential, no
|
||||
# agent-readable header file. The collector holding the credential achieves
|
||||
# nothing while anything else hands out a copy, so there is exactly one
|
||||
# holder and it is on the host.
|
||||
// {
|
||||
# In-cluster forge URL — the gateway vhost (`forge.<domain>`), which
|
||||
# nginx proxies to forgejo. Used both for internal API calls in
|
||||
|
|
|
|||
|
|
@ -320,7 +320,7 @@ in
|
|||
description = ''
|
||||
Loopback TCP port nginx serves its `stub_status` counters on, at
|
||||
`http://127.0.0.1:''${statusPort}/stub_status`. The hive's collector
|
||||
({option}`services.hyperhive.otel.enable`) reads it with its `nginx`
|
||||
(every hive runs one) reads it with its `nginx`
|
||||
receiver.
|
||||
|
||||
TCP rather than a unix socket because that receiver can only dial
|
||||
|
|
|
|||
|
|
@ -55,7 +55,6 @@ let
|
|||
h.gateway.enable
|
||||
h.gateway.dns.enable
|
||||
h.network.enable
|
||||
h.otel.enable
|
||||
h.snapshotStore.enable
|
||||
];
|
||||
in
|
||||
|
|
|
|||
|
|
@ -1,11 +1,8 @@
|
|||
# Hive-wide OTEL stats export, and the HIVE tier of the collector pair.
|
||||
# Set ONCE here at host level; the meta-flake renderer
|
||||
# (`hive-c0re/src/meta.rs::otel_config`) reads the HYPERHIVE_OTEL_* env
|
||||
# exported off hive-c0re's unit (see ./hive-c0re) and injects the matching
|
||||
# `services.hyperhive.agent.otel.*` build-time config into EVERY agent (mirroring the
|
||||
# CA-cert injection), so each agent's harness exports its own Claude Code
|
||||
# stats directly to the collector. There is no per-agent opt-in — this is
|
||||
# the single switch for the whole hive.
|
||||
# The HIVE tier of the collector pair: every hive runs one, on the host that
|
||||
# runs hive-c0re (`deploy.hive-controller.enable`). Agents export to it at
|
||||
# the fixed name `services.hyperhive.otel.collector.host`, which this hive's
|
||||
# resolver answers with its own bridge IP, so an agent's build carries
|
||||
# nothing hive-specific.
|
||||
#
|
||||
# This tier receives from this hive's agents and forwards to the swarm's
|
||||
# collector (./swarm-otel.nix). It holds no credential and picks no
|
||||
|
|
@ -41,6 +38,8 @@ let
|
|||
# The hive's own store identity. The collector authenticates to the swarm
|
||||
# as the hive, so it reads its secret as the hive: the hive leaf's policy
|
||||
# (`swarm-secret-client`'s `policy::render`) grants `swarm/hives/<hive>/*`.
|
||||
runsHive = config.services.hyperhive.otel.collector.enable;
|
||||
|
||||
hiveLeaf =
|
||||
config.services.hyperhive.deploy.bao.clientCertFile != null
|
||||
&& config.services.hyperhive.deploy.bao.clientKeyFile != null;
|
||||
|
|
@ -55,39 +54,19 @@ in
|
|||
name = "hive-otel";
|
||||
consumers = [ "opentelemetry-collector" ];
|
||||
hostUnit = true;
|
||||
enable = config.services.hyperhive.otel.enable;
|
||||
enable = runsHive;
|
||||
})
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "otel" "collector" "port" ]
|
||||
[ "services" "hyperhive" "swarm" "hiveOtel" "collectorPort" ]
|
||||
)
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "otel" "enable" ] ''
|
||||
Every hive runs its own telemetry collector, and every agent exports to
|
||||
it by default, so there is no switch. Remove this definition.
|
||||
'')
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "otel" "collector" "port" ] ''
|
||||
The hive collector's receiver port is fixed at 4318: agents dial that
|
||||
literal. Remove this definition.
|
||||
'')
|
||||
];
|
||||
|
||||
options.services.hyperhive.otel = {
|
||||
enable = lib.mkEnableOption ''
|
||||
hive-wide export of every agent's Claude Code stats (token usage,
|
||||
cost, tool calls) to an OTLP endpoint via Claude Code's built-in
|
||||
OpenTelemetry. One switch for all agents.
|
||||
|
||||
Enabling this also runs this hive's collector on the host: there is
|
||||
exactly one way telemetry leaves this hive, and it is through that
|
||||
collector. Agents export unauthenticated to
|
||||
{option}`services.hyperhive.otel.collector.host`, a name this hive's
|
||||
resolver answers with a bridge address only its own containers can
|
||||
reach.
|
||||
|
||||
That collector forwards to the swarm's
|
||||
({option}`services.hyperhive.deploy.swarm-otel.enable`), which holds the
|
||||
upstream credential and writes the swarm's store. So an agent never
|
||||
sees the credential, and neither does this tier.
|
||||
|
||||
⚠️ The collector is therefore in the path of all telemetry. It runs
|
||||
on the same host as the agents and restarts on failure, and
|
||||
telemetry is not the control plane, so degraded telemetry is not
|
||||
degraded operation — but the export does not survive independently
|
||||
of anything host-side
|
||||
'';
|
||||
|
||||
endpoint = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "";
|
||||
|
|
@ -171,25 +150,51 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
collector.enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
readOnly = true;
|
||||
default =
|
||||
config.services.hyperhive.deploy.hive-controller.enable
|
||||
|| config.services.hyperhive.deploy.allSwarmServices;
|
||||
defaultText = lib.literalExpression "deploy.hive-controller.enable || deploy.allSwarmServices";
|
||||
description = ''
|
||||
Whether this host runs the hive tier's collector. Not a switch:
|
||||
every host that runs a hive
|
||||
({option}`services.hyperhive.deploy.hive-controller.enable`) runs
|
||||
one, and so does a host carrying all the swarm's shared services
|
||||
({option}`services.hyperhive.deploy.allSwarmServices`), whose own
|
||||
containers export to it.
|
||||
|
||||
Agents export to it unauthenticated at
|
||||
{option}`services.hyperhive.otel.collector.host`, a name this
|
||||
hive's resolver answers with a bridge address only its own
|
||||
containers can reach. It forwards to the swarm's collector
|
||||
({option}`services.hyperhive.deploy.swarm-otel.enable`), which holds
|
||||
the upstream credential and writes the swarm's store, so neither an
|
||||
agent nor this tier ever sees the credential.
|
||||
|
||||
⚠️ The collector is in the path of all telemetry. It runs on the
|
||||
same host as the agents and restarts on failure; telemetry is not
|
||||
the control plane, so degraded telemetry is not degraded
|
||||
operation, but the export does not survive independently of
|
||||
anything host-side.
|
||||
'';
|
||||
};
|
||||
|
||||
collector.host = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
# Total on a null swarm domain, as in ./swarm-otel-service.nix: the
|
||||
# required-domain assertion should be what an operator sees.
|
||||
default = "hive-collector.${
|
||||
if config.services.hyperhive.swarm.domain == null then
|
||||
"invalid"
|
||||
else
|
||||
config.services.hyperhive.swarm.domain
|
||||
}";
|
||||
defaultText = lib.literalExpression ''"hive-collector.''${services.hyperhive.swarm.domain}"'';
|
||||
default = "hive-collector.hive";
|
||||
description = ''
|
||||
The name agents reach this hive's collector on. The same string on
|
||||
every hive: each hive's own resolver answers it with its own bridge
|
||||
IP, so an agent's build carries no hive-specific address.
|
||||
The name agents reach this hive's collector on. A literal, the same
|
||||
on every hive and in every swarm: each hive's own resolver answers
|
||||
it with its own bridge IP, so an agent's build carries neither a
|
||||
swarm domain nor a hive-specific address. The agent module's
|
||||
`services.hyperhive.agent.otel.endpoint` default spells the same
|
||||
name.
|
||||
|
||||
No hive can claim this name as its domain:
|
||||
`nix/reserved-hive-fragments.nix` forbids `hive` in any hive name.
|
||||
It sits outside every domain a hive or the swarm serves: a hive's
|
||||
domain is `<hiveName>.<swarm.domain>`, so it never ends in `.hive`.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
@ -274,12 +279,14 @@ in
|
|||
};
|
||||
};
|
||||
|
||||
# Shared by every hive's otel tier and its agents, so an agent's build
|
||||
# carries the same port whichever hive renders it.
|
||||
# Shared by every hive's otel tier and its agents. Read-only because the agent
|
||||
# module's `otel.endpoint` default spells the same literal, and a host that
|
||||
# moved the port would leave its agents dialling the old one.
|
||||
options.services.hyperhive.swarm.hiveOtel = {
|
||||
collectorPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 4318;
|
||||
readOnly = true;
|
||||
description = ''
|
||||
Port of every hive collector's OTLP/HTTP receiver, at that hive's
|
||||
`services.hyperhive.network.bridgeIp`: the port agents dial at
|
||||
|
|
@ -307,7 +314,7 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
config = lib.mkIf config.services.hyperhive.otel.enable (
|
||||
config = lib.mkIf runsHive (
|
||||
let
|
||||
otel = config.services.hyperhive.otel;
|
||||
autheliaCfg = config.services.hyperhive.swarm.authelia;
|
||||
|
|
@ -913,7 +920,7 @@ in
|
|||
# cleanly, retries forever, and reports nothing to anyone.
|
||||
assertion = senderAuth;
|
||||
message = ''
|
||||
services.hyperhive.otel.enable is true but this hive has no
|
||||
This hive runs a telemetry collector (every hive does) but has no
|
||||
identity to present to the swarm's collector:
|
||||
|
||||
services.hyperhive.otel.clientSecretFile = ${
|
||||
|
|
@ -948,7 +955,7 @@ in
|
|||
"none"
|
||||
]);
|
||||
message = ''
|
||||
services.hyperhive.otel.enable is true, which runs a journald
|
||||
This hive runs a telemetry collector, which runs a journald
|
||||
receiver reading /var/log/journal for hive-c0re's own crash-watch
|
||||
log line — but services.journald.storage is
|
||||
"${config.services.journald.storage}" on this host.
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ let
|
|||
# Whether the swarm collector's client is actually registered here.
|
||||
#
|
||||
# Asked of the client list rather than re-derived from the conditions
|
||||
# that produce it (`otel.enable`, a non-empty audience set, authelia
|
||||
# that produce it (a non-empty audience set, authelia
|
||||
# being co-located). A second copy of that predicate is a second thing
|
||||
# to keep in step, and the two drifting is not a build failure: a rule
|
||||
# naming an unregistered client is refused by authelia's *startup*
|
||||
|
|
|
|||
|
|
@ -1446,7 +1446,7 @@ let
|
|||
#
|
||||
# "Does the swarm have a collector" is not a question that can be false —
|
||||
# the swarm always has at least one instance of all the swarm level
|
||||
# services — so gating the forwarder on `otel.enable` or
|
||||
# services — so gating the forwarder on `otel.collector.enable` or
|
||||
# `deploy.swarm-otel.enable` would ask about the wrong host and render no
|
||||
# forwarder wherever the answer happened to be no.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ in
|
|||
{
|
||||
# `enable` is `services.hyperhive.deploy.swarm-otel.enable` — see ./deploy.nix.
|
||||
# ⚠️ That is the SWARM collector. The per-hive one keeps its own
|
||||
# `services.hyperhive.otel.enable` (./otel.nix) and is a different
|
||||
# `services.hyperhive.otel.collector.enable` (./otel.nix) and is a different
|
||||
# option entirely — every hive runs that one.
|
||||
options.services.hyperhive.swarm.otel = {
|
||||
|
||||
|
|
|
|||
|
|
@ -49,10 +49,6 @@ in
|
|||
# host. Same switch, same rule, one attribute path.
|
||||
config.services.hyperhive.deploy.matrix.enable = lib.mkDefault deployCfg.allSwarmServices;
|
||||
|
||||
# The collector that feeds the pair above (note: no `swarm.` prefix,
|
||||
# this is ./otel.nix's existing per-hive option).
|
||||
config.services.hyperhive.otel.enable = lib.mkDefault deployCfg.allSwarmServices;
|
||||
|
||||
# The rest of the shared services, from the same switch and for the same
|
||||
# reason.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ let
|
|||
# where this host runs a hive-tier collector on the bridge. The daemon adds
|
||||
# the repo and the PR to each `claude` run's resource attributes, and to
|
||||
# each cost record's attributes.
|
||||
otelEnv = lib.optionalAttrs otelCfg.enable {
|
||||
otelEnv = lib.optionalAttrs otelCfg.collector.enable {
|
||||
CLAUDE_CODE_ENABLE_TELEMETRY = "1";
|
||||
OTEL_METRICS_EXPORTER = "otlp";
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT = "http://${networkCfg.bridgeIp}:${toString hyperhiveCfg.swarm.hiveOtel.collectorPort}";
|
||||
|
|
|
|||
Loading…
Reference in a new issue