every hive runs its otel collector, and agents export to it by default
This commit is contained in:
parent
839efd1642
commit
13daf4c7dc
28 changed files with 245 additions and 303 deletions
|
|
@ -15,8 +15,8 @@
|
|||
let
|
||||
userName = config.services.hyperhive.agent.user.name;
|
||||
homeDir = "/home/${userName}";
|
||||
# Hive-wide OpenTelemetry config (host-driven; baked in per-agent by
|
||||
# meta.rs `otel_config`). Options declared in `otel.nix`, which also
|
||||
# OpenTelemetry config (on by default; extra labels baked in per-agent by
|
||||
# meta.rs). Options declared in `otel.nix`, which also
|
||||
# exports the generic OTLP environment this container's producers read.
|
||||
otelCfg = config.services.hyperhive.agent.otel;
|
||||
# Base claude-code environment applied to every agent regardless of OTEL.
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# OpenTelemetry wiring for an agent container: the per-agent options the
|
||||
# meta flake injects (host-driven from `services.hyperhive.otel.*`) plus
|
||||
# OpenTelemetry wiring for an agent container: the per-agent options (on by
|
||||
# default, exporting to the hive's collector by name) plus
|
||||
# `debug`, which the agent's own config sets, the
|
||||
# environment every OTLP producer inside the container reads, and the
|
||||
# collector that forwards this container's journal to the hive.
|
||||
|
|
@ -71,35 +71,35 @@ let
|
|||
logExporter = if cfg.protocol == "grpc" then "otlp" else "otlphttp";
|
||||
in
|
||||
{
|
||||
# OTEL stats export is configured ONCE at host level via
|
||||
# `services.hyperhive.otel.*` (see nix/host-modules/otel.nix) and
|
||||
# injected into every agent's build by the meta-flake renderer
|
||||
# (`hive-c0re/src/meta.rs::otel_config`). These per-agent options are
|
||||
# the build-time implementation surface that injection writes into;
|
||||
# they are not meant to be set directly in an agent.nix. Marked
|
||||
# `internal` so the host option is the only documented operator knob.
|
||||
# On by default: every hive runs a collector and answers
|
||||
# `hive-collector.hive` with its own bridge address, so the endpoint below
|
||||
# is the same string in every agent on every hive and nothing is rendered
|
||||
# into the meta flake. The port is the literal the host fixes at
|
||||
# `services.hyperhive.swarm.hiveOtel.collectorPort` (read-only there).
|
||||
# Marked `internal`: these are not meant to be set in an agent.nix, except
|
||||
# `debug`.
|
||||
options.services.hyperhive.agent.otel = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
default = true;
|
||||
internal = true;
|
||||
description = ''
|
||||
Export this agent's telemetry — Claude Code's stats (token
|
||||
usage, cost, tool calls) and anything pushed with
|
||||
`hive-metric` — to an OTLP endpoint. Each agent exports
|
||||
directly to the hive's collector, so it keeps working even when
|
||||
hive-c0re is down. Host-driven: set
|
||||
`services.hyperhive.otel.enable` instead.
|
||||
`hive-metric` — to the hive's collector. Each agent exports
|
||||
directly to it, so it keeps working even when hive-c0re is
|
||||
down.
|
||||
'';
|
||||
};
|
||||
|
||||
endpoint = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "";
|
||||
default = "http://hive-collector.hive:4318";
|
||||
internal = true;
|
||||
description = ''
|
||||
OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT`.
|
||||
Host-driven via `services.hyperhive.otel.endpoint`.
|
||||
The name is the one `services.hyperhive.otel.collector.host`
|
||||
holds, which the hive's resolver answers.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
@ -113,7 +113,8 @@ in
|
|||
internal = true;
|
||||
description = ''
|
||||
OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`.
|
||||
Host-driven via `services.hyperhive.otel.protocol`.
|
||||
The hive collector's receiver speaks OTLP/HTTP protobuf, so
|
||||
the default is the only value that reaches it.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -297,7 +297,7 @@ in
|
|||
|
||||
Named `swarm-otel` rather than `otel` because there are two
|
||||
collectors and the tier is the whole distinction:
|
||||
{option}`services.hyperhive.otel.enable` is the **hive-tier** one,
|
||||
{option}`services.hyperhive.otel.collector.enable` is the **hive-tier** one,
|
||||
which every hive runs. A bare `deploy.otel` would not say which
|
||||
it meant. A hive that does not run the swarm collector still runs
|
||||
its own, and reaches this one by name at
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ let
|
|||
# ./otel.nix: `fetches`, the collector loading the file this unit writes
|
||||
# under the hive's own leaf.
|
||||
swarm-bao-hive-otel-oidc =
|
||||
otel.enable
|
||||
otel.collector.enable
|
||||
&& hyperhiveCfg.hiveName != null
|
||||
&& havePair baoDeploy.clientCertFile baoDeploy.clientKeyFile
|
||||
&&
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ let
|
|||
waits = senderAuth && !fetches;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf (hyperhiveCfg.deploy.authelia.enable && otel.enable && waits) {
|
||||
config = lib.mkIf (hyperhiveCfg.deploy.authelia.enable && otel.collector.enable && waits) {
|
||||
systemd.services.hive-otel-client-secret = {
|
||||
after = [ "container@${autheliaMachine}.service" ];
|
||||
requires = [ "container@${autheliaMachine}.service" ];
|
||||
|
|
|
|||
|
|
@ -71,64 +71,34 @@ in
|
|||
# every agent. See services.hyperhive.swarm.github.enable.
|
||||
HYPERHIVE_GITHUB_DISABLED = "1";
|
||||
}
|
||||
// lib.optionalAttrs config.services.hyperhive.otel.enable (
|
||||
# OTEL config (`swarm.hiveOtel`, `deploy.hive-controller.otel`, gated on
|
||||
# this hive's `otel.enable`) -> read by meta.rs::otel_config and injected
|
||||
# as build-time `services.hyperhive.agent.otel.*` into every agent.
|
||||
# Endpoint presence is the enable signal on the meta side; the
|
||||
# optional fields are only emitted when set so absent values
|
||||
# don't render no-op env lines.
|
||||
let
|
||||
otel = config.services.hyperhive.otel;
|
||||
hiveOtel = config.services.hyperhive.swarm.hiveOtel;
|
||||
controllerOtel = config.services.hyperhive.deploy.hive-controller.otel;
|
||||
# The first hop, this hive's own collector, under the two spellings its
|
||||
# two kinds of caller need. Agents get the swarm-wide name, so their
|
||||
# builds carry nothing hive-specific; this daemon gets the bridge
|
||||
# address, because on the host the name may resolve to loopback
|
||||
# (`gateway.localHostsEntry`), where the receiver does not listen.
|
||||
agentFirstHop = "http://${otel.collector.host}:${toString hiveOtel.collectorPort}";
|
||||
hostFirstHop = "http://${config.services.hyperhive.network.bridgeIp}:${toString hiveOtel.collectorPort}";
|
||||
in
|
||||
{
|
||||
# `otel.endpoint` means "where telemetry ultimately goes" and keeps
|
||||
# that meaning; what agents are handed is the *first hop*, which is
|
||||
# always this hive's own collector. Deriving it rather than
|
||||
# redefining `endpoint` is what lets every existing deployment keep
|
||||
# its configured value untouched.
|
||||
HYPERHIVE_OTEL_ENDPOINT = agentFirstHop;
|
||||
# hive-c0re's OWN container-resource exporter (stats/otel_metrics.rs)
|
||||
# reads the STANDARD OTLP variable — the same one hive-metric and every
|
||||
# agent read — and lets the SDK resolve the URL, which appends the
|
||||
# signal path (`/v1/metrics`). Handing the SDK an address
|
||||
# programmatically instead takes it verbatim: it POSTs to the
|
||||
# collector's root, gets a 404 on every export, and says nothing,
|
||||
# because OTLP export failures go to an error handler no binary here
|
||||
# installs — the daemon logged "exporter enabled" and never delivered a
|
||||
# sample, for as long as it had that exporter. The variable above cannot
|
||||
# replace this one:
|
||||
# it is the agent-config transport meta.rs reads, and the SDK does not
|
||||
# know that name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT = hostFirstHop;
|
||||
# The first hop is the collector's OTLP/HTTP receiver, which speaks
|
||||
# protobuf regardless of what the upstream wants — `otel.protocol`
|
||||
# describes the *upstream* link, and the collector's own exporter is
|
||||
# what has to honour it (see nix/host-modules/otel.nix).
|
||||
HYPERHIVE_OTEL_PROTOCOL = "http/protobuf";
|
||||
}
|
||||
// lib.optionalAttrs (controllerOtel.extraResourceAttributes != "") {
|
||||
HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES = controllerOtel.extraResourceAttributes;
|
||||
}
|
||||
# HYPERHIVE_OTEL_HEADERS_CREDENTIAL is deliberately NOT emitted, and
|
||||
# its absence is the security half of this design: that variable would
|
||||
# put the upstream token into an agent's own settings.json. Nothing
|
||||
# delivers the token into the container — no nspawn credential, no
|
||||
# agent-readable header file.
|
||||
// {
|
||||
# hive-c0re's OWN container-resource exporter (stats/otel_metrics.rs) reads
|
||||
# the STANDARD OTLP variable — the same one hive-metric and every agent
|
||||
# read — and lets the SDK resolve the URL, which appends the signal path
|
||||
# (`/v1/metrics`). Handing the SDK an address programmatically instead
|
||||
# takes it verbatim: it POSTs to the collector's root, gets a 404 on every
|
||||
# export, and says nothing, because OTLP export failures go to an error
|
||||
# handler no binary here installs.
|
||||
#
|
||||
# The rule: the collector holding the credential achieves nothing while
|
||||
# anything else hands out a copy, so there is exactly one holder and it
|
||||
# is on the host.
|
||||
)
|
||||
# The bridge address, not `otel.collector.host`: on the host that name may
|
||||
# resolve to loopback (`gateway.localHostsEntry`), where the receiver does
|
||||
# not listen. Agents are not told an address at all; their module defaults
|
||||
# to the collector's name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT = "http://${config.services.hyperhive.network.bridgeIp}:${toString config.services.hyperhive.swarm.hiveOtel.collectorPort}";
|
||||
}
|
||||
//
|
||||
lib.optionalAttrs
|
||||
(config.services.hyperhive.deploy.hive-controller.otel.extraResourceAttributes != "")
|
||||
{
|
||||
HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES =
|
||||
config.services.hyperhive.deploy.hive-controller.otel.extraResourceAttributes;
|
||||
}
|
||||
# No upstream credential is emitted here, and its absence is the security half
|
||||
# of this design: it would put the upstream token where an agent could read
|
||||
# it. Nothing delivers the token into a container — no nspawn credential, no
|
||||
# agent-readable header file. The collector holding the credential achieves
|
||||
# nothing while anything else hands out a copy, so there is exactly one
|
||||
# holder and it is on the host.
|
||||
// {
|
||||
# In-cluster forge URL — the gateway vhost (`forge.<domain>`), which
|
||||
# nginx proxies to forgejo. Used both for internal API calls in
|
||||
|
|
|
|||
|
|
@ -320,7 +320,7 @@ in
|
|||
description = ''
|
||||
Loopback TCP port nginx serves its `stub_status` counters on, at
|
||||
`http://127.0.0.1:''${statusPort}/stub_status`. The hive's collector
|
||||
({option}`services.hyperhive.otel.enable`) reads it with its `nginx`
|
||||
(every hive runs one) reads it with its `nginx`
|
||||
receiver.
|
||||
|
||||
TCP rather than a unix socket because that receiver can only dial
|
||||
|
|
|
|||
|
|
@ -55,7 +55,6 @@ let
|
|||
h.gateway.enable
|
||||
h.gateway.dns.enable
|
||||
h.network.enable
|
||||
h.otel.enable
|
||||
h.snapshotStore.enable
|
||||
];
|
||||
in
|
||||
|
|
|
|||
|
|
@ -1,11 +1,8 @@
|
|||
# Hive-wide OTEL stats export, and the HIVE tier of the collector pair.
|
||||
# Set ONCE here at host level; the meta-flake renderer
|
||||
# (`hive-c0re/src/meta.rs::otel_config`) reads the HYPERHIVE_OTEL_* env
|
||||
# exported off hive-c0re's unit (see ./hive-c0re) and injects the matching
|
||||
# `services.hyperhive.agent.otel.*` build-time config into EVERY agent (mirroring the
|
||||
# CA-cert injection), so each agent's harness exports its own Claude Code
|
||||
# stats directly to the collector. There is no per-agent opt-in — this is
|
||||
# the single switch for the whole hive.
|
||||
# The HIVE tier of the collector pair: every hive runs one, on the host that
|
||||
# runs hive-c0re (`deploy.hive-controller.enable`). Agents export to it at
|
||||
# the fixed name `services.hyperhive.otel.collector.host`, which this hive's
|
||||
# resolver answers with its own bridge IP, so an agent's build carries
|
||||
# nothing hive-specific.
|
||||
#
|
||||
# This tier receives from this hive's agents and forwards to the swarm's
|
||||
# collector (./swarm-otel.nix). It holds no credential and picks no
|
||||
|
|
@ -41,6 +38,8 @@ let
|
|||
# The hive's own store identity. The collector authenticates to the swarm
|
||||
# as the hive, so it reads its secret as the hive: the hive leaf's policy
|
||||
# (`swarm-secret-client`'s `policy::render`) grants `swarm/hives/<hive>/*`.
|
||||
runsHive = config.services.hyperhive.otel.collector.enable;
|
||||
|
||||
hiveLeaf =
|
||||
config.services.hyperhive.deploy.bao.clientCertFile != null
|
||||
&& config.services.hyperhive.deploy.bao.clientKeyFile != null;
|
||||
|
|
@ -55,39 +54,19 @@ in
|
|||
name = "hive-otel";
|
||||
consumers = [ "opentelemetry-collector" ];
|
||||
hostUnit = true;
|
||||
enable = config.services.hyperhive.otel.enable;
|
||||
enable = runsHive;
|
||||
})
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "otel" "collector" "port" ]
|
||||
[ "services" "hyperhive" "swarm" "hiveOtel" "collectorPort" ]
|
||||
)
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "otel" "enable" ] ''
|
||||
Every hive runs its own telemetry collector, and every agent exports to
|
||||
it by default, so there is no switch. Remove this definition.
|
||||
'')
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "otel" "collector" "port" ] ''
|
||||
The hive collector's receiver port is fixed at 4318: agents dial that
|
||||
literal. Remove this definition.
|
||||
'')
|
||||
];
|
||||
|
||||
options.services.hyperhive.otel = {
|
||||
enable = lib.mkEnableOption ''
|
||||
hive-wide export of every agent's Claude Code stats (token usage,
|
||||
cost, tool calls) to an OTLP endpoint via Claude Code's built-in
|
||||
OpenTelemetry. One switch for all agents.
|
||||
|
||||
Enabling this also runs this hive's collector on the host: there is
|
||||
exactly one way telemetry leaves this hive, and it is through that
|
||||
collector. Agents export unauthenticated to
|
||||
{option}`services.hyperhive.otel.collector.host`, a name this hive's
|
||||
resolver answers with a bridge address only its own containers can
|
||||
reach.
|
||||
|
||||
That collector forwards to the swarm's
|
||||
({option}`services.hyperhive.deploy.swarm-otel.enable`), which holds the
|
||||
upstream credential and writes the swarm's store. So an agent never
|
||||
sees the credential, and neither does this tier.
|
||||
|
||||
⚠️ The collector is therefore in the path of all telemetry. It runs
|
||||
on the same host as the agents and restarts on failure, and
|
||||
telemetry is not the control plane, so degraded telemetry is not
|
||||
degraded operation — but the export does not survive independently
|
||||
of anything host-side
|
||||
'';
|
||||
|
||||
endpoint = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "";
|
||||
|
|
@ -171,25 +150,51 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
collector.enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
readOnly = true;
|
||||
default =
|
||||
config.services.hyperhive.deploy.hive-controller.enable
|
||||
|| config.services.hyperhive.deploy.allSwarmServices;
|
||||
defaultText = lib.literalExpression "deploy.hive-controller.enable || deploy.allSwarmServices";
|
||||
description = ''
|
||||
Whether this host runs the hive tier's collector. Not a switch:
|
||||
every host that runs a hive
|
||||
({option}`services.hyperhive.deploy.hive-controller.enable`) runs
|
||||
one, and so does a host carrying all the swarm's shared services
|
||||
({option}`services.hyperhive.deploy.allSwarmServices`), whose own
|
||||
containers export to it.
|
||||
|
||||
Agents export to it unauthenticated at
|
||||
{option}`services.hyperhive.otel.collector.host`, a name this
|
||||
hive's resolver answers with a bridge address only its own
|
||||
containers can reach. It forwards to the swarm's collector
|
||||
({option}`services.hyperhive.deploy.swarm-otel.enable`), which holds
|
||||
the upstream credential and writes the swarm's store, so neither an
|
||||
agent nor this tier ever sees the credential.
|
||||
|
||||
⚠️ The collector is in the path of all telemetry. It runs on the
|
||||
same host as the agents and restarts on failure; telemetry is not
|
||||
the control plane, so degraded telemetry is not degraded
|
||||
operation, but the export does not survive independently of
|
||||
anything host-side.
|
||||
'';
|
||||
};
|
||||
|
||||
collector.host = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
# Total on a null swarm domain, as in ./swarm-otel-service.nix: the
|
||||
# required-domain assertion should be what an operator sees.
|
||||
default = "hive-collector.${
|
||||
if config.services.hyperhive.swarm.domain == null then
|
||||
"invalid"
|
||||
else
|
||||
config.services.hyperhive.swarm.domain
|
||||
}";
|
||||
defaultText = lib.literalExpression ''"hive-collector.''${services.hyperhive.swarm.domain}"'';
|
||||
default = "hive-collector.hive";
|
||||
description = ''
|
||||
The name agents reach this hive's collector on. The same string on
|
||||
every hive: each hive's own resolver answers it with its own bridge
|
||||
IP, so an agent's build carries no hive-specific address.
|
||||
The name agents reach this hive's collector on. A literal, the same
|
||||
on every hive and in every swarm: each hive's own resolver answers
|
||||
it with its own bridge IP, so an agent's build carries neither a
|
||||
swarm domain nor a hive-specific address. The agent module's
|
||||
`services.hyperhive.agent.otel.endpoint` default spells the same
|
||||
name.
|
||||
|
||||
No hive can claim this name as its domain:
|
||||
`nix/reserved-hive-fragments.nix` forbids `hive` in any hive name.
|
||||
It sits outside every domain a hive or the swarm serves: a hive's
|
||||
domain is `<hiveName>.<swarm.domain>`, so it never ends in `.hive`.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
@ -274,12 +279,14 @@ in
|
|||
};
|
||||
};
|
||||
|
||||
# Shared by every hive's otel tier and its agents, so an agent's build
|
||||
# carries the same port whichever hive renders it.
|
||||
# Shared by every hive's otel tier and its agents. Read-only because the agent
|
||||
# module's `otel.endpoint` default spells the same literal, and a host that
|
||||
# moved the port would leave its agents dialling the old one.
|
||||
options.services.hyperhive.swarm.hiveOtel = {
|
||||
collectorPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 4318;
|
||||
readOnly = true;
|
||||
description = ''
|
||||
Port of every hive collector's OTLP/HTTP receiver, at that hive's
|
||||
`services.hyperhive.network.bridgeIp`: the port agents dial at
|
||||
|
|
@ -307,7 +314,7 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
config = lib.mkIf config.services.hyperhive.otel.enable (
|
||||
config = lib.mkIf runsHive (
|
||||
let
|
||||
otel = config.services.hyperhive.otel;
|
||||
autheliaCfg = config.services.hyperhive.swarm.authelia;
|
||||
|
|
@ -913,7 +920,7 @@ in
|
|||
# cleanly, retries forever, and reports nothing to anyone.
|
||||
assertion = senderAuth;
|
||||
message = ''
|
||||
services.hyperhive.otel.enable is true but this hive has no
|
||||
This hive runs a telemetry collector (every hive does) but has no
|
||||
identity to present to the swarm's collector:
|
||||
|
||||
services.hyperhive.otel.clientSecretFile = ${
|
||||
|
|
@ -948,7 +955,7 @@ in
|
|||
"none"
|
||||
]);
|
||||
message = ''
|
||||
services.hyperhive.otel.enable is true, which runs a journald
|
||||
This hive runs a telemetry collector, which runs a journald
|
||||
receiver reading /var/log/journal for hive-c0re's own crash-watch
|
||||
log line — but services.journald.storage is
|
||||
"${config.services.journald.storage}" on this host.
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ let
|
|||
# Whether the swarm collector's client is actually registered here.
|
||||
#
|
||||
# Asked of the client list rather than re-derived from the conditions
|
||||
# that produce it (`otel.enable`, a non-empty audience set, authelia
|
||||
# that produce it (a non-empty audience set, authelia
|
||||
# being co-located). A second copy of that predicate is a second thing
|
||||
# to keep in step, and the two drifting is not a build failure: a rule
|
||||
# naming an unregistered client is refused by authelia's *startup*
|
||||
|
|
|
|||
|
|
@ -1446,7 +1446,7 @@ let
|
|||
#
|
||||
# "Does the swarm have a collector" is not a question that can be false —
|
||||
# the swarm always has at least one instance of all the swarm level
|
||||
# services — so gating the forwarder on `otel.enable` or
|
||||
# services — so gating the forwarder on `otel.collector.enable` or
|
||||
# `deploy.swarm-otel.enable` would ask about the wrong host and render no
|
||||
# forwarder wherever the answer happened to be no.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ in
|
|||
{
|
||||
# `enable` is `services.hyperhive.deploy.swarm-otel.enable` — see ./deploy.nix.
|
||||
# ⚠️ That is the SWARM collector. The per-hive one keeps its own
|
||||
# `services.hyperhive.otel.enable` (./otel.nix) and is a different
|
||||
# `services.hyperhive.otel.collector.enable` (./otel.nix) and is a different
|
||||
# option entirely — every hive runs that one.
|
||||
options.services.hyperhive.swarm.otel = {
|
||||
|
||||
|
|
|
|||
|
|
@ -49,10 +49,6 @@ in
|
|||
# host. Same switch, same rule, one attribute path.
|
||||
config.services.hyperhive.deploy.matrix.enable = lib.mkDefault deployCfg.allSwarmServices;
|
||||
|
||||
# The collector that feeds the pair above (note: no `swarm.` prefix,
|
||||
# this is ./otel.nix's existing per-hive option).
|
||||
config.services.hyperhive.otel.enable = lib.mkDefault deployCfg.allSwarmServices;
|
||||
|
||||
# The rest of the shared services, from the same switch and for the same
|
||||
# reason.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ let
|
|||
# where this host runs a hive-tier collector on the bridge. The daemon adds
|
||||
# the repo and the PR to each `claude` run's resource attributes, and to
|
||||
# each cost record's attributes.
|
||||
otelEnv = lib.optionalAttrs otelCfg.enable {
|
||||
otelEnv = lib.optionalAttrs otelCfg.collector.enable {
|
||||
CLAUDE_CODE_ENABLE_TELEMETRY = "1";
|
||||
OTEL_METRICS_EXPORTER = "otlp";
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT = "http://${networkCfg.bridgeIp}:${toString hyperhiveCfg.swarm.hiveOtel.collectorPort}";
|
||||
|
|
|
|||
|
|
@ -47,7 +47,11 @@ let
|
|||
otel.metricIntervalMs = 10000;
|
||||
};
|
||||
|
||||
agentNoOtel = shared.agentBare;
|
||||
# Sets nothing: what an agent gets on a hive, whose meta flake renders no
|
||||
# otel line.
|
||||
agentDefault = shared.agentBare;
|
||||
|
||||
agentNoOtel = agent { otel.enable = false; };
|
||||
|
||||
agentSettings = machine: machine.services.opentelemetry-collector.settings;
|
||||
cases = [
|
||||
|
|
@ -100,11 +104,30 @@ let
|
|||
&& p.exporters != [ ]
|
||||
&& lib.all (e: (s.exporters ? ${e}) && s.exporters.${e}.endpoint == agentBridge) p.exporters;
|
||||
}
|
||||
{
|
||||
# 🎯 What the meta flake no longer says: an agent that sets nothing
|
||||
# exports, to a name rather than an address. The name carries no swarm
|
||||
# domain and no hive, so the same build is correct on every hive; the
|
||||
# resolver of the hive it runs on answers it. `agentNoOtel` above is the
|
||||
# control that these lookups find nothing where the switch is off.
|
||||
name = "an agent with no otel config exports to the fixed hive-collector name by default";
|
||||
ok =
|
||||
let
|
||||
endpoint = "http://hive-collector.hive:4318";
|
||||
env = agentDefault.systemd.globalEnvironment;
|
||||
s = agentSettings agentDefault;
|
||||
in
|
||||
agentDefault.services.hyperhive.agent.otel.enable
|
||||
&& env.OTEL_EXPORTER_OTLP_ENDPOINT == endpoint
|
||||
&& agentDefault.environment.variables.OTEL_EXPORTER_OTLP_ENDPOINT == endpoint
|
||||
&& lib.all (e: s.exporters.${e}.endpoint == endpoint) s.service.pipelines.logs.exporters
|
||||
&& !(agentNoOtel.systemd.globalEnvironment ? OTEL_EXPORTER_OTLP_ENDPOINT);
|
||||
}
|
||||
{
|
||||
# Presence control for the two cases above: with the switch off there
|
||||
# is no collector in the container at all, so their passing is about
|
||||
# the wiring rather than about a unit that renders regardless.
|
||||
name = "an agent that has not opted into telemetry runs no collector";
|
||||
name = "an agent that switched telemetry off runs no collector";
|
||||
ok = !agentNoOtel.services.opentelemetry-collector.enable;
|
||||
}
|
||||
{
|
||||
|
|
|
|||
|
|
@ -283,11 +283,11 @@ let
|
|||
{
|
||||
# The case the old hive-tier gate got wrong: this host runs the swarm's
|
||||
# collector and no hive collector at all, and the forwarder still exists
|
||||
# with an address that resolves. Gated on `otel.enable` it rendered
|
||||
# with an address that resolves. Gated on `otel.collector.enable` it rendered
|
||||
# nothing here, so the store's journal left no trace anywhere.
|
||||
name = "a store on a host with no hive collector still forwards";
|
||||
ok =
|
||||
!baoWithCollector.services.hyperhive.otel.enable
|
||||
!baoWithCollector.services.hyperhive.otel.collector.enable
|
||||
&& (baoForwarder baoWithCollector).enable
|
||||
&& (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint != "";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -233,18 +233,22 @@ let
|
|||
&& e.HIVE_AGENT_NATS_URL == e.HIVE_C0RE_NATS_URL;
|
||||
}
|
||||
{
|
||||
# What agents are handed lands in their builds, so it has to be the same
|
||||
# on every hive: a name, which this hive's resolver answers with the
|
||||
# bridge IP its collector binds. hive-c0re itself keeps the address.
|
||||
name = "the agents' collector address is a swarm-wide name this hive resolves to its bridge";
|
||||
# Agents are handed no address at all: their module defaults to a
|
||||
# literal name, the same on every hive and in every swarm, which this
|
||||
# hive's resolver answers with the bridge IP its collector binds.
|
||||
# hive-c0re itself keeps the address. `HYPERHIVE_OTEL_ENDPOINT` is gone,
|
||||
# and the bridge address next to it is the control that the lookup on
|
||||
# the same attrset finds what is there.
|
||||
name = "the agents' collector address is a fixed name this hive resolves to its bridge";
|
||||
ok =
|
||||
let
|
||||
e = allLocal.systemd.services.hive-c0re.environment;
|
||||
bridge = allLocal.services.hyperhive.network.bridgeIp;
|
||||
in
|
||||
e.HYPERHIVE_OTEL_ENDPOINT == "http://hive-collector.t.local:4318"
|
||||
!(e ? HYPERHIVE_OTEL_ENDPOINT)
|
||||
&& e.OTEL_EXPORTER_OTLP_ENDPOINT == "http://${bridge}:4318"
|
||||
&& lib.elem "/hive-collector.t.local/${bridge}" allLocal.services.dnsmasq.settings.address;
|
||||
&& allLocal.services.hyperhive.otel.collector.host == "hive-collector.hive"
|
||||
&& lib.elem "/hive-collector.hive/${bridge}" allLocal.services.dnsmasq.settings.address;
|
||||
}
|
||||
{
|
||||
# The agents mint against the swarm's IdP, the same endpoint the hive's
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ let
|
|||
;
|
||||
})
|
||||
hive
|
||||
hiveWithController
|
||||
runGroup
|
||||
otelSettings
|
||||
;
|
||||
|
|
@ -27,24 +28,22 @@ let
|
|||
# tier's, which lives in a container and is read through `otelSettings`.
|
||||
# The IdP is here too, with a hand-named secret file: the collector's wait
|
||||
# for that file exists beside authelia's container.
|
||||
hiveOtel = hive {
|
||||
otel.enable = true;
|
||||
hiveOtel = hiveWithController {
|
||||
otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
|
||||
deploy.authelia.enable = true;
|
||||
};
|
||||
|
||||
# A hive holding its own store leaf, placed by hand, and naming no secret
|
||||
# file: the collector's secret comes from the store under that leaf.
|
||||
hiveOtelLeaf = hive {
|
||||
otel.enable = true;
|
||||
hiveOtelLeaf = hiveWithController {
|
||||
gateway.enable = false;
|
||||
deploy.bao.clientCertFile = "/etc/pki/hive.pem";
|
||||
deploy.bao.clientKeyFile = "/etc/pki/hive-key.pem";
|
||||
};
|
||||
|
||||
# The IdP, the store and the collector on one host: the publisher that
|
||||
# writes the secret and the unit that reads it, in one config.
|
||||
hiveOtelAllLocal = hive {
|
||||
otel.enable = true;
|
||||
hiveOtelAllLocal = hiveWithController {
|
||||
deploy.bao.enable = true;
|
||||
deploy.authelia.enable = true;
|
||||
};
|
||||
|
|
@ -144,7 +143,7 @@ let
|
|||
{
|
||||
# The receiver, the pipeline entry and the nginx server it reads have to
|
||||
# agree on one address, and none of them may exist without a gateway on
|
||||
# the host. `hiveOtelLeaf` is the control: it runs no gateway, and the
|
||||
# the host. `hiveOtelLeaf` is the control: it opts out of the gateway, and the
|
||||
# clause asserting that keeps the control from going vacuous.
|
||||
name = "the hive collector reads the gateway's stub_status, and only where a gateway runs";
|
||||
ok =
|
||||
|
|
|
|||
|
|
@ -221,7 +221,7 @@ let
|
|||
in
|
||||
{
|
||||
matrix = h.deploy.matrix.enable;
|
||||
otel = h.otel.enable;
|
||||
otel = h.otel.collector.enable;
|
||||
authelia = h.deploy.authelia.enable;
|
||||
nats = h.deploy.nats.enable;
|
||||
swarm-otel = h.deploy.swarm-otel.enable;
|
||||
|
|
|
|||
|
|
@ -99,10 +99,9 @@ in
|
|||
};
|
||||
|
||||
# `clientSecretFile` is what ../host-modules/otel.nix's identity assertion
|
||||
# demands of any hive with the tier on.
|
||||
baoWithHiveOtel = hive {
|
||||
# demands of any host running a hive.
|
||||
baoWithHiveOtel = hiveWithController {
|
||||
deploy.bao.enable = true;
|
||||
otel.enable = true;
|
||||
otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -50,7 +50,6 @@ let
|
|||
deploy.hive-controller.enable = false;
|
||||
deploy.allSwarmServices = true;
|
||||
deploy.matrix.enable = false;
|
||||
otel.enable = false;
|
||||
deploy.authelia.enable = false;
|
||||
deploy.nats.enable = false;
|
||||
deploy.swarm-otel.enable = false;
|
||||
|
|
|
|||
Loading…
Reference in a new issue