From 13daf4c7dca2add4f95f24ae928ac0fa9508cbc6 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 18 Oct 2026 23:17:28 +0200 Subject: [PATCH] every hive runs its otel collector, and agents export to it by default --- docs/networking/network.md | 6 +- docs/scheduler/observability.md | 49 +++--- hive-c0re/src/meta.rs | 141 ++++++------------ hive-c0re/src/stats/otel_metrics.rs | 5 +- hive-c0re/src/test_env.rs | 6 +- hive-metric/README.md | 3 +- hive-metric/src/main.rs | 7 +- nix/agent-modules/claude-settings.nix | 4 +- nix/agent-modules/otel.nix | 35 ++--- nix/host-modules/deploy.nix | 2 +- .../glue-bao-readers-store-order.nix | 2 +- nix/host-modules/glue-hive-otel-authelia.nix | 2 +- nix/host-modules/hive-c0re/environment.nix | 84 ++++------- nix/host-modules/hive-gateway/options.nix | 2 +- nix/host-modules/hive-network.nix | 1 - nix/host-modules/otel.nix | 119 ++++++++------- nix/host-modules/swarm-authelia.nix | 2 +- nix/host-modules/swarm-bao.nix | 2 +- nix/host-modules/swarm-otel-service.nix | 2 +- nix/host-modules/swarm-required-services.nix | 4 - nix/host-modules/swarm-review-classifier.nix | 2 +- nix/module-eval/agent-otel.nix | 27 +++- nix/module-eval/bao-otel-collector.nix | 4 +- nix/module-eval/core-toggle.nix | 16 +- nix/module-eval/hive-otel.nix | 13 +- nix/module-eval/lib.nix | 2 +- nix/module-eval/shared.nix | 5 +- nix/module-eval/swarm-services-switch.nix | 1 - 28 files changed, 245 insertions(+), 303 deletions(-) diff --git a/docs/networking/network.md b/docs/networking/network.md index e6546918..ebd5f4cf 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -117,8 +117,8 @@ dnsmasq is **authoritative** for the hive's own zone (``) plus whatever swarm-service names this host contributes via `gateway.localNames` — `forge.` and `chat.` (matrix) when this host runs those services, `auth.` -when it runs authelia, and `hive-collector.` on every hive -with `otel.enable` — answering each with the bridge IP (where nginx +when it runs authelia, and `hive-collector.hive` on every hive +— answering each with the bridge IP (where nginx is reachable, and where the hive's own collector listens). Note forge and matrix are swarm-domain names, not sub-domains of ``: a swarm runs one forge and one homeserver, so their names belong to the swarm rather @@ -182,7 +182,7 @@ By default agents can only reach the host on 80/443 (+53 DNS), so a host-side service on another port — for example a dev OTLP collector you want agents to reach directly — is unreachable. (hyperhive's own services need none of this: a module that means to be reachable from agents opens -its own port here, which `otel.enable` and `nats.enable` both do. Note +its own port here, which the hive collector and `nats.enable` both do. Note that `otel.endpoint` is the _upstream_, which no agent ever dials. See `docs/scheduler/observability.md`.) diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index d5cadcea..ab701adf 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -15,16 +15,15 @@ hyperhive metrics catalogued below. Telemetry crosses two collectors, and which one you configure depends on what the host is: -| | runs where | receives from | does | -| ------------------------------------ | ---------------------- | --------------------------------- | ----------------------------------------------- | -| **swarm tier** — `deploy.swarm-otel` | once per swarm | every hive's collector | writes the swarm's stores and exports upstream | -| **hive tier** — `otel.enable` | every hive with agents | that hive's agents, on the bridge | forwards to the swarm tier. Holds no credential | +| | runs where | receives from | does | +| ------------------------------------ | -------------- | --------------------------------- | ----------------------------------------------- | +| **swarm tier** — `deploy.swarm-otel` | once per swarm | every hive's collector | writes the swarm's stores and exports upstream | +| **hive tier** — always on | every hive | that hive's agents, on the bridge | forwards to the swarm tier. Holds no credential | An all-local host runs both; the hop between them configures itself. ```nix services.hyperhive.otel = { - enable = true; endpoint = "https://collector.example.com/otel"; # the upstream headersCredential = "/run/secrets/otel-headers"; # only the swarm tier reads it }; @@ -32,20 +31,20 @@ services.hyperhive.otel = { ## Enabling export -`otel.enable` is the single gate on a hive: one switch in the host config -covers every agent container on it. +Every hive runs its collector and every agent exports to it, with nothing to +switch on: there is no per-hive or per-agent opt-in. `endpoint` is where telemetry ends up after it leaves the swarm — optional, because the swarm's own metrics store (`deploy.victoriametrics`) is a destination in its own right. With both, telemetry goes to both. See [`swarm/services.md`](../swarm/services.md#metrics-victoriametrics--grafana). **Telemetry leaves a hive exactly one way: through the collector that -`enable` starts on the host.** Agents never talk to `endpoint` themselves — +runs on the host.** Agents never talk to `endpoint` themselves — they export unauthenticated to a collector address only their own containers can reach. They dial it as -`http://hive-collector.:`, a name identical on -every hive, which each hive's own dnsmasq answers with its bridge IP, so an -agent's build carries no hive-specific address. The otel module opens that +`http://hive-collector.hive:4318`, a name identical on +every hive and every swarm, which each hive's own dnsmasq answers with its bridge IP, so an +agent's build carries no hive-specific address and no swarm domain. The otel module opens that port on the bridge itself. That collector forwards to the swarm's ([`swarm/services.md`](../swarm/services.md#telemetry-collector-otel)), the single process holding the upstream credential and the only writer to the @@ -170,7 +169,7 @@ a build error naming the reason rather than telemetry silently going nowhere. ## Network access Agent containers reach the host only on ports 80 and 443 (plus DNS). The -collector port needs nothing from you: `otel.enable` opens it on the bridge. +collector port needs nothing from you: the hive collector opens it on the bridge. To let agents reach some other host-local service of your own → [`networking/network.md`](../networking/network.md#reaching-host-services-exposehostports). @@ -180,7 +179,7 @@ The nix module (`nix/host-modules/otel.nix`) generates every `services.hyperhive.otel.*` option's full type, default, description and example into the [options reference](https://hyperhive.darkest.space/options/) (host options — `nix build .#docs-host` for a local render). The same module -declares the collector port every hive and its agents share, +fixes the collector port every hive and its agents share, `services.hyperhive.swarm.hiveOtel.collectorPort`, and each hive's own `services.hyperhive.deploy.hive-controller.otel.extraResourceAttributes`. That's the reference; this page covers what a per-option listing can't: the @@ -296,7 +295,7 @@ cleanly and the affected level button goes on returning zero rows: -When OTEL is enabled, **hive-c0re itself** also exports each agent +**hive-c0re itself** also exports each agent container's resource load — the same cgroup gauges — to this hive's own collector, exactly like an agent does and with no separate toggle. These come from the host, not the in-container Claude SDK, @@ -342,7 +341,7 @@ a crash and `tracing::warn!`s `"container crash detected"` — nothing else: no broker event, no per-hive dashboard banner. Both of those used to exist; this change removed them in favour of the log line. -When `otel.enable` is on, the hive tier's own collector runs a `journald` +The hive tier's own collector runs a `journald` receiver scoped to `hive-c0re.service` (no bind mount needed — unlike an agent's journal, this collector already shares a filesystem with the unit it reads, being a host systemd service rather than a container) and forwards @@ -353,18 +352,13 @@ matches it directly and pages the operator — no agent, and no per-hive dashboard, in the loop. Requires `services.journald.storage = "persistent"` (the NixOS default) on -every hive host with `otel.enable` set — asserted at eval time. - -⚠️ **A hive with `otel.enable` off has no operator-visible crash signal at -all** — the per-hive dashboard banner this replaces no longer exists, and -there is no fallback path. Turning on `otel.enable` is what makes a crash -reach anyone. +every hive host — asserted at eval time. ## Agent-emitted per-turn metrics (`hive-agent`) -When OTEL is enabled, the harness itself (`hive-agent`) exports one small set +The harness itself (`hive-agent`) exports one small set of metrics per claude turn, recorded the moment the turn ends (not polled). These are deliberately the fields Claude Code's own built-in export (see above) can't know about — the harness's own wall-clock timing, what woke the @@ -508,8 +502,7 @@ and the start of the response body. It repeats a persisting failure every ## Agent-emitted custom metrics (`hive-metric`) Agents can push arbitrary labeled metrics to the same OTEL collector via the -`hive-metric` CLI tool, available in every agent container when -`services.hyperhive.otel.enable = true`. +`hive-metric` CLI tool, available in every agent container. ### Usage @@ -544,11 +537,11 @@ hive-metric queue_depth 17 --type gauge hive-metric api_latency_ms 142.5 --type gauge --labels model=sonnet --labels tier=api ``` -### Error when OTEL isn't configured +### Error when an agent turns OTEL off -When `services.hyperhive.otel.enable = false` (the default), the -`OTEL_EXPORTER_OTLP_ENDPOINT` env var isn't set and `hive-metric` exits -with an informative error message. No silently dropped metrics. +An agent whose own config sets `services.hyperhive.agent.otel.enable = false` +has no `OTEL_EXPORTER_OTLP_ENDPOINT` env var, and `hive-metric` exits with an +informative error message. No silently dropped metrics. ### Wire format diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index ef7455d8..597495fa 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -561,42 +561,17 @@ fn hive_ca_source() -> Option { Some(path) } -/// Hive-wide OTEL config injected into every agent's build, read off -/// hive-c0re's own unit env (set from `services.hyperhive.otel.*`, -/// `services.hyperhive.swarm.hiveOtel.*` and -/// `services.hyperhive.deploy.hive-controller.otel.*`). A present, non-empty -/// `HYPERHIVE_OTEL_ENDPOINT` is the enable signal — the host module -/// asserts the endpoint is set whenever `otel.enable` is true, so -/// "endpoint present" == "OTEL on". The optional fields map to the -/// matching host options and are only carried when set. -struct OtelConfig { - endpoint: String, - protocol: String, - extra_resource_attributes: Option, -} - -/// Read the hive-wide OTEL config from env, or `None` when OTEL is off. -/// Mirrors `hive_ca_source` — host state surfaced to the meta renderer -/// so it can bake build-time `services.hyperhive.agent.otel.*` config into each agent -/// (the per-agent options the harness modules consume). Returns `None` -/// when the endpoint signal is absent so the renderer emits no -/// `services.hyperhive.agent.otel.*` lines and agents keep the disabled default. -fn otel_config() -> Option { - let endpoint = std::env::var("HYPERHIVE_OTEL_ENDPOINT") - .ok() - .filter(|v| !v.is_empty())?; - let protocol = std::env::var("HYPERHIVE_OTEL_PROTOCOL") +/// Extra `OTEL_RESOURCE_ATTRIBUTES` entries for every agent, read off +/// hive-c0re's own unit env +/// (`services.hyperhive.deploy.hive-controller.otel.extraResourceAttributes`). +/// `None` when unset or empty. This is the only OTEL value the renderer +/// writes into the flake: agents export to the hive collector by default and +/// find it by name, so there is no enable flag, endpoint or protocol to +/// carry. +fn otel_extra_resource_attributes() -> Option { + std::env::var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES") .ok() .filter(|v| !v.is_empty()) - .unwrap_or_else(|| "http/protobuf".to_owned()); - let extra_resource_attributes = std::env::var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES") - .ok() - .filter(|v| !v.is_empty()); - Some(OtelConfig { - endpoint, - protocol, - extra_resource_attributes, - }) } /// The ordered set of CA certs embedded next to the meta flake, as @@ -907,36 +882,20 @@ fn render_flake( ca_refs.join(" ") ); } - // Hive-wide OTEL stats export (`services.hyperhive.otel.*`): inject the - // build-time `services.hyperhive.agent.otel.*` config the harness modules consume (its - // otelEnv + otelExecStart wrapper + LoadCredential). Host-driven, so - // the same config lands on every agent; emitted only when enabled. - // Mirrors the CA-cert injection above — host state -> build-time agent - // module config. - if let Some(otel) = otel_config() { + // Extra OTEL resource labels (`deploy.hive-controller.otel.*`) are the one + // OTEL value rendered here; enablement and the collector endpoint are + // agent-module defaults. + if let Some(attrs) = otel_extra_resource_attributes() { let esc = |s: &str| s.replace('\\', "\\\\").replace('"', "\\\""); - out.push_str(" services.hyperhive.agent.otel.enable = true;\n"); let _ = writeln!( out, - " services.hyperhive.agent.otel.endpoint = \"{}\";", - esc(&otel.endpoint) + " services.hyperhive.agent.otel.extraResourceAttributes = \"{}\";", + esc(&attrs) ); - let _ = writeln!( - out, - " services.hyperhive.agent.otel.protocol = \"{}\";", - esc(&otel.protocol) - ); - if let Some(attrs) = &otel.extra_resource_attributes { - let _ = writeln!( - out, - " services.hyperhive.agent.otel.extraResourceAttributes = \"{}\";", - esc(attrs) - ); - } } // Agent-facing service URLs (`services.hyperhive.agent.forge.url`, `services.hyperhive.agent.matrix.url`): // emit the host's real values as build-time agent config, the same - // host-state -> agent-module shape as the otel block above. + // host-state -> agent-module shape as the CA block above. // // These options exist already, but until now *nothing set them*, so every // agent silently fell back to their `localhost:` defaults while the @@ -2140,16 +2099,16 @@ mod tests { } #[test] - fn render_flake_injects_otel_when_signalled() { - // services.hyperhive.otel.* -> HYPERHIVE_OTEL_* on hive-c0re's unit - // -> injected as build-time services.hyperhive.agent.otel.* into every agent. With - // no endpoint signal, no services.hyperhive.agent.otel lines are emitted (agents - // keep the the harness modules disabled default). + fn render_flake_carries_no_otel_enablement() { + // The agent module defaults otel on and finds the collector by name, so + // the flake carries no enable, endpoint or protocol line however the + // host's OTEL env is set. Only the per-hive resource labels are + // rendered, and only when set. // // Serialised against every other env-mutating test in the crate. - // `stats::otel_metrics::tests` perturbs HYPERHIVE_OTEL_ENDPOINT too, - // and lands in this same test binary — "no other test asserts on - // these" was true of this module and false of the process. + // `stats::otel_metrics::tests` perturbs OTEL env too, and lands in this + // same test binary — "no other test asserts on these" was true of this + // module and false of the process. let _env = crate::test_env::lock(); // SAFETY: serialised by the guard above; restored before returning. let render = || { @@ -2165,53 +2124,49 @@ mod tests { std::env::set_var("HYPERHIVE_OTEL_ENDPOINT", "https://c.example/otel"); std::env::set_var("HYPERHIVE_OTEL_PROTOCOL", "grpc"); } - let on_minimal = render(); + let plain = render(); unsafe { std::env::set_var( "HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES", "deployment.environment=prod", ); } - let on_full = render(); + let labelled = render(); unsafe { std::env::remove_var("HYPERHIVE_OTEL_ENDPOINT"); std::env::remove_var("HYPERHIVE_OTEL_PROTOCOL"); std::env::remove_var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES"); } - let off = render(); + // Presence control: the render is the real flake, carrying the pin. assert!( - on_minimal.contains("services.hyperhive.agent.otel.enable = true;"), - "otel enable must be injected:\n{on_minimal}" + plain.contains("rev=0123abcd"), + "the hyperhive pin line must be rendered:\n{plain}" + ); + for (name, flake) in [("plain", &plain), ("labelled", &labelled)] { + for needle in [ + "agent.otel.enable", + "agent.otel.endpoint", + "agent.otel.protocol", + "c.example", + ] { + assert!( + !flake.contains(needle), + "{name}: `{needle}` must not be rendered:\n{flake}" + ); + } + } + assert!( + !plain.contains("agent.otel.extraResourceAttributes"), + "extraResourceAttributes must not appear when unset:\n{plain}" ); assert!( - on_minimal - .contains("services.hyperhive.agent.otel.endpoint = \"https://c.example/otel\";"), - "otel endpoint must be injected:\n{on_minimal}" - ); - assert!( - on_minimal.contains("services.hyperhive.agent.otel.protocol = \"grpc\";"), - "otel protocol must be injected:\n{on_minimal}" - ); - // Optional fields absent when unset. - assert!( - !on_minimal.contains("services.hyperhive.agent.otel.extraResourceAttributes"), - "extraResourceAttributes must not appear when unset:\n{on_minimal}" - ); - - assert!( - on_full.contains( + labelled.contains( "services.hyperhive.agent.otel.extraResourceAttributes = \"deployment.environment=prod\";" ), - "extraResourceAttributes must be injected when set:\n{on_full}" - ); - - assert!( - !off.contains("services.hyperhive.agent.otel"), - "no otel lines when disabled:\n{off}" + "extraResourceAttributes must be rendered when set:\n{labelled}" ); } - #[test] fn render_flake_injects_github_disable_only_when_signalled() { // services.hyperhive.swarm.github.enable = false -> HYPERHIVE_GITHUB_DISABLED diff --git a/hive-c0re/src/stats/otel_metrics.rs b/hive-c0re/src/stats/otel_metrics.rs index fa01c52e..70745284 100644 --- a/hive-c0re/src/stats/otel_metrics.rs +++ b/hive-c0re/src/stats/otel_metrics.rs @@ -3,7 +3,7 @@ //! hive's own collector — the same first hop the agents use, named by the //! standard `OTEL_EXPORTER_OTLP_ENDPOINT` (see `nix/host-modules/hive-c0re`, //! which sets it to the collector's bridge address where agents get its -//! swarm-wide name). No toggle of +//! fixed name). No toggle of //! its own, and no credential: a hive's collector takes unauthenticated OTLP //! on the bridge, and the only hop that presents anything is the swarm tier's, //! which is the one that leaves the swarm. @@ -100,8 +100,7 @@ fn build_provider( hyperhive_flake: &str, ) -> Result { // http/json — the only OTLP transport this crate enables (matching - // hive-metric). The Claude SDK path honours `HYPERHIVE_OTEL_PROTOCOL` for - // its own export; this exporter is always http/json. + // hive-metric); this exporter is always http/json. // // The address is deliberately NOT passed here. The SDK reads // `OTEL_EXPORTER_OTLP_ENDPOINT` itself and appends the signal path diff --git a/hive-c0re/src/test_env.rs b/hive-c0re/src/test_env.rs index 3e5a95bb..f280eae9 100644 --- a/hive-c0re/src/test_env.rs +++ b/hive-c0re/src/test_env.rs @@ -12,15 +12,15 @@ //! sibling helper records why: two separate per-module mutexes serialise //! nothing against each other, and that produced a CI-only flake. The pair that //! motivated this one is `meta::tests` (which renders a flake from -//! `HYPERHIVE_OTEL_*`) and `stats::otel_metrics::tests` (which asserts which -//! variable enables the exporter) — different modules, same variable, same +//! `HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES`) and `stats::otel_metrics::tests` (which asserts which +//! variable enables the exporter) — different modules, same process env, same //! binary. //! //! ⚠️ **A green local run is weak evidence for this class.** An agent container //! has the hyperhive variables ambient-set, so a losing race still finds a //! plausible value; the nix sandbox strips them, so there the race can delete a //! variable out from under another thread. Reproduce the sandbox shape with -//! `env -u HYPERHIVE_OTEL_ENDPOINT cargo test -p hive-c0re`. +//! `env -u OTEL_EXPORTER_OTLP_ENDPOINT cargo test -p hive-c0re`. use std::sync::{Mutex, MutexGuard, PoisonError}; diff --git a/hive-metric/README.md b/hive-metric/README.md index 0a25a97d..47593d0c 100644 --- a/hive-metric/README.md +++ b/hive-metric/README.md @@ -20,7 +20,6 @@ The SDK reads standard OTEL environment variables automatically: - `OTEL_EXPORTER_OTLP_ENDPOINT` — collector URL (**required**) - `OTEL_RESOURCE_ATTRIBUTES` — resource labels (`k=v,...`) -The harness populates both per-agent when -`services.hyperhive.otel.enable = true`: an agent exports straight to +The harness populates both in every agent container: an agent exports straight to its own hive's collector. See `docs/scheduler/observability.md` for the collector setup and the metrics hyperhive exports. diff --git a/hive-metric/src/main.rs b/hive-metric/src/main.rs index 0e107031..de2ce134 100644 --- a/hive-metric/src/main.rs +++ b/hive-metric/src/main.rs @@ -7,8 +7,7 @@ //! - `OTEL_EXPORTER_OTLP_HEADERS` — auth headers (`Key=Value,...`) //! - `OTEL_RESOURCE_ATTRIBUTES` — resource labels (`k=v,...`) //! -//! The harness populates all of these per-agent when -//! `services.hyperhive.otel.enable = true`. +//! The harness populates all of these in every agent container. //! //! # Usage //! ```text @@ -25,7 +24,7 @@ use opentelemetry_sdk::metrics::Temporality; #[derive(Parser)] #[command( name = "hive-metric", - about = "Push a labeled metric to the OTEL collector (requires services.hyperhive.otel.enable = true)", + about = "Push a labeled metric to the hive's OTEL collector", // Let a leading-minus value parse as the `value` positional rather than // being rejected as an unknown flag — negative gauges are legitimate // (e.g. a delta), and it lets the counter-sign check below produce a clear @@ -90,7 +89,7 @@ fn main() -> Result<()> { if std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").is_err() { bail!( "OTEL_EXPORTER_OTLP_ENDPOINT not set — \ - enable OTEL in NixOS config (services.hyperhive.otel.enable = true)" + an agent that sets services.hyperhive.agent.otel.enable = false has no collector" ); } diff --git a/nix/agent-modules/claude-settings.nix b/nix/agent-modules/claude-settings.nix index c0dc3d6a..a5e38a2f 100644 --- a/nix/agent-modules/claude-settings.nix +++ b/nix/agent-modules/claude-settings.nix @@ -15,8 +15,8 @@ let userName = config.services.hyperhive.agent.user.name; homeDir = "/home/${userName}"; - # Hive-wide OpenTelemetry config (host-driven; baked in per-agent by - # meta.rs `otel_config`). Options declared in `otel.nix`, which also + # OpenTelemetry config (on by default; extra labels baked in per-agent by + # meta.rs). Options declared in `otel.nix`, which also # exports the generic OTLP environment this container's producers read. otelCfg = config.services.hyperhive.agent.otel; # Base claude-code environment applied to every agent regardless of OTEL. diff --git a/nix/agent-modules/otel.nix b/nix/agent-modules/otel.nix index 90af2d6a..d0f9046b 100644 --- a/nix/agent-modules/otel.nix +++ b/nix/agent-modules/otel.nix @@ -1,5 +1,5 @@ -# OpenTelemetry wiring for an agent container: the per-agent options the -# meta flake injects (host-driven from `services.hyperhive.otel.*`) plus +# OpenTelemetry wiring for an agent container: the per-agent options (on by +# default, exporting to the hive's collector by name) plus # `debug`, which the agent's own config sets, the # environment every OTLP producer inside the container reads, and the # collector that forwards this container's journal to the hive. @@ -71,35 +71,35 @@ let logExporter = if cfg.protocol == "grpc" then "otlp" else "otlphttp"; in { - # OTEL stats export is configured ONCE at host level via - # `services.hyperhive.otel.*` (see nix/host-modules/otel.nix) and - # injected into every agent's build by the meta-flake renderer - # (`hive-c0re/src/meta.rs::otel_config`). These per-agent options are - # the build-time implementation surface that injection writes into; - # they are not meant to be set directly in an agent.nix. Marked - # `internal` so the host option is the only documented operator knob. + # On by default: every hive runs a collector and answers + # `hive-collector.hive` with its own bridge address, so the endpoint below + # is the same string in every agent on every hive and nothing is rendered + # into the meta flake. The port is the literal the host fixes at + # `services.hyperhive.swarm.hiveOtel.collectorPort` (read-only there). + # Marked `internal`: these are not meant to be set in an agent.nix, except + # `debug`. options.services.hyperhive.agent.otel = { enable = lib.mkOption { type = lib.types.bool; - default = false; + default = true; internal = true; description = '' Export this agent's telemetry — Claude Code's stats (token usage, cost, tool calls) and anything pushed with - `hive-metric` — to an OTLP endpoint. Each agent exports - directly to the hive's collector, so it keeps working even when - hive-c0re is down. Host-driven: set - `services.hyperhive.otel.enable` instead. + `hive-metric` — to the hive's collector. Each agent exports + directly to it, so it keeps working even when hive-c0re is + down. ''; }; endpoint = lib.mkOption { type = lib.types.str; - default = ""; + default = "http://hive-collector.hive:4318"; internal = true; description = '' OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT`. - Host-driven via `services.hyperhive.otel.endpoint`. + The name is the one `services.hyperhive.otel.collector.host` + holds, which the hive's resolver answers. ''; }; @@ -113,7 +113,8 @@ in internal = true; description = '' OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`. - Host-driven via `services.hyperhive.otel.protocol`. + The hive collector's receiver speaks OTLP/HTTP protobuf, so + the default is the only value that reaches it. ''; }; diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index f3370360..5bb87489 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -297,7 +297,7 @@ in Named `swarm-otel` rather than `otel` because there are two collectors and the tier is the whole distinction: - {option}`services.hyperhive.otel.enable` is the **hive-tier** one, + {option}`services.hyperhive.otel.collector.enable` is the **hive-tier** one, which every hive runs. A bare `deploy.otel` would not say which it meant. A hive that does not run the swarm collector still runs its own, and reaches this one by name at diff --git a/nix/host-modules/glue-bao-readers-store-order.nix b/nix/host-modules/glue-bao-readers-store-order.nix index cd04e6d3..dda22c7f 100644 --- a/nix/host-modules/glue-bao-readers-store-order.nix +++ b/nix/host-modules/glue-bao-readers-store-order.nix @@ -41,7 +41,7 @@ let # ./otel.nix: `fetches`, the collector loading the file this unit writes # under the hive's own leaf. swarm-bao-hive-otel-oidc = - otel.enable + otel.collector.enable && hyperhiveCfg.hiveName != null && havePair baoDeploy.clientCertFile baoDeploy.clientKeyFile && diff --git a/nix/host-modules/glue-hive-otel-authelia.nix b/nix/host-modules/glue-hive-otel-authelia.nix index 7f4dd9a0..aa0daca0 100644 --- a/nix/host-modules/glue-hive-otel-authelia.nix +++ b/nix/host-modules/glue-hive-otel-authelia.nix @@ -31,7 +31,7 @@ let waits = senderAuth && !fetches; in { - config = lib.mkIf (hyperhiveCfg.deploy.authelia.enable && otel.enable && waits) { + config = lib.mkIf (hyperhiveCfg.deploy.authelia.enable && otel.collector.enable && waits) { systemd.services.hive-otel-client-secret = { after = [ "container@${autheliaMachine}.service" ]; requires = [ "container@${autheliaMachine}.service" ]; diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index e01c404e..aff9c44c 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -71,64 +71,34 @@ in # every agent. See services.hyperhive.swarm.github.enable. HYPERHIVE_GITHUB_DISABLED = "1"; } -// lib.optionalAttrs config.services.hyperhive.otel.enable ( - # OTEL config (`swarm.hiveOtel`, `deploy.hive-controller.otel`, gated on - # this hive's `otel.enable`) -> read by meta.rs::otel_config and injected - # as build-time `services.hyperhive.agent.otel.*` into every agent. - # Endpoint presence is the enable signal on the meta side; the - # optional fields are only emitted when set so absent values - # don't render no-op env lines. - let - otel = config.services.hyperhive.otel; - hiveOtel = config.services.hyperhive.swarm.hiveOtel; - controllerOtel = config.services.hyperhive.deploy.hive-controller.otel; - # The first hop, this hive's own collector, under the two spellings its - # two kinds of caller need. Agents get the swarm-wide name, so their - # builds carry nothing hive-specific; this daemon gets the bridge - # address, because on the host the name may resolve to loopback - # (`gateway.localHostsEntry`), where the receiver does not listen. - agentFirstHop = "http://${otel.collector.host}:${toString hiveOtel.collectorPort}"; - hostFirstHop = "http://${config.services.hyperhive.network.bridgeIp}:${toString hiveOtel.collectorPort}"; - in - { - # `otel.endpoint` means "where telemetry ultimately goes" and keeps - # that meaning; what agents are handed is the *first hop*, which is - # always this hive's own collector. Deriving it rather than - # redefining `endpoint` is what lets every existing deployment keep - # its configured value untouched. - HYPERHIVE_OTEL_ENDPOINT = agentFirstHop; - # hive-c0re's OWN container-resource exporter (stats/otel_metrics.rs) - # reads the STANDARD OTLP variable — the same one hive-metric and every - # agent read — and lets the SDK resolve the URL, which appends the - # signal path (`/v1/metrics`). Handing the SDK an address - # programmatically instead takes it verbatim: it POSTs to the - # collector's root, gets a 404 on every export, and says nothing, - # because OTLP export failures go to an error handler no binary here - # installs — the daemon logged "exporter enabled" and never delivered a - # sample, for as long as it had that exporter. The variable above cannot - # replace this one: - # it is the agent-config transport meta.rs reads, and the SDK does not - # know that name. - OTEL_EXPORTER_OTLP_ENDPOINT = hostFirstHop; - # The first hop is the collector's OTLP/HTTP receiver, which speaks - # protobuf regardless of what the upstream wants — `otel.protocol` - # describes the *upstream* link, and the collector's own exporter is - # what has to honour it (see nix/host-modules/otel.nix). - HYPERHIVE_OTEL_PROTOCOL = "http/protobuf"; - } - // lib.optionalAttrs (controllerOtel.extraResourceAttributes != "") { - HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES = controllerOtel.extraResourceAttributes; - } - # HYPERHIVE_OTEL_HEADERS_CREDENTIAL is deliberately NOT emitted, and - # its absence is the security half of this design: that variable would - # put the upstream token into an agent's own settings.json. Nothing - # delivers the token into the container — no nspawn credential, no - # agent-readable header file. +// { + # hive-c0re's OWN container-resource exporter (stats/otel_metrics.rs) reads + # the STANDARD OTLP variable — the same one hive-metric and every agent + # read — and lets the SDK resolve the URL, which appends the signal path + # (`/v1/metrics`). Handing the SDK an address programmatically instead + # takes it verbatim: it POSTs to the collector's root, gets a 404 on every + # export, and says nothing, because OTLP export failures go to an error + # handler no binary here installs. # - # The rule: the collector holding the credential achieves nothing while - # anything else hands out a copy, so there is exactly one holder and it - # is on the host. -) + # The bridge address, not `otel.collector.host`: on the host that name may + # resolve to loopback (`gateway.localHostsEntry`), where the receiver does + # not listen. Agents are not told an address at all; their module defaults + # to the collector's name. + OTEL_EXPORTER_OTLP_ENDPOINT = "http://${config.services.hyperhive.network.bridgeIp}:${toString config.services.hyperhive.swarm.hiveOtel.collectorPort}"; +} +// + lib.optionalAttrs + (config.services.hyperhive.deploy.hive-controller.otel.extraResourceAttributes != "") + { + HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES = + config.services.hyperhive.deploy.hive-controller.otel.extraResourceAttributes; + } +# No upstream credential is emitted here, and its absence is the security half +# of this design: it would put the upstream token where an agent could read +# it. Nothing delivers the token into a container — no nspawn credential, no +# agent-readable header file. The collector holding the credential achieves +# nothing while anything else hands out a copy, so there is exactly one +# holder and it is on the host. // { # In-cluster forge URL — the gateway vhost (`forge.`), which # nginx proxies to forgejo. Used both for internal API calls in diff --git a/nix/host-modules/hive-gateway/options.nix b/nix/host-modules/hive-gateway/options.nix index 2d4c0c70..fc16b92d 100644 --- a/nix/host-modules/hive-gateway/options.nix +++ b/nix/host-modules/hive-gateway/options.nix @@ -320,7 +320,7 @@ in description = '' Loopback TCP port nginx serves its `stub_status` counters on, at `http://127.0.0.1:''${statusPort}/stub_status`. The hive's collector - ({option}`services.hyperhive.otel.enable`) reads it with its `nginx` + (every hive runs one) reads it with its `nginx` receiver. TCP rather than a unix socket because that receiver can only dial diff --git a/nix/host-modules/hive-network.nix b/nix/host-modules/hive-network.nix index 21264644..1d902020 100644 --- a/nix/host-modules/hive-network.nix +++ b/nix/host-modules/hive-network.nix @@ -55,7 +55,6 @@ let h.gateway.enable h.gateway.dns.enable h.network.enable - h.otel.enable h.snapshotStore.enable ]; in diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index c527498f..cf284450 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -1,11 +1,8 @@ -# Hive-wide OTEL stats export, and the HIVE tier of the collector pair. -# Set ONCE here at host level; the meta-flake renderer -# (`hive-c0re/src/meta.rs::otel_config`) reads the HYPERHIVE_OTEL_* env -# exported off hive-c0re's unit (see ./hive-c0re) and injects the matching -# `services.hyperhive.agent.otel.*` build-time config into EVERY agent (mirroring the -# CA-cert injection), so each agent's harness exports its own Claude Code -# stats directly to the collector. There is no per-agent opt-in — this is -# the single switch for the whole hive. +# The HIVE tier of the collector pair: every hive runs one, on the host that +# runs hive-c0re (`deploy.hive-controller.enable`). Agents export to it at +# the fixed name `services.hyperhive.otel.collector.host`, which this hive's +# resolver answers with its own bridge IP, so an agent's build carries +# nothing hive-specific. # # This tier receives from this hive's agents and forwards to the swarm's # collector (./swarm-otel.nix). It holds no credential and picks no @@ -41,6 +38,8 @@ let # The hive's own store identity. The collector authenticates to the swarm # as the hive, so it reads its secret as the hive: the hive leaf's policy # (`swarm-secret-client`'s `policy::render`) grants `swarm/hives//*`. + runsHive = config.services.hyperhive.otel.collector.enable; + hiveLeaf = config.services.hyperhive.deploy.bao.clientCertFile != null && config.services.hyperhive.deploy.bao.clientKeyFile != null; @@ -55,39 +54,19 @@ in name = "hive-otel"; consumers = [ "opentelemetry-collector" ]; hostUnit = true; - enable = config.services.hyperhive.otel.enable; + enable = runsHive; }) - (lib.mkRenamedOptionModule - [ "services" "hyperhive" "otel" "collector" "port" ] - [ "services" "hyperhive" "swarm" "hiveOtel" "collectorPort" ] - ) + (lib.mkRemovedOptionModule [ "services" "hyperhive" "otel" "enable" ] '' + Every hive runs its own telemetry collector, and every agent exports to + it by default, so there is no switch. Remove this definition. + '') + (lib.mkRemovedOptionModule [ "services" "hyperhive" "otel" "collector" "port" ] '' + The hive collector's receiver port is fixed at 4318: agents dial that + literal. Remove this definition. + '') ]; options.services.hyperhive.otel = { - enable = lib.mkEnableOption '' - hive-wide export of every agent's Claude Code stats (token usage, - cost, tool calls) to an OTLP endpoint via Claude Code's built-in - OpenTelemetry. One switch for all agents. - - Enabling this also runs this hive's collector on the host: there is - exactly one way telemetry leaves this hive, and it is through that - collector. Agents export unauthenticated to - {option}`services.hyperhive.otel.collector.host`, a name this hive's - resolver answers with a bridge address only its own containers can - reach. - - That collector forwards to the swarm's - ({option}`services.hyperhive.deploy.swarm-otel.enable`), which holds the - upstream credential and writes the swarm's store. So an agent never - sees the credential, and neither does this tier. - - ⚠️ The collector is therefore in the path of all telemetry. It runs - on the same host as the agents and restarts on failure, and - telemetry is not the control plane, so degraded telemetry is not - degraded operation — but the export does not survive independently - of anything host-side - ''; - endpoint = lib.mkOption { type = lib.types.str; default = ""; @@ -171,25 +150,51 @@ in ''; }; + collector.enable = lib.mkOption { + type = lib.types.bool; + readOnly = true; + default = + config.services.hyperhive.deploy.hive-controller.enable + || config.services.hyperhive.deploy.allSwarmServices; + defaultText = lib.literalExpression "deploy.hive-controller.enable || deploy.allSwarmServices"; + description = '' + Whether this host runs the hive tier's collector. Not a switch: + every host that runs a hive + ({option}`services.hyperhive.deploy.hive-controller.enable`) runs + one, and so does a host carrying all the swarm's shared services + ({option}`services.hyperhive.deploy.allSwarmServices`), whose own + containers export to it. + + Agents export to it unauthenticated at + {option}`services.hyperhive.otel.collector.host`, a name this + hive's resolver answers with a bridge address only its own + containers can reach. It forwards to the swarm's collector + ({option}`services.hyperhive.deploy.swarm-otel.enable`), which holds + the upstream credential and writes the swarm's store, so neither an + agent nor this tier ever sees the credential. + + ⚠️ The collector is in the path of all telemetry. It runs on the + same host as the agents and restarts on failure; telemetry is not + the control plane, so degraded telemetry is not degraded + operation, but the export does not survive independently of + anything host-side. + ''; + }; + collector.host = lib.mkOption { type = lib.types.str; readOnly = true; - # Total on a null swarm domain, as in ./swarm-otel-service.nix: the - # required-domain assertion should be what an operator sees. - default = "hive-collector.${ - if config.services.hyperhive.swarm.domain == null then - "invalid" - else - config.services.hyperhive.swarm.domain - }"; - defaultText = lib.literalExpression ''"hive-collector.''${services.hyperhive.swarm.domain}"''; + default = "hive-collector.hive"; description = '' - The name agents reach this hive's collector on. The same string on - every hive: each hive's own resolver answers it with its own bridge - IP, so an agent's build carries no hive-specific address. + The name agents reach this hive's collector on. A literal, the same + on every hive and in every swarm: each hive's own resolver answers + it with its own bridge IP, so an agent's build carries neither a + swarm domain nor a hive-specific address. The agent module's + `services.hyperhive.agent.otel.endpoint` default spells the same + name. - No hive can claim this name as its domain: - `nix/reserved-hive-fragments.nix` forbids `hive` in any hive name. + It sits outside every domain a hive or the swarm serves: a hive's + domain is `.`, so it never ends in `.hive`. ''; }; @@ -274,12 +279,14 @@ in }; }; - # Shared by every hive's otel tier and its agents, so an agent's build - # carries the same port whichever hive renders it. + # Shared by every hive's otel tier and its agents. Read-only because the agent + # module's `otel.endpoint` default spells the same literal, and a host that + # moved the port would leave its agents dialling the old one. options.services.hyperhive.swarm.hiveOtel = { collectorPort = lib.mkOption { type = lib.types.port; default = 4318; + readOnly = true; description = '' Port of every hive collector's OTLP/HTTP receiver, at that hive's `services.hyperhive.network.bridgeIp`: the port agents dial at @@ -307,7 +314,7 @@ in ''; }; - config = lib.mkIf config.services.hyperhive.otel.enable ( + config = lib.mkIf runsHive ( let otel = config.services.hyperhive.otel; autheliaCfg = config.services.hyperhive.swarm.authelia; @@ -913,7 +920,7 @@ in # cleanly, retries forever, and reports nothing to anyone. assertion = senderAuth; message = '' - services.hyperhive.otel.enable is true but this hive has no + This hive runs a telemetry collector (every hive does) but has no identity to present to the swarm's collector: services.hyperhive.otel.clientSecretFile = ${ @@ -948,7 +955,7 @@ in "none" ]); message = '' - services.hyperhive.otel.enable is true, which runs a journald + This hive runs a telemetry collector, which runs a journald receiver reading /var/log/journal for hive-c0re's own crash-watch log line — but services.journald.storage is "${config.services.journald.storage}" on this host. diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index a88b5ac0..436a8b14 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -60,7 +60,7 @@ let # Whether the swarm collector's client is actually registered here. # # Asked of the client list rather than re-derived from the conditions - # that produce it (`otel.enable`, a non-empty audience set, authelia + # that produce it (a non-empty audience set, authelia # being co-located). A second copy of that predicate is a second thing # to keep in step, and the two drifting is not a build failure: a rule # naming an unregistered client is refused by authelia's *startup* diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index aa2626e6..a6ac67b7 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -1446,7 +1446,7 @@ let # # "Does the swarm have a collector" is not a question that can be false — # the swarm always has at least one instance of all the swarm level - # services — so gating the forwarder on `otel.enable` or + # services — so gating the forwarder on `otel.collector.enable` or # `deploy.swarm-otel.enable` would ask about the wrong host and render no # forwarder wherever the answer happened to be no. # diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index a6fc945a..be236266 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -35,7 +35,7 @@ in { # `enable` is `services.hyperhive.deploy.swarm-otel.enable` — see ./deploy.nix. # ⚠️ That is the SWARM collector. The per-hive one keeps its own - # `services.hyperhive.otel.enable` (./otel.nix) and is a different + # `services.hyperhive.otel.collector.enable` (./otel.nix) and is a different # option entirely — every hive runs that one. options.services.hyperhive.swarm.otel = { diff --git a/nix/host-modules/swarm-required-services.nix b/nix/host-modules/swarm-required-services.nix index c930e6d1..bdcac31c 100644 --- a/nix/host-modules/swarm-required-services.nix +++ b/nix/host-modules/swarm-required-services.nix @@ -49,10 +49,6 @@ in # host. Same switch, same rule, one attribute path. config.services.hyperhive.deploy.matrix.enable = lib.mkDefault deployCfg.allSwarmServices; - # The collector that feeds the pair above (note: no `swarm.` prefix, - # this is ./otel.nix's existing per-hive option). - config.services.hyperhive.otel.enable = lib.mkDefault deployCfg.allSwarmServices; - # The rest of the shared services, from the same switch and for the same # reason. # diff --git a/nix/host-modules/swarm-review-classifier.nix b/nix/host-modules/swarm-review-classifier.nix index e4d8496e..72b1b2f7 100644 --- a/nix/host-modules/swarm-review-classifier.nix +++ b/nix/host-modules/swarm-review-classifier.nix @@ -97,7 +97,7 @@ let # where this host runs a hive-tier collector on the bridge. The daemon adds # the repo and the PR to each `claude` run's resource attributes, and to # each cost record's attributes. - otelEnv = lib.optionalAttrs otelCfg.enable { + otelEnv = lib.optionalAttrs otelCfg.collector.enable { CLAUDE_CODE_ENABLE_TELEMETRY = "1"; OTEL_METRICS_EXPORTER = "otlp"; OTEL_EXPORTER_OTLP_ENDPOINT = "http://${networkCfg.bridgeIp}:${toString hyperhiveCfg.swarm.hiveOtel.collectorPort}"; diff --git a/nix/module-eval/agent-otel.nix b/nix/module-eval/agent-otel.nix index 5d75854b..a9df766f 100644 --- a/nix/module-eval/agent-otel.nix +++ b/nix/module-eval/agent-otel.nix @@ -47,7 +47,11 @@ let otel.metricIntervalMs = 10000; }; - agentNoOtel = shared.agentBare; + # Sets nothing: what an agent gets on a hive, whose meta flake renders no + # otel line. + agentDefault = shared.agentBare; + + agentNoOtel = agent { otel.enable = false; }; agentSettings = machine: machine.services.opentelemetry-collector.settings; cases = [ @@ -100,11 +104,30 @@ let && p.exporters != [ ] && lib.all (e: (s.exporters ? ${e}) && s.exporters.${e}.endpoint == agentBridge) p.exporters; } + { + # 🎯 What the meta flake no longer says: an agent that sets nothing + # exports, to a name rather than an address. The name carries no swarm + # domain and no hive, so the same build is correct on every hive; the + # resolver of the hive it runs on answers it. `agentNoOtel` above is the + # control that these lookups find nothing where the switch is off. + name = "an agent with no otel config exports to the fixed hive-collector name by default"; + ok = + let + endpoint = "http://hive-collector.hive:4318"; + env = agentDefault.systemd.globalEnvironment; + s = agentSettings agentDefault; + in + agentDefault.services.hyperhive.agent.otel.enable + && env.OTEL_EXPORTER_OTLP_ENDPOINT == endpoint + && agentDefault.environment.variables.OTEL_EXPORTER_OTLP_ENDPOINT == endpoint + && lib.all (e: s.exporters.${e}.endpoint == endpoint) s.service.pipelines.logs.exporters + && !(agentNoOtel.systemd.globalEnvironment ? OTEL_EXPORTER_OTLP_ENDPOINT); + } { # Presence control for the two cases above: with the switch off there # is no collector in the container at all, so their passing is about # the wiring rather than about a unit that renders regardless. - name = "an agent that has not opted into telemetry runs no collector"; + name = "an agent that switched telemetry off runs no collector"; ok = !agentNoOtel.services.opentelemetry-collector.enable; } { diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix index c05e56aa..a175be00 100644 --- a/nix/module-eval/bao-otel-collector.nix +++ b/nix/module-eval/bao-otel-collector.nix @@ -283,11 +283,11 @@ let { # The case the old hive-tier gate got wrong: this host runs the swarm's # collector and no hive collector at all, and the forwarder still exists - # with an address that resolves. Gated on `otel.enable` it rendered + # with an address that resolves. Gated on `otel.collector.enable` it rendered # nothing here, so the store's journal left no trace anywhere. name = "a store on a host with no hive collector still forwards"; ok = - !baoWithCollector.services.hyperhive.otel.enable + !baoWithCollector.services.hyperhive.otel.collector.enable && (baoForwarder baoWithCollector).enable && (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint != ""; } diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index c7d6ba08..033ac861 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -233,18 +233,22 @@ let && e.HIVE_AGENT_NATS_URL == e.HIVE_C0RE_NATS_URL; } { - # What agents are handed lands in their builds, so it has to be the same - # on every hive: a name, which this hive's resolver answers with the - # bridge IP its collector binds. hive-c0re itself keeps the address. - name = "the agents' collector address is a swarm-wide name this hive resolves to its bridge"; + # Agents are handed no address at all: their module defaults to a + # literal name, the same on every hive and in every swarm, which this + # hive's resolver answers with the bridge IP its collector binds. + # hive-c0re itself keeps the address. `HYPERHIVE_OTEL_ENDPOINT` is gone, + # and the bridge address next to it is the control that the lookup on + # the same attrset finds what is there. + name = "the agents' collector address is a fixed name this hive resolves to its bridge"; ok = let e = allLocal.systemd.services.hive-c0re.environment; bridge = allLocal.services.hyperhive.network.bridgeIp; in - e.HYPERHIVE_OTEL_ENDPOINT == "http://hive-collector.t.local:4318" + !(e ? HYPERHIVE_OTEL_ENDPOINT) && e.OTEL_EXPORTER_OTLP_ENDPOINT == "http://${bridge}:4318" - && lib.elem "/hive-collector.t.local/${bridge}" allLocal.services.dnsmasq.settings.address; + && allLocal.services.hyperhive.otel.collector.host == "hive-collector.hive" + && lib.elem "/hive-collector.hive/${bridge}" allLocal.services.dnsmasq.settings.address; } { # The agents mint against the swarm's IdP, the same endpoint the hive's diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 8c03419b..83f91d22 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -19,6 +19,7 @@ let ; }) hive + hiveWithController runGroup otelSettings ; @@ -27,24 +28,22 @@ let # tier's, which lives in a container and is read through `otelSettings`. # The IdP is here too, with a hand-named secret file: the collector's wait # for that file exists beside authelia's container. - hiveOtel = hive { - otel.enable = true; + hiveOtel = hiveWithController { otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret"; deploy.authelia.enable = true; }; # A hive holding its own store leaf, placed by hand, and naming no secret # file: the collector's secret comes from the store under that leaf. - hiveOtelLeaf = hive { - otel.enable = true; + hiveOtelLeaf = hiveWithController { + gateway.enable = false; deploy.bao.clientCertFile = "/etc/pki/hive.pem"; deploy.bao.clientKeyFile = "/etc/pki/hive-key.pem"; }; # The IdP, the store and the collector on one host: the publisher that # writes the secret and the unit that reads it, in one config. - hiveOtelAllLocal = hive { - otel.enable = true; + hiveOtelAllLocal = hiveWithController { deploy.bao.enable = true; deploy.authelia.enable = true; }; @@ -144,7 +143,7 @@ let { # The receiver, the pipeline entry and the nginx server it reads have to # agree on one address, and none of them may exist without a gateway on - # the host. `hiveOtelLeaf` is the control: it runs no gateway, and the + # the host. `hiveOtelLeaf` is the control: it opts out of the gateway, and the # clause asserting that keeps the control from going vacuous. name = "the hive collector reads the gateway's stub_status, and only where a gateway runs"; ok = diff --git a/nix/module-eval/lib.nix b/nix/module-eval/lib.nix index 2a0b4708..78819d9a 100644 --- a/nix/module-eval/lib.nix +++ b/nix/module-eval/lib.nix @@ -221,7 +221,7 @@ let in { matrix = h.deploy.matrix.enable; - otel = h.otel.enable; + otel = h.otel.collector.enable; authelia = h.deploy.authelia.enable; nats = h.deploy.nats.enable; swarm-otel = h.deploy.swarm-otel.enable; diff --git a/nix/module-eval/shared.nix b/nix/module-eval/shared.nix index 66452fca..ba54ad8a 100644 --- a/nix/module-eval/shared.nix +++ b/nix/module-eval/shared.nix @@ -99,10 +99,9 @@ in }; # `clientSecretFile` is what ../host-modules/otel.nix's identity assertion - # demands of any hive with the tier on. - baoWithHiveOtel = hive { + # demands of any host running a hive. + baoWithHiveOtel = hiveWithController { deploy.bao.enable = true; - otel.enable = true; otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret"; }; diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 4f5acde7..0b0b7f24 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -50,7 +50,6 @@ let deploy.hive-controller.enable = false; deploy.allSwarmServices = true; deploy.matrix.enable = false; - otel.enable = false; deploy.authelia.enable = false; deploy.nats.enable = false; deploy.swarm-otel.enable = false;