From 838cc9af9a959de053cac3d85e13b72a10ebc4db Mon Sep 17 00:00:00 2001 From: damocles Date: Tue, 23 Jun 2026 20:42:15 +0200 Subject: [PATCH] feat(#1930): move otel stats export to host-level services.hyperhive.otel --- hive-c0re/src/meta.rs | 161 +++++++++++++++++++++++++++++++++ nix/modules/hive-c0re.nix | 132 +++++++++++++++++++++++---- nix/templates/harness-base.nix | 48 ++++++---- 3 files changed, 303 insertions(+), 38 deletions(-) diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index 29a3a3c2..c5da4c40 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -656,6 +656,48 @@ fn peer_ca_sources() -> Vec { .collect() } +/// Hive-wide OTEL config injected into every agent's build, read off +/// hive-c0re's own unit env (set from `services.hyperhive.otel.*` in +/// `nix/modules/hive-c0re.nix`). A present, non-empty +/// `HYPERHIVE_OTEL_ENDPOINT` is the enable signal — the host module +/// asserts the endpoint is set whenever `otel.enable` is true, so +/// "endpoint present" == "OTEL on". The optional fields map to the +/// matching host options and are only carried when set. +struct OtelConfig { + endpoint: String, + protocol: String, + extra_resource_attributes: Option, + headers_credential: Option, +} + +/// Read the hive-wide OTEL config from env, or `None` when OTEL is off. +/// Mirrors `hive_ca_source` — host state surfaced to the meta renderer +/// so it can bake build-time `hyperhive.otel.*` config into each agent +/// (the per-agent options harness-base.nix consumes). Returns `None` +/// when the endpoint signal is absent so the renderer emits no +/// `hyperhive.otel.*` lines and agents keep the disabled default. +fn otel_config() -> Option { + let endpoint = std::env::var("HYPERHIVE_OTEL_ENDPOINT") + .ok() + .filter(|v| !v.is_empty())?; + let protocol = std::env::var("HYPERHIVE_OTEL_PROTOCOL") + .ok() + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| "http/protobuf".to_owned()); + let extra_resource_attributes = std::env::var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES") + .ok() + .filter(|v| !v.is_empty()); + let headers_credential = std::env::var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL") + .ok() + .filter(|v| !v.is_empty()); + Some(OtelConfig { + endpoint, + protocol, + extra_resource_attributes, + headers_credential, + }) +} + /// The ordered set of CA certs embedded next to the meta flake, as /// `(filename, host_source_path)`. The self-signed hive CA (when active) /// is `hive-ca.pem`; each peer CA is `peer-ca-.pem` in declaration @@ -907,6 +949,40 @@ where ca_refs.join(" ") ); } + // Hive-wide OTEL stats export (`services.hyperhive.otel.*`): inject the + // build-time `hyperhive.otel.*` config harness-base.nix consumes (its + // otelEnv + otelExecStart wrapper + LoadCredential). Host-driven, so + // the same config lands on every agent; emitted only when enabled. + // Mirrors the CA-cert injection above — host state -> build-time agent + // module config. + if let Some(otel) = otel_config() { + let esc = |s: &str| s.replace('\\', "\\\\").replace('"', "\\\""); + out.push_str(" hyperhive.otel.enable = true;\n"); + let _ = writeln!( + out, + " hyperhive.otel.endpoint = \"{}\";", + esc(&otel.endpoint) + ); + let _ = writeln!( + out, + " hyperhive.otel.protocol = \"{}\";", + esc(&otel.protocol) + ); + if let Some(attrs) = &otel.extra_resource_attributes { + let _ = writeln!( + out, + " hyperhive.otel.extraResourceAttributes = \"{}\";", + esc(attrs) + ); + } + if let Some(cred) = &otel.headers_credential { + let _ = writeln!( + out, + " hyperhive.otel.headersCredential = \"{}\";", + esc(cred) + ); + } + } out.push_str( r#" # The harness service inside the container runs as a # non-root unix user named after the agent (`damocles`, @@ -1445,4 +1521,89 @@ mod tests { "no certificateFiles reference without any CA signal:\n{without_ca}" ); } + + #[test] + fn render_flake_injects_otel_when_signalled() { + // services.hyperhive.otel.* -> HYPERHIVE_OTEL_* on hive-c0re's unit + // -> injected as build-time hyperhive.otel.* into every agent. With + // no endpoint signal, no hyperhive.otel lines are emitted (agents + // keep the harness-base disabled default). + // + // SAFETY: single-threaded mutation of process env vars no other + // test asserts on; restored before returning. + let render = || { + render_flake( + "github:example/hyperhive", + "path:/nix/store/aaaa-nixpkgs-source", + "path:/nix/store/bbbb-nixpkgs-unstable-source", + 8000, + "she/her", + &std::collections::HashMap::new(), + &[sample_spec("alice", false, 9001)], + ) + }; + unsafe { + std::env::remove_var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES"); + std::env::remove_var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL"); + std::env::set_var("HYPERHIVE_OTEL_ENDPOINT", "https://c.example/otel"); + std::env::set_var("HYPERHIVE_OTEL_PROTOCOL", "grpc"); + } + let on_minimal = render(); + unsafe { + std::env::set_var( + "HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES", + "deployment.environment=prod", + ); + std::env::set_var( + "HYPERHIVE_OTEL_HEADERS_CREDENTIAL", + "/run/secrets/otel-headers", + ); + } + let on_full = render(); + unsafe { + std::env::remove_var("HYPERHIVE_OTEL_ENDPOINT"); + std::env::remove_var("HYPERHIVE_OTEL_PROTOCOL"); + std::env::remove_var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES"); + std::env::remove_var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL"); + } + let off = render(); + + assert!( + on_minimal.contains("hyperhive.otel.enable = true;"), + "otel enable must be injected:\n{on_minimal}" + ); + assert!( + on_minimal.contains("hyperhive.otel.endpoint = \"https://c.example/otel\";"), + "otel endpoint must be injected:\n{on_minimal}" + ); + assert!( + on_minimal.contains("hyperhive.otel.protocol = \"grpc\";"), + "otel protocol must be injected:\n{on_minimal}" + ); + // Optional fields absent when unset. + assert!( + !on_minimal.contains("hyperhive.otel.extraResourceAttributes"), + "extraResourceAttributes must not appear when unset:\n{on_minimal}" + ); + assert!( + !on_minimal.contains("hyperhive.otel.headersCredential"), + "headersCredential must not appear when unset:\n{on_minimal}" + ); + + assert!( + on_full.contains( + "hyperhive.otel.extraResourceAttributes = \"deployment.environment=prod\";" + ), + "extraResourceAttributes must be injected when set:\n{on_full}" + ); + assert!( + on_full.contains("hyperhive.otel.headersCredential = \"/run/secrets/otel-headers\";"), + "headersCredential must be injected when set:\n{on_full}" + ); + + assert!( + !off.contains("hyperhive.otel"), + "no otel lines when disabled:\n{off}" + ); + } } diff --git a/nix/modules/hive-c0re.nix b/nix/modules/hive-c0re.nix index aae957bb..922e94f7 100644 --- a/nix/modules/hive-c0re.nix +++ b/nix/modules/hive-c0re.nix @@ -187,6 +187,75 @@ in ''; }; + # Hive-wide OTEL stats export. Set ONCE here at host level; the + # meta-flake renderer (`hive-c0re/src/meta.rs::otel_config`) reads the + # HYPERHIVE_OTEL_* env exported below off hive-c0re's unit and injects + # the matching `hyperhive.otel.*` build-time config into EVERY agent + # (mirroring the CA-cert injection), so each agent's harness exports + # its own Claude Code stats directly to the collector. There is no + # per-agent opt-in — this is the single switch for the whole hive. + options.services.hyperhive.otel = { + enable = lib.mkEnableOption '' + hive-wide export of every agent's Claude Code stats (token usage, + cost, tool calls) to an OTLP endpoint via Claude Code's built-in + OpenTelemetry. One switch for all agents; each harness exports + directly to the collector, so it keeps working even when hive-c0re + is down + ''; + + endpoint = lib.mkOption { + type = lib.types.str; + default = ""; + example = "https://collector.example.com/otel"; + description = '' + OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT` + for every agent. Required when `enable` is true. + ''; + }; + + protocol = lib.mkOption { + type = lib.types.enum [ + "http/protobuf" + "http/json" + "grpc" + ]; + default = "http/protobuf"; + description = '' + OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`. + ''; + }; + + headersCredential = lib.mkOption { + # `str`, not `path`: a `path`-typed relative literal is hash-copied + # into the world-readable nix store at eval time, defeating the + # point. Keep it a string + require an absolute runtime path so the + # secret is only ever read from disk by systemd at start. + type = lib.types.nullOr lib.types.str; + default = null; + example = "/run/secrets/otel-headers"; + description = '' + Absolute path to an operator-provided secret file whose contents + become `OTEL_EXPORTER_OTLP_HEADERS` (e.g. + `Authorization=Bearer `). Loaded via systemd + `LoadCredential` into each agent's unit-private credential store + at runtime, so the token is never copied into the nix store or + exposed in argv. Must be absolute. Leave null if the endpoint + needs no auth header. + ''; + }; + + extraResourceAttributes = lib.mkOption { + type = lib.types.str; + default = ""; + example = "deployment.environment=prod"; + description = '' + Extra comma-separated entries appended to + `OTEL_RESOURCE_ATTRIBUTES` after the built-in + `service.name` / `agent` / `hive` / `swarm` labels. + ''; + }; + }; + # Peer hives in the same swarm. Each entry declares a remote hive # reachable from this host. Serialised to JSON and injected as # `HYPERHIVE_PEERS` into the hive-c0re service and forwarded to agent @@ -720,24 +789,31 @@ in config.services.hyperhive.swarm.wireguard.listenPort ]; - assertions = lib.mkIf config.services.hyperhive.swarm.wireguard.enable [ - { - assertion = config.services.hyperhive.swarm.wireguard.privateKeyFile != null; - message = '' - services.hyperhive.swarm.wireguard.enable requires - services.hyperhive.swarm.wireguard.privateKeyFile to be set. - Generate a key: wg genkey > /etc/wireguard/hive.key - ''; - } - { - assertion = config.services.hyperhive.swarm.wireguard.address != ""; - message = '' - services.hyperhive.swarm.wireguard.enable requires - services.hyperhive.swarm.wireguard.address to be set - (e.g. "10.100.0.1/24"). - ''; - } - ]; + assertions = + lib.optionals config.services.hyperhive.swarm.wireguard.enable [ + { + assertion = config.services.hyperhive.swarm.wireguard.privateKeyFile != null; + message = '' + services.hyperhive.swarm.wireguard.enable requires + services.hyperhive.swarm.wireguard.privateKeyFile to be set. + Generate a key: wg genkey > /etc/wireguard/hive.key + ''; + } + { + assertion = config.services.hyperhive.swarm.wireguard.address != ""; + message = '' + services.hyperhive.swarm.wireguard.enable requires + services.hyperhive.swarm.wireguard.address to be set + (e.g. "10.100.0.1/24"). + ''; + } + ] + ++ lib.optionals config.services.hyperhive.otel.enable [ + { + assertion = config.services.hyperhive.otel.endpoint != ""; + message = "services.hyperhive.otel.enable is true but services.hyperhive.otel.endpoint is empty."; + } + ]; systemd.services.hive-c0re = { description = "hyperhive coordinator daemon"; @@ -790,6 +866,26 @@ in // lib.optionalAttrs (config.services.hyperhive.swarmName != null) { HYPERHIVE_SWARM_NAME = config.services.hyperhive.swarmName; } + // lib.optionalAttrs config.services.hyperhive.otel.enable ( + # Hive-wide OTEL config -> read by meta.rs::otel_config and + # injected as build-time `hyperhive.otel.*` into every agent. + # Endpoint presence is the enable signal on the meta side; the + # optional fields are only emitted when set so absent values + # don't render no-op env lines. + let + otel = config.services.hyperhive.otel; + in + { + HYPERHIVE_OTEL_ENDPOINT = otel.endpoint; + HYPERHIVE_OTEL_PROTOCOL = otel.protocol; + } + // lib.optionalAttrs (otel.extraResourceAttributes != "") { + HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES = otel.extraResourceAttributes; + } + // lib.optionalAttrs (otel.headersCredential != null) { + HYPERHIVE_OTEL_HEADERS_CREDENTIAL = otel.headersCredential; + } + ) // { # In-cluster forge URL — the gateway vhost (`forge.`), which # nginx proxies to forgejo. The forge is mandatory, so this is diff --git a/nix/templates/harness-base.nix b/nix/templates/harness-base.nix index dae0e750..c420d2c9 100644 --- a/nix/templates/harness-base.nix +++ b/nix/templates/harness-base.nix @@ -174,23 +174,34 @@ in visible = false; }; + # OTEL stats export is configured ONCE at host level via + # `services.hyperhive.otel.*` (see nix/modules/hive-c0re.nix) and + # injected into every agent's build by the meta-flake renderer + # (`hive-c0re/src/meta.rs::otel_config`). These per-agent options are + # the build-time implementation surface that injection writes into; + # they are not meant to be set directly in an agent.nix. Marked + # `internal` so the host option is the only documented operator knob. options.hyperhive.otel = { - enable = lib.mkEnableOption '' - exporting this agent's Claude Code stats (token usage, cost, tool - calls) to an OTLP endpoint via Claude Code's built-in OpenTelemetry. - Each agent's harness exports its own stats directly to the collector, - so it keeps working even when hive-c0re is down. Meant to be enabled - hive-wide (one switch for every agent) - there is no per-agent - opt-in flag beyond this option - ''; + enable = lib.mkOption { + type = lib.types.bool; + default = false; + internal = true; + description = '' + Export this agent's Claude Code stats (token usage, cost, tool + calls) to an OTLP endpoint via Claude Code's built-in + OpenTelemetry. Each agent's harness exports directly to the + collector, so it keeps working even when hive-c0re is down. + Host-driven: set `services.hyperhive.otel.enable` instead. + ''; + }; endpoint = lib.mkOption { type = lib.types.str; default = ""; - example = "https://collector.example.com/otel"; + internal = true; description = '' OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT`. - Required when `enable` is true. + Host-driven via `services.hyperhive.otel.endpoint`. ''; }; @@ -201,8 +212,10 @@ in "grpc" ]; default = "http/protobuf"; + internal = true; description = '' OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`. + Host-driven via `services.hyperhive.otel.protocol`. ''; }; @@ -214,27 +227,27 @@ in # is only ever read from disk by systemd at start, never nix-stored. type = lib.types.nullOr lib.types.str; default = null; - example = "/run/secrets/otel-headers"; + internal = true; description = '' Absolute path to an operator-provided secret file whose contents become `OTEL_EXPORTER_OTLP_HEADERS` (e.g. `Authorization=Bearer `). Loaded via systemd `LoadCredential` into the unit-private credential store at runtime, so the token is never copied into the nix store or - exposed in the process argv. Must be an absolute path (systemd - `LoadCredential` requires one). Leave null if the endpoint needs - no auth header. + exposed in the process argv. Host-driven via + `services.hyperhive.otel.headersCredential`. ''; }; extraResourceAttributes = lib.mkOption { type = lib.types.str; default = ""; - example = "deployment.environment=prod"; + internal = true; description = '' Extra comma-separated entries appended to `OTEL_RESOURCE_ATTRIBUTES` after the built-in `service.name` / `agent` / `hive` / `swarm` labels. + Host-driven via `services.hyperhive.otel.extraResourceAttributes`. ''; }; }; @@ -787,11 +800,6 @@ in ''; assertions = [ - # OTEL export needs an endpoint to point at. - { - assertion = !config.hyperhive.otel.enable || config.hyperhive.otel.endpoint != ""; - message = "hyperhive.otel.enable is true but hyperhive.otel.endpoint is empty."; - } # Guard the inputs-routed-as-output pattern: the agent flake.nix is # expected to set `_module.args.flakeInputs = builtins.removeAttrs inputs ["self"]`. # If `self` leaks into flakeInputs the agent gets a spurious attrset