feat(#1930): move otel stats export to host-level services.hyperhive.otel

hive-c0re/src/meta.rs

@@ -656,6 +656,48 @@ fn peer_ca_sources() -> Vec<String> {
         .collect()
 }
 
+/// Hive-wide OTEL config injected into every agent's build, read off
+/// hive-c0re's own unit env (set from `services.hyperhive.otel.*` in
+/// `nix/modules/hive-c0re.nix`). A present, non-empty
+/// `HYPERHIVE_OTEL_ENDPOINT` is the enable signal — the host module
+/// asserts the endpoint is set whenever `otel.enable` is true, so
+/// "endpoint present" == "OTEL on". The optional fields map to the
+/// matching host options and are only carried when set.
+struct OtelConfig {
+    endpoint: String,
+    protocol: String,
+    extra_resource_attributes: Option<String>,
+    headers_credential: Option<String>,
+}
+
+/// Read the hive-wide OTEL config from env, or `None` when OTEL is off.
+/// Mirrors `hive_ca_source` — host state surfaced to the meta renderer
+/// so it can bake build-time `hyperhive.otel.*` config into each agent
+/// (the per-agent options harness-base.nix consumes). Returns `None`
+/// when the endpoint signal is absent so the renderer emits no
+/// `hyperhive.otel.*` lines and agents keep the disabled default.
+fn otel_config() -> Option<OtelConfig> {
+    let endpoint = std::env::var("HYPERHIVE_OTEL_ENDPOINT")
+        .ok()
+        .filter(|v| !v.is_empty())?;
+    let protocol = std::env::var("HYPERHIVE_OTEL_PROTOCOL")
+        .ok()
+        .filter(|v| !v.is_empty())
+        .unwrap_or_else(|| "http/protobuf".to_owned());
+    let extra_resource_attributes = std::env::var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES")
+        .ok()
+        .filter(|v| !v.is_empty());
+    let headers_credential = std::env::var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL")
+        .ok()
+        .filter(|v| !v.is_empty());
+    Some(OtelConfig {
+        endpoint,
+        protocol,
+        extra_resource_attributes,
+        headers_credential,
+    })
+}
+
 /// The ordered set of CA certs embedded next to the meta flake, as
 /// `(filename, host_source_path)`. The self-signed hive CA (when active)
 /// is `hive-ca.pem`; each peer CA is `peer-ca-<N>.pem` in declaration
@@ -907,6 +949,40 @@ where
             ca_refs.join(" ")
         );
     }
+    // Hive-wide OTEL stats export (`services.hyperhive.otel.*`): inject the
+    // build-time `hyperhive.otel.*` config harness-base.nix consumes (its
+    // otelEnv + otelExecStart wrapper + LoadCredential). Host-driven, so
+    // the same config lands on every agent; emitted only when enabled.
+    // Mirrors the CA-cert injection above — host state -> build-time agent
+    // module config.
+    if let Some(otel) = otel_config() {
+        let esc = |s: &str| s.replace('\\', "\\\\").replace('"', "\\\"");
+        out.push_str("              hyperhive.otel.enable = true;\n");
+        let _ = writeln!(
+            out,
+            "              hyperhive.otel.endpoint = \"{}\";",
+            esc(&otel.endpoint)
+        );
+        let _ = writeln!(
+            out,
+            "              hyperhive.otel.protocol = \"{}\";",
+            esc(&otel.protocol)
+        );
+        if let Some(attrs) = &otel.extra_resource_attributes {
+            let _ = writeln!(
+                out,
+                "              hyperhive.otel.extraResourceAttributes = \"{}\";",
+                esc(attrs)
+            );
+        }
+        if let Some(cred) = &otel.headers_credential {
+            let _ = writeln!(
+                out,
+                "              hyperhive.otel.headersCredential = \"{}\";",
+                esc(cred)
+            );
+        }
+    }
     out.push_str(
         r#"              # The harness service inside the container runs as a
               # non-root unix user named after the agent (`damocles`,
@@ -1445,4 +1521,89 @@ mod tests {
             "no certificateFiles reference without any CA signal:\n{without_ca}"
         );
     }
+
+    #[test]
+    fn render_flake_injects_otel_when_signalled() {
+        // services.hyperhive.otel.* -> HYPERHIVE_OTEL_* on hive-c0re's unit
+        // -> injected as build-time hyperhive.otel.* into every agent. With
+        // no endpoint signal, no hyperhive.otel lines are emitted (agents
+        // keep the harness-base disabled default).
+        //
+        // SAFETY: single-threaded mutation of process env vars no other
+        // test asserts on; restored before returning.
+        let render = || {
+            render_flake(
+                "github:example/hyperhive",
+                "path:/nix/store/aaaa-nixpkgs-source",
+                "path:/nix/store/bbbb-nixpkgs-unstable-source",
+                8000,
+                "she/her",
+                &std::collections::HashMap::new(),
+                &[sample_spec("alice", false, 9001)],
+            )
+        };
+        unsafe {
+            std::env::remove_var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES");
+            std::env::remove_var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL");
+            std::env::set_var("HYPERHIVE_OTEL_ENDPOINT", "https://c.example/otel");
+            std::env::set_var("HYPERHIVE_OTEL_PROTOCOL", "grpc");
+        }
+        let on_minimal = render();
+        unsafe {
+            std::env::set_var(
+                "HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES",
+                "deployment.environment=prod",
+            );
+            std::env::set_var(
+                "HYPERHIVE_OTEL_HEADERS_CREDENTIAL",
+                "/run/secrets/otel-headers",
+            );
+        }
+        let on_full = render();
+        unsafe {
+            std::env::remove_var("HYPERHIVE_OTEL_ENDPOINT");
+            std::env::remove_var("HYPERHIVE_OTEL_PROTOCOL");
+            std::env::remove_var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES");
+            std::env::remove_var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL");
+        }
+        let off = render();
+
+        assert!(
+            on_minimal.contains("hyperhive.otel.enable = true;"),
+            "otel enable must be injected:\n{on_minimal}"
+        );
+        assert!(
+            on_minimal.contains("hyperhive.otel.endpoint = \"https://c.example/otel\";"),
+            "otel endpoint must be injected:\n{on_minimal}"
+        );
+        assert!(
+            on_minimal.contains("hyperhive.otel.protocol = \"grpc\";"),
+            "otel protocol must be injected:\n{on_minimal}"
+        );
+        // Optional fields absent when unset.
+        assert!(
+            !on_minimal.contains("hyperhive.otel.extraResourceAttributes"),
+            "extraResourceAttributes must not appear when unset:\n{on_minimal}"
+        );
+        assert!(
+            !on_minimal.contains("hyperhive.otel.headersCredential"),
+            "headersCredential must not appear when unset:\n{on_minimal}"
+        );
+
+        assert!(
+            on_full.contains(
+                "hyperhive.otel.extraResourceAttributes = \"deployment.environment=prod\";"
+            ),
+            "extraResourceAttributes must be injected when set:\n{on_full}"
+        );
+        assert!(
+            on_full.contains("hyperhive.otel.headersCredential = \"/run/secrets/otel-headers\";"),
+            "headersCredential must be injected when set:\n{on_full}"
+        );
+
+        assert!(
+            !off.contains("hyperhive.otel"),
+            "no otel lines when disabled:\n{off}"
+        );
+    }
 }

nix/modules/hive-c0re.nix

@@ -187,6 +187,75 @@ in
     '';
   };
 
+  # Hive-wide OTEL stats export. Set ONCE here at host level; the
+  # meta-flake renderer (`hive-c0re/src/meta.rs::otel_config`) reads the
+  # HYPERHIVE_OTEL_* env exported below off hive-c0re's unit and injects
+  # the matching `hyperhive.otel.*` build-time config into EVERY agent
+  # (mirroring the CA-cert injection), so each agent's harness exports
+  # its own Claude Code stats directly to the collector. There is no
+  # per-agent opt-in — this is the single switch for the whole hive.
+  options.services.hyperhive.otel = {
+    enable = lib.mkEnableOption ''
+      hive-wide export of every agent's Claude Code stats (token usage,
+      cost, tool calls) to an OTLP endpoint via Claude Code's built-in
+      OpenTelemetry. One switch for all agents; each harness exports
+      directly to the collector, so it keeps working even when hive-c0re
+      is down
+    '';
+
+    endpoint = lib.mkOption {
+      type = lib.types.str;
+      default = "";
+      example = "https://collector.example.com/otel";
+      description = ''
+        OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT`
+        for every agent. Required when `enable` is true.
+      '';
+    };
+
+    protocol = lib.mkOption {
+      type = lib.types.enum [
+        "http/protobuf"
+        "http/json"
+        "grpc"
+      ];
+      default = "http/protobuf";
+      description = ''
+        OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`.
+      '';
+    };
+
+    headersCredential = lib.mkOption {
+      # `str`, not `path`: a `path`-typed relative literal is hash-copied
+      # into the world-readable nix store at eval time, defeating the
+      # point. Keep it a string + require an absolute runtime path so the
+      # secret is only ever read from disk by systemd at start.
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      example = "/run/secrets/otel-headers";
+      description = ''
+        Absolute path to an operator-provided secret file whose contents
+        become `OTEL_EXPORTER_OTLP_HEADERS` (e.g.
+        `Authorization=Bearer <token>`). Loaded via systemd
+        `LoadCredential` into each agent's unit-private credential store
+        at runtime, so the token is never copied into the nix store or
+        exposed in argv. Must be absolute. Leave null if the endpoint
+        needs no auth header.
+      '';
+    };
+
+    extraResourceAttributes = lib.mkOption {
+      type = lib.types.str;
+      default = "";
+      example = "deployment.environment=prod";
+      description = ''
+        Extra comma-separated entries appended to
+        `OTEL_RESOURCE_ATTRIBUTES` after the built-in
+        `service.name` / `agent` / `hive` / `swarm` labels.
+      '';
+    };
+  };
+
   # Peer hives in the same swarm. Each entry declares a remote hive
   # reachable from this host. Serialised to JSON and injected as
   # `HYPERHIVE_PEERS` into the hive-c0re service and forwarded to agent
@@ -720,24 +789,31 @@ in
       config.services.hyperhive.swarm.wireguard.listenPort
     ];
 
-    assertions = lib.mkIf config.services.hyperhive.swarm.wireguard.enable [
+    assertions =
-      {
+      lib.optionals config.services.hyperhive.swarm.wireguard.enable [
-        assertion = config.services.hyperhive.swarm.wireguard.privateKeyFile != null;
+        {
-        message = ''
+          assertion = config.services.hyperhive.swarm.wireguard.privateKeyFile != null;
-          services.hyperhive.swarm.wireguard.enable requires
+          message = ''
-          services.hyperhive.swarm.wireguard.privateKeyFile to be set.
+            services.hyperhive.swarm.wireguard.enable requires
-          Generate a key: wg genkey > /etc/wireguard/hive.key
+            services.hyperhive.swarm.wireguard.privateKeyFile to be set.
-        '';
+            Generate a key: wg genkey > /etc/wireguard/hive.key
-      }
+          '';
-      {
+        }
-        assertion = config.services.hyperhive.swarm.wireguard.address != "";
+        {
-        message = ''
+          assertion = config.services.hyperhive.swarm.wireguard.address != "";
-          services.hyperhive.swarm.wireguard.enable requires
+          message = ''
-          services.hyperhive.swarm.wireguard.address to be set
+            services.hyperhive.swarm.wireguard.enable requires
-          (e.g. "10.100.0.1/24").
+            services.hyperhive.swarm.wireguard.address to be set
-        '';
+            (e.g. "10.100.0.1/24").
-      }
+          '';
-    ];
+        }
+      ]
+      ++ lib.optionals config.services.hyperhive.otel.enable [
+        {
+          assertion = config.services.hyperhive.otel.endpoint != "";
+          message = "services.hyperhive.otel.enable is true but services.hyperhive.otel.endpoint is empty.";
+        }
+      ];
 
     systemd.services.hive-c0re = {
       description = "hyperhive coordinator daemon";
@@ -790,6 +866,26 @@ in
       // lib.optionalAttrs (config.services.hyperhive.swarmName != null) {
         HYPERHIVE_SWARM_NAME = config.services.hyperhive.swarmName;
       }
+      // lib.optionalAttrs config.services.hyperhive.otel.enable (
+        # Hive-wide OTEL config -> read by meta.rs::otel_config and
+        # injected as build-time `hyperhive.otel.*` into every agent.
+        # Endpoint presence is the enable signal on the meta side; the
+        # optional fields are only emitted when set so absent values
+        # don't render no-op env lines.
+        let
+          otel = config.services.hyperhive.otel;
+        in
+        {
+          HYPERHIVE_OTEL_ENDPOINT = otel.endpoint;
+          HYPERHIVE_OTEL_PROTOCOL = otel.protocol;
+        }
+        // lib.optionalAttrs (otel.extraResourceAttributes != "") {
+          HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES = otel.extraResourceAttributes;
+        }
+        // lib.optionalAttrs (otel.headersCredential != null) {
+          HYPERHIVE_OTEL_HEADERS_CREDENTIAL = otel.headersCredential;
+        }
+      )
       // {
         # In-cluster forge URL — the gateway vhost (`forge.<domain>`), which
         # nginx proxies to forgejo. The forge is mandatory, so this is

nix/templates/harness-base.nix

@@ -174,23 +174,34 @@ in
     visible = false;
   };
 
+  # OTEL stats export is configured ONCE at host level via
+  # `services.hyperhive.otel.*` (see nix/modules/hive-c0re.nix) and
+  # injected into every agent's build by the meta-flake renderer
+  # (`hive-c0re/src/meta.rs::otel_config`). These per-agent options are
+  # the build-time implementation surface that injection writes into;
+  # they are not meant to be set directly in an agent.nix. Marked
+  # `internal` so the host option is the only documented operator knob.
   options.hyperhive.otel = {
-    enable = lib.mkEnableOption ''
+    enable = lib.mkOption {
-      exporting this agent's Claude Code stats (token usage, cost, tool
+      type = lib.types.bool;
-      calls) to an OTLP endpoint via Claude Code's built-in OpenTelemetry.
+      default = false;
-      Each agent's harness exports its own stats directly to the collector,
+      internal = true;
-      so it keeps working even when hive-c0re is down. Meant to be enabled
+      description = ''
-      hive-wide (one switch for every agent) - there is no per-agent
+        Export this agent's Claude Code stats (token usage, cost, tool
-      opt-in flag beyond this option
+        calls) to an OTLP endpoint via Claude Code's built-in
-    '';
+        OpenTelemetry. Each agent's harness exports directly to the
+        collector, so it keeps working even when hive-c0re is down.
+        Host-driven: set `services.hyperhive.otel.enable` instead.
+      '';
+    };
 
     endpoint = lib.mkOption {
       type = lib.types.str;
       default = "";
-      example = "https://collector.example.com/otel";
+      internal = true;
       description = ''
         OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT`.
-        Required when `enable` is true.
+        Host-driven via `services.hyperhive.otel.endpoint`.
       '';
     };
 
@@ -201,8 +212,10 @@ in
         "grpc"
       ];
       default = "http/protobuf";
+      internal = true;
       description = ''
         OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`.
+        Host-driven via `services.hyperhive.otel.protocol`.
       '';
     };
 
@@ -214,27 +227,27 @@ in
       # is only ever read from disk by systemd at start, never nix-stored.
       type = lib.types.nullOr lib.types.str;
       default = null;
-      example = "/run/secrets/otel-headers";
+      internal = true;
       description = ''
         Absolute path to an operator-provided secret file whose contents
         become `OTEL_EXPORTER_OTLP_HEADERS` (e.g.
         `Authorization=Bearer <token>`). Loaded via systemd
         `LoadCredential` into the unit-private credential store at
         runtime, so the token is never copied into the nix store or
-        exposed in the process argv. Must be an absolute path (systemd
+        exposed in the process argv. Host-driven via
-        `LoadCredential` requires one). Leave null if the endpoint needs
+        `services.hyperhive.otel.headersCredential`.
-        no auth header.
       '';
     };
 
     extraResourceAttributes = lib.mkOption {
       type = lib.types.str;
       default = "";
-      example = "deployment.environment=prod";
+      internal = true;
       description = ''
         Extra comma-separated entries appended to
         `OTEL_RESOURCE_ATTRIBUTES` after the built-in
         `service.name` / `agent` / `hive` / `swarm` labels.
+        Host-driven via `services.hyperhive.otel.extraResourceAttributes`.
       '';
     };
   };
@@ -787,11 +800,6 @@ in
     '';
 
     assertions = [
-      # OTEL export needs an endpoint to point at.
-      {
-        assertion = !config.hyperhive.otel.enable || config.hyperhive.otel.endpoint != "";
-        message = "hyperhive.otel.enable is true but hyperhive.otel.endpoint is empty.";
-      }
       # Guard the inputs-routed-as-output pattern: the agent flake.nix is
       # expected to set `_module.args.flakeInputs = builtins.removeAttrs inputs ["self"]`.
       # If `self` leaks into flakeInputs the agent gets a spurious attrset