hyperhive/nix/host-modules/otel.nix

# Hive-wide OTEL stats export. Set ONCE here at host level; the
# meta-flake renderer (`hive-c0re/src/meta.rs::otel_config`) reads the
# HYPERHIVE_OTEL_* env exported off hive-c0re's unit (see
# ./hive-c0re) and injects the matching `hyperhive.otel.*` build-time
# config into EVERY agent (mirroring the CA-cert injection), so each
# agent's harness exports its own Claude Code stats directly to the
# collector. There is no per-agent opt-in — this is the single switch
# for the whole hive.
{
  lib,
  config,
  ...
}:
{
  options.services.hyperhive.otel = {
    enable = lib.mkEnableOption ''
      hive-wide export of every agent's Claude Code stats (token usage,
      cost, tool calls) to an OTLP endpoint via Claude Code's built-in
      OpenTelemetry. One switch for all agents.

      Enabling this also runs a collector on the host: there is exactly
      one way telemetry leaves this hive, and it is through that
      collector. Agents export unauthenticated to a bridge address only
      their own containers can reach, and the collector is the only
      holder of the upstream credential — an agent never sees it.

      ⚠️ The collector is therefore in the path of all telemetry. It runs
      on the same host as the agents and restarts on failure, and
      telemetry is not the control plane, so degraded telemetry is not
      degraded operation — but the export no longer survives independently
      of anything host-side
    '';

    endpoint = lib.mkOption {
      type = lib.types.str;
      default = "";
      example = "https://collector.example.com/otel";
      description = ''
        OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT`
        for every agent. Required when `enable` is true.
      '';
    };

    protocol = lib.mkOption {
      type = lib.types.enum [
        "http/protobuf"
        "http/json"
        "grpc"
      ];
      default = "http/protobuf";
      description = ''
        OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`.
      '';
    };

    headersCredential = lib.mkOption {
      # `str`, not `path`: a `path`-typed relative literal is hash-copied
      # into the world-readable nix store at eval time, defeating the
      # point. Keep it a string + require an absolute runtime path so the
      # secret is only ever read from disk by systemd at start.
      type = lib.types.nullOr lib.types.str;
      default = null;
      example = "/run/secrets/otel-headers";
      description = ''
        Absolute path to an operator-provided secret file whose contents
        become `OTEL_EXPORTER_OTLP_HEADERS` (e.g.
        `Authorization=Bearer <token>`). hive-c0re forwards this host
        file into each agent container's credential store via
        systemd-nspawn `--load-credential=otel-headers:<path>`; the inner
        harness unit inherits it by name (`LoadCredential`), so the token
        is never copied into the nix store, the generated config, a bind
        mount, or argv. Must be absolute. Leave null if the endpoint
        needs no auth header. A configured-but-missing file is skipped
        with a log warning (OTEL still exports, without the auth header).
      '';
    };

    extraResourceAttributes = lib.mkOption {
      type = lib.types.str;
      default = "";
      example = "deployment.environment=prod";
      description = ''
        Extra comma-separated entries appended to
        `OTEL_RESOURCE_ATTRIBUTES` after the built-in
        `service.name` / `agent` / `hive` / `swarm` labels.
      '';
    };

    debug = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = ''
        Emit OTEL SDK diagnostic messages to every agent's stderr by
        setting `CLAUDE_CODE_OTEL_DIAG_STDERR=1`. Useful when
        troubleshooting collector connectivity or endpoint config;
        leave off in normal operation to avoid noise in agent logs.
        Only meaningful when `enable` is true.
      '';
    };

    collector.upstreamHeaderName = lib.mkOption {
      type = lib.types.str;
      default = "Authorization";
      description = ''
        Name of the HTTP header the collector sends upstream, whose
        *value* comes from `headersCredential`.

        The name is here and the value is not, and that split is forced
        rather than chosen: the collector models exporter headers as a
        static map, so rendering them means nix reading the value — the
        one thing `headersCredential` being a path exists to prevent.
        A name is public, a value is not.

        ⇒ exactly one header is expressible this way. A credential
        carrying several (`a=1,b=2`) would be read as a single value,
        which is why the shape is a named header rather than an opaque
        blob: a second header has to be *declared*, not smuggled.
      '';
    };

    collector.port = lib.mkOption {
      type = lib.types.port;
      default = 4318;
      description = ''
        Port the collector's OTLP/HTTP receiver listens on, at
        `services.hyperhive.network.bridgeIp`. 4318 is the OTLP/HTTP
        default.

        The port is contributed to
        `services.hyperhive.network.exposeHostPorts`, which opens it on
        the bridge interface only — so it is reachable from agent
        containers and not from the outside world.
      '';
    };

    metricIntervalMs = lib.mkOption {
      type = lib.types.nullOr lib.types.ints.positive;
      default = null;
      example = 10000;
      description = ''
        Metric export interval in milliseconds, set as
        `OTEL_METRIC_EXPORT_INTERVAL` for every agent. Claude Code's
        default is 60000 (60s). Leave `null` to use that default.

        Each agent runs claude as a short-lived per-turn process; claude
        force-flushes metrics on shutdown, so this is not required for
        metrics to be exported, but a lower value gives more frequent
        intermediate flushes within long turns. Cosmetic, not a
        correctness knob.
      '';
    };
  };

  config = lib.mkMerge [
    (lib.mkIf config.services.hyperhive.c0re.enable {
      assertions = lib.optionals config.services.hyperhive.otel.enable [
        {
          assertion = config.services.hyperhive.otel.endpoint != "";
          message = "services.hyperhive.otel.enable is true but services.hyperhive.otel.endpoint is empty.";
        }
      ];
    })

    (lib.mkIf config.services.hyperhive.otel.enable (
      (
        let
          otel = config.services.hyperhive.otel;
          listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}";
          # `otel.protocol` describes the UPSTREAM link and always did.
          # Inserting a collector splits the path in two, and the
          # upstream half is the one that has to keep honouring it — so
          # the exporter is chosen by it, rather than the option quietly
          # becoming "how agents talk to the collector". The agent half
          # is pinned to OTLP/HTTP by the receiver below (derived in
          # hive-c0re/environment.nix).
          grpcUpstream = otel.protocol == "grpc";
          upstreamName = if grpcUpstream then "otlp" else "otlphttp";
          upstream = {
            endpoint = otel.endpoint;
          }
          // lib.optionalAttrs (otel.headersCredential != null) {
            # The value is interpolated by the collector at runtime from
            # its environment, never by nix. `EnvironmentFile` below is
            # what puts it there. No credential configured means no
            # header at all — an upstream that needs no auth is a
            # legitimate deployment, and rendering `${env:…}` for a
            # variable nothing sets would send the literal.
            headers.${otel.collector.upstreamHeaderName} = "\${env:${otel.collector.upstreamHeaderName}}";
          }
          // lib.optionalAttrs (otel.protocol == "http/json") { encoding = "json"; };
        in
        {
          # Reachable from agent containers and nowhere else: this opens
          # the port on the bridge interface only.
          services.hyperhive.network.exposeHostPorts = [ otel.collector.port ];

          services.opentelemetry-collector = {
            enable = true;
            # `validateConfigFile` defaults to `isStorePath configFile`,
            # and `configFile` is null on the `settings` path — so the
            # upstream default is OFF for exactly the way this module
            # configures it. Turning it on runs `otelcol validate` at
            # build time, which is the collector checking its own config.
            # ⚠️ It parses; it does not prove a sample arrives.
            validateConfigFile = true;
            settings = {
              receivers.otlp.protocols.http.endpoint = listen;
              exporters.${upstreamName} = upstream;
              service.pipelines.metrics = {
                receivers = [ "otlp" ];
                exporters = [ upstreamName ];
              };
            };
          };

          # The credential file is already `NAME=value`, which is
          # systemd's EnvironmentFile format — so the secret reaches the
          # process as an environment variable without ever being read by
          # nix, written to the store, or passed in argv.
          systemd.services.opentelemetry-collector.serviceConfig =
            lib.optionalAttrs (otel.headersCredential != null)
              {
                EnvironmentFile = otel.headersCredential;
              };
        }
      )
    ))
  ];
}