hyperhive/nix/host-modules/otel.nix

# Hive-wide OTEL stats export. Set ONCE here at host level; the
# meta-flake renderer (`hive-c0re/src/meta.rs::otel_config`) reads the
# HYPERHIVE_OTEL_* env exported off hive-c0re's unit (see
# ./hive-c0re) and injects the matching `hyperhive.otel.*` build-time
# config into EVERY agent (mirroring the CA-cert injection), so each
# agent's harness exports its own Claude Code stats directly to the
# collector. There is no per-agent opt-in — this is the single switch
# for the whole hive.
{
  lib,
  config,
  ...
}:
{
  options.services.hyperhive.otel = {
    enable = lib.mkEnableOption ''
      hive-wide export of every agent's Claude Code stats (token usage,
      cost, tool calls) to an OTLP endpoint via Claude Code's built-in
      OpenTelemetry. One switch for all agents; each harness exports
      directly to the collector, so it keeps working even when hive-c0re
      is down
    '';

    endpoint = lib.mkOption {
      type = lib.types.str;
      default = "";
      example = "https://collector.example.com/otel";
      description = ''
        OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT`
        for every agent. Required when `enable` is true.
      '';
    };

    protocol = lib.mkOption {
      type = lib.types.enum [
        "http/protobuf"
        "http/json"
        "grpc"
      ];
      default = "http/protobuf";
      description = ''
        OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`.
      '';
    };

    headersCredential = lib.mkOption {
      # `str`, not `path`: a `path`-typed relative literal is hash-copied
      # into the world-readable nix store at eval time, defeating the
      # point. Keep it a string + require an absolute runtime path so the
      # secret is only ever read from disk by systemd at start.
      type = lib.types.nullOr lib.types.str;
      default = null;
      example = "/run/secrets/otel-headers";
      description = ''
        Absolute path to an operator-provided secret file whose contents
        become `OTEL_EXPORTER_OTLP_HEADERS` (e.g.
        `Authorization=Bearer <token>`). hive-c0re forwards this host
        file into each agent container's credential store via
        systemd-nspawn `--load-credential=otel-headers:<path>`; the inner
        harness unit inherits it by name (`LoadCredential`), so the token
        is never copied into the nix store, the generated config, a bind
        mount, or argv. Must be absolute. Leave null if the endpoint
        needs no auth header. A configured-but-missing file is skipped
        with a log warning (OTEL still exports, without the auth header).
      '';
    };

    extraResourceAttributes = lib.mkOption {
      type = lib.types.str;
      default = "";
      example = "deployment.environment=prod";
      description = ''
        Extra comma-separated entries appended to
        `OTEL_RESOURCE_ATTRIBUTES` after the built-in
        `service.name` / `agent` / `hive` / `swarm` labels.
      '';
    };

    debug = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = ''
        Emit OTEL SDK diagnostic messages to every agent's stderr by
        setting `CLAUDE_CODE_OTEL_DIAG_STDERR=1`. Useful when
        troubleshooting collector connectivity or endpoint config;
        leave off in normal operation to avoid noise in agent logs.
        Only meaningful when `enable` is true.
      '';
    };

    collector.enable = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = ''
        Run an OpenTelemetry collector on this host and have agents
        export to it instead of straight to `endpoint`.

        The point is the credential. Without this, every agent needs
        `headersCredential` in order to talk to the upstream — and the
        harness delivers it into the agent's own `settings.json`, where
        the agent can read it. With a collector the token stops at the
        host: the collector holds it, agents send unauthenticated to a
        bridge address only their own containers can reach.

        ⚠️ It also makes the collector a dependency in the export path.
        Today each harness exports directly, so telemetry survives
        anything host-side being down. That property is traded for the
        credential reduction; the collector is on the same host as the
        agents, so the window is small, but it is not zero.

        Off by default, and off means *absent*: no unit, no port, and
        `endpoint` keeps its current meaning for every agent.
      '';
    };

    collector.port = lib.mkOption {
      type = lib.types.port;
      default = 4318;
      description = ''
        Port the collector's OTLP/HTTP receiver listens on, at
        `services.hyperhive.network.bridgeIp`. 4318 is the OTLP/HTTP
        default.

        The port is contributed to
        `services.hyperhive.network.exposeHostPorts`, which opens it on
        the bridge interface only — so it is reachable from agent
        containers and not from the outside world.
      '';
    };

    metricIntervalMs = lib.mkOption {
      type = lib.types.nullOr lib.types.ints.positive;
      default = null;
      example = 10000;
      description = ''
        Metric export interval in milliseconds, set as
        `OTEL_METRIC_EXPORT_INTERVAL` for every agent. Claude Code's
        default is 60000 (60s). Leave `null` to use that default.

        Each agent runs claude as a short-lived per-turn process; claude
        force-flushes metrics on shutdown, so this is not required for
        metrics to be exported, but a lower value gives more frequent
        intermediate flushes within long turns. Cosmetic, not a
        correctness knob.
      '';
    };
  };

  config = lib.mkIf config.services.hyperhive.c0re.enable {
    assertions = lib.optionals config.services.hyperhive.otel.enable [
      {
        assertion = config.services.hyperhive.otel.endpoint != "";
        message = "services.hyperhive.otel.enable is true but services.hyperhive.otel.endpoint is empty.";
      }
    ];
  };
}