hyperhive/nix/agent-modules/otel.nix

# OpenTelemetry wiring for an agent container: the per-agent options the
# meta flake injects (host-driven from `services.hyperhive.otel.*`), and
# the environment every OTLP producer inside the container reads.
#
# Claude Code is one producer here, not the owner. `hive-metric` — and
# any future in-container exporter — reads the same endpoint, protocol
# and resource labels, so they are container environment rather than
# claude settings. Claude's own switches (its telemetry master flag,
# which signals it emits) stay in `claude-settings.nix`, which reads the
# options declared below.
{
  lib,
  config,
  ...
}:
let
  cfg = config.hyperhive.otel;
  userName = config.hyperhive.user.name;
  # Hive/swarm display names, read from the per-agent options meta.rs
  # renders (NOT from `environment.variables` — those carry the same
  # names at *runtime* only, so reading them here silently yielded
  # "unknown" on every agent while the process env held the right
  # answer). `null` means the hive did not name itself; "unknown" is then
  # an honest label rather than a guess.
  hiveDisplayName =
    if config.hyperhive.hiveName == null then "unknown" else config.hyperhive.hiveName;
  swarmDisplayName =
    if config.hyperhive.swarmName == null then "unknown" else config.hyperhive.swarmName;

  # Resource labels every producer in this container stamps on what it
  # emits. `service.name` names the container's role, not one binary
  # inside it: claude's samples and `hive-metric`'s both come from this
  # agent, and their metric names already tell them apart.
  resourceAttributes =
    "service.name=hyperhive-agent,agent=${userName},hive=${hiveDisplayName},swarm=${swarmDisplayName}"
    + lib.optionalString (cfg.extraResourceAttributes != "") ",${cfg.extraResourceAttributes}";

  # The variables an OTEL SDK reads on its own, in any language, in any
  # process here.
  #
  # There is no auth header among them, and no mechanism to add one. An
  # agent exports to the hive's own collector, which is the only thing
  # holding a credential for anything upstream; nothing an agent can read
  # is a secret to the swarm. An earlier revision forwarded the
  # operator's upstream token into this container and merged it into the
  # agent's own `~/.claude/settings.json` — which handed every agent the
  # hive's credential, and was removed with the direct-export path it
  # served.
  producerEnv = {
    OTEL_EXPORTER_OTLP_ENDPOINT = cfg.endpoint;
    OTEL_EXPORTER_OTLP_PROTOCOL = cfg.protocol;
    # Force CUMULATIVE temporality — Claude Code defaults to DELTA,
    # which Prometheus/Mimir-family backends (incl. grafana-lgtm)
    # silently drop without a deltatocumulative processor.
    OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
    OTEL_RESOURCE_ATTRIBUTES = resourceAttributes;
  }
  // lib.optionalAttrs (cfg.metricIntervalMs != null) {
    OTEL_METRIC_EXPORT_INTERVAL = toString cfg.metricIntervalMs;
  };
in
{
  # OTEL stats export is configured ONCE at host level via
  # `services.hyperhive.otel.*` (see nix/host-modules/otel.nix) and
  # injected into every agent's build by the meta-flake renderer
  # (`hive-c0re/src/meta.rs::otel_config`). These per-agent options are
  # the build-time implementation surface that injection writes into;
  # they are not meant to be set directly in an agent.nix. Marked
  # `internal` so the host option is the only documented operator knob.
  options.hyperhive.otel = {
    enable = lib.mkOption {
      type = lib.types.bool;
      default = false;
      internal = true;
      description = ''
        Export this agent's telemetry — Claude Code's stats (token
        usage, cost, tool calls) and anything pushed with
        `hive-metric` — to an OTLP endpoint. Each agent exports
        directly to the hive's collector, so it keeps working even when
        hive-c0re is down. Host-driven: set
        `services.hyperhive.otel.enable` instead.
      '';
    };

    endpoint = lib.mkOption {
      type = lib.types.str;
      default = "";
      internal = true;
      description = ''
        OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT`.
        Host-driven via `services.hyperhive.otel.endpoint`.
      '';
    };

    protocol = lib.mkOption {
      type = lib.types.enum [
        "http/protobuf"
        "http/json"
        "grpc"
      ];
      default = "http/protobuf";
      internal = true;
      description = ''
        OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`.
        Host-driven via `services.hyperhive.otel.protocol`.
      '';
    };

    extraResourceAttributes = lib.mkOption {
      type = lib.types.str;
      default = "";
      internal = true;
      description = ''
        Extra comma-separated entries appended to
        `OTEL_RESOURCE_ATTRIBUTES` after the built-in
        `service.name` / `agent` / `hive` / `swarm` labels.
        Host-driven via `services.hyperhive.otel.extraResourceAttributes`.
      '';
    };

    metricIntervalMs = lib.mkOption {
      type = lib.types.nullOr lib.types.ints.positive;
      default = null;
      internal = true;
      description = ''
        Metric export interval in milliseconds, set as
        `OTEL_METRIC_EXPORT_INTERVAL`. Null leaves the SDK default (60s
        for Claude Code). Host-driven via
        `services.hyperhive.otel.metricIntervalMs`.
      '';
    };

    debug = lib.mkOption {
      type = lib.types.bool;
      default = false;
      internal = true;
      description = ''
        Emit OTEL SDK diagnostics to stderr (`CLAUDE_CODE_OTEL_DIAG_STDERR=1`).
        Host-driven via `services.hyperhive.otel.debug`.
      '';
    };
  };

  config = lib.mkIf cfg.enable {
    # Two assignments, because they cover disjoint sets of processes and
    # neither implies the other:
    #
    # - `systemd.globalEnvironment` is merged into the `Environment=`
    #   lines of every generated unit, so the harness, the
    #   bash/matrix/MCP daemons and everything the bash tool spawns
    #   beneath them all get it. This is the assignment that matters:
    #   those daemons are claude's *siblings*, not its children, so
    #   nothing shipped inside claude's own settings could ever reach
    #   them, and `hive-metric` invoked from a tool call exited with
    #   "OTEL_EXPORTER_OTLP_ENDPOINT not set".
    # - `environment.variables` lands in `/etc/set-environment`, sourced
    #   by `/etc/profile` — login shells, i.e. `hivectl shell` and
    #   `hivectl choom`, which systemd does not start.
    #
    # `NIX_REMOTE` in `default.nix` is set both ways for this same
    # reason. The difference is visible on any running agent:
    # `HIVE_ASSETS_DIR` is an `environment.variables` entry and is absent
    # from `systemctl show hive-bash-daemon.service -p Environment`,
    # while `NIX_REMOTE` — the same value, also set globally — is there.
    systemd.globalEnvironment = producerEnv;
    environment.variables = producerEnv;
  };
}