# Hive-wide OTEL stats export, and the HIVE tier of the collector pair.
# Set ONCE here at host level; the meta-flake renderer
# (`hive-c0re/src/meta.rs::otel_config`) reads the HYPERHIVE_OTEL_* env
# exported off hive-c0re's unit (see ./hive-c0re) and injects the matching
# `hyperhive.otel.*` build-time config into EVERY agent (mirroring the
# CA-cert injection), so each agent's harness exports its own Claude Code
# stats directly to the collector. There is no per-agent opt-in — this is
# the single switch for the whole hive.
#
# This tier receives from this hive's agents and forwards to the swarm's
# collector (./swarm-otel.nix). It holds no credential and picks no
# destination: an agent's samples cross a hive boundary exactly once, and
# what happens after that is the swarm's decision, not a hive's. The
# upstream options declared below describe that far end and are read one
# tier up — they stay here because they mean what they have always meant.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  # This tier now reaches the swarm's collector by name through the
  # gateway (`swarm-otel.nix`'s `domain`) instead of a loopback URL, so it
  # needs the same hive-CA trust every other host consumer of an `https://`
  # swarm-service name needs — see `swarm-controller.nix` for the sibling
  # wiring this copies.
  #
  # `hostUnit`: `opentelemetry-collector` is a host systemd service, not a
  # container, so it reads the CA from the host path and the bundle oneshot
  # waits on `hive-tls-ca.service` itself. `enable`: `imports` is
  # unconditional at the host's top level, so without it a hive with this
  # tier off would still get a bundle oneshot and a phantom
  # `opentelemetry-collector` service holding an `SSL_CERT_FILE`.
  caTrust = import ./lib/hive-ca-trust.nix {
    inherit lib;
    tlsCfg = config.services.hyperhive.tls;
    gatewayCfg = config.services.hyperhive.gateway;
  };
in
{
  imports = [
    (caTrust.trustBundle {
      inherit pkgs;
      name = "hive-otel";
      consumers = [ "opentelemetry-collector" ];
      hostUnit = true;
      enable = config.services.hyperhive.otel.enable;
    })
  ];

  options.services.hyperhive.otel = {
    enable = lib.mkEnableOption ''
      hive-wide export of every agent's Claude Code stats (token usage,
      cost, tool calls) to an OTLP endpoint via Claude Code's built-in
      OpenTelemetry. One switch for all agents.

      Enabling this also runs this hive's collector on the host: there is
      exactly one way telemetry leaves this hive, and it is through that
      collector. Agents export unauthenticated to a bridge address only
      their own containers can reach.

      That collector forwards to the swarm's
      ({option}`services.hyperhive.swarm.otel.enable`), which holds the
      upstream credential and writes the swarm's store. So an agent never
      sees the credential, and neither does this tier.

      ⚠️ The collector is therefore in the path of all telemetry. It runs
      on the same host as the agents and restarts on failure, and
      telemetry is not the control plane, so degraded telemetry is not
      degraded operation — but the export no longer survives independently
      of anything host-side
    '';

    endpoint = lib.mkOption {
      type = lib.types.str;
      default = "";
      example = "https://collector.example.com/otel";
      description = ''
        Upstream OTLP endpoint: where telemetry ultimately goes, after it
        has left the swarm.

        Read by the swarm's collector
        ({option}`services.hyperhive.swarm.otel.enable`), which is the only
        tier that holds the upstream credential. An agent is handed the
        *first* hop instead — this hive's own collector — so this value is
        never given to a container.

        Optional. Leave it empty and the swarm's own metrics store
        ({option}`services.hyperhive.swarm.victoriametrics.enable`) is the
        destination; that is a complete deployment, not a degraded one.
        Set both and telemetry goes to both.
      '';
    };

    protocol = lib.mkOption {
      type = lib.types.enum [
        "http/protobuf"
        "http/json"
        "grpc"
      ];
      default = "http/protobuf";
      description = ''
        OTLP wire protocol for the **upstream** link, honoured by the
        swarm collector's exporter.

        Not what agents speak: their first hop is this hive's collector,
        whose OTLP/HTTP receiver takes protobuf whatever the upstream
        wants (see `hive-c0re/environment.nix`).
      '';
    };

    headersCredential = lib.mkOption {
      # `str`, not `path`: a `path`-typed relative literal is hash-copied
      # into the world-readable nix store at eval time, defeating the
      # point. Keep it a string + require an absolute runtime path so the
      # secret is only ever read from disk by systemd at start.
      type = lib.types.nullOr lib.types.str;
      default = null;
      example = "/run/secrets/otel-headers";
      description = ''
        Absolute path to an operator-provided secret file holding the
        upstream auth header as `NAME=value` (e.g.
        `Authorization=Bearer <token>`).

        **Only the swarm's collector reads this** — the one tier that
        talks to the upstream. It arrives as an `EnvironmentFile`, so the
        value is never read by nix, never copied into the store or the
        generated config, and never passed in argv; and it reaches
        neither an agent container nor this hive's own collector, which
        is the point of the tiers existing. Must be absolute.

        Leave null if the upstream needs no auth header; the collector
        then sends none rather than an empty one.
      '';
    };

    extraResourceAttributes = lib.mkOption {
      type = lib.types.str;
      default = "";
      example = "deployment.environment=prod";
      description = ''
        Extra comma-separated entries appended to
        `OTEL_RESOURCE_ATTRIBUTES` after the built-in
        `service.name` / `agent` / `hive` / `swarm` labels.
      '';
    };

    debug = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = ''
        Emit OTEL SDK diagnostic messages to every agent's stderr by
        setting `CLAUDE_CODE_OTEL_DIAG_STDERR=1`. Useful when
        troubleshooting collector connectivity or endpoint config;
        leave off in normal operation to avoid noise in agent logs.
        Only meaningful when `enable` is true.
      '';
    };

    collector.upstreamHeaderName = lib.mkOption {
      type = lib.types.str;
      default = "Authorization";
      description = ''
        Name of the HTTP header the collector sends upstream, whose
        *value* comes from `headersCredential`.

        The name is here and the value is not, and that split is forced
        rather than chosen: the collector models exporter headers as a
        static map, so rendering them means nix reading the value — the
        one thing `headersCredential` being a path exists to prevent.
        A name is public, a value is not.

        ⇒ exactly one header is expressible this way. A credential
        carrying several (`a=1,b=2`) would be read as a single value,
        which is why the shape is a named header rather than an opaque
        blob: a second header has to be *declared*, not smuggled.
      '';
    };

    collector.port = lib.mkOption {
      type = lib.types.port;
      default = 4318;
      description = ''
        Port the collector's OTLP/HTTP receiver listens on, at
        `services.hyperhive.network.bridgeIp`. 4318 is the OTLP/HTTP
        default.

        The port is contributed to
        `services.hyperhive.network.exposeHostPorts`, which opens it on
        the bridge interface only — so it is reachable from agent
        containers and not from the outside world.
      '';
    };

    scrapeTargets = lib.mkOption {
      type = lib.types.attrsOf lib.types.str;
      default = { };
      example = lib.literalExpression ''{ hive-forge = "127.0.0.1:3000"; }'';
      description = ''
        Prometheus exposition endpoints this hive's collector scrapes, as
        `<job name> = "<host>:<port>"`.

        **A service declares its own entry, from its own module, under its
        own `enable`** — the same rule as the swarm tier's option of the
        same name, and for the same reason: an entry exists only where the
        service that named it runs, so the scraper and its target are
        co-located by construction rather than by luck.

        ⚠️ **This tier, not the swarm one, is where a HIVE-owned target
        belongs.** The two are not interchangeable: a swarm service does
        not belong to a hive, so its samples must never acquire a `hive`
        label — which is why that scraper lives one tier up. Putting a
        hive-owned target there would either mislabel it or leave it
        unattributed.

        Scraping is unauthenticated on purpose. This receiver is a
        prometheus-to-OTLP converter sitting next to what it reads, in the
        same trust position as the agents already pushing to this
        collector; the authenticated hop is the collector's own export
        onward, which is one hop for the whole hive rather than one per
        target.

        Empty by default, and that is the shipped case — no scrape
        receiver is emitted at all, because a `prometheus` receiver with
        nothing to scrape is a config that renders, starts and collects
        nothing.
      '';
    };

    clientSecretFile = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default =
        if
          config.services.hyperhive.swarm.authelia.enable && config.services.hyperhive.hiveName != null
        then
          "${config.services.hyperhive.swarm.authelia.hostClientSecretDir}/"
          + "${config.services.hyperhive.swarm.authelia.hiveClientPrefix}${config.services.hyperhive.hiveName}.secret"
        else
          null;
      defaultText = lib.literalExpression ''"''${swarm.authelia.hostClientSecretDir}/''${swarm.authelia.hiveClientPrefix}''${hiveName}.secret" when this host runs the swarm's IdP, else null'';
      example = "/var/lib/secrets/hive-telemetry.secret";
      description = ''
        Absolute path to this hive's OAuth2 client secret, used to
        authenticate to the swarm's collector as this hive.

        **Whether this is set is what decides whether this tier
        authenticates at all.** A credential is the only thing that makes
        authenticated egress possible, so its presence is the condition
        rather than a second switch that could disagree with it.

        Defaults to the secret this host's own authelia minted, which is
        correct exactly when the swarm's IdP runs here. On a hive that is
        not that host, the file has to arrive some other way and this
        option names wherever it landed — the same manual-copy shape
        `services.hyperhive.swarm.authelia.oidc.hiveIdentities` documents,
        where delivering a secret to a hive that is not this host is
        deliberately not solved.

        Read by `LoadCredential`, so it is never evaluated by nix, never
        copied into the store and never passed in argv.
      '';
    };

    metricIntervalMs = lib.mkOption {
      type = lib.types.nullOr lib.types.ints.positive;
      default = null;
      example = 10000;
      description = ''
        Metric export interval in milliseconds, set as
        `OTEL_METRIC_EXPORT_INTERVAL` for every agent. Claude Code's
        default is 60000 (60s). Leave `null` to use that default.

        Each agent runs claude as a short-lived per-turn process; claude
        force-flushes metrics on shutdown, so this is not required for
        metrics to be exported, but a lower value gives more frequent
        intermediate flushes within long turns. Cosmetic, not a
        correctness knob.
      '';
    };
  };

  config = lib.mkIf config.services.hyperhive.otel.enable (
    let
      otel = config.services.hyperhive.otel;
      autheliaCfg = config.services.hyperhive.swarm.authelia;
      swarmOtelCfg = config.services.hyperhive.swarm.otel;
      hiveName = config.services.hyperhive.hiveName;
      listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}";
      swarmName = "otlphttp/swarm";
      authName = "oauth2client/swarm";

      # A hive always authenticates to the swarm's collector as itself, so
      # this is not a mode — it is a precondition, and the assertion below
      # is what enforces it. Kept as a name because several places have to
      # read "do we have what it takes", and an eval error from a null path
      # names this file rather than the option an operator has to set.
      senderAuth = otel.clientSecretFile != null && hiveName != null;

      # This hive's client id, and also the audience it must ASK for. Both
      # are `hiveClientPrefix` + the hive's name because that is the one
      # name the swarm already agrees on; the receiver one tier up derives
      # the same string.
      hiveClient = "${autheliaCfg.hiveClientPrefix}${toString hiveName}";

      # systemd exports `CREDENTIALS_DIRECTORY` to any unit with
      # `LoadCredential`, and the collector expands `${env:…}` at load. So
      # the secret reaches the process as a PATH resolved at runtime — nix
      # renders neither the value nor the directory, and nothing has to
      # hardcode `/run/credentials/<unit>`.
      credName = "swarm-client.secret";
      secretWaitUnit = "hive-otel-client-secret";
    in
    {
      # Reachable from agent containers and nowhere else: this opens
      # the port on the bridge interface only.
      services.hyperhive.network.exposeHostPorts = [ otel.collector.port ];

      services.opentelemetry-collector = {
        enable = true;
        # Contrib, matching the swarm tier (./swarm-otel.nix). The upstream
        # default build has no auth extensions at all, and this tier has to
        # *present* a credential to the swarm tier — `oauth2client` lives
        # only in contrib, so the package choice is what makes authenticated
        # egress expressible rather than a preference.
        #
        # Not a build-farm cost: contrib is fetched, not compiled.
        #
        # ⚠️ Read the note directly below before adding any extension here.
        # It describes precisely the trap this package unlocks: naming an
        # extension the build lacks passes `validate` and then kills the
        # collector at startup. With contrib the extensions exist — but the
        # gap it warns about (a green build proving nothing about whether
        # the process starts) is exactly why this module's auth wiring is
        # gated by a probe that runs both collectors, not by eval.
        package = pkgs.opentelemetry-collector-contrib;
        # `validateConfigFile` defaults to `isStorePath configFile`,
        # and `configFile` is null on the `settings` path — so the
        # upstream default is OFF for exactly the way this module
        # configures it. Turning it on runs `otelcol validate` at
        # build time, which is the collector checking its own config.
        # ⚠️ It is a PARSER, not a wiring check, and the gap is wider
        # than "no sample was sent": measured 2026-08-15, `validate`
        # ACCEPTS a receiver naming an auth extension that is absent
        # from the build, and the collector then dies at startup with
        # `Failed to start component`. So a green build does not
        # prove this config STARTS, never mind that a sample arrives.
        validateConfigFile = true;
        settings = {
          receivers = {
            otlp.protocols.http.endpoint = listen;
          }
          # Only when a service has actually declared a target. An enabled
          # `prometheus` receiver with an empty `scrape_configs` renders,
          # validates and starts perfectly while reading nothing — and the
          # empty set is the default, so that inert shape would be what
          # most hives deploy.
          // lib.optionalAttrs (otel.scrapeTargets != { }) {
            prometheus.config.scrape_configs = lib.mapAttrsToList (job: target: {
              job_name = job;
              static_configs = [ { targets = [ target ]; } ];
            }) otel.scrapeTargets;
          };

          # One destination, and it is the swarm's collector. This tier
          # holds no upstream credential and writes no store: it receives
          # from this hive's agents and forwards, which is the whole of
          # its job. Everything that decides where telemetry ultimately
          # goes lives one tier up, in ./swarm-otel.nix.
          exporters.${swarmName} = {
            # ⚠️ Plain `endpoint`, and the exporter beside this one in
            # ./swarm-otel.nix warns against exactly that spelling — read
            # both before "fixing" either. The difference is the far end,
            # not the exporter: `endpoint` is a BASE that otlphttp appends
            # `/v1/metrics` to, which is precisely the path an OTLP/HTTP
            # receiver serves. VictoriaMetrics is the odd one out, serving
            # OTLP at `/opentelemetry/api/v1/push`, and that is why the
            # store exporter needs `metrics_endpoint` while this one must
            # not have it.
            #
            # By name through the gateway, not a loopback literal: a
            # loopback literal is correct only while listener and caller
            # share a netns, an assumption that has cost this project two
            # outages, and it is exactly the split-host case a swarm
            # service name exists to make a config fact rather than a code
            # change. `https://` because that name resolves through the
            # gateway even on a co-located host — see `caTrust` above for
            # the trust half that makes this verify.
            # The hive's own path under the collector's single name. The
            # swarm tier gives each hive its own authenticated receiver and
            # routes to it by this prefix, so the path is not decoration —
            # it selects WHICH receiver, and therefore which hive the
            # samples get labelled as.
            endpoint = "https://${swarmOtelCfg.domain}" + lib.optionalString senderAuth "/${toString hiveName}";
          }
          // lib.optionalAttrs senderAuth { auth.authenticator = authName; };

          # ⚠️ An extension not listed here is INERT: the collector starts
          # clean and the exporter naming it sends nothing authenticated.
          service.extensions = lib.optional senderAuth authName;

          service.pipelines.metrics = {
            # EXTENDED, not replaced. Assigning here instead of appending
            # would drop `otlp` — the hive would stop receiving from its
            # own agents while still rendering a config the collector
            # starts cleanly on, so nothing would report the loss. The
            # `optional` matters too: a pipeline naming a receiver that no
            # longer exists is a startup failure, which `validateConfigFile`
            # below turns into a build failure rather than a broken deploy.
            receivers = [ "otlp" ] ++ lib.optional (otel.scrapeTargets != { }) "prometheus";
            exporters = [ swarmName ];
          };
        }
        // lib.optionalAttrs senderAuth {
          extensions.${authName} = {
            client_id = hiveClient;
            # A real key, measured against this collector version rather
            # than assumed — with a deliberate typo rejected in the same
            # run, so "accepted" is distinguishable from "ignores
            # everything". Keeps the secret out of nix entirely: the
            # collector opens the file itself.
            client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${credName}";
            token_url = "${toString autheliaCfg.url}/api/oidc/token";
            # ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted.
            # Registering it on the client only makes it permissible; a
            # token minted without asking carries `aud: []` and every
            # receiver refuses it — with a config that reads perfectly at
            # both ends. Measured against authelia 4.39.20.
            endpoint_params.audience = hiveClient;
          };
        };
      };

      # `LoadCredential` and not a copy-oneshot: this collector is a HOST
      # unit, so there is no container boundary to cross and therefore no
      # reason for a second on-disk copy of the secret. systemd hands it to
      # the process in a private tmpfs and exports the directory, which is
      # what the config above names.
      systemd.services.opentelemetry-collector = lib.optionalAttrs senderAuth {
        serviceConfig.LoadCredential = [ "${credName}:${otel.clientSecretFile}" ];

        # `requires`, not merely `after`: ordering without gating leaves the
        # collector starting anyway and failing on its own `LoadCredential`,
        # which is the failure the wait exists to prevent. Same reasoning
        # `lib/hive-ca-trust.nix` records for its bundle consumers.
        requires = [ "${secretWaitUnit}.service" ];
        after = [ "${secretWaitUnit}.service" ];
      };

      # Wait for the credential rather than racing it.
      #
      # `LoadCredential` naming a missing path is FATAL AT UNIT START, and the
      # secret is minted by authelia's first-boot generator inside its own
      # container — nothing orders a host unit against that. Worse, nixpkgs
      # sets `Restart = "always"` with no `RestartSec`, so the failure is
      # instant and the unit burns systemd's 5-starts-in-10s allowance in well
      # under a second, landing in `start-limit-hit` where it stops retrying
      # entirely and needs `systemctl reset-failed` by hand.
      #
      # 🔑 `Restart = always` reads like it makes this self-healing and does
      # the opposite: a SLOW-failing unit retries until the secret appears, a
      # FAST-failing one exhausts its limit before the thing it waits for can
      # exist. This oneshot converts the fast failure into a slow one, which
      # is what that restart policy is actually good at.
      #
      # Copied from `hive-forge-oidc-secret.service`, which solves exactly
      # this for the forge — see `docs/swarm/secrets.md`.
      systemd.services.${secretWaitUnit} = lib.mkIf senderAuth {
        description = "wait for this hive's telemetry client secret";
        # Deliberately says nothing about the collector: the CONSUMER declares
        # `requires` + `after` above, which both pulls this into the
        # transaction and gates on it. Stating the order from both sides too
        # would be one more thing to keep consistent for no added guarantee.
        #
        # Only when the minting container is on THIS host. Elsewhere the file
        # is operator-provided and there is no local unit to order against —
        # naming one that does not exist orders nothing, silently.
        after = lib.optional autheliaCfg.enable "container@${autheliaCfg.machine}.service";
        requires = lib.optional autheliaCfg.enable "container@${autheliaCfg.machine}.service";
        serviceConfig = {
          Type = "oneshot";
          RemainAfterExit = true;
          SyslogIdentifier = secretWaitUnit;
          # ⚠️ Longer than the bounded wait below. `DefaultTimeoutStartSec` is
          # 90s, so without this systemd kills the unit at 90 — before it can
          # emit the message naming the file it was waiting for.
          TimeoutStartSec = "180s";
        };
        path = [ pkgs.coreutils ];
        script = ''
          set -euo pipefail
          secret=${lib.escapeShellArg (toString otel.clientSecretFile)}

          # Bounded, then FAIL — never skip. A silent skip produces a
          # collector that starts and ships nothing, which is the failure
          # mode this whole path is trying not to have.
          for _ in $(seq 1 60); do
            [ -s "$secret" ] && exit 0
            sleep 2
          done
          echo "telemetry client secret $secret has not appeared after 120s" >&2
          exit 1
        '';
      };

      assertions = [
        {
          # A hive authenticates to the swarm's collector as itself — there
          # is no unauthenticated path to fall back to, so a missing
          # credential is a broken deployment rather than a quieter mode.
          # Caught here because the alternative is a collector that starts
          # cleanly, retries forever, and reports nothing to anyone.
          assertion = senderAuth;
          message = ''
            services.hyperhive.otel.enable is true but this hive has no
            identity to present to the swarm's collector:

              services.hyperhive.otel.clientSecretFile = ${
                if otel.clientSecretFile == null then "null" else otel.clientSecretFile
              }
              services.hyperhive.hiveName             = ${if hiveName == null then "null" else hiveName}

            Every hive authenticates as itself — that is what makes the
            `hive` label on its metrics mean anything — so both are
            required.

            On a host that runs the swarm's identity provider, the default
            already points at the secret authelia minted. On a hive that
            does not, copy that hive's secret across and name it here.
          '';
        }
      ];
    }
  );
}