# The swarm's telemetry collector: one per swarm, in a `swarm-otel`
# nixos-container beside the swarm's other shared services.
#
# Two tiers, and they are separate on purpose:
#
#   - `otel.nix` is the **hive** tier. It receives from this hive's agents
#     on the bridge and forwards, and it holds no upstream credential.
#   - this is the **swarm** tier. It is the only holder of the upstream
#     credential, the only writer to the swarm's metrics store, and the
#     place that will stamp `hive=` from the authenticated connection
#     rather than from anything a sender can choose.
#
# On a host that runs both, both processes run. They are not collapsed:
# all-local is a statement about *where* processes run, not about what
# shape the deployment has, and a local tier boundary that disappears is
# one the local deployment stops testing. `hive=` attribution is the
# property that would differ, and the ingest auth that makes it
# unforgeable is built on this boundary existing.
#
# A container rather than a second host unit, for the same reason every
# sibling swarm service is one — and because `services.opentelemetry-collector`
# is a singleton NixOS option, already spoken for on the host by the hive
# tier. A container gets its own evaluation and therefore its own collector.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.services.hyperhive.swarm.otel;
  swarmCfg = config.services.hyperhive.swarm;
  otelCfg = config.services.hyperhive.otel;
  vmCfg = config.services.hyperhive.swarm.victoriametrics;
  hyperhiveCfg = config.services.hyperhive;
  gatewayCfg = hyperhiveCfg.gateway;
  swarmDomain = hyperhiveCfg.swarm.domain;

  # Total on a null swarm domain for the same reason every sibling module is:
  # the required-domain assertion in hive-network.nix should be what an
  # operator sees, not a coercion error from here.
  domainBase = if swarmDomain == null then "invalid" else swarmDomain;
in
{
  options.services.hyperhive.swarm.otel = {
    enable = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = ''
        Run the swarm's telemetry collector on this host.

        Asserted from `swarm.enableRequiredServices` in
        ./swarm-required-services.nix, with the metrics pair this
        collector feeds: a swarm has one of these, and it belongs
        wherever the shared services live rather than on every hive.

        A hive that does not run it still runs its own hive-tier collector
        (`services.hyperhive.otel.enable`) and reaches this one by name, at
        {option}`services.hyperhive.swarm.otel.domain`.
      '';
    };

    machine = lib.mkOption {
      type = lib.types.str;
      readOnly = true;
      default = "swarm-otel";
      description = ''
        Name of the nixos-container this collector runs in — also the
        `machinectl` name, so other modules may read it rather than
        repeating the literal.
      '';
    };

    port = lib.mkOption {
      type = lib.types.port;
      default = 4319;
      description = ''
        Port this collector's OTLP/HTTP receiver listens on.

        ⚠️ **Deliberately not 4318**, the OTLP/HTTP default, because the
        hive tier already uses it (`services.hyperhive.otel.collector.port`)
        and every swarm container shares the host's network namespace. Two
        listeners claiming one port on one host is not a build failure —
        it is a runtime coin toss over which one gets it, with nothing in
        any log saying so. The same collision cost a release when grafana
        and the forge both defaulted to 3000.
      '';
    };

    telemetryPort = lib.mkOption {
      type = lib.types.port;
      default = 8889;
      description = ''
        Port this collector serves its **own** metrics on — queue depth,
        refused and dropped samples, exporter failures. How you find out
        that telemetry is being lost, so it is worth keeping rather than
        switching off.

        ⚠️ **Deliberately not 8889's neighbour 8888**, which is the
        collector's built-in default and therefore what the hive tier
        already binds. Two collectors share a network namespace whenever
        they are co-located, and unlike the OTLP port this one appears
        nowhere in either config — it is a default inside the binary, so
        nothing that compares configured ports can see the clash. The
        second collector to start simply dies with
        `bind: address already in use`.
      '';
    };

    domain = lib.mkOption {
      type = lib.types.str;
      default = "otel.${domainBase}";
      defaultText = lib.literalExpression ''"otel.''${services.hyperhive.swarm.domain}"'';
      description = ''
        Name the gateway serves this on. A sibling of the swarm's other
        service names, so the swarm-services sub-CA can issue for it — see
        `hive-tls.nix` for why a service name being a sibling rather than a
        child decides which CA may sign it.

        This is what the **hive** tier's exporter reaches — the hive
        collector is a plain producer against this name exactly like every
        other client of a swarm service, resolved locally by dnsmasq on a
        co-located host and over the real network otherwise. There is no
        separate loopback-vs-remote knob to get wrong: `swarm-nats` is the
        deliberate exception to this pattern (its cross-hive reach is the
        wireguard mesh, not the gateway), everything else in this swarm
        addresses its siblings by name.
      '';
    };
  };

  config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
    # The gateway name, inside `cfg.enable` — that guard is the load-bearing
    # part. Every hive in a swarm may know this collector exists, but only
    # the host that RUNS it may claim the name; a client hive declaring the
    # vhost would answer for a service it does not have.
    services.hyperhive.gateway.localNames = [ cfg.domain ];

    # OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as
    # every sibling swarm service: TLS terminates here, then plain http to
    # the co-located container over loopback (shared netns, like the store
    # this collector writes to).
    services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // {
      listen = gatewayCfg.lib.listen;
      extraConfig = gatewayCfg.lib.securityHeaders;
      locations."/" = {
        proxyPass = "http://127.0.0.1:${toString cfg.port}";
      };
    };

    assertions = [
      {
        # The tier exists to hold the upstream credential and to write the
        # swarm's store. With neither, it is a process that receives
        # samples and drops them — which looks healthy and loses data.
        assertion = otelCfg.endpoint != "" || vmCfg.enable;
        message = ''
          services.hyperhive.swarm.otel.enable is true but this collector
          has nowhere to send what it receives:
          services.hyperhive.otel.endpoint is empty and
          services.hyperhive.swarm.victoriametrics.enable is false.

          Set the endpoint to export upstream, or enable the swarm's
          metrics store.
        '';
      }
    ];

    containers.${cfg.machine} = {
      autoStart = true;
      ephemeral = false;
      # Shared host netns, like every sibling swarm service: the hive tier
      # reaches this collector, and this collector reaches the metrics
      # store, without either crossing a network boundary that would need
      # its own trust material.
      privateNetwork = false;

      # The upstream credential is operator-provided and lives on the host.
      # Read-only, and only when one is configured — binding a path that
      # does not exist makes nixos-container refuse to start the container,
      # which is a stall several layers from its cause.
      bindMounts = lib.optionalAttrs (otelCfg.headersCredential != null) {
        ${otelCfg.headersCredential} = {
          hostPath = otelCfg.headersCredential;
          isReadOnly = true;
        };
      };

      config =
        { ... }:
        {
          # This tier is the one that resolves an operator-configured
          # hostname: `otel.endpoint` is an external URL, and reaching it
          # is the entire reason this container holds a credential. The
          # `/etc/resolv.conf` nixos-containers copies in is a snapshot
          # taken once at boot, so without this the upstream export
          # depends on the host's file having been right at that instant.
          imports = [
            (import ./swarm-container-resolver.nix {
              inherit (config.services.hyperhive.network) bridgeIp;
              dnsConsumers = [ "opentelemetry-collector.service" ];
            })
          ];

          system.stateVersion = config.system.stateVersion;
          networking.firewall.enable = false;
          # Keep the host-copied /etc/resolv.conf intact — same reasoning
          # as the sibling swarm containers.
          networking.resolvconf.enable = lib.mkForce false;

          services.opentelemetry-collector = {
            enable = true;
            package = pkgs.opentelemetry-collector-contrib;
            # Runs `otelcol validate` at build time. ⚠️ A parser, not a
            # wiring check: it accepts a receiver naming an absent
            # extension, and the collector then dies at startup. A green
            # build does not prove this config starts, never mind that a
            # sample arrives — which is why this module's gate pushes a
            # real sample through both tiers into the store.
            validateConfigFile = true;
            settings = {
              receivers.otlp.protocols.http.endpoint = "127.0.0.1:${toString cfg.port}";

              exporters =
                lib.optionalAttrs vmCfg.enable {
                  # `metrics_endpoint`, NOT `endpoint`: the latter is a
                  # base that otlphttp appends `/v1/metrics` to, while
                  # VictoriaMetrics serves OTLP at
                  # `/opentelemetry/api/v1/push`. With `endpoint` the
                  # collector answers 200 to its own clients and posts the
                  # samples to a path that does not exist. Measured
                  # end-to-end, not read — `state/probe-3265-collector-to-vm.sh`.
                  "otlphttp/victoriametrics".metrics_endpoint =
                    "http://127.0.0.1:${toString vmCfg.port}/opentelemetry/api/v1/push";
                }
                // lib.optionalAttrs (otelCfg.endpoint != "") {
                  ${if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"} = {
                    endpoint = otelCfg.endpoint;
                  }
                  // lib.optionalAttrs (otelCfg.headersCredential != null) {
                    # Interpolated by the collector from its environment at
                    # runtime, never by nix: `EnvironmentFile` below is what
                    # puts it there, so the value is not read into the store.
                    headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}";
                  }
                  // lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; };
                };

              # Moves this collector's self-metrics off the built-in
              # default of `localhost:8888`, which the hive tier holds.
              #
              # ⚠️ `metrics.address` is the spelling that looks right and
              # is REJECTED by this collector version — measured, not
              # read: `'migration.MetricsConfigV030' has invalid keys:
              # address`. `readers` is the schema it accepts, and the
              # difference is a startup failure rather than a warning.
              service.telemetry.metrics.readers = [
                {
                  pull.exporter.prometheus = {
                    host = "127.0.0.1";
                    port = cfg.telemetryPort;
                  };
                }
              ];

              service.pipelines.metrics = {
                receivers = [ "otlp" ];
                # Fan-out, not a choice: with both configured the same
                # samples go upstream AND into the swarm's store. The store
                # is for looking at this swarm; the upstream is for whoever
                # aggregates across swarms, and neither replaces the other.
                exporters =
                  lib.optional (otelCfg.endpoint != "") (if otelCfg.protocol == "grpc" then "otlp" else "otlphttp")
                  ++ lib.optional vmCfg.enable "otlphttp/victoriametrics";
              };
            };
          };

          # The credential file is already `NAME=value`, systemd's
          # EnvironmentFile format — so the secret reaches the process as an
          # environment variable without being read by nix, written to the
          # store, or passed in argv.
          systemd.services.opentelemetry-collector.serviceConfig =
            lib.optionalAttrs (otelCfg.headersCredential != null)
              {
                EnvironmentFile = otelCfg.headersCredential;
              };
        };
    };
  };
}