hyperhive/nix/host-modules/swarm-otel.nix

# The swarm's telemetry collector: one per swarm, in a `swarm-otel`
# nixos-container beside the swarm's other shared services.
#
# Two tiers, and they are separate on purpose:
#
#   - `otel.nix` is the **hive** tier. It receives from this hive's agents
#     on the bridge and forwards, and it holds no upstream credential.
#   - this is the **swarm** tier. It is the only holder of the upstream
#     credential, the only writer to the swarm's metrics store, and (once
#     #3283 lands) the place that stamps `hive=` from the authenticated
#     connection rather than from anything a sender can choose.
#
# On a host that runs both, both processes run. They are not collapsed:
# all-local is a statement about *where* processes run, not about what
# shape the deployment has, and a local tier boundary that disappears is
# one the local deployment stops testing. `hive=` attribution is the
# property that would differ, and it is the one #3283 depends on.
#
# A container rather than a second host unit, for the same reason every
# sibling swarm service is one — and because `services.opentelemetry-collector`
# is a singleton NixOS option, already spoken for on the host by the hive
# tier. A container gets its own evaluation and therefore its own collector.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.services.hyperhive.swarm.otel;
  swarmCfg = config.services.hyperhive.swarm;
  otelCfg = config.services.hyperhive.otel;
  vmCfg = config.services.hyperhive.swarm.victoriametrics;
in
{
  options.services.hyperhive.swarm.otel = {
    enable = lib.mkOption {
      type = lib.types.bool;
      default = swarmCfg.enableRequiredServices;
      defaultText = lib.literalExpression "services.hyperhive.swarm.enableRequiredServices";
      description = ''
        Run the swarm's telemetry collector on this host.

        Derived from `swarm.enableRequiredServices` like the swarm's other
        shared services: a swarm has one of these, and it belongs wherever
        the shared services live rather than on every hive.

        A hive that does not run it still runs its own hive-tier collector
        (`services.hyperhive.otel.enable`) and points it here with
        {option}`services.hyperhive.swarm.otel.url`.
      '';
    };

    machine = lib.mkOption {
      type = lib.types.str;
      default = "swarm-otel";
      description = ''
        Name of the nixos-container this collector runs in — also the
        `machinectl` name, so other modules may read it rather than
        repeating the literal.
      '';
    };

    port = lib.mkOption {
      type = lib.types.port;
      default = 4319;
      description = ''
        Port this collector's OTLP/HTTP receiver listens on.

        ⚠️ **Deliberately not 4318**, the OTLP/HTTP default, because the
        hive tier already uses it (`services.hyperhive.otel.collector.port`)
        and every swarm container shares the host's network namespace. Two
        listeners claiming one port on one host is not a build failure —
        it is a runtime coin toss over which one gets it, with nothing in
        any log saying so. The same collision cost a release when grafana
        and the forge both defaulted to 3000.
      '';
    };

    url = lib.mkOption {
      type = lib.types.str;
      default = "http://127.0.0.1:${toString cfg.port}";
      defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString config.services.hyperhive.swarm.otel.port}"'';
      description = ''
        Where the **hive** tier sends what it receives — this collector's
        OTLP/HTTP base URL.

        The default addresses it on loopback, which is correct while the
        two tiers share a host: every swarm container runs in the host's
        network namespace, so a swarm service is reachable there exactly
        as the metrics store already is.

        ⚠️ That default is a *default*, not an assumption baked into the
        exporter. A hive whose swarm collector runs elsewhere sets this to
        that host's address, and nothing else changes — a loopback literal
        written directly into the exporter would have made the split-host
        case a code change instead of a config one.
      '';
    };
  };

  config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
    assertions = [
      {
        # The tier exists to hold the upstream credential and to write the
        # swarm's store. With neither, it is a process that receives
        # samples and drops them — which looks healthy and loses data.
        assertion = otelCfg.endpoint != "" || vmCfg.enable;
        message = ''
          services.hyperhive.swarm.otel.enable is true but this collector
          has nowhere to send what it receives:
          services.hyperhive.otel.endpoint is empty and
          services.hyperhive.swarm.victoriametrics.enable is false.

          Set the endpoint to export upstream, or enable the swarm's
          metrics store.
        '';
      }
    ];

    containers.${cfg.machine} = {
      autoStart = true;
      ephemeral = false;
      # Shared host netns, like every sibling swarm service: the hive tier
      # reaches this collector, and this collector reaches the metrics
      # store, without either crossing a network boundary that would need
      # its own trust material.
      privateNetwork = false;

      # The upstream credential is operator-provided and lives on the host.
      # Read-only, and only when one is configured — binding a path that
      # does not exist makes nixos-container refuse to start the container,
      # which is a stall several layers from its cause.
      bindMounts = lib.optionalAttrs (otelCfg.headersCredential != null) {
        ${otelCfg.headersCredential} = {
          hostPath = otelCfg.headersCredential;
          isReadOnly = true;
        };
      };

      config =
        { ... }:
        {
          system.stateVersion = config.system.stateVersion;
          networking.firewall.enable = false;
          # Keep the host-copied /etc/resolv.conf intact — same reasoning
          # as the sibling swarm containers.
          networking.resolvconf.enable = lib.mkForce false;

          services.opentelemetry-collector = {
            enable = true;
            package = pkgs.opentelemetry-collector-contrib;
            # Runs `otelcol validate` at build time. ⚠️ A parser, not a
            # wiring check: it accepts a receiver naming an absent
            # extension, and the collector then dies at startup. A green
            # build does not prove this config starts, never mind that a
            # sample arrives — which is why this module's gate pushes a
            # real sample through both tiers into the store.
            validateConfigFile = true;
            settings = {
              receivers.otlp.protocols.http.endpoint = "127.0.0.1:${toString cfg.port}";

              exporters =
                lib.optionalAttrs vmCfg.enable {
                  # `metrics_endpoint`, NOT `endpoint`: the latter is a
                  # base that otlphttp appends `/v1/metrics` to, while
                  # VictoriaMetrics serves OTLP at
                  # `/opentelemetry/api/v1/push`. With `endpoint` the
                  # collector answers 200 to its own clients and posts the
                  # samples to a path that does not exist. Measured
                  # end-to-end, not read — `state/probe-3265-collector-to-vm.sh`.
                  "otlphttp/victoriametrics".metrics_endpoint =
                    "http://127.0.0.1:${toString vmCfg.port}/opentelemetry/api/v1/push";
                }
                // lib.optionalAttrs (otelCfg.endpoint != "") {
                  ${if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"} = {
                    endpoint = otelCfg.endpoint;
                  }
                  // lib.optionalAttrs (otelCfg.headersCredential != null) {
                    # Interpolated by the collector from its environment at
                    # runtime, never by nix: `EnvironmentFile` below is what
                    # puts it there, so the value is not read into the store.
                    headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}";
                  }
                  // lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; };
                };

              service.pipelines.metrics = {
                receivers = [ "otlp" ];
                # Fan-out, not a choice: with both configured the same
                # samples go upstream AND into the swarm's store. The store
                # is for looking at this swarm; the upstream is for whoever
                # aggregates across swarms, and neither replaces the other.
                exporters =
                  lib.optional (otelCfg.endpoint != "") (if otelCfg.protocol == "grpc" then "otlp" else "otlphttp")
                  ++ lib.optional vmCfg.enable "otlphttp/victoriametrics";
              };
            };
          };

          # The credential file is already `NAME=value`, systemd's
          # EnvironmentFile format — so the secret reaches the process as an
          # environment variable without being read by nix, written to the
          # store, or passed in argv.
          systemd.services.opentelemetry-collector.serviceConfig =
            lib.optionalAttrs (otelCfg.headersCredential != null)
              {
                EnvironmentFile = otelCfg.headersCredential;
              };
        };
    };
  };
}