hyperhive/nix/module-eval/hive-otel.nix

# `checks.module-eval-hive-otel` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
  pkgs,
  lib,
  self,
  nixosSystem,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    runGroup
    otelSettings
    ;

  # This hive's own collector, which is a HOST service — unlike the swarm
  # tier's, which lives in a container and is read through `otelSettings`.
  hiveOtel = hive {
    otel.enable = true;
    otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
  };

  hiveOtelPipelines = hiveOtel.services.opentelemetry-collector.settings.service.pipelines;

  hiveOtelSettings = hiveOtel.services.opentelemetry-collector.settings;

  # The hive tier's rendered scrape list. Same reasoning as `scrapeJob` for
  # the swarm tier — the option is one string, what prometheus is handed is a
  # job — but this collector is a host service, so the path to it differs.
  hiveScrapeJob =
    job:
    lib.findFirst (
      c: c.job_name == job
    ) null hiveOtelSettings.receivers.prometheus.config.scrape_configs;
  # Duplicated from bao-otel-collector.nix — a case here needs it too.

  # The store with and without a collector on the same host. `scrapeTargets`
  # is only ever read by a local collector, so the metrics endpoint is a
  # function of the pairing rather than of the store.
  baoWithCollector = hive {
    deploy.bao.enable = true;
    deploy.swarm-otel.enable = true;
    # A second job declared as bare `host:port`, so the pair of cases below
    # reads one rendered scrape list: the store's entry carries a path, this
    # one carries none.
    swarm.otel.scrapeTargets.plain = "127.0.0.1:9999";
  };

  # Also duplicated from bao-otel-collector.nix, where it is the fixture for
  # the store standing beside a HIVE collector. That co-location is exactly
  # the topology the port case below needs: two collectors, one netns, and
  # nothing but their self-telemetry ports keeping them apart.
  # `clientSecretFile` is what ../host-modules/otel.nix's identity assertion
  # demands of any hive with the tier on.
  baoWithHiveOtel = hive {
    deploy.bao.enable = true;
    otel.enable = true;
    otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
  };

  # The collector INSIDE the store's container — neither the host's nor the
  # swarm tier's.
  baoForwarderSettings =
    machine: machine.containers.swarm-bao.config.services.opentelemetry-collector.settings;
  cases = [
    {
      # The tier in the middle. Its OTLP receiver takes both signals on one
      # port, so without this pipeline an agent's push is answered 404 on
      # `/v1/logs` — and a forwarder retrying into a 404 is indistinguish-
      # able from one with nothing to send. Compared against the metrics
      # pipeline's exporters rather than a name, so the two signals cannot
      # drift to different destinations.
      name = "the hive collector forwards logs upstream, not only metrics";
      ok =
        (hiveOtelPipelines ? logs)
        && hiveOtelPipelines.metrics.exporters != [ ]
        && hiveOtelPipelines.logs.receivers == [ "otlp" ]
        && hiveOtelPipelines.logs.exporters == hiveOtelPipelines.metrics.exporters;
    }
    {
      # `deltatocumulative` is metrics-only: naming it in a logs pipeline
      # kills the collector at startup rather than doing nothing. The second
      # clause is the control — the metrics pipeline still names it, so a
      # pass means the two processor lists differ rather than that the
      # processor left the module.
      name = "the hive collector keeps the metrics-only processor out of its logs pipeline";
      ok =
        !(builtins.elem "deltatocumulative" hiveOtelPipelines.logs.processors)
        && builtins.elem "deltatocumulative" hiveOtelPipelines.metrics.processors;
    }
    {
      # The counters that say telemetry is being LOST — refused, failed,
      # queue depth — are served on loopback and reach no store unless
      # something reads them. Read off the rendered job rather than the
      # option: only the job is what prometheus actually requests.
      name = "the hive collector scrapes its own telemetry endpoint";
      ok =
        let
          j = hiveScrapeJob "collector";
        in
        j != null && j.static_configs == [ { targets = [ "127.0.0.1:8888" ]; } ];
    }
    {
      # A `prometheus` receiver no pipeline names collects nothing while
      # rendering and starting perfectly, so the scrape above is inert
      # without this. The path is newly reachable: until the hive tier had a
      # target of its own, this receiver was never emitted on any hive.
      name = "the hive metrics pipeline names the prometheus receiver the self-scrape needs";
      ok = builtins.elem "prometheus" hiveOtelPipelines.metrics.receivers;
    }
    {
      # `metrics.address` is the spelling that looks right and is rejected by
      # this collector version. The first clause is the control: without it a
      # missing telemetry block would pass the port check vacuously.
      name = "the hive collector binds its telemetry port through readers, not address";
      ok =
        let
          m = hiveOtelSettings.service.telemetry.metrics;
        in
        !(m ? address) && (lib.head m.readers).pull.exporter.prometheus.port == 8888;
    }
    {
      # A bind that loses the race with the outgoing process is the whole
      # failure: nixpkgs ships `Restart = "always"` with no `RestartSec`, so
      # without this the unit spends its five default attempts inside two
      # seconds and lands in `start-limit-hit`, where it stops retrying. The
      # third clause is the one that has to hold — `StartLimit*` are `[Unit]`
      # settings that systemd ignores under `[Service]`, so a bound written
      # into `serviceConfig` renders, deploys and does nothing. Asserted
      # where nixpkgs puts it rather than where it was written, same as
      # ./bao-grants.nix.
      name = "the hive collector backs off a failed bind from [Unit], not [Service]";
      ok =
        let
          u = hiveOtel.systemd.services.opentelemetry-collector;
        in
        u.serviceConfig.RestartSec or 0 > 0
        && toString u.unitConfig.StartLimitBurst == "12"
        && !(u.serviceConfig ? StartLimitBurst)
        # The window has to outlast every attempt, or the burst is unreachable.
        && u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
    }
    {
      # Collectors share a network namespace whenever they are co-located,
      # and this port appears in no config the port-collision assertion can
      # read — so equal defaults mean the second to start dies at `bind()`.
      # Pinned as a case rather than an assertion: enforcing it belongs with
      # the other port checks, not here.
      #
      # THREE tiers, because the store's container grew a collector of its
      # own and this case could not see it: an UNDECLARED port is not an
      # absent one, it is the binary's 8888, which is the hive tier's. That
      # is why `portOf` falls back to 8888 rather than throwing — a tier that
      # declares nothing has to fail this case loudly, not evaluate away.
      name = "no two collector tiers claim the same self-telemetry port";
      ok =
        let
          portOf =
            s:
            (lib.head (s.service.telemetry.metrics.readers or [ { } ])).pull.exporter.prometheus.port or 8888;
          hivePort = portOf hiveOtelSettings;
          swarmPort = portOf (otelSettings baoWithCollector);
          baoPort = portOf (baoForwarderSettings baoWithHiveOtel);
        in
        hivePort != swarmPort && hivePort != baoPort && swarmPort != baoPort;
    }
  ];
in
runGroup "hive-otel" cases