hyperhive/nix/module-eval/bao-otel-collector.nix

# `checks.module-eval-bao-otel-collector` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
  pkgs,
  lib,
  self,
  nixosSystem,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    runGroup
    baoSettings
    ;

  baoTwoAddresses = hive {
    deploy.bao.enable = true;
    deploy.bao.extraListenAddresses = [ "10.0.0.1" ];
    # Pinned, not incidental: the case counting these listeners is about the
    # declared addresses, and a collector on this host would add one of its own.
    deploy.swarm-otel.enable = false;
  };

  # The store with and without a collector on the same host. `scrapeTargets`
  # is only ever read by a local collector, so the metrics endpoint is a
  # function of the pairing rather than of the store.
  baoWithCollector = hive {
    deploy.bao.enable = true;
    deploy.swarm-otel.enable = true;
    # A second job declared as bare `host:port`, so the pair of cases below
    # reads one rendered scrape list: the store's entry carries a path, this
    # one carries none.
    swarm.otel.scrapeTargets.plain = "127.0.0.1:9999";
  };

  baoNoCollector = hive {
    deploy.bao.enable = true;
    deploy.swarm-otel.enable = false;
  };

  # The store on a host whose HIVE collector is running — the only state in
  # which the container forwards its own journal, since that collector is the
  # hop it forwards to. `clientSecretFile` is what ../host-modules/otel.nix's
  # identity assertion demands of any hive with the tier on. `swarm-otel` is
  # deliberately left off: the forwarder is a function of the first hop
  # existing, and pairing the two fixtures would make a case that passed on the
  # wrong condition.
  baoWithHiveOtel = hive {
    deploy.bao.enable = true;
    otel.enable = true;
    otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
  };

  # The forwarder INSIDE the store's container, not the host's collector and
  # not the swarm tier's — three collectors in this tree, and only this one can
  # see the store's journal.
  baoForwarder = machine: machine.containers.swarm-bao.config.services.opentelemetry-collector;

  # The scrape list prometheus is handed, not the option a service declared:
  # the address, the path and the query are one string on the way in and three
  # fields on the way out, and only the second shape is what gets requested.
  scrapeJob =
    machine: job:
    lib.findFirst (c: c.job_name == job) null
      machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs;
  cases = [
    {
      # Same gap one tier up, and it needs its own arm: this collector
      # already had six scrape targets, so a pass here is about the seventh
      # rather than about the receiver existing at all.
      name = "the swarm collector scrapes its own telemetry endpoint";
      ok =
        let
          j = scrapeJob baoWithCollector "collector";
        in
        j != null && j.static_configs == [ { targets = [ "127.0.0.1:8889" ]; } ];
    }
    {
      # The store stays behind the passthrough rather than beside it: loopback
      # plus whatever was declared, never the bridge. A store that also bound
      # the bridge itself would collide with the listener above, and the
      # colliding one is nginx — the whole gateway, not just this port.
      name = "the store binds loopback and its declared addresses, never the bridge";
      ok =
        let
          l = (baoSettings baoTwoAddresses).listener;
        in
        l.loopback.address == "127.0.0.1:8200" && l.extra-1.address == "10.0.0.1:8200";
    }
    {
      # Control for the case above: these settings are rendered per deployment,
      # not constants a passing case could be indifferent to.
      name = "a declared extra address renders a second listener beside loopback";
      ok = builtins.length (builtins.attrNames (baoSettings baoTwoAddresses).listener) == 2;
    }
    {
      # Retention is what serves the endpoint at all, so the listener alone
      # would be a port that answers 404.
      name = "a store beside a collector serves metrics on its own listener";
      ok =
        let
          s = baoSettings baoWithCollector;
        in
        s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s";
    }
    {
      # openbao serves no `/metrics` at all, so a scrape of the default path
      # 404s: the store looks like a dead exporter, and every panel built on
      # it renders empty rather than erroring.
      name = "the store's scrape asks for the path openbao serves";
      ok =
        let
          j = scrapeJob baoWithCollector "bao";
        in
        (j.metrics_path or "") == "/v1/sys/metrics" && (j.params.format or [ ]) == [ "prometheus" ];
    }
    {
      # Presence control for the case above: both fields are omitted rather
      # than defaulted, so a target declared as bare `host:port` renders what
      # it rendered before the path grammar existed.
      name = "a target with no path renders neither metrics_path nor params";
      ok =
        let
          j = scrapeJob baoWithCollector "plain";
        in
        j != null && !(j ? metrics_path) && !(j ? params);
    }
    {
      # Absence arm. Unauthenticated by design, so it must not exist where
      # nothing reads it.
      name = "a store with no collector beside it serves no metrics";
      ok =
        let
          s = baoSettings baoNoCollector;
        in
        !(s.listener ? metrics) && !(s ? telemetry);
    }
    {
      # The store's journal reaches a reader through a collector of its own,
      # and every one of these fields is silent when wrong: the runtime journal
      # is the receiver's own default and is empty here, an unlisted extension
      # is inert so the cursor silently stops persisting, and a pipeline is
      # free to name none of it.
      name = "the store's container forwards its own journal";
      ok =
        let
          s = (baoForwarder baoWithHiveOtel).settings;
          p = s.service.pipelines.logs;
        in
        s.receivers.journald.directory == "/var/log/journal"
        && s.receivers.journald.storage == "file_storage"
        && s.service.extensions == [ "file_storage" ]
        && p.receivers == [ "journald" ]
        && p.exporters == [ "otlphttp" ]
        && s ? exporters.otlphttp;
    }
    {
      # The whole journal, which is what the shared collector's unit allowlist
      # is not. A `units` list here would render and deploy perfectly while
      # shipping only the units someone remembered to name — the failure this
      # forwarder exists to end.
      name = "the store's forwarder filters no units";
      ok = !((baoForwarder baoWithHiveOtel).settings.receivers.journald ? units);
    }
    {
      # Both ends of the first hop, on ONE host, because a mismatch between
      # them is silent in both directions: the exporter retries into nothing
      # and the receiver never hears from it.
      name = "the store's forwarder exports to this hive's own collector";
      ok =
        (baoForwarder baoWithHiveOtel).settings.exporters.otlphttp.endpoint
        == "http://${baoWithHiveOtel.services.opentelemetry-collector.settings.receivers.otlp.protocols.http.endpoint}";
    }
    {
      # Absence arm — `baoNoCollector` runs neither tier, so there is no first
      # hop on this host at all. A forwarder rendered anyway would start, find
      # nothing listening, and retry forever while reporting healthy.
      name = "a store with no hive collector renders no forwarder";
      ok = !(baoForwarder baoNoCollector).enable;
    }
  ];
in
runGroup "bao-otel-collector" cases