hyperhive/nix/module-eval/bao-otel-collector.nix

# `checks.module-eval-bao-otel-collector` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
  pkgs,
  lib,
  self,
  nixosSystem,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    runGroup
    baoSettings
    ;

  baoTwoAddresses = hive {
    deploy.bao.enable = true;
    deploy.bao.extraListenAddresses = [ "10.0.0.1" ];
    # Pinned, not incidental: the case counting these listeners is about the
    # declared addresses, and a collector on this host would add one of its own.
    deploy.swarm-otel.enable = false;
  };

  # The store with and without a collector on the same host. `scrapeTargets`
  # is only ever read by a local collector, so the metrics endpoint is a
  # function of the pairing rather than of the store.
  baoWithCollector = hive {
    deploy.bao.enable = true;
    deploy.swarm-otel.enable = true;
    # A second job declared as bare `host:port`, so the pair of cases below
    # reads one rendered scrape list: the store's entry carries a path, this
    # one carries none.
    swarm.otel.scrapeTargets.plain = "127.0.0.1:9999";
  };

  baoNoCollector = hive {
    deploy.bao.enable = true;
    deploy.swarm-otel.enable = false;
  };

  # The store on a host whose HIVE collector is running and whose SWARM one is
  # not. The forwarder is a function of the swarm tier, so this fixture pins
  # the negative: a hive collector beside the store buys it nothing.
  # `clientSecretFile` is what ../host-modules/otel.nix's identity assertion
  # demands of any hive with the tier on.
  baoWithHiveOtel = hive {
    deploy.bao.enable = true;
    otel.enable = true;
    otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
  };

  # The forwarder INSIDE the store's container, not the host's collector and
  # not the swarm tier's — three collectors in this tree, and only this one can
  # see the store's journal.
  baoForwarder = machine: machine.containers.swarm-bao.config.services.opentelemetry-collector;

  # The scrape list prometheus is handed, not the option a service declared:
  # the address, the path and the query are one string on the way in and three
  # fields on the way out, and only the second shape is what gets requested.
  scrapeJob =
    machine: job:
    lib.findFirst (c: c.job_name == job) null
      machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs;
  cases = [
    {
      # Same gap one tier up, and it needs its own arm: this collector
      # already had six scrape targets, so a pass here is about the seventh
      # rather than about the receiver existing at all.
      name = "the swarm collector scrapes its own telemetry endpoint";
      ok =
        let
          j = scrapeJob baoWithCollector "collector";
        in
        j != null && j.static_configs == [ { targets = [ "127.0.0.1:8889" ]; } ];
    }
    {
      # The store stays behind the passthrough rather than beside it: loopback
      # plus whatever was declared, never the bridge. A store that also bound
      # the bridge itself would collide with the listener above, and the
      # colliding one is nginx — the whole gateway, not just this port.
      name = "the store binds loopback and its declared addresses, never the bridge";
      ok =
        let
          l = (baoSettings baoTwoAddresses).listener;
        in
        l.loopback.address == "127.0.0.1:8200" && l.extra-1.address == "10.0.0.1:8200";
    }
    {
      # Control for the case above: these settings are rendered per deployment,
      # not constants a passing case could be indifferent to.
      name = "a declared extra address renders a second listener beside loopback";
      ok = builtins.length (builtins.attrNames (baoSettings baoTwoAddresses).listener) == 2;
    }
    {
      # Retention is what serves the endpoint at all, so the listener alone
      # would be a port that answers 404.
      name = "a store beside a collector serves metrics on its own listener";
      ok =
        let
          s = baoSettings baoWithCollector;
        in
        s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s";
    }
    {
      # openbao serves no `/metrics` at all, so a scrape of the default path
      # 404s: the store looks like a dead exporter, and every panel built on
      # it renders empty rather than erroring.
      name = "the store's scrape asks for the path openbao serves";
      ok =
        let
          j = scrapeJob baoWithCollector "bao";
        in
        (j.metrics_path or "") == "/v1/sys/metrics" && (j.params.format or [ ]) == [ "prometheus" ];
    }
    {
      # Presence control for the case above: both fields are omitted rather
      # than defaulted, so a target declared as bare `host:port` renders what
      # it rendered before the path grammar existed.
      name = "a target with no path renders neither metrics_path nor params";
      ok =
        let
          j = scrapeJob baoWithCollector "plain";
        in
        j != null && !(j ? metrics_path) && !(j ? params);
    }
    {
      # Absence arm. Unauthenticated by design, so it must not exist where
      # nothing reads it.
      name = "a store with no collector beside it serves no metrics";
      ok =
        let
          s = baoSettings baoNoCollector;
        in
        !(s.listener ? metrics) && !(s ? telemetry);
    }
    {
      # The store's journal reaches a reader through a collector of its own,
      # and every one of these fields is silent when wrong: the runtime journal
      # is the receiver's own default and is empty here, an unlisted extension
      # is inert so the cursor silently stops persisting, and a pipeline is
      # free to name none of it.
      name = "the store's container forwards its own journal";
      ok =
        let
          s = (baoForwarder baoWithCollector).settings;
          p = s.service.pipelines.logs;
        in
        s.receivers.journald.directory == "/var/log/journal"
        && s.receivers.journald.storage == "file_storage"
        && s.service.extensions == [ "file_storage" ]
        && p.receivers == [ "journald" ]
        && p.exporters == [ "otlphttp" ]
        && s ? exporters.otlphttp;
    }
    {
      # The whole journal, which is what the shared collector's unit allowlist
      # is not. A `units` list here would render and deploy perfectly while
      # shipping only the units someone remembered to name — the failure this
      # forwarder exists to end.
      name = "the store's forwarder filters no units";
      ok = !((baoForwarder baoWithCollector).settings.receivers.journald ? units);
    }
    {
      # Both ends of the hop, because a mismatch between them is silent in both
      # directions: the exporter retries into a 404 and the receiver never
      # hears from it. Not "on one host" any more — the far end is the gateway
      # vhost the swarm collector serves its name on, which is the whole point
      # of addressing it by name rather than by a bridge address.
      name = "the store's forwarder exports to the swarm collector's own route";
      ok =
        let
          otel = baoWithCollector.services.hyperhive.swarm.otel;
          vhost = baoWithCollector.services.nginx.virtualHosts.${otel.domain};
        in
        (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint
        == "https://${otel.domain}/${otel.producerName}"
        && vhost.locations ? "/${otel.producerName}/";
    }
    {
      # The case the old hive-tier gate got wrong: this host runs the swarm's
      # collector and no hive collector at all, and the forwarder still exists
      # with an address that resolves. Gated on `otel.enable` it rendered
      # nothing here, so the store's journal left no trace anywhere.
      name = "a store on a host with no hive collector still forwards";
      ok =
        !baoWithCollector.services.hyperhive.otel.enable
        && (baoForwarder baoWithCollector).enable
        && (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint != "";
    }
    {
      # Absence arm — no swarm collector, so nothing this side of the gateway
      # serves that name. A forwarder rendered anyway would start, find nothing
      # listening, and retry forever while reporting healthy. `baoWithHiveOtel`
      # is the pointed half: a hive collector beside the store is not a reason
      # to forward, which is exactly what the old gate assumed.
      name = "a store with no swarm collector renders no forwarder";
      ok = !(baoForwarder baoNoCollector).enable && !(baoForwarder baoWithHiveOtel).enable;
    }
  ];
in
runGroup "bao-otel-collector" cases