hyperhive/nix/module-eval/bao-otel-collector.nix

# `checks.module-eval-bao-otel-collector` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
  pkgs,
  lib,
  self,
  nixosSystem,
  shared,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    runGroup
    baoSettings
    forceServiceConfigs
    selfScrapes
    ;

  baoTwoAddresses = hive {
    deploy.bao.enable = true;
    deploy.bao.extraListenAddresses = [ "10.0.0.1" ];
    # Pinned, not incidental: the case counting these listeners is about the
    # declared API addresses. ⚠️ Leaving the swarm tier off does not keep the
    # METRICS listener out of the count — that one is a function of the
    # store, not of a collector's placement — so the case below subtracts it
    # by name.
    deploy.swarm-otel.enable = false;
  };

  # The store with and without a collector on the same host. `scrapeTargets`
  # is only ever read by a local collector, so the metrics endpoint is a
  # function of the pairing rather than of the store.
  inherit (shared) baoWithCollector;

  # The SPLIT topology, and the reason it is not called `baoNoCollector` any
  # more: a swarm always runs every swarm-level service somewhere, so this host
  # having `deploy.swarm-otel.enable = false` means the collector is on ANOTHER
  # host, never that the swarm has none. Both of those read the same locally,
  # and only one of them is a real deployment — which is what the two gates
  # this PR removed each mistook for "there is nothing to forward to".
  baoOtelElsewhere = shared.baoOnly;

  # The same split, plus a HIVE collector on the store's host — the pairing the
  # first gate read as permission to forward. It is a third collector with a
  # different job, so it changes neither whether the store forwards nor where.
  inherit (shared) baoWithHiveOtel;

  # The forwarder INSIDE the store's container, not the host's collector and
  # not the swarm tier's — three collectors in this tree, and only this one can
  # see the store's journal.
  baoForwarder = machine: machine.containers.swarm-bao.config.services.opentelemetry-collector;

  # The scrape list prometheus is handed, not the option a service declared:
  # the address, the path and the query are one string on the way in and three
  # fields on the way out, and only the second shape is what gets requested.
  #
  # ⚠️ Two of these now, aimed at two different containers, and the split is
  # the point: the store's own job left the swarm collector for the
  # one inside its container, so a case that looked in the old place would
  # find `null` and a case that only looked in the new one would not notice
  # the store still being declared to both.
  scrapeJobIn =
    container: machine: job:
    lib.findFirst (c: c.job_name == job) null (
      machine.containers.${container}.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs
        or [ ]
    );
  scrapeJob = scrapeJobIn "swarm-otel";
  baoScrapeJob = scrapeJobIn "swarm-bao";
  cases = [
    {
      # Every unit in the container merges without a priority collision, not
      # only the keys the cases here read.
      name = "the store's container units merge without a priority collision";
      ok = forceServiceConfigs baoWithCollector.containers.swarm-bao.config;
    }
    {
      # Same gap one tier up, and it needs its own arm: this collector
      # already had six scrape targets, so a pass here is about the seventh
      # rather than about the receiver existing at all.
      name = "the swarm collector scrapes its own telemetry endpoint";
      ok =
        let
          j = scrapeJob baoWithCollector "collector";
        in
        j != null && j.static_configs == [ { targets = [ "127.0.0.1:8889" ]; } ];
    }
    {
      # The store stays behind the passthrough rather than beside it: loopback
      # plus whatever was declared, never the bridge. A store that also bound
      # the bridge itself would collide with the listener above, and the
      # colliding one is nginx — the whole gateway, not just this port.
      name = "the store binds loopback and its declared addresses, never the bridge";
      ok =
        let
          l = (baoSettings baoTwoAddresses).listener;
        in
        l.loopback.address == "127.0.0.1:8200" && l.extra-1.address == "10.0.0.1:8200";
    }
    {
      # Control for the case above: these settings are rendered per deployment,
      # not constants a passing case could be indifferent to. The metrics and
      # UI listeners are excluded by name — each answers on a loopback port of
      # its own and is not one of the declared addresses this case is counting.
      name = "a declared extra address renders a second listener beside loopback";
      ok =
        builtins.length (
          builtins.filter (
            n:
            !(builtins.elem n [
              "metrics"
              "ui"
            ])
          ) (builtins.attrNames (baoSettings baoTwoAddresses).listener)
        ) == 2;
    }
    {
      # Retention is what serves the endpoint at all, so the listener alone
      # would be a port that answers 404.
      name = "a store beside a collector serves metrics on its own listener";
      ok =
        let
          s = baoSettings baoWithCollector;
        in
        s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s";
    }
    {
      # openbao serves no `/metrics` at all, so a scrape of the default path
      # 404s: the store looks like a dead exporter, and every panel built on
      # it renders empty rather than erroring. The query is the other half —
      # `/v1/sys/metrics` answers JSON without it, which parses as no series
      # rather than as an error.
      name = "the store's scrape asks for the path openbao serves";
      ok =
        let
          j = baoScrapeJob baoWithCollector "bao";
        in
        j != null
        && (j.metrics_path or "") == "/v1/sys/metrics"
        && (j.params.format or [ ]) == [ "prometheus" ];
    }
    {
      # 🎯 Needs BOTH halves: the job is in the store's own container, and it
      # is not declared to the swarm collector. Only the second half can catch
      # a scrape declared in both places — two scrapers of one endpoint,
      # double-counted and neither obviously wrong.
      name = "the store's metrics are scraped by its own collector, not the swarm's";
      ok = baoScrapeJob baoWithCollector "bao" != null && scrapeJob baoWithCollector "bao" == null;
    }
    {
      # The scrape target is the loopback listener, not the API one: dialing
      # the API listener would need a client certificate the collector has no
      # way to present, and the fix for that would be relaxing
      # `tls_require_and_verify_client_cert` — which is the trade this address
      # exists to avoid. Pinned so a later edit has to argue with it.
      name = "the store's collector scrapes loopback, never a wider address";
      ok =
        let
          j = baoScrapeJob baoWithCollector "bao";
          s = baoSettings baoWithCollector;
        in
        j.static_configs == [ { targets = [ "127.0.0.1:8202" ]; } ]
        && s.listener.metrics.address == "127.0.0.1:8202";
    }
    {
      # A pipeline is typed: a prometheus receiver named in `logs` is rejected
      # at startup, and metrics with no pipeline at all are collected and
      # dropped. The exporter is pinned alongside because "no otel bypass"
      # means both signals leave by the one hop.
      name = "the store's collector has a metrics pipeline out the same hop as its logs";
      ok =
        let
          s = (baoForwarder baoWithCollector).settings;
          m = s.service.pipelines.metrics;
        in
        m.receivers == [ "prometheus" ]
        && m.exporters == [ "otlphttp" ]
        && m.exporters == s.service.pipelines.logs.exporters;
    }
    {
      # The store's collector reports its own health through a pipeline of its
      # own: the store pipeline above upserts `service.name`, and sharing it
      # would relabel these series away from the collector dashboard.
      name = "the store's collector scrapes its own telemetry endpoint on loopback";
      ok =
        let
          s = (baoForwarder baoWithCollector).settings;
          port = baoWithCollector.services.hyperhive.deploy.bao.otel.telemetryPort;
        in
        selfScrapes s port { container = "bao"; }
        && s.service.pipelines.metrics.receivers == [ "prometheus" ]
        && s.service.pipelines.metrics.processors == [ "resource" ];
    }
    {
      # Presence control for the case above: both fields are omitted rather
      # than defaulted, so a target declared as bare `host:port` renders what
      # it rendered before the path grammar existed.
      name = "a target with no path renders neither metrics_path nor params";
      ok =
        let
          j = scrapeJob baoWithCollector "plain";
        in
        j != null && !(j ? metrics_path) && !(j ? params);
    }
    {
      # 🩸 The store's metrics endpoint does not depend on where the swarm
      # collector runs. The endpoint is unauthenticated, so it should not exist
      # where nothing reads it; the reader ships inside the container, so every
      # store has one. Gating the endpoint on a collector beside the store
      # would render a store whose metrics go nowhere, indistinguishable from
      # a store nobody had asked to scrape. The endpoint is loopback in a
      # netns whose only other occupants are the store and its collector.
      name = "a store whose swarm collector is elsewhere still serves and scrapes its metrics";
      ok =
        let
          s = baoSettings baoOtelElsewhere;
        in
        s.listener.metrics.address == "127.0.0.1:8202"
        && (s.telemetry.prometheus_retention_time or "0s") != "0s"
        && baoScrapeJob baoOtelElsewhere "bao" != null;
    }
    {
      # The store's journal reaches a reader through a collector of its own,
      # and every one of these fields is silent when wrong: the runtime journal
      # is the receiver's own default and is empty here, an unlisted extension
      # is inert so the cursor silently stops persisting, and a pipeline is
      # free to name none of it.
      name = "the store's container forwards its own journal";
      ok =
        let
          s = (baoForwarder baoWithCollector).settings;
          p = s.service.pipelines.logs;
        in
        s.receivers.journald.directory == "/var/log/journal"
        && s.receivers.journald.storage == "file_storage"
        &&
          s.service.extensions == [
            "file_storage"
            "oauth2client/swarm-bao"
          ]
        && p.receivers == [ "journald" ]
        && p.exporters == [ "otlphttp" ]
        && s ? exporters.otlphttp;
    }
    {
      # 🩸 The receiver read NOTHING before this, on every deploy there has
      # ever been: `journalctl --follow` (the receiver passes `--follow`
      # unconditionally) scopes itself to the current boot unless `--merge` is
      # also given, and under `--link-journal=host` the directory below is the
      # HOST's journal tree, where this container's current boot has no entry.
      # journalctl exited 1 with "No journal boot entry found for the specified
      # boot (+0)" and the receiver restarted it every ~2s forever.
      #
      # Both halves, because each alone is satisfiable by the broken config:
      # `merge` is only a fix while the directory stays the host-linked one,
      # and the directory is only readable while `merge` clears the boot scope.
      name = "the store's forwarder reads the host-linked journal without a boot filter";
      ok =
        let
          j = (baoForwarder baoWithCollector).settings.receivers.journald;
        in
        (j.merge or false) == true && j.directory == "/var/log/journal";
    }
    {
      # The empty-directory trap: `host` mode bind-mounts a host directory
      # journald never actually writes into, so `merge`/`directory` above
      # would keep validating and starting while the receiver reads nothing —
      # silently, same as every other failure mode here. Guards against the
      # flag that links it.
      name = "the store's container does not link its journal onto the host";
      ok = !(lib.elem "--link-journal=host" (baoWithCollector.containers.swarm-bao.extraFlags or [ ]));
    }
    {
      # The whole journal. A `units` list here would render and deploy
      # perfectly while shipping only the units someone remembered to name —
      # the failure this forwarder exists to end.
      name = "the store's forwarder filters no units";
      ok = !((baoForwarder baoWithCollector).settings.receivers.journald ? units);
    }
    {
      # Both ends of the hop, because a mismatch between them is silent in both
      # directions: the exporter retries into a 404 and the receiver never
      # hears from it. Not "on one host" — the far end is the gateway
      # vhost the swarm collector serves its name on, which is the whole point
      # of addressing it by name rather than by a bridge address.
      name = "the store's forwarder exports to the swarm collector's own route";
      ok =
        let
          otel = baoWithCollector.services.hyperhive.swarm.otel;
          vhost = baoWithCollector.services.nginx.virtualHosts.${otel.domain};
        in
        (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint
        == "https://${otel.domain}/${otel.storeProducerName}"
        && vhost.locations ? "/${otel.storeProducerName}/";
    }
    {
      # The case the old hive-tier gate got wrong: this host runs the swarm's
      # collector and no hive collector at all, and the forwarder still exists
      # with an address that resolves. Gated on `otel.collector.enable` it rendered
      # nothing here, so the store's journal left no trace anywhere.
      name = "a store on a host with no hive collector still forwards";
      ok =
        !baoWithCollector.services.hyperhive.otel.collector.enable
        && (baoForwarder baoWithCollector).enable
        && (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint != "";
    }
    {
      # 🎯 The split topology, which is what both removed gates got wrong and
      # neither earlier test could see: the swarm's collector runs on ANOTHER
      # host, so nothing local says it exists — and it exists anyway, because
      # every swarm-level service runs somewhere in the swarm. The forwarder
      # has to render here and has to be addressed by the collector's swarm
      # name, which is reachable from a host that serves no vhost for it.
      #
      # Both halves matter. Without `enable` this is the old bug; without the
      # endpoint it is a forwarder that renders unconditionally and points at
      # nothing. The `!(vhost ? domain)` conjunct is what makes the fixture
      # genuinely remote rather than a co-located host in disguise — the
      # premise the co-located fixtures cannot test.
      name = "a store forwards by name from a host that does not run the swarm collector";
      ok =
        let
          otel = baoOtelElsewhere.services.hyperhive.swarm.otel;
          route = "https://${otel.domain}/${otel.storeProducerName}";
        in
        !(baoOtelElsewhere.services.nginx.virtualHosts ? ${otel.domain})
        && (baoForwarder baoOtelElsewhere).enable
        && (baoForwarder baoOtelElsewhere).settings.exporters.otlphttp.endpoint == route
        # …and a hive collector standing beside the store neither supplies the
        # reason to forward nor changes the addressee: same route, same tier.
        && (baoForwarder baoWithHiveOtel).enable
        && (baoForwarder baoWithHiveOtel).settings.exporters.otlphttp.endpoint == route;
    }
    {
      # 🎯 The credential half of the hop, and the arm that says it
      # DELIVERS rather than merely resolves. The receiver at the far end
      # takes an OIDC token and nothing else, so an exporter with no
      # authenticator is a forwarder that retries into 401s while every other
      # case here passes. Both halves: the extension has to exist AND the
      # exporter has to name it.
      name = "the store's forwarder presents a credential on the hop it exports over";
      ok =
        let
          s = (baoForwarder baoWithCollector).settings;
          auth = "oauth2client/swarm-bao";
        in
        s.exporters.otlphttp.auth.authenticator == auth
        && s.extensions ? ${auth}
        && lib.elem auth s.service.extensions;
    }
    {
      # Its OWN client, never the swarm collector's and never the
      # controller's: one identity per principal, and this one runs on a host
      # neither of those two does. The audience is the same id because the
      # receiver checks exactly that — a token minted without ASKING for it
      # carries `aud: []` and is refused with a config that reads correctly at
      # both ends.
      name = "the forwarder authenticates as a principal of its own";
      ok =
        let
          bao = baoWithCollector.services.hyperhive.swarm.bao;
          e = (baoForwarder baoWithCollector).settings.extensions."oauth2client/swarm-bao";
        in
        e.client_id == bao.otel.clientId
        && e.endpoint_params.audience == bao.otel.clientId
        && bao.otel.clientId != baoWithCollector.services.hyperhive.swarm.otel.clientId
        && bao.otel.clientId != baoWithCollector.services.hyperhive.swarm.controller.queueClientId;
    }
    {
      # A path systemd resolves at runtime, never a value and never a store
      # path: the collector runs under `DynamicUser` and opens this file
      # itself, so the credential arrives through `LoadCredential` and the
      # config names only the directory systemd exports.
      name = "the forwarder's client secret reaches it as a credential, not a value";
      ok =
        let
          container = baoWithCollector.containers.swarm-bao.config;
          e = (baoForwarder baoWithCollector).settings.extensions."oauth2client/swarm-bao";
          unit = container.systemd.services.opentelemetry-collector.serviceConfig;
        in
        e.client_secret_file == "\${env:CREDENTIALS_DIRECTORY}/oidc-client-secret"
        &&
          unit.LoadCredential == [
            "oidc-client-secret:/run/swarm-bao-forwarder-oidc/swarm-bao-collector.secret"
          ];
    }
    {
      # A copy on persistent disk outlives the secret's rotation and lands in
      # backups of the container's root. The file is deliberately not created
      # ahead of the fetch: a placeholder would start the forwarder and have
      # its pushes refused instead of leaving it down until the store answers.
      name = "the forwarder's client secret is written to the host's /run only, and the old copy removed";
      ok =
        let
          m = baoWithCollector;
          script = m.systemd.services.swarm-bao-forwarder-oidc.script;
          mount = m.containers.swarm-bao.bindMounts."/run/swarm-bao-forwarder-oidc";
          rules = m.systemd.tmpfiles.rules;
        in
        lib.hasInfix "atomic_write_secret 0400 root:root /run/swarm-bao-forwarder-oidc/swarm-bao-collector.secret" script
        && !(lib.hasInfix "/var/lib/nixos-containers" script)
        && !(lib.hasInfix "/var/lib/swarm-bao-otel-oidc" script)
        && mount.hostPath == "/run/swarm-bao-forwarder-oidc"
        && mount.isReadOnly
        && lib.elem "d /run/swarm-bao-forwarder-oidc 0700 root root -" rules
        && lib.elem "r /var/lib/nixos-containers/swarm-bao/var/lib/swarm-bao-otel-oidc/swarm-bao-collector.secret" rules
        && !(lib.any (r: lib.hasPrefix "f /run/swarm-bao-forwarder-oidc" r) rules)
        && lib.elem "systemd-tmpfiles-resetup.service" m.systemd.services."container@swarm-bao".after;
    }
    {
      # Same 403-not-a-miss property the grafana and matrix readers are pinned
      # for: the reader's grant is one path under `services`, so a secret
      # filed under the hive that happens to run the store would be refused
      # rather than missing, however correct the path reads.
      name = "the forwarder's secret is read from the prefix the mint writes";
      ok =
        let
          s = baoWithCollector.systemd.services.swarm-bao-forwarder-oidc.script;
        in
        lib.hasInfix "secret/swarm/services/swarm-bao-collector/oidc/client" s
        && !(lib.hasInfix "secret/swarm/hives/" s);
    }
    {
      # The delivery unit is ordered AFTER the store's container, not before
      # it: the store it reads lives in the container it delivers into, so a
      # `before` edge is a wait on a process that cannot start until this
      # finishes. Pinned because the unit it was copied from does the
      # opposite, and copying that line too would deadlock every cold boot.
      name = "the delivery unit waits for the store rather than blocking it";
      ok =
        let
          u = baoWithCollector.systemd.services.swarm-bao-forwarder-oidc;
        in
        lib.elem "container@swarm-bao.service" u.after
        && !(lib.elem "container@swarm-bao.service" (u.before or [ ]))
        && lib.elem "swarm-bao-pki.service" u.wants
        && !(lib.elem "swarm-bao-pki.service" u.requires);
    }
    {
      # The far end of the credential. An authenticator checks ONE audience,
      # so the store's forwarder gets a receiver of its own rather than a
      # second sender into the route whose audience belongs to
      # `swarm-controller` — and every one of these four is silent when
      # wrong: an unlisted extension is inert, a receiver in no pipeline
      # drops what it accepts, and a mismatched audience is a healthy 401.
      name = "the swarm collector admits this principal on a receiver of its own";
      ok =
        let
          otel = baoWithCollector.services.hyperhive.swarm.otel;
          s = baoWithCollector.containers.swarm-otel.config.services.opentelemetry-collector.settings;
          name = otel.storeProducerName;
        in
        s.receivers."otlp/${name}".protocols.http.auth.authenticator == "oidc/${name}"
        &&
          s.extensions."oidc/${name}".audience == baoWithCollector.services.hyperhive.swarm.bao.otel.clientId
        && lib.elem "oidc/${name}" s.service.extensions
        && lib.elem "otlp/${name}" s.service.pipelines."metrics/${otel.producerName}".receivers
        && lib.elem "otlp/${name}" s.service.pipelines."logs/${otel.producerName}".receivers;
    }
    {
      # A client authelia has never heard of mints nothing, so the publisher
      # has nothing to copy and the delivery unit reads an empty path
      # forever. Registered where AUTHELIA runs, which is why it is a glue
      # module and not a line in the store's own config — this fixture runs
      # no store at all and still has to register it.
      name = "the forwarder's client is registered wherever authelia runs";
      ok =
        let
          inherit (shared) autheliaOnly;
          clients = autheliaOnly.services.hyperhive.swarm.authelia.oidc.clients;
          c = lib.findFirst (c: c.id == "swarm-bao-collector") null clients;
        in
        !autheliaOnly.services.hyperhive.deploy.bao.enable
        && c != null
        && c.audience == [ "swarm-bao-collector" ]
        # ⚠️ What makes the token READABLE by the receiver at all: authelia's
        # default is an opaque handle, and an `oidc` extension verifies
        # offline against the provider's JWKS.
        && c.accessTokenSignedResponseAlg == "RS256";
    }
  ];
in
runGroup "bao-otel-collector" cases