hyperhive/nix/module-eval/swarm-otel-identity.nix

# `checks.module-eval-swarm-otel-identity` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
  pkgs,
  lib,
  self,
  nixosSystem,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    runGroup
    otelSettings
    ;

  # A collector holding no store identity at all. Unlike Grafana's mirror
  # image, this is not a refused shape: the collector still receives every
  # hive's telemetry with nothing to push authenticated with, which is the
  # already-supported degrade `haveCollectorSecret` names above the module's
  # `let`. What this fixture is for is checking the reading unit itself does
  # not render, rather than rendering with an env var nothing filled in.
  otelNoIdentity = hive {
    deploy.swarm-otel.enable = true;
    swarm.authelia.url = "https://auth.example.invalid";
    deploy.forgejo.sso.clientSecretFile = "/var/lib/forgejo-oidc/by-hand.secret";
  };

  # 🩸 The shape the degrade above must not swallow: a collector on a host that
  # demonstrably READS the store — it holds the hive's own leaf — and is missing
  # only the collector's own pair. Before the four-way split this host rendered
  # the reading unit off `clientCertFile` alone, so the silence it gets now is a
  # regression rather than the supported degrade `otelNoIdentity` stands for. The
  # two fixtures differ in exactly that one pair, which is what lets the arms
  # below separate a refusal from a degrade.
  otelReaderMissingLeaf = hive {
    deploy.swarm-otel.enable = true;
    swarm.authelia.url = "https://auth.example.invalid";
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
  };

  # The same deployment named in full, which must stay buildable. Pairs with the
  # fixture above so the refusal is pinned to the omission and not to reading a
  # remote store at all.
  otelReaderNamedInFull = hive {
    deploy.swarm-otel.enable = true;
    swarm.authelia.url = "https://auth.example.invalid";
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
    deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
    deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
  };

  # Did the module refuse this host, and over which option. An assertion is a
  # config VALUE until something forces it — `.config` never throws — so a
  # fixture in a refused state stays evaluable and the refusal reads back as
  # data. Matched on the option name the message names rather than on its prose:
  # the option name is the part an operator has to act on, and a message that
  # stopped naming it would be the actual defect.
  refusedOver = m: option: lib.any (a: !a.assertion && lib.hasInfix option a.message) m.assertions;

  # authelia somewhere else, the credential delivered by hand. Whether this
  # collector authenticates must follow the credential, never another
  # service's placement.
  #
  # The `swarm.otel.clientSecretFile` below is the PRE-RENAME path. It predates
  # the split and is deliberately left spelled that way: it makes this fixture
  # the old-path case for that option too, so dropping its rename entry fails
  # the eval here rather than only in a real operator's config.
  otelRemoteAuthelia = hive {
    deploy.swarm-otel.enable = true;
    deploy.authelia.enable = false;
    # Where that elsewhere IS. Running no IdP does not mean knowing no IdP:
    # the authenticator this fixture exists to render puts this address in its
    # `token_url`, so a hive with a secret and no URL has a credential it can
    # present nowhere.
    swarm.authelia.url = "https://auth.example.invalid";
    swarm.otel.clientSecretFile = "/var/lib/swarm-otel-oidc/by-hand.secret";
  };

  # A collector whose ONLY scrape work is published: loopback targets forced
  # empty, one published job declared. Unreachable in a real deploy today —
  # the module seeds `scrapeTargets.collector` under its own `enable`, so the
  # loopback set is never empty on its own — which is exactly why the arm
  # below needs a fixture that takes that seeding away. `mkForce` is what
  # does it, and it leaves the collector itself enabled: the state under test
  # is a running collector with no self-scrape, not an absent one.
  otelOnlyPublished = hive {
    deploy.swarm-otel.enable = true;
    swarm.otel.scrapeTargets = lib.mkForce { };
    swarm.otel.publishedScrapeTargets.remote = "https://remote.t.local/metrics";
  };

  # Two hives in the roster, which no other fixture here has: every one of
  # them declares `swarm.hives.h1` alone, so a per-hive arm written against
  # one of those passes on a hardcoded literal.
  otelTwoHives = hive {
    deploy.swarm-otel.enable = true;
    deploy.authelia.enable = true;
    swarm.hives.h2.domain = "h2.t.local";
  };
  cases = [
    {
      # The collector's non-assertion, the deliberate mirror of Grafana's
      # assertion two cases up: a host with no store identity is a supported,
      # merely degraded shape here, so the reading unit simply does not exist
      # rather than refusing the build. `haveCollectorSecret` is what the
      # degrade already reads, unchanged by this slice.
      name = "a collector with no store identity renders no reading unit, and is not refused";
      ok =
        !(otelNoIdentity.systemd.services ? swarm-bao-otel-oidc)
        && otelNoIdentity.services.hyperhive.deploy.swarm-otel.clientSecretFile == null
        && !(lib.any (a: !a.assertion) otelNoIdentity.assertions);
    }
    {
      # The collector's half of the same split, and a different arm from the
      # authenticator case below: this one reads the PATH the unit loads, so a
      # reader left on a source that is non-null but wrong still fails. The
      # fixture spells the option its pre-rename way, so it covers the rename
      # entry at the same time.
      name = "a config written against the pre-rename otel secret path still loads it as a credential";
      ok =
        lib.any (c: lib.hasInfix "/var/lib/swarm-otel-oidc/by-hand.secret" c)
          otelRemoteAuthelia.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential;
    }
    {
      # The collector authenticates because it HOLDS a credential, not because
      # authelia happens to share its host. Gating on the other service's
      # placement renders a collector that pushes unauthenticated wherever
      # authelia lives elsewhere — one of the supported shapes.
      name = "a collector with a hand-delivered secret authenticates without authelia beside it";
      ok =
        let
          s = otelSettings otelRemoteAuthelia;
        in
        (s.exporters."otlphttp/victoriametrics" ? auth)
        && builtins.elem "oauth2client/victoriametrics" s.service.extensions;
    }
    {
      # Defining a receiver and attaching it are two separate lists, and the
      # two gates were spelled differently: the receiver appeared for either
      # scrape option, the pipeline only for the loopback one. A published-
      # only collector therefore rendered scrape configs that reached no
      # pipeline — requested, parsed, delivered nowhere, and valid enough to
      # deploy. The receiver clause is what stops the arm passing for the
      # wrong reason, by an empty `prometheus` never rendering at all.
      name = "a published-only collector attaches its prometheus receiver to the swarm pipeline";
      ok =
        let
          s = otelSettings otelOnlyPublished;
        in
        otelOnlyPublished.services.hyperhive.swarm.otel.scrapeTargets == { }
        && otelOnlyPublished.services.hyperhive.swarm.otel.publishedScrapeTargets != { }
        && (s.receivers ? prometheus)
        && builtins.elem "prometheus" s.service.pipelines."metrics/swarm".receivers;
    }
    {
      # Read against the roster the fixture declares rather than against
      # names spelled here: an arm naming `h1` passes on a single-hive
      # config however the mapping is written. The length clause is what
      # makes the `all` mean anything — over an empty roster it holds
      # vacuously.
      name = "the swarm collector routes every hive's logs, not just one";
      ok =
        let
          p = (otelSettings otelTwoHives).service.pipelines;
          hives = lib.attrNames otelTwoHives.services.hyperhive.swarm.hives;
        in
        lib.length hives == 2
        && lib.all (h: (p ? "logs/${h}") && p."logs/${h}".receivers == [ "otlp/${h}" ]) hives;
    }
    {
      # The same split as the metrics case above — defining an exporter and
      # naming it are two lists — plus the half one shared list cannot have:
      # the metrics store's exporter renders perfectly well inside a logs
      # pipeline and posts journal records at an ingest route that is not
      # for them.
      name = "every logs pipeline sends to the log store and to no metrics one";
      ok =
        let
          s = otelSettings otelTwoHives;
          logPipes = lib.filterAttrs (n: _: lib.hasPrefix "logs/" n) s.service.pipelines;
          used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues logPipes));
        in
        logPipes != { }
        && builtins.elem "otlphttp/victorialogs" used
        && !(builtins.elem "otlphttp/victoriametrics" used)
        && lib.all (e: s.exporters ? ${e}) used;
    }
    {
      # 🩸 The refusal, and the only way this module's omission is visible at
      # all: the reading unit is gated on its own leaf, so the regression is a
      # green build with the unit absent — and the missing unit is the evidence.
      # Read as a refusal naming both options, so an operator acts on the
      # message without opening the nix.
      name = "a store reader missing the collector's own leaf is refused, naming both options";
      ok =
        refusedOver otelReaderMissingLeaf "otelOidcClientCertFile"
        && refusedOver otelReaderMissingLeaf "otelOidcClientKeyFile";
    }
    {
      # ⚠️ The arm that keeps the refusal from eating the degrade beside it. A
      # collector on a host with NO store identity still receives every hive's
      # telemetry and is a supported deployment — `otelNoIdentity` is that
      # fixture, and it differs from the refused one only in the hive's own
      # leaf. Widening the gate to the collector's leaf alone would reject it,
      # which is what this pins.
      name = "a collector with no store identity at all is not refused over the collector's leaf";
      ok =
        !(refusedOver otelNoIdentity "otelOidcClientCertFile")
        && !(refusedOver otelNoIdentity "otelOidcClientKeyFile");
    }
    {
      # The other two shapes that must stay buildable: the same remote reader
      # named in full, and a host that mints its own PKI and therefore has all
      # eight as defaults from ./host-modules/glue-bao-tls.nix.
      name = "neither a fully-named remote reader nor a store host is refused";
      ok =
        !(refusedOver otelReaderNamedInFull "otelOidcClientCertFile")
        && !(refusedOver otelRemoteAuthelia "otelOidcClientCertFile");
    }
  ];
in
runGroup "swarm-otel-identity" cases