hyperhive/nix/module-eval/swarm-otel-core.nix

# `checks.module-eval-swarm-otel-core` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
  pkgs,
  lib,
  self,
  nixosSystem,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    carriesJournaldSeverity
    runGroup
    otelSettings
    ;

  # A swarm collector on a host that runs NEITHER store — the fully-spread
  # shape from docs/swarm/services.md, and the one the old per-host gates made
  # inexpressible. It is the whole point of the cases below that this hive is
  # not a degenerate configuration but a supported one.
  otelNoStores = hive {
    deploy.swarm-otel.enable = true;
    deploy.authelia.enable = true;
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
    deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
    deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
    deploy.victoriametrics.enable = false;
    deploy.victorialogs.enable = false;
  };

  # The collector beside authelia, reading its own OIDC secret out of the
  # store like every other collector — the cert pair here is not scenery, it
  # is the arm that would catch the deleted co-located copy unit coming back.
  otelBaoWithAuthelia = hive {
    deploy.swarm-otel.enable = true;
    deploy.authelia.enable = true;
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
    deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
    deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
  };

  # The same collector with the IdP on ANOTHER host and a store leaf placed by
  # hand. Identical to the fixture above in everything the delivery path
  # reads, which is the point.
  otelBaoRemoteAuthelia = hive {
    deploy.swarm-otel.enable = true;
    swarm.authelia.url = "https://auth.example.invalid";
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
    deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
    deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
  };

  # The collector beside the store holding a bootstrap token: the one shape in
  # which every unit an apply can leave failed renders on the same host.
  otelApplyPath = hive {
    deploy.swarm-otel.enable = true;
    deploy.authelia.enable = true;
    deploy.bao.enable = true;
    deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
  };

  # Units on the path a deploy takes to TLS, the store's grants and the
  # collector itself. A failure among them silences ingest, so without their
  # journals the store can show that ingest stopped but not which unit
  # stopped it.
  applyPathUnits = [
    "container@swarm-otel"
    "hive-tls-ca"
    "swarm-services-cert"
    "hive-gateway-self-signed-cert"
    "swarm-bao-controller-policy"
    "swarm-bao-secret-publisher-policy"
    "swarm-bao-matrix-ctl-policy"
    "swarm-bao-matrix-token-policy"
    "swarm-bao-queue-agent-policy"
    "swarm-bao-grafana-oidc-policy"
    "swarm-bao-otel-oidc-policy"
    "swarm-bao-services-issuer-policy"
  ];
  cases = [
    {
      # Listed AND defined, because a name that matches nothing is not an
      # error anywhere: a unit renamed out from under its entry would pass a
      # membership check and still never reach the store.
      name = "every unit on the apply path is defined and ships its journal";
      ok =
        let
          m = otelApplyPath;
        in
        lib.all (
          u: builtins.elem u m.services.hyperhive.swarm.otel.journaldUnits && m.systemd.services ? ${u}
        ) applyPathUnits;
    }
    {
      # Transient, so nothing here defines it and only membership can be
      # pinned: nixos-rebuild names the unit it runs the activation in.
      name = "the activation's journal ships beside the units it starts";
      ok = builtins.elem "nixos-rebuild-switch-to-configuration" otelApplyPath.services.hyperhive.swarm.otel.journaldUnits;
    }
    {
      # 🩸 The arm that guards the ruling this slice landed under, the
      # collector's half of ./swarm-grafana.nix's own. There is ONE delivery
      # route: the store reader, on every host that runs the collector and
      # holds a store identity. The negative names the deleted unit rather
      # than a generic absence, because the way this regresses is someone
      # re-adding the co-located copy as an optimisation.
      name = "the collector's OIDC secret has exactly one delivery unit, the store reader, in both topologies";
      ok =
        let
          local = otelBaoWithAuthelia.systemd.services;
          remote = otelBaoRemoteAuthelia.systemd.services;
        in
        local ? swarm-bao-otel-oidc
        && remote ? swarm-bao-otel-oidc
        && !(local ? swarm-otel-oidc-secret)
        && !(remote ? swarm-otel-oidc-secret);
    }
    {
      # Same 403-not-a-miss reason as grafana's arm above: the reader's grant
      # covers the `services` prefix, so a path outside it is refused rather
      # than empty, however correct it reads.
      name = "the collector's OIDC secret is read from the prefix the publisher writes";
      ok =
        let
          s = otelBaoRemoteAuthelia.systemd.services.swarm-bao-otel-oidc.script;
        in
        lib.hasInfix "secret/swarm/services/swarm-collector/oidc/client" s
        && !(lib.hasInfix "secret/swarm/hives/" s);
    }
    {
      # The defect itself. These exporters used to be gated on the stores'
      # PER-HOST enables, so a collector that did not share a host with them
      # rendered none at all and dropped everything it received, from every
      # hive — silently, because an absent exporter is not an error.
      name = "a collector that hosts neither store still exports to both";
      ok =
        let
          e = (otelSettings otelNoStores).exporters;
        in
        (e ? "otlphttp/victoriametrics") && (e ? "otlphttp/victorialogs");
    }
    {
      # A swarm has one of each store, so the address is a swarm-level name.
      # A loopback literal here is the co-location assumption written back in,
      # and it renders, deploys and reports healthy while reaching nothing.
      name = "the store exporters address the stores by name, never by loopback";
      ok =
        let
          e = (otelSettings otelNoStores).exporters;
          m = e."otlphttp/victoriametrics".metrics_endpoint;
          l = e."otlphttp/victorialogs".logs_endpoint;
        in
        !(lib.hasInfix "127.0.0.1" m)
        && !(lib.hasInfix "127.0.0.1" l)
        && lib.hasInfix "metrics.t.local" m
        && lib.hasInfix "logs.t.local" l;
    }
    {
      # `_HOSTNAME` cannot separate machines on its own: a hostname is a
      # config value two of them can share, and then every stream for a unit
      # name merges into one.
      name = "the log stream is keyed by machine, not only by a hostname every container shares";
      ok =
        let
          l = (otelSettings otelNoStores).exporters."otlphttp/victorialogs".logs_endpoint;
          field = f: lib.hasInfix ("_stream_fields=" + f) l || lib.hasInfix ("," + f) l;
        in
        field "_MACHINE_ID" && field "_SYSTEMD_UNIT" && !(field "_NOSUCHFIELD");
    }
    {
      # Defining an exporter and REFERENCING it are two separate lists, and
      # the second is where the original gate also lived. An exporter no
      # pipeline names is as silent as one that does not exist — this case
      # exists because a mutation that restored only the reference-side gate
      # left every other case here green.
      name = "every pipeline that has a store exporter defined actually sends to it";
      ok =
        let
          s = otelSettings otelNoStores;
          used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues s.service.pipelines));
        in
        builtins.elem "otlphttp/victoriametrics" used && builtins.elem "otlphttp/victorialogs" used;
    }
    {
      # An authenticator an exporter names but `service.extensions` omits is
      # INERT — the collector starts clean and pushes unauthenticated until
      # something at the far end refuses it. Checked as a set relation rather
      # than by naming the two, so it keeps holding for exporters not written
      # yet.
      name = "every exporter authenticator is listed in service.extensions";
      ok =
        let
          s = otelSettings otelNoStores;
          named = lib.filter (v: v != null) (
            lib.mapAttrsToList (_: e: e.auth.authenticator or null) s.exporters
          );
        in
        named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named;
    }
    {
      # The host-journal sibling of ./agent-otel.nix's wiring case, which
      # carries the full reasoning. Same question, different receiver: this
      # one reads the HOST's journal rather than a container's, and the two
      # are unrelated config — a fixed stanza there, a parameterised block
      # inside `containers.swarm-otel` here — so one losing its parser while
      # the other keeps one is a real and silent state.
      #
      # Contents are not this case's business. The table both receivers import
      # is asserted once, in ./journald-severity.nix.
      name = "the swarm collector's journald receiver carries the shared PRIORITY mapping";
      ok = carriesJournaldSeverity (otelSettings otelNoStores).receivers.journald;
    }
  ];
in
runGroup "swarm-otel-core" cases