# `checks.module-eval-hive-otel` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
  pkgs,
  lib,
  self,
  nixosSystem,
  shared,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    runGroup
    otelSettings
    ;

  # This hive's own collector, which is a HOST service — unlike the swarm
  # tier's, which lives in a container and is read through `otelSettings`.
  # The IdP is here too, with a hand-named secret file: the collector's wait
  # for that file exists beside authelia's container.
  hiveOtel = hive {
    otel.enable = true;
    otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
    deploy.authelia.enable = true;
  };

  # A hive holding its own store leaf, placed by hand, and naming no secret
  # file: the collector's secret comes from the store under that leaf.
  hiveOtelLeaf = hive {
    otel.enable = true;
    deploy.bao.clientCertFile = "/etc/pki/hive.pem";
    deploy.bao.clientKeyFile = "/etc/pki/hive-key.pem";
  };

  # The IdP, the store and the collector on one host: the publisher that
  # writes the secret and the unit that reads it, in one config.
  hiveOtelAllLocal = hive {
    otel.enable = true;
    deploy.bao.enable = true;
    deploy.authelia.enable = true;
  };

  hiveOtelFetch = machine: machine.systemd.services.swarm-bao-hive-otel-oidc;

  hiveOtelPipelines = hiveOtel.services.opentelemetry-collector.settings.service.pipelines;

  hiveOtelSettings = hiveOtel.services.opentelemetry-collector.settings;

  # The hive tier's rendered scrape list. Same reasoning as `scrapeJob` for
  # the swarm tier — the option is one string, what prometheus is handed is a
  # job — but this collector is a host service, so the path to it differs.
  hiveScrapeJob =
    job:
    lib.findFirst (
      c: c.job_name == job
    ) null hiveOtelSettings.receivers.prometheus.config.scrape_configs;

  # The store with and without a collector on the same host. `scrapeTargets`
  # is only ever read by a local collector, so the metrics endpoint is a
  # function of the pairing rather than of the store.
  inherit (shared) baoWithCollector;

  # The store standing beside a HIVE collector: the topology the port case
  # below needs, two collectors in one netns with nothing but their
  # self-telemetry ports keeping them apart.
  inherit (shared) baoWithHiveOtel;

  # The collector INSIDE the store's container — neither the host's nor the
  # swarm tier's.
  baoForwarderSettings =
    machine: machine.containers.swarm-bao.config.services.opentelemetry-collector.settings;
  cases = [
    {
      # The tier in the middle. Its OTLP receiver takes both signals on one
      # port, so without this pipeline an agent's push is answered 404 on
      # `/v1/logs` — and a forwarder retrying into a 404 is indistinguish-
      # able from one with nothing to send. `journald` joined the receiver
      # list for this collector's own host journal (hive-c0re's crash-watch
      # log line, see ./otel.nix) — both have to be present, in either
      # order, so this stays a set check rather than a list-equality one.
      # Compared against the metrics pipeline's exporters rather than a
      # name, so the two signals cannot drift to different destinations.
      name = "the hive collector forwards logs upstream, not only metrics";
      ok =
        (hiveOtelPipelines ? logs)
        && hiveOtelPipelines.metrics.exporters != [ ]
        &&
          lib.sort builtins.lessThan hiveOtelPipelines.logs.receivers == [
            "journald"
            "otlp"
          ]
        && hiveOtelPipelines.logs.exporters == hiveOtelPipelines.metrics.exporters;
    }
    {
      # `deltatocumulative` is metrics-only: naming it in a logs pipeline
      # kills the collector at startup rather than doing nothing. The second
      # clause is the control — the metrics pipeline still names it, so a
      # pass means the two processor lists differ rather than that the
      # processor left the module.
      name = "the hive collector keeps the metrics-only processor out of its logs pipeline";
      ok =
        !(builtins.elem "deltatocumulative" hiveOtelPipelines.logs.processors)
        && builtins.elem "deltatocumulative" hiveOtelPipelines.metrics.processors;
    }
    {
      # `otelcol validate` refuses a `service.extensions` entry with no
      # definition, and that runs when the unit is built, not in this suite.
      # `hiveOtel` authenticates upstream, so both extensions are listed;
      # the length clause keeps the check from passing on an empty list.
      name = "the hive collector defines every extension its service lists";
      ok =
        lib.length hiveOtelSettings.service.extensions == 2
        && lib.all (e: hiveOtelSettings.extensions ? ${e}) hiveOtelSettings.service.extensions;
    }
    {
      # The counters that say telemetry is being LOST — refused, failed,
      # queue depth — are served on loopback and reach no store unless
      # something reads them. Read off the rendered job rather than the
      # option: only the job is what prometheus actually requests.
      name = "the hive collector scrapes its own telemetry endpoint";
      ok =
        let
          j = hiveScrapeJob "collector";
        in
        j != null && j.static_configs == [ { targets = [ "127.0.0.1:8888" ]; } ];
    }
    {
      # A `prometheus` receiver no pipeline names collects nothing while
      # rendering and starting perfectly, so the scrape above is inert
      # without this. The path is newly reachable: until the hive tier had a
      # target of its own, this receiver was never emitted on any hive.
      name = "the hive metrics pipeline names the prometheus receiver the self-scrape needs";
      ok = builtins.elem "prometheus" hiveOtelPipelines.metrics.receivers;
    }
    {
      # The receiver, the pipeline entry and the nginx server it reads have to
      # agree on one address, and none of them may exist without a gateway on
      # the host. `hiveOtelLeaf` is the control: it runs no gateway, and the
      # clause asserting that keeps the control from going vacuous.
      name = "the hive collector reads the gateway's stub_status, and only where a gateway runs";
      ok =
        let
          nginxConf = hiveOtel.services.nginx.appendHttpConfig;
          leafSettings = hiveOtelLeaf.services.opentelemetry-collector.settings;
        in
        hiveOtel.services.hyperhive.gateway.enable
        && hiveOtelSettings.receivers.nginx.endpoint == "http://127.0.0.1:8085/stub_status"
        && builtins.elem "nginx" hiveOtelPipelines.metrics.receivers
        && lib.hasInfix "listen 127.0.0.1:8085;" nginxConf
        && lib.hasInfix "location = /stub_status {" nginxConf
        && !hiveOtelLeaf.services.hyperhive.gateway.enable
        && !(leafSettings.receivers ? nginx)
        && !(builtins.elem "nginx" leafSettings.service.pipelines.metrics.receivers);
    }
    {
      # `metrics.address` is the spelling that looks right and is rejected by
      # this collector version. The first clause is the control: without it a
      # missing telemetry block would pass the port check vacuously.
      name = "the hive collector binds its telemetry port through readers, not address";
      ok =
        let
          m = hiveOtelSettings.service.telemetry.metrics;
        in
        !(m ? address) && (lib.head m.readers).pull.exporter.prometheus.port == 8888;
    }
    {
      # A bind that loses the race with the outgoing process is the whole
      # failure: nixpkgs ships `Restart = "always"` with no `RestartSec`, so
      # without this the unit spends its five default attempts inside two
      # seconds and lands in `start-limit-hit`, where it stops retrying. The
      # third clause is the one that has to hold — `StartLimit*` are `[Unit]`
      # settings that systemd ignores under `[Service]`, so a bound written
      # into `serviceConfig` renders, deploys and does nothing. Asserted
      # where nixpkgs puts it rather than where it was written, same as
      # ./bao-grants.nix.
      name = "the hive collector backs off a failed bind from [Unit], not [Service]";
      ok =
        let
          u = hiveOtel.systemd.services.opentelemetry-collector;
        in
        u.serviceConfig.RestartSec or 0 > 0
        && toString u.unitConfig.StartLimitBurst == "12"
        && !(u.serviceConfig ? StartLimitBurst)
        # The window has to outlast every attempt, or the burst is unreachable.
        && u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
    }
    {
      # Collectors share a network namespace whenever they are co-located,
      # and this port appears in no config the port-collision assertion can
      # read — so equal defaults mean the second to start dies at `bind()`.
      # Pinned as a case rather than an assertion: enforcing it belongs with
      # the other port checks, not here.
      #
      # THREE tiers, because the store's container grew a collector of its
      # own and this case could not see it: an UNDECLARED port is not an
      # absent one, it is the binary's 8888, which is the hive tier's. That
      # is why `portOf` falls back to 8888 rather than throwing — a tier that
      # declares nothing has to fail this case loudly, not evaluate away.
      name = "no two collector tiers claim the same self-telemetry port";
      ok =
        let
          portOf =
            s:
            (lib.head (s.service.telemetry.metrics.readers or [ { } ])).pull.exporter.prometheus.port or 8888;
          hivePort = portOf hiveOtelSettings;
          swarmPort = portOf (otelSettings baoWithCollector);
          baoPort = portOf (baoForwarderSettings baoWithHiveOtel);
        in
        hivePort != swarmPort && hivePort != baoPort && swarmPort != baoPort;
    }
    {
      # The collector authenticates as the hive, so it reads its secret as
      # the hive: the leaf `deploy.bao.clientCertFile` names, whose policy
      # grants `swarm/hives/<hive>/*`. The last two clauses tie the unit's
      # output to what the collector loads.
      name = "the hive collector's secret is fetched from the store under the hive's own leaf";
      ok =
        let
          u = hiveOtelFetch hiveOtelLeaf;
          b = hiveOtelLeaf.services.hyperhive.deploy.bao;
          file = hiveOtelLeaf.services.hyperhive.otel.clientSecretFile;
          s = builtins.unsafeDiscardStringContext u.script;
        in
        hiveOtelLeaf.systemd.services ? swarm-bao-hive-otel-oidc
        && u.environment.BAO_CLIENT_CERT == b.clientCertFile
        && u.environment.BAO_CLIENT_KEY == b.clientKeyFile
        && lib.hasInfix "bao login -method=cert" s
        && lib.hasInfix "bao kv get -field=value secret/swarm/hives/h1/oidc/client" s
        && lib.hasInfix "atomic_write_secret 0400 \"\" ${file} " s
        && builtins.elem "swarm-client.secret:${file}" hiveOtelLeaf.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
        && builtins.elem "opentelemetry-collector.service" u.wantedBy
        && builtins.elem "opentelemetry-collector.service" u.before;
    }
    {
      # `LoadCredential` copies the file at the collector's start, so a value
      # landed later reaches it only through a restart. Retries every 30s
      # (./lib/store-retry.nix) while the store is down or the secret
      # unpublished.
      name = "the hive collector's fetch retries and restarts the collector when the file is newer";
      ok =
        let
          u = hiveOtelFetch hiveOtelLeaf;
          s = builtins.unsafeDiscardStringContext u.script;
        in
        u.serviceConfig.Restart == "on-failure"
        && u.serviceConfig.RestartSec == 30
        && toString u.unitConfig.StartLimitBurst == "2880"
        && lib.hasInfix "refresh_unit opentelemetry-collector.service /var/lib/hive-otel-oidc/hive-h1.secret" s
        && lib.hasInfix "ActiveEnterTimestamp" s
        && lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' s;
    }
    {
      # On a first boot the file does not exist until the fetch succeeds, so
      # the collector fails `LoadCredential` into its start limit, i.e. the
      # `failed` state. Only `reset-failed` clears that limit, so the fetch
      # has to reset before it starts; `try-restart` acts on no failed unit.
      # A never-active collector has an empty `ActiveEnterTimestamp`, read as
      # 0, so the file always counts as newer. The collector only `wants` the
      # fetch, so an unreachable store does not hold it back.
      name = "the hive collector's fetch revives a collector that hit its start limit without the file";
      ok =
        let
          s = builtins.unsafeDiscardStringContext (hiveOtelFetch hiveOtelLeaf).script;
          c = hiveOtelLeaf.systemd.services.opentelemetry-collector;
          failedArm = lib.last (lib.splitString ''is-failed --quiet "$unit"; then'' s);
        in
        lib.hasInfix ''is-failed --quiet "$unit"; then'' s
        && lib.hasInfix ''reset-failed "$unit"'' failedArm
        && lib.hasInfix ''start --no-block "$unit"'' (
          lib.last (lib.splitString ''reset-failed "$unit"'' failedArm)
        )
        && lib.hasInfix "started_us=0" s
        && !(builtins.elem "swarm-bao-hive-otel-oidc.service" (c.requires or [ ]));
    }
    {
      # The store is the one route, on the IdP's own host too: no default into
      # authelia's tree and no unit waiting for a file to appear there.
      name = "the hive collector reads no secret out of authelia's tree and waits on no copied file";
      ok =
        let
          file = hiveOtelAllLocal.services.hyperhive.otel.clientSecretFile;
          c = hiveOtelAllLocal.systemd.services.opentelemetry-collector;
        in
        !(hiveOtelAllLocal.systemd.services ? hive-otel-client-secret)
        && !(hiveOtelLeaf.systemd.services ? hive-otel-client-secret)
        && !(lib.hasInfix "authelia" file)
        && hiveOtelAllLocal.systemd.services ? swarm-bao-hive-otel-oidc
        && !(builtins.elem "hive-otel-client-secret.service" (c.requires or [ ]));
    }
    {
      # The writer and the reader spell the path independently; this is the
      # one place that sees both rendered.
      name = "the hive collector reads the path the secret publisher writes";
      ok =
        let
          fetch = builtins.unsafeDiscardStringContext (hiveOtelFetch hiveOtelAllLocal).script;
          publish = builtins.unsafeDiscardStringContext hiveOtelAllLocal.systemd.services.swarm-secret-publish.script;
          path = "secret/swarm/hives/h1/oidc/client";
        in
        lib.hasInfix "bao kv get -field=value ${path}" fetch && lib.hasInfix "bao kv put ${path} " publish;
    }
    {
      # The store's units exist only on its host. The PKI unit is wanted, never
      # required: a refused pki start fails a requirer's job with `dependency`.
      name = "the hive collector's fetch waits for the store's container and PKI unit only where the store runs here";
      ok =
        let
          storeUnits = m: [
            "swarm-bao-pki.service"
            "container@${m.services.hyperhive.swarm.bao.machine}.service"
          ];
          here = hiveOtelFetch hiveOtelAllLocal;
          elsewhere = hiveOtelFetch hiveOtelLeaf;
        in
        lib.all (n: lib.elem n here.after) (storeUnits hiveOtelAllLocal)
        && lib.elem "swarm-bao-pki.service" here.wants
        && !(lib.elem "swarm-bao-pki.service" here.requires)
        && lib.elem "container@${hiveOtelAllLocal.services.hyperhive.swarm.bao.machine}.service" here.wants
        && !(lib.any (n: lib.elem n (elsewhere.after ++ elsewhere.wants ++ elsewhere.requires)) (
          storeUnits hiveOtelLeaf
        ));
    }
    {
      # A secret delivered some other way is the operator's: no unit fetches
      # into a file the collector does not load, or restarts it on one. Nothing
      # restarts the collector when that file lands, so it waits for the file
      # instead of failing `LoadCredential` into its start limit.
      name = "a hand-named collector secret renders no store fetch, and the collector waits for the file";
      ok =
        let
          waitsFor =
            machine:
            let
              c = machine.systemd.services.opentelemetry-collector;
              w = machine.systemd.services.hive-otel-client-secret;
            in
            builtins.elem "hive-otel-client-secret.service" c.requires
            && builtins.elem "hive-otel-client-secret.service" c.after
            && lib.hasInfix "/var/lib/hive-otel-oidc/client.secret" w.script;
        in
        !(hiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
        && !(baoWithHiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
        && builtins.elem "swarm-client.secret:/var/lib/hive-otel-oidc/client.secret" baoWithHiveOtel.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
        && waitsFor hiveOtel
        && waitsFor baoWithHiveOtel;
    }
    {
      # ../host-modules/glue-hive-otel-authelia.nix orders the wait after
      # authelia's container. `hiveOtel` has the IdP; `baoWithHiveOtel` waits
      # on the same kind of file without it, and must not name a container
      # that is not there — `Requires=` on an absent unit fails the job.
      name = "the hive collector's secret wait is ordered after authelia's container only where authelia runs here";
      ok =
        let
          container = "container@${hiveOtel.services.hyperhive.swarm.authelia.machine}.service";
          wait = machine: machine.systemd.services.hive-otel-client-secret;
        in
        builtins.elem container (wait hiveOtel).after
        && builtins.elem container (wait hiveOtel).requires
        && !(builtins.elem container (wait baoWithHiveOtel).after)
        && !(builtins.elem container (wait baoWithHiveOtel).requires);
    }
  ];
in
runGroup "hive-otel" cases