# `checks.module-eval-grafana` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
  pkgs,
  lib,
  self,
  nixosSystem,
  shared,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    runGroup
    ;

  grafanaHere = hive {
    deploy.grafana.enable = true;
    deploy.grafana.datasourceUrl = "http://127.0.0.1:19999";
    deploy.grafana.logsDatasourceUrl = "http://127.0.0.1:19998";
    deploy.grafana.plugins = [ ];
    deploy.grafana.package = pkgs.emptyDirectory;
  };

  # A host that tries to move the socket directory; reading the option throws.
  grafanaSocketDirSet = hive {
    deploy.grafana.enable = true;
    deploy.grafana.socketDir = "/run/test-grafana-sock";
    deploy.grafana.plugins = [ ];
    deploy.grafana.package = pkgs.emptyDirectory;
  };

  # Same pairing `nix/module-eval/bao-otel-collector.nix` uses for its own
  # cases: the store's container with its own collector on, so
  # `.containers.swarm-bao.config` carries a real, evaluated
  # `opentelemetry-collector.settings` rather than one this file would have
  # to hand-assemble.
  baoWithCollector = hive {
    deploy.bao.enable = true;
    deploy.swarm-otel.enable = true;
  };

  # The metrics UI beside the IdP. It reads its secret out of the store like
  # every other Grafana host, so it needs a store identity like every other
  # Grafana host — the cert pair here is not scenery, it is the arm that would
  # have caught the deleted co-located copy unit coming back.
  grafanaWithAuthelia = hive {
    deploy.grafana.enable = true;
    deploy.grafana.plugins = [ ];
    deploy.grafana.package = pkgs.emptyDirectory;
    deploy.authelia.enable = true;
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
    deploy.bao.grafanaOidcClientCertFile = "/etc/pki/bao-grafana-oidc.pem";
    deploy.bao.grafanaOidcClientKeyFile = "/etc/pki/bao-grafana-oidc-key.pem";
  };

  # The same UI with the IdP on ANOTHER host and a store leaf placed by hand.
  # Knowing an IdP is not running one: `swarm.authelia.url` is what says this
  # swarm has SSO, and nothing about this host does. Identical to the fixture
  # above in everything the delivery path reads, which is the point.
  grafanaRemoteAuthelia = hive {
    deploy.grafana.enable = true;
    deploy.grafana.plugins = [ ];
    deploy.grafana.package = pkgs.emptyDirectory;
    swarm.authelia.url = "https://auth.example.invalid";
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
    deploy.bao.grafanaOidcClientCertFile = "/etc/pki/bao-grafana-oidc.pem";
    deploy.bao.grafanaOidcClientKeyFile = "/etc/pki/bao-grafana-oidc-key.pem";
  };

  # A Grafana host holding no store identity. The module refuses it with an
  # assertion naming the two options to set, rather than going QUIET: no OIDC
  # block can render and the container's login form is off regardless, so
  # there would be no way in and nothing would fail. Only the identity is
  # missing, so an arm below can name which refusal fired.
  grafanaNoIdentity = hive {
    deploy.grafana.enable = true;
    deploy.grafana.plugins = [ ];
    deploy.grafana.package = pkgs.emptyDirectory;
    swarm.authelia.url = "https://auth.example.invalid";
    # Declares a vhost that binds its own `listen`, which the gateway refuses.
    gateway.vhosts."own-listen.t.local".listen = [ ];
  };

  # Did ./host-modules/swarm-grafana.nix refuse this host, and for which of its
  # two reasons. An assertion is a config VALUE until something forces it —
  # `.config` never throws — so a fixture in a state the module refuses is
  # evaluable and the refusal is readable as data. That is what lets a case
  # check that a misconfiguration is REPORTED, rather than only that it is not
  # silently accepted.
  #
  # Matched on the option name the message names, not on its prose, so the
  # wording stays rewordable: the option name is the part an operator has to
  # act on, and a message that stopped naming it would be the actual defect.
  grafanaRefusedFor =
    m: option:
    lib.any (
      a:
      !a.assertion
      && lib.hasInfix "services.hyperhive.deploy.grafana.enable requires" a.message
      && lib.hasInfix option a.message
    ) m.assertions;
  cases = [
    {
      # The socket's directory is its access control, so it must stay dedicated
      # to this one socket. `readOnly` is what makes that inexpressible: a
      # definition anywhere — here at the `deploy` path — has to fail the eval,
      # and an unset host has to render the default into nginx's upstream.
      name = "grafana's socket directory cannot be set and renders its default";
      ok =
        !(builtins.tryEval grafanaSocketDirSet.services.hyperhive.deploy.grafana.socketDir).success
        &&
          grafanaHere.services.nginx.virtualHosts.${grafanaHere.services.hyperhive.swarm.grafana.domain}.locations."= /metrics".proxyPass
          == "http://unix:/run/swarm-grafana/grafana.sock:/metrics";
    }
    {
      # Grafana serves `/metrics` unauthenticated, so the published name gates
      # it on the collector's auth subrequest: no `error_page 401 =302` (a
      # scraper would parse a login page as metrics) and no `/__hive_authelia`.
      # `=` is what catches the normalised variants and still leaves the rest
      # of Grafana proxied.
      name = "grafana's published vhost serves /metrics only behind the collector's auth subrequest";
      ok =
        let
          g = grafanaHere.services.hyperhive.swarm.grafana;
          locs = grafanaHere.services.nginx.virtualHosts.${g.domain}.locations;
          checksOf = l: lib.filter (lib.hasPrefix "auth_request") (lib.splitString "\n" l.extraConfig);
          authz = locs."= /__grafana_metrics_authz";
        in
        locs ? "= /metrics"
        && locs."= /metrics".return == null
        && checksOf locs."= /metrics" == [ "auth_request /__grafana_metrics_authz;" ]
        && !(lib.hasInfix "error_page" locs."= /metrics".extraConfig)
        && authz.proxyPass == "https://auth.t.local/api/authz/auth-request"
        && lib.hasInfix "internal;" authz.extraConfig
        && !(grafanaHere.services.nginx.virtualHosts ? "grafana-metrics")
        && lib.hasPrefix "http://unix:" locs."/".proxyPass;
    }
    {
      # This fixture enables grafana and NOT authelia, so the login-form toggle
      # has to follow "grafana requires SSO" rather than "both services are on
      # this host". Its vhost is on the public gateway, so a password box there
      # is a way in whatever the topology.
      name = "grafana disables its local login form even where authelia is not on this host";
      ok = grafanaHere.containers.swarm-grafana.config.services.grafana.settings.auth.disable_login_form;
    }
    {
      # The absence class this whole file is for, and the reported defect in one
      # arm: the OIDC block hung off "authelia is on this host", so the split
      # deployment got a Grafana with no SSO settings and no login form — no way
      # in at all. The block is emitted in every deployment now, so the negative
      # arm is not "no block elsewhere" but "the two do not name the same IdP":
      # each host's block has to point at the URL the SWARM names, and a block
      # built from `deploy.authelia` rather than `swarm.authelia.url` would pass
      # a presence check on both fixtures while sending one of them nowhere.
      name = "grafana's OIDC block names the swarm's IdP, wherever that IdP runs";
      ok =
        let
          oauth = m: m.containers.swarm-grafana.config.services.grafana.settings."auth.generic_oauth";
          remote = oauth grafanaRemoteAuthelia;
          local = oauth grafanaWithAuthelia;
        in
        remote.enabled
        && lib.hasInfix "https://auth.example.invalid/api/oidc/token" remote.token_url
        && local.enabled
        && lib.hasInfix "https://auth.t.local/api/oidc/token" local.token_url
        && !(lib.hasInfix "auth.example.invalid" local.token_url);
    }
    {
      # 🩸 The arm that guards the ruling this slice landed under. There is ONE
      # delivery route: the store reader, on every host that runs Grafana. The
      # negative names the deleted unit rather than a generic absence, because
      # the way this regresses is someone re-adding the co-located copy as an
      # optimisation — a second writer of one path, and a second shape of "the
      # secret is wrong" to debug.
      name = "grafana's OIDC secret has exactly one delivery unit, the store reader, in both topologies";
      ok =
        let
          local = grafanaWithAuthelia.systemd.services;
          remote = grafanaRemoteAuthelia.systemd.services;
        in
        local ? swarm-bao-grafana-oidc
        && remote ? swarm-bao-grafana-oidc
        && !(local ? swarm-grafana-oidc-secret)
        && !(remote ? swarm-grafana-oidc-secret);
    }
    {
      # What the deleted warning became. The shape is unchanged — a Grafana host
      # holding no store leaf — but silence there is a container nobody can log
      # into for a reason no log names, and a warning is read back by nothing.
      # The second arm is what makes this a refusal about the IDENTITY: this
      # fixture names an IdP, so a message about `swarm.authelia.url` here would
      # mean the two assertions had been collapsed into one conjunction.
      name = "a grafana host with no store identity is refused, naming the options to set";
      ok =
        grafanaRefusedFor grafanaNoIdentity "deploy.bao.grafanaOidcClientCertFile"
        && grafanaRefusedFor grafanaNoIdentity "deploy.bao.grafanaOidcClientKeyFile"
        && !(grafanaRefusedFor grafanaNoIdentity "swarm.authelia.url");
    }
    {
      # Without this the arm above proves nothing: a refusal that fires on
      # every host is not a check, and both of these are hosts a swarm is
      # expected to have. Read through the same helper, so a message that
      # stopped naming its option would fail the arms above rather than pass
      # this one by accident.
      name = "neither grafana refusal fires on a correctly configured host, co-located or not";
      ok =
        !(grafanaRefusedFor grafanaWithAuthelia "services.hyperhive.swarm.authelia.url")
        && !(grafanaRefusedFor grafanaWithAuthelia "deploy.bao.grafanaOidcClientCertFile")
        && !(grafanaRefusedFor grafanaRemoteAuthelia "services.hyperhive.swarm.authelia.url")
        && !(grafanaRefusedFor grafanaRemoteAuthelia "deploy.bao.grafanaOidcClientCertFile");
    }
    {
      # Grafana's API takes basic auth whatever the login form says, so the
      # built-in admin's password is the only thing between the public vhost
      # and server-admin. Upstream sets it only when it creates the database,
      # so both halves are pinned: the setting a new database reads, and the
      # reset every start applies to whichever database exists. Without the
      # reset, a database created before the setting keeps `admin`.
      #
      # The reset's `--config` must be the file the server reads, or it resets
      # a different database: compared by store path against the server's own
      # command line.
      name = "grafana's admin password is minted into the store and applied to new and existing databases";
      ok =
        let
          unit = grafanaRemoteAuthelia.systemd.services.swarm-bao-grafana-admin;
          grafana = grafanaRemoteAuthelia.containers.swarm-grafana.config.services.grafana;
          service = grafanaRemoteAuthelia.containers.swarm-grafana.config.systemd.services.grafana;
          resetConfig = lib.head (
            lib.head (lib.filter lib.isList (builtins.split "--config ([^ ]+)" service.preStart))
          );
        in
        lib.hasInfix "bao kv put -cas=0 secret/swarm/services/swarm-grafana/admin/password value=-" unit.script
        && lib.hasInfix "openssl rand -hex 32" unit.script
        && lib.hasInfix "atomic_write_secret 0400 196:0 /run/swarm-grafana-secrets/admin-password" unit.script
        && lib.elem "container@swarm-grafana.service" unit.before
        && unit.environment.BAO_CLIENT_CERT == "/etc/pki/bao-grafana-oidc.pem"
        && grafana.settings.security.admin_password == "$__file{/run/swarm-grafana-secrets/admin-password}"
        && lib.hasInfix "admin reset-admin-password --password-from-stdin < /run/swarm-grafana-secrets/admin-password" service.preStart
        && lib.hasInfix "-config ${resetConfig}" service.script;
    }
    {
      # swarm-controller reads alerts with this token, so the account it
      # belongs to is created and kept a Viewer, the token lands at the path
      # the controller's grant reads, and an attempt against a Grafana or a
      # store that never answers ends and retries. The timer re-runs it, so
      # the unit must not stay active.
      name = "the controller's Grafana token is a Viewer account's, stored under Grafana's leaf, and each attempt is bounded";
      ok =
        let
          unit = grafanaRemoteAuthelia.systemd.services.swarm-bao-grafana-controller-token;
          s = unit.script;
        in
        !(lib.elem (toString (unit.serviceConfig.TimeoutStartSec or "infinity")) [
          "infinity"
          "0"
        ])
        && lib.hasInfix "curl --silent --show-error --connect-timeout 5 --max-time 15" s
        && lib.hasInfix "--data '{\"name\":\"swarm-controller\",\"role\":\"Viewer\",\"isDisabled\":false}'" s
        && lib.hasInfix "--data '{\"role\":\"Viewer\",\"isDisabled\":false}'" s
        && lib.hasInfix "bao kv put secret/swarm/controller/swarm-controller/grafana/token value=-" s
        && lib.hasInfix "--unix-socket /run/swarm-grafana/grafana.sock" s
        && unit.environment.BAO_CLIENT_CERT == "/etc/pki/bao-grafana-oidc.pem"
        && unit.serviceConfig.Restart == "on-failure"
        && !(unit.serviceConfig.RemainAfterExit or false)
        && lib.elem "swarm-bao-grafana-admin.service" unit.after
        && grafanaRemoteAuthelia.systemd.timers ? swarm-bao-grafana-controller-token;
    }
    {
      # Same 403-not-a-miss reason as the matrix and queue arms below: the
      # reader's grant covers the `services` prefix, so a path outside it is
      # refused rather than empty, however correct it reads. The negative arm is
      # the rename this is exposed to — a secret filed under the hive that runs
      # the service instead of under the service itself.
      name = "grafana's OIDC secret is read from the prefix the publisher writes";
      ok =
        let
          s = grafanaRemoteAuthelia.systemd.services.swarm-bao-grafana-oidc.script;
        in
        lib.hasInfix "secret/swarm/services/swarm-grafana/oidc/client" s
        && !(lib.hasInfix "secret/swarm/hives/" s);
    }
    {
      # A copy on persistent disk outlives the secret's rotation and lands in
      # backups of the container's root.
      name = "grafana's OIDC secret, admin password and metrics bearer are written to the host's /run only, and the old copies removed";
      ok =
        let
          m = grafanaRemoteAuthelia;
          services = m.systemd.services;
          mount = m.containers.swarm-grafana.bindMounts."/run/swarm-grafana-secrets";
          rules = m.systemd.tmpfiles.rules;
          guest = m.containers.swarm-grafana.config.services.grafana;
          persistent = s: lib.hasInfix "/var/lib/nixos-containers" s || lib.hasInfix "/var/lib/grafana-" s;
          oidc = services.swarm-bao-grafana-oidc.script;
          admin = services.swarm-bao-grafana-admin.script;
          bearer = services.swarm-bao-grafana-metrics-bearer.script;
          # The bind source must exist when the store refuses the login, so the
          # directory is made ahead of it in every delivery script.
          dirBeforeLogin =
            s:
            lib.hasInfix "install -d -m 0755 /run/swarm-grafana-secrets" (
              lib.head (lib.splitString "bao login -method=cert" s)
            );
        in
        lib.hasInfix "atomic_write_secret 0400 196:0 /run/swarm-grafana-secrets/swarm-grafana.secret" oidc
        && lib.hasInfix "atomic_write_secret 0400 196:0 /run/swarm-grafana-secrets/metrics-bearer" bearer
        && dirBeforeLogin oidc
        && dirBeforeLogin admin
        && dirBeforeLogin bearer
        && !(persistent oidc)
        && !(persistent admin)
        && !(persistent bearer)
        && !(persistent services.swarm-bao-grafana-controller-token.script)
        && lib.hasInfix "< /run/swarm-grafana-secrets/admin-password" services.swarm-bao-grafana-controller-token.script
        && mount.hostPath == "/run/swarm-grafana-secrets"
        && mount.isReadOnly
        &&
          guest.settings."auth.generic_oauth".client_secret
          == "$__file{/run/swarm-grafana-secrets/swarm-grafana.secret}"
        && lib.any (
          d: d.secureJsonData.httpHeaderValue1 or null == "$__file{/run/swarm-grafana-secrets/metrics-bearer}"
        ) guest.provision.datasources.settings.datasources
        && lib.elem "d /run/swarm-grafana-secrets 0755 root root -" rules
        && lib.elem "r /var/lib/nixos-containers/swarm-grafana/var/lib/grafana-oidc/swarm-grafana.secret" rules
        && lib.elem "r /var/lib/nixos-containers/swarm-grafana/var/lib/grafana-admin/password" rules
        && lib.elem "r /var/lib/nixos-containers/swarm-grafana/var/lib/grafana-metrics/bearer" rules
        && !(lib.any (lib.hasPrefix "f /run/swarm-grafana-secrets") rules);
    }
    {
      # Both halves of the co-location assumption, which was one host's
      # `deploy.*` answering a question about the whole swarm: the identities
      # were minted only where the queue happened to run, and the token
      # endpoint was known only where the IdP happened to run.
      name = "hive identities and the token endpoint do not depend on which host runs what";
      ok =
        let
          autheliaNoQueue = shared.autheliaOnly;
        in
        lib.elem "hive-h1" (map (c: c.id) autheliaNoQueue.services.hyperhive.swarm.authelia.oidc.clients)
        &&
          grafanaRemoteAuthelia.services.hyperhive.swarm.statusPublish.tokenEndpoint
          == "https://auth.example.invalid/api/oidc/token";
    }
    {
      # The CI board is in the directory the provider scans (`bao.json` is the
      # control that the entries are read at all), and every metric it queries
      # is a name the poller pushes: a renamed metric on either side is a board
      # of empty panels.
      name = "the CI dashboard ships, and queries the metric names the CI run poller pushes";
      ok =
        let
          provider = lib.head grafanaWithAuthelia.containers.swarm-grafana.config.services.grafana.provision.dashboards.settings.providers;
          shipped = lib.attrNames provider.options.path.entries;
          board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/ci.json);
          exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels;
          queried = lib.unique (
            lib.concatMap (
              e: map lib.head (lib.filter lib.isList (builtins.split ''"(hyperhive\.ci\.[a-z_.]+)"'' e))
            ) exprs
          );
          pushed = builtins.readFile ../../swarm-ci-metrics/src/sample.rs;
        in
        lib.elem "ci.json" shipped
        && lib.elem "bao.json" shipped
        &&
          lib.sort lib.lessThan queried == [
            "hyperhive.ci.run.duration"
            "hyperhive.ci.run.queue_wait"
          ]
        && lib.all (m: lib.hasInfix "\"${m}\"" pushed) queried
        && lib.any (v: v.name == "repo") board.templating.list;
    }
    {
      # The standing check on the collector's PRIORITY mapping — whether a line
      # ever reaches the log store carrying no severity. It is shipped config
      # rather than something an operator imports by hand, which also means it
      # can be deleted by hand. Read out of the dashboard the provisioner
      # actually ships (`dashboardDir` in ../host-modules/swarm-grafana.nix
      # names this file), and asserted on the QUERY rather than the title,
      # because a panel that keeps its title and loses its expression renders
      # an empty graph that looks exactly like zero prioless lines.
      name = "the logs dashboard still counts lines that arrive with no severity";
      ok =
        let
          board = builtins.fromJSON (
            builtins.readFile ../host-modules/swarm-grafana/dashboards/logstore.json
          );
          exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels;
          counts = lib.filter (e: lib.hasInfix "severity_text" e && lib.hasInfix "stats count()" e) exprs;
        in
        # Both arms of the split: lines that HAD a priority and arrived without
        # a severity anyway (the regression line, which must reach zero), and
        # lines that never had one to map — Claude Code's own OTLP telemetry,
        # which no mapping can reach. Folded into one number the second would
        # keep the panel off zero forever and hide the first.
        lib.any (e: lib.hasInfix "PRIORITY:*" e) counts
        && lib.any (e: lib.hasInfix "PRIORITY:\"\"" e) counts;
    }
    {
      # The logs datasource reaches the store through its swarm name — a
      # loopback URL works only where Grafana and the store share a host — and
      # sends the metrics datasource's bearer, which the store's `/select/logsql/`
      # route checks.
      name = "the logs datasource targets the log store's swarm name and sends the metrics bearer";
      ok =
        let
          datasources =
            grafanaWithAuthelia.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources;
          metrics = lib.findFirst (d: d.uid == "swarm-victoriametrics") null datasources;
          logs = lib.findFirst (d: d.uid == "swarm-victorialogs") null datasources;
        in
        logs.url == "https://${grafanaWithAuthelia.services.hyperhive.swarm.victorialogs.domain}"
        && !(lib.hasInfix "127.0.0.1" logs.url)
        && logs.jsonData.httpHeaderName1 == "Authorization"
        && logs.secureJsonData.httpHeaderValue1 == metrics.secureJsonData.httpHeaderValue1;
    }
    {
      # An external plugin inherits only Grafana's proxy variables unless its
      # id is listed here, and without `SSL_CERT_FILE` every logs query fails
      # `x509: unknown authority`. Compared against the provisioned
      # datasource's `type`, so renaming one side without the other fails.
      name = "the logs datasource's plugin is handed Grafana's environment";
      ok =
        let
          grafana = grafanaWithAuthelia.containers.swarm-grafana.config.services.grafana;
          logs = lib.findFirst (
            d: d.uid == "swarm-victorialogs"
          ) null grafana.provision.datasources.settings.datasources;
        in
        logs != null
        && lib.elem logs.type (lib.toList (grafana.settings.plugins.forward_host_env_vars or [ ]));
    }
    {
      # The reader's half of the same mapping. The collector writes a severity
      # TEXT; nothing downstream reads it as a level unless the datasource is
      # told which field holds it, and that telling is a list of rules rather
      # than a field name — so it goes stale one severity at a time. Asserted
      # against ../journald-severity.nix rather than a literal list here,
      # because the way this regresses is a severity added to the parser and
      # not to Grafana: every line still arrives, the new one is just
      # unfilterable, and no query errors to say so.
      #
      # `enabled` and the canonical `level` spelling are pinned alongside
      # because both fail silently in the direction of "provisions fine,
      # returns nothing" — see the comment on `logLevelRules` in
      # ../host-modules/swarm-grafana.nix.
      name = "the logs datasource maps every severity the collector can emit to a log level";
      ok =
        let
          sources =
            grafanaHere.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources;
          logs = lib.head (lib.filter (d: d.uid == "swarm-victorialogs") sources);
          rules = logs.jsonData.logLevelRules or [ ];
          # `overwrite_text` makes the stored text the OpenTelemetry short name
          # for each mapped severity, which is the parser's own key uppercased.
          emitted = map lib.toUpper (lib.attrNames (lib.head (import ../journald-severity.nix)).mapping);
          canonical = [
            "critical"
            "error"
            "warning"
            "info"
            "debug"
            "trace"
          ];
        in
        rules != [ ]
        && lib.all (r: r.field == "severity_text" && r.enabled == true && lib.elem r.level canonical) rules
        && lib.all (text: lib.any (r: r.value == text) rules) emitted;
    }
    {
      # Harness turn failures reach the operator only through this rule, so
      # its absence is silent. Asserted on the query, not the title: the
      # exact `_msg` is what keeps subagent turn failures out, and a phrase
      # match would still provision and fire.
      name = "the agent turn-failure alert rule is provisioned against the harness's log line";
      ok =
        let
          groups =
            grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
          rules = lib.concatMap (g: g.rules) groups;
          rule = lib.findFirst (r: r.uid == "hh-agent-turn-failed") null rules;
          expr = (lib.head rule.data).model.expr;
        in
        rule != null
        && lib.hasInfix ''_SYSTEMD_UNIT:="hive-agent.service"'' expr
        && lib.hasInfix ''_msg:="turn failed"'' expr
        && lib.hasInfix "stats by (hive, agent)" expr;
    }
    {
      # The hive-health alerts query metric names that hive-c0re spells in
      # Rust. A rename on either side provisions cleanly and the rule then
      # reads nothing, so it never fires. Each name here is checked against
      # the exporter's source rather than a copy of it, and each rule must
      # query the metrics datasource with an instant query.
      # 🔬 Invert-proof: rename a gauge in `otel_metrics.rs` and this fails.
      name = "the hive-health alert rules query metrics hive-c0re exports";
      ok =
        let
          groups =
            grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
          rules = lib.concatMap (g: g.rules) groups;
          source = builtins.readFile ../../hive-c0re/src/stats/otel_metrics.rs;
          expected = {
            "hh-hive-degraded" = "hyperhive.hive.degraded";
            "hh-hive-warning" = "hyperhive.hive.warning";
            "hh-host-disk-high" = "hyperhive.hive.nix_store.utilization";
            "hh-host-disk-critical" = "hyperhive.hive.nix_store.utilization";
          };
          check =
            uid: metric:
            let
              rule = lib.findFirst (r: r.uid == uid) null rules;
              query = (lib.head rule.data).model;
            in
            rule != null
            && lib.hasInfix ''{__name__="${metric}"'' query.expr
            && lib.hasInfix ''"${metric}"'' source
            && query.instant
            && (lib.head rule.data).datasourceUid == "swarm-victoriametrics";
        in
        lib.all (uid: check uid expected.${uid}) (builtins.attrNames expected);
    }
    {
      # 🩸 The regression this guards: moving bao's metrics scrape into its
      # own in-container collector made that collector stamp every exported
      # series `service.name=swarm-bao`, while every panel in the shipped
      # bao.json still queried the literal `"service.name"="bao"` — delivery
      # could be perfect and no panel would ever match. Fixed by renaming it
      # in the collector rather than in the dashboard: `swarm-bao.nix`'s
      # `collectorServiceName` now stamps `"bao"` directly. This case reads
      # the collector's OWN evaluated config — not a hand-copied literal —
      # so it fails again the moment that binding and the dashboard drift
      # apart in either direction.
      # 🔬 Invert-proof: change `collectorServiceName` in swarm-bao.nix back
      # to `cfg.machine` and this case fails.
      name = "the bao dashboard's selectors match the service.name bao's own collector stamps";
      ok =
        let
          attrs =
            baoWithCollector.containers.swarm-bao.config.services.opentelemetry-collector.settings.processors.resource.attributes;
          stamped = lib.findFirst (a: a.key == "service.name") null attrs;
          board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/bao.json);
          exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels;
          selectors = lib.filter (e: lib.hasInfix "service.name" e) exprs;
        in
        stamped != null
        && builtins.length selectors == 12
        && lib.all (e: lib.hasInfix "\"service.name\"=\"${stamped.value}\"" e) selectors;
    }
    {
      # A module declares its vhost under `gateway.vhosts` and the gateway adds
      # what it owns: the listen set, the cert for the name and the security
      # headers ahead of the module's own `extraConfig`.
      name = "grafana's vhost is rendered with the gateway's listen set, TLS and security headers";
      ok =
        let
          gw = grafanaHere.services.hyperhive.gateway;
          v = grafanaHere.services.nginx.virtualHosts.${grafanaHere.services.hyperhive.swarm.grafana.domain};
          # nginx's `listen` entries come back with defaults filled in
          # (`extraParameters`, `proxyProtocol`), so compare the fields set.
          listenOf = map (l: {
            inherit (l) addr port;
            ssl = l.ssl or false;
          });
        in
        listenOf v.listen == listenOf gw.lib.listen
        && lib.hasPrefix gw.lib.securityHeaders v.extraConfig
        && v.addSSL
        && !(v ? certificate);
    }
    {
      # A vhost that sets `listen` itself would bind a port the gateway does
      # not consider its own, so the eval refuses it and names the vhost.
      name = "a gateway vhost that sets its own listen is refused";
      ok = lib.any (
        a:
        !a.assertion
        && lib.hasInfix "sets attributes the gateway owns" a.message
        && lib.hasInfix "own-listen.t.local.listen" a.message
      ) grafanaNoIdentity.assertions;
    }
  ];
in
runGroup "grafana" cases