hyperhive/nix/module-eval/grafana.nix

# `checks.module-eval-grafana` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
  pkgs,
  lib,
  self,
  nixosSystem,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    runGroup
    ;

  grafanaOldPath = hive {
    deploy.grafana.enable = true;
    swarm.grafana.socketDir = "/run/test-grafana-sock";
    swarm.grafana.datasourceUrl = "http://127.0.0.1:19999";
    swarm.grafana.logsDatasourceUrl = "http://127.0.0.1:19998";
    swarm.grafana.plugins = [ ];
    swarm.grafana.package = pkgs.emptyDirectory;
  };

  # The metrics UI beside the IdP. It reads its secret out of the store like
  # every other Grafana host, so it needs a store identity like every other
  # Grafana host — the cert pair here is not scenery, it is the arm that would
  # have caught the deleted co-located copy unit coming back.
  grafanaWithAuthelia = hive {
    deploy.grafana.enable = true;
    deploy.grafana.plugins = [ ];
    deploy.grafana.package = pkgs.emptyDirectory;
    deploy.authelia.enable = true;
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
  };

  # The same UI with the IdP on ANOTHER host and a store leaf placed by hand.
  # Knowing an IdP is not running one: `swarm.authelia.url` is what says this
  # swarm has SSO, and nothing about this host does. Identical to the fixture
  # above in everything the delivery path reads, which is the point.
  grafanaRemoteAuthelia = hive {
    deploy.grafana.enable = true;
    deploy.grafana.plugins = [ ];
    deploy.grafana.package = pkgs.emptyDirectory;
    swarm.authelia.url = "https://auth.example.invalid";
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
  };

  # A Grafana host holding no store identity. This used to be the shape the
  # module went QUIET on — no OIDC block, a warning, and a container whose
  # login form is off regardless, so no way in and nothing failed. It is kept
  # rather than deleted because the shape is still reachable by an operator;
  # what changed is the deliverable, from a warning nothing reads back to a
  # refusal naming the two options to set. Only the identity is missing, so an
  # arm below can name which refusal fired.
  grafanaNoIdentity = hive {
    deploy.grafana.enable = true;
    deploy.grafana.plugins = [ ];
    deploy.grafana.package = pkgs.emptyDirectory;
    swarm.authelia.url = "https://auth.example.invalid";
  };

  # The mirror image: the identity is placed, and the swarm names no IdP. The
  # other half of "SSO must always be configured", and isolated the same way —
  # exactly one thing wrong, so the arm reads one refusal.
  grafanaNoSso = hive {
    deploy.grafana.enable = true;
    deploy.grafana.plugins = [ ];
    deploy.grafana.package = pkgs.emptyDirectory;
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
  };

  # Did ./host-modules/swarm-grafana.nix refuse this host, and for which of its
  # two reasons. An assertion is a config VALUE until something forces it —
  # `.config` never throws — so a fixture in a state the module refuses is
  # evaluable and the refusal is readable as data. That is what lets a case
  # check that a misconfiguration is REPORTED, rather than only that it is not
  # silently accepted.
  #
  # Matched on the option name the message names, not on its prose, so the
  # wording stays rewordable: the option name is the part an operator has to
  # act on, and a message that stopped naming it would be the actual defect.
  grafanaRefusedFor =
    m: option:
    lib.any (
      a:
      !a.assertion
      && lib.hasInfix "services.hyperhive.deploy.grafana.enable requires" a.message
      && lib.hasInfix option a.message
    ) m.assertions;
  cases = [
    {
      # This fixture enables grafana and NOT authelia, which is the shape the
      # login form used to stay enabled in: the toggle read "both services are
      # on this host" rather than "grafana requires SSO". Grafana ships an
      # `admin`/`admin` account and its vhost is on the public gateway, so a
      # password box there is a way in whatever the topology.
      name = "grafana disables its local login form even where authelia is not on this host";
      ok =
        grafanaOldPath.containers.swarm-grafana.config.services.grafana.settings.auth.disable_login_form;
    }
    {
      # The absence class this whole file is for, and the reported defect in one
      # arm: the OIDC block hung off "authelia is on this host", so the split
      # deployment got a Grafana with no SSO settings and no login form — no way
      # in at all. The block is emitted in every deployment now, so the negative
      # arm is not "no block elsewhere" but "the two do not name the same IdP":
      # each host's block has to point at the URL the SWARM names, and a block
      # built from `deploy.authelia` rather than `swarm.authelia.url` would pass
      # a presence check on both fixtures while sending one of them nowhere.
      name = "grafana's OIDC block names the swarm's IdP, wherever that IdP runs";
      ok =
        let
          oauth = m: m.containers.swarm-grafana.config.services.grafana.settings."auth.generic_oauth";
          remote = oauth grafanaRemoteAuthelia;
          local = oauth grafanaWithAuthelia;
        in
        remote.enabled
        && lib.hasInfix "https://auth.example.invalid/api/oidc/token" remote.token_url
        && local.enabled
        && lib.hasInfix "https://auth.t.local/api/oidc/token" local.token_url
        && !(lib.hasInfix "auth.example.invalid" local.token_url);
    }
    {
      # 🩸 The arm that guards the ruling this slice landed under. There is ONE
      # delivery route: the store reader, on every host that runs Grafana. The
      # negative names the deleted unit rather than a generic absence, because
      # the way this regresses is someone re-adding the co-located copy as an
      # optimisation — a second writer of one path, and a second shape of "the
      # secret is wrong" to debug.
      name = "grafana's OIDC secret has exactly one delivery unit, the store reader, in both topologies";
      ok =
        let
          local = grafanaWithAuthelia.systemd.services;
          remote = grafanaRemoteAuthelia.systemd.services;
        in
        local ? swarm-bao-grafana-oidc
        && remote ? swarm-bao-grafana-oidc
        && !(local ? swarm-grafana-oidc-secret)
        && !(remote ? swarm-grafana-oidc-secret);
    }
    {
      # What the deleted warning became. The shape is unchanged — a Grafana host
      # holding no store leaf — but silence there is a container nobody can log
      # into for a reason no log names, and a warning is read back by nothing.
      # The second arm is what makes this a refusal about the IDENTITY: this
      # fixture names an IdP, so a message about `swarm.authelia.url` here would
      # mean the two assertions had been collapsed into one conjunction.
      name = "a grafana host with no store identity is refused, naming the options to set";
      ok =
        grafanaRefusedFor grafanaNoIdentity "deploy.bao.clientCertFile"
        && grafanaRefusedFor grafanaNoIdentity "deploy.bao.clientKeyFile"
        && !(grafanaRefusedFor grafanaNoIdentity "swarm.authelia.url");
    }
    {
      # "SSO must always be configured", as an eval-time refusal rather than a
      # gate. A null URL used to drop the OIDC block silently, and
      # `disable_login_form` is unconditional a hundred lines below it, so that
      # combination produced a Grafana with no SSO and no password box — an
      # outage whose cause is a boolean that evaluated to false at build time
      # and left no trace. Same isolation as the arm above, mirrored.
      name = "a grafana host in a swarm with no IdP is refused, naming swarm.authelia.url";
      ok =
        grafanaRefusedFor grafanaNoSso "services.hyperhive.swarm.authelia.url"
        && !(grafanaRefusedFor grafanaNoSso "deploy.bao.clientCertFile");
    }
    {
      # Without this the two arms above prove nothing: a refusal that fires on
      # every host is not a check, and both of these are hosts a swarm is
      # expected to have. Read through the same helper, so a message that
      # stopped naming its option would fail the arms above rather than pass
      # this one by accident.
      name = "neither grafana refusal fires on a correctly configured host, co-located or not";
      ok =
        !(grafanaRefusedFor grafanaWithAuthelia "services.hyperhive.swarm.authelia.url")
        && !(grafanaRefusedFor grafanaWithAuthelia "deploy.bao.clientCertFile")
        && !(grafanaRefusedFor grafanaRemoteAuthelia "services.hyperhive.swarm.authelia.url")
        && !(grafanaRefusedFor grafanaRemoteAuthelia "deploy.bao.clientCertFile");
    }
    {
      # Same 403-not-a-miss reason as the matrix and queue arms below: the
      # reader's grant covers the `services` prefix, so a path outside it is
      # refused rather than empty, however correct it reads. The negative arm is
      # the rename this is exposed to — a secret filed under the hive that runs
      # the service instead of under the service itself.
      name = "grafana's OIDC secret is read from the prefix the publisher writes";
      ok =
        let
          s = grafanaRemoteAuthelia.systemd.services.swarm-bao-grafana-oidc.script;
        in
        lib.hasInfix "secret/swarm/services/swarm-grafana/oidc/client" s
        && !(lib.hasInfix "secret/swarm/hives/" s);
    }
    {
      # Both halves of the co-location assumption, which was one host's
      # `deploy.*` answering a question about the whole swarm: the identities
      # were minted only where the queue happened to run, and the token
      # endpoint was known only where the IdP happened to run.
      name = "hive identities and the token endpoint do not depend on which host runs what";
      ok =
        let
          autheliaNoQueue = hive { deploy.authelia.enable = true; };
        in
        lib.elem "hive-h1" (map (c: c.id) autheliaNoQueue.services.hyperhive.swarm.authelia.oidc.clients)
        &&
          grafanaRemoteAuthelia.services.hyperhive.swarm.statusPublish.tokenEndpoint
          == "https://auth.example.invalid/api/oidc/token";
    }
    {
      # The standing check on the collector's PRIORITY mapping — whether a line
      # ever reaches the log store carrying no severity. It is shipped config
      # rather than something an operator imports by hand, which also means it
      # can be deleted by hand. Read out of the dashboard the provisioner
      # actually ships (`dashboardDir` in ../host-modules/swarm-grafana.nix
      # names this file), and asserted on the QUERY rather than the title,
      # because a panel that keeps its title and loses its expression renders
      # an empty graph that looks exactly like zero prioless lines.
      name = "the logs dashboard still counts lines that arrive with no severity";
      ok =
        let
          board = builtins.fromJSON (
            builtins.readFile ../host-modules/swarm-grafana/dashboards/logstore.json
          );
          exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels;
          counts = lib.filter (e: lib.hasInfix "severity_text" e && lib.hasInfix "stats count()" e) exprs;
        in
        # Both arms of the split: lines that HAD a priority and arrived without
        # a severity anyway (the regression line, which must reach zero), and
        # lines that never had one to map — Claude Code's own OTLP telemetry,
        # which no mapping can reach. Folded into one number the second would
        # keep the panel off zero forever and hide the first.
        lib.any (e: lib.hasInfix "PRIORITY:*" e) counts
        && lib.any (e: lib.hasInfix "PRIORITY:\"\"" e) counts;
    }
    {
      # The reader's half of the same mapping. The collector writes a severity
      # TEXT; nothing downstream reads it as a level unless the datasource is
      # told which field holds it, and that telling is a list of rules rather
      # than a field name — so it goes stale one severity at a time. Asserted
      # against ../journald-severity.nix rather than a literal list here,
      # because the way this regresses is a severity added to the parser and
      # not to Grafana: every line still arrives, the new one is just
      # unfilterable, and no query errors to say so.
      #
      # `enabled` and the canonical `level` spelling are pinned alongside
      # because both fail silently in the direction of "provisions fine,
      # returns nothing" — see the comment on `logLevelRules` in
      # ../host-modules/swarm-grafana.nix.
      name = "the logs datasource maps every severity the collector can emit to a log level";
      ok =
        let
          sources =
            grafanaOldPath.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources;
          logs = lib.head (lib.filter (d: d.uid == "swarm-victorialogs") sources);
          rules = logs.jsonData.logLevelRules or [ ];
          # `overwrite_text` makes the stored text the OpenTelemetry short name
          # for each mapped severity, which is the parser's own key uppercased.
          emitted = map lib.toUpper (lib.attrNames (lib.head (import ../journald-severity.nix)).mapping);
          canonical = [
            "critical"
            "error"
            "warning"
            "info"
            "debug"
            "trace"
          ];
        in
        rules != [ ]
        && lib.all (r: r.field == "severity_text" && r.enabled == true && lib.elem r.level canonical) rules
        && lib.all (text: lib.any (r: r.value == text) rules) emitted;
    }
  ];
in
runGroup "grafana" cases