hyperhive/nix/module-eval.nix

# `checks.module-eval` — the flake check that covers **nix**.
#
# Why this exists: every other check in ./checks.nix is a Rust
# derivation, so a `.nix`-only diff moves no hash, every check is a
# cache hit, and `nix flake check` reports green **without evaluating
# what changed**. This one's derivation hash is a function of the
# evaluated *results* below, so a nix change that flips a property
# rebuilds it and the builder fails naming that property.
#
# ## What belongs here, and what does not
#
# Anything expressible as a module `assertion` **should be one instead**:
# an assertion fires at deploy time for a real operator, not only in CI.
# What cannot be an assertion is the **absence class** — "a hive that
# hasn't opted in renders exactly what it did before", "this unit does
# not exist unless X". Those are claims about the *rendered config*
# rather than about a config being invalid, so they need an evaluator.
#
# ⚠️ **Cases are named by the PROPERTY they defend, never by the ticket
# that prompted them.** A case named after a ticket has the ticket's
# lifetime; a case named after a property lives as long as the property.
#
# ⚠️ **This check evaluates. It does not execute.** Where the artifact is
# a command line, an HTTP request or a certificate, a value assertion
# cannot stand in — those need something that *runs* them. And a case
# that needs a **rendered file** must stub the packages that file drags
# in (`swarm.ui.package = pkgs.emptyDirectory`), or it costs a full
# frontend build to answer a question about a listen directive.
{
  pkgs,
  lib,
  self,
  nixosSystem,
}:
let
  # Stub host, same shape ./docs/default.nix already uses: enough for a
  # `nixosSystem` to evaluate, nothing that pulls a real disk or
  # bootloader in.
  hive =
    extra:
    (nixosSystem {
      system = pkgs.stdenv.hostPlatform.system;
      modules = [
        self.nixosModules.default
        {
          fileSystems."/" = {
            device = "/dev/null";
            fsType = "tmpfs";
          };
          boot.loader.grub.enable = false;
          system.stateVersion = "25.11";
          # `recursiveUpdate`, not `//`: a plain `//` only merges the
          # *top-level* keys of `extra` in, so an `extra` that touches a
          # nested attr under an existing top-level key (e.g. `swarm.*`)
          # silently drops every sibling under that key instead of merging
          # into it — the same class of bug as the `services.hyperhive`
          # double-nesting mistake this stub already had to dodge once,
          # just one level down. `recursiveUpdate` merges nested attrsets
          # all the way down instead.
          services.hyperhive = lib.recursiveUpdate {
            enable = true;
            hiveName = "h1";
            swarm.domain = "t.local";
            swarm.hives.h1.domain = "h1.t.local";
          } extra;
        }
      ];
    }).config;

  allLocal = hive { deploy.singleHostSwarm = true; };
  bare = hive { };
  withCi = hive { deploy.forgejo.ci.enable = true; };

  baoPkcs11 = hive {
    deploy.bao.enable = true;
    deploy.bao.seal = "pkcs11";
  };
  baoShamir = hive {
    deploy.bao.enable = true;
    deploy.bao.seal = "shamir";
  };
  baoExplicitCerts = hive {
    deploy.bao.enable = true;
    deploy.bao.serverCertFile = "/etc/pki/bao.pem";
    deploy.bao.serverKeyFile = "/etc/pki/bao-key.pem";
  };
  # The store and a service that reads from it, versus the store alone. The
  # pair is what makes the reader's absence arm mean anything.
  baoWithMatrix = hive {
    deploy.bao.enable = true;
    deploy.matrix.enable = true;
  };
  # A hive that reads from a store it does not run: no `deploy.bao.enable`, so
  # nothing here mints a leaf and the operator names one placed by hand. The
  # deployment this pairing exists to serve, and the one that was previously
  # inexpressible — the gate asked whether the store was a neighbour.
  baoRemoteReader = hive {
    deploy.matrix.enable = true;
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
  };
  # The same hive with the identity taken away, which separates "a homeserver
  # is deployed" from "this host can authenticate to the store".
  matrixNoBaoIdentity = hive { deploy.matrix.enable = true; };

  baoNames = machine: machine.services.hyperhive.gateway.localNames;

  baoTwoAddresses = hive {
    deploy.bao.enable = true;
    deploy.bao.extraListenAddresses = [ "10.0.0.1" ];
    # Pinned, not incidental: the case counting these listeners is about the
    # declared addresses, and a collector on this host would add one of its own.
    deploy.swarm-otel.enable = false;
  };
  # The store with and without a collector on the same host. `scrapeTargets`
  # is only ever read by a local collector, so the metrics endpoint is a
  # function of the pairing rather than of the store.
  baoWithCollector = hive {
    deploy.bao.enable = true;
    deploy.swarm-otel.enable = true;
    # A second job declared as bare `host:port`, so the pair of cases below
    # reads one rendered scrape list: the store's entry carries a path, this
    # one carries none.
    swarm.otel.scrapeTargets.plain = "127.0.0.1:9999";
  };
  baoNoCollector = hive {
    deploy.bao.enable = true;
    deploy.swarm-otel.enable = false;
  };

  # The config file openbao parses, not the nix that produces it: a setting it
  # requires is absent here without anything in the module system minding, so
  # the daemon's own startup is otherwise the first reader.
  baoSettings = machine: machine.containers.swarm-bao.config.services.openbao.settings;

  # The store's units live inside its container, so the gates below have to
  # look there rather than at the host's service set.
  baoUnits = machine: machine.containers.swarm-bao.config.systemd.services;

  # The scrape list prometheus is handed, not the option a service declared:
  # the address, the path and the query are one string on the way in and three
  # fields on the way out, and only the second shape is what gets requested.
  scrapeJob =
    machine: job:
    lib.findFirst (c: c.job_name == job) null
      machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs;

  # A priority collision is a property of the *option*, not
  # of the merged value's interior — nix throws the moment the value is
  # demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is
  # both necessary and sufficient. `deepSeq` over-specifies this: it keeps
  # walking *into* the resulting value after the merge already succeeded,
  # and a package/derivation-shaped value's `override`/`overrideAttrs`
  # self-reference sends it into nixpkgs' fixpoint machinery and blows the
  # stack (measured — this is not a hypothetical).
  forceCiServiceConfigs =
    let
      svcs = withCi.containers.hive-ci.config.systemd.services;
      vals = lib.concatMap (s: builtins.attrValues (s.serviceConfig or { })) (builtins.attrValues svcs);
    in
    builtins.foldl' (acc: v: builtins.seq v acc) true vals;

  # Each case: a name stating the property, and `ok`.
  cases = [
    {
      name = "a hive that has not opted into all-local runs no swarm controller";
      ok = !bare.services.hyperhive.deploy.swarm-controller.enable;
    }
    {
      name = "the all-local mode turns the swarm controller on";
      ok = allLocal.services.hyperhive.deploy.swarm-controller.enable;
    }
    {
      # The gateway's per-name issuer choice. If this ever collapses to a
      # constant, every swarm-service vhost serves a certificate its CA
      # is name-constrained out of — which evaluates cleanly and fails in
      # a browser.
      name = "a swarm service name gets the swarm-services leaf and the default server does not";
      ok =
        let
          l = allLocal.services.hyperhive.gateway.lib;
        in
        (l.tlsFor "t.local").sslCertificate != (l.tlsFor "_").sslCertificate;
    }
    {
      # nixos asserts when a vhost declares both, so this is also a
      # statement that the `removeAttrs` upstream of it still happens.
      name = "the swarm UI vhost forces TLS instead of merely adding it";
      ok =
        let
          v = allLocal.services.nginx.virtualHosts."t.local";
        in
        v.forceSSL && !(v.addSSL or false);
    }
    {
      name = "a hive with matrix off serves no matrix discovery endpoint";
      ok =
        !(builtins.hasAttr "= /.well-known/matrix/client" bare.services.nginx.virtualHosts."_".locations);
    }
    {
      # main got eval-borked twice by this exact class of bug (once on the
      # unit's `Restart` key, once on `RestartSec`) — a nixpkgs bump to
      # `gitea-actions-runner.nix` adds a plain `serviceConfig.*`
      # definition that collides with one of ours, and nix refuses to
      # merge two plain definitions at *host* eval. No other check
      # instantiates a host with `containers.hive-ci` actually enabled, so
      # the collision only surfaces on operator deploy, not in CI.
      name = "the CI container's unit definitions merge without a priority collision";
      ok = forceCiServiceConfigs;
    }
    {
      # The store's seal is spread over six gates — the stanza, the
      # provisioning unit, two bind mounts, a device and an EnvironmentFile.
      # Rendering only some of them is the dangerous state: a store that
      # says hardware-backed and seals with a software key, which no
      # assertion can catch because every value is individually valid.
      name = "a shamir store renders no TPM provisioning unit";
      ok = !(baoUnits baoShamir ? swarm-bao-token);
    }
    {
      # Presence control for the case above. Without it, a typo in the
      # option name would satisfy the absence arm forever. The second half is
      # the fix itself: the unit has to run where openbao's `DynamicUser` is
      # allocated, and a host unit writing the same bytes has no name to hand
      # them to.
      name = "a pkcs11 store provisions the token in the container, not on the host";
      ok = (baoUnits baoPkcs11 ? swarm-bao-token) && !(baoPkcs11.systemd.services ? swarm-bao-token);
    }
    {
      # `allowedDevices` renders `DeviceAllow=` and nothing else — nspawn
      # mounts its own /dev and cannot create device nodes, so permission to
      # use a device that was never bound in opens nothing. Neither half
      # fails on its own, which is why they are asserted as a pair.
      name = "a pkcs11 store gets the TPM device bound in, not merely allowed";
      ok =
        let
          c = baoPkcs11.containers.swarm-bao;
        in
        (c.bindMounts ? "/dev/tpmrm0") && builtins.any (d: d.node == "/dev/tpmrm0") c.allowedDevices;
    }
    {
      # The stanza's label and the label the unit creates are two literals that
      # have to name one object, and the mechanism is asserted with them
      # because it is valid only for an RSA key: openbao takes AES-GCM or
      # RSA-OAEP and a TPM 2.0 has neither GCM nor an opinion about which the
      # seal asked for. Every wrong combination renders and deploys, and
      # surfaces as a pkcs11 error at `operator init`.
      name = "the seal asks for the RSA key the provisioning unit creates";
      ok =
        let
          p = (baoSettings baoPkcs11).seal.pkcs11;
        in
        (p.mechanism or "") == "CKM_RSA_PKCS_OAEP"
        && lib.hasInfix "--algorithm=rsa2048 --key-label=${p.key_label or ""}" (
          (baoUnits baoPkcs11).swarm-bao-token.script or ""
        );
    }
    {
      # `DynamicUser` implies `ProtectSystem=strict`, so the token directory is
      # read-only to the seal however it is owned, and the group is the only
      # handle on a uid allocated at start. Dropping either surfaces as a
      # pkcs11 error deep in a library, naming neither the mount nor the user.
      name = "the store's seal may write the token directory, and is in both its groups";
      ok =
        let
          sc = (baoUnits baoPkcs11).openbao.serviceConfig;
        in
        # `or [ ]` rather than a bare select: the interesting mutation is the
        # key being gone, and a select would abort the whole run with a nix
        # trace instead of failing this case by name.
        builtins.elem "/var/lib/swarm-bao-token" (sc.ReadWritePaths or [ ])
        && builtins.elem "swarm-bao-token" (sc.SupplementaryGroups or [ ])
        && builtins.elem "swarm-bao-tpm" (sc.SupplementaryGroups or [ ]);
    }
    {
      # The device node belongs to the HOST and is matched by NUMBER, while the
      # unit that opens it lives in the container — so the two sides holding
      # the same gid is the entire mechanism. Letting either side auto-allocate
      # renders cleanly, deploys cleanly, and leaves a 0660 node the seal
      # cannot open. Compared rather than each checked against a literal: the
      # property is that they AGREE, not what they agree on.
      name = "the TPM group has the same gid on the host and inside the container";
      ok =
        let
          host = baoPkcs11.users.groups.swarm-bao-tpm.gid or null;
          inner = baoPkcs11.containers.swarm-bao.config.users.groups.swarm-bao-tpm.gid or null;
        in
        host != null && host == inner;
    }
    {
      # Absence arm for the case above — a shamir store never opens a TPM, so
      # it must not claim a device node's group. Without this, pinning the gid
      # unconditionally would look identical.
      name = "a shamir store claims no TPM device group";
      ok = !(baoShamir.users.groups ? swarm-bao-tpm);
    }
    {
      # The store's mTLS identity is a separate trust domain from both CAs in
      # this tree, because it must not come from an authority the store will
      # itself distribute. What supplies it is the glue, which mints a CA of
      # the store's own — so an enabled store has all three paths, and if this
      # ever reads null again the store stops coming up on its own.
      name = "a deployed store is given its own certificate, key and client CA";
      ok =
        let
          b = baoPkcs11.services.hyperhive.deploy.bao;
        in
        b.serverCertFile != null && b.serverKeyFile != null && b.clientCaFile != null;
    }
    {
      # Everything the glue sets is `mkDefault`, and this is the case that
      # says so: a deployment whose certificates come from somewhere the glue
      # has never heard of must win. Also the presence control for the case
      # above — a renamed option would read `null` on both and satisfy
      # neither, but only this one names a value.
      name = "an operator's own certificate path beats the glue's default";
      ok = baoExplicitCerts.services.hyperhive.deploy.bao.serverCertFile == "/etc/pki/bao.pem";
    }
    {
      # The store's first reader. Its unit belongs to the pairing, not to
      # either service: matrix must not learn the store exists, and the store
      # must not know who reads it.
      name = "a store deployed beside the homeserver fetches its registration token";
      ok = baoWithMatrix.systemd.services ? swarm-bao-matrix-token;
    }
    {
      # Absence arm. A store with nothing to serve renders no reader, so the
      # unit is a function of the PAIRING rather than of the store — which is
      # the property that makes it glue instead of a feature of either side.
      name = "a store with no homeserver beside it renders no token reader";
      ok = !(baoPkcs11.systemd.services ? swarm-bao-matrix-token);
    }
    {
      name = "a hive that names a client identity reads from a store it does not run";
      ok = baoRemoteReader.systemd.services ? swarm-bao-matrix-token;
    }
    {
      # Absence arm for the one above, and the reason the gate is the identity
      # rather than the homeserver: without it, deploying matrix anywhere would
      # render a reader that cannot authenticate.
      name = "a homeserver with no way to authenticate to the store renders no token reader";
      ok = !(matrixNoBaoIdentity.systemd.services ? swarm-bao-matrix-token);
    }
    {
      # `Requires=` on a unit that does not exist fails the job, and nothing
      # local mints certificates off-host — so this orders against nothing.
      # Eval cannot see that failure; only the empty list here stands in for it.
      name = "an off-host reader requires no unit the store's host would have provided";
      # Membership first, then the value: indexing a missing unit throws, and a
      # table that reports which property broke must not be the thing that dies.
      ok =
        let
          s = baoRemoteReader.systemd.services;
        in
        s ? swarm-bao-matrix-token && s.swarm-bao-matrix-token.requires == [ ];
    }
    {
      # Presence control for the case above: the list is conditional, not gone.
      name = "a co-located reader still orders after the local pki unit";
      ok =
        let
          s = baoWithMatrix.systemd.services;
        in
        s ? swarm-bao-matrix-token && s.swarm-bao-matrix-token.requires == [ "swarm-bao-pki.service" ];
    }
    {
      # The name a reader dials has to resolve where the store runs; a
      # multi-host swarm resolves it upstream instead.
      name = "the store's host answers for the store's name";
      ok = builtins.elem "bao.t.local" (baoNames baoPkcs11);
    }
    {
      # Absence arm, and the one that matters: claiming a name this host does
      # not serve points every local reader at the wrong machine.
      name = "a hive that does not run the store claims no name for it";
      ok = !(builtins.elem "bao.t.local" (baoNames bare));
    }
    {
      # Raft refuses to start without it, and says so in a message that names
      # neither the setting nor the stanza.
      name = "the store advertises a cluster address";
      ok = lib.hasPrefix "https://" ((baoSettings baoPkcs11).cluster_addr or "");
    }
    {
      # Control for the case above: these settings are rendered per deployment,
      # not constants a passing case could be indifferent to.
      name = "a declared extra address renders a second listener beside loopback";
      ok = builtins.length (builtins.attrNames (baoSettings baoTwoAddresses).listener) == 2;
    }
    {
      # Retention is what serves the endpoint at all, so the listener alone
      # would be a port that answers 404.
      name = "a store beside a collector serves metrics on its own listener";
      ok =
        let
          s = baoSettings baoWithCollector;
        in
        s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s";
    }
    {
      # openbao serves no `/metrics` at all, so a scrape of the default path
      # 404s: the store looks like a dead exporter, and every panel built on
      # it renders empty rather than erroring.
      name = "the store's scrape asks for the path openbao serves";
      ok =
        let
          j = scrapeJob baoWithCollector "bao";
        in
        (j.metrics_path or "") == "/v1/sys/metrics" && (j.params.format or [ ]) == [ "prometheus" ];
    }
    {
      # Presence control for the case above: both fields are omitted rather
      # than defaulted, so a target declared as bare `host:port` renders what
      # it rendered before the path grammar existed.
      name = "a target with no path renders neither metrics_path nor params";
      ok =
        let
          j = scrapeJob baoWithCollector "plain";
        in
        j != null && !(j ? metrics_path) && !(j ? params);
    }
    {
      # Absence arm. Unauthenticated by design, so it must not exist where
      # nothing reads it.
      name = "a store with no collector beside it serves no metrics";
      ok =
        let
          s = baoSettings baoNoCollector;
        in
        !(s.listener ? metrics) && !(s ? telemetry);
    }
  ];

  bad = builtins.filter (c: !c.ok) cases;
  # Escaped, because a name is prose and prose contains apostrophes. Hand-quoting
  # broke the builder mid-report on the first such name that failed — and only
  # ever on failure, so every green run agreed the reporter was fine.
  report = lib.concatMapStringsSep "\n" (
    c: "  echo ${lib.escapeShellArg "FAILED: ${c.name}"} >&2"
  ) bad;
in
# The results are embedded in the builder text on purpose: that is what
# makes this derivation's hash depend on them, so a nix-only change that
# flips a case cannot be answered from cache.
pkgs.runCommand "hyperhive-module-eval" { } ''
  ${report}
  ${
    if bad == [ ] then
      "echo '${toString (builtins.length cases)} module properties hold' && touch $out"
    else
      "echo 'module-eval: ${toString (builtins.length bad)} of ${toString (builtins.length cases)} properties broke' >&2 && exit 1"
  }
''