hyperhive/nix/module-eval/semi-spread.nix

# `checks.module-eval-semi-spread` — see ./lib.nix for the shared rationale
# (why this suite exists, naming convention, "evaluates not executes").
#
# Evaluates the five machines of docs/examples/semi-spread/ exactly as that
# directory's files describe them, so the example the handbook points at
# cannot stop evaluating, and pins the shape the handbook page promises: which
# machine runs what, what is open on the public interface, and what only on the
# VPN.
{
  pkgs,
  lib,
  self,
  nixosSystem,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    baoSettings
    runGroup
    ;

  example = self + "/docs/examples/semi-spread";

  # What the example's files leave to the machine's own configuration: its
  # host name, which picks the VPN entry and, for a hive, the hive. `role` is
  # the file the machine adds to common.nix.
  machine =
    hostName: role:
    (nixosSystem {
      modules = [
        (pkgs.path + "/nixos/modules/misc/nixpkgs/read-only.nix")
        { nixpkgs.pkgs = pkgs; }
        self.nixosModules.default
        {
          fileSystems."/" = {
            device = "/dev/null";
            fsType = "tmpfs";
          };
          boot.loader.grub.enable = false;
          system.stateVersion = "25.11";
          networking.hostName = hostName;
        }
        (example + "/common.nix")
        (example + "/${role}.nix")
      ];
    }).config;

  central = machine "swarm-central" "swarm-central";
  services = machine "swarm-services" "swarm-services";
  heavyduty = machine "swarm-heavyduty" "swarm-heavyduty";
  hiveOne = machine "hive-one" "hive";
  hiveTwo = machine "hive-two" "hive";
  all = {
    swarm-central = central;
    swarm-services = services;
    swarm-heavyduty = heavyduty;
    hive-one = hiveOne;
    hive-two = hiveTwo;
  };

  mesh = import (example + "/mesh.nix");

  # What `system.build.toplevel` checks before it builds anything: every module
  # assertion holds. The example also renders no warning. Evaluating the
  # toplevel itself would add the package set of five full systems to every
  # run of this suite.
  evaluates =
    m:
    let
      r = builtins.tryEval (lib.all (a: a.assertion) m.assertions && m.warnings == [ ]);
    in
    r.success && r.value;

  # The TCP ports open on every interface, and those open on `wg0` alone.
  publicTcp = m: m.networking.firewall.allowedTCPPorts;
  vpnTcp = m: (m.networking.firewall.interfaces.wg0 or { allowedTCPPorts = [ ]; }).allowedTCPPorts;

  deploy = m: m.services.hyperhive.deploy;
  swarmOf = m: m.services.hyperhive.swarm;
  https = m: m.services.hyperhive.gateway.httpsPort;

  # The names a machine's `/etc/hosts` and its resolver's extra hosts file
  # resolve to the VPN, as a set of `address -> names`.
  hostsOf = m: m.networking.hosts;
  resolverTable =
    m: lib.concatMapStrings builtins.readFile (m.services.dnsmasq.settings.addn-hosts or [ ]);
  names = m: lib.concatLists (lib.attrValues (lib.filterAttrs (a: _: a != "127.0.0.2") (hostsOf m)));

  vhosts = m: builtins.attrNames m.services.nginx.virtualHosts;

  # The `listen` entries of one vhost, as `addr:port`.
  listens =
    m: vhost: map (l: "${l.addr}:${toString l.port}") m.services.nginx.virtualHosts.${vhost}.listen;

  # Both ports on each address.
  bothPorts =
    addrs:
    lib.concatMap (a: [
      "${a}:80"
      "${a}:443"
    ]) addrs;

  # The private addresses of a machine as nginx spells them: its VPN IPv4 and
  # bracketed IPv6 address, and the hive bridge.
  privateAddrs = name: m: [
    mesh.${name}.address
    "[${mesh.${name}.address6}]"
    m.services.hyperhive.network.bridgeIp
  ];

  # The vhosts that listen on the wildcard, and those that listen on private
  # addresses only. The catch-all `_` is neither: it is not a service.
  publicVhosts =
    m:
    lib.sort (a: b: a < b) (
      lib.filter (n: n != "_" && lib.elem "0.0.0.0:443" (listens m n)) (vhosts m)
    );
  privateVhosts =
    m:
    lib.sort (a: b: a < b) (
      lib.filter (n: n != "_" && !lib.elem "0.0.0.0:443" (listens m n)) (vhosts m)
    );
  sorted = lib.sort (a: b: a < b);

  # What each machine is meant to serve to everyone, and what only inside.
  expectedPublic = {
    swarm-central = [ (swarmOf central).domain ];
    swarm-services = [
      (swarmOf services).authelia.domain
      (swarmOf services).forge.domain
      (swarmOf services).matrix.gatewayHost
    ];
    swarm-heavyduty = [ ];
    hive-one = [ ];
    hive-two = [ ];
  };
  expectedPrivate = {
    swarm-central = [ (swarmOf central).bao.ui.domain ];
    swarm-services = with swarmOf services; [
      grafana.domain
      nats.domain
      otel.domain
      victorialogs.domain
      victoriametrics.domain
    ];
    swarm-heavyduty = [ (swarmOf heavyduty).attic.ownDomain ];
    hive-one = [
      (swarmOf hiveOne).dashboardDomains.one
      (swarmOf hiveOne).hives.one.domain
    ];
    hive-two = [
      (swarmOf hiveTwo).dashboardDomains.two
      (swarmOf hiveTwo).hives.two.domain
    ];
  };

  cases = [
    # Every machine evaluates with its assertions holding. `tryEval`
    # turns a refusal into a failed case that names the machine.
    {
      name = "swarm-central evaluates with its assertions holding";
      ok = evaluates central;
    }
    {
      name = "swarm-services evaluates with its assertions holding";
      ok = evaluates services;
    }
    {
      name = "swarm-heavyduty evaluates with its assertions holding";
      ok = evaluates heavyduty;
    }
    {
      name = "hive-one evaluates with its assertions holding";
      ok = evaluates hiveOne;
    }
    {
      name = "hive-two evaluates with its assertions holding";
      ok = evaluates hiveTwo;
    }

    # Who runs what. Each machine's list is exact: a service that moved would
    # show here on the machine that gained it and the one that lost it.
    {
      name = "swarm-central runs the controller and the store, and none of the services placed elsewhere";
      ok =
        let
          d = deploy central;
        in
        d.swarm-controller.enable
        && d.bao.enable
        && !d.forgejo.enable
        && !d.matrix.enable
        && !d.nats.enable
        && !d.attic.enable
        && !d.builder.enable
        && !d.hive-controller.enable;
    }
    {
      name = "swarm-services runs the forge, matrix, SSO, queue and the metrics and log stack, not the store";
      ok =
        let
          d = deploy services;
        in
        lib.all (x: x) [
          d.forgejo.enable
          d.matrix.enable
          d.authelia.enable
          d.nats.enable
          d.swarm-otel.enable
          d.victoriametrics.enable
          d.grafana.enable
          d.victorialogs.enable
          d.review-classifier.enable
          d.notify-router.enable
          d.claude-login.enable
          d.ci-metrics.enable
          (!d.bao.enable)
          (!d.swarm-controller.enable)
          (!d.attic.enable)
          (!d.builder.enable)
          (!d.ci-runner.enable)
          (!d.hive-controller.enable)
        ];
    }
    {
      name = "swarm-heavyduty runs the builder, the CI runner and the cache, and nothing else";
      ok =
        let
          d = deploy heavyduty;
        in
        d.builder.enable
        && d.ci-runner.enable
        && d.attic.enable
        && !d.forgejo.enable
        && !d.matrix.enable
        && !d.nats.enable
        && !d.bao.enable
        && !d.swarm-controller.enable
        && !d.hive-controller.enable;
    }
    {
      name = "hive-one and hive-two run a hive each, under different names, and no shared service";
      ok =
        let
          d1 = deploy hiveOne;
          d2 = deploy hiveTwo;
          none =
            d:
            !(
              d.forgejo.enable || d.matrix.enable || d.nats.enable || d.bao.enable || d.swarm-controller.enable
            );
        in
        d1.hive-controller.enable
        && d2.hive-controller.enable
        && d1.hive-controller.hiveName == "one"
        && d2.hive-controller.hiveName == "two"
        && none d1
        && none d2;
    }
    {
      # `swarm.*` is the one part that has to be identical, and it comes from a
      # single file; the case reads it back from every machine.
      name = "every machine holds the same swarm name, hives and service names";
      ok =
        let
          view = m: {
            inherit (swarmOf m) domain serviceDomains;
            hives = builtins.attrNames (swarmOf m).hives;
          };
        in
        lib.all (m: view m == view central) (lib.attrValues all);
    }

    # The public interface. The machines serving the forge, matrix and the
    # swarm UI open the gateway to everyone; the others open nothing but the
    # VPN's own port.
    {
      name = "swarm-central and swarm-services open the gateway on every interface";
      ok =
        publicTcp central == [
          80
          443
        ]
        &&
          publicTcp services == [
            80
            443
          ];
    }
    {
      # Presence control for the case above: the same read finds the ports
      # where they are open, so an empty list here means closed.
      name = "swarm-heavyduty and the hives open no TCP port on every interface";
      ok =
        publicTcp heavyduty == [ ]
        && publicTcp hiveOne == [ ]
        && publicTcp hiveTwo == [ ]
        && vpnTcp heavyduty == [ (https heavyduty) ]
        && vpnTcp hiveOne == [ (https hiveOne) ]
        && vpnTcp hiveTwo == [ (https hiveTwo) ];
    }
    {
      name = "every machine opens only the VPN's UDP port on every interface";
      ok = lib.all (m: m.networking.firewall.allowedUDPPorts == [ 51820 ]) (lib.attrValues all);
    }
    {
      name = "swarm-central serves the swarm UI and swarm-services serves the forge and matrix";
      ok =
        lib.elem (swarmOf central).domain (vhosts central)
        && lib.elem (swarmOf services).forge.domain (vhosts services)
        && lib.elem (swarmOf services).matrix.gatewayHost (vhosts services)
        # The cache is on a machine with nothing public.
        && lib.elem (swarmOf heavyduty).attic.ownDomain (vhosts heavyduty)
        && !lib.elem (swarmOf heavyduty).attic.ownDomain (vhosts services);
    }

    # What only the VPN reaches.
    {
      name = "the secret store's port is open on wg0 and on no other external interface";
      ok =
        let
          port = (swarmOf central).bao.port;
        in
        vpnTcp central == [ port ] && !lib.elem port (publicTcp central);
    }
    {
      name = "the secret store listens on its VPN address and requires a client certificate there";
      ok =
        let
          l = (baoSettings central).listener.extra-1 or { };
        in
        (l.address or null) == "${mesh.swarm-central.address}:${toString (swarmOf central).bao.port}"
        && (l.tls_require_and_verify_client_cert or false);
    }
    {
      name = "the queue's port is open on wg0 and on no other external interface";
      ok =
        let
          port = (swarmOf services).nats.port;
        in
        lib.elem port (vpnTcp services) && !lib.elem port (publicTcp services);
    }

    # The VPN itself.
    {
      name = "every machine peers with the other four over wg0, with its key outside the store";
      ok = lib.all (
        m:
        let
          wg = m.networking.wireguard.interfaces.wg0;
        in
        builtins.length wg.peers == 4
        && wg.privateKey == null
        && wg.privateKeyFile != null
        && !lib.hasPrefix "/nix/store" wg.privateKeyFile
      ) (lib.attrValues all);
    }
    {
      name = "each machine takes its own address from the shared VPN table";
      ok = lib.all (
        n: lib.elem "${mesh.${n}.address}/24" all.${n}.networking.wireguard.interfaces.wg0.ips
      ) (builtins.attrNames all);
    }

    # DNS. A machine lists the names others serve, at those machines' VPN
    # addresses, and leaves out its own.
    {
      name = "a hive resolves the store, queue and cache names to the machines that serve them";
      ok =
        let
          t = hostsOf hiveOne;
        in
        lib.elem (swarmOf hiveOne).bao.domain t.${mesh.swarm-central.address}
        && lib.elem (swarmOf hiveOne).nats.domain t.${mesh.swarm-services.address}
        && lib.elem (swarmOf hiveOne).attic.ownDomain t.${mesh.swarm-heavyduty.address};
    }
    {
      name = "a machine leaves out the names it serves itself";
      ok =
        !lib.elem (swarmOf central).bao.domain (names central)
        && !lib.elem (swarmOf services).nats.domain (names services)
        && !lib.elem (swarmOf heavyduty).attic.ownDomain (names heavyduty)
        && !lib.elem "one.hives.${(swarmOf hiveOne).domain}" (names hiveOne)
        # Control: another hive's dashboard is in the table.
        && lib.elem "two.hives.${(swarmOf hiveOne).domain}" (names hiveOne);
    }
    {
      # Containers ask the hive's resolver, which ignores /etc/hosts, so the
      # same table has to reach it through its extra hosts file.
      name = "the hive resolver carries the same table as /etc/hosts";
      ok =
        let
          table = resolverTable hiveOne;
        in
        hostsOf hiveOne != { }
        && lib.all (
          address: lib.hasInfix "${address} ${lib.concatStringsSep " " (hostsOf hiveOne).${address}}" table
        ) (builtins.attrNames (lib.filterAttrs (a: _: a != "127.0.0.2") (hostsOf hiveOne)))
        # Control: the file is not one every machine shares.
        && !lib.hasInfix (swarmOf central).bao.domain (resolverTable central);
    }

    # Which vhosts answer where. Public means the wildcard, IPv4 and IPv6;
    # private means the machine's VPN addresses and its bridge, and nothing
    # else.
    {
      name = "only the swarm UI, SSO login, forge and matrix answer on the public address";
      ok = lib.all (n: publicVhosts all.${n} == sorted expectedPublic.${n}) (builtins.attrNames all);
    }
    {
      # Presence control for the case above: the same read finds every vhost
      # of the private set, so an empty public list is not a misread.
      name = "every other vhost answers on the private addresses only";
      ok = lib.all (
        n:
        let
          m = all.${n};
          private = privateAddrs n m;
        in
        privateVhosts m == sorted expectedPrivate.${n}
        && lib.all (v: sorted (listens m v) == sorted (bothPorts private)) expectedPrivate.${n}
      ) (builtins.attrNames all);
    }
    {
      name = "public vhosts listen on 0.0.0.0 and [::] and also on the private addresses";
      ok =
        lib.all (
          n:
          let
            m = all.${n};
            want = bothPorts (
              [
                "0.0.0.0"
                "[::]"
              ]
              ++ privateAddrs n m
            );
          in
          lib.all (v: sorted (listens m v) == sorted want) expectedPublic.${n}
        ) (builtins.attrNames all)
        # Control: the lists above are not empty.
        && expectedPublic.swarm-services != [ ];
    }
    {
      name = "the catch-all listens on the private addresses too, and the wildcard";
      ok = lib.all (
        n:
        let
          m = all.${n};
        in
        sorted (listens m "_") == sorted (
          bothPorts (
            [
              "0.0.0.0"
              "[::]"
            ]
            ++ privateAddrs n m
          )
        )
      ) (builtins.attrNames all);
    }
    {
      name = "the private addresses hold the VPN's IPv4 and IPv6 address of the machine, and the bridge";
      ok = lib.all (
        n:
        all.${n}.services.hyperhive.gateway.privateListenAddresses == [
          mesh.${n}.address
          mesh.${n}.address6
          all.${n}.services.hyperhive.network.bridgeIp
        ]
      ) (builtins.attrNames all);
    }
    {
      name = "the VPN carries IPv6: each machine has its address and a /128 per peer";
      ok = lib.all (
        n:
        let
          wg = all.${n}.networking.wireguard.interfaces.wg0;
        in
        lib.elem "${mesh.${n}.address6}/64" wg.ips
        && lib.all (p: lib.any (a: lib.hasSuffix "/128" a) p.allowedIPs) wg.peers
      ) (builtins.attrNames all);
    }
    {
      # The store's certificate is signed by its own CA, which only the store's
      # host has by default; every machine without the store names the copy.
      name = "every machine without the store checks it against the CA copied from swarm-central";
      ok =
        lib.all (n: all.${n}.services.hyperhive.deploy.bao.serverCaFile == "/var/lib/swarm-pki/ca.pem") [
          "swarm-services"
          "swarm-heavyduty"
          "hive-one"
          "hive-two"
        ]
        # Control: the machine with the store keeps the CA it minted.
        && central.services.hyperhive.deploy.bao.serverCaFile == "/var/lib/swarm-bao-pki/ca.pem";
    }
    {
      name = "a hive resolves the internal names to the IPv6 addresses as well";
      ok =
        let
          t = hostsOf hiveOne;
        in
        lib.elem (swarmOf hiveOne).bao.domain (t.${mesh.swarm-central.address6} or [ ])
        && lib.elem (swarmOf hiveOne).grafana.domain (t.${mesh.swarm-services.address6} or [ ])
        && lib.hasInfix "${mesh.swarm-heavyduty.address6} ${(swarmOf hiveOne).attic.ownDomain}" (
          resolverTable hiveOne
        );
    }
  ];
in
runGroup "semi-spread" cases