# `checks.module-eval-semi-spread` — see ./lib.nix for the shared rationale # (why this suite exists, naming convention, "evaluates not executes"). # # Evaluates the five machines of docs/examples/semi-spread/ exactly as that # directory's files describe them, so the example the handbook points at # cannot stop evaluating, and pins the shape the handbook page promises: which # machine runs what, what is open on the public interface, and what only on the # VPN. { pkgs, lib, self, nixosSystem, }: let inherit (import ./lib.nix { inherit pkgs lib self nixosSystem ; }) baoSettings runGroup ; example = self + "/docs/examples/semi-spread"; # What the example's files leave to the machine's own configuration: its # host name, which picks the VPN entry and, for a hive, the hive. `role` is # the file the machine adds to common.nix. machine = hostName: role: (nixosSystem { modules = [ (pkgs.path + "/nixos/modules/misc/nixpkgs/read-only.nix") { nixpkgs.pkgs = pkgs; } self.nixosModules.default { fileSystems."/" = { device = "/dev/null"; fsType = "tmpfs"; }; boot.loader.grub.enable = false; system.stateVersion = "25.11"; networking.hostName = hostName; } (example + "/common.nix") (example + "/${role}.nix") ]; }).config; central = machine "swarm-central" "swarm-central"; services = machine "swarm-services" "swarm-services"; heavyduty = machine "swarm-heavyduty" "swarm-heavyduty"; hiveOne = machine "hive-one" "hive"; hiveTwo = machine "hive-two" "hive"; all = { swarm-central = central; swarm-services = services; swarm-heavyduty = heavyduty; hive-one = hiveOne; hive-two = hiveTwo; }; mesh = import (example + "/mesh.nix"); # What `system.build.toplevel` checks before it builds anything: every module # assertion holds. The example also renders no warning. Evaluating the # toplevel itself would add the package set of five full systems to every # run of this suite. evaluates = m: let r = builtins.tryEval (lib.all (a: a.assertion) m.assertions && m.warnings == [ ]); in r.success && r.value; # The TCP ports open on every interface, and those open on `wg0` alone. publicTcp = m: m.networking.firewall.allowedTCPPorts; vpnTcp = m: (m.networking.firewall.interfaces.wg0 or { allowedTCPPorts = [ ]; }).allowedTCPPorts; deploy = m: m.services.hyperhive.deploy; swarmOf = m: m.services.hyperhive.swarm; https = m: m.services.hyperhive.gateway.httpsPort; # The names a machine's `/etc/hosts` and its resolver's extra hosts file # resolve to the VPN, as a set of `address -> names`. hostsOf = m: m.networking.hosts; resolverTable = m: lib.concatMapStrings builtins.readFile (m.services.dnsmasq.settings.addn-hosts or [ ]); names = m: lib.concatLists (lib.attrValues (lib.filterAttrs (a: _: a != "127.0.0.2") (hostsOf m))); vhosts = m: builtins.attrNames m.services.nginx.virtualHosts; cases = [ # Every machine evaluates with its assertions holding. `tryEval` # turns a refusal into a failed case that names the machine. { name = "swarm-central evaluates with its assertions holding"; ok = evaluates central; } { name = "swarm-services evaluates with its assertions holding"; ok = evaluates services; } { name = "swarm-heavyduty evaluates with its assertions holding"; ok = evaluates heavyduty; } { name = "hive-one evaluates with its assertions holding"; ok = evaluates hiveOne; } { name = "hive-two evaluates with its assertions holding"; ok = evaluates hiveTwo; } # Who runs what. Each machine's list is exact: a service that moved would # show here on the machine that gained it and the one that lost it. { name = "swarm-central runs the controller and the store, and none of the services placed elsewhere"; ok = let d = deploy central; in d.swarm-controller.enable && d.bao.enable && !d.forgejo.enable && !d.matrix.enable && !d.nats.enable && !d.attic.enable && !d.builder.enable && !d.hive-controller.enable; } { name = "swarm-services runs the forge, matrix, SSO, queue and the metrics and log stack, not the store"; ok = let d = deploy services; in lib.all (x: x) [ d.forgejo.enable d.matrix.enable d.authelia.enable d.nats.enable d.swarm-otel.enable d.victoriametrics.enable d.grafana.enable d.victorialogs.enable d.review-classifier.enable d.notify-router.enable d.claude-login.enable d.ci-metrics.enable (!d.bao.enable) (!d.swarm-controller.enable) (!d.attic.enable) (!d.builder.enable) (!d.ci-runner.enable) (!d.hive-controller.enable) ]; } { name = "swarm-heavyduty runs the builder, the CI runner and the cache, and nothing else"; ok = let d = deploy heavyduty; in d.builder.enable && d.ci-runner.enable && d.attic.enable && !d.forgejo.enable && !d.matrix.enable && !d.nats.enable && !d.bao.enable && !d.swarm-controller.enable && !d.hive-controller.enable; } { name = "hive-one and hive-two run a hive each, under different names, and no shared service"; ok = let d1 = deploy hiveOne; d2 = deploy hiveTwo; none = d: !( d.forgejo.enable || d.matrix.enable || d.nats.enable || d.bao.enable || d.swarm-controller.enable ); in d1.hive-controller.enable && d2.hive-controller.enable && d1.hive-controller.hiveName == "one" && d2.hive-controller.hiveName == "two" && none d1 && none d2; } { # `swarm.*` is the one part that has to be identical, and it comes from a # single file; the case reads it back from every machine. name = "every machine holds the same swarm name, hives and service names"; ok = let view = m: { inherit (swarmOf m) domain serviceDomains; hives = builtins.attrNames (swarmOf m).hives; }; in lib.all (m: view m == view central) (lib.attrValues all); } # The public interface. The machines serving the forge, matrix and the # swarm UI open the gateway to everyone; the others open nothing but the # VPN's own port. { name = "swarm-central and swarm-services open the gateway on every interface"; ok = publicTcp central == [ 80 443 ] && publicTcp services == [ 80 443 ]; } { # Presence control for the case above: the same read finds the ports # where they are open, so an empty list here means closed. name = "swarm-heavyduty and the hives open no TCP port on every interface"; ok = publicTcp heavyduty == [ ] && publicTcp hiveOne == [ ] && publicTcp hiveTwo == [ ] && vpnTcp heavyduty == [ (https heavyduty) ] && vpnTcp hiveOne == [ (https hiveOne) ] && vpnTcp hiveTwo == [ (https hiveTwo) ]; } { name = "every machine opens only the VPN's UDP port on every interface"; ok = lib.all (m: m.networking.firewall.allowedUDPPorts == [ 51820 ]) (lib.attrValues all); } { name = "swarm-central serves the swarm UI and swarm-services serves the forge and matrix"; ok = lib.elem (swarmOf central).domain (vhosts central) && lib.elem (swarmOf services).forge.domain (vhosts services) && lib.elem (swarmOf services).matrix.gatewayHost (vhosts services) # The cache is on a machine with nothing public. && lib.elem (swarmOf heavyduty).attic.ownDomain (vhosts heavyduty) && !lib.elem (swarmOf heavyduty).attic.ownDomain (vhosts services); } # What only the VPN reaches. { name = "the secret store's port is open on wg0 and on no other external interface"; ok = let port = (swarmOf central).bao.port; in vpnTcp central == [ port ] && !lib.elem port (publicTcp central); } { name = "the secret store listens on its VPN address and requires a client certificate there"; ok = let l = (baoSettings central).listener.extra-1 or { }; in (l.address or null) == "${mesh.swarm-central.address}:${toString (swarmOf central).bao.port}" && (l.tls_require_and_verify_client_cert or false); } { name = "the queue's port is open on wg0 and on no other external interface"; ok = let port = (swarmOf services).nats.port; in lib.elem port (vpnTcp services) && !lib.elem port (publicTcp services); } # The VPN itself. { name = "every machine peers with the other four over wg0, with its key outside the store"; ok = lib.all ( m: let wg = m.networking.wireguard.interfaces.wg0; in builtins.length wg.peers == 4 && wg.privateKey == null && wg.privateKeyFile != null && !lib.hasPrefix "/nix/store" wg.privateKeyFile ) (lib.attrValues all); } { name = "each machine takes its own address from the shared VPN table"; ok = lib.all ( n: lib.elem "${mesh.${n}.address}/24" all.${n}.networking.wireguard.interfaces.wg0.ips ) (builtins.attrNames all); } # DNS. A machine lists the names others serve, at those machines' VPN # addresses, and leaves out its own. { name = "a hive resolves the store, queue and cache names to the machines that serve them"; ok = let t = hostsOf hiveOne; in lib.elem (swarmOf hiveOne).bao.domain t.${mesh.swarm-central.address} && lib.elem (swarmOf hiveOne).nats.domain t.${mesh.swarm-services.address} && lib.elem (swarmOf hiveOne).attic.ownDomain t.${mesh.swarm-heavyduty.address}; } { name = "a machine leaves out the names it serves itself"; ok = !lib.elem (swarmOf central).bao.domain (names central) && !lib.elem (swarmOf services).nats.domain (names services) && !lib.elem (swarmOf heavyduty).attic.ownDomain (names heavyduty) && !lib.elem "one.hives.${(swarmOf hiveOne).domain}" (names hiveOne) # Control: another hive's dashboard is in the table. && lib.elem "two.hives.${(swarmOf hiveOne).domain}" (names hiveOne); } { # Containers ask the hive's resolver, which ignores /etc/hosts, so the # same table has to reach it through its extra hosts file. name = "the hive resolver carries the same table as /etc/hosts"; ok = let table = resolverTable hiveOne; in hostsOf hiveOne != { } && lib.all ( address: lib.hasInfix "${address} ${lib.concatStringsSep " " (hostsOf hiveOne).${address}}" table ) (builtins.attrNames (lib.filterAttrs (a: _: a != "127.0.0.2") (hostsOf hiveOne))) # Control: the file is not one every machine shares. && !lib.hasInfix (swarmOf central).bao.domain (resolverTable central); } ]; in runGroup "semi-spread" cases