345 lines
11 KiB
Nix
345 lines
11 KiB
Nix
# `checks.module-eval-semi-spread` — see ./lib.nix for the shared rationale
|
|
# (why this suite exists, naming convention, "evaluates not executes").
|
|
#
|
|
# Evaluates the five machines of docs/examples/semi-spread/ exactly as that
|
|
# directory's files describe them, so the example the handbook points at
|
|
# cannot stop evaluating, and pins the shape the handbook page promises: which
|
|
# machine runs what, what is open on the public interface, and what only on the
|
|
# VPN.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
baoSettings
|
|
runGroup
|
|
;
|
|
|
|
example = self + "/docs/examples/semi-spread";
|
|
|
|
# What the example's files leave to the machine's own configuration: its
|
|
# host name, which picks the VPN entry and, for a hive, the hive. `role` is
|
|
# the file the machine adds to common.nix.
|
|
machine =
|
|
hostName: role:
|
|
(nixosSystem {
|
|
modules = [
|
|
(pkgs.path + "/nixos/modules/misc/nixpkgs/read-only.nix")
|
|
{ nixpkgs.pkgs = pkgs; }
|
|
self.nixosModules.default
|
|
{
|
|
fileSystems."/" = {
|
|
device = "/dev/null";
|
|
fsType = "tmpfs";
|
|
};
|
|
boot.loader.grub.enable = false;
|
|
system.stateVersion = "25.11";
|
|
networking.hostName = hostName;
|
|
}
|
|
(example + "/common.nix")
|
|
(example + "/${role}.nix")
|
|
];
|
|
}).config;
|
|
|
|
central = machine "swarm-central" "swarm-central";
|
|
services = machine "swarm-services" "swarm-services";
|
|
heavyduty = machine "swarm-heavyduty" "swarm-heavyduty";
|
|
hiveOne = machine "hive-one" "hive";
|
|
hiveTwo = machine "hive-two" "hive";
|
|
all = {
|
|
swarm-central = central;
|
|
swarm-services = services;
|
|
swarm-heavyduty = heavyduty;
|
|
hive-one = hiveOne;
|
|
hive-two = hiveTwo;
|
|
};
|
|
|
|
mesh = import (example + "/mesh.nix");
|
|
|
|
# What `system.build.toplevel` checks before it builds anything: every module
|
|
# assertion holds. The example also renders no warning. Evaluating the
|
|
# toplevel itself would add the package set of five full systems to every
|
|
# run of this suite.
|
|
evaluates =
|
|
m:
|
|
let
|
|
r = builtins.tryEval (lib.all (a: a.assertion) m.assertions && m.warnings == [ ]);
|
|
in
|
|
r.success && r.value;
|
|
|
|
# The TCP ports open on every interface, and those open on `wg0` alone.
|
|
publicTcp = m: m.networking.firewall.allowedTCPPorts;
|
|
vpnTcp = m: (m.networking.firewall.interfaces.wg0 or { allowedTCPPorts = [ ]; }).allowedTCPPorts;
|
|
|
|
deploy = m: m.services.hyperhive.deploy;
|
|
swarmOf = m: m.services.hyperhive.swarm;
|
|
https = m: m.services.hyperhive.gateway.httpsPort;
|
|
|
|
# The names a machine's `/etc/hosts` and its resolver's extra hosts file
|
|
# resolve to the VPN, as a set of `address -> names`.
|
|
hostsOf = m: m.networking.hosts;
|
|
resolverTable =
|
|
m: lib.concatMapStrings builtins.readFile (m.services.dnsmasq.settings.addn-hosts or [ ]);
|
|
names = m: lib.concatLists (lib.attrValues (lib.filterAttrs (a: _: a != "127.0.0.2") (hostsOf m)));
|
|
|
|
vhosts = m: builtins.attrNames m.services.nginx.virtualHosts;
|
|
|
|
cases = [
|
|
# Every machine evaluates with its assertions holding. `tryEval`
|
|
# turns a refusal into a failed case that names the machine.
|
|
{
|
|
name = "swarm-central evaluates with its assertions holding";
|
|
ok = evaluates central;
|
|
}
|
|
{
|
|
name = "swarm-services evaluates with its assertions holding";
|
|
ok = evaluates services;
|
|
}
|
|
{
|
|
name = "swarm-heavyduty evaluates with its assertions holding";
|
|
ok = evaluates heavyduty;
|
|
}
|
|
{
|
|
name = "hive-one evaluates with its assertions holding";
|
|
ok = evaluates hiveOne;
|
|
}
|
|
{
|
|
name = "hive-two evaluates with its assertions holding";
|
|
ok = evaluates hiveTwo;
|
|
}
|
|
|
|
# Who runs what. Each machine's list is exact: a service that moved would
|
|
# show here on the machine that gained it and the one that lost it.
|
|
{
|
|
name = "swarm-central runs the controller and the store, and none of the services placed elsewhere";
|
|
ok =
|
|
let
|
|
d = deploy central;
|
|
in
|
|
d.swarm-controller.enable
|
|
&& d.bao.enable
|
|
&& !d.forgejo.enable
|
|
&& !d.matrix.enable
|
|
&& !d.nats.enable
|
|
&& !d.attic.enable
|
|
&& !d.builder.enable
|
|
&& !d.hive-controller.enable;
|
|
}
|
|
{
|
|
name = "swarm-services runs the forge, matrix, SSO, queue and the metrics and log stack, not the store";
|
|
ok =
|
|
let
|
|
d = deploy services;
|
|
in
|
|
lib.all (x: x) [
|
|
d.forgejo.enable
|
|
d.matrix.enable
|
|
d.authelia.enable
|
|
d.nats.enable
|
|
d.swarm-otel.enable
|
|
d.victoriametrics.enable
|
|
d.grafana.enable
|
|
d.victorialogs.enable
|
|
d.review-classifier.enable
|
|
d.notify-router.enable
|
|
d.claude-login.enable
|
|
d.ci-metrics.enable
|
|
(!d.bao.enable)
|
|
(!d.swarm-controller.enable)
|
|
(!d.attic.enable)
|
|
(!d.builder.enable)
|
|
(!d.ci-runner.enable)
|
|
(!d.hive-controller.enable)
|
|
];
|
|
}
|
|
{
|
|
name = "swarm-heavyduty runs the builder, the CI runner and the cache, and nothing else";
|
|
ok =
|
|
let
|
|
d = deploy heavyduty;
|
|
in
|
|
d.builder.enable
|
|
&& d.ci-runner.enable
|
|
&& d.attic.enable
|
|
&& !d.forgejo.enable
|
|
&& !d.matrix.enable
|
|
&& !d.nats.enable
|
|
&& !d.bao.enable
|
|
&& !d.swarm-controller.enable
|
|
&& !d.hive-controller.enable;
|
|
}
|
|
{
|
|
name = "hive-one and hive-two run a hive each, under different names, and no shared service";
|
|
ok =
|
|
let
|
|
d1 = deploy hiveOne;
|
|
d2 = deploy hiveTwo;
|
|
none =
|
|
d:
|
|
!(
|
|
d.forgejo.enable || d.matrix.enable || d.nats.enable || d.bao.enable || d.swarm-controller.enable
|
|
);
|
|
in
|
|
d1.hive-controller.enable
|
|
&& d2.hive-controller.enable
|
|
&& d1.hive-controller.hiveName == "one"
|
|
&& d2.hive-controller.hiveName == "two"
|
|
&& none d1
|
|
&& none d2;
|
|
}
|
|
{
|
|
# `swarm.*` is the one part that has to be identical, and it comes from a
|
|
# single file; the case reads it back from every machine.
|
|
name = "every machine holds the same swarm name, hives and service names";
|
|
ok =
|
|
let
|
|
view = m: {
|
|
inherit (swarmOf m) domain serviceDomains;
|
|
hives = builtins.attrNames (swarmOf m).hives;
|
|
};
|
|
in
|
|
lib.all (m: view m == view central) (lib.attrValues all);
|
|
}
|
|
|
|
# The public interface. The machines serving the forge, matrix and the
|
|
# swarm UI open the gateway to everyone; the others open nothing but the
|
|
# VPN's own port.
|
|
{
|
|
name = "swarm-central and swarm-services open the gateway on every interface";
|
|
ok =
|
|
publicTcp central == [
|
|
80
|
|
443
|
|
]
|
|
&&
|
|
publicTcp services == [
|
|
80
|
|
443
|
|
];
|
|
}
|
|
{
|
|
# Presence control for the case above: the same read finds the ports
|
|
# where they are open, so an empty list here means closed.
|
|
name = "swarm-heavyduty and the hives open no TCP port on every interface";
|
|
ok =
|
|
publicTcp heavyduty == [ ]
|
|
&& publicTcp hiveOne == [ ]
|
|
&& publicTcp hiveTwo == [ ]
|
|
&& vpnTcp heavyduty == [ (https heavyduty) ]
|
|
&& vpnTcp hiveOne == [ (https hiveOne) ]
|
|
&& vpnTcp hiveTwo == [ (https hiveTwo) ];
|
|
}
|
|
{
|
|
name = "every machine opens only the VPN's UDP port on every interface";
|
|
ok = lib.all (m: m.networking.firewall.allowedUDPPorts == [ 51820 ]) (lib.attrValues all);
|
|
}
|
|
{
|
|
name = "swarm-central serves the swarm UI and swarm-services serves the forge and matrix";
|
|
ok =
|
|
lib.elem (swarmOf central).domain (vhosts central)
|
|
&& lib.elem (swarmOf services).forge.domain (vhosts services)
|
|
&& lib.elem (swarmOf services).matrix.gatewayHost (vhosts services)
|
|
# The cache is on a machine with nothing public.
|
|
&& lib.elem (swarmOf heavyduty).attic.ownDomain (vhosts heavyduty)
|
|
&& !lib.elem (swarmOf heavyduty).attic.ownDomain (vhosts services);
|
|
}
|
|
|
|
# What only the VPN reaches.
|
|
{
|
|
name = "the secret store's port is open on wg0 and on no other external interface";
|
|
ok =
|
|
let
|
|
port = (swarmOf central).bao.port;
|
|
in
|
|
vpnTcp central == [ port ] && !lib.elem port (publicTcp central);
|
|
}
|
|
{
|
|
name = "the secret store listens on its VPN address and requires a client certificate there";
|
|
ok =
|
|
let
|
|
l = (baoSettings central).listener.extra-1 or { };
|
|
in
|
|
(l.address or null) == "${mesh.swarm-central.address}:${toString (swarmOf central).bao.port}"
|
|
&& (l.tls_require_and_verify_client_cert or false);
|
|
}
|
|
{
|
|
name = "the queue's port is open on wg0 and on no other external interface";
|
|
ok =
|
|
let
|
|
port = (swarmOf services).nats.port;
|
|
in
|
|
lib.elem port (vpnTcp services) && !lib.elem port (publicTcp services);
|
|
}
|
|
|
|
# The VPN itself.
|
|
{
|
|
name = "every machine peers with the other four over wg0, with its key outside the store";
|
|
ok = lib.all (
|
|
m:
|
|
let
|
|
wg = m.networking.wireguard.interfaces.wg0;
|
|
in
|
|
builtins.length wg.peers == 4
|
|
&& wg.privateKey == null
|
|
&& wg.privateKeyFile != null
|
|
&& !lib.hasPrefix "/nix/store" wg.privateKeyFile
|
|
) (lib.attrValues all);
|
|
}
|
|
{
|
|
name = "each machine takes its own address from the shared VPN table";
|
|
ok = lib.all (
|
|
n: lib.elem "${mesh.${n}.address}/24" all.${n}.networking.wireguard.interfaces.wg0.ips
|
|
) (builtins.attrNames all);
|
|
}
|
|
|
|
# DNS. A machine lists the names others serve, at those machines' VPN
|
|
# addresses, and leaves out its own.
|
|
{
|
|
name = "a hive resolves the store, queue and cache names to the machines that serve them";
|
|
ok =
|
|
let
|
|
t = hostsOf hiveOne;
|
|
in
|
|
lib.elem (swarmOf hiveOne).bao.domain t.${mesh.swarm-central.address}
|
|
&& lib.elem (swarmOf hiveOne).nats.domain t.${mesh.swarm-services.address}
|
|
&& lib.elem (swarmOf hiveOne).attic.ownDomain t.${mesh.swarm-heavyduty.address};
|
|
}
|
|
{
|
|
name = "a machine leaves out the names it serves itself";
|
|
ok =
|
|
!lib.elem (swarmOf central).bao.domain (names central)
|
|
&& !lib.elem (swarmOf services).nats.domain (names services)
|
|
&& !lib.elem (swarmOf heavyduty).attic.ownDomain (names heavyduty)
|
|
&& !lib.elem "one.hives.${(swarmOf hiveOne).domain}" (names hiveOne)
|
|
# Control: another hive's dashboard is in the table.
|
|
&& lib.elem "two.hives.${(swarmOf hiveOne).domain}" (names hiveOne);
|
|
}
|
|
{
|
|
# Containers ask the hive's resolver, which ignores /etc/hosts, so the
|
|
# same table has to reach it through its extra hosts file.
|
|
name = "the hive resolver carries the same table as /etc/hosts";
|
|
ok =
|
|
let
|
|
table = resolverTable hiveOne;
|
|
in
|
|
hostsOf hiveOne != { }
|
|
&& lib.all (
|
|
address: lib.hasInfix "${address} ${lib.concatStringsSep " " (hostsOf hiveOne).${address}}" table
|
|
) (builtins.attrNames (lib.filterAttrs (a: _: a != "127.0.0.2") (hostsOf hiveOne)))
|
|
# Control: the file is not one every machine shares.
|
|
&& !lib.hasInfix (swarmOf central).bao.domain (resolverTable central);
|
|
}
|
|
];
|
|
in
|
|
runGroup "semi-spread" cases
|