add a semi-spread example deploy and a check that evaluates it
This commit is contained in:
atlas
mara
parent
793d987a88
commit
538da5a13a
1 changed files with 875 additions and 1 deletions
345
nix/module-eval/semi-spread.nix
Normal file
345
nix/module-eval/semi-spread.nix
Normal file
|
|
@ -0,0 +1,345 @@
|
|||
# `checks.module-eval-semi-spread` — see ./lib.nix for the shared rationale
|
||||
# (why this suite exists, naming convention, "evaluates not executes").
|
||||
#
|
||||
# Evaluates the five machines of docs/examples/semi-spread/ exactly as that
|
||||
# directory's files describe them, so the example the handbook points at
|
||||
# cannot stop evaluating, and pins the shape the handbook page promises: which
|
||||
# machine runs what, what is open on the public interface, and what only on the
|
||||
# VPN.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
self,
|
||||
nixosSystem,
|
||||
}:
|
||||
let
|
||||
inherit
|
||||
(import ./lib.nix {
|
||||
inherit
|
||||
pkgs
|
||||
lib
|
||||
self
|
||||
nixosSystem
|
||||
;
|
||||
})
|
||||
baoSettings
|
||||
runGroup
|
||||
;
|
||||
|
||||
example = self + "/docs/examples/semi-spread";
|
||||
|
||||
# What the example's files leave to the machine's own configuration: its
|
||||
# host name, which picks the VPN entry and, for a hive, the hive. `role` is
|
||||
# the file the machine adds to common.nix.
|
||||
machine =
|
||||
hostName: role:
|
||||
(nixosSystem {
|
||||
modules = [
|
||||
(pkgs.path + "/nixos/modules/misc/nixpkgs/read-only.nix")
|
||||
{ nixpkgs.pkgs = pkgs; }
|
||||
self.nixosModules.default
|
||||
{
|
||||
fileSystems."/" = {
|
||||
device = "/dev/null";
|
||||
fsType = "tmpfs";
|
||||
};
|
||||
boot.loader.grub.enable = false;
|
||||
system.stateVersion = "25.11";
|
||||
networking.hostName = hostName;
|
||||
}
|
||||
(example + "/common.nix")
|
||||
(example + "/${role}.nix")
|
||||
];
|
||||
}).config;
|
||||
|
||||
central = machine "swarm-central" "swarm-central";
|
||||
services = machine "swarm-services" "swarm-services";
|
||||
heavyduty = machine "swarm-heavyduty" "swarm-heavyduty";
|
||||
hiveOne = machine "hive-one" "hive";
|
||||
hiveTwo = machine "hive-two" "hive";
|
||||
all = {
|
||||
swarm-central = central;
|
||||
swarm-services = services;
|
||||
swarm-heavyduty = heavyduty;
|
||||
hive-one = hiveOne;
|
||||
hive-two = hiveTwo;
|
||||
};
|
||||
|
||||
mesh = import (example + "/mesh.nix");
|
||||
|
||||
# What `system.build.toplevel` checks before it builds anything: every module
|
||||
# assertion holds. The example also renders no warning. Evaluating the
|
||||
# toplevel itself would add the package set of five full systems to every
|
||||
# run of this suite.
|
||||
evaluates =
|
||||
m:
|
||||
let
|
||||
r = builtins.tryEval (lib.all (a: a.assertion) m.assertions && m.warnings == [ ]);
|
||||
in
|
||||
r.success && r.value;
|
||||
|
||||
# The TCP ports open on every interface, and those open on `wg0` alone.
|
||||
publicTcp = m: m.networking.firewall.allowedTCPPorts;
|
||||
vpnTcp = m: (m.networking.firewall.interfaces.wg0 or { allowedTCPPorts = [ ]; }).allowedTCPPorts;
|
||||
|
||||
deploy = m: m.services.hyperhive.deploy;
|
||||
swarmOf = m: m.services.hyperhive.swarm;
|
||||
https = m: m.services.hyperhive.gateway.httpsPort;
|
||||
|
||||
# The names a machine's `/etc/hosts` and its resolver's extra hosts file
|
||||
# resolve to the VPN, as a set of `address -> names`.
|
||||
hostsOf = m: m.networking.hosts;
|
||||
resolverTable =
|
||||
m: lib.concatMapStrings builtins.readFile (m.services.dnsmasq.settings.addn-hosts or [ ]);
|
||||
names = m: lib.concatLists (lib.attrValues (lib.filterAttrs (a: _: a != "127.0.0.2") (hostsOf m)));
|
||||
|
||||
vhosts = m: builtins.attrNames m.services.nginx.virtualHosts;
|
||||
|
||||
cases = [
|
||||
# Every machine evaluates with its assertions holding. `tryEval`
|
||||
# turns a refusal into a failed case that names the machine.
|
||||
{
|
||||
name = "swarm-central evaluates with its assertions holding";
|
||||
ok = evaluates central;
|
||||
}
|
||||
{
|
||||
name = "swarm-services evaluates with its assertions holding";
|
||||
ok = evaluates services;
|
||||
}
|
||||
{
|
||||
name = "swarm-heavyduty evaluates with its assertions holding";
|
||||
ok = evaluates heavyduty;
|
||||
}
|
||||
{
|
||||
name = "hive-one evaluates with its assertions holding";
|
||||
ok = evaluates hiveOne;
|
||||
}
|
||||
{
|
||||
name = "hive-two evaluates with its assertions holding";
|
||||
ok = evaluates hiveTwo;
|
||||
}
|
||||
|
||||
# Who runs what. Each machine's list is exact: a service that moved would
|
||||
# show here on the machine that gained it and the one that lost it.
|
||||
{
|
||||
name = "swarm-central runs the controller and the store, and none of the services placed elsewhere";
|
||||
ok =
|
||||
let
|
||||
d = deploy central;
|
||||
in
|
||||
d.swarm-controller.enable
|
||||
&& d.bao.enable
|
||||
&& !d.forgejo.enable
|
||||
&& !d.matrix.enable
|
||||
&& !d.nats.enable
|
||||
&& !d.attic.enable
|
||||
&& !d.builder.enable
|
||||
&& !d.hive-controller.enable;
|
||||
}
|
||||
{
|
||||
name = "swarm-services runs the forge, matrix, SSO, queue and the metrics and log stack, not the store";
|
||||
ok =
|
||||
let
|
||||
d = deploy services;
|
||||
in
|
||||
lib.all (x: x) [
|
||||
d.forgejo.enable
|
||||
d.matrix.enable
|
||||
d.authelia.enable
|
||||
d.nats.enable
|
||||
d.swarm-otel.enable
|
||||
d.victoriametrics.enable
|
||||
d.grafana.enable
|
||||
d.victorialogs.enable
|
||||
d.review-classifier.enable
|
||||
d.notify-router.enable
|
||||
d.claude-login.enable
|
||||
d.ci-metrics.enable
|
||||
(!d.bao.enable)
|
||||
(!d.swarm-controller.enable)
|
||||
(!d.attic.enable)
|
||||
(!d.builder.enable)
|
||||
(!d.ci-runner.enable)
|
||||
(!d.hive-controller.enable)
|
||||
];
|
||||
}
|
||||
{
|
||||
name = "swarm-heavyduty runs the builder, the CI runner and the cache, and nothing else";
|
||||
ok =
|
||||
let
|
||||
d = deploy heavyduty;
|
||||
in
|
||||
d.builder.enable
|
||||
&& d.ci-runner.enable
|
||||
&& d.attic.enable
|
||||
&& !d.forgejo.enable
|
||||
&& !d.matrix.enable
|
||||
&& !d.nats.enable
|
||||
&& !d.bao.enable
|
||||
&& !d.swarm-controller.enable
|
||||
&& !d.hive-controller.enable;
|
||||
}
|
||||
{
|
||||
name = "hive-one and hive-two run a hive each, under different names, and no shared service";
|
||||
ok =
|
||||
let
|
||||
d1 = deploy hiveOne;
|
||||
d2 = deploy hiveTwo;
|
||||
none =
|
||||
d:
|
||||
!(
|
||||
d.forgejo.enable || d.matrix.enable || d.nats.enable || d.bao.enable || d.swarm-controller.enable
|
||||
);
|
||||
in
|
||||
d1.hive-controller.enable
|
||||
&& d2.hive-controller.enable
|
||||
&& d1.hive-controller.hiveName == "one"
|
||||
&& d2.hive-controller.hiveName == "two"
|
||||
&& none d1
|
||||
&& none d2;
|
||||
}
|
||||
{
|
||||
# `swarm.*` is the one part that has to be identical, and it comes from a
|
||||
# single file; the case reads it back from every machine.
|
||||
name = "every machine holds the same swarm name, hives and service names";
|
||||
ok =
|
||||
let
|
||||
view = m: {
|
||||
inherit (swarmOf m) domain serviceDomains;
|
||||
hives = builtins.attrNames (swarmOf m).hives;
|
||||
};
|
||||
in
|
||||
lib.all (m: view m == view central) (lib.attrValues all);
|
||||
}
|
||||
|
||||
# The public interface. The machines serving the forge, matrix and the
|
||||
# swarm UI open the gateway to everyone; the others open nothing but the
|
||||
# VPN's own port.
|
||||
{
|
||||
name = "swarm-central and swarm-services open the gateway on every interface";
|
||||
ok =
|
||||
publicTcp central == [
|
||||
80
|
||||
443
|
||||
]
|
||||
&&
|
||||
publicTcp services == [
|
||||
80
|
||||
443
|
||||
];
|
||||
}
|
||||
{
|
||||
# Presence control for the case above: the same read finds the ports
|
||||
# where they are open, so an empty list here means closed.
|
||||
name = "swarm-heavyduty and the hives open no TCP port on every interface";
|
||||
ok =
|
||||
publicTcp heavyduty == [ ]
|
||||
&& publicTcp hiveOne == [ ]
|
||||
&& publicTcp hiveTwo == [ ]
|
||||
&& vpnTcp heavyduty == [ (https heavyduty) ]
|
||||
&& vpnTcp hiveOne == [ (https hiveOne) ]
|
||||
&& vpnTcp hiveTwo == [ (https hiveTwo) ];
|
||||
}
|
||||
{
|
||||
name = "every machine opens only the VPN's UDP port on every interface";
|
||||
ok = lib.all (m: m.networking.firewall.allowedUDPPorts == [ 51820 ]) (lib.attrValues all);
|
||||
}
|
||||
{
|
||||
name = "swarm-central serves the swarm UI and swarm-services serves the forge and matrix";
|
||||
ok =
|
||||
lib.elem (swarmOf central).domain (vhosts central)
|
||||
&& lib.elem (swarmOf services).forge.domain (vhosts services)
|
||||
&& lib.elem (swarmOf services).matrix.gatewayHost (vhosts services)
|
||||
# The cache is on a machine with nothing public.
|
||||
&& lib.elem (swarmOf heavyduty).attic.ownDomain (vhosts heavyduty)
|
||||
&& !lib.elem (swarmOf heavyduty).attic.ownDomain (vhosts services);
|
||||
}
|
||||
|
||||
# What only the VPN reaches.
|
||||
{
|
||||
name = "the secret store's port is open on wg0 and on no other external interface";
|
||||
ok =
|
||||
let
|
||||
port = (swarmOf central).bao.port;
|
||||
in
|
||||
vpnTcp central == [ port ] && !lib.elem port (publicTcp central);
|
||||
}
|
||||
{
|
||||
name = "the secret store listens on its VPN address and requires a client certificate there";
|
||||
ok =
|
||||
let
|
||||
l = (baoSettings central).listener.extra-1 or { };
|
||||
in
|
||||
(l.address or null) == "${mesh.swarm-central.address}:${toString (swarmOf central).bao.port}"
|
||||
&& (l.tls_require_and_verify_client_cert or false);
|
||||
}
|
||||
{
|
||||
name = "the queue's port is open on wg0 and on no other external interface";
|
||||
ok =
|
||||
let
|
||||
port = (swarmOf services).nats.port;
|
||||
in
|
||||
lib.elem port (vpnTcp services) && !lib.elem port (publicTcp services);
|
||||
}
|
||||
|
||||
# The VPN itself.
|
||||
{
|
||||
name = "every machine peers with the other four over wg0, with its key outside the store";
|
||||
ok = lib.all (
|
||||
m:
|
||||
let
|
||||
wg = m.networking.wireguard.interfaces.wg0;
|
||||
in
|
||||
builtins.length wg.peers == 4
|
||||
&& wg.privateKey == null
|
||||
&& wg.privateKeyFile != null
|
||||
&& !lib.hasPrefix "/nix/store" wg.privateKeyFile
|
||||
) (lib.attrValues all);
|
||||
}
|
||||
{
|
||||
name = "each machine takes its own address from the shared VPN table";
|
||||
ok = lib.all (
|
||||
n: lib.elem "${mesh.${n}.address}/24" all.${n}.networking.wireguard.interfaces.wg0.ips
|
||||
) (builtins.attrNames all);
|
||||
}
|
||||
|
||||
# DNS. A machine lists the names others serve, at those machines' VPN
|
||||
# addresses, and leaves out its own.
|
||||
{
|
||||
name = "a hive resolves the store, queue and cache names to the machines that serve them";
|
||||
ok =
|
||||
let
|
||||
t = hostsOf hiveOne;
|
||||
in
|
||||
lib.elem (swarmOf hiveOne).bao.domain t.${mesh.swarm-central.address}
|
||||
&& lib.elem (swarmOf hiveOne).nats.domain t.${mesh.swarm-services.address}
|
||||
&& lib.elem (swarmOf hiveOne).attic.ownDomain t.${mesh.swarm-heavyduty.address};
|
||||
}
|
||||
{
|
||||
name = "a machine leaves out the names it serves itself";
|
||||
ok =
|
||||
!lib.elem (swarmOf central).bao.domain (names central)
|
||||
&& !lib.elem (swarmOf services).nats.domain (names services)
|
||||
&& !lib.elem (swarmOf heavyduty).attic.ownDomain (names heavyduty)
|
||||
&& !lib.elem "one.hives.${(swarmOf hiveOne).domain}" (names hiveOne)
|
||||
# Control: another hive's dashboard is in the table.
|
||||
&& lib.elem "two.hives.${(swarmOf hiveOne).domain}" (names hiveOne);
|
||||
}
|
||||
{
|
||||
# Containers ask the hive's resolver, which ignores /etc/hosts, so the
|
||||
# same table has to reach it through its extra hosts file.
|
||||
name = "the hive resolver carries the same table as /etc/hosts";
|
||||
ok =
|
||||
let
|
||||
table = resolverTable hiveOne;
|
||||
in
|
||||
hostsOf hiveOne != { }
|
||||
&& lib.all (
|
||||
address: lib.hasInfix "${address} ${lib.concatStringsSep " " (hostsOf hiveOne).${address}}" table
|
||||
) (builtins.attrNames (lib.filterAttrs (a: _: a != "127.0.0.2") (hostsOf hiveOne)))
|
||||
# Control: the file is not one every machine shares.
|
||||
&& !lib.hasInfix (swarmOf central).bao.domain (resolverTable central);
|
||||
}
|
||||
];
|
||||
in
|
||||
runGroup "semi-spread" cases
|
||||
Loading…
Reference in a new issue