From 1c3118e9cdc086d5f0014c2fc7f914ca56a7c0e8 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 23:05:08 +0100 Subject: [PATCH] add two-vm wg-hive mesh test to vmTests --- docs/scheduler/ci.md | 4 +- flake.nix | 2 +- nix/vm-tests.nix | 7 ++- nix/vm-tests/wg-hive.nix | 120 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 nix/vm-tests/wg-hive.nix diff --git a/docs/scheduler/ci.md b/docs/scheduler/ci.md index 0ab65698..9bf4e3a5 100644 --- a/docs/scheduler/ci.md +++ b/docs/scheduler/ci.md @@ -120,7 +120,9 @@ by hand, start the workflow from the forge UI (Actions, vm-nightly, Run workflow, optional `test` input), or run `nix build .#vmTests.x86_64-linux.` on a host with KVM. Current tests: `agent-ephemeral-rootfs` (the switch of an agent container to an ephemeral -root, through the real hive-priv unit) and `hive-squid`. +root, through the real hive-priv unit), `hive-squid`, and `wg-hive` (two hives +peered over the `wg-hive` mesh: handshake, traffic, and a wrong-key peer that +must not handshake). ## Configuration reference diff --git a/flake.nix b/flake.nix index a78e6ec2..813c77db 100644 --- a/flake.nix +++ b/flake.nix @@ -231,7 +231,7 @@ vmTests = forAllSystems ( { pkgs, system, ... }: import ./nix/vm-tests.nix { - inherit pkgs; + inherit pkgs self; hivePriv = self.packages.${system}.hive-priv; } ); diff --git a/nix/vm-tests.nix b/nix/vm-tests.nix index aba292c8..9d726608 100644 --- a/nix/vm-tests.nix +++ b/nix/vm-tests.nix @@ -1,8 +1,13 @@ # NixOS VM tests: one derivation per test, exposed as `vmTests..`. # Kept out of `checks` because each boots VMs and needs KVM; the nightly # workflow `.forgejo/workflows/vm-nightly.yml` builds them. -{ pkgs, hivePriv }: +{ + pkgs, + hivePriv, + self, +}: { agent-ephemeral-rootfs = import ./vm-tests/agent-ephemeral-rootfs.nix { inherit pkgs hivePriv; }; + wg-hive = import ./vm-tests/wg-hive.nix { inherit pkgs self; }; hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs; }; } diff --git a/nix/vm-tests/wg-hive.nix b/nix/vm-tests/wg-hive.nix new file mode 100644 index 00000000..b909f438 --- /dev/null +++ b/nix/vm-tests/wg-hive.nix @@ -0,0 +1,120 @@ +# `vmTests..wg-hive`: two hives peered over `wg-hive`, with real traffic across the tunnel. +# Unlike ../module-eval, which only reads the rendered `networking.wireguard` +# options, this boots both hosts and sends packets. +# +# Keys are minted by a throwaway derivation at build time and nothing is +# committed. The module takes peer public keys as eval-time strings +# (`swarm.hives..wireguardPublicKey`), so they are read back from +# the derivation's output. +{ + pkgs, + self, +}: +let + mkKey = + name: + pkgs.runCommand "wg-hive-test-key-${name}" { nativeBuildInputs = [ pkgs.wireguard-tools ]; } '' + mkdir $out + wg genkey > $out/private + wg pubkey < $out/private > $out/public + ''; + keys = { + h1 = mkKey "h1"; + h2 = mkKey "h2"; + # Never installed on any node; stands in for a peer key that does not + # match the real peer's private key. + wrong = mkKey "wrong"; + }; + pub = name: pkgs.lib.removeSuffix "\n" (builtins.readFile "${keys.${name}}/public"); + + # The test driver numbers nodes on the test vlan in alphabetical order. + lanAddr = { + h1 = "192.168.1.1"; + h2 = "192.168.1.2"; + }; + meshAddr = { + h1 = "10.100.0.1"; + h2 = "10.100.0.2"; + }; + + swarmHives = pkgs.lib.mapAttrs (name: _: { + domain = "${name}.t.local"; + wireguardPublicKey = pub name; + wireguardAddress = "${meshAddr.${name}}/32"; + wireguardEndpoint = "${lanAddr.${name}}:51820"; + }) lanAddr; + + node = name: { + imports = [ self.nixosModules.default ]; + services.hyperhive = { + swarm.domain = "t.local"; + swarm.hives = swarmHives; + deploy.hive-controller.hiveName = name; + deploy.wireguard = { + enable = true; + address = "${meshAddr.${name}}/24"; + privateKeyFile = "${keys.${name}}/private"; + persistentKeepalive = 1; + }; + }; + # The listener the TCP check connects to, open on the mesh only. + networking.firewall.interfaces.wg-hive.allowedTCPPorts = [ 9000 ]; + environment.systemPackages = [ + pkgs.wireguard-tools + pkgs.python3 + pkgs.curl + ]; + }; +in +pkgs.testers.runNixOSTest { + name = "wg-hive"; + + nodes = { + h1 = node "h1"; + h2 = node "h2"; + }; + + testScript = '' + start_all() + for m in (h1, h2): + m.wait_for_unit("wireguard-wg-hive.service") + m.succeed("ip link show wg-hive") + + # Traffic makes the tunnel handshake. + h1.succeed("ping -c 3 -W 5 ${meshAddr.h2}") + h2.succeed("ping -c 3 -W 5 ${meshAddr.h1}") + for m in (h1, h2): + stamps = m.succeed("wg show wg-hive latest-handshakes").split() + assert len(stamps) == 2 and int(stamps[1]) > 0, f"no handshake: {stamps}" + + # The peer's mesh address is routed through the tunnel, its plain + # address is not. + via_mesh = h1.succeed("ip route get ${meshAddr.h2}") + assert "dev wg-hive" in via_mesh, via_mesh + via_lan = h1.succeed("ip route get ${lanAddr.h2}") + assert "dev wg-hive" not in via_lan, via_lan + + # A TCP connect to a listener on the peer's mesh address, and the + # tunnel's own counters moved. + h2.succeed( + "systemd-run --unit=wg-listener ${pkgs.python3}/bin/python3" + " -m http.server 9000 --bind ${meshAddr.h2}" + ) + h2.wait_for_open_port(9000, "${meshAddr.h2}") + h1.succeed("${pkgs.curl}/bin/curl -sf --max-time 5 http://${meshAddr.h2}:9000/") + transfer = h1.succeed("wg show wg-hive transfer").split() + assert int(transfer[1]) > 0 and int(transfer[2]) > 0, transfer + + # Negative control: h2 swaps h1's public key for one that does not match + # h1's private key. Same endpoint, same address, so the only difference + # from the working peer above is the key, and no handshake may happen. + h2.succeed("wg set wg-hive peer ${pub "h1"} remove") + h2.succeed( + "wg set wg-hive peer ${pub "wrong"} allowed-ips ${meshAddr.h1}/32" + " endpoint ${lanAddr.h1}:51820 persistent-keepalive 1" + ) + h2.fail("ping -c 3 -W 2 ${meshAddr.h1}") + stamps = h2.succeed("wg show wg-hive latest-handshakes").split() + assert stamps == ["${pub "wrong"}", "0"], f"handshake with wrong key: {stamps}" + ''; +}