add two-vm wg-hive mesh test to vmTests

docs/scheduler/ci.md

@@ -120,7 +120,9 @@ by hand, start the workflow from the forge UI (Actions, vm-nightly, Run
 workflow, optional `test` input), or run
 `nix build .#vmTests.x86_64-linux.<name>` on a host with KVM. Current tests:
 `agent-ephemeral-rootfs` (the switch of an agent container to an ephemeral
-root, through the real hive-priv unit) and `hive-squid`.
+root, through the real hive-priv unit), `hive-squid`, and `wg-hive` (two hives
+peered over the `wg-hive` mesh: handshake, traffic, and a wrong-key peer that
+must not handshake).
 
 ## Configuration reference
 

flake.nix

@@ -231,7 +231,7 @@
       vmTests = forAllSystems (
         { pkgs, system, ... }:
         import ./nix/vm-tests.nix {
-          inherit pkgs;
+          inherit pkgs self;
           hivePriv = self.packages.${system}.hive-priv;
         }
       );

nix/vm-tests.nix

@@ -1,8 +1,13 @@
 # NixOS VM tests: one derivation per test, exposed as `vmTests.<system>.<name>`.
 # Kept out of `checks` because each boots VMs and needs KVM; the nightly
 # workflow `.forgejo/workflows/vm-nightly.yml` builds them.
-{ pkgs, hivePriv }:
+{
+  pkgs,
+  hivePriv,
+  self,
+}:
 {
   agent-ephemeral-rootfs = import ./vm-tests/agent-ephemeral-rootfs.nix { inherit pkgs hivePriv; };
+  wg-hive = import ./vm-tests/wg-hive.nix { inherit pkgs self; };
   hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs; };
 }

nix/vm-tests/wg-hive.nix

@@ -0,0 +1,120 @@
+# `vmTests.<system>.wg-hive`: two hives peered over `wg-hive`, with real traffic across the tunnel.
+# Unlike ../module-eval, which only reads the rendered `networking.wireguard`
+# options, this boots both hosts and sends packets.
+#
+# Keys are minted by a throwaway derivation at build time and nothing is
+# committed. The module takes peer public keys as eval-time strings
+# (`swarm.hives.<name>.wireguardPublicKey`), so they are read back from
+# the derivation's output.
+{
+  pkgs,
+  self,
+}:
+let
+  mkKey =
+    name:
+    pkgs.runCommand "wg-hive-test-key-${name}" { nativeBuildInputs = [ pkgs.wireguard-tools ]; } ''
+      mkdir $out
+      wg genkey > $out/private
+      wg pubkey < $out/private > $out/public
+    '';
+  keys = {
+    h1 = mkKey "h1";
+    h2 = mkKey "h2";
+    # Never installed on any node; stands in for a peer key that does not
+    # match the real peer's private key.
+    wrong = mkKey "wrong";
+  };
+  pub = name: pkgs.lib.removeSuffix "\n" (builtins.readFile "${keys.${name}}/public");
+
+  # The test driver numbers nodes on the test vlan in alphabetical order.
+  lanAddr = {
+    h1 = "192.168.1.1";
+    h2 = "192.168.1.2";
+  };
+  meshAddr = {
+    h1 = "10.100.0.1";
+    h2 = "10.100.0.2";
+  };
+
+  swarmHives = pkgs.lib.mapAttrs (name: _: {
+    domain = "${name}.t.local";
+    wireguardPublicKey = pub name;
+    wireguardAddress = "${meshAddr.${name}}/32";
+    wireguardEndpoint = "${lanAddr.${name}}:51820";
+  }) lanAddr;
+
+  node = name: {
+    imports = [ self.nixosModules.default ];
+    services.hyperhive = {
+      swarm.domain = "t.local";
+      swarm.hives = swarmHives;
+      deploy.hive-controller.hiveName = name;
+      deploy.wireguard = {
+        enable = true;
+        address = "${meshAddr.${name}}/24";
+        privateKeyFile = "${keys.${name}}/private";
+        persistentKeepalive = 1;
+      };
+    };
+    # The listener the TCP check connects to, open on the mesh only.
+    networking.firewall.interfaces.wg-hive.allowedTCPPorts = [ 9000 ];
+    environment.systemPackages = [
+      pkgs.wireguard-tools
+      pkgs.python3
+      pkgs.curl
+    ];
+  };
+in
+pkgs.testers.runNixOSTest {
+  name = "wg-hive";
+
+  nodes = {
+    h1 = node "h1";
+    h2 = node "h2";
+  };
+
+  testScript = ''
+    start_all()
+    for m in (h1, h2):
+        m.wait_for_unit("wireguard-wg-hive.service")
+        m.succeed("ip link show wg-hive")
+
+    # Traffic makes the tunnel handshake.
+    h1.succeed("ping -c 3 -W 5 ${meshAddr.h2}")
+    h2.succeed("ping -c 3 -W 5 ${meshAddr.h1}")
+    for m in (h1, h2):
+        stamps = m.succeed("wg show wg-hive latest-handshakes").split()
+        assert len(stamps) == 2 and int(stamps[1]) > 0, f"no handshake: {stamps}"
+
+    # The peer's mesh address is routed through the tunnel, its plain
+    # address is not.
+    via_mesh = h1.succeed("ip route get ${meshAddr.h2}")
+    assert "dev wg-hive" in via_mesh, via_mesh
+    via_lan = h1.succeed("ip route get ${lanAddr.h2}")
+    assert "dev wg-hive" not in via_lan, via_lan
+
+    # A TCP connect to a listener on the peer's mesh address, and the
+    # tunnel's own counters moved.
+    h2.succeed(
+        "systemd-run --unit=wg-listener ${pkgs.python3}/bin/python3"
+        " -m http.server 9000 --bind ${meshAddr.h2}"
+    )
+    h2.wait_for_open_port(9000, "${meshAddr.h2}")
+    h1.succeed("${pkgs.curl}/bin/curl -sf --max-time 5 http://${meshAddr.h2}:9000/")
+    transfer = h1.succeed("wg show wg-hive transfer").split()
+    assert int(transfer[1]) > 0 and int(transfer[2]) > 0, transfer
+
+    # Negative control: h2 swaps h1's public key for one that does not match
+    # h1's private key. Same endpoint, same address, so the only difference
+    # from the working peer above is the key, and no handshake may happen.
+    h2.succeed("wg set wg-hive peer ${pub "h1"} remove")
+    h2.succeed(
+        "wg set wg-hive peer ${pub "wrong"} allowed-ips ${meshAddr.h1}/32"
+        " endpoint ${lanAddr.h1}:51820 persistent-keepalive 1"
+    )
+    h2.fail("ping -c 3 -W 2 ${meshAddr.h1}")
+    stamps = h2.succeed("wg show wg-hive latest-handshakes").split()
+    assert stamps == ["${pub "wrong"}", "0"], f"handshake with wrong key: {stamps}"
+  '';
+}