# `vmTests..hive-egress-antispoof`: the source check of # ../host-modules/hive-egress.nix on a real agent bridge, with two nspawn # containers on it that take their addresses from the hive's dnsmasq # (../host-modules/hive-gateway/dnsmasq.nix' lease script, the real # `hive-priv egress-load`, `egress-pin` and `egress-lease`). `hive-squid` # cannot host this: its stand-in bridge is one NIC, so every agent shares one # port. # # What it pins: # - the default verdict is `drop`, and each container takes its first lease # (DHCP from `0.0.0.0`, the reply, the first ARP and IP traffic) with it in # place; each port is bound to its container's MAC and each lease is in the # table, without the test adding either; # - a container sending from its own address reaches the host and the other # container (the control for every drop below); # - another container's IPv4 address, another container's MAC, an IPv6 address # that is not link-local and a DHCP message with another's `chaddr` are # dropped, and each leaves a `hive-egress-spoof ` line that names the port; # - a container that keeps talking across renewals of a 2-minute lease is # never dropped, and its pair is still in `lease_v4` after the first lease # and its margin would have ended; # - a DHCP renewal, a link-local IPv6 packet and a restored MAC go through; # - a pair deleted by hand comes back at the next renewal and rebind; a DHCP # client message with the port's chaddr passes from any source address, to # the broadcast or bridge address only; a two-hour wall-clock step keeps # the pairs dnsmasq prunes; a release removes its pair; # - a port with no bound MAC sends nothing, and the timer binds it again; # - `antiSpoof = "log"`, reached by a reload of the unit, brings the bindings # and the leases with the new table (nothing is dropped in between), # lets the same spoofs through and logs them. { pkgs, hivePriv }: let bridge = "hive0"; gateway = "10.42.0.1"; # A DHCP DISCOVER by hand, with the `chaddr` given as the first argument, # unicast to dnsmasq. A second argument names another destination, a third # a source address, and a message with a source address leaves from port 68 # like a client's. The container shares the host's store, so it runs from # there. bootp = pkgs.writeText "bootp.pl" '' use IO::Socket::INET; use Socket; my $mac = pack("H12", join("", split(/:/, $ARGV[0]))); my ($dest, $src) = ($ARGV[1] || "${gateway}", $ARGV[2]); my @bind = $src ? (LocalAddr => $src, LocalPort => 68, ReuseAddr => 1) : (); my $s = IO::Socket::INET->new(Proto => "udp", @bind) or die "socket: $!"; my $head = pack("C4 N n n a4 a4 a4 a4", 1, 1, 6, 0, 4660, 0, 0, "", "", "", ""); my $msg = $head . $mac . ("\0" x 10) . ("\0" x 192) . pack("H8", "63825363") . pack("C4", 53, 1, 1, 255); $s->send($msg, 0, pack_sockaddr_in(67, inet_aton($dest))) or die "send: $!"; ''; hiveStubs = { lib, ... }: { options.services.hyperhive = { deploy.hive-controller.package = lib.mkOption { type = lib.types.package; }; # Off: hive-priv.nix then adds only the link the pin command runs by. deploy.hive-controller.enable = lib.mkOption { type = lib.types.bool; default = false; }; swarm.forge.domain = lib.mkOption { type = lib.types.str; default = "forge.t.local"; }; network = { enable = lib.mkOption { type = lib.types.bool; default = true; }; bridgeIp = lib.mkOption { type = lib.types.str; default = gateway; }; # Set by hive-egress.nix; nothing here opens it. exposeHostPorts = lib.mkOption { type = lib.types.listOf lib.types.port; default = [ ]; }; bridgeName = lib.mkOption { type = lib.types.str; default = bridge; }; }; }; config.services.hyperhive.deploy.hive-controller.package = hivePriv; }; # A container that takes its address by DHCP, as the agents do. container = { autoStart = true; privateNetwork = true; hostBridge = bridge; config = { lib, pkgs, ... }: { system.stateVersion = "25.11"; networking.useDHCP = lib.mkForce true; networking.dhcpcd.extraConfig = "noipv4ll"; networking.dhcpcd.wait = "ipv4"; networking.firewall.allowedUDPPorts = [ 68 ]; environment.systemPackages = [ pkgs.perl pkgs.socat ]; }; }; in pkgs.testers.runNixOSTest { name = "hive-egress-antispoof"; nodes.host = { config, lib, ... }: { imports = [ hiveStubs ../host-modules/hive-egress.nix ../host-modules/hive-priv.nix ]; virtualisation.memorySize = 2048; # Explicit: the tests below are about what `drop` does. services.hyperhive.deploy.hive-controller.egress.antiSpoof = "drop"; networking.bridges.${bridge}.interfaces = [ ]; networking.interfaces.${bridge}.ipv4.addresses = [ { address = gateway; prefixLength = 24; } ]; # The test's own datagram listener, and dnsmasq's DHCP port. networking.firewall.interfaces.${bridge} = { allowedUDPPorts = [ 67 9999 ]; }; services.dnsmasq = { enable = true; resolveLocalQueries = false; settings = { interface = bridge; bind-interfaces = true; port = 0; # dnsmasq's shortest lease: the test runs through renewals in minutes. dhcp-range = "10.42.0.2,10.42.0.254,2m"; dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases"; dhcp-script = toString config.services.hyperhive.network.egress.leaseScript; script-on-renewal = true; log-dhcp = true; }; }; systemd.services.dnsmasq = { wants = [ "network-addresses-${bridge}.service" ]; after = [ "network-addresses-${bridge}.service" ]; }; environment.systemPackages = [ pkgs.socat ]; containers.a = container; containers.b = container; specialisation.logmode.configuration = { services.hyperhive.deploy.hive-controller.egress.antiSpoof = lib.mkForce "log"; }; }; testScript = '' import re import shlex TABLE = "bridge hive_egress" def ct(name, cmd): return host.succeed(f"nixos-container run {name} -- sh -c {shlex.quote(cmd)}").strip() def ct_ok(name, cmd): return host.execute(f"nixos-container run {name} -- sh -c {shlex.quote(cmd)}")[0] == 0 def mac(name): return ct(name, "cat /sys/class/net/eth0/address") def ipv4(name): out = ct(name, "ip -4 -o addr show dev eth0 scope global") found = re.search(r"inet (10\.42\.0\.\d+)/", out) assert found, out return found.group(1) def spoof_lines(pattern=""): out = host.succeed("journalctl -k -o cat --no-pager | grep 'hive-egress-spoof ' || true") return [line for line in out.splitlines() if re.search(pattern, line)] def udp_reaches_host(name, src, v6=False): """One datagram from the container, sent from `src`. True when the host got it.""" host.succeed("rm -f /tmp/rx") host.succeed( "systemd-run --unit=rx --collect " f"timeout 6 sh -c 'socat -u UDP{6 if v6 else 4}-RECV:9999 - > /tmp/rx'" ) host.sleep(1) if v6: ll = host.succeed("ip -6 -o addr show dev ${bridge} scope link").split()[3].split("/")[0] target = f"UDP6-SENDTO:[{ll}%eth0]:9999,bind=[{src}]" else: target = f"UDP4-SENDTO:${gateway}:9999,bind={src}" ct_ok(name, f"echo hi | socat -u - {target}") host.sleep(1) got = host.succeed("cat /tmp/rx").strip() == "hi" host.execute("systemctl stop rx.service") return got def acks_of_cmd(mac_addr): return "$(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPACK(${bridge}) .* " + mac_addr + "')" def acks_of(mac_addr): return int(host.succeed("echo " + acks_of_cmd(mac_addr)).strip()) def stale_dels(): return host.succeed("journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del .* skipped (stale)' || true").strip() host.start() host.wait_for_unit("multi-user.target") host.wait_for_unit("dnsmasq.service") host.wait_for_unit("hive-egress.service") # `drop` is set explicitly on the host above, loaded before either container # exists: the first lease of each container below is taken with it in place. assert "counter drop" in host.succeed(f"nft list chain {TABLE} spoof"), "antiSpoof = drop is loaded" for name in ("a", "b"): host.wait_for_unit(f"container@{name}.service") host.wait_until_succeeds( f"nixos-container run {name} -- sh -c 'ip -4 -o addr show dev eth0 | grep -q \"inet 10.42.0.\"'" ) MAC_A, MAC_B = mac("a"), mac("b") IP_A, IP_B = ipv4("a"), ipv4("b") assert MAC_A != MAC_B and IP_A != IP_B T0 = int(host.succeed("date +%s").strip()) with subtest("each port is bound to its container's MAC and each lease is loaded, with no help from the test"): host.wait_until_succeeds(f"nft list set {TABLE} port_mac | grep -q '\"vb-a\" . {MAC_A}'") host.wait_until_succeeds(f"nft list set {TABLE} port_mac | grep -q '\"vb-b\" . {MAC_B}'") host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'") host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'") assert not spoof_lines(), spoof_lines() with subtest("a container sending from its own address reaches the host and the other container"): assert ct_ok("a", "ping -c1 -W2 ${gateway}") assert ct_ok("a", f"ping -c1 -W2 {IP_B}") assert ct_ok("b", f"ping -c1 -W2 {IP_A}") assert udp_reaches_host("a", IP_A), "control: a datagram from a's own address arrives" assert not spoof_lines(), spoof_lines() with subtest("a container that keeps talking across renewals is never dropped, and its pair outlives the first lease and its margin"): # The leases run 2 minutes and the pair outlives its lease by 300 # seconds (LEASE_MARGIN_SECS in hive-priv's egress.rs). The pairs were # loaded before T0, so `T0 + 420` is the latest the first load alone # would have kept them; the loop runs past it, and only the refresh at # each renewal keeps them in the set. def now(): return int(host.succeed("date +%s").strip()) def ack_count(ip): return host.succeed("journalctl -u dnsmasq -o cat --no-pager").count("DHCPACK(${bridge}) " + ip) while now() < T0 + 450: assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "a's traffic is dropped while it renews" assert ct_ok("b", "ping -c1 -W2 ${gateway}"), "b's traffic is dropped while it renews" host.sleep(15) assert not spoof_lines(), spoof_lines() assert ack_count(IP_A) >= 4 and ack_count(IP_B) >= 4, "each container renewed several times" host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'") host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'") assert udp_reaches_host("a", IP_A), "a's own address still reaches the host after its first lease ended" with subtest("another container's IPv4 address is dropped, and the line names the port"): ct("a", f"ip addr add {IP_B}/24 dev eth0") assert not udp_reaches_host("a", IP_B), "a datagram from b's address must not arrive" host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*SRC=" + IP_B.replace(".", "\\.") + " '") assert not spoof_lines(r"IN=vb-b "), "b sent nothing wrong" ct("a", f"ip addr del {IP_B}/24 dev eth0") assert udp_reaches_host("a", IP_A), "a's own address still works" with subtest("an IPv6 address that is not link-local is dropped, a link-local one is not"): ct("a", "ip -6 addr add fd00::bad/64 dev eth0 nodad") assert not udp_reaches_host("a", "fd00::bad", v6=True) host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*SRC=fd00:0*:0*:0*:0*:0*:0*:0*bad '") ll_a = ct("a", "ip -6 -o addr show dev eth0 scope link").split()[3].split("/")[0] assert udp_reaches_host("a", ll_a, v6=True), "control: a link-local source goes through" ct("a", "ip -6 addr del fd00::bad/64 dev eth0") with subtest("another container's MAC is dropped, and the restored MAC works again"): ct("a", f"ip link set dev eth0 address {MAC_B}") assert not ct_ok("a", "ping -c1 -W2 ${gateway}") host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*MAC=[0-9a-f:]*" + MAC_B + ":'") ct("a", f"ip link set dev eth0 address {MAC_A}") assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "the port stays bound to the MAC it had" assert ct_ok("b", "ping -c1 -W2 ${gateway}"), "b was never affected" with subtest("another container's MAC and IPv4 address together are dropped, so the leased pair alone is no credential"): # A static neighbour entry for the host, so the datagram needs no ARP: # only the check of the port's MAC is left between it and the host. host_mac = host.succeed("cat /sys/class/net/${bridge}/address").strip() ct("a", f"ip neigh replace ${gateway} lladdr {host_mac} dev eth0 nud permanent") ct("a", f"ip link set dev eth0 address {MAC_B}") ct("a", f"ip addr add {IP_B}/24 dev eth0") seen = len(spoof_lines(r"IN=vb-a ")) assert not udp_reaches_host("a", IP_B), "b's MAC and address from a's port must not arrive" assert len(spoof_lines(r"IN=vb-a ")) > seen, "and it is logged against a's port" ct("a", f"ip addr del {IP_B}/24 dev eth0") ct("a", f"ip link set dev eth0 address {MAC_A}") ct("a", "ip neigh del ${gateway} dev eth0") assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "a works again with its own pair" with subtest("a DHCP message with another container's chaddr is dropped, one with its own is answered, and a renewal works"): def discovers(mac_addr): out = host.succeed("journalctl -u dnsmasq -o cat --no-pager") return len(re.findall(r"DHCPDISCOVER\(${bridge}\) " + mac_addr, out)) before_b, before_a = discovers(MAC_B), discovers(MAC_A) ct("a", f"perl ${bootp} {MAC_B}") ct("a", f"perl ${bootp} {MAC_A}") host.wait_until_succeeds(f"journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPDISCOVER(${bridge}) {MAC_A}' | grep -qv '^{before_a}$'") assert discovers(MAC_A) == before_a + 1, "the message with a's own chaddr reached dnsmasq" assert discovers(MAC_B) == before_b, "the message with b's chaddr did not" host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*DPT=67 '") acks = host.succeed("journalctl -u dnsmasq -o cat --no-pager").count("DHCPACK(${bridge}) " + IP_A) ct("a", "dhcpcd --rebind eth0") host.wait_until_succeeds( "test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPACK(${bridge}) " + IP_A + "') -gt " + str(acks) ) assert ipv4("a") == IP_A, "the renewal kept the address" host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'") assert ct_ok("a", "ping -c1 -W2 ${gateway}") with subtest("a port with no bound MAC sends nothing, and the timer binds it again"): host.succeed(f"nft flush set {TABLE} port_mac") assert not ct_ok("a", "ping -c1 -W2 ${gateway}") assert not ct_ok("b", "ping -c1 -W2 ${gateway}") # Nothing starts the pin unit by hand: the timer finds the unbound ports. host.wait_until_succeeds("nixos-container run a -- ping -c1 -W2 ${gateway}") host.wait_until_succeeds("nixos-container run b -- ping -c1 -W2 ${gateway}") with subtest("a pair that left the set comes back at the container's next renewal and its next rebind, and nothing is dropped"): # The renewal is a unicast REQUEST from the leased address, the rebind a # broadcast one from it. Neither can match the pair: it is gone. for verb in ("renew", "rebind"): host.succeed(f"nft delete element {TABLE} lease_v4 {{ {MAC_A} . {IP_A} }}") host.fail(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'") assert not ct_ok("a", "ping -c1 -W2 ${gateway}"), "control: a is cut off without its pair" assert ct_ok("b", "ping -c1 -W2 ${gateway}"), "control: b is not" drops_before = len(spoof_lines(r"IN=vb-a .*DPT=67 ")) acks = ack_count(IP_A) ct("a", f"dhcpcd --{verb} eth0") host.wait_until_succeeds(f"test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPACK(${bridge}) {IP_A}') -gt {acks}", timeout=30) host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=30) assert ct_ok("a", "ping -c1 -W2 ${gateway}"), f"a reaches the host again after its {verb}" assert len(spoof_lines(r"IN=vb-a .*DPT=67 ")) == drops_before, f"the {verb} was not dropped" with subtest("a DHCP client message with the port's own chaddr passes from any source address to the broadcast or bridge address only"): host_mac = host.succeed("cat /sys/class/net/${bridge}/address").strip() ct("a", f"ip neigh replace ${gateway} lladdr {host_mac} dev eth0 nud permanent") ct("a", f"ip neigh replace {IP_B} lladdr {MAC_B} dev eth0 nud permanent") ct("a", f"ip addr add {IP_B}/24 dev eth0") discovers_a = host.succeed("journalctl -u dnsmasq -o cat --no-pager").count("DHCPDISCOVER(${bridge}) " + MAC_A) ct("a", f"perl ${bootp} {MAC_A} ${gateway} {IP_B}") host.wait_until_succeeds( "test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPDISCOVER(${bridge}) " + MAC_A + "') -gt " + str(discovers_a), timeout=30, ) seen = len(spoof_lines(r"IN=vb-a .*DST=" + IP_B.replace(".", r"\.") + " .*DPT=67 ")) ct("a", f"perl ${bootp} {MAC_A} {IP_B} {IP_B}") host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*DST=" + IP_B.replace(".", "\\.") + " .*DPT=67 '", timeout=30) assert len(spoof_lines(r"IN=vb-a .*DST=" + IP_B.replace(".", r"\.") + " .*DPT=67 ")) > seen, "to another container's address it is dropped" ct("a", f"ip addr del {IP_B}/24 dev eth0") ct("a", f"ip neigh del {IP_B} dev eth0") ct("a", "ip neigh del ${gateway} dev eth0") with subtest("a step of the wall clock that makes dnsmasq prune every lease leaves the pairs in the set and the containers reaching the host"): # The qemu guest has no suspend. A step of the wall clock is the same # divergence: dnsmasq counts wall time and expires every lease, while # the nft timeouts and the containers' leases count monotonic time. assert ct_ok("a", "ping -c1 -W2 ${gateway}") and ct_ok("b", "ping -c1 -W2 ${gateway}") drops_before = len(spoof_lines(r"IN=vb-[ab] ")) skipped = int(stale_dels()) host.succeed("date -s @$(( $(date +%s) + 7200 ))") # Any DHCP packet makes dnsmasq prune: a DISCOVER with a's own chaddr. ct("a", "perl ${bootp} " + MAC_A) host.wait_until_succeeds("test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del .* skipped (stale)') -ge " + str(skipped + 2), timeout=60) host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'") host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'") assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "a reaches the host after dnsmasq forgot its lease" assert ct_ok("b", "ping -c1 -W2 ${gateway}"), "b reaches the host after dnsmasq forgot its lease" assert not host.succeed("journalctl -u dnsmasq -o cat --no-pager | grep 'egress-lease: del .* applied' || true").strip(), "no delete was applied" # Each container then renews against a dnsmasq that has forgotten its # lease, and ends with a lease again. It keeps reaching the host. acks_a, acks_b = acks_of(MAC_A), acks_of(MAC_B) for name in ("a", "b"): ct(name, "dhcpcd --renew eth0") host.wait_until_succeeds(f"test {acks_of_cmd(MAC_A)} -gt {acks_a} && test {acks_of_cmd(MAC_B)} -gt {acks_b}", timeout=90) IP_A, IP_B = ipv4("a"), ipv4("b") host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=60) host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'", timeout=60) assert len(spoof_lines(r"IN=vb-[ab] ")) == drops_before, "nothing of a or b was dropped" with subtest("a release still removes the pair at once"): # A fresh lease first: the release must name an expiry in the future. acks = ack_count(IP_A) ct("a", "dhcpcd --rebind eth0") host.wait_until_succeeds(f"test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPACK(${bridge}) {IP_A}') -gt {acks}", timeout=30) host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'") ct("a", "dhcpcd -k eth0") host.wait_until_fails(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=30) host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'") host.wait_until_succeeds("journalctl -u dnsmasq -o cat --no-pager | grep -E 'egress-lease: del mac=" + MAC_A + " .* applied'", timeout=30) # a asks again, as its container would after a restart. ct("a", "systemctl restart dhcpcd") host.wait_until_succeeds("nixos-container run a -- ping -c1 -W2 ${gateway}", timeout=60) IP_A = ipv4("a") host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=30) with subtest("antiSpoof = log keeps the bindings and the leases, lets a spoofed datagram through and logs it"): host.succeed("/run/current-system/specialisation/logmode/bin/switch-to-configuration test") assert "counter accept" in host.succeed(f"nft list chain {TABLE} spoof"), "the table was reloaded in log mode" # No waiting: the reload put the sets in with the table. host.succeed(f"nft list set {TABLE} port_mac | grep -q '\"vb-a\" . {MAC_A}'") host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'") assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "control: own address, right after the reload" seen = len(spoof_lines()) ct("a", f"ip addr add {IP_B}/24 dev eth0") assert udp_reaches_host("a", IP_B), "log mode lets the spoofed datagram through" host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep 'hive-egress-spoof ' | wc -l | grep -qv '^" + str(seen) + "$'") assert spoof_lines(r"IN=vb-a .*SRC=" + IP_B.replace(".", r"\.") + " ") ''; }