mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

egress anti-spoof: renewals from the leased address pass, a stale lease delete is skipped

Browse source
This commit is contained in:
atlas 2026-11-14 20:55:07 +01:00 • committed by mara
commit 1cef99e992
1 changed files with 305 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

126
nix/vm-tests/hive-egress-antispoof.nix
View file

@ -2,8 +2,9 @@
# ../host-modules/hive-egress.nix on a real agent bridge, with two nspawn
# containers on it that take their addresses from the hive's dnsmasq
# (../host-modules/hive-gateway/dnsmasq.nix' lease script, the real
# `hive-priv egress-load`, `egress-pin` and `egress-lease`). `hive-squid` cannot host this: its
# stand-in bridge is one NIC, so every agent shares one port.
# `hive-priv egress-load`, `egress-pin` and `egress-lease`). `hive-squid`
# cannot host this: its stand-in bridge is one NIC, so every agent shares one
# port.
#
# What it pins:
# - the default verdict is `drop`, and each container takes its first lease
@ -15,10 +16,14 @@
# - another container's IPv4 address, another container's MAC, an IPv6 address
# that is not link-local and a DHCP message with another's `chaddr` are
# dropped, and each leaves a `hive-egress-spoof ` line that names the port;
# - a container that keeps talking across several renewals of a 2-minute lease
# is never dropped, and its pair is still in `lease_v4` after the first
# lease and its margin would have ended;
# - a container that keeps talking across renewals of a 2-minute lease is
# never dropped, and its pair is still in `lease_v4` after the first lease
# and its margin would have ended;
# - a DHCP renewal, a link-local IPv6 packet and a restored MAC go through;
# - a pair deleted by hand comes back at the next renewal and rebind; a DHCP
# client message with the port's chaddr passes from any source address, to
# the broadcast or bridge address only; a two-hour wall-clock step keeps
# the pairs dnsmasq prunes; a release removes its pair;
# - a port with no bound MAC sends nothing, and the timer binds it again;
# - `antiSpoof = "log"`, reached by a reload of the unit, brings the bindings
# and the leases with the new table (nothing is dropped in between),
@ -28,16 +33,21 @@ let
bridge = "hive0";
gateway = "10.42.0.1";
# A DHCP DISCOVER by hand, unicast to dnsmasq, with the `chaddr` given as
# the argument. The container shares the host's store, so it runs from there.
# A DHCP DISCOVER by hand, with the `chaddr` given as the first argument,
# unicast to dnsmasq. A second argument names another destination, a third
# a source address, and a message with a source address leaves from port 68
# like a client's. The container shares the host's store, so it runs from
# there.
bootp = pkgs.writeText "bootp.pl" ''
use IO::Socket::INET;
use Socket;
my $mac = pack("H12", join("", split(/:/, $ARGV[0])));
my $s = IO::Socket::INET->new(Proto => "udp") or die "socket: $!";
my ($dest, $src) = ($ARGV[1] || "${gateway}", $ARGV[2]);
my @bind = $src ? (LocalAddr => $src, LocalPort => 68, ReuseAddr => 1) : ();
my $s = IO::Socket::INET->new(Proto => "udp", @bind) or die "socket: $!";
my $head = pack("C4 N n n a4 a4 a4 a4", 1, 1, 6, 0, 4660, 0, 0, "", "", "", "");
my $msg = $head . $mac . ("\0" x 10) . ("\0" x 192) . pack("H8", "63825363") . pack("C4", 53, 1, 1, 255);
$s->send($msg, 0, pack_sockaddr_in(67, inet_aton("${gateway}"))) or die "send: $!";
$s->send($msg, 0, pack_sockaddr_in(67, inet_aton($dest))) or die "send: $!";
'';
hiveStubs =
@ -59,6 +69,15 @@ let
type = lib.types.bool;
default = true;
};
bridgeIp = lib.mkOption {
type = lib.types.str;
default = gateway;
};
# Set by hive-egress.nix; nothing here opens it.
exposeHostPorts = lib.mkOption {
type = lib.types.listOf lib.types.port;
default = [ ];
};
bridgeName = lib.mkOption {
type = lib.types.str;
default = bridge;
@ -100,6 +119,8 @@ pkgs.testers.runNixOSTest {
../host-modules/hive-priv.nix
];
virtualisation.memorySize = 2048;
# Explicit: the tests below are about what `drop` does.
services.hyperhive.network.egress.antiSpoof = "drop";
networking.bridges.${bridge}.interfaces = [ ];
networking.interfaces.${bridge}.ipv4.addresses = [
{
@ -187,6 +208,15 @@ pkgs.testers.runNixOSTest {
host.execute("systemctl stop rx.service")
return got
def acks_of_cmd(mac_addr):
return "$(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPACK(${bridge}) .* " + mac_addr + "')"
def acks_of(mac_addr):
return int(host.succeed("echo " + acks_of_cmd(mac_addr)).strip())
def stale_dels():
return host.succeed("journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del .* skipped (stale)' || true").strip()
host.start()
host.wait_for_unit("multi-user.target")
host.wait_for_unit("dnsmasq.service")
@ -305,6 +335,84 @@ pkgs.testers.runNixOSTest {
host.wait_until_succeeds("nixos-container run a -- ping -c1 -W2 ${gateway}")
host.wait_until_succeeds("nixos-container run b -- ping -c1 -W2 ${gateway}")
with subtest("a pair that left the set comes back at the container's next renewal and its next rebind, and nothing is dropped"):
# The renewal is a unicast REQUEST from the leased address, the rebind a
# broadcast one from it. Neither can match the pair: it is gone.
for verb in ("renew", "rebind"):
host.succeed(f"nft delete element {TABLE} lease_v4 {{ {MAC_A} . {IP_A} }}")
host.fail(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
assert not ct_ok("a", "ping -c1 -W2 ${gateway}"), "control: a is cut off without its pair"
assert ct_ok("b", "ping -c1 -W2 ${gateway}"), "control: b is not"
drops_before = len(spoof_lines(r"IN=vb-a .*DPT=67 "))
acks = ack_count(IP_A)
ct("a", f"dhcpcd --{verb} eth0")
host.wait_until_succeeds(f"test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPACK(${bridge}) {IP_A}') -gt {acks}", timeout=30)
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=30)
assert ct_ok("a", "ping -c1 -W2 ${gateway}"), f"a reaches the host again after its {verb}"
assert len(spoof_lines(r"IN=vb-a .*DPT=67 ")) == drops_before, f"the {verb} was not dropped"
with subtest("a DHCP client message with the port's own chaddr passes from any source address to the broadcast or bridge address only"):
host_mac = host.succeed("cat /sys/class/net/${bridge}/address").strip()
ct("a", f"ip neigh replace ${gateway} lladdr {host_mac} dev eth0 nud permanent")
ct("a", f"ip neigh replace {IP_B} lladdr {MAC_B} dev eth0 nud permanent")
ct("a", f"ip addr add {IP_B}/24 dev eth0")
discovers_a = host.succeed("journalctl -u dnsmasq -o cat --no-pager").count("DHCPDISCOVER(${bridge}) " + MAC_A)
ct("a", f"perl ${bootp} {MAC_A} ${gateway} {IP_B}")
host.wait_until_succeeds(
"test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPDISCOVER(${bridge}) " + MAC_A + "') -gt " + str(discovers_a),
timeout=30,
)
seen = len(spoof_lines(r"IN=vb-a .*DST=" + IP_B.replace(".", r"\.") + " .*DPT=67 "))
ct("a", f"perl ${bootp} {MAC_A} {IP_B} {IP_B}")
host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*DST=" + IP_B.replace(".", "\\.") + " .*DPT=67 '", timeout=30)
assert len(spoof_lines(r"IN=vb-a .*DST=" + IP_B.replace(".", r"\.") + " .*DPT=67 ")) > seen, "to another container's address it is dropped"
ct("a", f"ip addr del {IP_B}/24 dev eth0")
ct("a", f"ip neigh del {IP_B} dev eth0")
ct("a", "ip neigh del ${gateway} dev eth0")
with subtest("a step of the wall clock that makes dnsmasq prune every lease leaves the pairs in the set and the containers reaching the host"):
# The qemu guest has no suspend. A step of the wall clock is the same
# divergence: dnsmasq counts wall time and expires every lease, while
# the nft timeouts and the containers' leases count monotonic time.
assert ct_ok("a", "ping -c1 -W2 ${gateway}") and ct_ok("b", "ping -c1 -W2 ${gateway}")
drops_before = len(spoof_lines(r"IN=vb-[ab] "))
skipped = int(stale_dels())
host.succeed("date -s @$(( $(date +%s) + 7200 ))")
# Any DHCP packet makes dnsmasq prune: a DISCOVER with a's own chaddr.
ct("a", "perl ${bootp} " + MAC_A)
host.wait_until_succeeds("test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del .* skipped (stale)') -ge " + str(skipped + 2), timeout=60)
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "a reaches the host after dnsmasq forgot its lease"
assert ct_ok("b", "ping -c1 -W2 ${gateway}"), "b reaches the host after dnsmasq forgot its lease"
assert not host.succeed("journalctl -u dnsmasq -o cat --no-pager | grep 'egress-lease: del .* applied' || true").strip(), "no delete was applied"
# Each container then renews against a dnsmasq that has forgotten its
# lease, and ends with a lease again. It keeps reaching the host.
acks_a, acks_b = acks_of(MAC_A), acks_of(MAC_B)
for name in ("a", "b"):
ct(name, "dhcpcd --renew eth0")
host.wait_until_succeeds(f"test {acks_of_cmd(MAC_A)} -gt {acks_a} && test {acks_of_cmd(MAC_B)} -gt {acks_b}", timeout=90)
IP_A, IP_B = ipv4("a"), ipv4("b")
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=60)
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'", timeout=60)
assert len(spoof_lines(r"IN=vb-[ab] ")) == drops_before, "nothing of a or b was dropped"
with subtest("a release still removes the pair at once"):
# A fresh lease first: the release must name an expiry in the future.
acks = ack_count(IP_A)
ct("a", "dhcpcd --rebind eth0")
host.wait_until_succeeds(f"test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPACK(${bridge}) {IP_A}') -gt {acks}", timeout=30)
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
ct("a", "dhcpcd -k eth0")
host.wait_until_fails(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=30)
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
host.wait_until_succeeds("journalctl -u dnsmasq -o cat --no-pager | grep -E 'egress-lease: del mac=" + MAC_A + " .* applied'", timeout=30)
# a asks again, as its container would after a restart.
ct("a", "systemctl restart dhcpcd")
host.wait_until_succeeds("nixos-container run a -- ping -c1 -W2 ${gateway}", timeout=60)
IP_A = ipv4("a")
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=30)
with subtest("antiSpoof = log keeps the bindings and the leases, lets a spoofed datagram through and logs it"):
host.succeed("/run/current-system/specialisation/logmode/bin/switch-to-configuration test")
assert "counter accept" in host.succeed(f"nft list chain {TABLE} spoof"), "the table was reloaded in log mode"