Watch
0
0
Fork
You've already forked hyperhive
0
hyperhive/nix/vm-tests/hive-egress-antispoof.nix

559 lines
30 KiB
Nix

# `vmTests.<system>.hive-egress-antispoof`: the source check of
# ../host-modules/hive-egress.nix on a real agent bridge, with two nspawn
# containers on it that take their addresses from the hive's dnsmasq
# (../host-modules/hive-gateway/dnsmasq.nix' lease script, the real
# `hive-priv egress-load`, `egress-pin` and `egress-lease`). `hive-squid`
# cannot host this: its stand-in bridge is one NIC, so every agent shares one
# port.
#
# What it pins:
# - the verdict is `drop` (set here; the module default is `log`), and each
# container takes its first lease (DHCP from `0.0.0.0`, the reply, the first
# ARP and IP traffic) with it in place; each port is bound to its
# container's MAC and each lease is in the table, unaided by the test;
# - a container sending from its own address reaches the host and the other
# container (the control for every drop below);
# - another container's IPv4 address, another container's MAC, an IPv6 address
# that is not link-local and a DHCP message with another's `chaddr` are
# dropped, and each leaves a `hive-egress-spoof ` line that names the port;
# - a container that keeps talking across renewals of a 2-minute lease is
# never dropped, and its pair is still in `lease_v4` after the first lease
# and its margin would have ended;
# - a DHCP renewal, a link-local IPv6 packet and a restored MAC go through;
# - a pair deleted by hand comes back at the next renewal and rebind; a DHCP
# client message with the port's chaddr passes from any source address, to
# the broadcast or bridge address only; a two-hour wall-clock step keeps
# the pairs dnsmasq prunes; a release removes its pair;
# - a port with no bound MAC sends nothing, and the timer binds it again;
# - `antiSpoof = "log"`, reached by a reload of the unit, brings the bindings
# and the leases with the new table (nothing is dropped in between),
# lets the same spoofs through and logs them.
{ pkgs, hivePriv }:
let
bridge = "hive0";
gateway = "10.42.0.1";
# A DHCP DISCOVER by hand, with the `chaddr` given as the first argument,
# unicast to dnsmasq. A second argument names another destination, a third
# a source address, and a message with a source address leaves from port 68
# like a client's. The container shares the host's store, so it runs from
# there.
bootp = pkgs.writeText "bootp.pl" ''
use IO::Socket::INET;
use Socket;
my $mac = pack("H12", join("", split(/:/, $ARGV[0])));
my ($dest, $src) = ($ARGV[1] || "${gateway}", $ARGV[2]);
my @bind = $src ? (LocalAddr => $src, LocalPort => 68, ReuseAddr => 1) : ();
my $s = IO::Socket::INET->new(Proto => "udp", @bind) or die "socket: $!";
my $head = pack("C4 N n n a4 a4 a4 a4", 1, 1, 6, 0, 4660, 0, 0, "", "", "", "");
my $msg = $head . $mac . ("\0" x 10) . ("\0" x 192) . pack("H8", "63825363") . pack("C4", 53, 1, 1, 255);
$s->send($msg, 0, pack_sockaddr_in(67, inet_aton($dest))) or die "send: $!";
'';
hiveStubs =
{ lib, ... }:
{
options.services.hyperhive = {
deploy.hive-controller.package = lib.mkOption { type = lib.types.package; };
# Off: hive-priv.nix then adds only the link the pin command runs by.
deploy.hive-controller.enable = lib.mkOption {
type = lib.types.bool;
default = false;
};
swarm.forge.domain = lib.mkOption {
type = lib.types.str;
default = "forge.t.local";
};
network = {
enable = lib.mkOption {
type = lib.types.bool;
default = true;
};
bridgeIp = lib.mkOption {
type = lib.types.str;
default = gateway;
};
# Set by hive-egress.nix; nothing here opens it.
exposeHostPorts = lib.mkOption {
type = lib.types.listOf lib.types.port;
default = [ ];
};
bridgeName = lib.mkOption {
type = lib.types.str;
default = bridge;
};
};
};
config.services.hyperhive.deploy.hive-controller.package = hivePriv;
};
# A container that takes its address by DHCP, as the agents do.
container = {
autoStart = true;
privateNetwork = true;
hostBridge = bridge;
config =
{ lib, pkgs, ... }:
{
system.stateVersion = "25.11";
networking.useDHCP = lib.mkForce true;
networking.dhcpcd.extraConfig = "noipv4ll";
networking.dhcpcd.wait = "ipv4";
networking.firewall.allowedUDPPorts = [ 68 ];
environment.systemPackages = [
pkgs.perl
pkgs.socat
];
};
};
in
pkgs.testers.runNixOSTest {
name = "hive-egress-antispoof";
nodes.host =
{ config, lib, ... }:
{
imports = [
hiveStubs
../host-modules/hive-egress.nix
../host-modules/hive-priv.nix
];
virtualisation.memorySize = 2048;
# Explicit: the tests below are about what `drop` does.
services.hyperhive.deploy.hive-controller.egress.antiSpoof = "drop";
networking.bridges.${bridge}.interfaces = [ ];
networking.interfaces.${bridge}.ipv4.addresses = [
{
address = gateway;
prefixLength = 24;
}
];
# The test's own datagram listener, and dnsmasq's DHCP port.
networking.firewall.interfaces.${bridge} = {
allowedUDPPorts = [
67
9999
];
};
services.dnsmasq = {
enable = true;
resolveLocalQueries = false;
settings = {
interface = bridge;
bind-interfaces = true;
port = 0;
# dnsmasq's shortest lease: the test runs through renewals in minutes.
dhcp-range = "10.42.0.2,10.42.0.254,2m";
dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases";
dhcp-script = toString config.services.hyperhive.network.egress.leaseScript;
script-on-renewal = true;
log-dhcp = true;
};
};
systemd.services.dnsmasq = {
wants = [ "network-addresses-${bridge}.service" ];
after = [ "network-addresses-${bridge}.service" ];
};
# `nft` for the test's own reads of the tables; the units run it by store path.
environment.systemPackages = [
pkgs.socat
pkgs.nftables
];
containers.a = container;
containers.b = container;
specialisation.logmode.configuration = {
services.hyperhive.deploy.hive-controller.egress.antiSpoof = lib.mkForce "log";
};
};
testScript = ''
import ipaddress
import json
import re
import shlex
TABLE = "bridge hive_egress"
def ct(name, cmd):
return host.succeed(f"nixos-container run {name} -- sh -c {shlex.quote(cmd)}").strip()
def ct_ok(name, cmd):
return host.execute(f"nixos-container run {name} -- sh -c {shlex.quote(cmd)}")[0] == 0
def mac(name):
return ct(name, "cat /sys/class/net/eth0/address")
def ipv4(name):
out = ct(name, "ip -4 -o addr show dev eth0 scope global")
found = re.search(r"inet (10\.42\.0\.\d+)/", out)
assert found, out
return found.group(1)
# A port with no bound MAC sends nothing, so a container's first frames after
# start (MLD reports from `::`) can be refused before its port is pinned and
# for a moment after. Spoof lines of a port up to this long after its pin
# line are start-up refusals and are not counted; the forge issue "a
# container's first frames are sometimes refused by the anti-spoof MAC check
# right after its port is reported bound" tracks them.
PIN_SETTLE_US = 2_000_000
# port -> journal monotonic stamp (µs) up to which its spoof lines are start-up refusals
QUIET_AFTER_US = {}
def journal_entries(*args):
"""(monotonic µs, message) of each journal entry, one boot, kernel and userspace on one clock."""
out = host.succeed("journalctl --no-pager -o json " + " ".join(args))
entries = [json.loads(line) for line in out.splitlines()]
return [(int(e["__MONOTONIC_TIMESTAMP"]), e["MESSAGE"]) for e in entries if isinstance(e.get("MESSAGE"), str)]
def startup_refusal(stamp, message):
found = re.search(r"IN=(\S+)", message)
return found is not None and stamp < QUIET_AFTER_US.get(found.group(1), 0)
def spoof_entries():
return [(t, m) for t, m in journal_entries("-k") if "hive-egress-spoof " in m]
def wait_ipv4(name):
"""`name`'s address once dhcpcd has assigned it. An ACK counted in the journal comes first: dhcpcd probes the offered address for seconds before it takes it."""
host.wait_until_succeeds(
f"nixos-container run {name} -- sh -c 'ip -4 -o addr show dev eth0 scope global | grep -q \"inet 10.42.0.\"'",
timeout=60,
)
return ipv4(name)
def spoof_lines(pattern=""):
return [m for t, m in spoof_entries() if re.search(pattern, m) and not startup_refusal(t, m)]
def udp_reaches_host(name, src, v6=False):
"""One datagram from the container, sent from `src`. True when the host got it.
The kernel refuses to bind a link-local source without a scope id, so one gets `%eth0`.
When nothing arrives, the sender's exit status and output and the container's neighbour table are printed.
"""
host.succeed("rm -f /tmp/rx")
host.succeed(
"systemd-run --unit=rx --collect "
f"${pkgs.coreutils}/bin/timeout 6 ${pkgs.runtimeShell} -c '${pkgs.socat}/bin/socat -u UDP{6 if v6 else 4}-RECV:9999 - > /tmp/rx'"
)
host.sleep(1)
if v6:
ll = host.succeed("ip -6 -o addr show dev ${bridge} scope link").split()[3].split("/")[0]
bound = f"{src}%eth0" if ipaddress.ip_address(src).is_link_local else src
target = f"UDP6-SENDTO:[{ll}%eth0]:9999,bind=[{bound}]"
else:
target = f"UDP4-SENDTO:${gateway}:9999,bind={src}"
sent_status, sent = host.execute(
f"nixos-container run {name} -- sh -c {shlex.quote(f'echo hi | socat -u - {target} 2>&1')}"
)
host.sleep(1)
got = host.succeed("cat /tmp/rx").strip() == "hi"
if not got:
neighbours = host.execute(f"nixos-container run {name} -- ip neigh show dev eth0")[1]
print(
f"udp_reaches_host({name}, {src}): nothing arrived; sender exit {sent_status}, "
f"sender output {sent.strip()!r}; {name}'s neighbours:\n{neighbours.strip()}"
)
host.execute("systemctl stop rx.service")
return got
def spoof_verdict():
"""The verdict of the spoof chain's last rule, the one every packet that gets past the log rules reaches."""
out = host.succeed(f"nft list chain {TABLE} spoof")
found = re.findall(r"counter packets \d+ bytes \d+ (drop|accept)$", out, re.M)
assert found, out
return found[-1]
def own_pair_again():
"""a's address once it reaches the host with its own MAC again, and its pair is loaded.
While a held another container's MAC its DHCP client declined its
lease and asked again, so the address it ends with may not be the one
it started with. a's own neighbour entry for the gateway is flushed:
its ARP requests were refused meanwhile, so the entry may still be
failed. The host never saw those frames and has nothing to flush.
"""
ct("a", "ip -4 neigh flush dev eth0")
host.wait_until_succeeds("nixos-container run a -- ping -c1 -W2 ${gateway}", timeout=60)
ip = ipv4("a")
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {ip}'", timeout=60)
return ip
def acks_of_cmd(mac_addr):
return "$(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPACK(${bridge}) .* " + mac_addr + "')"
def acks_of(mac_addr):
return int(host.succeed("echo " + acks_of_cmd(mac_addr)).strip())
def stale_dels():
return host.succeed("journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del .* skipped (stale)' || true").strip()
def applied_dels(who=""):
"""How many deletes the lease script applied so far, all or `mac=<mac> ` only.
A count is taken before the step under test and compared after: the
journal holds the lines of every earlier subtest.
"""
return int(host.succeed(f"journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del {who}.* applied' || true").strip())
host.start()
host.wait_for_unit("multi-user.target")
host.wait_for_unit("dnsmasq.service")
host.wait_for_unit("hive-egress.service")
# `drop` is set explicitly on the host above, loaded before either container
# exists: the first lease of each container below is taken with it in place.
assert spoof_verdict() == "drop", "antiSpoof = drop is loaded"
for name in ("a", "b"):
host.wait_for_unit(f"container@{name}.service")
host.wait_until_succeeds(
f"nixos-container run {name} -- sh -c 'ip -4 -o addr show dev eth0 | grep -q \"inet 10.42.0.\"'"
)
MAC_A, MAC_B = mac("a"), mac("b")
IP_A, IP_B = ipv4("a"), ipv4("b")
assert MAC_A != MAC_B and IP_A != IP_B
T0 = int(host.succeed("date +%s").strip())
with subtest("each port is bound to its container's MAC and each lease is loaded, with no help from the test"):
host.wait_until_succeeds(f"nft list set {TABLE} port_mac | grep -q '\"vb-a\" . {MAC_A}'")
host.wait_until_succeeds(f"nft list set {TABLE} port_mac | grep -q '\"vb-b\" . {MAC_B}'")
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
# The pin line is printed after the set changes, so wait for it too.
for port, mac_addr in (("vb-a", MAC_A), ("vb-b", MAC_B)):
host.wait_until_succeeds(f"journalctl -b -o cat --no-pager | grep -q 'egress-pin: {port} bound to {mac_addr} '")
journal = journal_entries("-b")
for port, mac_addr in (("vb-a", MAC_A), ("vb-b", MAC_B)):
stamps = [t for t, m in journal if m.startswith(f"egress-pin: {port} bound to {mac_addr} ")]
assert stamps, f"no pin line for {port}"
QUIET_AFTER_US[port] = min(stamps) + PIN_SETTLE_US
for stamp, message in spoof_entries():
if startup_refusal(stamp, message):
found = re.search(r"IN=(\S+)", message)
assert found is not None, message
port = found.group(1)
print(f"tolerated start-up refusal, {(stamp - QUIET_AFTER_US[port]) / 1e6:+.3f}s from the end of {port}'s settle: {message}")
assert not spoof_lines(), spoof_lines()
with subtest("a container sending from its own address reaches the host and the other container"):
assert ct_ok("a", "ping -c1 -W2 ${gateway}")
assert ct_ok("a", f"ping -c1 -W2 {IP_B}")
assert ct_ok("b", f"ping -c1 -W2 {IP_A}")
assert udp_reaches_host("a", IP_A), "control: a datagram from a's own address arrives"
assert not spoof_lines(), spoof_lines()
with subtest("a container that keeps talking across renewals is never dropped, and its pair outlives the first lease and its margin"):
# The leases run 2 minutes and the pair outlives its lease by 300
# seconds (LEASE_MARGIN_SECS in hive-priv's egress.rs). The pairs were
# loaded before T0, so `T0 + 420` is the latest the first load alone
# would have kept them; the loop runs past it, and only the refresh at
# each renewal keeps them in the set.
def now():
return int(host.succeed("date +%s").strip())
def ack_count(ip):
return host.succeed("journalctl -u dnsmasq -o cat --no-pager").count("DHCPACK(${bridge}) " + ip)
while now() < T0 + 450:
assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "a's traffic is dropped while it renews"
assert ct_ok("b", "ping -c1 -W2 ${gateway}"), "b's traffic is dropped while it renews"
host.sleep(15)
assert not spoof_lines(), spoof_lines()
assert ack_count(IP_A) >= 4 and ack_count(IP_B) >= 4, "each container renewed several times"
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
assert udp_reaches_host("a", IP_A), "a's own address still reaches the host after its first lease ended"
with subtest("another container's IPv4 address is dropped, and the line names the port"):
ct("a", f"ip addr add {IP_B}/24 dev eth0")
assert not udp_reaches_host("a", IP_B), "a datagram from b's address must not arrive"
host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*SRC=" + IP_B.replace(".", "\\.") + " '")
assert not spoof_lines(r"IN=vb-b "), "b sent nothing wrong"
ct("a", f"ip addr del {IP_B}/24 dev eth0")
# Resolve the host again from a's own address, so the control does not
# ride on an ARP entry the spoofed source touched.
ct("a", "ip -4 neigh flush dev eth0")
ct_ok("a", "ping -c1 -W2 ${gateway}")
assert udp_reaches_host("a", IP_A), "a's own address still works"
with subtest("an IPv6 address that is not link-local is dropped, a link-local one is not"):
ct("a", "ip -6 addr add fd00::bad/64 dev eth0 nodad")
assert not udp_reaches_host("a", "fd00::bad", v6=True)
host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*SRC=fd00:0*:0*:0*:0*:0*:0*:0*bad '")
# The kernel resolves the host's link-local address with neighbour
# solicits from the datagram's source, which the table drops for
# fd00::bad. Take that source away and resolve the host again from a
# legitimate one, so the control does not wait on the failed entry.
ct("a", "ip -6 addr del fd00::bad/64 dev eth0")
ct("a", "ip -6 neigh flush dev eth0")
host_ll = host.succeed("ip -6 -o addr show dev ${bridge} scope link").split()[3].split("/")[0]
ct_ok("a", f"ping -6 -c1 -W2 {host_ll}%eth0")
ll_a = ct("a", "ip -6 -o addr show dev eth0 scope link").split()[3].split("/")[0]
assert udp_reaches_host("a", ll_a, v6=True), "control: a link-local source goes through"
with subtest("another container's MAC is dropped, and the restored MAC works again"):
ct("a", f"ip link set dev eth0 address {MAC_B}")
assert not ct_ok("a", "ping -c1 -W2 ${gateway}")
# a cannot resolve the gateway with b's MAC, so what it sends is ARP, which is
# logged with `MACSRC=`; a frame with an IP header carries the MAC in `MAC=`.
host.wait_until_succeeds(
"journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*(MAC=[0-9a-f:]*" + MAC_B + ":|MACSRC=" + MAC_B + " )'",
timeout=60,
)
ct("a", f"ip link set dev eth0 address {MAC_A}")
IP_A = own_pair_again()
assert ct_ok("b", "ping -c1 -W2 ${gateway}"), "b was never affected"
with subtest("another container's MAC and IPv4 address together are dropped, so the leased pair alone is no credential"):
ct("a", f"ip link set dev eth0 address {MAC_B}")
ct("a", f"ip addr add {IP_B}/24 dev eth0")
# A static neighbour entry for the host, so the datagram needs no ARP:
# only the check of the port's MAC is left between it and the host. It
# is set after the MAC change, which flushes every entry, static ones too.
host_mac = host.succeed("cat /sys/class/net/${bridge}/address").strip()
ct("a", f"ip neigh replace ${gateway} lladdr {host_mac} dev eth0 nud permanent")
assert "PERMANENT" in ct("a", "ip neigh show to ${gateway} dev eth0"), "the static entry is in place"
# An IP frame from b's MAC and address; a's ARP lines would not show it.
datagram = r"IN=vb-a .*MAC=[0-9a-f:]*" + MAC_B + r":.*SRC=" + IP_B.replace(".", r"\.") + r" .*DPT=9999 "
seen = len(spoof_lines(datagram))
assert not udp_reaches_host("a", IP_B), "b's MAC and address from a's port must not arrive"
assert len(spoof_lines(datagram)) > seen, "and it is logged against a's port"
ct("a", f"ip addr del {IP_B}/24 dev eth0")
ct("a", f"ip link set dev eth0 address {MAC_A}")
# The MAC change flushes the neighbour table, static entries included, so
# there may be nothing left to remove; `nud all` is also what matches a static one.
ct("a", "ip neigh flush to ${gateway} dev eth0 nud all")
IP_A = own_pair_again()
with subtest("a DHCP message with another container's chaddr is dropped, one with its own is answered, and a renewal works"):
def discovers(mac_addr):
out = host.succeed("journalctl -u dnsmasq -o cat --no-pager")
return len(re.findall(r"DHCPDISCOVER\(${bridge}\) " + mac_addr, out))
before_b, before_a = discovers(MAC_B), discovers(MAC_A)
dhcp_drops = int(host.succeed("journalctl -k -o cat --no-pager | grep -cE 'hive-egress-spoof .*IN=vb-a .*DPT=67 ' || true").strip())
ct("a", f"perl ${bootp} {MAC_B}")
ct("a", f"perl ${bootp} {MAC_A}")
host.wait_until_succeeds(f"journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPDISCOVER(${bridge}) {MAC_A}' | grep -qv '^{before_a}$'")
assert discovers(MAC_A) == before_a + 1, "the message with a's own chaddr reached dnsmasq"
assert discovers(MAC_B) == before_b, "the message with b's chaddr did not"
host.wait_until_succeeds("test $(journalctl -k -o cat --no-pager | grep -cE 'hive-egress-spoof .*IN=vb-a .*DPT=67 ') -gt " + str(dhcp_drops))
acks = host.succeed("journalctl -u dnsmasq -o cat --no-pager").count("DHCPACK(${bridge}) " + IP_A)
ct("a", "dhcpcd --rebind eth0")
host.wait_until_succeeds(
"test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPACK(${bridge}) " + IP_A + "') -gt " + str(acks)
)
assert wait_ipv4("a") == IP_A, "the renewal kept the address"
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
assert ct_ok("a", "ping -c1 -W2 ${gateway}")
with subtest("a port with no bound MAC sends nothing, and the timer binds it again"):
host.succeed(f"nft flush set {TABLE} port_mac")
assert not ct_ok("a", "ping -c1 -W2 ${gateway}")
assert not ct_ok("b", "ping -c1 -W2 ${gateway}")
# Nothing starts the pin unit by hand: the timer finds the unbound ports.
host.wait_until_succeeds("nixos-container run a -- ping -c1 -W2 ${gateway}")
host.wait_until_succeeds("nixos-container run b -- ping -c1 -W2 ${gateway}")
with subtest("a pair that left the set comes back at the container's next renewal and its next rebind, and nothing is dropped"):
# The renewal is a unicast REQUEST from the leased address, the rebind a
# broadcast one from it. Neither can match the pair: it is gone. A static
# neighbour entry for the host keeps the unicast renewal from needing an
# ARP request, which the missing pair would drop.
host_mac = host.succeed("cat /sys/class/net/${bridge}/address").strip()
ct("a", f"ip neigh replace ${gateway} lladdr {host_mac} dev eth0 nud permanent")
for verb in ("renew", "rebind"):
host.succeed(f"nft delete element {TABLE} lease_v4 {{ {MAC_A} . {IP_A} }}")
host.fail(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
assert not ct_ok("a", "ping -c1 -W2 ${gateway}"), "control: a is cut off without its pair"
assert ct_ok("b", "ping -c1 -W2 ${gateway}"), "control: b is not"
drops_before = len(spoof_lines(r"IN=vb-a .*DPT=67 "))
acks = ack_count(IP_A)
ct("a", f"dhcpcd --{verb} eth0")
host.wait_until_succeeds(f"test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPACK(${bridge}) {IP_A}') -gt {acks}", timeout=30)
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=30)
assert ct_ok("a", "ping -c1 -W2 ${gateway}"), f"a reaches the host again after its {verb}"
assert len(spoof_lines(r"IN=vb-a .*DPT=67 ")) == drops_before, f"the {verb} was not dropped"
ct("a", "ip neigh del ${gateway} dev eth0")
with subtest("a DHCP client message with the port's own chaddr passes from any source address to the broadcast or bridge address only"):
# IP_B is a's own address for this subtest, so the other destination is
# a third one, with a static neighbour entry to spare the ARP request.
FAR = "10.42.0.250"
far_re = FAR.replace(".", r"\.")
host_mac = host.succeed("cat /sys/class/net/${bridge}/address").strip()
ct("a", f"ip neigh replace ${gateway} lladdr {host_mac} dev eth0 nud permanent")
ct("a", f"ip neigh replace {FAR} lladdr {MAC_B} dev eth0 nud permanent")
ct("a", f"ip addr add {IP_B}/24 dev eth0")
discovers_a = host.succeed("journalctl -u dnsmasq -o cat --no-pager").count("DHCPDISCOVER(${bridge}) " + MAC_A)
ct("a", f"perl ${bootp} {MAC_A} ${gateway} {IP_B}")
host.wait_until_succeeds(
"test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPDISCOVER(${bridge}) " + MAC_A + "') -gt " + str(discovers_a),
timeout=30,
)
seen = len(spoof_lines(r"IN=vb-a .*DST=" + far_re + " .*DPT=67 "))
ct("a", f"perl ${bootp} {MAC_A} {FAR} {IP_B}")
host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*DST=" + far_re + " .*DPT=67 '", timeout=30)
assert len(spoof_lines(r"IN=vb-a .*DST=" + far_re + " .*DPT=67 ")) > seen, "to any other address it is dropped"
ct("a", f"ip addr del {IP_B}/24 dev eth0")
ct("a", f"ip neigh del {FAR} dev eth0")
ct("a", "ip neigh del ${gateway} dev eth0")
with subtest("a step of the wall clock that makes dnsmasq prune every lease leaves the pairs in the set and the containers reaching the host"):
# The qemu guest has no suspend. A step of the wall clock is the same
# divergence: dnsmasq counts wall time and expires every lease, while
# the nft timeouts and the containers' leases count monotonic time.
assert ct_ok("a", "ping -c1 -W2 ${gateway}") and ct_ok("b", "ping -c1 -W2 ${gateway}")
drops_before = len(spoof_lines(r"IN=vb-[ab] "))
skipped = int(stale_dels())
applied = applied_dels()
host.succeed("date -s @$(( $(date +%s) + 7200 ))")
# Any DHCP packet makes dnsmasq prune: a DISCOVER with a's own chaddr.
ct("a", "perl ${bootp} " + MAC_A)
host.wait_until_succeeds("test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del .* skipped (stale)') -ge " + str(skipped + 2), timeout=60)
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "a reaches the host after dnsmasq forgot its lease"
assert ct_ok("b", "ping -c1 -W2 ${gateway}"), "b reaches the host after dnsmasq forgot its lease"
assert applied_dels() == applied, "no delete was applied"
# Each container then renews against a dnsmasq that has forgotten its
# lease, and ends with a lease again. It keeps reaching the host.
acks_a, acks_b = acks_of(MAC_A), acks_of(MAC_B)
for name in ("a", "b"):
ct(name, "dhcpcd --renew eth0")
host.wait_until_succeeds(f"test {acks_of_cmd(MAC_A)} -gt {acks_a} && test {acks_of_cmd(MAC_B)} -gt {acks_b}", timeout=90)
IP_A, IP_B = wait_ipv4("a"), wait_ipv4("b")
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=60)
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'", timeout=60)
assert len(spoof_lines(r"IN=vb-[ab] ")) == drops_before, "nothing of a or b was dropped"
with subtest("a release still removes the pair at once"):
# A fresh lease first: the release must name an expiry in the future.
acks = ack_count(IP_A)
ct("a", "dhcpcd --rebind eth0")
host.wait_until_succeeds(f"test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPACK(${bridge}) {IP_A}') -gt {acks}", timeout=30)
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
released = applied_dels(f"mac={MAC_A} ")
ct("a", "dhcpcd -k eth0")
host.wait_until_fails(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=30)
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
host.wait_until_succeeds(
"test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del mac=" + MAC_A + " .* applied') -gt " + str(released),
timeout=30,
)
# a asks again, as its container would after a restart.
ct("a", "systemctl restart dhcpcd")
host.wait_until_succeeds("nixos-container run a -- ping -c1 -W2 ${gateway}", timeout=60)
IP_A = ipv4("a")
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=30)
with subtest("antiSpoof = log keeps the bindings and the leases, lets a spoofed datagram through and logs it"):
host.succeed("/run/current-system/specialisation/logmode/bin/switch-to-configuration test")
assert spoof_verdict() == "accept", "the table was reloaded in log mode"
# No waiting: the reload put the sets in with the table.
host.succeed(f"nft list set {TABLE} port_mac | grep -q '\"vb-a\" . {MAC_A}'")
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "control: own address, right after the reload"
seen = int(host.succeed("journalctl -k -o cat --no-pager | grep -c 'hive-egress-spoof ' || true").strip())
ct("a", f"ip addr add {IP_B}/24 dev eth0")
assert udp_reaches_host("a", IP_B), "log mode lets the spoofed datagram through"
host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep 'hive-egress-spoof ' | wc -l | grep -qv '^" + str(seen) + "$'")
assert spoof_lines(r"IN=vb-a .*SRC=" + IP_B.replace(".", r"\.") + " ")
'';
}