dnsmasq relays the hook's stderr at LOG_DEBUG as dnsmasq-script, and the field log shipper's journald receiver keeps info and above, so no lease add or renewal was visible in the field. The hook now also writes each line to /dev/log at info priority with identifier egress-lease. The stderr print stays. The VM test reads the lines back with journalctl -t egress-lease -p info.
572 lines
31 KiB
Nix
572 lines
31 KiB
Nix
# `vmTests.<system>.hive-egress-antispoof`: the source check of
|
|
# ../host-modules/hive-egress.nix on a real agent bridge, with two nspawn
|
|
# containers on it that take their addresses from the hive's dnsmasq
|
|
# (../host-modules/hive-gateway/dnsmasq.nix' lease script, the real
|
|
# `hive-priv egress-load`, `egress-pin` and `egress-lease`). `hive-squid`
|
|
# cannot host this: its stand-in bridge is one NIC, so every agent shares one
|
|
# port.
|
|
#
|
|
# What it pins:
|
|
# - the verdict is `drop` (set here; the module default is `log`), and each
|
|
# container takes its first lease (DHCP from `0.0.0.0`, the reply, the first
|
|
# ARP and IP traffic) with it in place; each port is bound to its
|
|
# container's MAC and each lease is in the table, unaided by the test;
|
|
# - a container sending from its own address reaches the host and the other
|
|
# container (the control for every drop below);
|
|
# - another container's IPv4 address, another container's MAC, an IPv6 address
|
|
# that is not link-local and a DHCP message with another's `chaddr` are
|
|
# dropped, and each leaves a `hive-egress-spoof ` line that names the port;
|
|
# - a container that keeps talking across renewals of a 2-minute lease is
|
|
# never dropped, and its pair is still in `lease_v4` after the first lease
|
|
# and its margin would have ended;
|
|
# - a DHCP renewal, a link-local IPv6 packet and a restored MAC go through;
|
|
# - a pair deleted by hand comes back at the next renewal and rebind; a DHCP
|
|
# client message with the port's chaddr passes from any source address, to
|
|
# the broadcast or bridge address only; a two-hour wall-clock step keeps
|
|
# the pairs dnsmasq prunes; a release removes its pair;
|
|
# - a port with no bound MAC sends nothing, and the timer binds it again;
|
|
# - `antiSpoof = "log"`, reached by a reload of the unit, brings the bindings
|
|
# and the leases with the new table (nothing is dropped in between),
|
|
# lets the same spoofs through and logs them.
|
|
{ pkgs, hivePriv }:
|
|
let
|
|
bridge = "hive0";
|
|
gateway = "10.42.0.1";
|
|
|
|
# A DHCP DISCOVER by hand, with the `chaddr` given as the first argument,
|
|
# unicast to dnsmasq. A second argument names another destination, a third
|
|
# a source address, and a message with a source address leaves from port 68
|
|
# like a client's. The container shares the host's store, so it runs from
|
|
# there.
|
|
bootp = pkgs.writeText "bootp.pl" ''
|
|
use IO::Socket::INET;
|
|
use Socket;
|
|
my $mac = pack("H12", join("", split(/:/, $ARGV[0])));
|
|
my ($dest, $src) = ($ARGV[1] || "${gateway}", $ARGV[2]);
|
|
my @bind = $src ? (LocalAddr => $src, LocalPort => 68, ReuseAddr => 1) : ();
|
|
my $s = IO::Socket::INET->new(Proto => "udp", @bind) or die "socket: $!";
|
|
my $head = pack("C4 N n n a4 a4 a4 a4", 1, 1, 6, 0, 4660, 0, 0, "", "", "", "");
|
|
my $msg = $head . $mac . ("\0" x 10) . ("\0" x 192) . pack("H8", "63825363") . pack("C4", 53, 1, 1, 255);
|
|
$s->send($msg, 0, pack_sockaddr_in(67, inet_aton($dest))) or die "send: $!";
|
|
'';
|
|
|
|
hiveStubs =
|
|
{ lib, ... }:
|
|
{
|
|
options.services.hyperhive = {
|
|
deploy.hive-controller.package = lib.mkOption { type = lib.types.package; };
|
|
# Off: hive-priv.nix then adds only the link the pin command runs by.
|
|
deploy.hive-controller.enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
};
|
|
swarm.forge.domain = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "forge.t.local";
|
|
};
|
|
network = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = true;
|
|
};
|
|
bridgeIp = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = gateway;
|
|
};
|
|
# Set by hive-egress.nix; nothing here opens it.
|
|
exposeHostPorts = lib.mkOption {
|
|
type = lib.types.listOf lib.types.port;
|
|
default = [ ];
|
|
};
|
|
bridgeName = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = bridge;
|
|
};
|
|
};
|
|
};
|
|
config.services.hyperhive.deploy.hive-controller.package = hivePriv;
|
|
};
|
|
|
|
# A container that takes its address by DHCP, as the agents do.
|
|
container = {
|
|
autoStart = true;
|
|
privateNetwork = true;
|
|
hostBridge = bridge;
|
|
config =
|
|
{ lib, pkgs, ... }:
|
|
{
|
|
system.stateVersion = "25.11";
|
|
networking.useDHCP = lib.mkForce true;
|
|
networking.dhcpcd.extraConfig = "noipv4ll";
|
|
networking.dhcpcd.wait = "ipv4";
|
|
networking.firewall.allowedUDPPorts = [ 68 ];
|
|
environment.systemPackages = [
|
|
pkgs.perl
|
|
pkgs.socat
|
|
];
|
|
};
|
|
};
|
|
in
|
|
pkgs.testers.runNixOSTest {
|
|
name = "hive-egress-antispoof";
|
|
|
|
# Without this the driver allows an hour, the runner's whole job limit.
|
|
globalTimeout = 1800;
|
|
|
|
nodes.host =
|
|
{ config, lib, ... }:
|
|
{
|
|
imports = [
|
|
hiveStubs
|
|
../host-modules/hive-egress.nix
|
|
../host-modules/hive-priv.nix
|
|
];
|
|
virtualisation.memorySize = 2048;
|
|
# Explicit: the tests below are about what `drop` does.
|
|
services.hyperhive.deploy.hive-controller.egress.antiSpoof = "drop";
|
|
networking.bridges.${bridge}.interfaces = [ ];
|
|
networking.interfaces.${bridge}.ipv4.addresses = [
|
|
{
|
|
address = gateway;
|
|
prefixLength = 24;
|
|
}
|
|
];
|
|
# The test's own datagram listener, and dnsmasq's DHCP port.
|
|
networking.firewall.interfaces.${bridge} = {
|
|
allowedUDPPorts = [
|
|
67
|
|
9999
|
|
];
|
|
};
|
|
services.dnsmasq = {
|
|
enable = true;
|
|
resolveLocalQueries = false;
|
|
settings = {
|
|
interface = bridge;
|
|
bind-interfaces = true;
|
|
port = 0;
|
|
# dnsmasq's shortest lease: the test runs through renewals in minutes.
|
|
dhcp-range = "10.42.0.2,10.42.0.254,2m";
|
|
dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases";
|
|
dhcp-script = toString config.services.hyperhive.network.egress.leaseScript;
|
|
script-on-renewal = true;
|
|
log-dhcp = true;
|
|
};
|
|
};
|
|
systemd.services.dnsmasq = {
|
|
wants = [ "network-addresses-${bridge}.service" ];
|
|
after = [ "network-addresses-${bridge}.service" ];
|
|
};
|
|
# `nft` for the test's own reads of the tables; the units run it by store path.
|
|
environment.systemPackages = [
|
|
pkgs.socat
|
|
pkgs.nftables
|
|
];
|
|
|
|
containers.a = container;
|
|
containers.b = container;
|
|
|
|
specialisation.logmode.configuration = {
|
|
services.hyperhive.deploy.hive-controller.egress.antiSpoof = lib.mkForce "log";
|
|
};
|
|
};
|
|
|
|
testScript = ''
|
|
import ipaddress
|
|
import json
|
|
import re
|
|
import shlex
|
|
|
|
TABLE = "bridge hive_egress"
|
|
|
|
def ct(name, cmd):
|
|
return host.succeed(f"nixos-container run {name} -- sh -c {shlex.quote(cmd)}").strip()
|
|
|
|
def ct_ok(name, cmd):
|
|
return host.execute(f"nixos-container run {name} -- sh -c {shlex.quote(cmd)}")[0] == 0
|
|
|
|
def mac(name):
|
|
return ct(name, "cat /sys/class/net/eth0/address")
|
|
|
|
def ipv4(name):
|
|
out = ct(name, "ip -4 -o addr show dev eth0 scope global")
|
|
found = re.search(r"inet (10\.42\.0\.\d+)/", out)
|
|
assert found, out
|
|
return found.group(1)
|
|
|
|
# A port with no bound MAC sends nothing, so a container's first frames after
|
|
# start (MLD reports from `::`) can be refused before its port is pinned and
|
|
# for a moment after. Spoof lines of a port up to this long after its pin
|
|
# line are start-up refusals and are not counted; the forge issue "a
|
|
# container's first frames are sometimes refused by the anti-spoof MAC check
|
|
# right after its port is reported bound" tracks them.
|
|
PIN_SETTLE_US = 2_000_000
|
|
# port -> journal monotonic stamp (µs) up to which its spoof lines are start-up refusals
|
|
QUIET_AFTER_US = {}
|
|
|
|
def journal_entries(*args):
|
|
"""(monotonic µs, message) of each journal entry, one boot, kernel and userspace on one clock."""
|
|
out = host.succeed("journalctl --no-pager -o json " + " ".join(args))
|
|
entries = [json.loads(line) for line in out.splitlines()]
|
|
return [(int(e["__MONOTONIC_TIMESTAMP"]), e["MESSAGE"]) for e in entries if isinstance(e.get("MESSAGE"), str)]
|
|
|
|
def startup_refusal(stamp, message):
|
|
found = re.search(r"IN=(\S+)", message)
|
|
return found is not None and stamp < QUIET_AFTER_US.get(found.group(1), 0)
|
|
|
|
def spoof_entries():
|
|
return [(t, m) for t, m in journal_entries("-k") if "hive-egress-spoof " in m]
|
|
|
|
def wait_ipv4(name):
|
|
"""`name`'s address once dhcpcd has assigned it. An ACK counted in the journal comes first: dhcpcd probes the offered address for seconds before it takes it."""
|
|
host.wait_until_succeeds(
|
|
f"nixos-container run {name} -- sh -c 'ip -4 -o addr show dev eth0 scope global | grep -q \"inet 10.42.0.\"'",
|
|
timeout=60,
|
|
)
|
|
return ipv4(name)
|
|
|
|
def spoof_lines(pattern=""):
|
|
return [m for t, m in spoof_entries() if re.search(pattern, m) and not startup_refusal(t, m)]
|
|
|
|
def udp_reaches_host(name, src, v6=False):
|
|
"""One datagram from the container, sent from `src`. True when the host got it.
|
|
|
|
The kernel refuses to bind a link-local source without a scope id, so one gets `%eth0`.
|
|
When nothing arrives, the sender's exit status and output and the container's neighbour table are printed.
|
|
"""
|
|
host.succeed("rm -f /tmp/rx")
|
|
host.succeed(
|
|
"systemd-run --unit=rx --collect "
|
|
f"${pkgs.coreutils}/bin/timeout 6 ${pkgs.runtimeShell} -c '${pkgs.socat}/bin/socat -u UDP{6 if v6 else 4}-RECV:9999 - > /tmp/rx'"
|
|
)
|
|
host.sleep(1)
|
|
if v6:
|
|
ll = host.succeed("ip -6 -o addr show dev ${bridge} scope link").split()[3].split("/")[0]
|
|
bound = f"{src}%eth0" if ipaddress.ip_address(src).is_link_local else src
|
|
target = f"UDP6-SENDTO:[{ll}%eth0]:9999,bind=[{bound}]"
|
|
else:
|
|
target = f"UDP4-SENDTO:${gateway}:9999,bind={src}"
|
|
sent_status, sent = host.execute(
|
|
f"nixos-container run {name} -- sh -c {shlex.quote(f'echo hi | socat -u - {target} 2>&1')}"
|
|
)
|
|
host.sleep(1)
|
|
got = host.succeed("cat /tmp/rx").strip() == "hi"
|
|
if not got:
|
|
neighbours = host.execute(f"nixos-container run {name} -- ip neigh show dev eth0")[1]
|
|
print(
|
|
f"udp_reaches_host({name}, {src}): nothing arrived; sender exit {sent_status}, "
|
|
f"sender output {sent.strip()!r}; {name}'s neighbours:\n{neighbours.strip()}"
|
|
)
|
|
host.execute("systemctl stop rx.service")
|
|
return got
|
|
|
|
def spoof_verdict():
|
|
"""The verdict of the spoof chain's last rule, the one every packet that gets past the log rules reaches."""
|
|
out = host.succeed(f"nft list chain {TABLE} spoof")
|
|
found = re.findall(r"counter packets \d+ bytes \d+ (drop|accept)$", out, re.M)
|
|
assert found, out
|
|
return found[-1]
|
|
|
|
def own_pair_again():
|
|
"""a's address once it reaches the host with its own MAC again, and its pair is loaded.
|
|
|
|
While a held another container's MAC its DHCP client declined its
|
|
lease and asked again, so the address it ends with may not be the one
|
|
it started with. a's own neighbour entry for the gateway is flushed:
|
|
its ARP requests were refused meanwhile, so the entry may still be
|
|
failed. The host never saw those frames and has nothing to flush.
|
|
"""
|
|
ct("a", "ip -4 neigh flush dev eth0")
|
|
host.wait_until_succeeds("nixos-container run a -- ping -c1 -W2 ${gateway}", timeout=60)
|
|
ip = ipv4("a")
|
|
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {ip}'", timeout=60)
|
|
return ip
|
|
|
|
def acks_of_cmd(mac_addr):
|
|
return "$(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPACK(${bridge}) .* " + mac_addr + "')"
|
|
|
|
def acks_of(mac_addr):
|
|
return int(host.succeed("echo " + acks_of_cmd(mac_addr)).strip())
|
|
|
|
# What `hive-priv egress-lease` leaves in the journal: one line per lease event, under its
|
|
# own identifier at info priority, which is as low as the log shipper reads. `-u dnsmasq`
|
|
# would also match dnsmasq's debug-priority copy of the same stderr line, which the shipper
|
|
# drops and which would count each event twice.
|
|
LEASE_LOG = "journalctl -t egress-lease -p info -o cat --no-pager"
|
|
|
|
def stale_dels():
|
|
return host.succeed(LEASE_LOG + " | grep -c 'egress-lease: del .* skipped (stale)' || true").strip()
|
|
|
|
def applied_dels(who=""):
|
|
"""How many deletes the lease script applied so far, all or `mac=<mac> ` only.
|
|
|
|
A count is taken before the step under test and compared after: the
|
|
journal holds the lines of every earlier subtest.
|
|
"""
|
|
return int(host.succeed(f"{LEASE_LOG} | grep -c 'egress-lease: del {who}.* applied' || true").strip())
|
|
|
|
host.start()
|
|
host.wait_for_unit("multi-user.target")
|
|
host.wait_for_unit("dnsmasq.service")
|
|
host.wait_for_unit("hive-egress.service")
|
|
# `drop` is set explicitly on the host above, loaded before either container
|
|
# exists: the first lease of each container below is taken with it in place.
|
|
assert spoof_verdict() == "drop", "antiSpoof = drop is loaded"
|
|
for name in ("a", "b"):
|
|
host.wait_for_unit(f"container@{name}.service")
|
|
host.wait_until_succeeds(
|
|
f"nixos-container run {name} -- sh -c 'ip -4 -o addr show dev eth0 | grep -q \"inet 10.42.0.\"'"
|
|
)
|
|
MAC_A, MAC_B = mac("a"), mac("b")
|
|
IP_A, IP_B = ipv4("a"), ipv4("b")
|
|
assert MAC_A != MAC_B and IP_A != IP_B
|
|
T0 = int(host.succeed("date +%s").strip())
|
|
|
|
with subtest("each port is bound to its container's MAC and each lease is loaded, with no help from the test"):
|
|
host.wait_until_succeeds(f"nft list set {TABLE} port_mac | grep -q '\"vb-a\" . {MAC_A}'")
|
|
host.wait_until_succeeds(f"nft list set {TABLE} port_mac | grep -q '\"vb-b\" . {MAC_B}'")
|
|
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
|
|
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
|
|
# The pin line is printed after the set changes, so wait for it too.
|
|
for port, mac_addr in (("vb-a", MAC_A), ("vb-b", MAC_B)):
|
|
host.wait_until_succeeds(f"journalctl -b -o cat --no-pager | grep -q 'egress-pin: {port} bound to {mac_addr} '")
|
|
journal = journal_entries("-b")
|
|
for port, mac_addr in (("vb-a", MAC_A), ("vb-b", MAC_B)):
|
|
stamps = [t for t, m in journal if m.startswith(f"egress-pin: {port} bound to {mac_addr} ")]
|
|
assert stamps, f"no pin line for {port}"
|
|
QUIET_AFTER_US[port] = min(stamps) + PIN_SETTLE_US
|
|
for stamp, message in spoof_entries():
|
|
if startup_refusal(stamp, message):
|
|
found = re.search(r"IN=(\S+)", message)
|
|
assert found is not None, message
|
|
port = found.group(1)
|
|
print(f"tolerated start-up refusal, {(stamp - QUIET_AFTER_US[port]) / 1e6:+.3f}s from the end of {port}'s settle: {message}")
|
|
assert not spoof_lines(), spoof_lines()
|
|
# Each lease left its line in the journal where the shipper can read it, with the
|
|
# container's MAC and address in it.
|
|
for mac_addr, ip_addr in ((MAC_A, IP_A), (MAC_B, IP_B)):
|
|
host.wait_until_succeeds(f"{LEASE_LOG} | grep -E 'egress-lease: (add|old) mac={mac_addr} ip={ip_addr} '")
|
|
|
|
with subtest("a container sending from its own address reaches the host and the other container"):
|
|
assert ct_ok("a", "ping -c1 -W2 ${gateway}")
|
|
assert ct_ok("a", f"ping -c1 -W2 {IP_B}")
|
|
assert ct_ok("b", f"ping -c1 -W2 {IP_A}")
|
|
assert udp_reaches_host("a", IP_A), "control: a datagram from a's own address arrives"
|
|
assert not spoof_lines(), spoof_lines()
|
|
|
|
with subtest("a container that keeps talking across renewals is never dropped, and its pair outlives the first lease and its margin"):
|
|
# The leases run 2 minutes and the pair outlives its lease by 300
|
|
# seconds (LEASE_MARGIN_SECS in hive-priv's egress.rs). The pairs were
|
|
# loaded before T0, so `T0 + 420` is the latest the first load alone
|
|
# would have kept them; the loop runs past it, and only the refresh at
|
|
# each renewal keeps them in the set.
|
|
def now():
|
|
return int(host.succeed("date +%s").strip())
|
|
def ack_count(ip):
|
|
return host.succeed("journalctl -u dnsmasq -o cat --no-pager").count("DHCPACK(${bridge}) " + ip)
|
|
while now() < T0 + 450:
|
|
assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "a's traffic is dropped while it renews"
|
|
assert ct_ok("b", "ping -c1 -W2 ${gateway}"), "b's traffic is dropped while it renews"
|
|
host.sleep(15)
|
|
assert not spoof_lines(), spoof_lines()
|
|
assert ack_count(IP_A) >= 4 and ack_count(IP_B) >= 4, "each container renewed several times"
|
|
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
|
|
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
|
|
assert udp_reaches_host("a", IP_A), "a's own address still reaches the host after its first lease ended"
|
|
|
|
with subtest("another container's IPv4 address is dropped, and the line names the port"):
|
|
ct("a", f"ip addr add {IP_B}/24 dev eth0")
|
|
assert not udp_reaches_host("a", IP_B), "a datagram from b's address must not arrive"
|
|
host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*SRC=" + IP_B.replace(".", "\\.") + " '")
|
|
assert not spoof_lines(r"IN=vb-b "), "b sent nothing wrong"
|
|
ct("a", f"ip addr del {IP_B}/24 dev eth0")
|
|
# Resolve the host again from a's own address, so the control does not
|
|
# ride on an ARP entry the spoofed source touched.
|
|
ct("a", "ip -4 neigh flush dev eth0")
|
|
ct_ok("a", "ping -c1 -W2 ${gateway}")
|
|
assert udp_reaches_host("a", IP_A), "a's own address still works"
|
|
|
|
with subtest("an IPv6 address that is not link-local is dropped, a link-local one is not"):
|
|
ct("a", "ip -6 addr add fd00::bad/64 dev eth0 nodad")
|
|
assert not udp_reaches_host("a", "fd00::bad", v6=True)
|
|
host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*SRC=fd00:0*:0*:0*:0*:0*:0*:0*bad '")
|
|
# The kernel resolves the host's link-local address with neighbour
|
|
# solicits from the datagram's source, which the table drops for
|
|
# fd00::bad. Take that source away and resolve the host again from a
|
|
# legitimate one, so the control does not wait on the failed entry.
|
|
ct("a", "ip -6 addr del fd00::bad/64 dev eth0")
|
|
ct("a", "ip -6 neigh flush dev eth0")
|
|
host_ll = host.succeed("ip -6 -o addr show dev ${bridge} scope link").split()[3].split("/")[0]
|
|
ct_ok("a", f"ping -6 -c1 -W2 {host_ll}%eth0")
|
|
ll_a = ct("a", "ip -6 -o addr show dev eth0 scope link").split()[3].split("/")[0]
|
|
assert udp_reaches_host("a", ll_a, v6=True), "control: a link-local source goes through"
|
|
|
|
with subtest("another container's MAC is dropped, and the restored MAC works again"):
|
|
ct("a", f"ip link set dev eth0 address {MAC_B}")
|
|
assert not ct_ok("a", "ping -c1 -W2 ${gateway}")
|
|
# a cannot resolve the gateway with b's MAC, so what it sends is ARP, which is
|
|
# logged with `MACSRC=`; a frame with an IP header carries the MAC in `MAC=`.
|
|
host.wait_until_succeeds(
|
|
"journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*(MAC=[0-9a-f:]*" + MAC_B + ":|MACSRC=" + MAC_B + " )'",
|
|
timeout=60,
|
|
)
|
|
ct("a", f"ip link set dev eth0 address {MAC_A}")
|
|
IP_A = own_pair_again()
|
|
assert ct_ok("b", "ping -c1 -W2 ${gateway}"), "b was never affected"
|
|
|
|
with subtest("another container's MAC and IPv4 address together are dropped, so the leased pair alone is no credential"):
|
|
ct("a", f"ip link set dev eth0 address {MAC_B}")
|
|
ct("a", f"ip addr add {IP_B}/24 dev eth0")
|
|
# A static neighbour entry for the host, so the datagram needs no ARP:
|
|
# only the check of the port's MAC is left between it and the host. It
|
|
# is set after the MAC change, which flushes every entry, static ones too.
|
|
host_mac = host.succeed("cat /sys/class/net/${bridge}/address").strip()
|
|
ct("a", f"ip neigh replace ${gateway} lladdr {host_mac} dev eth0 nud permanent")
|
|
assert "PERMANENT" in ct("a", "ip neigh show to ${gateway} dev eth0"), "the static entry is in place"
|
|
# An IP frame from b's MAC and address; a's ARP lines would not show it.
|
|
datagram = r"IN=vb-a .*MAC=[0-9a-f:]*" + MAC_B + r":.*SRC=" + IP_B.replace(".", r"\.") + r" .*DPT=9999 "
|
|
seen = len(spoof_lines(datagram))
|
|
assert not udp_reaches_host("a", IP_B), "b's MAC and address from a's port must not arrive"
|
|
assert len(spoof_lines(datagram)) > seen, "and it is logged against a's port"
|
|
ct("a", f"ip addr del {IP_B}/24 dev eth0")
|
|
ct("a", f"ip link set dev eth0 address {MAC_A}")
|
|
# The MAC change flushes the neighbour table, static entries included, so
|
|
# there may be nothing left to remove; `nud all` is also what matches a static one.
|
|
ct("a", "ip neigh flush to ${gateway} dev eth0 nud all")
|
|
IP_A = own_pair_again()
|
|
|
|
with subtest("a DHCP message with another container's chaddr is dropped, one with its own is answered, and a renewal works"):
|
|
def discovers(mac_addr):
|
|
out = host.succeed("journalctl -u dnsmasq -o cat --no-pager")
|
|
return len(re.findall(r"DHCPDISCOVER\(${bridge}\) " + mac_addr, out))
|
|
before_b, before_a = discovers(MAC_B), discovers(MAC_A)
|
|
dhcp_drops = int(host.succeed("journalctl -k -o cat --no-pager | grep -cE 'hive-egress-spoof .*IN=vb-a .*DPT=67 ' || true").strip())
|
|
ct("a", f"perl ${bootp} {MAC_B}")
|
|
ct("a", f"perl ${bootp} {MAC_A}")
|
|
host.wait_until_succeeds(f"journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPDISCOVER(${bridge}) {MAC_A}' | grep -qv '^{before_a}$'")
|
|
assert discovers(MAC_A) == before_a + 1, "the message with a's own chaddr reached dnsmasq"
|
|
assert discovers(MAC_B) == before_b, "the message with b's chaddr did not"
|
|
host.wait_until_succeeds("test $(journalctl -k -o cat --no-pager | grep -cE 'hive-egress-spoof .*IN=vb-a .*DPT=67 ') -gt " + str(dhcp_drops))
|
|
acks = host.succeed("journalctl -u dnsmasq -o cat --no-pager").count("DHCPACK(${bridge}) " + IP_A)
|
|
ct("a", "dhcpcd --rebind eth0")
|
|
host.wait_until_succeeds(
|
|
"test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPACK(${bridge}) " + IP_A + "') -gt " + str(acks)
|
|
)
|
|
assert wait_ipv4("a") == IP_A, "the renewal kept the address"
|
|
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
|
|
assert ct_ok("a", "ping -c1 -W2 ${gateway}")
|
|
|
|
with subtest("a port with no bound MAC sends nothing, and the timer binds it again"):
|
|
host.succeed(f"nft flush set {TABLE} port_mac")
|
|
assert not ct_ok("a", "ping -c1 -W2 ${gateway}")
|
|
assert not ct_ok("b", "ping -c1 -W2 ${gateway}")
|
|
# Nothing starts the pin unit by hand: the timer finds the unbound ports.
|
|
host.wait_until_succeeds("nixos-container run a -- ping -c1 -W2 ${gateway}")
|
|
host.wait_until_succeeds("nixos-container run b -- ping -c1 -W2 ${gateway}")
|
|
|
|
with subtest("a pair that left the set comes back at the container's next renewal and its next rebind, and nothing is dropped"):
|
|
# The renewal is a unicast REQUEST from the leased address, the rebind a
|
|
# broadcast one from it. Neither can match the pair: it is gone. A static
|
|
# neighbour entry for the host keeps the unicast renewal from needing an
|
|
# ARP request, which the missing pair would drop.
|
|
host_mac = host.succeed("cat /sys/class/net/${bridge}/address").strip()
|
|
ct("a", f"ip neigh replace ${gateway} lladdr {host_mac} dev eth0 nud permanent")
|
|
for verb in ("renew", "rebind"):
|
|
host.succeed(f"nft delete element {TABLE} lease_v4 {{ {MAC_A} . {IP_A} }}")
|
|
host.fail(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
|
|
assert not ct_ok("a", "ping -c1 -W2 ${gateway}"), "control: a is cut off without its pair"
|
|
assert ct_ok("b", "ping -c1 -W2 ${gateway}"), "control: b is not"
|
|
drops_before = len(spoof_lines(r"IN=vb-a .*DPT=67 "))
|
|
acks = ack_count(IP_A)
|
|
ct("a", f"dhcpcd --{verb} eth0")
|
|
host.wait_until_succeeds(f"test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPACK(${bridge}) {IP_A}') -gt {acks}", timeout=30)
|
|
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=30)
|
|
assert ct_ok("a", "ping -c1 -W2 ${gateway}"), f"a reaches the host again after its {verb}"
|
|
assert len(spoof_lines(r"IN=vb-a .*DPT=67 ")) == drops_before, f"the {verb} was not dropped"
|
|
ct("a", "ip neigh del ${gateway} dev eth0")
|
|
|
|
with subtest("a DHCP client message with the port's own chaddr passes from any source address to the broadcast or bridge address only"):
|
|
# IP_B is a's own address for this subtest, so the other destination is
|
|
# a third one, with a static neighbour entry to spare the ARP request.
|
|
FAR = "10.42.0.250"
|
|
far_re = FAR.replace(".", r"\.")
|
|
host_mac = host.succeed("cat /sys/class/net/${bridge}/address").strip()
|
|
ct("a", f"ip neigh replace ${gateway} lladdr {host_mac} dev eth0 nud permanent")
|
|
ct("a", f"ip neigh replace {FAR} lladdr {MAC_B} dev eth0 nud permanent")
|
|
ct("a", f"ip addr add {IP_B}/24 dev eth0")
|
|
discovers_a = host.succeed("journalctl -u dnsmasq -o cat --no-pager").count("DHCPDISCOVER(${bridge}) " + MAC_A)
|
|
ct("a", f"perl ${bootp} {MAC_A} ${gateway} {IP_B}")
|
|
host.wait_until_succeeds(
|
|
"test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPDISCOVER(${bridge}) " + MAC_A + "') -gt " + str(discovers_a),
|
|
timeout=30,
|
|
)
|
|
seen = len(spoof_lines(r"IN=vb-a .*DST=" + far_re + " .*DPT=67 "))
|
|
ct("a", f"perl ${bootp} {MAC_A} {FAR} {IP_B}")
|
|
host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*DST=" + far_re + " .*DPT=67 '", timeout=30)
|
|
assert len(spoof_lines(r"IN=vb-a .*DST=" + far_re + " .*DPT=67 ")) > seen, "to any other address it is dropped"
|
|
ct("a", f"ip addr del {IP_B}/24 dev eth0")
|
|
ct("a", f"ip neigh del {FAR} dev eth0")
|
|
ct("a", "ip neigh del ${gateway} dev eth0")
|
|
|
|
with subtest("a step of the wall clock that makes dnsmasq prune every lease leaves the pairs in the set and the containers reaching the host"):
|
|
# The qemu guest has no suspend. A step of the wall clock is the same
|
|
# divergence: dnsmasq counts wall time and expires every lease, while
|
|
# the nft timeouts and the containers' leases count monotonic time.
|
|
assert ct_ok("a", "ping -c1 -W2 ${gateway}") and ct_ok("b", "ping -c1 -W2 ${gateway}")
|
|
drops_before = len(spoof_lines(r"IN=vb-[ab] "))
|
|
skipped = int(stale_dels())
|
|
applied = applied_dels()
|
|
host.succeed("date -s @$(( $(date +%s) + 7200 ))")
|
|
# Any DHCP packet makes dnsmasq prune: a DISCOVER with a's own chaddr.
|
|
ct("a", "perl ${bootp} " + MAC_A)
|
|
host.wait_until_succeeds("test $(" + LEASE_LOG + " | grep -c 'egress-lease: del .* skipped (stale)') -ge " + str(skipped + 2), timeout=60)
|
|
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
|
|
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
|
|
assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "a reaches the host after dnsmasq forgot its lease"
|
|
assert ct_ok("b", "ping -c1 -W2 ${gateway}"), "b reaches the host after dnsmasq forgot its lease"
|
|
assert applied_dels() == applied, "no delete was applied"
|
|
# Each container then renews against a dnsmasq that has forgotten its
|
|
# lease, and ends with a lease again. It keeps reaching the host.
|
|
acks_a, acks_b = acks_of(MAC_A), acks_of(MAC_B)
|
|
for name in ("a", "b"):
|
|
ct(name, "dhcpcd --renew eth0")
|
|
host.wait_until_succeeds(f"test {acks_of_cmd(MAC_A)} -gt {acks_a} && test {acks_of_cmd(MAC_B)} -gt {acks_b}", timeout=90)
|
|
IP_A, IP_B = wait_ipv4("a"), wait_ipv4("b")
|
|
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=60)
|
|
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'", timeout=60)
|
|
assert len(spoof_lines(r"IN=vb-[ab] ")) == drops_before, "nothing of a or b was dropped"
|
|
|
|
with subtest("a release still removes the pair at once"):
|
|
# A fresh lease first: the release must name an expiry in the future.
|
|
acks = ack_count(IP_A)
|
|
ct("a", "dhcpcd --rebind eth0")
|
|
host.wait_until_succeeds(f"test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPACK(${bridge}) {IP_A}') -gt {acks}", timeout=30)
|
|
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
|
|
released = applied_dels(f"mac={MAC_A} ")
|
|
ct("a", "dhcpcd -k eth0")
|
|
host.wait_until_fails(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=30)
|
|
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
|
|
host.wait_until_succeeds(
|
|
"test $(" + LEASE_LOG + " | grep -c 'egress-lease: del mac=" + MAC_A + " .* applied') -gt " + str(released),
|
|
timeout=30,
|
|
)
|
|
# a asks again, as its container would after a restart.
|
|
ct("a", "systemctl restart dhcpcd")
|
|
host.wait_until_succeeds("nixos-container run a -- ping -c1 -W2 ${gateway}", timeout=60)
|
|
IP_A = ipv4("a")
|
|
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=30)
|
|
|
|
with subtest("antiSpoof = log keeps the bindings and the leases, lets a spoofed datagram through and logs it"):
|
|
host.succeed("/run/current-system/specialisation/logmode/bin/switch-to-configuration test")
|
|
assert spoof_verdict() == "accept", "the table was reloaded in log mode"
|
|
# No waiting: the reload put the sets in with the table.
|
|
host.succeed(f"nft list set {TABLE} port_mac | grep -q '\"vb-a\" . {MAC_A}'")
|
|
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
|
|
assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "control: own address, right after the reload"
|
|
seen = int(host.succeed("journalctl -k -o cat --no-pager | grep -c 'hive-egress-spoof ' || true").strip())
|
|
ct("a", f"ip addr add {IP_B}/24 dev eth0")
|
|
assert udp_reaches_host("a", IP_B), "log mode lets the spoofed datagram through"
|
|
host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep 'hive-egress-spoof ' | wc -l | grep -qv '^" + str(seen) + "$'")
|
|
assert spoof_lines(r"IN=vb-a .*SRC=" + IP_B.replace(".", r"\.") + " ")
|
|
'';
|
|
}
|