mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

egress: refresh a leased pair on dhcp renewal and default anti-spoof to log

Browse source 
dnsmasq runs the dhcp-script on a renewal only with script-on-renewal, so a pair loaded at the first lease timed out while the lease kept renewing and the bridge table dropped the container's traffic. Set script-on-renewal, time pairs out remaining + 300s, default antiSpoof to log, and run the vm test through renewals with drop.
This commit is contained in:
atlas 2026-11-13 23:55:07 +01:00
commit 5189e8fb4b
5 changed files with 107 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

17
nix/host-modules/hive-egress.nix
View file

@ -26,8 +26,8 @@
# may use are not known until a container runs, so they are set contents.
# `hive-priv egress-load` puts the current ones into the same transaction as the
# tables; `egress-pin` (per new port from udev, and from a timer for the ports
# still unbound) and dnsmasq's `dhcp-script` (per lease) keep them current.
# `hive-priv`'s `egress.rs` documents how.
# still unbound) and dnsmasq's `dhcp-script` (per lease and renewal) keep them
# current. `hive-priv`'s `egress.rs` documents how.
{
lib,
config,
@ -200,7 +200,7 @@ let
# as the 48 bits that sit at the `chaddr` field of a DHCP message. A port
# with no pair sends nothing, DHCP included.
# - `lease_v4` holds the leased (MAC, IPv4) pairs; each times out with its
# lease.
# lease, and a renewal moves the timeout.
# - IPv6 has no leases: a source is link-local, or `::` in the packets that
# duplicate address detection and multicast listener reports send before an
# address exists. The bridge carries no other IPv6 traffic.
@ -335,7 +335,7 @@ in
"drop"
"log"
];
default = "drop";
default = "log";
description = ''
What the `bridge hive_egress` table does with a frame whose source is not
the sending container's own. Every port of the agent bridge is checked:
@ -350,8 +350,10 @@ in
cannot log gets a line with the prefix `hive-egress-spoof-unmetered `
from a limit shared by all ports. A container sending from its leased
address is never affected, so nothing legitimate breaks.
- `log`: the same lines and counters, and the frame goes through. For
finding out why a legitimate container is refused on a live hive.
- `log` (the default): the same lines and counters, and the frame goes
through. For finding out why a legitimate container is refused on a
live hive. The default goes back to `drop` once the renewal fix
(dnsmasq runs the lease script on a renewal) is proven live.
With `log`, the identity squid takes from the source of an intercepted
flow (`egress.intercept`) can be spoofed: use it to diagnose, not to run.
@ -448,7 +450,8 @@ in
default = leaseScript;
defaultText = lib.literalMD "the script that loads a lease into the bridge table";
description = ''
The program dnsmasq runs on each lease change (its `dhcp-script`).
The program dnsmasq runs on each lease change and renewal (its
`dhcp-script` with `script-on-renewal`).
'';
};

6
nix/host-modules/hive-gateway/dnsmasq.nix
View file

@ -84,8 +84,12 @@ in
# `hive-priv egress-load` reads this file back (hive-egress.nix).
dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases";
# Each lease change updates the leased (MAC, IPv4) pairs the bridge
# anti-spoofing table checks sources against (hive-egress.nix).
# anti-spoofing table checks sources against (hive-egress.nix). A pair
# times out with its lease, so the script must also run on a renewal:
# without `script-on-renewal` dnsmasq runs it only for a new lease, and a
# renewed one leaves the table when the first lease would have ended.
dhcp-script = toString networkCfg.egress.leaseScript;
script-on-renewal = true;
# No explicit upstream: non-hive queries follow dnsmasq's
# resolv.conf default — the host's own `/etc/resolv.conf`, so the
# hive always uses the host's resolvers and follows them live with

29
nix/module-eval/core-toggle.nix
View file

@ -73,12 +73,12 @@ let
# The two settings of `egress.forward` the ordinary host (`log`) does not
# render, for ../host-modules/hive-egress.nix: `accept` leaves the table
# without a forward chain, `reject` renders the chain that refuses. The
# `accept` host also sets `egress.antiSpoof = "log"`, the one setting the
# ordinary host (`drop`) does not render.
# `accept` host also sets `egress.antiSpoof = "drop"`, the one setting the
# ordinary host (`log`) does not render.
egressForwardAccept = hive {
network.enable = true;
network.egress.forward = "accept";
network.egress.antiSpoof = "log";
network.egress.antiSpoof = "drop";
};
egressForwardReject = hiveWithController {
network.egress.forward = "reject";
@ -1587,17 +1587,17 @@ let
# its own, and logs it. The ruleset is read as text: the spoof chain
# ends every path in `drop`, and the accepts that remain are the ones that
# let the leased source through, which is the presence arm.
name = "the default egress table drops frames whose source is not the port's own";
name = "egress.antiSpoof drop renders the bridge table that drops frames whose source is not the port's own";
ok =
let
text = bare.services.hyperhive.network.egress.ruleset;
text = egressForwardAccept.services.hyperhive.network.egress.ruleset;
bridgeText = lib.last (lib.splitString "table bridge hive_egress" text);
section =
name: lib.head (lib.splitString "chain" (lib.last (lib.splitString "chain ${name} {" bridgeText)));
spoof = lib.filter (l: l != "" && l != "}") (map lib.trim (lib.splitString "\n" (section "spoof")));
ip4 = section "l3_v4";
in
bare.services.hyperhive.network.egress.antiSpoof == "drop"
egressForwardAccept.services.hyperhive.network.egress.antiSpoof == "drop"
&& lib.hasInfix "hook prerouting priority -300" bridgeText
&& lib.hasInfix ''meta ibrname != "hive-br0" accept'' bridgeText
&& lib.hasInfix "iifname . ether saddr != @port_mac goto spoof" bridgeText
@ -1614,10 +1614,10 @@ let
&& lib.hasInfix "goto spoof" ip4;
}
{
# `log` renders the same chains with `accept` where `drop` was: the lines
# and counters stay, the frame goes through. The `accept` host is the
# only fixture that sets it; `bare` is the control that the two differ.
name = "egress.antiSpoof log renders the spoof chain with accept and no drop";
# `log`, the default, renders the same chains with `accept` where `drop`
# was: the lines and counters stay, the frame goes through. The `accept`
# host sets `drop` and is the control that the two differ.
name = "egress.antiSpoof defaults to log and renders the spoof chain with accept and no drop";
ok =
let
textOf = m: m.services.hyperhive.network.egress.ruleset;
@ -1629,14 +1629,14 @@ let
lib.filter (l: l != "" && l != "}") (
map lib.trim (lib.splitString "\n" (lib.head (lib.splitString "chain" t)))
);
logged = spoofOf egressForwardAccept;
logged = spoofOf bare;
in
egressForwardAccept.services.hyperhive.network.egress.antiSpoof == "log"
bare.services.hyperhive.network.egress.antiSpoof == "log"
&& lib.length logged == 3
&& lib.all (l: lib.hasSuffix " accept" l) logged
&& lib.hasInfix ''log prefix "hive-egress-spoof "'' (textOf egressForwardAccept)
&& lib.hasInfix ''log prefix "hive-egress-spoof "'' (textOf bare)
&& !(lib.any (l: lib.hasSuffix " drop" l) logged)
&& lib.all (l: lib.hasSuffix " drop" l) (spoofOf bare);
&& lib.all (l: lib.hasSuffix " drop" l) (spoofOf egressForwardAccept);
}
{
# The sets are filled at run time, from four places: the unit at start and
@ -1664,6 +1664,7 @@ let
&& lib.elem (toString bare.services.hyperhive.network.egress.leaseScript) (
lib.toList bare.services.dnsmasq.settings.dhcp-script
)
&& lib.toList bare.services.dnsmasq.settings.script-on-renewal == [ true ]
&& !(centralToggleOff.systemd.services ? "hive-egress-pin@")
&& !(centralToggleOff.systemd ? timers && centralToggleOff.systemd.timers ? hive-egress-pin);
}

1
nix/vm-test-typecheck.nix
View file

@ -128,6 +128,7 @@ let
tests = {
inherit (vmTests)
hive-squid
hive-egress-antispoof
swarm-archive
agent-ephemeral-rootfs
full-setup

29
nix/vm-tests/hive-egress-antispoof.nix
View file

@ -13,6 +13,9 @@
# - another container's IPv4 address, another container's MAC, an IPv6 address
# that is not link-local and a DHCP message with another's `chaddr` are
# dropped, and each leaves a `hive-egress-spoof ` line that names the port;
# - a container that keeps talking across several renewals of a 2-minute lease
# is never dropped, and its pair is still in `lease_v4` after the first
# lease and its margin would have ended, with `antiSpoof = "drop"`;
# - a DHCP renewal, a link-local IPv6 packet and a restored MAC go through;
# - a port with no bound MAC sends nothing, and the timer binds it again;
# - `antiSpoof = "log"`, reached by a reload of the unit, brings the bindings
@ -95,6 +98,7 @@ pkgs.testers.runNixOSTest {
../host-modules/hive-priv.nix
];
virtualisation.memorySize = 2048;
services.hyperhive.network.egress.antiSpoof = "drop";
networking.bridges.${bridge}.interfaces = [ ];
networking.interfaces.${bridge}.ipv4.addresses = [
{
@ -116,9 +120,11 @@ pkgs.testers.runNixOSTest {
interface = bridge;
bind-interfaces = true;
port = 0;
dhcp-range = "10.42.0.2,10.42.0.254,1h";
# dnsmasq's shortest lease: the test runs through renewals in minutes.
dhcp-range = "10.42.0.2,10.42.0.254,2m";
dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases";
dhcp-script = toString config.services.hyperhive.network.egress.leaseScript;
script-on-renewal = true;
log-dhcp = true;
};
};
@ -192,6 +198,7 @@ pkgs.testers.runNixOSTest {
MAC_A, MAC_B = mac("a"), mac("b")
IP_A, IP_B = ipv4("a"), ipv4("b")
assert MAC_A != MAC_B and IP_A != IP_B
T0 = int(host.succeed("date +%s").strip())
with subtest("each port is bound to its container's MAC and each lease is loaded, with no help from the test"):
host.wait_until_succeeds(f"nft list set {TABLE} port_mac | grep -q '\"vb-a\" . {MAC_A}'")
@ -207,6 +214,26 @@ pkgs.testers.runNixOSTest {
assert udp_reaches_host("a", IP_A), "control: a datagram from a's own address arrives"
assert not spoof_lines(), spoof_lines()
with subtest("a container that keeps talking across renewals is never dropped, and its pair outlives the first lease and its margin"):
# The leases run 2 minutes and the pair outlives its lease by 300
# seconds (LEASE_MARGIN_SECS in hive-priv's egress.rs). The pairs were
# loaded before T0, so `T0 + 420` is the latest the first load alone
# would have kept them; the loop runs past it, and only the refresh at
# each renewal keeps them in the set.
def now():
return int(host.succeed("date +%s").strip())
def ack_count(ip):
return host.succeed("journalctl -u dnsmasq -o cat --no-pager").count("DHCPACK(${bridge}) " + ip)
while now() < T0 + 450:
assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "a's traffic is dropped while it renews"
assert ct_ok("b", "ping -c1 -W2 ${gateway}"), "b's traffic is dropped while it renews"
host.sleep(15)
assert not spoof_lines(), spoof_lines()
assert ack_count(IP_A) >= 4 and ack_count(IP_B) >= 4, "each container renewed several times"
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
assert udp_reaches_host("a", IP_A), "a's own address still reaches the host after its first lease ended"
with subtest("another container's IPv4 address is dropped, and the line names the port"):
ct("a", f"ip addr add {IP_B}/24 dev eth0")
assert not udp_reaches_host("a", IP_B), "a datagram from b's address must not arrive"