From 5189e8fb4b223bcedae091df068562f3fbcc9e51 Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 13 Nov 2026 23:55:07 +0100 Subject: [PATCH] egress: refresh a leased pair on dhcp renewal and default anti-spoof to log dnsmasq runs the dhcp-script on a renewal only with script-on-renewal, so a pair loaded at the first lease timed out while the lease kept renewing and the bridge table dropped the container's traffic. Set script-on-renewal, time pairs out remaining + 300s, default antiSpoof to log, and run the vm test through renewals with drop. --- docs/4-implementation/networking/network.md | 21 ++++++---- hive-priv/src/egress.rs | 43 +++++++++++++++++---- nix/host-modules/hive-egress.nix | 17 ++++---- nix/host-modules/hive-gateway/dnsmasq.nix | 6 ++- nix/module-eval/core-toggle.nix | 29 +++++++------- nix/vm-test-typecheck.nix | 1 + nix/vm-tests/hive-egress-antispoof.nix | 29 +++++++++++++- 7 files changed, 107 insertions(+), 39 deletions(-) diff --git a/docs/4-implementation/networking/network.md b/docs/4-implementation/networking/network.md index 80180dbb0..3645d49d9 100644 --- a/docs/4-implementation/networking/network.md +++ b/docs/4-implementation/networking/network.md @@ -803,7 +803,7 @@ Every container on the agent bridge is root in its own network namespace and shares one L2 segment with the others, so it can send with another container's MAC or address. `services.hyperhive.network.egress.antiSpoof` sets what the `bridge hive_egress` table (`nix/host-modules/hive-egress.nix`) does about it. It -takes `drop` (the default) or `log`. Two tables, not one: the bridge carries traffic between two +takes `drop` or `log` (the default, until the renewal fix has run on a live hive). Two tables, not one: the bridge carries traffic between two ports, and the `inet` hooks never see it as coming from a port. Only a `bridge`-family hook at `prerouting` sees which veth a frame came in on, so the source check lives in that table and the host-side filtering stays in @@ -837,7 +837,10 @@ source check lives in that table and the host-side filtering stays in that a binding already holds, so a newcomer can't evict an established container. If two new ports claim one MAC, neither gets a binding. dnsmasq's `dhcp-script` (`hive-priv egress-lease`) adds each lease as a (MAC, address) - pair that times out with the lease. + pair that times out with the lease plus 5 minutes. `script-on-renewal` makes + dnsmasq run it on every renewal too, which loads the pair again with the + new remaining time; without it a pair would leave the table when the first + lease ended, while dnsmasq kept renewing the lease. - **The window.** The binding takes the MAC the container's link has when the command first looks, normally within a second of the veth appearing. Until then root in the container could change its own MAC, but in that time only @@ -856,12 +859,14 @@ source check lives in that table and the host-side filtering stays in sent; the port names the sender, because a bound port sends only its own MAC. The counters in `nft list chain bridge hive_egress spoof` count every refused frame. -- **Why `drop` is the default.** A container that sends from its leased address - and its own MAC is never refused, so `drop` breaks nothing a container does - legitimately, and without it the egress log can't tell agents apart. `log` is - for a live hive where the table refuses a legitimate container (a binding that - failed shows as `hive-egress-pin` errors in the journal and `IN=` lines for - that port): the same lines and counters, nothing blocked. +- **Why `log` is the default.** A container that sends from its leased address + and its own MAC is never refused by `drop`, but only while the pair stays in + the table. Until a live hive has run through many lease renewals with + `script-on-renewal` set without a `hive-egress-spoof` line for a legitimate + container, `log` keeps the lines and the counters and blocks nothing. `drop` + is what the table is for: without it the egress log can't tell agents apart. + A binding that failed shows as `hive-egress-pin` errors in the journal and + `IN=` lines for that port. - **What it doesn't cover.** A container can still send from any IPv6 link-local address and answer DHCP as a server on its own leased address. diff --git a/hive-priv/src/egress.rs b/hive-priv/src/egress.rs index 7d3e88c13..5f40d11ba 100644 --- a/hive-priv/src/egress.rs +++ b/hive-priv/src/egress.rs @@ -71,6 +71,10 @@ const POLL: Duration = Duration::from_millis(250); /// A lease timeout when dnsmasq names no remaining time. const DEFAULT_LEASE_SECS: u64 = 3600; +/// Seconds a leased pair outlives the lease it was loaded for. A renewal whose +/// script run comes a little late then finds the pair still in the set. +const LEASE_MARGIN_SECS: u64 = 300; + /// Where the bindings are kept between passes, when `HIVE_EGRESS_STATE` names /// no other file. It is on tmpfs: a reboot starts every port afresh. const STATE: &str = "/run/hive-egress-pins"; @@ -408,9 +412,10 @@ fn whois() -> Result<()> { /// The nft input for one lease event, or `None` for an event that changes /// nothing: `init`, an IPv6 address, an address or MAC that does not parse. /// -/// `add` and `old` replace the pair with one that times out after -/// `remaining_secs`; `del` removes it. The first `add element` makes the -/// `delete element` that follows succeed whether or not the pair was there. +/// `add` and `old` replace the pair with one that times out `remaining_secs` +/// plus [`LEASE_MARGIN_SECS`] from now, so every renewal moves the timeout +/// forward; `del` removes it. The first `add element` makes the `delete +/// element` that follows succeed whether or not the pair was there. pub(crate) fn lease_script( action: &str, mac: &str, @@ -421,11 +426,12 @@ pub(crate) fn lease_script( let ip: std::net::Ipv4Addr = ip.parse().ok()?; let set = format!("{TABLE} lease_v4"); let pair = format!("{mac} . {ip}"); + let timeout = remaining_secs.saturating_add(LEASE_MARGIN_SECS); match action { "add" | "old" => Some(format!( "add element {set} {{ {pair} }}\n\ delete element {set} {{ {pair} }}\n\ - add element {set} {{ {pair} timeout {remaining_secs}s }}\n" + add element {set} {{ {pair} timeout {timeout}s }}\n" )), "del" => Some(format!( "add element {set} {{ {pair} }}\ndelete element {set} {{ {pair} }}\n" @@ -1033,7 +1039,7 @@ mod tests { let script = &calls[0]; assert!(script.starts_with("table bridge hive_egress { }")); assert!(script.contains("port_mac { \"vb-a\" . 02:00:00:00:00:0a }")); - assert!(script.contains("lease_v4 { 02:00:00:00:00:0a . 10.42.0.10 timeout 90s }")); + assert!(script.contains("lease_v4 { 02:00:00:00:00:0a . 10.42.0.10 timeout 390s }")); assert!(!script.contains("flush set"), "the sets are new"); } @@ -1239,7 +1245,7 @@ mod tests { #[test] fn lease_events_become_nft_input() { let add = lease_script("add", "02:00:00:00:00:0a", "10.42.0.10", 90).unwrap(); - assert!(add.ends_with("add element bridge hive_egress lease_v4 { 02:00:00:00:00:0a . 10.42.0.10 timeout 90s }\n")); + assert!(add.ends_with("add element bridge hive_egress lease_v4 { 02:00:00:00:00:0a . 10.42.0.10 timeout 390s }\n")); assert!(add.contains("delete element")); assert_eq!( lease_script("old", "02:00:00:00:00:0a", "10.42.0.10", 90), @@ -1258,6 +1264,27 @@ mod tests { assert_eq!(lease_script("add", "nonsense", "10.42.0.10", 90), None); } + #[test] + fn a_renewal_moves_the_timeout_to_the_new_remaining_time_plus_the_margin() { + let mac = "02:00:00:00:00:0a"; + let first = lease_script("add", mac, "10.42.0.10", 120).unwrap(); + let renewed = lease_script("old", mac, "10.42.0.10", 120).unwrap(); + let later = lease_script("old", mac, "10.42.0.10", 3600).unwrap(); + assert!(first.ends_with("10.42.0.10 timeout 420s }\n")); + assert_eq!(renewed, first, "old and add load the same pair"); + assert!(later.ends_with("10.42.0.10 timeout 3900s }\n")); + assert!( + later.starts_with("add element bridge hive_egress lease_v4 { 02:00:00:00:00:0a . 10.42.0.10 }\ndelete element"), + "the old timeout is deleted first, so the new one replaces it" + ); + } + + #[test] + fn a_remaining_time_that_overflows_with_the_margin_does_not_wrap() { + let script = lease_script("old", "02:00:00:00:00:0a", "10.42.0.10", u64::MAX).unwrap(); + assert!(script.contains(&format!("timeout {}s", u64::MAX))); + } + #[test] fn seeding_skips_expired_and_malformed_leases() { let leases = "2000 02:00:00:00:00:0a 10.42.0.10 a *\n\ @@ -1265,9 +1292,9 @@ mod tests { 0 02:00:00:00:00:0c 10.42.0.12 c *\n\ garbage\n"; let script = seed_script(leases, 1000); - assert!(script.contains("02:00:00:00:00:0a . 10.42.0.10 timeout 1000s")); + assert!(script.contains("02:00:00:00:00:0a . 10.42.0.10 timeout 1300s")); assert!(!script.contains("10.42.0.11")); - assert!(script.contains("10.42.0.12 timeout 3600s")); + assert!(script.contains("10.42.0.12 timeout 3900s")); } #[test] diff --git a/nix/host-modules/hive-egress.nix b/nix/host-modules/hive-egress.nix index fc04c0ab2..e094586ed 100644 --- a/nix/host-modules/hive-egress.nix +++ b/nix/host-modules/hive-egress.nix @@ -26,8 +26,8 @@ # may use are not known until a container runs, so they are set contents. # `hive-priv egress-load` puts the current ones into the same transaction as the # tables; `egress-pin` (per new port from udev, and from a timer for the ports -# still unbound) and dnsmasq's `dhcp-script` (per lease) keep them current. -# `hive-priv`'s `egress.rs` documents how. +# still unbound) and dnsmasq's `dhcp-script` (per lease and renewal) keep them +# current. `hive-priv`'s `egress.rs` documents how. { lib, config, @@ -200,7 +200,7 @@ let # as the 48 bits that sit at the `chaddr` field of a DHCP message. A port # with no pair sends nothing, DHCP included. # - `lease_v4` holds the leased (MAC, IPv4) pairs; each times out with its - # lease. + # lease, and a renewal moves the timeout. # - IPv6 has no leases: a source is link-local, or `::` in the packets that # duplicate address detection and multicast listener reports send before an # address exists. The bridge carries no other IPv6 traffic. @@ -335,7 +335,7 @@ in "drop" "log" ]; - default = "drop"; + default = "log"; description = '' What the `bridge hive_egress` table does with a frame whose source is not the sending container's own. Every port of the agent bridge is checked: @@ -350,8 +350,10 @@ in cannot log gets a line with the prefix `hive-egress-spoof-unmetered ` from a limit shared by all ports. A container sending from its leased address is never affected, so nothing legitimate breaks. - - `log`: the same lines and counters, and the frame goes through. For - finding out why a legitimate container is refused on a live hive. + - `log` (the default): the same lines and counters, and the frame goes + through. For finding out why a legitimate container is refused on a + live hive. The default goes back to `drop` once the renewal fix + (dnsmasq runs the lease script on a renewal) is proven live. With `log`, the identity squid takes from the source of an intercepted flow (`egress.intercept`) can be spoofed: use it to diagnose, not to run. @@ -448,7 +450,8 @@ in default = leaseScript; defaultText = lib.literalMD "the script that loads a lease into the bridge table"; description = '' - The program dnsmasq runs on each lease change (its `dhcp-script`). + The program dnsmasq runs on each lease change and renewal (its + `dhcp-script` with `script-on-renewal`). ''; }; diff --git a/nix/host-modules/hive-gateway/dnsmasq.nix b/nix/host-modules/hive-gateway/dnsmasq.nix index a9844503d..f8f02729f 100644 --- a/nix/host-modules/hive-gateway/dnsmasq.nix +++ b/nix/host-modules/hive-gateway/dnsmasq.nix @@ -84,8 +84,12 @@ in # `hive-priv egress-load` reads this file back (hive-egress.nix). dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases"; # Each lease change updates the leased (MAC, IPv4) pairs the bridge - # anti-spoofing table checks sources against (hive-egress.nix). + # anti-spoofing table checks sources against (hive-egress.nix). A pair + # times out with its lease, so the script must also run on a renewal: + # without `script-on-renewal` dnsmasq runs it only for a new lease, and a + # renewed one leaves the table when the first lease would have ended. dhcp-script = toString networkCfg.egress.leaseScript; + script-on-renewal = true; # No explicit upstream: non-hive queries follow dnsmasq's # resolv.conf default — the host's own `/etc/resolv.conf`, so the # hive always uses the host's resolvers and follows them live with diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index eb14b0b91..dc2cf4601 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -73,12 +73,12 @@ let # The two settings of `egress.forward` the ordinary host (`log`) does not # render, for ../host-modules/hive-egress.nix: `accept` leaves the table # without a forward chain, `reject` renders the chain that refuses. The - # `accept` host also sets `egress.antiSpoof = "log"`, the one setting the - # ordinary host (`drop`) does not render. + # `accept` host also sets `egress.antiSpoof = "drop"`, the one setting the + # ordinary host (`log`) does not render. egressForwardAccept = hive { network.enable = true; network.egress.forward = "accept"; - network.egress.antiSpoof = "log"; + network.egress.antiSpoof = "drop"; }; egressForwardReject = hiveWithController { network.egress.forward = "reject"; @@ -1587,17 +1587,17 @@ let # its own, and logs it. The ruleset is read as text: the spoof chain # ends every path in `drop`, and the accepts that remain are the ones that # let the leased source through, which is the presence arm. - name = "the default egress table drops frames whose source is not the port's own"; + name = "egress.antiSpoof drop renders the bridge table that drops frames whose source is not the port's own"; ok = let - text = bare.services.hyperhive.network.egress.ruleset; + text = egressForwardAccept.services.hyperhive.network.egress.ruleset; bridgeText = lib.last (lib.splitString "table bridge hive_egress" text); section = name: lib.head (lib.splitString "chain" (lib.last (lib.splitString "chain ${name} {" bridgeText))); spoof = lib.filter (l: l != "" && l != "}") (map lib.trim (lib.splitString "\n" (section "spoof"))); ip4 = section "l3_v4"; in - bare.services.hyperhive.network.egress.antiSpoof == "drop" + egressForwardAccept.services.hyperhive.network.egress.antiSpoof == "drop" && lib.hasInfix "hook prerouting priority -300" bridgeText && lib.hasInfix ''meta ibrname != "hive-br0" accept'' bridgeText && lib.hasInfix "iifname . ether saddr != @port_mac goto spoof" bridgeText @@ -1614,10 +1614,10 @@ let && lib.hasInfix "goto spoof" ip4; } { - # `log` renders the same chains with `accept` where `drop` was: the lines - # and counters stay, the frame goes through. The `accept` host is the - # only fixture that sets it; `bare` is the control that the two differ. - name = "egress.antiSpoof log renders the spoof chain with accept and no drop"; + # `log`, the default, renders the same chains with `accept` where `drop` + # was: the lines and counters stay, the frame goes through. The `accept` + # host sets `drop` and is the control that the two differ. + name = "egress.antiSpoof defaults to log and renders the spoof chain with accept and no drop"; ok = let textOf = m: m.services.hyperhive.network.egress.ruleset; @@ -1629,14 +1629,14 @@ let lib.filter (l: l != "" && l != "}") ( map lib.trim (lib.splitString "\n" (lib.head (lib.splitString "chain" t))) ); - logged = spoofOf egressForwardAccept; + logged = spoofOf bare; in - egressForwardAccept.services.hyperhive.network.egress.antiSpoof == "log" + bare.services.hyperhive.network.egress.antiSpoof == "log" && lib.length logged == 3 && lib.all (l: lib.hasSuffix " accept" l) logged - && lib.hasInfix ''log prefix "hive-egress-spoof "'' (textOf egressForwardAccept) + && lib.hasInfix ''log prefix "hive-egress-spoof "'' (textOf bare) && !(lib.any (l: lib.hasSuffix " drop" l) logged) - && lib.all (l: lib.hasSuffix " drop" l) (spoofOf bare); + && lib.all (l: lib.hasSuffix " drop" l) (spoofOf egressForwardAccept); } { # The sets are filled at run time, from four places: the unit at start and @@ -1664,6 +1664,7 @@ let && lib.elem (toString bare.services.hyperhive.network.egress.leaseScript) ( lib.toList bare.services.dnsmasq.settings.dhcp-script ) + && lib.toList bare.services.dnsmasq.settings.script-on-renewal == [ true ] && !(centralToggleOff.systemd.services ? "hive-egress-pin@") && !(centralToggleOff.systemd ? timers && centralToggleOff.systemd.timers ? hive-egress-pin); } diff --git a/nix/vm-test-typecheck.nix b/nix/vm-test-typecheck.nix index 506170333..3acec075c 100644 --- a/nix/vm-test-typecheck.nix +++ b/nix/vm-test-typecheck.nix @@ -128,6 +128,7 @@ let tests = { inherit (vmTests) hive-squid + hive-egress-antispoof swarm-archive agent-ephemeral-rootfs full-setup diff --git a/nix/vm-tests/hive-egress-antispoof.nix b/nix/vm-tests/hive-egress-antispoof.nix index 7eefe97a5..264ee068d 100644 --- a/nix/vm-tests/hive-egress-antispoof.nix +++ b/nix/vm-tests/hive-egress-antispoof.nix @@ -13,6 +13,9 @@ # - another container's IPv4 address, another container's MAC, an IPv6 address # that is not link-local and a DHCP message with another's `chaddr` are # dropped, and each leaves a `hive-egress-spoof ` line that names the port; +# - a container that keeps talking across several renewals of a 2-minute lease +# is never dropped, and its pair is still in `lease_v4` after the first +# lease and its margin would have ended, with `antiSpoof = "drop"`; # - a DHCP renewal, a link-local IPv6 packet and a restored MAC go through; # - a port with no bound MAC sends nothing, and the timer binds it again; # - `antiSpoof = "log"`, reached by a reload of the unit, brings the bindings @@ -95,6 +98,7 @@ pkgs.testers.runNixOSTest { ../host-modules/hive-priv.nix ]; virtualisation.memorySize = 2048; + services.hyperhive.network.egress.antiSpoof = "drop"; networking.bridges.${bridge}.interfaces = [ ]; networking.interfaces.${bridge}.ipv4.addresses = [ { @@ -116,9 +120,11 @@ pkgs.testers.runNixOSTest { interface = bridge; bind-interfaces = true; port = 0; - dhcp-range = "10.42.0.2,10.42.0.254,1h"; + # dnsmasq's shortest lease: the test runs through renewals in minutes. + dhcp-range = "10.42.0.2,10.42.0.254,2m"; dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases"; dhcp-script = toString config.services.hyperhive.network.egress.leaseScript; + script-on-renewal = true; log-dhcp = true; }; }; @@ -192,6 +198,7 @@ pkgs.testers.runNixOSTest { MAC_A, MAC_B = mac("a"), mac("b") IP_A, IP_B = ipv4("a"), ipv4("b") assert MAC_A != MAC_B and IP_A != IP_B + T0 = int(host.succeed("date +%s").strip()) with subtest("each port is bound to its container's MAC and each lease is loaded, with no help from the test"): host.wait_until_succeeds(f"nft list set {TABLE} port_mac | grep -q '\"vb-a\" . {MAC_A}'") @@ -207,6 +214,26 @@ pkgs.testers.runNixOSTest { assert udp_reaches_host("a", IP_A), "control: a datagram from a's own address arrives" assert not spoof_lines(), spoof_lines() + with subtest("a container that keeps talking across renewals is never dropped, and its pair outlives the first lease and its margin"): + # The leases run 2 minutes and the pair outlives its lease by 300 + # seconds (LEASE_MARGIN_SECS in hive-priv's egress.rs). The pairs were + # loaded before T0, so `T0 + 420` is the latest the first load alone + # would have kept them; the loop runs past it, and only the refresh at + # each renewal keeps them in the set. + def now(): + return int(host.succeed("date +%s").strip()) + def ack_count(ip): + return host.succeed("journalctl -u dnsmasq -o cat --no-pager").count("DHCPACK(${bridge}) " + ip) + while now() < T0 + 450: + assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "a's traffic is dropped while it renews" + assert ct_ok("b", "ping -c1 -W2 ${gateway}"), "b's traffic is dropped while it renews" + host.sleep(15) + assert not spoof_lines(), spoof_lines() + assert ack_count(IP_A) >= 4 and ack_count(IP_B) >= 4, "each container renewed several times" + host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'") + host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'") + assert udp_reaches_host("a", IP_A), "a's own address still reaches the host after its first lease ended" + with subtest("another container's IPv4 address is dropped, and the line names the port"): ct("a", f"ip addr add {IP_B}/24 dev eth0") assert not udp_reaches_host("a", IP_B), "a datagram from b's address must not arrive"