From 60aede950b6c64e74726cb397bb8cc5984e546a3 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 14 Nov 2026 12:35:51 +0100 Subject: [PATCH] egress anti-spoof: drop by default again --- docs/4-implementation/networking/network.md | 16 +++++++------ nix/host-modules/hive-egress.nix | 10 ++++---- nix/module-eval/core-toggle.nix | 26 ++++++++++----------- nix/vm-tests/hive-egress-antispoof.nix | 10 +++++--- 4 files changed, 33 insertions(+), 29 deletions(-) diff --git a/docs/4-implementation/networking/network.md b/docs/4-implementation/networking/network.md index 3645d49d9..a1a6fe911 100644 --- a/docs/4-implementation/networking/network.md +++ b/docs/4-implementation/networking/network.md @@ -803,7 +803,7 @@ Every container on the agent bridge is root in its own network namespace and shares one L2 segment with the others, so it can send with another container's MAC or address. `services.hyperhive.network.egress.antiSpoof` sets what the `bridge hive_egress` table (`nix/host-modules/hive-egress.nix`) does about it. It -takes `drop` or `log` (the default, until the renewal fix has run on a live hive). Two tables, not one: the bridge carries traffic between two +takes `drop` (the default) or `log`. Two tables, not one: the bridge carries traffic between two ports, and the `inet` hooks never see it as coming from a port. Only a `bridge`-family hook at `prerouting` sees which veth a frame came in on, so the source check lives in that table and the host-side filtering stays in @@ -859,12 +859,14 @@ source check lives in that table and the host-side filtering stays in sent; the port names the sender, because a bound port sends only its own MAC. The counters in `nft list chain bridge hive_egress spoof` count every refused frame. -- **Why `log` is the default.** A container that sends from its leased address - and its own MAC is never refused by `drop`, but only while the pair stays in - the table. Until a live hive has run through many lease renewals with - `script-on-renewal` set without a `hive-egress-spoof` line for a legitimate - container, `log` keeps the lines and the counters and blocks nothing. `drop` - is what the table is for: without it the egress log can't tell agents apart. +- **Reading the log.** A container that sends from its leased address and its + own MAC never gets a line, so a line is a frame to look at. A container that + has no lease yet logs a few lines in its first seconds: DNS queries from + `0.0.0.0`, and the DHCP replies and ICMP redirects its own stack reflects + while the address isn't configured. They stop at the first lease, and `drop` + loses nothing by refusing them. Set `antiSpoof = "log"` to let the frames + through while you check whether a refusal is a false positive. `drop` is what + the table is for: without it the egress log can't tell agents apart. A binding that failed shows as `hive-egress-pin` errors in the journal and `IN=` lines for that port. - **What it doesn't cover.** A container can still send from any IPv6 diff --git a/nix/host-modules/hive-egress.nix b/nix/host-modules/hive-egress.nix index e094586ed..f038cf168 100644 --- a/nix/host-modules/hive-egress.nix +++ b/nix/host-modules/hive-egress.nix @@ -335,7 +335,7 @@ in "drop" "log" ]; - default = "log"; + default = "drop"; description = '' What the `bridge hive_egress` table does with a frame whose source is not the sending container's own. Every port of the agent bridge is checked: @@ -345,15 +345,13 @@ in `0.0.0.0` whose `chaddr` is the port's MAC, ARP probes, and IPv6 link-local sources (and `::` in neighbour discovery and listener reports). - - `drop`: the frame is dropped and logged with the prefix + - `drop` (the default): the frame is dropped and logged with the prefix `hive-egress-spoof `, rate limited per port. A packet the per-port limit cannot log gets a line with the prefix `hive-egress-spoof-unmetered ` from a limit shared by all ports. A container sending from its leased address is never affected, so nothing legitimate breaks. - - `log` (the default): the same lines and counters, and the frame goes - through. For finding out why a legitimate container is refused on a - live hive. The default goes back to `drop` once the renewal fix - (dnsmasq runs the lease script on a renewal) is proven live. + - `log`: the same lines and counters, and the frame goes through. For + finding out whether a suspected false positive of `drop` is one. With `log`, the identity squid takes from the source of an intercepted flow (`egress.intercept`) can be spoofed: use it to diagnose, not to run. diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index dc2cf4601..f4e8aa60e 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -73,12 +73,12 @@ let # The two settings of `egress.forward` the ordinary host (`log`) does not # render, for ../host-modules/hive-egress.nix: `accept` leaves the table # without a forward chain, `reject` renders the chain that refuses. The - # `accept` host also sets `egress.antiSpoof = "drop"`, the one setting the - # ordinary host (`log`) does not render. + # `accept` host also sets `egress.antiSpoof = "log"`, the one setting the + # ordinary host (`drop`) does not render. egressForwardAccept = hive { network.enable = true; network.egress.forward = "accept"; - network.egress.antiSpoof = "drop"; + network.egress.antiSpoof = "log"; }; egressForwardReject = hiveWithController { network.egress.forward = "reject"; @@ -1587,17 +1587,17 @@ let # its own, and logs it. The ruleset is read as text: the spoof chain # ends every path in `drop`, and the accepts that remain are the ones that # let the leased source through, which is the presence arm. - name = "egress.antiSpoof drop renders the bridge table that drops frames whose source is not the port's own"; + name = "egress.antiSpoof defaults to drop and renders the bridge table that drops frames whose source is not the port's own"; ok = let - text = egressForwardAccept.services.hyperhive.network.egress.ruleset; + text = bare.services.hyperhive.network.egress.ruleset; bridgeText = lib.last (lib.splitString "table bridge hive_egress" text); section = name: lib.head (lib.splitString "chain" (lib.last (lib.splitString "chain ${name} {" bridgeText))); spoof = lib.filter (l: l != "" && l != "}") (map lib.trim (lib.splitString "\n" (section "spoof"))); ip4 = section "l3_v4"; in - egressForwardAccept.services.hyperhive.network.egress.antiSpoof == "drop" + bare.services.hyperhive.network.egress.antiSpoof == "drop" && lib.hasInfix "hook prerouting priority -300" bridgeText && lib.hasInfix ''meta ibrname != "hive-br0" accept'' bridgeText && lib.hasInfix "iifname . ether saddr != @port_mac goto spoof" bridgeText @@ -1614,10 +1614,10 @@ let && lib.hasInfix "goto spoof" ip4; } { - # `log`, the default, renders the same chains with `accept` where `drop` + # `log` renders the same chains with `accept` where `drop` # was: the lines and counters stay, the frame goes through. The `accept` - # host sets `drop` and is the control that the two differ. - name = "egress.antiSpoof defaults to log and renders the spoof chain with accept and no drop"; + # host sets `log` and `bare` is the control that the two differ. + name = "egress.antiSpoof log renders the spoof chain with accept and no drop"; ok = let textOf = m: m.services.hyperhive.network.egress.ruleset; @@ -1629,14 +1629,14 @@ let lib.filter (l: l != "" && l != "}") ( map lib.trim (lib.splitString "\n" (lib.head (lib.splitString "chain" t))) ); - logged = spoofOf bare; + logged = spoofOf egressForwardAccept; in - bare.services.hyperhive.network.egress.antiSpoof == "log" + egressForwardAccept.services.hyperhive.network.egress.antiSpoof == "log" && lib.length logged == 3 && lib.all (l: lib.hasSuffix " accept" l) logged - && lib.hasInfix ''log prefix "hive-egress-spoof "'' (textOf bare) + && lib.hasInfix ''log prefix "hive-egress-spoof "'' (textOf egressForwardAccept) && !(lib.any (l: lib.hasSuffix " drop" l) logged) - && lib.all (l: lib.hasSuffix " drop" l) (spoofOf egressForwardAccept); + && lib.all (l: lib.hasSuffix " drop" l) (spoofOf bare); } { # The sets are filled at run time, from four places: the unit at start and diff --git a/nix/vm-tests/hive-egress-antispoof.nix b/nix/vm-tests/hive-egress-antispoof.nix index 264ee068d..a4eb537e2 100644 --- a/nix/vm-tests/hive-egress-antispoof.nix +++ b/nix/vm-tests/hive-egress-antispoof.nix @@ -6,7 +6,9 @@ # stand-in bridge is one NIC, so every agent shares one port. # # What it pins: -# - each port is bound to its container's MAC and each lease is in the +# - the default verdict is `drop`, and each container takes its first lease +# (DHCP from `0.0.0.0`, the reply, the first ARP and IP traffic) with it in +# place; each port is bound to its container's MAC and each lease is in the # table, without the test adding either; # - a container sending from its own address reaches the host and the other # container (the control for every drop below); @@ -15,7 +17,7 @@ # dropped, and each leaves a `hive-egress-spoof ` line that names the port; # - a container that keeps talking across several renewals of a 2-minute lease # is never dropped, and its pair is still in `lease_v4` after the first -# lease and its margin would have ended, with `antiSpoof = "drop"`; +# lease and its margin would have ended; # - a DHCP renewal, a link-local IPv6 packet and a restored MAC go through; # - a port with no bound MAC sends nothing, and the timer binds it again; # - `antiSpoof = "log"`, reached by a reload of the unit, brings the bindings @@ -98,7 +100,6 @@ pkgs.testers.runNixOSTest { ../host-modules/hive-priv.nix ]; virtualisation.memorySize = 2048; - services.hyperhive.network.egress.antiSpoof = "drop"; networking.bridges.${bridge}.interfaces = [ ]; networking.interfaces.${bridge}.ipv4.addresses = [ { @@ -190,6 +191,9 @@ pkgs.testers.runNixOSTest { host.wait_for_unit("multi-user.target") host.wait_for_unit("dnsmasq.service") host.wait_for_unit("hive-egress.service") + # The default verdict, loaded before either container exists: the first + # lease of each container below is taken with the drop rule in place. + assert "counter drop" in host.succeed(f"nft list chain {TABLE} spoof"), "the default is drop" for name in ("a", "b"): host.wait_for_unit(f"container@{name}.service") host.wait_until_succeeds(