From 96334845f04b5c58feac1fc4097bd11bdc9cb408 Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 19 Nov 2026 14:52:16 +0100 Subject: [PATCH] hive-egress: the spoof log line says which check failed Each refused frame now goes through its own chain, spoof_, whose log prefix names the check that refused it (mac, vlan, dhcp, v4, v6, arp). The unmetered log lines of all reasons share one named limit, spoof_unmetered, so the budget stays one for all ports and reasons. --- docs/4-implementation/networking/network.md | 17 +++--- nix/host-modules/hive-egress.nix | 64 +++++++++++++++------ nix/module-eval/core-toggle.nix | 51 +++++++++++----- 3 files changed, 93 insertions(+), 39 deletions(-) diff --git a/docs/4-implementation/networking/network.md b/docs/4-implementation/networking/network.md index 016e969e9..697df0978 100644 --- a/docs/4-implementation/networking/network.md +++ b/docs/4-implementation/networking/network.md @@ -864,13 +864,16 @@ source check lives in that table and the host-side filtering stays in - **The log.** A refused frame (all of them with `drop`, the same ones with `log`) gets a kernel log line with the prefix `hive-egress-spoof `, rate limited per port: a burst of 20, then one a second. A frame its port can't log - gets a line with the prefix `hive-egress-spoof-unmetered ` instead, from a limit - shared by all ports: 5 lines a second after a burst of 50. Read them with - `journalctl -k -g hive-egress-spoof`. `IN=` is the port, `vb-` (`vb-h-` for - an agent), and `SRC` and `MAC=::` are what the container - sent; the port names the sender, because a bound port sends only its own MAC. - The counters in `nft list chain bridge hive_egress spoof` count every refused - frame. + gets a line with the prefix `hive-egress-spoof-unmetered ` instead, from one + limit shared by all ports and reasons: 5 lines a second after a burst of 50. + Read them with `journalctl -k -g hive-egress-spoof`. The word after the + prefix is the check that refused the frame: `mac` (the port isn't bound to + the source MAC), `vlan`, `dhcp` (the `chaddr` isn't the port's), `v4`, `v6` or + `arp`; each has its own chain, `spoof_`, with its own counters. `IN=` + is the port, `vb-` (`vb-h-` for an agent), and `SRC` and + `MAC=::` are what the container sent; the port names the + sender, because a bound port sends only its own MAC. The counters in + `nft list chain bridge hive_egress spoof` count every refused frame. - **Why `log` is the default.** A container that sends from its leased address and its own MAC is never refused by `drop`, but only while the pair stays in the table, and a host suspend/resume empties it. `log` keeps the lines and diff --git a/nix/host-modules/hive-egress.nix b/nix/host-modules/hive-egress.nix index f1368e27e..e2a51b49c 100644 --- a/nix/host-modules/hive-egress.nix +++ b/nix/host-modules/hive-egress.nix @@ -190,8 +190,8 @@ let # The bridge table. A frame of a port goes through `prerouting` in this # order: the port must be bound to the frame's source MAC, then the frame's - # protocol decides which source address it may carry. Everything else ends in - # `spoof`, which logs and then drops or, with `antiSpoof = "log"`, accepts. + # protocol decides which source address it may carry. Everything else logs in + # `spoof_`, then `spoof` drops it (`antiSpoof = "log"`: accepts). # # - `port_mac` and `port_chaddr` hold the bound (port, MAC) pairs, the second # as the 48 bits that sit at the `chaddr` field of a DHCP message. A port @@ -215,9 +215,31 @@ let # Logging is per ingress port, in the style of the forward chain: a burst of # 20 lines, then one a second, and a port is forgotten after a minute. A # packet its port cannot log (over its rate, or the set is full) falls to the - # unmetered line, which has one budget for all ports. It logs - # packets, not flows: the bridge hook has no connection tracking. + # unmetered line, which has one budget for all ports and reasons: the named + # limit `spoof_unmetered`. It logs packets, not flows: the bridge hook has no + # connection tracking. spoofVerdict = if egressCfg.antiSpoof == "drop" then "drop" else "accept"; + + # The log prefix names the check that refused the frame: `mac` (the port is + # not bound to the source MAC), `vlan`, `dhcp` (the `chaddr` is not the + # port's), `v4`, `v6`, and `arp` (the ARP sender's MAC or address is not the + # port's). Each reason has its own chain and counters; `spoof` holds the + # verdict and one counter for every refused frame. + spoofReasons = [ + "mac" + "vlan" + "dhcp" + "v4" + "v6" + "arp" + ]; + spoofChain = reason: '' + chain spoof_${reason} { + update @spoof_ports { iifname limit rate ${logRate} } counter log prefix "hive-egress-spoof ${reason} " goto spoof + limit name "spoof_unmetered" counter log prefix "hive-egress-spoof-unmetered ${reason} " goto spoof + counter goto spoof + } + ''; bridgeTable = '' table bridge hive_egress { set port_mac { type ifname . ether_addr; } @@ -225,37 +247,39 @@ let set lease_v4 { type ether_addr . ipv4_addr; flags timeout; size 65536; } set spoof_ports { type ifname; flags dynamic; timeout 1m; size 4096; } + limit spoof_unmetered { rate ${unmeteredRate} } + chain spoof { - update @spoof_ports { iifname limit rate ${logRate} } counter log prefix "hive-egress-spoof " ${spoofVerdict} - limit rate ${unmeteredRate} counter log prefix "hive-egress-spoof-unmetered " ${spoofVerdict} counter ${spoofVerdict} } + ${lib.concatMapStringsSep "\n" spoofChain spoofReasons} + chain l3_v4 { - udp dport 67 iifname . @th,288,48 != @port_chaddr goto spoof + udp dport 67 iifname . @th,288,48 != @port_chaddr goto spoof_dhcp ip daddr { 255.255.255.255, ${cfg.bridgeIp} } udp sport 68 udp dport 67 accept ether saddr . ip saddr @lease_v4 accept - goto spoof + goto spoof_v4 } chain l3_v6 { ip6 saddr fe80::/10 accept ip6 saddr :: icmpv6 type { nd-neighbor-solicit, mld2-listener-report } accept - goto spoof + goto spoof_v6 } chain l3_arp { - iifname . arp saddr ether != @port_mac goto spoof + iifname . arp saddr ether != @port_mac goto spoof_arp arp saddr ip 0.0.0.0 accept arp saddr ether . arp saddr ip @lease_v4 accept - goto spoof + goto spoof_arp } chain prerouting { type filter hook prerouting priority -300; policy accept; meta ibrname != "${br}" accept - iifname . ether saddr != @port_mac goto spoof - ether type { 8021q, 8021ad } goto spoof + iifname . ether saddr != @port_mac goto spoof_mac + ether type { 8021q, 8021ad } goto spoof_vlan ether type ip goto l3_v4 ether type ip6 goto l3_v6 ether type arp goto l3_arp @@ -347,12 +371,14 @@ in sources (and `::` in neighbour discovery and listener reports). - `drop`: the frame is dropped and logged with the prefix - `hive-egress-spoof `, rate limited per port. A packet the per-port limit - cannot log gets a line with the prefix `hive-egress-spoof-unmetered ` - from a limit shared by all ports. A container sending from its leased - address is unaffected while its (MAC, IP) pair is in the lease set; a - lost pair, as after a host suspend/resume, cuts the container off until - it leases again. + `hive-egress-spoof `, rate limited per port; the reason is the + check that failed: `mac`, `vlan`, `dhcp`, `v4`, `v6` or `arp`. A packet + the per-port limit cannot log gets a line with the prefix + `hive-egress-spoof-unmetered `, from one limit shared by all + ports and reasons. A container sending from its leased address is + unaffected while its (MAC, IP) pair is in the lease set; a lost pair, + as after a host suspend/resume, cuts the container off until it leases + again. - `log` (the default): the same lines and counters, and the frame goes through. For finding out why a legitimate container is refused on a live hive. It is the default because the lease set does not yet survive diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 3165a68f7..0ff868188 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -1647,9 +1647,10 @@ let } { # The bridge table drops what a port sends from an address that is not - # its own, and logs it. The ruleset is read as text: the spoof chain - # ends every path in `drop`, and the accepts that remain are the ones that - # let the leased source through, which is the presence arm. + # its own, and logs it. The ruleset is read as text: each `spoof_` + # chain logs and goes to `spoof`, which ends every path in `drop`, and the + # accepts that remain are the ones that let the leased source through, + # which is the presence arm. name = "egress.antiSpoof drop renders the bridge table that drops frames whose source is not the port's own"; ok = let @@ -1659,22 +1660,46 @@ let name: lib.head (lib.splitString "chain" (lib.last (lib.splitString "chain ${name} {" bridgeText))); spoof = lib.filter (l: l != "" && l != "}") (map lib.trim (lib.splitString "\n" (section "spoof"))); ip4 = section "l3_v4"; + reasons = [ + "mac" + "vlan" + "dhcp" + "v4" + "v6" + "arp" + ]; + reasonLines = + r: lib.filter (l: l != "" && l != "}") (map lib.trim (lib.splitString "\n" (section "spoof_${r}"))); in egressForwardAccept.services.hyperhive.deploy.hive-controller.egress.antiSpoof == "drop" && lib.hasInfix "hook prerouting priority -300" bridgeText && lib.hasInfix ''meta ibrname != "hive-br0" accept'' bridgeText - && lib.hasInfix "iifname . ether saddr != @port_mac goto spoof" bridgeText - && lib.hasInfix "ether type { 8021q, 8021ad } goto spoof" bridgeText - && lib.hasInfix "udp dport 67 iifname . @th,288,48 != @port_chaddr goto spoof" bridgeText + && lib.hasInfix "iifname . ether saddr != @port_mac goto spoof_mac" bridgeText + && lib.hasInfix "ether type { 8021q, 8021ad } goto spoof_vlan" bridgeText + && lib.hasInfix "udp dport 67 iifname . @th,288,48 != @port_chaddr goto spoof_dhcp" bridgeText && lib.hasInfix "ether saddr . ip saddr @lease_v4 accept" bridgeText && lib.hasInfix "ip6 saddr fe80::/10 accept" bridgeText && lib.hasInfix "arp saddr ether . arp saddr ip @lease_v4 accept" bridgeText - && lib.hasInfix ''log prefix "hive-egress-spoof "'' bridgeText - && lib.hasInfix ''log prefix "hive-egress-spoof-unmetered "'' bridgeText - && lib.hasInfix "update @spoof_ports { iifname limit rate 1/second burst 20 packets }" bridgeText - && lib.length spoof == 3 + && lib.all ( + r: + lib.hasInfix ''log prefix "hive-egress-spoof ${r} "'' (section "spoof_${r}") + && lib.hasInfix ''log prefix "hive-egress-spoof-unmetered ${r} "'' (section "spoof_${r}") + && lib.hasInfix ''limit name "spoof_unmetered" counter log prefix "hive-egress-spoof-unmetered ${r} "'' ( + section "spoof_${r}" + ) + && lib.hasInfix "update @spoof_ports { iifname limit rate 1/second burst 20 packets }" ( + section "spoof_${r}" + ) + && lib.length (reasonLines r) == 3 + && lib.all (l: lib.hasSuffix " goto spoof" l) (reasonLines r) + ) reasons + && lib.hasInfix "limit spoof_unmetered { rate 5/second burst 50 packets }" bridgeText + && lib.length (lib.splitString "limit spoof_unmetered {" bridgeText) == 2 + && lib.length spoof == 1 && lib.all (l: lib.hasSuffix " drop" l) spoof - && lib.hasInfix "goto spoof" ip4; + && lib.hasInfix "goto spoof_v4" ip4 + && lib.hasInfix "goto spoof_v6" (section "l3_v6") + && lib.hasInfix "goto spoof_arp" (section "l3_arp"); } { # `log`, the default, renders the same chains with `accept` where `drop` @@ -1695,9 +1720,9 @@ let logged = spoofOf bare; in bare.services.hyperhive.deploy.hive-controller.egress.antiSpoof == "log" - && lib.length logged == 3 + && lib.length logged == 1 && lib.all (l: lib.hasSuffix " accept" l) logged - && lib.hasInfix ''log prefix "hive-egress-spoof "'' (textOf bare) + && lib.hasInfix ''log prefix "hive-egress-spoof mac "'' (textOf bare) && !(lib.any (l: lib.hasSuffix " drop" l) logged) && lib.all (l: lib.hasSuffix " drop" l) (spoofOf egressForwardAccept); }