From c4ac01ef8857ea8b8e694f0bf6b6653753bcf7fe Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 13 Nov 2026 02:36:38 +0100 Subject: [PATCH] agent bridge: drop frames whose source MAC or address isn't the container's own (egress.antiSpoof) --- docs/4-implementation/networking/network.md | 77 +++++- nix/host-modules/hive-egress.nix | 201 +++++++++++++- nix/host-modules/hive-gateway/dnsmasq.nix | 4 + nix/module-eval/core-toggle.nix | 85 +++++- nix/vm-tests.nix | 1 + nix/vm-tests/hive-egress-antispoof.nix | 288 ++++++++++++++++++++ nix/vm-tests/hive-squid.nix | 5 + 7 files changed, 636 insertions(+), 25 deletions(-) create mode 100644 nix/vm-tests/hive-egress-antispoof.nix diff --git a/docs/4-implementation/networking/network.md b/docs/4-implementation/networking/network.md index cc966df8d..cc1cb7988 100644 --- a/docs/4-implementation/networking/network.md +++ b/docs/4-implementation/networking/network.md @@ -519,6 +519,7 @@ the nix side sets up unconditionally: | Internet NAT | `networking.nat { enable = true; internalInterfaces = [ bridgeName ]; }` — MASQUERADE on packets leaving via any external NIC | | Loopback DROP | `nix/host-modules/hive-egress.nix` — own nftables table `inet hive_egress` (whichever firewall the host runs): drops bridge → `127.0.0.0/8` and `::1` traffic; the kernel default `route_localnet=0` is the first layer | | Forward log | `nix/host-modules/hive-egress.nix` — the same table's `forward` chain logs and counts what the host routes from the bridge to the uplink; it blocks nothing (see _Agent bridge egress log_ below) | +| Source check | `nix/host-modules/hive-egress.nix` — the `bridge hive_egress` table drops a frame whose source MAC or address differs from the sending container's own (see _Agent bridge source check_ below) | | Gateway access | `networking.firewall.interfaces..allowedTCPPorts = [ 80 443 ]` — lets isolated agents (private netns, veth on bridge) reach nginx on the host | | c0re signal | `HIVE_NETWORK_BRIDGE`, `HIVE_NETWORK_SUBNET` in `systemd.services.hive-c0re.environment` — both **required**; `hive-c0re` refuses to start without them | @@ -577,7 +578,8 @@ that never touches the network stack. The nix side also enables IP forwarding + NAT (agents reach the internet through the host) and drops bridge-subnet → loopback traffic from its own -nftables table (`inet hive_egress`, `nix/host-modules/hive-egress.nix`; +nftables table (`inet hive_egress`, `nix/host-modules/hive-egress.nix`, which also +holds the `bridge hive_egress` table of _Agent bridge source check_; defence-in-depth against a compromised agent reaching the c0re dashboard HTTP at `127.0.0.1`). Agents have no legitimate reason to reach the dashboard over loopback — the hive-c0re admin socket is a UDS, not TCP. @@ -616,14 +618,71 @@ and counts every packet. ` `. Look the `SRC` address up in the third column; the hostname names the container. A lease lasts one hour, so look an address up soon after the line appears. -- **A hint, not evidence.** Until anti-spoofing lands, a container that forges - its source address can mislead the log. It can file its flows under another - agent's address, use up that agent's burst, or fill the table of addresses so - that its own flows only reach the shared limit. It can also influence the - lease behind the address. Use the log to find out what agents send directly - now. Don't treat it as proof of which agent sent a flow, or as proof that a - flow is absent. Anti-spoofing has to land before the log serves as the soak - exit for enforcement. +- **Trust.** A frame that carries another container's source address never + reaches the forward chain while `antiSpoof` is `drop` (see _Agent bridge + source check_), so `SRC` names the container that sent the flow. In `log` + mode a container can still forge it, so a log from a host that ran in `log` + mode during the soak doesn't show who sent a flow. + +### Agent bridge source check + +Every container on the agent bridge is root in its own network namespace and +shares one L2 segment with the others, so it can send with another container's +MAC or address. `services.hyperhive.network.egress.antiSpoof` sets what the +`bridge hive_egress` table (`nix/host-modules/hive-egress.nix`) does about it. It +takes `drop` (the default) or `log`. Two tables, not one: the bridge carries traffic between two +ports, and the `inet` hooks never see it as coming from a port. Only a +`bridge`-family hook at `prerouting` sees which veth a frame came in on, so the +source check lives in that table and the host-side filtering stays in +`inet hive_egress`. + +- **What a port may send.** The frame's MAC is the one bound to the port. + Then, by protocol: IPv4 from an address leased to that MAC, or DHCP from + `0.0.0.0` to `255.255.255.255`; ARP whose sender is the port's MAC and an + address leased to it, or an ARP probe from `0.0.0.0`; IPv6 from a link-local + address (and from `::` in neighbour solicitations and listener reports, which + precede an address). A message to UDP port 67 must carry the port's own MAC as + its `chaddr`, so a container can't lease, renew or release an address for + another. The table refuses a VLAN tag, and everything else counts as a spoof. +- **A port nobody owns sends nothing.** Not even DHCP. A container's port is + bound a moment after its veth appears, and the container's DHCP client + retries until then. +- **Where the bindings come from.** Addresses are dynamic, so the table's sets + the commands below fill them at run time. `hive-priv egress-pin` binds a port to the MAC of the + link at the other end of its veth pair, found through the container systemd + registers for it. udev starts it for each new `vb-*` port + (`hive-egress-pin@.service`), and the `hive-egress` unit starts it for + every running container on each start and reload. A binding stays with the + port until the port goes: root in the container can't move the port to + another MAC afterwards (state in `/run/hive-egress-pins`). If two ports have + the same MAC, neither gets a binding. dnsmasq's `dhcp-script` (`hive-priv +egress-lease`) adds each lease as a (MAC, address) pair that times out with + the lease. +- **The window.** The binding takes the MAC the container's link has when the + command first looks, normally within a second of the veth appearing. Until + then root in the container could change its own MAC, but in that time only + the container's early init runs, which is nix-built, and the agent's code + starts later. A container that did change its MAC first would get its own + lease for the new MAC, not another container's: a MAC that two ports have + gets no binding, and a lease follows the `chaddr` the port may send. +- **The log.** A refused frame (all of them with `drop`, the same ones with + `log`) gets a kernel log line with the prefix `hive-egress-spoof `, rate + limited per port: a burst of 20, then one a second. A frame its port can't log + gets a line with the prefix `hive-egress-spoof-unmetered ` instead, from a limit + shared by all ports: 5 lines a second after a burst of 50. Read them with + `journalctl -k -g hive-egress-spoof`. `IN=` is the port, `vb-` (`vb-h-` for + an agent), and `SRC` and `MAC=::` are what the container + sent; the port names the sender, because a bound port sends only its own MAC. + The counters in `nft list chain bridge hive_egress spoof` count every refused + frame. +- **Why `drop` is the default.** A container that sends from its leased address + and its own MAC is never refused, so `drop` breaks nothing a container does + legitimately, and without it the egress log can't tell agents apart. `log` is + for a live hive where the table refuses a legitimate container (a binding that + failed shows as `hive-egress-pin` errors in the journal and `IN=` lines for + that port): the same lines and counters, nothing blocked. +- **What it doesn't cover.** A container can still send from any IPv6 + link-local address and answer DHCP as a server on its own leased address. Enforcement comes with later changes in the same series: the log only shows what such a change would cut. diff --git a/nix/host-modules/hive-egress.nix b/nix/host-modules/hive-egress.nix index cd91cff1a..8f8e4867f 100644 --- a/nix/host-modules/hive-egress.nix +++ b/nix/host-modules/hive-egress.nix @@ -1,17 +1,26 @@ -# The hive's own nftables table, `inet hive_egress`, loaded by a systemd -# oneshot. It is independent of the NixOS firewall: `networking.firewall` -# renders only the rules of the backend in use (`iptables` unless -# `networking.nftables.enable` is set), so a rule that has to hold on every -# host lives here, in a table no firewall backend writes. Only -# `networking.nftables.flushRuleset` removes it; the unit re-applies it then. +# The hive's own nftables tables, `inet hive_egress` and `bridge hive_egress`, +# loaded by a systemd oneshot. They are independent of the NixOS firewall: +# `networking.firewall` renders only the rules of the backend in use +# (`iptables` unless `networking.nftables.enable` is set), so a rule that has +# to hold on every host lives here, in tables no firewall backend writes. Only +# `networking.nftables.flushRuleset` removes them; the unit re-applies them then. # -# The table holds two chains. `input`: traffic from the agent bridge to host +# The `inet` table holds two chains. `input`: traffic from the agent bridge to host # loopback is dropped. The kernel does the same by default (a packet for # 127.0.0.0/8 arriving on a non-loopback interface is a martian while # `route_localnet` is 0); this rule is the second layer, and the assertion # below keeps the first one on. `forward` (`egress.forward = "log"`): traffic # the host routes from the agent bridge to anywhere else is logged and # counted, never blocked. +# +# The `bridge` table checks the source address of every frame a port of the +# agent bridge sends (`egress.antiSpoof`). It is a `bridge`-family table because +# traffic between two ports is bridged: the `inet` hooks see it as coming from +# the bridge device, and only a bridge hook knows the port it came in on +# (`iifname` is the veth). Which MAC a port may use and which IPv4 address a MAC +# may use are not known when the ruleset loads, so they are set contents that +# `hive-priv egress-pin` (per port, from udev) and dnsmasq's `dhcp-script` +# (per lease) fill; `hive-priv`'s `egress.rs` documents how. { lib, config, @@ -26,6 +35,22 @@ let # The mesh interface (swarm-wireguard.nix): inter-hive traffic is not egress. meshInterface = "wg-hive"; + # The sub-commands of hive-priv that fill the bridge table's sets. The path is + # the one every platform container's unit uses (hive-priv.nix points it at the + # current build). + hivePriv = (import ./lib/container-rootfs-check.nix { inherit config; }).hivePriv; + pinCommand = args: "${hivePriv} egress-pin ${args}"; + pinPath = [ + pkgs.nftables + pkgs.iproute2 + pkgs.util-linux + ]; + pinEnvironment.HIVE_EGRESS_BRIDGE = br; + leaseScript = pkgs.writeShellScript "hive-egress-lease" '' + export PATH=${lib.makeBinPath [ pkgs.nftables ]} + exec ${hivePriv} egress-lease "$@" + ''; + # Logging is per source address, so one busy agent cannot use up the budget # of another: a burst of 20 lines, then one a second, and an address is # forgotten after a minute without a new flow. The rule accepts after it @@ -64,12 +89,86 @@ let } ''; - # One transaction: `add` + `delete` drop an old copy of the table and the - # definition recreates it, so the file loads the same on first start, on a + # The bridge table. A frame of a port goes through `prerouting` in this + # order: the port must be bound to the frame's source MAC, then the frame's + # protocol decides which source address it may carry. Everything else ends in + # `spoof`, which logs and then drops or, with `antiSpoof = "log"`, accepts. + # + # - `port_mac` and `port_chaddr` hold the bound (port, MAC) pairs, the second + # as the 48 bits that sit at the `chaddr` field of a DHCP message. A port + # with no pair sends nothing, DHCP included. + # - `lease_v4` holds the leased (MAC, IPv4) pairs; each times out with its + # lease. + # - IPv6 has no leases: a source is link-local, or `::` in the packets that + # duplicate address detection and multicast listener reports send before an + # address exists. The bridge carries no other IPv6 traffic. + # - DHCP: any message to port 67 must carry the sender's own MAC as `chaddr`, + # so a port cannot lease, renew or release another's address; a client with + # no address yet sends from 0.0.0.0 to the broadcast address. + # - ARP: the sender's MAC is the port's, and its address 0.0.0.0 (a probe) or + # leased to that MAC. + # - A VLAN tag is refused: a tagged frame would carry its packet past the + # protocol checks. The bridge uses no VLANs. + # + # Logging is per ingress port, in the style of the forward chain: a burst of + # 20 lines, then one a second, and a port is forgotten after a minute. A + # packet its port cannot log (over its rate, or the set is full) falls to the + # unmetered line, which has one budget for all ports. It logs + # packets, not flows: the bridge hook has no connection tracking. + spoofVerdict = if cfg.egress.antiSpoof == "drop" then "drop" else "accept"; + bridgeTable = '' + table bridge hive_egress { + set port_mac { type ifname . ether_addr; } + set port_chaddr { typeof iifname . @th,288,48; } + set lease_v4 { type ether_addr . ipv4_addr; flags timeout; size 65536; } + set spoof_ports { type ifname; flags dynamic; timeout 1m; size 4096; } + + chain spoof { + update @spoof_ports { iifname limit rate ${logRate} } counter log prefix "hive-egress-spoof " ${spoofVerdict} + limit rate ${unmeteredRate} counter log prefix "hive-egress-spoof-unmetered " ${spoofVerdict} + counter ${spoofVerdict} + } + + chain l3_v4 { + udp dport 67 iifname . @th,288,48 != @port_chaddr goto spoof + ip saddr 0.0.0.0 ip daddr 255.255.255.255 udp sport 68 udp dport 67 accept + ether saddr . ip saddr @lease_v4 accept + goto spoof + } + + chain l3_v6 { + ip6 saddr fe80::/10 accept + ip6 saddr :: icmpv6 type { nd-neighbor-solicit, mld2-listener-report } accept + goto spoof + } + + chain l3_arp { + iifname . arp saddr ether != @port_mac goto spoof + arp saddr ip 0.0.0.0 accept + arp saddr ether . arp saddr ip @lease_v4 accept + goto spoof + } + + chain prerouting { + type filter hook prerouting priority -300; policy accept; + meta ibrname != "${br}" accept + iifname . ether saddr != @port_mac goto spoof + ether type { 8021q, 8021ad } goto spoof + ether type ip goto l3_v4 + ether type ip6 goto l3_v6 + ether type arp goto l3_arp + } + } + ''; + + # One transaction: `add` + `delete` drop an old copy of each table and the + # definitions recreate them, so the file loads the same on first start, on a # reload and after something else flushed the ruleset. rulesetText = '' add table inet hive_egress delete table inet hive_egress + add table bridge hive_egress + delete table bridge hive_egress table inet hive_egress { chain input { @@ -79,6 +178,8 @@ let } ${lib.optionalString (cfg.egress.forward == "log") forwardChain} } + + ${bridgeTable} ''; routeLocalnetKeys = [ @@ -117,12 +218,48 @@ in ''; }; + options.services.hyperhive.network.egress.antiSpoof = lib.mkOption { + type = lib.types.enum [ + "drop" + "log" + ]; + default = "drop"; + description = '' + What the `bridge hive_egress` table does with a frame whose source is not + the sending container's own. Every port of the agent bridge is checked: + the frame's MAC must be the one bound to the port when its container + started, and its IPv4 address the one leased to that MAC. A port with no + bound MAC sends nothing. Allowed besides the leased address: DHCP from + `0.0.0.0` whose `chaddr` is the port's MAC, ARP probes, and IPv6 link-local + sources (and `::` in neighbour discovery and listener reports). + + - `drop`: the frame is dropped and logged with the prefix + `hive-egress-spoof `, rate limited per port. A packet the per-port limit + cannot log gets a line with the prefix `hive-egress-spoof-unmetered ` + from a limit shared by all ports. A container sending from its leased + address is never affected, so nothing legitimate breaks. + - `log`: the same lines and counters, and the frame goes through. For + finding out why a legitimate container is refused on a live hive. + ''; + }; + + options.services.hyperhive.network.egress.leaseScript = lib.mkOption { + type = lib.types.path; + internal = true; + readOnly = true; + default = leaseScript; + defaultText = lib.literalMD "the script that loads a lease into the bridge table"; + description = '' + The program dnsmasq runs on each lease change (its `dhcp-script`). + ''; + }; + options.services.hyperhive.network.egress.ruleset = lib.mkOption { type = lib.types.lines; internal = true; readOnly = true; default = rulesetText; - defaultText = lib.literalMD "the `inet hive_egress` table definition"; + defaultText = lib.literalMD "the `inet hive_egress` and `bridge hive_egress` table definitions"; description = '' The nft file the `hive-egress` unit loads, as text. ''; @@ -151,23 +288,59 @@ in ]; systemd.services.hive-egress = { - description = "hive nftables table (agent bridge egress log, host loopback drop)"; + description = "hive nftables tables (agent bridge egress log and source check, host loopback drop)"; wantedBy = [ "multi-user.target" ]; wants = [ "network-pre.target" ]; before = [ "network-pre.target" ]; # `nftables.service` runs `flush ruleset` where `networking.nftables.flushRuleset` # is on: start after it, follow its restarts and re-apply on its reload. - after = [ "nftables.service" ]; + # The tmpfiles units link `hivePriv`. + after = [ + "nftables.service" + "systemd-tmpfiles-setup.service" + "systemd-tmpfiles-resetup.service" + ]; partOf = [ "nftables.service" ]; unitConfig.ReloadPropagatedFrom = [ "nftables.service" ]; reloadIfChanged = true; + path = pinPath; + environment = pinEnvironment; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = "${nft} -f ${ruleset}"; - ExecReload = "${nft} -f ${ruleset}"; - ExecStop = "-${nft} delete table inet hive_egress"; + # The sets of a freshly loaded bridge table are empty: bind the ports of + # the running containers and load the leases from dnsmasq's file. + ExecStartPost = pinCommand "--seed"; + ExecReload = [ + "${nft} -f ${ruleset}" + (pinCommand "--seed") + ]; + ExecStop = [ + "-${nft} delete table inet hive_egress" + "-${nft} delete table bridge hive_egress" + ]; }; }; + + # Binds one port of the bridge once its veth exists: the run waits for the + # container to register with systemd-machined. + systemd.services."hive-egress-pin@" = { + description = "bind agent bridge port %i to its container's MAC"; + wants = [ "hive-egress.service" ]; + after = [ "hive-egress.service" ]; + path = pinPath; + environment = pinEnvironment; + serviceConfig = { + Type = "oneshot"; + ExecStart = pinCommand "%i"; + }; + }; + + # nspawn names the host end of a container's veth `vb-` when the + # container joins a bridge. + services.udev.extraRules = '' + SUBSYSTEM=="net", ACTION=="add", KERNEL=="vb-*", TAG+="systemd", ENV{SYSTEMD_WANTS}+="hive-egress-pin@%k.service" + ''; }; } diff --git a/nix/host-modules/hive-gateway/dnsmasq.nix b/nix/host-modules/hive-gateway/dnsmasq.nix index 017b05a9d..7f33b208d 100644 --- a/nix/host-modules/hive-gateway/dnsmasq.nix +++ b/nix/host-modules/hive-gateway/dnsmasq.nix @@ -81,7 +81,11 @@ in # bridgeIp/bridgePrefixLength. All containers (agents and service # containers such as swarm-ci-runner) receive their IPs dynamically. dhcp-range = "${networkCfg.dhcpRangeStart},${networkCfg.dhcpRangeEnd},1h"; + # `hive-priv egress-pin --seed` reads this file back (hive-egress.nix). dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases"; + # Each lease change updates the leased (MAC, IPv4) pairs the bridge + # anti-spoofing table checks sources against (hive-egress.nix). + dhcp-script = toString networkCfg.egress.leaseScript; # No explicit upstream: non-hive queries follow dnsmasq's # resolv.conf default — the host's own `/etc/resolv.conf`, so the # hive always uses the host's resolvers and follows them live with diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 1801da103..bd39bdb7e 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -72,10 +72,13 @@ let # The two settings of `egress.forward` the ordinary host (`log`) does not # render, for ../host-modules/hive-egress.nix: `accept` leaves the table - # without a forward chain, `drop` is refused by an assertion. + # without a forward chain, `drop` is refused by an assertion. The `accept` + # host also sets `egress.antiSpoof = "log"`, the one setting the ordinary + # host (`drop`) does not render. egressForwardAccept = hive { network.enable = true; network.egress.forward = "accept"; + network.egress.antiSpoof = "log"; }; egressForwardDrop = hive { network.enable = true; @@ -1529,7 +1532,8 @@ let unit != null && unit.serviceConfig.Type == "oneshot" && lib.hasInfix "hive-egress.nft" unit.serviceConfig.ExecStart - && lib.hasInfix "delete table inet hive_egress" unit.serviceConfig.ExecStop + && lib.hasInfix "delete table inet hive_egress" (toString unit.serviceConfig.ExecStop) + && lib.hasInfix "delete table bridge hive_egress" (toString unit.serviceConfig.ExecStop) && bare.networking.firewall.extraInputRules == "" && !(centralToggleOff.systemd.services ? hive-egress); } @@ -1563,6 +1567,83 @@ let && !(lib.any (l: lib.hasSuffix " drop" l || lib.hasSuffix " reject" l) rules) && lib.hasInfix "chain input" text; } + { + # The bridge table drops what a port sends from an address that is not + # its own, and logs it. The ruleset is read as text: the spoof chain + # ends every path in `drop`, and the accepts that remain are the ones that + # let the leased source through, which is the presence arm. + name = "the default egress table drops frames whose source is not the port's own"; + ok = + let + text = bare.services.hyperhive.network.egress.ruleset; + bridgeText = lib.last (lib.splitString "table bridge hive_egress" text); + section = + name: lib.head (lib.splitString "chain" (lib.last (lib.splitString "chain ${name} {" bridgeText))); + spoof = lib.filter (l: l != "" && l != "}") (map lib.trim (lib.splitString "\n" (section "spoof"))); + ip4 = section "l3_v4"; + in + bare.services.hyperhive.network.egress.antiSpoof == "drop" + && lib.hasInfix "hook prerouting priority -300" bridgeText + && lib.hasInfix ''meta ibrname != "hive-br0" accept'' bridgeText + && lib.hasInfix "iifname . ether saddr != @port_mac goto spoof" bridgeText + && lib.hasInfix "ether type { 8021q, 8021ad } goto spoof" bridgeText + && lib.hasInfix "udp dport 67 iifname . @th,288,48 != @port_chaddr goto spoof" bridgeText + && lib.hasInfix "ether saddr . ip saddr @lease_v4 accept" bridgeText + && lib.hasInfix "ip6 saddr fe80::/10 accept" bridgeText + && lib.hasInfix "arp saddr ether . arp saddr ip @lease_v4 accept" bridgeText + && lib.hasInfix ''log prefix "hive-egress-spoof "'' bridgeText + && lib.hasInfix ''log prefix "hive-egress-spoof-unmetered "'' bridgeText + && lib.hasInfix "update @spoof_ports { iifname limit rate 1/second burst 20 packets }" bridgeText + && lib.length spoof == 3 + && lib.all (l: lib.hasSuffix " drop" l) spoof + && lib.hasInfix "goto spoof" ip4; + } + { + # `log` renders the same chains with `accept` where `drop` was: the lines + # and counters stay, the frame goes through. The `accept` host is the + # only fixture that sets it; `bare` is the control that the two differ. + name = "egress.antiSpoof log renders the spoof chain with accept and no drop"; + ok = + let + textOf = m: m.services.hyperhive.network.egress.ruleset; + spoofOf = + m: + let + t = lib.last (lib.splitString "chain spoof {" (textOf m)); + in + lib.filter (l: l != "" && l != "}") ( + map lib.trim (lib.splitString "\n" (lib.head (lib.splitString "chain" t))) + ); + logged = spoofOf egressForwardAccept; + in + egressForwardAccept.services.hyperhive.network.egress.antiSpoof == "log" + && lib.length logged == 3 + && lib.all (l: lib.hasSuffix " accept" l) logged + && lib.hasInfix ''log prefix "hive-egress-spoof "'' (textOf egressForwardAccept) + && !(lib.any (l: lib.hasSuffix " drop" l) logged) + && lib.all (l: lib.hasSuffix " drop" l) (spoofOf bare); + } + { + # The sets are filled at run time, from three places: the unit at start + # and reload, the udev rule for each new port, and dnsmasq for each + # lease. Each is present on `bare`; the host without the bridge has none. + name = "the bridge table's sets are filled by the unit, a udev rule per port and dnsmasq's lease script"; + ok = + let + unit = bare.systemd.services.hive-egress; + pin = bare.systemd.services."hive-egress-pin@"; + in + lib.hasInfix "egress-pin --seed" unit.serviceConfig.ExecStartPost + && lib.any (lib.hasInfix "egress-pin --seed") unit.serviceConfig.ExecReload + && unit.environment.HIVE_EGRESS_BRIDGE == "hive-br0" + && lib.hasInfix "egress-pin %i" pin.serviceConfig.ExecStart + && lib.hasInfix ''KERNEL=="vb-*"'' bare.services.udev.extraRules + && lib.hasInfix "hive-egress-pin@%k.service" bare.services.udev.extraRules + && + bare.services.dnsmasq.settings.dhcp-script + == toString bare.services.hyperhive.network.egress.leaseScript + && !(centralToggleOff.systemd.services ? "hive-egress-pin@"); + } { # `accept` renders the table without the forward chain and without the # log prefix; the input chain is the presence arm that the text it diff --git a/nix/vm-tests.nix b/nix/vm-tests.nix index bf7f90329..67790c44c 100644 --- a/nix/vm-tests.nix +++ b/nix/vm-tests.nix @@ -10,6 +10,7 @@ agent-ephemeral-rootfs = import ./vm-tests/agent-ephemeral-rootfs.nix { inherit pkgs hivePriv; }; wg-hive = import ./vm-tests/wg-hive.nix { inherit pkgs self; }; hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs; }; + hive-egress-antispoof = import ./vm-tests/hive-egress-antispoof.nix { inherit pkgs hivePriv; }; swarm-archive = import ./vm-tests/swarm-archive.nix { inherit pkgs; }; swarm-builder-offline = import ./vm-tests/swarm-builder-offline.nix { inherit pkgs self; }; full-setup = import ./vm-tests/full-setup.nix { inherit pkgs self; }; diff --git a/nix/vm-tests/hive-egress-antispoof.nix b/nix/vm-tests/hive-egress-antispoof.nix new file mode 100644 index 000000000..91b69b6f8 --- /dev/null +++ b/nix/vm-tests/hive-egress-antispoof.nix @@ -0,0 +1,288 @@ +# `vmTests..hive-egress-antispoof`: the source check of +# ../host-modules/hive-egress.nix on a real agent bridge, with two nspawn +# containers on it that take their addresses from the hive's dnsmasq +# (../host-modules/hive-gateway/dnsmasq.nix' lease script, the real +# `hive-priv egress-pin` and `egress-lease`). `hive-squid` cannot host this: its +# stand-in bridge is one NIC, so every agent shares one port. +# +# What it pins: +# - each port is bound to its container's MAC and each lease is in the +# table, without the test adding either; +# - a container sending from its own address reaches the host and the other +# container (the control for every drop below); +# - another container's IPv4 address, another container's MAC, an IPv6 address +# that is not link-local and a DHCP message with another's `chaddr` are +# dropped, and each leaves a `hive-egress-spoof ` line that names the port; +# - a DHCP renewal, a link-local IPv6 packet and a restored MAC go through; +# - a port with no bound MAC sends nothing, and the pin unit binds it again; +# - `antiSpoof = "log"`, reached by a reload of the unit, keeps the bindings +# and the leases, lets the same spoofs through and logs them. +{ pkgs, hivePriv }: +let + bridge = "hive0"; + gateway = "10.42.0.1"; + + # A DHCP DISCOVER by hand, unicast to dnsmasq, with the `chaddr` given as + # the argument. The container shares the host's store, so it runs from there. + bootp = pkgs.writeText "bootp.pl" '' + use IO::Socket::INET; + use Socket; + my $mac = pack("H12", join("", split(/:/, $ARGV[0]))); + my $s = IO::Socket::INET->new(Proto => "udp") or die "socket: $!"; + my $head = pack("C4 N n n a4 a4 a4 a4", 1, 1, 6, 0, 4660, 0, 0, "", "", "", ""); + my $msg = $head . $mac . ("\0" x 10) . ("\0" x 192) . pack("H8", "63825363") . pack("C4", 53, 1, 1, 255); + $s->send($msg, 0, pack_sockaddr_in(67, inet_aton("${gateway}"))) or die "send: $!"; + ''; + + hiveStubs = + { lib, ... }: + { + options.services.hyperhive = { + deploy.hive-controller.package = lib.mkOption { type = lib.types.package; }; + # Off: hive-priv.nix then adds only the link the pin command runs by. + deploy.hive-controller.enable = lib.mkOption { + type = lib.types.bool; + default = false; + }; + swarm.forge.domain = lib.mkOption { + type = lib.types.str; + default = "forge.t.local"; + }; + network = { + enable = lib.mkOption { + type = lib.types.bool; + default = true; + }; + bridgeName = lib.mkOption { + type = lib.types.str; + default = bridge; + }; + }; + }; + config.services.hyperhive.deploy.hive-controller.package = hivePriv; + }; + + # A container that takes its address by DHCP, as the agents do. + container = { + autoStart = true; + privateNetwork = true; + hostBridge = bridge; + config = + { lib, pkgs, ... }: + { + system.stateVersion = "25.11"; + networking.useDHCP = lib.mkForce true; + networking.dhcpcd.extraConfig = "noipv4ll"; + networking.dhcpcd.wait = "ipv4"; + networking.firewall.allowedUDPPorts = [ 68 ]; + environment.systemPackages = [ + pkgs.perl + pkgs.socat + ]; + }; + }; +in +pkgs.testers.runNixOSTest { + name = "hive-egress-antispoof"; + + nodes.host = + { config, lib, ... }: + { + imports = [ + hiveStubs + ../host-modules/hive-egress.nix + ../host-modules/hive-priv.nix + ]; + virtualisation.memorySize = 2048; + networking.bridges.${bridge}.interfaces = [ ]; + networking.interfaces.${bridge}.ipv4.addresses = [ + { + address = gateway; + prefixLength = 24; + } + ]; + # The test's own datagram listener, and dnsmasq's DHCP port. + networking.firewall.interfaces.${bridge} = { + allowedUDPPorts = [ + 67 + 9999 + ]; + }; + services.dnsmasq = { + enable = true; + resolveLocalQueries = false; + settings = { + interface = bridge; + bind-interfaces = true; + port = 0; + dhcp-range = "10.42.0.2,10.42.0.254,1h"; + dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases"; + dhcp-script = toString config.services.hyperhive.network.egress.leaseScript; + log-dhcp = true; + }; + }; + systemd.services.dnsmasq = { + wants = [ "network-addresses-${bridge}.service" ]; + after = [ "network-addresses-${bridge}.service" ]; + }; + environment.systemPackages = [ pkgs.socat ]; + + containers.a = container; + containers.b = container; + + specialisation.logmode.configuration = { + services.hyperhive.network.egress.antiSpoof = lib.mkForce "log"; + }; + }; + + testScript = '' + import re + import shlex + + TABLE = "bridge hive_egress" + + def ct(name, cmd): + return host.succeed(f"nixos-container run {name} -- sh -c {shlex.quote(cmd)}").strip() + + def ct_ok(name, cmd): + return host.execute(f"nixos-container run {name} -- sh -c {shlex.quote(cmd)}")[0] == 0 + + def mac(name): + return ct(name, "cat /sys/class/net/eth0/address") + + def ipv4(name): + out = ct(name, "ip -4 -o addr show dev eth0 scope global") + found = re.search(r"inet (10\.42\.0\.\d+)/", out) + assert found, out + return found.group(1) + + def spoof_lines(pattern=""): + out = host.succeed("journalctl -k -o cat --no-pager | grep 'hive-egress-spoof ' || true") + return [line for line in out.splitlines() if re.search(pattern, line)] + + def udp_reaches_host(name, src, v6=False): + """One datagram from the container, sent from `src`. True when the host got it.""" + host.succeed("rm -f /tmp/rx") + host.succeed( + "systemd-run --unit=rx --collect " + f"timeout 6 sh -c 'socat -u UDP{6 if v6 else 4}-RECV:9999 - > /tmp/rx'" + ) + host.sleep(1) + if v6: + ll = host.succeed("ip -6 -o addr show dev ${bridge} scope link").split()[3].split("/")[0] + target = f"UDP6-SENDTO:[{ll}%eth0]:9999,bind=[{src}]" + else: + target = f"UDP4-SENDTO:${gateway}:9999,bind={src}" + ct_ok(name, f"echo hi | socat -u - {target}") + host.sleep(1) + got = host.succeed("cat /tmp/rx").strip() == "hi" + host.execute("systemctl stop rx.service") + return got + + host.start() + host.wait_for_unit("multi-user.target") + host.wait_for_unit("dnsmasq.service") + host.wait_for_unit("hive-egress.service") + for name in ("a", "b"): + host.wait_for_unit(f"container@{name}.service") + host.wait_until_succeeds( + f"nixos-container run {name} -- sh -c 'ip -4 -o addr show dev eth0 | grep -q \"inet 10.42.0.\"'" + ) + MAC_A, MAC_B = mac("a"), mac("b") + IP_A, IP_B = ipv4("a"), ipv4("b") + assert MAC_A != MAC_B and IP_A != IP_B + + with subtest("each port is bound to its container's MAC and each lease is loaded, with no help from the test"): + host.wait_until_succeeds(f"nft list set {TABLE} port_mac | grep -q '\"vb-a\" . {MAC_A}'") + host.wait_until_succeeds(f"nft list set {TABLE} port_mac | grep -q '\"vb-b\" . {MAC_B}'") + host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'") + host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'") + assert not spoof_lines(), spoof_lines() + + with subtest("a container sending from its own address reaches the host and the other container"): + assert ct_ok("a", "ping -c1 -W2 ${gateway}") + assert ct_ok("a", f"ping -c1 -W2 {IP_B}") + assert ct_ok("b", f"ping -c1 -W2 {IP_A}") + assert udp_reaches_host("a", IP_A), "control: a datagram from a's own address arrives" + assert not spoof_lines(), spoof_lines() + + with subtest("another container's IPv4 address is dropped, and the line names the port"): + ct("a", f"ip addr add {IP_B}/24 dev eth0") + assert not udp_reaches_host("a", IP_B), "a datagram from b's address must not arrive" + host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*SRC=" + IP_B.replace(".", "\\.") + " '") + assert not spoof_lines(r"IN=vb-b "), "b sent nothing wrong" + ct("a", f"ip addr del {IP_B}/24 dev eth0") + assert udp_reaches_host("a", IP_A), "a's own address still works" + + with subtest("an IPv6 address that is not link-local is dropped, a link-local one is not"): + ct("a", "ip -6 addr add fd00::bad/64 dev eth0 nodad") + assert not udp_reaches_host("a", "fd00::bad", v6=True) + host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*SRC=fd00:0*:0*:0*:0*:0*:0*:0*bad '") + ll_a = ct("a", "ip -6 -o addr show dev eth0 scope link").split()[3].split("/")[0] + assert udp_reaches_host("a", ll_a, v6=True), "control: a link-local source goes through" + ct("a", "ip -6 addr del fd00::bad/64 dev eth0") + + with subtest("another container's MAC is dropped, and the restored MAC works again"): + ct("a", f"ip link set dev eth0 address {MAC_B}") + assert not ct_ok("a", "ping -c1 -W2 ${gateway}") + host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*MAC=[0-9a-f:]*" + MAC_B + ":'") + ct("a", f"ip link set dev eth0 address {MAC_A}") + assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "the port stays bound to the MAC it had" + assert ct_ok("b", "ping -c1 -W2 ${gateway}"), "b was never affected" + + with subtest("another container's MAC and IPv4 address together are dropped, so the leased pair alone is no credential"): + # A static neighbour entry for the host, so the datagram needs no ARP: + # only the check of the port's MAC is left between it and the host. + host_mac = host.succeed("cat /sys/class/net/${bridge}/address").strip() + ct("a", f"ip neigh replace ${gateway} lladdr {host_mac} dev eth0 nud permanent") + ct("a", f"ip link set dev eth0 address {MAC_B}") + ct("a", f"ip addr add {IP_B}/24 dev eth0") + seen = len(spoof_lines(r"IN=vb-a ")) + assert not udp_reaches_host("a", IP_B), "b's MAC and address from a's port must not arrive" + assert len(spoof_lines(r"IN=vb-a ")) > seen, "and it is logged against a's port" + ct("a", f"ip addr del {IP_B}/24 dev eth0") + ct("a", f"ip link set dev eth0 address {MAC_A}") + ct("a", "ip neigh del ${gateway} dev eth0") + assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "a works again with its own pair" + + with subtest("a DHCP message with another container's chaddr is dropped, one with its own is answered, and a renewal works"): + def discovers(mac_addr): + out = host.succeed("journalctl -u dnsmasq -o cat --no-pager") + return len(re.findall(r"DHCPDISCOVER\(${bridge}\) " + mac_addr, out)) + before_b, before_a = discovers(MAC_B), discovers(MAC_A) + ct("a", f"perl ${bootp} {MAC_B}") + ct("a", f"perl ${bootp} {MAC_A}") + host.wait_until_succeeds(f"journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPDISCOVER(${bridge}) {MAC_A}' | grep -qv '^{before_a}$'") + assert discovers(MAC_A) == before_a + 1, "the message with a's own chaddr reached dnsmasq" + assert discovers(MAC_B) == before_b, "the message with b's chaddr did not" + host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep -E 'hive-egress-spoof .*IN=vb-a .*DPT=67 '") + acks = host.succeed("journalctl -u dnsmasq -o cat --no-pager").count("DHCPACK(${bridge}) " + IP_A) + ct("a", "dhcpcd --rebind eth0") + host.wait_until_succeeds( + "test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'DHCPACK(${bridge}) " + IP_A + "') -gt " + str(acks) + ) + assert ipv4("a") == IP_A, "the renewal kept the address" + host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'") + assert ct_ok("a", "ping -c1 -W2 ${gateway}") + + with subtest("a port with no bound MAC sends nothing, and the pin unit binds it again"): + host.succeed(f"nft flush set {TABLE} port_mac") + assert not ct_ok("a", "ping -c1 -W2 ${gateway}") + assert not ct_ok("b", "ping -c1 -W2 ${gateway}") + host.succeed("systemctl start hive-egress-pin@vb-a.service hive-egress-pin@vb-b.service") + host.wait_until_succeeds("nixos-container run a -- ping -c1 -W2 ${gateway}") + host.wait_until_succeeds("nixos-container run b -- ping -c1 -W2 ${gateway}") + + with subtest("antiSpoof = log keeps the bindings and the leases, lets a spoofed datagram through and logs it"): + host.succeed("/run/current-system/specialisation/logmode/bin/switch-to-configuration test") + assert "counter accept" in host.succeed(f"nft list chain {TABLE} spoof"), "the table was reloaded in log mode" + host.wait_until_succeeds(f"nft list set {TABLE} port_mac | grep -q '\"vb-a\" . {MAC_A}'") + host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'") + assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "control: own address, after the reload" + seen = len(spoof_lines()) + ct("a", f"ip addr add {IP_B}/24 dev eth0") + assert udp_reaches_host("a", IP_B), "log mode lets the spoofed datagram through" + host.wait_until_succeeds("journalctl -k -o cat --no-pager | grep 'hive-egress-spoof ' | wc -l | grep -qv '^" + str(seen) + "$'") + assert spoof_lines(r"IN=vb-a .*SRC=" + IP_B.replace(".", r"\.") + " ") + ''; +} diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index a38b797f4..3ba24ed80 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -197,6 +197,11 @@ pkgs.testers.runNixOSTest { ); # The gateway: nginx answers on the bridge, which agents reach directly. networking.firewall.interfaces.eth1.allowedTCPPorts = [ 443 ]; + # hive-priv.nix links the binary the hive-egress unit runs to bind bridge + # ports; `eth1` is no bridge here, so a stub that binds nothing stands in. + systemd.tmpfiles.rules = [ + "L+ /run/hive-rootfs-check - - - - ${pkgs.writeShellScript "hive-priv-stub" "exit 0"}" + ]; # Renders the users file the way hive-c0re does. environment.systemPackages = [ pkgs.mkpasswd