mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

egress anti-spoofing: keep established bindings, load tables with bindings and leases in one transaction

Browse source
This commit is contained in:
atlas 2026-11-13 02:53:37 +01:00 • committed by mara
commit 70179b6c1f
5 changed files with 400 additions and 134 deletions
Download patch file Download diff file Expand all files Collapse all files

48
nix/host-modules/hive-egress.nix
View file

@ -18,9 +18,11 @@
# traffic between two ports is bridged: the `inet` hooks see it as coming from
# the bridge device, and only a bridge hook knows the port it came in on
# (`iifname` is the veth). Which MAC a port may use and which IPv4 address a MAC
# may use are not known when the ruleset loads, so they are set contents that
# `hive-priv egress-pin` (per port, from udev) and dnsmasq's `dhcp-script`
# (per lease) fill; `hive-priv`'s `egress.rs` documents how.
# may use are not known until a container runs, so they are set contents.
# `hive-priv egress-load` puts the current ones into the same transaction as the
# tables; `egress-pin` (per new port from udev, and from a timer for the ports
# still unbound) and dnsmasq's `dhcp-script` (per lease) keep them current.
# `hive-priv`'s `egress.rs` documents how.
{
lib,
config,
@ -39,7 +41,8 @@ let
# the one every platform container's unit uses (hive-priv.nix points it at the
# current build).
hivePriv = (import ./lib/container-rootfs-check.nix { inherit config; }).hivePriv;
pinCommand = args: "${hivePriv} egress-pin ${args}";
pinCommand = args: lib.trim "${hivePriv} egress-pin ${args}";
loadCommand = "${hivePriv} egress-load ${ruleset}";
pinPath = [
pkgs.nftables
pkgs.iproute2
@ -308,14 +311,11 @@ in
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = "${nft} -f ${ruleset}";
# The sets of a freshly loaded bridge table are empty: bind the ports of
# the running containers and load the leases from dnsmasq's file.
ExecStartPost = pinCommand "--seed";
ExecReload = [
"${nft} -f ${ruleset}"
(pinCommand "--seed")
];
# The tables load with the bindings of the running containers' ports and
# the leases of dnsmasq's file in the same transaction, so a reload never
# leaves the bridge with empty sets.
ExecStart = loadCommand;
ExecReload = loadCommand;
ExecStop = [
"-${nft} delete table inet hive_egress"
"-${nft} delete table bridge hive_egress"
@ -337,6 +337,30 @@ in
};
};
# Looks again at the ports still unbound. A port a rival MAC kept unbound
# sends nothing, so nothing else would tell the hive it can be bound now.
# Ports with a binding cost a listing of the bridge and no container is read.
systemd.services.hive-egress-pin = {
description = "bind the agent bridge ports that have no binding yet";
wants = [ "hive-egress.service" ];
after = [ "hive-egress.service" ];
path = pinPath;
environment = pinEnvironment;
serviceConfig = {
Type = "oneshot";
ExecStart = pinCommand "";
};
};
systemd.timers.hive-egress-pin = {
description = "bind the agent bridge ports that have no binding yet";
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = "1min";
OnUnitActiveSec = "30s";
AccuracySec = "5s";
};
};
# nspawn names the host end of a container's veth `vb-<machine>` when the
# container joins a bridge.
services.udev.extraRules = ''

2
nix/host-modules/hive-gateway/dnsmasq.nix
View file

@ -81,7 +81,7 @@ in
# bridgeIp/bridgePrefixLength. All containers (agents and service
# containers such as swarm-ci-runner) receive their IPs dynamically.
dhcp-range = "${networkCfg.dhcpRangeStart},${networkCfg.dhcpRangeEnd},1h";
# `hive-priv egress-pin --seed` reads this file back (hive-egress.nix).
# `hive-priv egress-load` reads this file back (hive-egress.nix).
dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases";
# Each lease change updates the leased (MAC, IPv4) pairs the bridge
# anti-spoofing table checks sources against (hive-egress.nix).

22
nix/module-eval/core-toggle.nix
View file

@ -1624,17 +1624,24 @@ let
&& lib.all (l: lib.hasSuffix " drop" l) (spoofOf bare);
}
{
# The sets are filled at run time, from three places: the unit at start
# and reload, the udev rule for each new port, and dnsmasq for each
# lease. Each is present on `bare`; the host without the bridge has none.
name = "the bridge table's sets are filled by the unit, a udev rule per port and dnsmasq's lease script";
# The sets are filled at run time, from four places: the unit at start and
# reload (tables, bindings and leases in one transaction), the udev rule
# for each new port, a timer for the ports still unbound, and dnsmasq for
# each lease. Each is present on `bare`; the host without the bridge has
# none.
name = "the bridge table's sets load with the table, and a udev rule, a timer and dnsmasq's lease script keep them current";
ok =
let
unit = bare.systemd.services.hive-egress;
pin = bare.systemd.services."hive-egress-pin@";
sweep = bare.systemd.services.hive-egress-pin;
in
lib.hasInfix "egress-pin --seed" unit.serviceConfig.ExecStartPost
&& lib.any (lib.hasInfix "egress-pin --seed") unit.serviceConfig.ExecReload
lib.hasInfix "egress-load " unit.serviceConfig.ExecStart
&& lib.hasInfix "hive-egress.nft" unit.serviceConfig.ExecStart
&& unit.serviceConfig.ExecReload == unit.serviceConfig.ExecStart
&& !(unit.serviceConfig ? ExecStartPost)
&& lib.hasSuffix "egress-pin" sweep.serviceConfig.ExecStart
&& bare.systemd.timers.hive-egress-pin.timerConfig.OnUnitActiveSec == "30s"
&& unit.environment.HIVE_EGRESS_BRIDGE == "hive-br0"
&& lib.hasInfix "egress-pin %i" pin.serviceConfig.ExecStart
&& lib.hasInfix ''KERNEL=="vb-*"'' bare.services.udev.extraRules
@ -1642,7 +1649,8 @@ let
&&
bare.services.dnsmasq.settings.dhcp-script
== toString bare.services.hyperhive.network.egress.leaseScript
&& !(centralToggleOff.systemd.services ? "hive-egress-pin@");
&& !(centralToggleOff.systemd.services ? "hive-egress-pin@")
&& !(centralToggleOff.systemd ? timers && centralToggleOff.systemd.timers ? hive-egress-pin);
}
{
# `accept` renders the table without the forward chain and without the

20
nix/vm-tests/hive-egress-antispoof.nix
View file

@ -2,7 +2,7 @@
# ../host-modules/hive-egress.nix on a real agent bridge, with two nspawn
# containers on it that take their addresses from the hive's dnsmasq
# (../host-modules/hive-gateway/dnsmasq.nix' lease script, the real
# `hive-priv egress-pin` and `egress-lease`). `hive-squid` cannot host this: its
# `hive-priv egress-load`, `egress-pin` and `egress-lease`). `hive-squid` cannot host this: its
# stand-in bridge is one NIC, so every agent shares one port.
#
# What it pins:
@ -14,9 +14,10 @@
# that is not link-local and a DHCP message with another's `chaddr` are
# dropped, and each leaves a `hive-egress-spoof ` line that names the port;
# - a DHCP renewal, a link-local IPv6 packet and a restored MAC go through;
# - a port with no bound MAC sends nothing, and the pin unit binds it again;
# - `antiSpoof = "log"`, reached by a reload of the unit, keeps the bindings
# and the leases, lets the same spoofs through and logs them.
# - a port with no bound MAC sends nothing, and the timer binds it again;
# - `antiSpoof = "log"`, reached by a reload of the unit, brings the bindings
# and the leases with the new table (nothing is dropped in between),
# lets the same spoofs through and logs them.
{ pkgs, hivePriv }:
let
bridge = "hive0";
@ -265,20 +266,21 @@ pkgs.testers.runNixOSTest {
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
assert ct_ok("a", "ping -c1 -W2 ${gateway}")
with subtest("a port with no bound MAC sends nothing, and the pin unit binds it again"):
with subtest("a port with no bound MAC sends nothing, and the timer binds it again"):
host.succeed(f"nft flush set {TABLE} port_mac")
assert not ct_ok("a", "ping -c1 -W2 ${gateway}")
assert not ct_ok("b", "ping -c1 -W2 ${gateway}")
host.succeed("systemctl start hive-egress-pin@vb-a.service hive-egress-pin@vb-b.service")
# Nothing starts the pin unit by hand: the timer finds the unbound ports.
host.wait_until_succeeds("nixos-container run a -- ping -c1 -W2 ${gateway}")
host.wait_until_succeeds("nixos-container run b -- ping -c1 -W2 ${gateway}")
with subtest("antiSpoof = log keeps the bindings and the leases, lets a spoofed datagram through and logs it"):
host.succeed("/run/current-system/specialisation/logmode/bin/switch-to-configuration test")
assert "counter accept" in host.succeed(f"nft list chain {TABLE} spoof"), "the table was reloaded in log mode"
host.wait_until_succeeds(f"nft list set {TABLE} port_mac | grep -q '\"vb-a\" . {MAC_A}'")
host.wait_until_succeeds(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "control: own address, after the reload"
# No waiting: the reload put the sets in with the table.
host.succeed(f"nft list set {TABLE} port_mac | grep -q '\"vb-a\" . {MAC_A}'")
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "control: own address, right after the reload"
seen = len(spoof_lines())
ct("a", f"ip addr add {IP_B}/24 dev eth0")
assert udp_reaches_host("a", IP_B), "log mode lets the spoofed datagram through"

10
nix/vm-tests/hive-squid.nix
View file

@ -197,10 +197,14 @@ pkgs.testers.runNixOSTest {
);
# The gateway: nginx answers on the bridge, which agents reach directly.
networking.firewall.interfaces.eth1.allowedTCPPorts = [ 443 ];
# hive-priv.nix links the binary the hive-egress unit runs to bind bridge
# ports; `eth1` is no bridge here, so a stub that binds nothing stands in.
# hive-priv.nix links the binary the hive-egress unit loads its tables with;
# `eth1` is no bridge here, so a stub that only loads the ruleset (no
# bindings, no leases) stands in.
systemd.tmpfiles.rules = [
"L+ /run/hive-rootfs-check - - - - ${pkgs.writeShellScript "hive-priv-stub" "exit 0"}"
"L+ /run/hive-rootfs-check - - - - ${pkgs.writeShellScript "hive-priv-stub" ''
[ "$1" = egress-load ] && exec ${pkgs.nftables}/bin/nft -f "$2"
exit 0
''}"
];
# Renders the users file the way hive-c0re does.
environment.systemPackages = [