mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

agent bridge: drop frames whose source MAC or address isn't the container's own (egress.antiSpoof)

Browse source
This commit is contained in:
atlas 2026-11-13 02:36:38 +01:00 • committed by mara
commit c4ac01ef88
2 changed files with 636 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

201
nix/host-modules/hive-egress.nix
View file

@ -1,17 +1,26 @@
# The hive's own nftables table, `inet hive_egress`, loaded by a systemd
# oneshot. It is independent of the NixOS firewall: `networking.firewall`
# renders only the rules of the backend in use (`iptables` unless
# `networking.nftables.enable` is set), so a rule that has to hold on every
# host lives here, in a table no firewall backend writes. Only
# `networking.nftables.flushRuleset` removes it; the unit re-applies it then.
# The hive's own nftables tables, `inet hive_egress` and `bridge hive_egress`,
# loaded by a systemd oneshot. They are independent of the NixOS firewall:
# `networking.firewall` renders only the rules of the backend in use
# (`iptables` unless `networking.nftables.enable` is set), so a rule that has
# to hold on every host lives here, in tables no firewall backend writes. Only
# `networking.nftables.flushRuleset` removes them; the unit re-applies them then.
#
# The table holds two chains. `input`: traffic from the agent bridge to host
# The `inet` table holds two chains. `input`: traffic from the agent bridge to host
# loopback is dropped. The kernel does the same by default (a packet for
# 127.0.0.0/8 arriving on a non-loopback interface is a martian while
# `route_localnet` is 0); this rule is the second layer, and the assertion
# below keeps the first one on. `forward` (`egress.forward = "log"`): traffic
# the host routes from the agent bridge to anywhere else is logged and
# counted, never blocked.
#
# The `bridge` table checks the source address of every frame a port of the
# agent bridge sends (`egress.antiSpoof`). It is a `bridge`-family table because
# traffic between two ports is bridged: the `inet` hooks see it as coming from
# the bridge device, and only a bridge hook knows the port it came in on
# (`iifname` is the veth). Which MAC a port may use and which IPv4 address a MAC
# may use are not known when the ruleset loads, so they are set contents that
# `hive-priv egress-pin` (per port, from udev) and dnsmasq's `dhcp-script`
# (per lease) fill; `hive-priv`'s `egress.rs` documents how.
{
lib,
config,
@ -26,6 +35,22 @@ let
# The mesh interface (swarm-wireguard.nix): inter-hive traffic is not egress.
meshInterface = "wg-hive";
# The sub-commands of hive-priv that fill the bridge table's sets. The path is
# the one every platform container's unit uses (hive-priv.nix points it at the
# current build).
hivePriv = (import ./lib/container-rootfs-check.nix { inherit config; }).hivePriv;
pinCommand = args: "${hivePriv} egress-pin ${args}";
pinPath = [
pkgs.nftables
pkgs.iproute2
pkgs.util-linux
];
pinEnvironment.HIVE_EGRESS_BRIDGE = br;
leaseScript = pkgs.writeShellScript "hive-egress-lease" ''
export PATH=${lib.makeBinPath [ pkgs.nftables ]}
exec ${hivePriv} egress-lease "$@"
'';
# Logging is per source address, so one busy agent cannot use up the budget
# of another: a burst of 20 lines, then one a second, and an address is
# forgotten after a minute without a new flow. The rule accepts after it
@ -64,12 +89,86 @@ let
}
'';
# One transaction: `add` + `delete` drop an old copy of the table and the
# definition recreates it, so the file loads the same on first start, on a
# The bridge table. A frame of a port goes through `prerouting` in this
# order: the port must be bound to the frame's source MAC, then the frame's
# protocol decides which source address it may carry. Everything else ends in
# `spoof`, which logs and then drops or, with `antiSpoof = "log"`, accepts.
#
# - `port_mac` and `port_chaddr` hold the bound (port, MAC) pairs, the second
# as the 48 bits that sit at the `chaddr` field of a DHCP message. A port
# with no pair sends nothing, DHCP included.
# - `lease_v4` holds the leased (MAC, IPv4) pairs; each times out with its
# lease.
# - IPv6 has no leases: a source is link-local, or `::` in the packets that
# duplicate address detection and multicast listener reports send before an
# address exists. The bridge carries no other IPv6 traffic.
# - DHCP: any message to port 67 must carry the sender's own MAC as `chaddr`,
# so a port cannot lease, renew or release another's address; a client with
# no address yet sends from 0.0.0.0 to the broadcast address.
# - ARP: the sender's MAC is the port's, and its address 0.0.0.0 (a probe) or
# leased to that MAC.
# - A VLAN tag is refused: a tagged frame would carry its packet past the
# protocol checks. The bridge uses no VLANs.
#
# Logging is per ingress port, in the style of the forward chain: a burst of
# 20 lines, then one a second, and a port is forgotten after a minute. A
# packet its port cannot log (over its rate, or the set is full) falls to the
# unmetered line, which has one budget for all ports. It logs
# packets, not flows: the bridge hook has no connection tracking.
spoofVerdict = if cfg.egress.antiSpoof == "drop" then "drop" else "accept";
bridgeTable = ''
table bridge hive_egress {
set port_mac { type ifname . ether_addr; }
set port_chaddr { typeof iifname . @th,288,48; }
set lease_v4 { type ether_addr . ipv4_addr; flags timeout; size 65536; }
set spoof_ports { type ifname; flags dynamic; timeout 1m; size 4096; }
chain spoof {
update @spoof_ports { iifname limit rate ${logRate} } counter log prefix "hive-egress-spoof " ${spoofVerdict}
limit rate ${unmeteredRate} counter log prefix "hive-egress-spoof-unmetered " ${spoofVerdict}
counter ${spoofVerdict}
}
chain l3_v4 {
udp dport 67 iifname . @th,288,48 != @port_chaddr goto spoof
ip saddr 0.0.0.0 ip daddr 255.255.255.255 udp sport 68 udp dport 67 accept
ether saddr . ip saddr @lease_v4 accept
goto spoof
}
chain l3_v6 {
ip6 saddr fe80::/10 accept
ip6 saddr :: icmpv6 type { nd-neighbor-solicit, mld2-listener-report } accept
goto spoof
}
chain l3_arp {
iifname . arp saddr ether != @port_mac goto spoof
arp saddr ip 0.0.0.0 accept
arp saddr ether . arp saddr ip @lease_v4 accept
goto spoof
}
chain prerouting {
type filter hook prerouting priority -300; policy accept;
meta ibrname != "${br}" accept
iifname . ether saddr != @port_mac goto spoof
ether type { 8021q, 8021ad } goto spoof
ether type ip goto l3_v4
ether type ip6 goto l3_v6
ether type arp goto l3_arp
}
}
'';
# One transaction: `add` + `delete` drop an old copy of each table and the
# definitions recreate them, so the file loads the same on first start, on a
# reload and after something else flushed the ruleset.
rulesetText = ''
add table inet hive_egress
delete table inet hive_egress
add table bridge hive_egress
delete table bridge hive_egress
table inet hive_egress {
chain input {
@ -79,6 +178,8 @@ let
}
${lib.optionalString (cfg.egress.forward == "log") forwardChain}
}
${bridgeTable}
'';
routeLocalnetKeys = [
@ -117,12 +218,48 @@ in
'';
};
options.services.hyperhive.network.egress.antiSpoof = lib.mkOption {
type = lib.types.enum [
"drop"
"log"
];
default = "drop";
description = ''
What the `bridge hive_egress` table does with a frame whose source is not
the sending container's own. Every port of the agent bridge is checked:
the frame's MAC must be the one bound to the port when its container
started, and its IPv4 address the one leased to that MAC. A port with no
bound MAC sends nothing. Allowed besides the leased address: DHCP from
`0.0.0.0` whose `chaddr` is the port's MAC, ARP probes, and IPv6 link-local
sources (and `::` in neighbour discovery and listener reports).
- `drop`: the frame is dropped and logged with the prefix
`hive-egress-spoof `, rate limited per port. A packet the per-port limit
cannot log gets a line with the prefix `hive-egress-spoof-unmetered `
from a limit shared by all ports. A container sending from its leased
address is never affected, so nothing legitimate breaks.
- `log`: the same lines and counters, and the frame goes through. For
finding out why a legitimate container is refused on a live hive.
'';
};
options.services.hyperhive.network.egress.leaseScript = lib.mkOption {
type = lib.types.path;
internal = true;
readOnly = true;
default = leaseScript;
defaultText = lib.literalMD "the script that loads a lease into the bridge table";
description = ''
The program dnsmasq runs on each lease change (its `dhcp-script`).
'';
};
options.services.hyperhive.network.egress.ruleset = lib.mkOption {
type = lib.types.lines;
internal = true;
readOnly = true;
default = rulesetText;
defaultText = lib.literalMD "the `inet hive_egress` table definition";
defaultText = lib.literalMD "the `inet hive_egress` and `bridge hive_egress` table definitions";
description = ''
The nft file the `hive-egress` unit loads, as text.
'';
@ -151,23 +288,59 @@ in
];
systemd.services.hive-egress = {
description = "hive nftables table (agent bridge egress log, host loopback drop)";
description = "hive nftables tables (agent bridge egress log and source check, host loopback drop)";
wantedBy = [ "multi-user.target" ];
wants = [ "network-pre.target" ];
before = [ "network-pre.target" ];
# `nftables.service` runs `flush ruleset` where `networking.nftables.flushRuleset`
# is on: start after it, follow its restarts and re-apply on its reload.
after = [ "nftables.service" ];
# The tmpfiles units link `hivePriv`.
after = [
"nftables.service"
"systemd-tmpfiles-setup.service"
"systemd-tmpfiles-resetup.service"
];
partOf = [ "nftables.service" ];
unitConfig.ReloadPropagatedFrom = [ "nftables.service" ];
reloadIfChanged = true;
path = pinPath;
environment = pinEnvironment;
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = "${nft} -f ${ruleset}";
ExecReload = "${nft} -f ${ruleset}";
ExecStop = "-${nft} delete table inet hive_egress";
# The sets of a freshly loaded bridge table are empty: bind the ports of
# the running containers and load the leases from dnsmasq's file.
ExecStartPost = pinCommand "--seed";
ExecReload = [
"${nft} -f ${ruleset}"
(pinCommand "--seed")
];
ExecStop = [
"-${nft} delete table inet hive_egress"
"-${nft} delete table bridge hive_egress"
];
};
};
# Binds one port of the bridge once its veth exists: the run waits for the
# container to register with systemd-machined.
systemd.services."hive-egress-pin@" = {
description = "bind agent bridge port %i to its container's MAC";
wants = [ "hive-egress.service" ];
after = [ "hive-egress.service" ];
path = pinPath;
environment = pinEnvironment;
serviceConfig = {
Type = "oneshot";
ExecStart = pinCommand "%i";
};
};
# nspawn names the host end of a container's veth `vb-<machine>` when the
# container joins a bridge.
services.udev.extraRules = ''
SUBSYSTEM=="net", ACTION=="add", KERNEL=="vb-*", TAG+="systemd", ENV{SYSTEMD_WANTS}+="hive-egress-pin@%k.service"
'';
};
}

4
nix/host-modules/hive-gateway/dnsmasq.nix
View file

@ -81,7 +81,11 @@ in
# bridgeIp/bridgePrefixLength. All containers (agents and service
# containers such as swarm-ci-runner) receive their IPs dynamically.
dhcp-range = "${networkCfg.dhcpRangeStart},${networkCfg.dhcpRangeEnd},1h";
# `hive-priv egress-pin --seed` reads this file back (hive-egress.nix).
dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases";
# Each lease change updates the leased (MAC, IPv4) pairs the bridge
# anti-spoofing table checks sources against (hive-egress.nix).
dhcp-script = toString networkCfg.egress.leaseScript;
# No explicit upstream: non-hive queries follow dnsmasq's
# resolv.conf default — the host's own `/etc/resolv.conf`, so the
# hive always uses the host's resolvers and follows them live with