346 lines
14 KiB
Nix
346 lines
14 KiB
Nix
# The hive's own nftables tables, `inet hive_egress` and `bridge hive_egress`,
|
|
# loaded by a systemd oneshot. They are independent of the NixOS firewall:
|
|
# `networking.firewall` renders only the rules of the backend in use
|
|
# (`iptables` unless `networking.nftables.enable` is set), so a rule that has
|
|
# to hold on every host lives here, in tables no firewall backend writes. Only
|
|
# `networking.nftables.flushRuleset` removes them; the unit re-applies them then.
|
|
#
|
|
# The `inet` table holds two chains. `input`: traffic from the agent bridge to host
|
|
# loopback is dropped. The kernel does the same by default (a packet for
|
|
# 127.0.0.0/8 arriving on a non-loopback interface is a martian while
|
|
# `route_localnet` is 0); this rule is the second layer, and the assertion
|
|
# below keeps the first one on. `forward` (`egress.forward = "log"`): traffic
|
|
# the host routes from the agent bridge to anywhere else is logged and
|
|
# counted, never blocked.
|
|
#
|
|
# The `bridge` table checks the source address of every frame a port of the
|
|
# agent bridge sends (`egress.antiSpoof`). It is a `bridge`-family table because
|
|
# traffic between two ports is bridged: the `inet` hooks see it as coming from
|
|
# the bridge device, and only a bridge hook knows the port it came in on
|
|
# (`iifname` is the veth). Which MAC a port may use and which IPv4 address a MAC
|
|
# may use are not known when the ruleset loads, so they are set contents that
|
|
# `hive-priv egress-pin` (per port, from udev) and dnsmasq's `dhcp-script`
|
|
# (per lease) fill; `hive-priv`'s `egress.rs` documents how.
|
|
{
|
|
lib,
|
|
config,
|
|
pkgs,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.network;
|
|
nft = lib.getExe pkgs.nftables;
|
|
br = cfg.bridgeName;
|
|
|
|
# The mesh interface (swarm-wireguard.nix): inter-hive traffic is not egress.
|
|
meshInterface = "wg-hive";
|
|
|
|
# The sub-commands of hive-priv that fill the bridge table's sets. The path is
|
|
# the one every platform container's unit uses (hive-priv.nix points it at the
|
|
# current build).
|
|
hivePriv = (import ./lib/container-rootfs-check.nix { inherit config; }).hivePriv;
|
|
pinCommand = args: "${hivePriv} egress-pin ${args}";
|
|
pinPath = [
|
|
pkgs.nftables
|
|
pkgs.iproute2
|
|
pkgs.util-linux
|
|
];
|
|
pinEnvironment.HIVE_EGRESS_BRIDGE = br;
|
|
leaseScript = pkgs.writeShellScript "hive-egress-lease" ''
|
|
export PATH=${lib.makeBinPath [ pkgs.nftables ]}
|
|
exec ${hivePriv} egress-lease "$@"
|
|
'';
|
|
|
|
# Logging is per source address, so one busy agent cannot use up the budget
|
|
# of another: a burst of 20 lines, then one a second, and an address is
|
|
# forgotten after a minute without a new flow. The rule accepts after it
|
|
# logs. A flow the rule cannot log falls through to the unmetered rule below
|
|
# it: the address was over its rate, or the set is full (`update` then fails
|
|
# and the rule stops matching), which a container spoofing source addresses
|
|
# can bring about. Only that case reaches the unmetered rule, so a flow
|
|
# gets one line at most. The unmetered rule has one budget for all
|
|
# addresses, 5 lines a second after a burst of 50, about 100 MB of journal a
|
|
# day when it never rests.
|
|
logRate = "1/second burst 20 packets";
|
|
unmeteredRate = "5/second burst 50 packets";
|
|
fromBridge = ''iifname "${br}" oifname != "${br}"'';
|
|
logRule =
|
|
family: addr:
|
|
''${fromBridge} ct state new update @log_${family} { ${addr} saddr limit rate ${logRate} } log prefix "hive-egress-would-drop " accept'';
|
|
|
|
# Priority `filter - 10` runs before the NixOS firewall's forward handling,
|
|
# which sits at `filter` on both backends (`FORWARD` of the iptables backend,
|
|
# the `forward` chain of `nixos-fw`). An `accept` here ends this chain only,
|
|
# so the chain neither allows what those refuse nor refuses anything itself.
|
|
# Traffic to the host's own addresses (squid, dnsmasq, the gateway) is
|
|
# delivered locally and never reaches the forward hook.
|
|
forwardChain = ''
|
|
set log_v4 { type ipv4_addr; flags dynamic; timeout 1m; size 4096; }
|
|
set log_v6 { type ipv6_addr; flags dynamic; timeout 1m; size 4096; }
|
|
|
|
chain forward {
|
|
type filter hook forward priority filter - 10; policy accept;
|
|
iifname "${br}" oifname "${meshInterface}" accept
|
|
${fromBridge} counter
|
|
${logRule "v4" "ip"}
|
|
${logRule "v6" "ip6"}
|
|
${fromBridge} ct state new limit rate ${unmeteredRate} log prefix "hive-egress-would-drop-unmetered "
|
|
${fromBridge} accept
|
|
}
|
|
'';
|
|
|
|
# The bridge table. A frame of a port goes through `prerouting` in this
|
|
# order: the port must be bound to the frame's source MAC, then the frame's
|
|
# protocol decides which source address it may carry. Everything else ends in
|
|
# `spoof`, which logs and then drops or, with `antiSpoof = "log"`, accepts.
|
|
#
|
|
# - `port_mac` and `port_chaddr` hold the bound (port, MAC) pairs, the second
|
|
# as the 48 bits that sit at the `chaddr` field of a DHCP message. A port
|
|
# with no pair sends nothing, DHCP included.
|
|
# - `lease_v4` holds the leased (MAC, IPv4) pairs; each times out with its
|
|
# lease.
|
|
# - IPv6 has no leases: a source is link-local, or `::` in the packets that
|
|
# duplicate address detection and multicast listener reports send before an
|
|
# address exists. The bridge carries no other IPv6 traffic.
|
|
# - DHCP: any message to port 67 must carry the sender's own MAC as `chaddr`,
|
|
# so a port cannot lease, renew or release another's address; a client with
|
|
# no address yet sends from 0.0.0.0 to the broadcast address.
|
|
# - ARP: the sender's MAC is the port's, and its address 0.0.0.0 (a probe) or
|
|
# leased to that MAC.
|
|
# - A VLAN tag is refused: a tagged frame would carry its packet past the
|
|
# protocol checks. The bridge uses no VLANs.
|
|
#
|
|
# Logging is per ingress port, in the style of the forward chain: a burst of
|
|
# 20 lines, then one a second, and a port is forgotten after a minute. A
|
|
# packet its port cannot log (over its rate, or the set is full) falls to the
|
|
# unmetered line, which has one budget for all ports. It logs
|
|
# packets, not flows: the bridge hook has no connection tracking.
|
|
spoofVerdict = if cfg.egress.antiSpoof == "drop" then "drop" else "accept";
|
|
bridgeTable = ''
|
|
table bridge hive_egress {
|
|
set port_mac { type ifname . ether_addr; }
|
|
set port_chaddr { typeof iifname . @th,288,48; }
|
|
set lease_v4 { type ether_addr . ipv4_addr; flags timeout; size 65536; }
|
|
set spoof_ports { type ifname; flags dynamic; timeout 1m; size 4096; }
|
|
|
|
chain spoof {
|
|
update @spoof_ports { iifname limit rate ${logRate} } counter log prefix "hive-egress-spoof " ${spoofVerdict}
|
|
limit rate ${unmeteredRate} counter log prefix "hive-egress-spoof-unmetered " ${spoofVerdict}
|
|
counter ${spoofVerdict}
|
|
}
|
|
|
|
chain l3_v4 {
|
|
udp dport 67 iifname . @th,288,48 != @port_chaddr goto spoof
|
|
ip saddr 0.0.0.0 ip daddr 255.255.255.255 udp sport 68 udp dport 67 accept
|
|
ether saddr . ip saddr @lease_v4 accept
|
|
goto spoof
|
|
}
|
|
|
|
chain l3_v6 {
|
|
ip6 saddr fe80::/10 accept
|
|
ip6 saddr :: icmpv6 type { nd-neighbor-solicit, mld2-listener-report } accept
|
|
goto spoof
|
|
}
|
|
|
|
chain l3_arp {
|
|
iifname . arp saddr ether != @port_mac goto spoof
|
|
arp saddr ip 0.0.0.0 accept
|
|
arp saddr ether . arp saddr ip @lease_v4 accept
|
|
goto spoof
|
|
}
|
|
|
|
chain prerouting {
|
|
type filter hook prerouting priority -300; policy accept;
|
|
meta ibrname != "${br}" accept
|
|
iifname . ether saddr != @port_mac goto spoof
|
|
ether type { 8021q, 8021ad } goto spoof
|
|
ether type ip goto l3_v4
|
|
ether type ip6 goto l3_v6
|
|
ether type arp goto l3_arp
|
|
}
|
|
}
|
|
'';
|
|
|
|
# One transaction: `add` + `delete` drop an old copy of each table and the
|
|
# definitions recreate them, so the file loads the same on first start, on a
|
|
# reload and after something else flushed the ruleset.
|
|
rulesetText = ''
|
|
add table inet hive_egress
|
|
delete table inet hive_egress
|
|
add table bridge hive_egress
|
|
delete table bridge hive_egress
|
|
|
|
table inet hive_egress {
|
|
chain input {
|
|
type filter hook input priority filter - 10; policy accept;
|
|
iifname "${br}" ip daddr 127.0.0.0/8 counter drop
|
|
iifname "${br}" ip6 daddr ::1 counter drop
|
|
}
|
|
${lib.optionalString (cfg.egress.forward == "log") forwardChain}
|
|
}
|
|
|
|
${bridgeTable}
|
|
'';
|
|
|
|
routeLocalnetKeys = [
|
|
"net.ipv4.conf.all.route_localnet"
|
|
"net.ipv4.conf.default.route_localnet"
|
|
"net.ipv4.conf.${cfg.bridgeName}.route_localnet"
|
|
];
|
|
isOn =
|
|
v:
|
|
builtins.elem v [
|
|
1
|
|
"1"
|
|
true
|
|
];
|
|
ruleset = pkgs.writeText "hive-egress.nft" cfg.egress.ruleset;
|
|
in
|
|
{
|
|
options.services.hyperhive.network.egress.forward = lib.mkOption {
|
|
type = lib.types.enum [
|
|
"accept"
|
|
"log"
|
|
"drop"
|
|
];
|
|
default = "log";
|
|
description = ''
|
|
What the `inet hive_egress` table does with traffic the host forwards
|
|
from the agent bridge to anywhere but the bridge and the mesh.
|
|
|
|
- `accept`: nothing; the table has no forward chain.
|
|
- `log`: each new flow is logged with the prefix `hive-egress-would-drop `,
|
|
rate limited per source address, and counted. A flow the per-source limit
|
|
cannot log gets a line with the prefix `hive-egress-would-drop-unmetered `
|
|
from a limit shared by all addresses. Nothing is blocked. The source
|
|
address maps to an agent through the hive's DHCP leases.
|
|
- `drop`: not available yet; the evaluation refuses it.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.network.egress.antiSpoof = lib.mkOption {
|
|
type = lib.types.enum [
|
|
"drop"
|
|
"log"
|
|
];
|
|
default = "drop";
|
|
description = ''
|
|
What the `bridge hive_egress` table does with a frame whose source is not
|
|
the sending container's own. Every port of the agent bridge is checked:
|
|
the frame's MAC must be the one bound to the port when its container
|
|
started, and its IPv4 address the one leased to that MAC. A port with no
|
|
bound MAC sends nothing. Allowed besides the leased address: DHCP from
|
|
`0.0.0.0` whose `chaddr` is the port's MAC, ARP probes, and IPv6 link-local
|
|
sources (and `::` in neighbour discovery and listener reports).
|
|
|
|
- `drop`: the frame is dropped and logged with the prefix
|
|
`hive-egress-spoof `, rate limited per port. A packet the per-port limit
|
|
cannot log gets a line with the prefix `hive-egress-spoof-unmetered `
|
|
from a limit shared by all ports. A container sending from its leased
|
|
address is never affected, so nothing legitimate breaks.
|
|
- `log`: the same lines and counters, and the frame goes through. For
|
|
finding out why a legitimate container is refused on a live hive.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.network.egress.leaseScript = lib.mkOption {
|
|
type = lib.types.path;
|
|
internal = true;
|
|
readOnly = true;
|
|
default = leaseScript;
|
|
defaultText = lib.literalMD "the script that loads a lease into the bridge table";
|
|
description = ''
|
|
The program dnsmasq runs on each lease change (its `dhcp-script`).
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.network.egress.ruleset = lib.mkOption {
|
|
type = lib.types.lines;
|
|
internal = true;
|
|
readOnly = true;
|
|
default = rulesetText;
|
|
defaultText = lib.literalMD "the `inet hive_egress` and `bridge hive_egress` table definitions";
|
|
description = ''
|
|
The nft file the `hive-egress` unit loads, as text.
|
|
'';
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = cfg.egress.forward != "drop";
|
|
message = ''
|
|
services.hyperhive.network.egress.forward = "drop" is not available yet:
|
|
the interception that lets an agent's web traffic through squid has not
|
|
landed, so a drop would cut every direct connection with no replacement.
|
|
Use "log" or "accept".
|
|
'';
|
|
}
|
|
{
|
|
assertion = !(lib.any (k: isOn (config.boot.kernel.sysctl.${k} or 0)) routeLocalnetKeys);
|
|
message = ''
|
|
boot.kernel.sysctl must not turn on route_localnet (${lib.concatStringsSep ", " routeLocalnetKeys}).
|
|
With it on, the kernel routes a packet from the agent bridge to 127.0.0.0/8
|
|
instead of dropping it as a martian, which leaves the hive_egress
|
|
table as the only barrier between an agent and the host's loopback services.
|
|
'';
|
|
}
|
|
];
|
|
|
|
systemd.services.hive-egress = {
|
|
description = "hive nftables tables (agent bridge egress log and source check, host loopback drop)";
|
|
wantedBy = [ "multi-user.target" ];
|
|
wants = [ "network-pre.target" ];
|
|
before = [ "network-pre.target" ];
|
|
# `nftables.service` runs `flush ruleset` where `networking.nftables.flushRuleset`
|
|
# is on: start after it, follow its restarts and re-apply on its reload.
|
|
# The tmpfiles units link `hivePriv`.
|
|
after = [
|
|
"nftables.service"
|
|
"systemd-tmpfiles-setup.service"
|
|
"systemd-tmpfiles-resetup.service"
|
|
];
|
|
partOf = [ "nftables.service" ];
|
|
unitConfig.ReloadPropagatedFrom = [ "nftables.service" ];
|
|
reloadIfChanged = true;
|
|
path = pinPath;
|
|
environment = pinEnvironment;
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = "${nft} -f ${ruleset}";
|
|
# The sets of a freshly loaded bridge table are empty: bind the ports of
|
|
# the running containers and load the leases from dnsmasq's file.
|
|
ExecStartPost = pinCommand "--seed";
|
|
ExecReload = [
|
|
"${nft} -f ${ruleset}"
|
|
(pinCommand "--seed")
|
|
];
|
|
ExecStop = [
|
|
"-${nft} delete table inet hive_egress"
|
|
"-${nft} delete table bridge hive_egress"
|
|
];
|
|
};
|
|
};
|
|
|
|
# Binds one port of the bridge once its veth exists: the run waits for the
|
|
# container to register with systemd-machined.
|
|
systemd.services."hive-egress-pin@" = {
|
|
description = "bind agent bridge port %i to its container's MAC";
|
|
wants = [ "hive-egress.service" ];
|
|
after = [ "hive-egress.service" ];
|
|
path = pinPath;
|
|
environment = pinEnvironment;
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
ExecStart = pinCommand "%i";
|
|
};
|
|
};
|
|
|
|
# nspawn names the host end of a container's veth `vb-<machine>` when the
|
|
# container joins a bridge.
|
|
services.udev.extraRules = ''
|
|
SUBSYSTEM=="net", ACTION=="add", KERNEL=="vb-*", TAG+="systemd", ENV{SYSTEMD_WANTS}+="hive-egress-pin@%k.service"
|
|
'';
|
|
};
|
|
}
|