# The hive's own nftables tables, `inet hive_egress` and `bridge hive_egress`, # loaded by a systemd oneshot. They are independent of the NixOS firewall: # `networking.firewall` renders only the rules of the backend in use # (`iptables` unless `networking.nftables.enable` is set), so a rule that has # to hold on every host lives here, in tables no firewall backend writes. Only # `networking.nftables.flushRuleset` removes them; the unit re-applies them then. # # The `inet` table holds three chains. `input`: traffic from the agent bridge to host # loopback is dropped. The kernel does the same by default (a packet for # 127.0.0.0/8 arriving on a non-loopback interface is a martian while # `route_localnet` is 0); this rule is the second layer, and the assertion # below keeps the first one on. `forward` (`egress.forward`): traffic from the # agent bridge to anywhere else is logged and counted (`log`) or logged and # refused (`reject`). `intercept` (`egress.intercept.enable`, set by # hive-squid.nix): the web traffic of the bridge's agents is redirected into # squid's intercept ports (port 443 through the ClientHello relay). Both leave # alone the ports in `direct_mac`, the service containers and the unrestricted # agents. `hive-priv egress-pin` and `egress-load` fill that set from the # bridge table's bindings, so a MAC in no binding is intercepted and refused. # # The `bridge` table checks the source address of every frame a port of the # agent bridge sends (`egress.antiSpoof`). It is a `bridge`-family table because # traffic between two ports is bridged: the `inet` hooks see it as coming from # the bridge device, and only a bridge hook knows the port it came in on # (`iifname` is the veth). Which MAC a port may use and which IPv4 address a MAC # may use are not known until a container runs, so they are set contents. # `hive-priv egress-load` puts the current ones into the same transaction as the # tables; `egress-pin` (per new port from udev, and from a timer for the ports # still unbound) and dnsmasq's `dhcp-script` (per lease) keep them current. # `hive-priv`'s `egress.rs` documents how. { lib, config, pkgs, ... }: let cfg = config.services.hyperhive.network; nft = lib.getExe pkgs.nftables; br = cfg.bridgeName; # The mesh interface (swarm-wireguard.nix): inter-hive traffic is not egress. meshInterface = "wg-hive"; # With `refuseEch`, the redirect of port 443 ends at `hive-priv # egress-ech-relay`, which refuses a ClientHello that carries the ECH # extension and sends the rest on to squid's HTTPS intercept port. The relay # connects to the address the agent dialed, bound to a source address of the # pool and carrying the mark; the `output` chain DNATs those connections to # squid. `egress_ech.rs` derives the pool address from the agent's and # `egress-whois` maps it back, so the pool is the one defined there. intercept = cfg.egress.intercept; refuseEch = intercept.enable && intercept.refuseEch; relayPool = "198.18.0.0/15"; relayMark = "0x45434800"; relayCommand = "${hivePriv} egress-ech-relay ${cfg.bridgeIp}:${toString intercept.relayPort} ${relayMark}"; # The sub-commands of hive-priv that fill the bridge table's sets. The path is # the one every platform container's unit uses (hive-priv.nix points it at the # current build). hivePriv = (import ./lib/container-rootfs-check.nix { inherit config; }).hivePriv; pinCommand = args: lib.trim "${hivePriv} egress-pin ${args}"; loadCommand = "${hivePriv} egress-load ${ruleset}"; pinPath = [ pkgs.nftables pkgs.iproute2 pkgs.util-linux ]; pinEnvironment = { HIVE_EGRESS_BRIDGE = br; } // lib.optionalAttrs (cfg.egress.intercept.unrestrictedFile != null) { HIVE_EGRESS_UNRESTRICTED = toString cfg.egress.intercept.unrestrictedFile; }; leaseScript = pkgs.writeShellScript "hive-egress-lease" '' export PATH=${lib.makeBinPath [ pkgs.nftables ]} exec ${hivePriv} egress-lease "$@" ''; # Logging is per source address, so one busy agent cannot use up the budget # of another: a burst of 20 lines, then one a second, and an address is # forgotten after a minute without a new flow. The rule decides after it # logs. A flow the rule cannot log falls through to the unmetered rule below # it: the address was over its rate, or the set is full (`update` then fails # and the rule stops matching), which a container spoofing source addresses # can bring about. Only that case reaches the unmetered rule, so a flow # gets one line at most. The unmetered rule has one budget for all # addresses, 5 lines a second after a burst of 50, about 100 MB of journal a # day when it never rests. logRate = "1/second burst 20 packets"; unmeteredRate = "5/second burst 50 packets"; fromBridge = ''iifname "${br}" oifname != "${br}"''; # `log` accepts what it logs. `reject` logs and refuses in one rule, so a # line always names a refused flow. A TCP flow gets a reset, which fails the # connect at once (to a TCP socket an ICMP error is a soft error, and the # connect would stall); anything else gets ICMP admin-prohibited. A rule that # refuses is therefore a pair: one for TCP, one for the rest. isReject = cfg.egress.forward == "reject"; logPrefix = if isReject then "hive-egress-reject" else "hive-egress-would-drop"; refuse = { tcp = "meta l4proto tcp"; other = "meta l4proto != tcp"; }; verdict = { tcp = "reject with tcp reset"; other = "reject with icmpx type admin-prohibited"; }; decide = statements: if isReject then map ( k: lib.concatStringsSep " " ( lib.filter (x: x != "") [ fromBridge refuse.${k} statements verdict.${k} ] ) ) [ "tcp" "other" ] else [ "${fromBridge} ${statements} accept" ]; logRules = family: addr: decide ''ct state new update @log_${family} { ${addr} saddr limit rate ${logRate} } log prefix "${logPrefix} "''; # In `log` mode the unmetered rule has no verdict: the chain's last rule is # the verdict for every flow that gets past the logging. unmeteredRules = let statements = ''ct state new limit rate ${unmeteredRate} log prefix "${logPrefix}-unmetered "''; in if isReject then decide statements else [ "${fromBridge} ${statements}" ]; # What no rule above decided, a packet that is no new flow included. finalRules = if isReject then decide "" else [ "${fromBridge} accept" ]; # IPv4 TCP from the agent bridge to ports 80 and 443. `fib daddr oifname` is # the interface the route to the destination leaves by: the bridge (another # container), `lo` (an address of the host: `fib daddr type local`) and the # mesh interface are not egress. The redirect rewrites the destination to # the bridge address and the port of squid's intercept listener (for port 443 # with `refuseEch`, of the relay), which reads the original destination from # conntrack. IPv6 is left alone: the agent bridge has no IPv6 addressing. interceptChain = '' chain intercept { type nat hook prerouting priority dstnat; policy accept; iifname "${br}" meta nfproto ipv4 ether saddr @direct_mac return iifname "${br}" meta nfproto ipv4 fib daddr type local return iifname "${br}" meta nfproto ipv4 fib daddr oifname { "${br}", "${meshInterface}" } return iifname "${br}" meta nfproto ipv4 tcp dport 80 counter redirect to :${toString cfg.egress.intercept.httpPort} iifname "${br}" meta nfproto ipv4 tcp dport 443 counter redirect to :${ toString (if refuseEch then intercept.relayPort else intercept.httpsPort) } } '' + lib.optionalString refuseEch '' chain intercept_relay { type nat hook output priority dstnat; policy accept; meta nfproto ipv4 ip saddr ${relayPool} meta mark ${relayMark} tcp dport 443 counter dnat ip to ${cfg.bridgeIp}:${toString intercept.httpsPort} } ''; # Priority `filter - 10` runs before the NixOS firewall's forward handling, # which sits at `filter` on both backends (`FORWARD` of the iptables backend, # the `forward` chain of `nixos-fw`). An `accept` here ends this chain only, # so it allows nothing those refuse; a `reject` ends the packet. The mesh # and the MACs in `direct_mac` (the unrestricted agents and the service # containers, the traffic the intercept chain leaves alone) come first in # both modes, so a `log` line names what `reject` would refuse. Traffic to # the host's own addresses (squid, dnsmasq, the gateway) is delivered # locally and never reaches the forward hook. forwardChain = '' set log_v4 { type ipv4_addr; flags dynamic; timeout 1m; size 4096; } set log_v6 { type ipv6_addr; flags dynamic; timeout 1m; size 4096; } chain forward { type filter hook forward priority filter - 10; policy accept; iifname "${br}" oifname "${meshInterface}" accept ${fromBridge} ether saddr @direct_mac accept ${fromBridge} counter ${lib.concatStringsSep "\n " ( logRules "v4" "ip" ++ logRules "v6" "ip6" ++ unmeteredRules ++ finalRules )} } ''; # The bridge table. A frame of a port goes through `prerouting` in this # order: the port must be bound to the frame's source MAC, then the frame's # protocol decides which source address it may carry. Everything else ends in # `spoof`, which logs and then drops or, with `antiSpoof = "log"`, accepts. # # - `port_mac` and `port_chaddr` hold the bound (port, MAC) pairs, the second # as the 48 bits that sit at the `chaddr` field of a DHCP message. A port # with no pair sends nothing, DHCP included. # - `lease_v4` holds the leased (MAC, IPv4) pairs; each times out with its # lease. # - IPv6 has no leases: a source is link-local, or `::` in the packets that # duplicate address detection and multicast listener reports send before an # address exists. The bridge carries no other IPv6 traffic. # - DHCP: any message to port 67 must carry the sender's own MAC as `chaddr`, # so a port cannot lease, renew or release another's address; a client with # no address yet sends from 0.0.0.0 to the broadcast address. # - ARP: the sender's MAC is the port's, and its address 0.0.0.0 (a probe) or # leased to that MAC. # - A VLAN tag is refused: a tagged frame would carry its packet past the # protocol checks. The bridge uses no VLANs. # # Logging is per ingress port, in the style of the forward chain: a burst of # 20 lines, then one a second, and a port is forgotten after a minute. A # packet its port cannot log (over its rate, or the set is full) falls to the # unmetered line, which has one budget for all ports. It logs # packets, not flows: the bridge hook has no connection tracking. spoofVerdict = if cfg.egress.antiSpoof == "drop" then "drop" else "accept"; bridgeTable = '' table bridge hive_egress { set port_mac { type ifname . ether_addr; } set port_chaddr { typeof iifname . @th,288,48; } set lease_v4 { type ether_addr . ipv4_addr; flags timeout; size 65536; } set spoof_ports { type ifname; flags dynamic; timeout 1m; size 4096; } chain spoof { update @spoof_ports { iifname limit rate ${logRate} } counter log prefix "hive-egress-spoof " ${spoofVerdict} limit rate ${unmeteredRate} counter log prefix "hive-egress-spoof-unmetered " ${spoofVerdict} counter ${spoofVerdict} } chain l3_v4 { udp dport 67 iifname . @th,288,48 != @port_chaddr goto spoof ip saddr 0.0.0.0 ip daddr 255.255.255.255 udp sport 68 udp dport 67 accept ether saddr . ip saddr @lease_v4 accept goto spoof } chain l3_v6 { ip6 saddr fe80::/10 accept ip6 saddr :: icmpv6 type { nd-neighbor-solicit, mld2-listener-report } accept goto spoof } chain l3_arp { iifname . arp saddr ether != @port_mac goto spoof arp saddr ip 0.0.0.0 accept arp saddr ether . arp saddr ip @lease_v4 accept goto spoof } chain prerouting { type filter hook prerouting priority -300; policy accept; meta ibrname != "${br}" accept iifname . ether saddr != @port_mac goto spoof ether type { 8021q, 8021ad } goto spoof ether type ip goto l3_v4 ether type ip6 goto l3_v6 ether type arp goto l3_arp } } ''; # One transaction: `add` + `delete` drop an old copy of each table and the # definitions recreate them, so the file loads the same on first start, on a # reload and after something else flushed the ruleset. rulesetText = '' add table inet hive_egress delete table inet hive_egress add table bridge hive_egress delete table bridge hive_egress table inet hive_egress { chain input { type filter hook input priority filter - 10; policy accept; iifname "${br}" ip daddr 127.0.0.0/8 counter drop iifname "${br}" ip6 daddr ::1 counter drop ${lib.optionalString refuseEch ''iifname "${br}" ip saddr ${relayPool} counter drop''} } set direct_mac { type ether_addr; } ${lib.optionalString cfg.egress.intercept.enable interceptChain} ${lib.optionalString (cfg.egress.forward != "accept") forwardChain} } ${bridgeTable} ''; routeLocalnetKeys = [ "net.ipv4.conf.all.route_localnet" "net.ipv4.conf.default.route_localnet" "net.ipv4.conf.${cfg.bridgeName}.route_localnet" ]; isOn = v: builtins.elem v [ 1 "1" true ]; ruleset = pkgs.writeText "hive-egress.nft" cfg.egress.ruleset; in { options.services.hyperhive.network.egress.forward = lib.mkOption { type = lib.types.enum [ "accept" "log" "reject" ]; default = "log"; description = '' What the `inet hive_egress` table does with traffic the host forwards from the agent bridge to anywhere but the bridge and the mesh. In `log` and `reject` the traffic of the MACs in the `direct_mac` set (the unrestricted agents and the service containers) is accepted and never logged. - `accept`: nothing; the table has no forward chain. - `log`: each new flow is logged with the prefix `hive-egress-would-drop `, rate limited per source address, and counted. A flow the per-source limit cannot log gets a line with the prefix `hive-egress-would-drop-unmetered ` from a limit shared by all addresses. Nothing is blocked. The lines are the flows `reject` would refuse. The source address maps to an agent through the hive's DHCP leases. - `reject`: the same lines with the prefix `hive-egress-reject ` and `hive-egress-reject-unmetered `, and every other packet from the bridge is refused as well: a TCP flow with a reset, anything else with ICMP administratively prohibited. Only the web traffic that the intercept chain redirects into squid still gets out. ''; }; options.services.hyperhive.network.egress.antiSpoof = lib.mkOption { type = lib.types.enum [ "drop" "log" ]; default = "drop"; description = '' What the `bridge hive_egress` table does with a frame whose source is not the sending container's own. Every port of the agent bridge is checked: the frame's MAC must be the one bound to the port when its container started, and its IPv4 address the one leased to that MAC. A port with no bound MAC sends nothing. Allowed besides the leased address: DHCP from `0.0.0.0` whose `chaddr` is the port's MAC, ARP probes, and IPv6 link-local sources (and `::` in neighbour discovery and listener reports). - `drop`: the frame is dropped and logged with the prefix `hive-egress-spoof `, rate limited per port. A packet the per-port limit cannot log gets a line with the prefix `hive-egress-spoof-unmetered ` from a limit shared by all ports. A container sending from its leased address is never affected, so nothing legitimate breaks. - `log`: the same lines and counters, and the frame goes through. For finding out why a legitimate container is refused on a live hive. With `log`, the identity squid takes from the source of an intercepted flow (`egress.intercept`) can be spoofed: use it to diagnose, not to run. ''; }; options.services.hyperhive.network.egress.intercept = { enable = lib.mkEnableOption "the redirect of the agents' web traffic into squid's intercept ports"; httpPort = lib.mkOption { type = lib.types.port; default = 3129; description = '' The port on the bridge address where squid listens for the redirected TCP port 80 traffic. ''; }; httpsPort = lib.mkOption { type = lib.types.port; default = 3130; description = '' The port on the bridge address where squid listens for TCP port 443 traffic: what the ClientHello relay sends on, or with `refuseEch` off the redirected traffic itself. ''; }; relayPort = lib.mkOption { type = lib.types.port; default = 3131; description = '' The port on the bridge address where `hive-priv egress-ech-relay` listens for the redirected TCP port 443 traffic (`refuseEch`). ''; }; refuseEch = lib.mkOption { type = lib.types.bool; default = true; description = '' Refuse a TLS ClientHello that carries the `encrypted_client_hello` extension on the redirected port 443 traffic. Squid 7.6 sees only the outer name of such a hello, so without this an agent that holds an ECH configuration could dial an allowed name and reach another site behind the same address. `hive-priv egress-ech-relay` reads the first TLS record of each redirected connection and, if the hello is clean, sends the connection on to squid. The extension of a client with no configuration (GREASE, which Chrome and Firefox send by default) looks the same as a real one on the wire, so it is refused too: those clients fail on the redirected path and work through the proxy variables. An agent in `log` mode is observed instead: its hello goes on and the relay logs `hive-egress-ech-would-refuse`. Turn this off to send port 443 straight to squid, with ECH unchecked. ''; }; agentAllowFile = lib.mkOption { type = lib.types.nullOr lib.types.path; default = null; description = '' The squid include `hive-c0re` renders with each agent's own rules, in which an agent in `log` mode has an `acl agent_log_ proxy_auth ` line. The ClientHello relay (`refuseEch`) reads it to tell a `log` agent, whose ECH hello is let through and logged as `hive-egress-ech-would-refuse`, from a restricted one, whose hello is refused. Squid reads the same file for the same split. `null` or an unreadable file lists no `log` agent, so every agent's ECH hello is refused. ''; }; unrestrictedFile = lib.mkOption { type = lib.types.nullOr lib.types.path; default = null; description = '' The file hive-c0re renders with the names of the agents that hold the `unrestricted_internet` flag, one per line. The `direct_mac` set of the `inet hive_egress` table holds the MACs of the ports of the agents it lists and of every container that is not an agent, and the redirect skips them. `null` or an unreadable file lists no agent, so every agent's web traffic is redirected. The file is read when a port is bound, on each pass of the 30 second timer and when the file changes. ''; }; }; options.services.hyperhive.network.egress.leaseScript = lib.mkOption { type = lib.types.path; internal = true; readOnly = true; default = leaseScript; defaultText = lib.literalMD "the script that loads a lease into the bridge table"; description = '' The program dnsmasq runs on each lease change (its `dhcp-script`). ''; }; options.services.hyperhive.network.egress.ruleset = lib.mkOption { type = lib.types.lines; internal = true; readOnly = true; default = rulesetText; defaultText = lib.literalMD "the `inet hive_egress` and `bridge hive_egress` table definitions"; description = '' The nft file the `hive-egress` unit loads, as text. ''; }; config = lib.mkIf cfg.enable { assertions = [ { assertion = !refuseEch || lib.toInt (lib.last (lib.splitString "/" cfg.networkCidr)) >= 15; message = '' services.hyperhive.network.bridgePrefixLength must be 15 or more while egress.intercept.refuseEch is on: the relay names a flow's source with the low 17 bits of the agent's address. ''; } { assertion = cfg.egress.forward != "reject" || cfg.egress.intercept.enable; message = '' services.hyperhive.network.egress.forward = "reject" needs services.hyperhive.network.egress.intercept.enable = true: without the intercept nothing redirects the agents' web traffic into squid, and the reject would refuse all of it. Enable the intercept (hive-squid.nix does on a hive controller host), or keep forward = "log". ''; } { assertion = !(lib.any (k: isOn (config.boot.kernel.sysctl.${k} or 0)) routeLocalnetKeys); message = '' boot.kernel.sysctl must not turn on route_localnet (${lib.concatStringsSep ", " routeLocalnetKeys}). With it on, the kernel routes a packet from the agent bridge to 127.0.0.0/8 instead of dropping it as a martian, which leaves the hive_egress table as the only barrier between an agent and the host's loopback services. ''; } ]; services.hyperhive.network.exposeHostPorts = lib.mkIf refuseEch [ intercept.relayPort ]; # The relay is not part of the tables: with it down, the redirect of port 443 # reaches a closed port and the agent's connection is refused. systemd.services.hive-egress-ech-relay = lib.mkIf refuseEch { description = "refuse TLS hellos with encrypted client hello on the way to squid's HTTPS intercept port"; wantedBy = [ "multi-user.target" ]; wants = [ "hive-egress.service" ]; after = [ "hive-egress.service" "systemd-tmpfiles-setup.service" "systemd-tmpfiles-resetup.service" ]; environment = lib.optionalAttrs (intercept.agentAllowFile != null) { HIVE_EGRESS_AGENT_ALLOW = toString intercept.agentAllowFile; }; serviceConfig = { # The pool is local, so the relay can bind to a source address in it. ExecStartPre = "+${pkgs.iproute2}/bin/ip route replace local ${relayPool} dev lo"; ExecStart = relayCommand; ExecStopPost = "+-${pkgs.iproute2}/bin/ip route del local ${relayPool} dev lo"; # The bridge address may not be assigned yet at first start. Restart = "on-failure"; RestartSec = 5; # The packet mark needs CAP_NET_ADMIN. The include is `squid`'s group's. DynamicUser = true; SupplementaryGroups = lib.optional (intercept.agentAllowFile != null) "squid"; AmbientCapabilities = "CAP_NET_ADMIN"; CapabilityBoundingSet = "CAP_NET_ADMIN"; NoNewPrivileges = true; ProtectSystem = "strict"; ProtectHome = true; PrivateTmp = true; PrivateDevices = true; ProtectKernelTunables = true; ProtectKernelModules = true; ProtectKernelLogs = true; ProtectControlGroups = true; RestrictAddressFamilies = "AF_INET"; RestrictNamespaces = true; RestrictRealtime = true; RestrictSUIDSGID = true; LockPersonality = true; SystemCallArchitectures = "native"; }; }; systemd.services.hive-egress = { description = "hive nftables tables (agent bridge egress log and source check, host loopback drop)"; wantedBy = [ "multi-user.target" ]; wants = [ "network-pre.target" ]; before = [ "network-pre.target" ]; # `nftables.service` runs `flush ruleset` where `networking.nftables.flushRuleset` # is on: start after it, follow its restarts and re-apply on its reload. # The tmpfiles units link `hivePriv`. after = [ "nftables.service" "systemd-tmpfiles-setup.service" "systemd-tmpfiles-resetup.service" ]; partOf = [ "nftables.service" ]; unitConfig.ReloadPropagatedFrom = [ "nftables.service" ]; reloadIfChanged = true; path = pinPath; environment = pinEnvironment; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; # The tables load with the bindings of the running containers' ports and # the leases of dnsmasq's file in the same transaction, so a reload never # leaves the bridge with empty sets. ExecStart = loadCommand; ExecReload = loadCommand; ExecStop = [ "-${nft} delete table inet hive_egress" "-${nft} delete table bridge hive_egress" ]; }; }; # Binds one port of the bridge once its veth exists: the run waits for the # container to register with systemd-machined. systemd.services."hive-egress-pin@" = { description = "bind agent bridge port %i to its container's MAC"; wants = [ "hive-egress.service" ]; after = [ "hive-egress.service" ]; path = pinPath; environment = pinEnvironment; serviceConfig = { Type = "oneshot"; ExecStart = pinCommand "%i"; }; }; # Looks again at the ports still unbound. A port a rival MAC kept unbound # sends nothing, so nothing else would tell the hive it can be bound now. # Ports with a binding cost a listing of the bridge and no container is read. systemd.services.hive-egress-pin = { description = "bind the agent bridge ports that have no binding yet"; wants = [ "hive-egress.service" ]; after = [ "hive-egress.service" ]; path = pinPath; environment = pinEnvironment; serviceConfig = { Type = "oneshot"; ExecStart = pinCommand ""; }; }; systemd.timers.hive-egress-pin = { description = "bind the agent bridge ports that have no binding yet"; wantedBy = [ "timers.target" ]; timerConfig = { OnBootSec = "1min"; OnUnitActiveSec = "30s"; AccuracySec = "5s"; }; }; # A pass also refills `direct_mac`, so a change of the unrestricted list # takes effect at once instead of at the next tick of the timer. systemd.paths.hive-egress-direct = lib.mkIf (cfg.egress.intercept.unrestrictedFile != null) { description = "Watch the unrestricted agents list for the egress intercept exemptions"; wantedBy = [ "paths.target" ]; pathConfig = { PathChanged = toString cfg.egress.intercept.unrestrictedFile; Unit = "hive-egress-pin.service"; }; }; # nspawn names the host end of a container's veth `vb-` when the # container joins a bridge. services.udev.extraRules = '' SUBSYSTEM=="net", ACTION=="add", KERNEL=="vb-*", TAG+="systemd", ENV{SYSTEMD_WANTS}+="hive-egress-pin@%k.service" ''; }; }