# The hive's own nftables tables, `inet hive_egress` and `bridge hive_egress`, # loaded by a systemd oneshot. They are independent of the NixOS firewall: # `networking.firewall` renders only the rules of the backend in use # (`iptables` unless `networking.nftables.enable` is set), so a rule that has # to hold on every host lives here, in tables no firewall backend writes. Only # `networking.nftables.flushRuleset` removes them; the unit re-applies them then. # # The `inet` table holds three chains. `input`: traffic from the agent bridge to host # loopback is dropped. The kernel does the same by default (a packet for # 127.0.0.0/8 arriving on a non-loopback interface is a martian while # `route_localnet` is 0); this rule is the second layer, and the assertion # below keeps the first one on. `forward` (`egress.forward`): traffic from the # agent bridge to anywhere else is logged and counted (`log`) or logged and # refused (`reject`). `intercept` (`egress.intercept.enable`, set by # hive-squid.nix): the web traffic of the bridge's agents is redirected into # squid's intercept ports (port 443 through the ClientHello relay). Both leave # alone the ports in `direct_mac`, the service containers and the unrestricted # agents. `hive-priv egress-pin` and `egress-load` fill that set from the # bridge table's bindings, so a MAC in no binding is intercepted and refused. # # The `bridge` table checks the source address of every frame a port of the # agent bridge sends (`egress.antiSpoof`). It is a `bridge`-family table because # traffic between two ports is bridged: the `inet` hooks see it as coming from # the bridge device, and only a bridge hook knows the port it came in on # (`iifname` is the veth). Which MAC a port may use and which IPv4 address a MAC # may use are not known until a container runs, so they are set contents. # `hive-priv egress-load` puts the current ones into the same transaction as the # tables; `egress-pin` (per new port from udev, and from a timer for the ports # still unbound) and dnsmasq's `dhcp-script` (per lease and renewal) keep them # current. `hive-priv`'s `egress.rs` documents how. { lib, config, pkgs, ... }: let cfg = config.services.hyperhive.network; egressCfg = config.services.hyperhive.deploy.hive-controller.egress; nft = lib.getExe pkgs.nftables; br = cfg.bridgeName; # With `refuseEch`, the redirect of port 443 ends at `hive-priv # egress-ech-relay`, which refuses a ClientHello that carries the ECH # extension and sends the rest on to squid's HTTPS intercept port. The relay # connects to the address the agent dialed, bound to a source address of the # pool and carrying the mark; the `output` chain DNATs those connections to # squid. `egress_ech.rs` derives the pool address from the agent's and # `egress-whois` maps it back, so the pool is the one defined there. intercept = egressCfg.intercept; refuseEch = intercept.enable && intercept.refuseEch; relayPool = "198.18.0.0/15"; relayMark = "0x45434800"; relayCommand = "${hivePriv} egress-ech-relay ${cfg.bridgeIp}:${toString intercept.relayPort} ${relayMark}"; # The sub-commands of hive-priv that fill the bridge table's sets. The path is # the one every platform container's unit uses (hive-priv.nix points it at the # current build). hivePriv = (import ./lib/container-rootfs-check.nix { inherit config; }).hivePriv; pinCommand = args: lib.trim "${hivePriv} egress-pin ${args}"; loadCommand = "${hivePriv} egress-load ${ruleset}"; pinPath = [ pkgs.nftables pkgs.iproute2 pkgs.util-linux ]; # systemd gives a oneshot unit no start timeout unless it is set. A pass takes # the lock for up to 30 s and runs each child for up to 10 s (hive-priv's # `egress.rs`); this is above both, and kills a pass that still hangs so the # timer and udev can start the next. pinTimeout = "90s"; pinEnvironment = { HIVE_EGRESS_BRIDGE = br; } // lib.optionalAttrs (egressCfg.intercept.unrestrictedFile != null) { HIVE_EGRESS_UNRESTRICTED = toString egressCfg.intercept.unrestrictedFile; } // lib.optionalAttrs (egressCfg.connectionsFile != null) { HIVE_EGRESS_CONNECTIONS = toString egressCfg.connectionsFile; }; leaseScript = pkgs.writeShellScript "hive-egress-lease" '' export PATH=${lib.makeBinPath [ pkgs.nftables ]} exec ${hivePriv} egress-lease "$@" ''; # Logging is per source address, so one busy agent cannot use up the budget # of another: a burst of 20 lines, then one a second, and an address is # forgotten after a minute without a new flow. The rule decides after it # logs. A flow the rule cannot log falls through to the unmetered rule below # it: the address was over its rate, or the set is full (`update` then fails # and the rule stops matching), which a container spoofing source addresses # can bring about. Only that case reaches the unmetered rule, so a flow # gets one line at most. The unmetered rule has one budget for all # addresses, 5 lines a second after a burst of 50, about 100 MB of journal a # day when it never rests. logRate = "1/second burst 20 packets"; unmeteredRate = "5/second burst 50 packets"; fromBridge = ''iifname "${br}" oifname != "${br}"''; # `log` accepts what it logs. `reject` logs and refuses in one rule, so a # line always names a refused flow. A TCP flow gets a reset, which fails the # connect at once (to a TCP socket an ICMP error is a soft error, and the # connect would stall); anything else gets ICMP admin-prohibited. A rule that # refuses is therefore a pair: one for TCP, one for the rest. isReject = egressCfg.forward == "reject"; logPrefix = if isReject then "hive-egress-reject" else "hive-egress-would-drop"; refuse = { tcp = "meta l4proto tcp"; other = "meta l4proto != tcp"; }; verdict = { tcp = "reject with tcp reset"; other = "reject with icmpx type admin-prohibited"; }; decide = statements: if isReject then map ( k: lib.concatStringsSep " " ( lib.filter (x: x != "") [ fromBridge refuse.${k} statements verdict.${k} ] ) ) [ "tcp" "other" ] else [ "${fromBridge} ${statements} accept" ]; logRules = family: addr: decide ''ct state new update @log_${family} { ${addr} saddr limit rate ${logRate} } log prefix "${logPrefix} "''; # In `log` mode the unmetered rule has no verdict: the chain's last rule is # the verdict for every flow that gets past the logging. unmeteredRules = let statements = ''ct state new limit rate ${unmeteredRate} log prefix "${logPrefix}-unmetered "''; in if isReject then decide statements else [ "${fromBridge} ${statements}" ]; # What no rule above decided, a packet that is no new flow included. finalRules = if isReject then decide "" else [ "${fromBridge} accept" ]; # IPv4 TCP from the agent bridge to ports 80 and 443. `fib daddr oifname` is # the interface the route to the destination leaves by: the bridge (another # container) and `lo` (an address of the host: `fib daddr type local`) are # not egress. The redirect rewrites the destination to # the bridge address and the port of squid's intercept listener (for port 443 # with `refuseEch`, of the relay), which reads the original destination from # conntrack. IPv6 is left alone: the agent bridge has no IPv6 addressing. interceptChain = '' chain intercept { type nat hook prerouting priority dstnat; policy accept; iifname "${br}" meta nfproto ipv4 ether saddr @direct_mac return iifname "${br}" meta nfproto ipv4 fib daddr type local return iifname "${br}" meta nfproto ipv4 fib daddr oifname "${br}" return iifname "${br}" meta nfproto ipv4 tcp dport 80 counter redirect to :${toString egressCfg.intercept.httpPort} iifname "${br}" meta nfproto ipv4 tcp dport 443 counter redirect to :${ toString (if refuseEch then intercept.relayPort else intercept.httpsPort) } } '' + lib.optionalString refuseEch '' chain intercept_relay { type nat hook output priority dstnat; policy accept; meta nfproto ipv4 ip saddr ${relayPool} meta mark ${relayMark} tcp dport 443 counter dnat ip to ${cfg.bridgeIp}:${toString intercept.httpsPort} } ''; # Priority `filter - 10` runs before the NixOS firewall's forward handling, # which sits at `filter` on both backends (`FORWARD` of the iptables backend, # the `forward` chain of `nixos-fw`). An `accept` here ends this chain only, # so it allows nothing those refuse; a `reject` ends the packet. The MACs # in `direct_mac` (the unrestricted agents and the service # containers, the traffic the intercept chain leaves alone) come first in # both modes, so a `log` line names what `reject` would refuse. The TCP # connections in `agent_allow` (an agent's `ssh_connections`, resolved by # hive-c0re) are accepted next, with the same effect. Traffic to # the host's own addresses (squid, dnsmasq, the gateway) is delivered # locally and never reaches the forward hook. forwardChain = '' set log_v4 { type ipv4_addr; flags dynamic; timeout 1m; size 4096; } set log_v6 { type ipv6_addr; flags dynamic; timeout 1m; size 4096; } chain forward { type filter hook forward priority filter - 10; policy accept; ${fromBridge} ether saddr @direct_mac accept ${fromBridge} meta nfproto ipv4 meta l4proto tcp ether saddr . ip daddr . tcp dport @agent_allow accept ${fromBridge} counter ${lib.concatStringsSep "\n " ( logRules "v4" "ip" ++ logRules "v6" "ip6" ++ unmeteredRules ++ finalRules )} } ''; # The bridge table. A frame of a port goes through `prerouting` in this # order: the port must be bound to the frame's source MAC, then the frame's # protocol decides which source address it may carry. Everything else logs in # `spoof_`, then `spoof` drops it (`antiSpoof = "log"`: accepts). # # - `port_mac` and `port_chaddr` hold the bound (port, MAC) pairs, the second # as the 48 bits that sit at the `chaddr` field of a DHCP message. A port # with no pair sends nothing, DHCP included. # - `lease_v4` holds the leased (MAC, IPv4) pairs; each times out with its # lease, and a renewal moves the timeout. # - IPv6 has no leases: a source is link-local, or `::` in the packets that # duplicate address detection and multicast listener reports send before an # address exists. The bridge carries no other IPv6 traffic. # - DHCP: any message to port 67 must carry the sender's own MAC as `chaddr`, # so a port cannot lease, renew or release another's address. A client # message to the broadcast address or the bridge address then passes from # any source address: a client with no address sends from 0.0.0.0, and one # whose pair has left `lease_v4` renews (unicast) or rebinds (broadcast) # from its own address. The reply puts the pair back. # - ARP: the sender's MAC is the port's, and its address 0.0.0.0 (a probe) or # leased to that MAC. # - A VLAN tag is refused: a tagged frame would carry its packet past the # protocol checks. The bridge uses no VLANs. # # Logging is per ingress port, in the style of the forward chain: a burst of # 20 lines, then one a second, and a port is forgotten after a minute. A # packet its port cannot log (over its rate, or the set is full) falls to the # unmetered line, which has one budget for all ports and reasons: the named # limit `spoof_unmetered`. It logs packets, not flows: the bridge hook has no # connection tracking. spoofVerdict = if egressCfg.antiSpoof == "drop" then "drop" else "accept"; # The log prefix names the check that refused the frame: `mac` (the port is # not bound to the source MAC), `vlan`, `dhcp` (the `chaddr` is not the # port's), `v4`, `v6`, and `arp` (the ARP sender's MAC or address is not the # port's). Each reason has its own chain and counters; `spoof` holds the # verdict and one counter for every refused frame. spoofReasons = [ "mac" "vlan" "dhcp" "v4" "v6" "arp" ]; spoofChain = reason: '' chain spoof_${reason} { update @spoof_ports { iifname limit rate ${logRate} } counter log prefix "hive-egress-spoof ${reason} " goto spoof limit name "spoof_unmetered" counter log prefix "hive-egress-spoof-unmetered ${reason} " goto spoof counter goto spoof } ''; bridgeTable = '' table bridge hive_egress { set port_mac { type ifname . ether_addr; } set port_chaddr { typeof iifname . @th,288,48; } set lease_v4 { type ether_addr . ipv4_addr; flags timeout; size 65536; } set spoof_ports { type ifname; flags dynamic; timeout 1m; size 4096; } limit spoof_unmetered { rate ${unmeteredRate} } chain spoof { counter ${spoofVerdict} } ${lib.concatMapStringsSep "\n" spoofChain spoofReasons} chain l3_v4 { udp dport 67 iifname . @th,288,48 != @port_chaddr goto spoof_dhcp ip daddr { 255.255.255.255, ${cfg.bridgeIp} } udp sport 68 udp dport 67 accept ether saddr . ip saddr @lease_v4 accept goto spoof_v4 } chain l3_v6 { ip6 saddr fe80::/10 accept ip6 saddr :: icmpv6 type { nd-neighbor-solicit, mld2-listener-report } accept goto spoof_v6 } chain l3_arp { iifname . arp saddr ether != @port_mac goto spoof_arp arp saddr ip 0.0.0.0 accept arp saddr ether . arp saddr ip @lease_v4 accept goto spoof_arp } chain prerouting { type filter hook prerouting priority -300; policy accept; meta ibrname != "${br}" accept iifname . ether saddr != @port_mac goto spoof_mac ether type { 8021q, 8021ad } goto spoof_vlan ether type ip goto l3_v4 ether type ip6 goto l3_v6 ether type arp goto l3_arp } } ''; # One transaction: `add` + `delete` drop an old copy of each table and the # definitions recreate them, so the file loads the same on first start, on a # reload and after something else flushed the ruleset. rulesetText = '' add table inet hive_egress delete table inet hive_egress add table bridge hive_egress delete table bridge hive_egress table inet hive_egress { chain input { type filter hook input priority filter - 10; policy accept; iifname "${br}" ip daddr 127.0.0.0/8 counter drop iifname "${br}" ip6 daddr ::1 counter drop ${lib.optionalString refuseEch ''iifname "${br}" ip saddr ${relayPool} counter drop''} } set direct_mac { type ether_addr; } set agent_allow { type ether_addr . ipv4_addr . inet_service; } ${lib.optionalString egressCfg.intercept.enable interceptChain} ${lib.optionalString (egressCfg.forward != "accept") forwardChain} } ${bridgeTable} ''; routeLocalnetKeys = [ "net.ipv4.conf.all.route_localnet" "net.ipv4.conf.default.route_localnet" "net.ipv4.conf.${cfg.bridgeName}.route_localnet" ]; isOn = v: builtins.elem v [ 1 "1" true ]; ruleset = pkgs.writeText "hive-egress.nft" cfg.egress.ruleset; in { options.services.hyperhive.deploy.hive-controller.egress.forward = lib.mkOption { type = lib.types.enum [ "accept" "log" "reject" ]; default = "log"; description = '' What the `inet hive_egress` table does with traffic the host forwards from the agent bridge to anywhere but the bridge. In `log` and `reject` the traffic of the MACs in the `direct_mac` set (the unrestricted agents and the service containers) is accepted and never logged, and so is TCP from an agent to an address and port in the `agent_allow` set (`connectionsFile`). - `accept`: nothing; the table has no forward chain. - `log`: each new flow is logged with the prefix `hive-egress-would-drop `, rate limited per source address, and counted. A flow the per-source limit cannot log gets a line with the prefix `hive-egress-would-drop-unmetered ` from a limit shared by all addresses. Nothing is blocked. The lines are the flows `reject` would refuse. The source address maps to an agent through the hive's DHCP leases. - `reject`: the same lines with the prefix `hive-egress-reject ` and `hive-egress-reject-unmetered `, and every other packet from the bridge is refused as well: a TCP flow with a reset, anything else with ICMP administratively prohibited. Only the web traffic that the intercept chain redirects into squid still gets out. ''; }; options.services.hyperhive.deploy.hive-controller.egress.connectionsFile = lib.mkOption { type = lib.types.nullOr lib.types.path; default = null; description = '' The file hive-c0re renders with the SSH connections each agent may open, one ` ` per line: the addresses behind the `host` of each connection in the permissions record of an agent whose `ssh` switch is on. `hive-priv egress-pin` turns each line into elements of the `agent_allow` set of the `inet hive_egress` table, keyed by the MAC of the agent's port, and the forward chain accepts TCP to an address and port in it before it logs or refuses. The file is read when a port is bound, on each pass of the 30 second timer and when the file changes. `null` leaves the set alone, empty: no agent has an allowance. ''; }; options.services.hyperhive.deploy.hive-controller.egress.antiSpoof = lib.mkOption { type = lib.types.enum [ "drop" "log" ]; default = "log"; description = '' What the `bridge hive_egress` table does with a frame whose source is not the sending container's own. Every port of the agent bridge is checked: the frame's MAC must be the one bound to the port when its container started, and its IPv4 address the one leased to that MAC. A port with no bound MAC sends nothing. Allowed besides the leased address: DHCP from a DHCP client message to the broadcast or bridge address from any source address whose `chaddr` is the port's MAC, ARP probes, and IPv6 link-local sources (and `::` in neighbour discovery and listener reports). - `drop`: the frame is dropped and logged with the prefix `hive-egress-spoof `, rate limited per port; the reason is the check that failed: `mac`, `vlan`, `dhcp`, `v4`, `v6` or `arp`. A packet the per-port limit cannot log gets a line with the prefix `hive-egress-spoof-unmetered `, from one limit shared by all ports and reasons. A container sending from its leased address is unaffected while its (MAC, IP) pair is in the lease set; a lost pair, as after a host suspend/resume, cuts the container off until it leases again. - `log` (the default): the same lines and counters, and the frame goes through. For finding out why a legitimate container is refused on a live hive. It is the default because the lease set does not yet survive a host suspend/resume; `drop` returns once a resume test proves it and the `egress-lease: … skipped (stale)` lines confirm the renewal fix on a live hive. With `log`, the identity squid takes from the source of an intercepted flow (`egress.intercept`) can be spoofed: use it to diagnose, not to run. ''; }; options.services.hyperhive.deploy.hive-controller.egress.intercept = { enable = lib.mkEnableOption "the redirect of the agents' web traffic into squid's intercept ports"; httpPort = lib.mkOption { type = lib.types.port; default = 3129; description = '' The port on the bridge address where squid listens for the redirected TCP port 80 traffic. ''; }; httpsPort = lib.mkOption { type = lib.types.port; default = 3130; description = '' The port on the bridge address where squid listens for TCP port 443 traffic: what the ClientHello relay sends on, or with `refuseEch` off the redirected traffic itself. ''; }; relayPort = lib.mkOption { type = lib.types.port; default = 3131; description = '' The port on the bridge address where `hive-priv egress-ech-relay` listens for the redirected TCP port 443 traffic (`refuseEch`). ''; }; refuseEch = lib.mkOption { type = lib.types.bool; default = true; description = '' Refuse a TLS ClientHello that carries the `encrypted_client_hello` extension on the redirected port 443 traffic. Squid 7.6 sees only the outer name of such a hello, so without this an agent that holds an ECH configuration could dial an allowed name and reach another site behind the same address. `hive-priv egress-ech-relay` reads the first TLS record of each redirected connection and, if the hello is clean, sends the connection on to squid. The extension of a client with no configuration (GREASE, which Chrome and Firefox send by default) looks the same as a real one on the wire, so it is refused too: those clients fail on the redirected path and work through the proxy variables. An agent in `log` mode is observed instead: its hello goes on and the relay logs `hive-egress-ech-would-refuse`. Turn this off to send port 443 straight to squid, with ECH unchecked. ''; }; agentAllowFile = lib.mkOption { type = lib.types.nullOr lib.types.path; default = null; description = '' The squid include `hive-c0re` renders with each agent's own rules, in which an agent in `log` mode has an `acl agent_log_ proxy_auth ` line. The ClientHello relay (`refuseEch`) reads it to tell a `log` agent, whose ECH hello is let through and logged as `hive-egress-ech-would-refuse`, from a restricted one, whose hello is refused. Squid reads the same file for the same split. `null` or an unreadable file lists no `log` agent, so every agent's ECH hello is refused. ''; }; unrestrictedFile = lib.mkOption { type = lib.types.nullOr lib.types.path; default = null; description = '' The file hive-c0re renders with the names of the agents that hold the `unrestricted_internet` flag, one per line. The `direct_mac` set of the `inet hive_egress` table holds the MACs of the ports of the agents it lists and of every container that is not an agent, and the redirect skips them. `null` or an unreadable file lists no agent, so every agent's web traffic is redirected. The file is read when a port is bound, on each pass of the 30 second timer and when the file changes. ''; }; }; options.services.hyperhive.network.egress.leaseScript = lib.mkOption { type = lib.types.path; internal = true; readOnly = true; default = leaseScript; defaultText = lib.literalMD "the script that loads a lease into the bridge table"; description = '' The program dnsmasq runs on each lease change and renewal (its `dhcp-script` with `script-on-renewal`). ''; }; options.services.hyperhive.network.egress.ruleset = lib.mkOption { type = lib.types.lines; internal = true; readOnly = true; default = rulesetText; defaultText = lib.literalMD "the `inet hive_egress` and `bridge hive_egress` table definitions"; description = '' The nft file the `hive-egress` unit loads, as text. ''; }; config = lib.mkIf cfg.enable { assertions = [ { assertion = !refuseEch || lib.toInt (lib.last (lib.splitString "/" cfg.networkCidr)) >= 15; message = '' services.hyperhive.network.bridgePrefixLength must be 15 or more while egress.intercept.refuseEch is on: the relay names a flow's source with the low 17 bits of the agent's address. ''; } { assertion = egressCfg.forward != "reject" || egressCfg.intercept.enable; message = '' services.hyperhive.deploy.hive-controller.egress.forward = "reject" needs services.hyperhive.deploy.hive-controller.egress.intercept.enable = true: without the intercept nothing redirects the agents' web traffic into squid, and the reject would refuse all of it. Enable the intercept (hive-squid.nix does on a hive controller host), or keep forward = "log". ''; } { assertion = !(lib.any (k: isOn (config.boot.kernel.sysctl.${k} or 0)) routeLocalnetKeys); message = '' boot.kernel.sysctl must not turn on route_localnet (${lib.concatStringsSep ", " routeLocalnetKeys}). With it on, the kernel routes a packet from the agent bridge to 127.0.0.0/8 instead of dropping it as a martian, which leaves the hive_egress table as the only barrier between an agent and the host's loopback services. ''; } ]; services.hyperhive.network.exposeHostPorts = lib.mkIf refuseEch [ intercept.relayPort ]; # The relay is not part of the tables: with it down, the redirect of port 443 # reaches a closed port and the agent's connection is refused. systemd.services.hive-egress-ech-relay = lib.mkIf refuseEch { description = "refuse TLS hellos with encrypted client hello on the way to squid's HTTPS intercept port"; wantedBy = [ "multi-user.target" ]; wants = [ "hive-egress.service" ]; after = [ "hive-egress.service" "systemd-tmpfiles-setup.service" "systemd-tmpfiles-resetup.service" ]; environment = lib.optionalAttrs (intercept.agentAllowFile != null) { HIVE_EGRESS_AGENT_ALLOW = toString intercept.agentAllowFile; }; serviceConfig = { # The pool is local, so the relay can bind to a source address in it. ExecStartPre = "+${pkgs.iproute2}/bin/ip route replace local ${relayPool} dev lo"; ExecStart = relayCommand; ExecStopPost = "+-${pkgs.iproute2}/bin/ip route del local ${relayPool} dev lo"; # The bridge address may not be assigned yet at first start. Restart = "on-failure"; RestartSec = 5; # The packet mark needs CAP_NET_ADMIN. The include is `squid`'s group's. DynamicUser = true; SupplementaryGroups = lib.optional (intercept.agentAllowFile != null) "squid"; AmbientCapabilities = "CAP_NET_ADMIN"; CapabilityBoundingSet = "CAP_NET_ADMIN"; NoNewPrivileges = true; ProtectSystem = "strict"; ProtectHome = true; PrivateTmp = true; PrivateDevices = true; ProtectKernelTunables = true; ProtectKernelModules = true; ProtectKernelLogs = true; ProtectControlGroups = true; RestrictAddressFamilies = "AF_INET"; RestrictNamespaces = true; RestrictRealtime = true; RestrictSUIDSGID = true; LockPersonality = true; SystemCallArchitectures = "native"; }; }; systemd.services.hive-egress = { description = "hive nftables tables (agent bridge egress log and source check, host loopback drop)"; wantedBy = [ "multi-user.target" ]; wants = [ "network-pre.target" ]; before = [ "network-pre.target" ]; # `nftables.service` runs `flush ruleset` where `networking.nftables.flushRuleset` # is on: start after it, follow its restarts and re-apply on its reload. # The tmpfiles units link `hivePriv`. after = [ "nftables.service" "systemd-tmpfiles-setup.service" "systemd-tmpfiles-resetup.service" ]; partOf = [ "nftables.service" ]; unitConfig.ReloadPropagatedFrom = [ "nftables.service" ]; reloadIfChanged = true; path = pinPath; environment = pinEnvironment; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; # The tables load with the bindings of the running containers' ports and # the leases of dnsmasq's file in the same transaction, so a reload never # leaves the bridge with empty sets. ExecStart = loadCommand; ExecReload = loadCommand; ExecStop = [ "-${nft} delete table inet hive_egress" "-${nft} delete table bridge hive_egress" ]; }; }; # Binds one port of the bridge once its veth exists: the run waits for the # container to register with systemd-machined. systemd.services."hive-egress-pin@" = { description = "bind agent bridge port %i to its container's MAC"; wants = [ "hive-egress.service" ]; after = [ "hive-egress.service" ]; path = pinPath; environment = pinEnvironment; serviceConfig = { Type = "oneshot"; ExecStart = pinCommand "%i"; TimeoutStartSec = pinTimeout; }; }; # Looks again at the ports still unbound. A port a rival MAC kept unbound # sends nothing, so nothing else would tell the hive it can be bound now. # Ports with a binding cost a listing of the bridge and no container is read. systemd.services.hive-egress-pin = { description = "bind the agent bridge ports that have no binding yet"; wants = [ "hive-egress.service" ]; after = [ "hive-egress.service" ]; path = pinPath; environment = pinEnvironment; serviceConfig = { Type = "oneshot"; ExecStart = pinCommand ""; TimeoutStartSec = pinTimeout; }; }; systemd.timers.hive-egress-pin = { description = "bind the agent bridge ports that have no binding yet"; wantedBy = [ "timers.target" ]; timerConfig = { OnBootSec = "1min"; OnUnitActiveSec = "30s"; AccuracySec = "5s"; }; }; # A pass also refills `direct_mac`, so a change of the unrestricted list # takes effect at once instead of at the next tick of the timer. systemd.paths.hive-egress-direct = lib.mkIf (egressCfg.intercept.unrestrictedFile != null) { description = "Watch the unrestricted agents list for the egress intercept exemptions"; wantedBy = [ "paths.target" ]; pathConfig = { PathChanged = toString egressCfg.intercept.unrestrictedFile; Unit = "hive-egress-pin.service"; }; }; # The same pass refills `agent_allow`, so a connection an operator declares # is allowed at once. systemd.paths.hive-egress-connections = lib.mkIf (egressCfg.connectionsFile != null) { description = "Watch the agents' allowed connections list for the egress forward chain"; wantedBy = [ "paths.target" ]; pathConfig = { PathChanged = toString egressCfg.connectionsFile; Unit = "hive-egress-pin.service"; }; }; # nspawn names the host end of a container's veth `vb-` when the # container joins a bridge. services.udev.extraRules = '' SUBSYSTEM=="net", ACTION=="add", KERNEL=="vb-*", TAG+="systemd", ENV{SYSTEMD_WANTS}+="hive-egress-pin@%k.service" ''; }; }