log the agent bridge's forwarded flows in the hive_egress table

A forward chain in the hive-owned nft table logs each new flow from the agent bridge to the uplink (v4 and v6) and counts every packet. It never blocks. The mesh interface is skipped. Logging is rate limited per source address; a flow that limit cannot log gets one line from a shared limit with its own prefix. services.hyperhive.network.egress.forward selects accept, log (default) or drop; drop is refused at evaluation until the interception it needs exists.

nix/host-modules/hive-egress.nix

@@ -5,11 +5,13 @@
 # host lives here, in a table no firewall backend writes. Only
 # `networking.nftables.flushRuleset` removes it; the unit re-applies it then.
 #
-# The table holds one chain: traffic from the agent bridge to host
+# The table holds two chains. `input`: traffic from the agent bridge to host
 # loopback is dropped. The kernel does the same by default (a packet for
 # 127.0.0.0/8 arriving on a non-loopback interface is a martian while
 # `route_localnet` is 0); this rule is the second layer, and the assertion
-# below keeps the first one on.
+# below keeps the first one on. `forward` (`egress.forward = "log"`): traffic
+# the host routes from the agent bridge to anywhere else is logged and
+# counted, never blocked.
 {
   lib,
   config,
@@ -19,20 +21,63 @@
 let
   cfg = config.services.hyperhive.network;
   nft = lib.getExe pkgs.nftables;
+  br = cfg.bridgeName;
+
+  # The mesh interface (swarm-wireguard.nix): inter-hive traffic is not egress.
+  meshInterface = "wg-hive";
+
+  # Logging is per source address, so one busy agent cannot use up the budget
+  # of another: a burst of 20 lines, then one a second, and an address is
+  # forgotten after a minute without a new flow. The rule accepts after it
+  # logs. A flow the rule cannot log falls through to the unmetered rule below
+  # it: the address was over its rate, or the set is full (`update` then fails
+  # and the rule stops matching), which a container spoofing source addresses
+  # can bring about. Only that case reaches the unmetered rule, so a flow
+  # gets one line at most. The unmetered rule has one budget for all
+  # addresses, 5 lines a second after a burst of 50, about 100 MB of journal a
+  # day when it never rests.
+  logRate = "1/second burst 20 packets";
+  unmeteredRate = "5/second burst 50 packets";
+  fromBridge = ''iifname "${br}" oifname != "${br}"'';
+  logRule =
+    family: addr:
+    ''${fromBridge} ct state new update @log_${family} { ${addr} saddr limit rate ${logRate} } log prefix "hive-egress-would-drop " accept'';
+
+  # Priority `filter - 10` runs before the NixOS firewall's forward handling,
+  # which sits at `filter` on both backends (`FORWARD` of the iptables backend,
+  # the `forward` chain of `nixos-fw`). An `accept` here ends this chain only,
+  # so the chain neither allows what those refuse nor refuses anything itself.
+  # Traffic to the host's own addresses (squid, dnsmasq, the gateway) is
+  # delivered locally and never reaches the forward hook.
+  forwardChain = ''
+    set log_v4 { type ipv4_addr; flags dynamic; timeout 1m; size 4096; }
+    set log_v6 { type ipv6_addr; flags dynamic; timeout 1m; size 4096; }
+
+    chain forward {
+      type filter hook forward priority filter - 10; policy accept;
+      iifname "${br}" oifname "${meshInterface}" accept
+      ${fromBridge} counter
+      ${logRule "v4" "ip"}
+      ${logRule "v6" "ip6"}
+      ${fromBridge} ct state new limit rate ${unmeteredRate} log prefix "hive-egress-would-drop-unmetered "
+      ${fromBridge} accept
+    }
+  '';
 
   # One transaction: `add` + `delete` drop an old copy of the table and the
   # definition recreates it, so the file loads the same on first start, on a
   # reload and after something else flushed the ruleset.
-  ruleset = pkgs.writeText "hive-egress.nft" ''
+  rulesetText = ''
     add table inet hive_egress
     delete table inet hive_egress
 
     table inet hive_egress {
       chain input {
         type filter hook input priority filter - 10; policy accept;
-        iifname "${cfg.bridgeName}" ip daddr 127.0.0.0/8 counter drop
-        iifname "${cfg.bridgeName}" ip6 daddr ::1 counter drop
+        iifname "${br}" ip daddr 127.0.0.0/8 counter drop
+        iifname "${br}" ip6 daddr ::1 counter drop
       }
+      ${lib.optionalString (cfg.egress.forward == "log") forwardChain}
     }
   '';
 
@@ -48,10 +93,52 @@ let
       "1"
       true
     ];
+  ruleset = pkgs.writeText "hive-egress.nft" cfg.egress.ruleset;
 in
 {
+  options.services.hyperhive.network.egress.forward = lib.mkOption {
+    type = lib.types.enum [
+      "accept"
+      "log"
+      "drop"
+    ];
+    default = "log";
+    description = ''
+      What the `inet hive_egress` table does with traffic the host forwards
+      from the agent bridge to anywhere but the bridge and the mesh.
+
+      - `accept`: nothing; the table has no forward chain.
+      - `log`: each new flow is logged with the prefix `hive-egress-would-drop `,
+        rate limited per source address, and counted. A flow the per-source limit
+        cannot log gets a line with the prefix `hive-egress-would-drop-unmetered `
+        from a limit shared by all addresses. Nothing is blocked. The source
+        address maps to an agent through the hive's DHCP leases.
+      - `drop`: not available yet; the evaluation refuses it.
+    '';
+  };
+
+  options.services.hyperhive.network.egress.ruleset = lib.mkOption {
+    type = lib.types.lines;
+    internal = true;
+    readOnly = true;
+    default = rulesetText;
+    defaultText = lib.literalMD "the `inet hive_egress` table definition";
+    description = ''
+      The nft file the `hive-egress` unit loads, as text.
+    '';
+  };
+
   config = lib.mkIf cfg.enable {
     assertions = [
+      {
+        assertion = cfg.egress.forward != "drop";
+        message = ''
+          services.hyperhive.network.egress.forward = "drop" is not available yet:
+          the interception that lets an agent's web traffic through squid has not
+          landed, so a drop would cut every direct connection with no replacement.
+          Use "log" or "accept".
+        '';
+      }
       {
         assertion = !(lib.any (k: isOn (config.boot.kernel.sysctl.${k} or 0)) routeLocalnetKeys);
         message = ''
@@ -64,7 +151,7 @@ in
     ];
 
     systemd.services.hive-egress = {
-      description = "hive nftables table (agent bridge to host loopback drop)";
+      description = "hive nftables table (agent bridge egress log, host loopback drop)";
       wantedBy = [ "multi-user.target" ];
       wants = [ "network-pre.target" ];
       before = [ "network-pre.target" ];