load the agent-bridge to loopback drop in its own nft table

docs/4-implementation/networking/network.md

@@ -214,7 +214,7 @@ The hive's own collector needs none of this: agents dial it by
 
 This is **firewall-only**: the host service must bind an address
 reachable from the bridge — `0.0.0.0` or the bridge IP — not loopback
-only. The bridge→`127.0.0.0/8` DROP rule (below) is unchanged, so a
+only. The bridge→`127.0.0.0/8` DROP rule (below) still applies, so a
 service bound to `127.0.0.1` only stays unreachable; rebind it to
 `0.0.0.0`.
 
@@ -512,13 +512,13 @@ Each agent container runs in a private network namespace with a dedicated
 veth pair attached to the bridge. The following table summarises what
 the nix side sets up unconditionally:
 
-| effect         | mechanism                                                                                                                                               |
+| effect         | mechanism                                                                                                                                                                                                               |
-| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| IP forwarding  | `boot.kernel.sysctl."net.ipv4.ip_forward" = 1`                                                                                                          |
+| IP forwarding  | `boot.kernel.sysctl."net.ipv4.ip_forward" = 1`                                                                                                                                                                          |
-| Internet NAT   | `networking.nat { enable = true; internalInterfaces = [ bridgeName ]; }` — MASQUERADE on packets leaving via any external NIC                           |
+| Internet NAT   | `networking.nat { enable = true; internalInterfaces = [ bridgeName ]; }` — MASQUERADE on packets leaving via any external NIC                                                                                           |
-| Loopback DROP  | `networking.firewall.extraInputRules` — drops bridge-subnet → `127.0.0.0/8` traffic; defence-in-depth against routing table leaks                       |
+| Loopback DROP  | `nix/host-modules/hive-egress.nix` — own nftables table `inet hive_egress` (whichever firewall the host runs): drops bridge → `127.0.0.0/8` and `::1` traffic; the kernel default `route_localnet=0` is the first layer |
-| Gateway access | `networking.firewall.interfaces.<bridge>.allowedTCPPorts = [ 80 443 ]` — lets isolated agents (private netns, veth on bridge) reach nginx on the host   |
+| Gateway access | `networking.firewall.interfaces.<bridge>.allowedTCPPorts = [ 80 443 ]` — lets isolated agents (private netns, veth on bridge) reach nginx on the host                                                                   |
-| c0re signal    | `HIVE_NETWORK_BRIDGE`, `HIVE_NETWORK_SUBNET` in `systemd.services.hive-c0re.environment` — both **required**; `hive-c0re` refuses to start without them |
+| c0re signal    | `HIVE_NETWORK_BRIDGE`, `HIVE_NETWORK_SUBNET` in `systemd.services.hive-c0re.environment` — both **required**; `hive-c0re` refuses to start without them                                                                 |
 
 `HIVE_NETWORK_SUBNET` is the host-side bridge IP + prefix (e.g.
 `10.42.0.1/24`), **not** the canonical network address. The Rust side
@@ -574,8 +574,9 @@ map](#network-map) above. `PRIVATE_NETWORK=1` has no effect on a path
 that never touches the network stack.
 
 The nix side also enables IP forwarding + NAT (agents reach the internet
-through the host) and drops bridge-subnet → loopback traffic (defence-in-depth
+through the host) and drops bridge-subnet → loopback traffic from its own
-against a compromised agent reaching the c0re dashboard HTTP at
+nftables table (`inet hive_egress`, `nix/host-modules/hive-egress.nix`;
+defence-in-depth against a compromised agent reaching the c0re dashboard HTTP at
 `127.0.0.1`). Agents have no legitimate reason to reach the dashboard over
 loopback — the hive-c0re admin socket is a UDS, not TCP.
 

nix/host-modules/default.nix

@@ -21,6 +21,7 @@
     ./hive-matrix-service.nix
     ./hive-matrix.nix
     ./hive-network.nix
+    ./hive-egress.nix
     ./hive-priv.nix
     ./hive-archive-backup.nix
     ./hive-priv-cache.nix

nix/host-modules/hive-egress.nix

@@ -0,0 +1,86 @@
+# The hive's own nftables table, `inet hive_egress`, loaded by a systemd
+# oneshot. It is independent of the NixOS firewall: `networking.firewall`
+# renders only the rules of the backend in use (`iptables` unless
+# `networking.nftables.enable` is set), so a rule that has to hold on every
+# host lives here, in a table no firewall backend writes. Only
+# `networking.nftables.flushRuleset` removes it; the unit re-applies it then.
+#
+# The table holds one chain: traffic from the agent bridge to host
+# loopback is dropped. The kernel does the same by default (a packet for
+# 127.0.0.0/8 arriving on a non-loopback interface is a martian while
+# `route_localnet` is 0); this rule is the second layer, and the assertion
+# below keeps the first one on.
+{
+  lib,
+  config,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.hyperhive.network;
+  nft = lib.getExe pkgs.nftables;
+
+  # One transaction: `add` + `delete` drop an old copy of the table and the
+  # definition recreates it, so the file loads the same on first start, on a
+  # reload and after something else flushed the ruleset.
+  ruleset = pkgs.writeText "hive-egress.nft" ''
+    add table inet hive_egress
+    delete table inet hive_egress
+
+    table inet hive_egress {
+      chain input {
+        type filter hook input priority filter - 10; policy accept;
+        iifname "${cfg.bridgeName}" ip daddr 127.0.0.0/8 counter drop
+        iifname "${cfg.bridgeName}" ip6 daddr ::1 counter drop
+      }
+    }
+  '';
+
+  routeLocalnetKeys = [
+    "net.ipv4.conf.all.route_localnet"
+    "net.ipv4.conf.default.route_localnet"
+    "net.ipv4.conf.${cfg.bridgeName}.route_localnet"
+  ];
+  isOn =
+    v:
+    builtins.elem v [
+      1
+      "1"
+      true
+    ];
+in
+{
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = !(lib.any (k: isOn (config.boot.kernel.sysctl.${k} or 0)) routeLocalnetKeys);
+        message = ''
+          boot.kernel.sysctl must not turn on route_localnet (${lib.concatStringsSep ", " routeLocalnetKeys}).
+          With it on, the kernel routes a packet from the agent bridge to 127.0.0.0/8
+          instead of dropping it as a martian, which leaves the hive_egress
+          table as the only barrier between an agent and the host's loopback services.
+        '';
+      }
+    ];
+
+    systemd.services.hive-egress = {
+      description = "hive nftables table (agent bridge to host loopback drop)";
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "network-pre.target" ];
+      before = [ "network-pre.target" ];
+      # `nftables.service` runs `flush ruleset` where `networking.nftables.flushRuleset`
+      # is on: start after it, follow its restarts and re-apply on its reload.
+      after = [ "nftables.service" ];
+      partOf = [ "nftables.service" ];
+      unitConfig.ReloadPropagatedFrom = [ "nftables.service" ];
+      reloadIfChanged = true;
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        ExecStart = "${nft} -f ${ruleset}";
+        ExecReload = "${nft} -f ${ruleset}";
+        ExecStop = "-${nft} delete table inet hive_egress";
+      };
+    };
+  };
+}

nix/host-modules/hive-network.nix

@@ -122,7 +122,7 @@ in
 
         **The host service must bind an address reachable from the bridge** —
         `0.0.0.0` or the bridge IP (`bridgeIp`) — not loopback-only. The
-        bridge→`127.0.0.0/8` DROP rule (defence-in-depth) is unchanged: this
+        bridge→`127.0.0.0/8` DROP rule (`hive-egress.nix`) still applies: this
         only opens the firewall, it does not bridge loopback. A service that
         binds `127.0.0.1` only is still unreachable; rebind it to `0.0.0.0`.
 
@@ -276,12 +276,6 @@ in
         internalInterfaces = [ cfg.bridgeName ];
       };
 
-      # Defence-in-depth: DROP bridge→loopback so compromised agents can't
-      # reach host-loopback services even via routing table leaks.
-      networking.firewall.extraInputRules = ''
-        ip saddr ${cfg.bridgeIp}/${toString cfg.bridgePrefixLength} ip daddr 127.0.0.0/8 drop
-      '';
-
       # Allow isolated agents to reach the gateway (nginx on the host, shared
       # netns). Port 80 covers `http://forge.<domain>`, per-agent UI proxies,
       # and any other HTTP services the gateway fronts. Port 443 for HTTPS.
@@ -304,7 +298,7 @@ in
     # Host port exposure: open each `exposeHostPorts` entry on the bridge
     # firewall so agents can reach a host service at `<bridgeIp>:P`. The host
     # service must bind `0.0.0.0` or the bridge IP (a loopback-only bind stays
-    # unreachable — the bridge→127.0.0.0/8 DROP rule above is unchanged). This
+    # unreachable — the bridge→127.0.0.0/8 DROP rule in hive-egress.nix still applies). This
     # is firewall-only by design: a host service that binds `0.0.0.0` already
     # serves the bridge IP, so an extra bridge-IP proxy would only collide
     # (EADDRINUSE) with it. Merges with the [ 80 443 ] gateway ports above.

nix/module-eval/core-toggle.nix

@@ -63,6 +63,13 @@ let
 
   inherit (shared) forgeHere;
 
+  # A host that turns `route_localnet` on, for the assertion in
+  # ../host-modules/hive-egress.nix: the kernel then routes a bridge packet
+  # for 127.0.0.0/8 instead of dropping it, so the config is refused.
+  routeLocalnetOn = hiveWith [
+    { boot.kernel.sysctl."net.ipv4.conf.all.route_localnet" = 1; }
+  ] { network.enable = true; };
+
   # Every hive defaults the agents' queue address to the queue's name, so the
   # only hive without one is a hive told to have none. Also the host still
   # setting the four removed host options; no case on its queue reads them.
@@ -1497,6 +1504,33 @@ let
         && bare.services.dnsmasq.enable
         && bare.networking.bridges ? hive-br0;
     }
+    {
+      # The bridge→loopback drop is a unit of its own that loads its own nft
+      # table, so it does not depend on which firewall backend the host runs.
+      # `bare` is the presence arm; the firewall option the old rule used
+      # stays empty, so the rule has one home.
+      name = "the bridge to loopback drop is its own unit where the bridge is up and absent elsewhere";
+      ok =
+        let
+          unit = bare.systemd.services.hive-egress or null;
+        in
+        unit != null
+        && unit.serviceConfig.Type == "oneshot"
+        && lib.hasInfix "hive-egress.nft" unit.serviceConfig.ExecStart
+        && lib.hasInfix "delete table inet hive_egress" unit.serviceConfig.ExecStop
+        && bare.networking.firewall.extraInputRules == ""
+        && !(centralToggleOff.systemd.services ? hive-egress);
+    }
+    {
+      # The host that turns `route_localnet` on is refused by an assertion
+      # naming it; the ordinary host, which leaves it alone, is not.
+      name = "route_localnet on the sysctl is refused on a host with the bridge up";
+      ok =
+        let
+          refused = m: lib.any (a: !a.assertion && lib.hasInfix "route_localnet" a.message) m.assertions;
+        in
+        refused routeLocalnetOn && !(refused bare);
+    }
     {
       # Refused where a hive runs and nowhere else. A name is an identifier
       # only for a hive; the hive host is the control that the lookup finds the

nix/vm-tests/hive-squid.nix

@@ -8,6 +8,9 @@
 # `proxy` imports the squid module against stub options for the few hive
 # options it reads, so no hive-c0re build is pulled in; the module-eval suite
 # covers the real option wiring.
+#
+# It also covers ../host-modules/hive-egress.nix: `proxy` loads the real module, so the
+# agent's dials to the host's loopback run against the table the module installs.
 { pkgs }:
 let
   bridgeIp = "192.168.1.1";
@@ -37,6 +40,10 @@ let
             type = lib.types.bool;
             default = true;
           };
+          bridgeName = lib.mkOption {
+            type = lib.types.str;
+            default = "eth1";
+          };
         };
         otel = {
           scrapeTargets = lib.mkOption {
@@ -114,9 +121,12 @@ pkgs.testers.runNixOSTest {
         imports = [
           hiveStubs
           ../host-modules/hive-squid.nix
+          ../host-modules/hive-egress.nix
           # Listeners a leaked destination would reach: loopback (`::` lands on `::1`) and the bridge.
           (listen443 [
             "127.0.0.1"
+            # An address the agent can route to the proxy: the agent's own lo owns 127.0.0.1 itself.
+            "127.0.0.5"
             "[::1]"
             bridgeIp
           ])
@@ -174,7 +184,10 @@ pkgs.testers.runNixOSTest {
         # The gateway: nginx answers on the bridge, which agents reach directly.
         networking.firewall.interfaces.eth1.allowedTCPPorts = [ 443 ];
         # Renders the users file the way hive-c0re does.
-        environment.systemPackages = [ pkgs.mkpasswd ];
+        environment.systemPackages = [
+          pkgs.mkpasswd
+          pkgs.nftables
+        ];
         services.hyperhive.deploy.hive-controller.squid.selfcheck.unrestrictedCanary = "other.t.local";
         services.hyperhive.deploy.hive-controller.squid.selfcheck.canary = "cache.t.local";
         services.hyperhive.deploy.hive-controller.squid.baseAllowedDomains = [
@@ -684,5 +697,43 @@ pkgs.testers.runNixOSTest {
         write_users(ATLAS, UMA, SELFCHECK, SELFCHECK_U)
         assert connect("forge.t.local", ATLAS) == "200", "the file appeared"
         assert connect("other.t.local", UMA) == "200", "listed, and now logged in"
+
+    def dial(addr):
+        return agent.execute(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/443'")[0] == 0
+
+    def dropped():
+        # Packets the table's v4 rule has dropped so far.
+        out = proxy.succeed("nft list chain inet hive_egress input")
+        found = re.search(r"ip daddr 127\.0\.0\.0/8 counter packets (\d+)", out)
+        assert found, out
+        return int(found.group(1))
+
+    with checked("the agent bridge cannot reach the host's loopback, and the table is what stops it"):
+        assert dial("${bridgeIp}"), "control: a listener on the bridge IP answers"
+        # The agent has a route for an address the proxy owns on lo, and both
+        # ends accept loopback addresses on the link, so only the table is left
+        # between the dial and the listener.
+        for vm, dev in ((agent, "eth1"), (proxy, "eth1")):
+            vm.succeed(
+                "sysctl -w net.ipv4.conf.all.route_localnet=1 net.ipv4.conf.all.rp_filter=0 "
+                f"net.ipv4.conf.{dev}.route_localnet=1 net.ipv4.conf.{dev}.rp_filter=0"
+            )
+        agent.succeed("ip route add 127.0.0.5/32 via ${bridgeIp} table local")
+        proxy.succeed("nft delete table inet hive_egress")
+        assert dial("127.0.0.5"), "control: with the table gone the loopback listener answers"
+        proxy.succeed("systemctl reload hive-egress.service")
+        before = dropped()
+        assert not dial("127.0.0.5"), "the table drops the dial"
+        assert dropped() > before, "the drop rule counted it"
+        assert dial("${bridgeIp}"), "the bridge IP still answers"
+        # A firewall reload rewrites the NixOS chains and leaves this table alone.
+        proxy.succeed("systemctl reload firewall.service")
+        assert not dial("127.0.0.5"), "still dropped after a firewall reload"
+        assert dial("${bridgeIp}"), "the bridge IP still answers after the reload"
+        # With the kernel default back, the dial is refused with no table either.
+        proxy.succeed("sysctl -w net.ipv4.conf.all.route_localnet=0 net.ipv4.conf.eth1.route_localnet=0")
+        proxy.succeed("nft delete table inet hive_egress")
+        assert not dial("127.0.0.5"), "route_localnet=0 refuses it alone"
+        proxy.succeed("systemctl reload hive-egress.service")
   '';
 }