diff --git a/docs/4-implementation/networking/network.md b/docs/4-implementation/networking/network.md index 0434533d1..cd5d17c6c 100644 --- a/docs/4-implementation/networking/network.md +++ b/docs/4-implementation/networking/network.md @@ -214,7 +214,7 @@ The hive's own collector needs none of this: agents dial it by This is **firewall-only**: the host service must bind an address reachable from the bridge — `0.0.0.0` or the bridge IP — not loopback -only. The bridge→`127.0.0.0/8` DROP rule (below) is unchanged, so a +only. The bridge→`127.0.0.0/8` DROP rule (below) still applies, so a service bound to `127.0.0.1` only stays unreachable; rebind it to `0.0.0.0`. @@ -512,13 +512,13 @@ Each agent container runs in a private network namespace with a dedicated veth pair attached to the bridge. The following table summarises what the nix side sets up unconditionally: -| effect | mechanism | -| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | -| IP forwarding | `boot.kernel.sysctl."net.ipv4.ip_forward" = 1` | -| Internet NAT | `networking.nat { enable = true; internalInterfaces = [ bridgeName ]; }` — MASQUERADE on packets leaving via any external NIC | -| Loopback DROP | `networking.firewall.extraInputRules` — drops bridge-subnet → `127.0.0.0/8` traffic; defence-in-depth against routing table leaks | -| Gateway access | `networking.firewall.interfaces..allowedTCPPorts = [ 80 443 ]` — lets isolated agents (private netns, veth on bridge) reach nginx on the host | -| c0re signal | `HIVE_NETWORK_BRIDGE`, `HIVE_NETWORK_SUBNET` in `systemd.services.hive-c0re.environment` — both **required**; `hive-c0re` refuses to start without them | +| effect | mechanism | +| -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| IP forwarding | `boot.kernel.sysctl."net.ipv4.ip_forward" = 1` | +| Internet NAT | `networking.nat { enable = true; internalInterfaces = [ bridgeName ]; }` — MASQUERADE on packets leaving via any external NIC | +| Loopback DROP | `nix/host-modules/hive-egress.nix` — own nftables table `inet hive_egress` (whichever firewall the host runs): drops bridge → `127.0.0.0/8` and `::1` traffic; the kernel default `route_localnet=0` is the first layer | +| Gateway access | `networking.firewall.interfaces..allowedTCPPorts = [ 80 443 ]` — lets isolated agents (private netns, veth on bridge) reach nginx on the host | +| c0re signal | `HIVE_NETWORK_BRIDGE`, `HIVE_NETWORK_SUBNET` in `systemd.services.hive-c0re.environment` — both **required**; `hive-c0re` refuses to start without them | `HIVE_NETWORK_SUBNET` is the host-side bridge IP + prefix (e.g. `10.42.0.1/24`), **not** the canonical network address. The Rust side @@ -574,8 +574,9 @@ map](#network-map) above. `PRIVATE_NETWORK=1` has no effect on a path that never touches the network stack. The nix side also enables IP forwarding + NAT (agents reach the internet -through the host) and drops bridge-subnet → loopback traffic (defence-in-depth -against a compromised agent reaching the c0re dashboard HTTP at +through the host) and drops bridge-subnet → loopback traffic from its own +nftables table (`inet hive_egress`, `nix/host-modules/hive-egress.nix`; +defence-in-depth against a compromised agent reaching the c0re dashboard HTTP at `127.0.0.1`). Agents have no legitimate reason to reach the dashboard over loopback — the hive-c0re admin socket is a UDS, not TCP. diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 63ef2dec2..89dd19ce2 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -21,6 +21,7 @@ ./hive-matrix-service.nix ./hive-matrix.nix ./hive-network.nix + ./hive-egress.nix ./hive-priv.nix ./hive-archive-backup.nix ./hive-priv-cache.nix diff --git a/nix/host-modules/hive-egress.nix b/nix/host-modules/hive-egress.nix new file mode 100644 index 000000000..a45dfcd13 --- /dev/null +++ b/nix/host-modules/hive-egress.nix @@ -0,0 +1,86 @@ +# The hive's own nftables table, `inet hive_egress`, loaded by a systemd +# oneshot. It is independent of the NixOS firewall: `networking.firewall` +# renders only the rules of the backend in use (`iptables` unless +# `networking.nftables.enable` is set), so a rule that has to hold on every +# host lives here, in a table no firewall backend writes. Only +# `networking.nftables.flushRuleset` removes it; the unit re-applies it then. +# +# The table holds one chain: traffic from the agent bridge to host +# loopback is dropped. The kernel does the same by default (a packet for +# 127.0.0.0/8 arriving on a non-loopback interface is a martian while +# `route_localnet` is 0); this rule is the second layer, and the assertion +# below keeps the first one on. +{ + lib, + config, + pkgs, + ... +}: +let + cfg = config.services.hyperhive.network; + nft = lib.getExe pkgs.nftables; + + # One transaction: `add` + `delete` drop an old copy of the table and the + # definition recreates it, so the file loads the same on first start, on a + # reload and after something else flushed the ruleset. + ruleset = pkgs.writeText "hive-egress.nft" '' + add table inet hive_egress + delete table inet hive_egress + + table inet hive_egress { + chain input { + type filter hook input priority filter - 10; policy accept; + iifname "${cfg.bridgeName}" ip daddr 127.0.0.0/8 counter drop + iifname "${cfg.bridgeName}" ip6 daddr ::1 counter drop + } + } + ''; + + routeLocalnetKeys = [ + "net.ipv4.conf.all.route_localnet" + "net.ipv4.conf.default.route_localnet" + "net.ipv4.conf.${cfg.bridgeName}.route_localnet" + ]; + isOn = + v: + builtins.elem v [ + 1 + "1" + true + ]; +in +{ + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = !(lib.any (k: isOn (config.boot.kernel.sysctl.${k} or 0)) routeLocalnetKeys); + message = '' + boot.kernel.sysctl must not turn on route_localnet (${lib.concatStringsSep ", " routeLocalnetKeys}). + With it on, the kernel routes a packet from the agent bridge to 127.0.0.0/8 + instead of dropping it as a martian, which leaves the hive_egress + table as the only barrier between an agent and the host's loopback services. + ''; + } + ]; + + systemd.services.hive-egress = { + description = "hive nftables table (agent bridge to host loopback drop)"; + wantedBy = [ "multi-user.target" ]; + wants = [ "network-pre.target" ]; + before = [ "network-pre.target" ]; + # `nftables.service` runs `flush ruleset` where `networking.nftables.flushRuleset` + # is on: start after it, follow its restarts and re-apply on its reload. + after = [ "nftables.service" ]; + partOf = [ "nftables.service" ]; + unitConfig.ReloadPropagatedFrom = [ "nftables.service" ]; + reloadIfChanged = true; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = "${nft} -f ${ruleset}"; + ExecReload = "${nft} -f ${ruleset}"; + ExecStop = "-${nft} delete table inet hive_egress"; + }; + }; + }; +} diff --git a/nix/host-modules/hive-network.nix b/nix/host-modules/hive-network.nix index 928e5858c..a2c7a3070 100644 --- a/nix/host-modules/hive-network.nix +++ b/nix/host-modules/hive-network.nix @@ -122,7 +122,7 @@ in **The host service must bind an address reachable from the bridge** — `0.0.0.0` or the bridge IP (`bridgeIp`) — not loopback-only. The - bridge→`127.0.0.0/8` DROP rule (defence-in-depth) is unchanged: this + bridge→`127.0.0.0/8` DROP rule (`hive-egress.nix`) still applies: this only opens the firewall, it does not bridge loopback. A service that binds `127.0.0.1` only is still unreachable; rebind it to `0.0.0.0`. @@ -276,12 +276,6 @@ in internalInterfaces = [ cfg.bridgeName ]; }; - # Defence-in-depth: DROP bridge→loopback so compromised agents can't - # reach host-loopback services even via routing table leaks. - networking.firewall.extraInputRules = '' - ip saddr ${cfg.bridgeIp}/${toString cfg.bridgePrefixLength} ip daddr 127.0.0.0/8 drop - ''; - # Allow isolated agents to reach the gateway (nginx on the host, shared # netns). Port 80 covers `http://forge.`, per-agent UI proxies, # and any other HTTP services the gateway fronts. Port 443 for HTTPS. @@ -304,7 +298,7 @@ in # Host port exposure: open each `exposeHostPorts` entry on the bridge # firewall so agents can reach a host service at `:P`. The host # service must bind `0.0.0.0` or the bridge IP (a loopback-only bind stays - # unreachable — the bridge→127.0.0.0/8 DROP rule above is unchanged). This + # unreachable — the bridge→127.0.0.0/8 DROP rule in hive-egress.nix still applies). This # is firewall-only by design: a host service that binds `0.0.0.0` already # serves the bridge IP, so an extra bridge-IP proxy would only collide # (EADDRINUSE) with it. Merges with the [ 80 443 ] gateway ports above. diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index d99763ec1..612b5f5a4 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -63,6 +63,13 @@ let inherit (shared) forgeHere; + # A host that turns `route_localnet` on, for the assertion in + # ../host-modules/hive-egress.nix: the kernel then routes a bridge packet + # for 127.0.0.0/8 instead of dropping it, so the config is refused. + routeLocalnetOn = hiveWith [ + { boot.kernel.sysctl."net.ipv4.conf.all.route_localnet" = 1; } + ] { network.enable = true; }; + # Every hive defaults the agents' queue address to the queue's name, so the # only hive without one is a hive told to have none. Also the host still # setting the four removed host options; no case on its queue reads them. @@ -1497,6 +1504,33 @@ let && bare.services.dnsmasq.enable && bare.networking.bridges ? hive-br0; } + { + # The bridge→loopback drop is a unit of its own that loads its own nft + # table, so it does not depend on which firewall backend the host runs. + # `bare` is the presence arm; the firewall option the old rule used + # stays empty, so the rule has one home. + name = "the bridge to loopback drop is its own unit where the bridge is up and absent elsewhere"; + ok = + let + unit = bare.systemd.services.hive-egress or null; + in + unit != null + && unit.serviceConfig.Type == "oneshot" + && lib.hasInfix "hive-egress.nft" unit.serviceConfig.ExecStart + && lib.hasInfix "delete table inet hive_egress" unit.serviceConfig.ExecStop + && bare.networking.firewall.extraInputRules == "" + && !(centralToggleOff.systemd.services ? hive-egress); + } + { + # The host that turns `route_localnet` on is refused by an assertion + # naming it; the ordinary host, which leaves it alone, is not. + name = "route_localnet on the sysctl is refused on a host with the bridge up"; + ok = + let + refused = m: lib.any (a: !a.assertion && lib.hasInfix "route_localnet" a.message) m.assertions; + in + refused routeLocalnetOn && !(refused bare); + } { # Refused where a hive runs and nowhere else. A name is an identifier # only for a hive; the hive host is the control that the lookup finds the diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index 5c23725a8..db9deafd0 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -8,6 +8,9 @@ # `proxy` imports the squid module against stub options for the few hive # options it reads, so no hive-c0re build is pulled in; the module-eval suite # covers the real option wiring. +# +# It also covers ../host-modules/hive-egress.nix: `proxy` loads the real module, so the +# agent's dials to the host's loopback run against the table the module installs. { pkgs }: let bridgeIp = "192.168.1.1"; @@ -37,6 +40,10 @@ let type = lib.types.bool; default = true; }; + bridgeName = lib.mkOption { + type = lib.types.str; + default = "eth1"; + }; }; otel = { scrapeTargets = lib.mkOption { @@ -114,9 +121,12 @@ pkgs.testers.runNixOSTest { imports = [ hiveStubs ../host-modules/hive-squid.nix + ../host-modules/hive-egress.nix # Listeners a leaked destination would reach: loopback (`::` lands on `::1`) and the bridge. (listen443 [ "127.0.0.1" + # An address the agent can route to the proxy: the agent's own lo owns 127.0.0.1 itself. + "127.0.0.5" "[::1]" bridgeIp ]) @@ -174,7 +184,10 @@ pkgs.testers.runNixOSTest { # The gateway: nginx answers on the bridge, which agents reach directly. networking.firewall.interfaces.eth1.allowedTCPPorts = [ 443 ]; # Renders the users file the way hive-c0re does. - environment.systemPackages = [ pkgs.mkpasswd ]; + environment.systemPackages = [ + pkgs.mkpasswd + pkgs.nftables + ]; services.hyperhive.deploy.hive-controller.squid.selfcheck.unrestrictedCanary = "other.t.local"; services.hyperhive.deploy.hive-controller.squid.selfcheck.canary = "cache.t.local"; services.hyperhive.deploy.hive-controller.squid.baseAllowedDomains = [ @@ -684,5 +697,43 @@ pkgs.testers.runNixOSTest { write_users(ATLAS, UMA, SELFCHECK, SELFCHECK_U) assert connect("forge.t.local", ATLAS) == "200", "the file appeared" assert connect("other.t.local", UMA) == "200", "listed, and now logged in" + + def dial(addr): + return agent.execute(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/443'")[0] == 0 + + def dropped(): + # Packets the table's v4 rule has dropped so far. + out = proxy.succeed("nft list chain inet hive_egress input") + found = re.search(r"ip daddr 127\.0\.0\.0/8 counter packets (\d+)", out) + assert found, out + return int(found.group(1)) + + with checked("the agent bridge cannot reach the host's loopback, and the table is what stops it"): + assert dial("${bridgeIp}"), "control: a listener on the bridge IP answers" + # The agent has a route for an address the proxy owns on lo, and both + # ends accept loopback addresses on the link, so only the table is left + # between the dial and the listener. + for vm, dev in ((agent, "eth1"), (proxy, "eth1")): + vm.succeed( + "sysctl -w net.ipv4.conf.all.route_localnet=1 net.ipv4.conf.all.rp_filter=0 " + f"net.ipv4.conf.{dev}.route_localnet=1 net.ipv4.conf.{dev}.rp_filter=0" + ) + agent.succeed("ip route add 127.0.0.5/32 via ${bridgeIp} table local") + proxy.succeed("nft delete table inet hive_egress") + assert dial("127.0.0.5"), "control: with the table gone the loopback listener answers" + proxy.succeed("systemctl reload hive-egress.service") + before = dropped() + assert not dial("127.0.0.5"), "the table drops the dial" + assert dropped() > before, "the drop rule counted it" + assert dial("${bridgeIp}"), "the bridge IP still answers" + # A firewall reload rewrites the NixOS chains and leaves this table alone. + proxy.succeed("systemctl reload firewall.service") + assert not dial("127.0.0.5"), "still dropped after a firewall reload" + assert dial("${bridgeIp}"), "the bridge IP still answers after the reload" + # With the kernel default back, the dial is refused with no table either. + proxy.succeed("sysctl -w net.ipv4.conf.all.route_localnet=0 net.ipv4.conf.eth1.route_localnet=0") + proxy.succeed("nft delete table inet hive_egress") + assert not dial("127.0.0.5"), "route_localnet=0 refuses it alone" + proxy.succeed("systemctl reload hive-egress.service") ''; }