From b89c5f5334c936ff133f3c7fb821e59eebf59d5a Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 3 Jun 2026 09:39:53 +0200 Subject: [PATCH] network: fix stale assertion messages; move prose to docs/network.md --- docs/network.md | 46 +++++++++++++++++++++ nix/modules/hive-network.nix | 78 +++++++----------------------------- 2 files changed, 61 insertions(+), 63 deletions(-) diff --git a/docs/network.md b/docs/network.md index 4e67b470..79f03ff7 100644 --- a/docs/network.md +++ b/docs/network.md @@ -95,6 +95,52 @@ agent containers. interface only. Other interfaces stay closed. The hive resolver isn't an external-facing service. +## Container isolation + +`services.hyperhive.network.isolateContainers` (default `false`) flips +agent containers from shared host netns to private netns. Set only after +`enable = true` is stable in production — an assertion blocks the reverse. + +### What the nix side does when `isolateContainers = true` + +| effect | mechanism | +|---|---| +| IP forwarding | `boot.kernel.sysctl."net.ipv4.ip_forward" = 1` | +| Internet NAT | `networking.nat { enable = true; internalInterfaces = [ bridgeName ]; }` — MASQUERADE on packets leaving via any external NIC | +| Loopback DROP | `networking.firewall.extraInputRules` — drops bridge-subnet → `127.0.0.0/8` traffic; defence-in-depth against routing table leaks | +| c0re signal | `HIVE_NETWORK_ISOLATION=1`, `HIVE_NETWORK_BRIDGE`, `HIVE_NETWORK_SUBNET` in `systemd.services.hive-c0re.environment` | + +`HIVE_NETWORK_SUBNET` is the host-side bridge IP + prefix (e.g. +`10.42.0.1/24`), **not** the canonical network address. The Rust side +must normalise (bitwise-AND with mask) before subnet membership checks or +address arithmetic. + +### What the Rust side does + +`hive-c0re` reads `HIVE_NETWORK_ISOLATION` and, when set, passes +`PRIVATE_NETWORK=1`, `LOCAL_ADDRESS=`, and +`HOST_BRIDGE=` via `lifecycle::set_nspawn_flags` when +creating or updating containers. Each agent gets a deterministic IP +derived from its name so the address is reproducible across destroy/recreate. + +### Prerequisites before flipping on + +- All agents must have `hyperhive.web.useUnixSocket = true`. Agents that + still bind TCP on `0.0.0.0:` will be reachable at their bridge IP + from other agents on the same subnet — defeating the isolation goal. The + gateway routes via unix sockets so gateway reach is unaffected. + +### Migration behaviour + +Containers are destroyed and re-created when the flag flips. Agent state +under `/agents//state/` is bind-mounted and survives; the container +rootfs is recreated cleanly from the nix store. + +### Follow-up + +Issue #1119 tracks defaulting `isolateContainers` to `true` once the +bridge is stable in production. + ## Cross-references - `docs/gateway.md` — vhost map + the gateway container's other duties diff --git a/nix/modules/hive-network.nix b/nix/modules/hive-network.nix index b64fb186..66725bc6 100644 --- a/nix/modules/hive-network.nix +++ b/nix/modules/hive-network.nix @@ -150,8 +150,8 @@ in services.hyperhive.network.enable = true requires services.hyperhive.domain to be set — the resolver needs a domain to be authoritative for. Either pin a hostname - (`services.hyperhive.domain = "example.com";`) or leave - `network.enable` at its default of false. + (`services.hyperhive.domain = "example.com";`) or set + `services.hyperhive.network.enable = false` explicitly. ''; } { @@ -161,18 +161,15 @@ in services.hyperhive.gateway.enable = true — the dnsmasq resolver runs inside the hive-gateway container (single front-door for both DNS and HTTP). Enable the gateway or - leave `network.enable` at its default of false. + set `services.hyperhive.network.enable = false` explicitly. ''; } ]; - # Bridge interface on the host. Empty interfaces list = purely - # virtual bridge (no slave NICs attached). Per-agent veth pairs - # will join this bridge once #14 lands; at v1 it stands alone. + # Virtual bridge — veth pairs attach when isolateContainers flips on. networking.bridges.${cfg.bridgeName}.interfaces = [ ]; - # Host-side IP assignment on the bridge. This is what dnsmasq - # (inside the gateway container, shared host netns) binds on. + # Bridge IP — dnsmasq (in the gateway container) binds here. networking.interfaces.${cfg.bridgeName}.ipv4.addresses = [ { address = cfg.bridgeIp; @@ -180,19 +177,15 @@ in } ]; - # Open the resolver port in the host firewall for traffic from - # the bridge subnet only. Other interfaces stay closed — - # external DNS-amplification surface is not exposed. + # DNS only on the bridge interface — no external amplification surface. networking.firewall.interfaces.${cfg.bridgeName} = { allowedUDPPorts = [ 53 ]; allowedTCPPorts = [ 53 ]; }; }) - # Guard: isolateContainers requires the bridge to be up. This arm is - # unconditional on cfg.enable so the assertion fires even when an - # operator sets isolateContainers=true but forgets enable=true (the - # combined-condition arm below would silently do nothing in that case). + # Guard: fires unconditionally on isolateContainers so the assertion + # is not silently swallowed when enable=false. (lib.mkIf cfg.isolateContainers { assertions = [ { @@ -206,66 +199,25 @@ in ]; }) - # Container network isolation (#14 v1). Ships as a separate overlay - # on top of the base bridge config (which stays unconditional) so - # operators can stand the bridge + resolver up first, validate - # everything, then flip isolation on independently. + # Container isolation overlay — see docs/network.md#container-isolation. (lib.mkIf (cfg.enable && cfg.isolateContainers) { - # IP forwarding — agents need to route through the bridge to reach - # the internet. NixOS firewall's `nat.enable` sets this too, but - # making it explicit here keeps the intent visible alongside the - # NAT rule. + # Agents route internet traffic via the bridge; NAT masquerades their RFC-1918 IPs. boot.kernel.sysctl."net.ipv4.ip_forward" = 1; - - # NAT/masquerade: translate agent bridge IPs → host's outbound - # IP for internet-bound traffic. Without masquerade, packets from - # 10.42.0.X arrive at external servers with an RFC-1918 source - # that can't be routed back. networking.nat = { enable = true; - # `internalInterfaces` causes `MASQUERADE` on packets from the - # bridge leaving via any external interface. Only traffic from - # agents crosses the bridge — hive-gateway / forge / matrix - # stay on host netns and don't need NAT. internalInterfaces = [ cfg.bridgeName ]; }; - # DROP traffic from the bridge subnet to host loopback addresses. - # Defence-in-depth: the c0re dashboard already binds 127.0.0.1 - # (not 0.0.0.0), so bridge-sourced traffic can't reach it via - # the bridge IP. But a misconfigured service that slips to - # 0.0.0.0 would otherwise be reachable. The DROP rule closes that - # window. Use `extraInputRules` (nftables `input` chain, priority - # 0, same ruleset as `allowedTCPPorts`) so it's processed before - # the accept rules for bridge-side DNS we added above. - # - # The rule fires only when an agent container has a bridge IP - # (i.e. after the Rust side also ships `PRIVATE_NETWORK=1`); - # until then all containers share host netns and no traffic - # originates from 10.42.0.0/24 so this is a dead letter. + # Defence-in-depth: DROP bridge→loopback so compromised agents can't + # reach host-loopback services even via routing table leaks. networking.firewall.extraInputRules = '' ip saddr ${cfg.bridgeIp}/${toString cfg.bridgePrefixLength} ip daddr 127.0.0.0/8 drop ''; - # Signal to the c0re Rust side that container isolation is - # enabled. c0re reads `HIVE_NETWORK_ISOLATION` from its service - # environment and uses it in `lifecycle::set_nspawn_flags` to set - # `PRIVATE_NETWORK=1`, `LOCAL_ADDRESS=`, and - # `HOST_BRIDGE=` in each agent's container config. - # Also forwards the bridge name + subnet so c0re can wire the - # veth without hardcoding. - # - # `systemd.services.hive-c0re.environment` is an attrset; NixOS - # merges contributions from all modules that set it, so this - # cross-module injection is idiomatic and doesn't require a - # dedicated option in hive-c0re.nix. - # - # Note: HIVE_NETWORK_SUBNET is the host-side bridge IP + prefix - # length (e.g. "10.42.0.1/24"), not the canonical network address - # ("10.42.0.0/24"). The Rust side must normalize (bitwise AND the - # IP with the mask) before using it for subnet membership checks or - # address arithmetic. + # Tells hive-c0re to pass PRIVATE_NETWORK + bridge settings to each + # container. HIVE_NETWORK_SUBNET is host-bridge IP/prefix, not canonical + # network address — the Rust side normalises before subnet arithmetic. systemd.services.hive-c0re.environment = { HIVE_NETWORK_ISOLATION = "1"; HIVE_NETWORK_BRIDGE = cfg.bridgeName;