diff --git a/docs/network.md b/docs/network.md index 4e67b470..e7bb1f87 100644 --- a/docs/network.md +++ b/docs/network.md @@ -95,6 +95,47 @@ agent containers. interface only. Other interfaces stay closed. The hive resolver isn't an external-facing service. +## Container isolation + +`services.hyperhive.network.isolateContainers` (default `false`) flips +agent containers from shared host netns to private netns. Set only after +`enable = true` is stable in production — an assertion blocks the reverse. + +### What the nix side does when `isolateContainers = true` + +| effect | mechanism | +|---|---| +| IP forwarding | `boot.kernel.sysctl."net.ipv4.ip_forward" = 1` | +| Internet NAT | `networking.nat { enable = true; internalInterfaces = [ bridgeName ]; }` — MASQUERADE on packets leaving via any external NIC | +| Loopback DROP | `networking.firewall.extraInputRules` — drops bridge-subnet → `127.0.0.0/8` traffic; defence-in-depth against routing table leaks | +| c0re signal | `HIVE_NETWORK_ISOLATION=1`, `HIVE_NETWORK_BRIDGE`, `HIVE_NETWORK_SUBNET` in `systemd.services.hive-c0re.environment` | + +`HIVE_NETWORK_SUBNET` is the host-side bridge IP + prefix (e.g. +`10.42.0.1/24`), **not** the canonical network address. The Rust side +must normalise (bitwise-AND with mask) before subnet membership checks or +address arithmetic. + +### What the Rust side does + +`hive-c0re` reads `HIVE_NETWORK_ISOLATION` and, when set, passes +`PRIVATE_NETWORK=1`, `LOCAL_ADDRESS=`, and +`HOST_BRIDGE=` via `lifecycle::set_nspawn_flags` when +creating or updating containers. Each agent gets a deterministic IP +derived from its name so the address is reproducible across destroy/recreate. + +### Prerequisites before flipping on + +- All agents must have `hyperhive.web.useUnixSocket = true`. Agents that + still bind TCP on `0.0.0.0:` will be reachable at their bridge IP + from other agents on the same subnet — defeating the isolation goal. The + gateway routes via unix sockets so gateway reach is unaffected. + +### Migration behaviour + +Containers are destroyed and re-created when the flag flips. Agent state +under `/agents//state/` is bind-mounted and survives; the container +rootfs is recreated cleanly from the nix store. + ## Cross-references - `docs/gateway.md` — vhost map + the gateway container's other duties diff --git a/nix/modules/hive-network.nix b/nix/modules/hive-network.nix index e304230b..66725bc6 100644 --- a/nix/modules/hive-network.nix +++ b/nix/modules/hive-network.nix @@ -15,18 +15,22 @@ in options.services.hyperhive.network = { enable = lib.mkOption { type = lib.types.bool; - default = false; - example = true; + default = config.services.hyperhive.enable; + defaultText = lib.literalExpression "config.services.hyperhive.enable"; + example = false; description = '' Stand up the hive-internal bridge + dnsmasq resolver. - Off by default while v1 phases in. When enabled: - a bridge interface (`bridgeName`) appears on the host with - `bridgeIp` assigned, and the hive-gateway container runs a - dnsmasq listening on that IP for `` + + Defaults to `config.services.hyperhive.enable` so it comes + on automatically with the rest of hyperhive. Requires + `services.hyperhive.domain` to be set — the dnsmasq resolver + is authoritative for `` and its sub-domains. + When enabled: a bridge interface (`bridgeName`) appears on the + host with `bridgeIp` assigned, and the hive-gateway container + runs a dnsmasq listening on that IP for `` + sub-domains. Agent containers still default to shared host - netns at v1 — the endpoint is up but only used once #14 - lands and flips containers to a private netns + veth peer - on this bridge. + netns — the endpoint is up but only used once + `isolateContainers = true` flips containers to private netns + + veth peers on this bridge. ''; }; @@ -87,52 +91,138 @@ in regardless of upstream choice. ''; }; - }; - config = lib.mkIf cfg.enable { - assertions = [ - { - assertion = config.services.hyperhive.domain != null; - message = '' - services.hyperhive.network.enable = true requires - services.hyperhive.domain to be set — the resolver needs a - domain to be authoritative for. Either pin a hostname - (`services.hyperhive.domain = "example.com";`) or leave - `network.enable` at its default of false. - ''; - } - { - assertion = config.services.hyperhive.gateway.enable; - message = '' - services.hyperhive.network.enable = true requires - services.hyperhive.gateway.enable = true — the dnsmasq - resolver runs inside the hive-gateway container (single - front-door for both DNS and HTTP). Enable the gateway or - leave `network.enable` at its default of false. - ''; - } - ]; + isolateContainers = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = '' + Flip agent containers from shared host netns to private netns + (#14). When true, each agent container gets a dedicated veth + pair attached to `bridgeName` and a deterministic IP from + the bridge subnet. The bridge (already up when `enable = true`) + becomes the sole routed path between the host and agent + containers. - # Bridge interface on the host. Empty interfaces list = purely - # virtual bridge (no slave NICs attached). Per-agent veth pairs - # will join this bridge once #14 lands; at v1 it stands alone. - networking.bridges.${cfg.bridgeName}.interfaces = [ ]; + The host-side nix effect (this option) is: + - Sets `HIVE_NETWORK_ISOLATION=1` in the c0re service env so + the Rust lifecycle knows to pass `--private-network` + + bridge settings when creating/updating containers. + - Enables IP forwarding + NAT so agents can reach the internet + through the host. + - Adds a firewall rule DROP'ing traffic from the bridge subnet + to the host's loopback addresses — defence-in-depth so a + compromised agent can't reach the c0re dashboard (already + bound to 127.0.0.1) or other host-loopback services even if + the routing table somehow leaks. + - Allows HTTP/HTTPS (80/443) traffic from the bridge subnet to + the host so agents can reach the gateway container (shared + host netns, proxies the operator's per-agent UI). - # Host-side IP assignment on the bridge. This is what dnsmasq - # (inside the gateway container, shared host netns) binds on. - networking.interfaces.${cfg.bridgeName}.ipv4.addresses = [ - { - address = cfg.bridgeIp; - prefixLength = cfg.bridgePrefixLength; - } - ]; + **Prerequisite**: all agents must have + `hyperhive.web.useUnixSocket = true` before enabling isolation. + Agents that still bind TCP on `0.0.0.0:` will be + reachable at their bridge IP from other agents on the same + subnet — defeating the isolation goal. The gateway routes via + unix sockets so gateway reach still works regardless. - # Open the resolver port in the host firewall for traffic from - # the bridge subnet only. Other interfaces stay closed — - # external DNS-amplification surface is not exposed. - networking.firewall.interfaces.${cfg.bridgeName} = { - allowedUDPPorts = [ 53 ]; - allowedTCPPorts = [ 53 ]; + **Migration**: containers are destroyed and re-created when + the network isolation flag flips. Operator state under + `/agents//state/` is bind-mounted and survives; the + container rootfs (nix store paths) is recreated cleanly. + + **Rust counterpart**: `hive-c0re` reads `HIVE_NETWORK_ISOLATION` + and `HIVE_NETWORK_BRIDGE` from its service env and uses them in + `lifecycle::set_nspawn_flags` to configure `PRIVATE_NETWORK`, + `LOCAL_ADDRESS`, and `HOST_BRIDGE` in each container's + `nixos-containers/.conf`. See `docs/network.md` for the + full design. + ''; }; }; + + config = lib.mkMerge [ + (lib.mkIf cfg.enable { + assertions = [ + { + assertion = config.services.hyperhive.domain != null; + message = '' + services.hyperhive.network.enable = true requires + services.hyperhive.domain to be set — the resolver needs a + domain to be authoritative for. Either pin a hostname + (`services.hyperhive.domain = "example.com";`) or set + `services.hyperhive.network.enable = false` explicitly. + ''; + } + { + assertion = config.services.hyperhive.gateway.enable; + message = '' + services.hyperhive.network.enable = true requires + services.hyperhive.gateway.enable = true — the dnsmasq + resolver runs inside the hive-gateway container (single + front-door for both DNS and HTTP). Enable the gateway or + set `services.hyperhive.network.enable = false` explicitly. + ''; + } + ]; + + # Virtual bridge — veth pairs attach when isolateContainers flips on. + networking.bridges.${cfg.bridgeName}.interfaces = [ ]; + + # Bridge IP — dnsmasq (in the gateway container) binds here. + networking.interfaces.${cfg.bridgeName}.ipv4.addresses = [ + { + address = cfg.bridgeIp; + prefixLength = cfg.bridgePrefixLength; + } + ]; + + # DNS only on the bridge interface — no external amplification surface. + networking.firewall.interfaces.${cfg.bridgeName} = { + allowedUDPPorts = [ 53 ]; + allowedTCPPorts = [ 53 ]; + }; + }) + + # Guard: fires unconditionally on isolateContainers so the assertion + # is not silently swallowed when enable=false. + (lib.mkIf cfg.isolateContainers { + assertions = [ + { + assertion = cfg.enable; + message = '' + services.hyperhive.network.isolateContainers = true requires + services.hyperhive.network.enable = true (the bridge and + resolver must be running before isolation is flipped on). + ''; + } + ]; + }) + + # Container isolation overlay — see docs/network.md#container-isolation. + (lib.mkIf (cfg.enable && cfg.isolateContainers) { + + # Agents route internet traffic via the bridge; NAT masquerades their RFC-1918 IPs. + boot.kernel.sysctl."net.ipv4.ip_forward" = 1; + networking.nat = { + enable = true; + internalInterfaces = [ cfg.bridgeName ]; + }; + + # Defence-in-depth: DROP bridge→loopback so compromised agents can't + # reach host-loopback services even via routing table leaks. + networking.firewall.extraInputRules = '' + ip saddr ${cfg.bridgeIp}/${toString cfg.bridgePrefixLength} ip daddr 127.0.0.0/8 drop + ''; + + # Tells hive-c0re to pass PRIVATE_NETWORK + bridge settings to each + # container. HIVE_NETWORK_SUBNET is host-bridge IP/prefix, not canonical + # network address — the Rust side normalises before subnet arithmetic. + systemd.services.hive-c0re.environment = { + HIVE_NETWORK_ISOLATION = "1"; + HIVE_NETWORK_BRIDGE = cfg.bridgeName; + HIVE_NETWORK_SUBNET = "${cfg.bridgeIp}/${toString cfg.bridgePrefixLength}"; + }; + }) + ]; }