diff --git a/docs/network.md b/docs/network.md index e7bb1f87..4e67b470 100644 --- a/docs/network.md +++ b/docs/network.md @@ -95,47 +95,6 @@ agent containers. interface only. Other interfaces stay closed. The hive resolver isn't an external-facing service. -## Container isolation - -`services.hyperhive.network.isolateContainers` (default `false`) flips -agent containers from shared host netns to private netns. Set only after -`enable = true` is stable in production — an assertion blocks the reverse. - -### What the nix side does when `isolateContainers = true` - -| effect | mechanism | -|---|---| -| IP forwarding | `boot.kernel.sysctl."net.ipv4.ip_forward" = 1` | -| Internet NAT | `networking.nat { enable = true; internalInterfaces = [ bridgeName ]; }` — MASQUERADE on packets leaving via any external NIC | -| Loopback DROP | `networking.firewall.extraInputRules` — drops bridge-subnet → `127.0.0.0/8` traffic; defence-in-depth against routing table leaks | -| c0re signal | `HIVE_NETWORK_ISOLATION=1`, `HIVE_NETWORK_BRIDGE`, `HIVE_NETWORK_SUBNET` in `systemd.services.hive-c0re.environment` | - -`HIVE_NETWORK_SUBNET` is the host-side bridge IP + prefix (e.g. -`10.42.0.1/24`), **not** the canonical network address. The Rust side -must normalise (bitwise-AND with mask) before subnet membership checks or -address arithmetic. - -### What the Rust side does - -`hive-c0re` reads `HIVE_NETWORK_ISOLATION` and, when set, passes -`PRIVATE_NETWORK=1`, `LOCAL_ADDRESS=`, and -`HOST_BRIDGE=` via `lifecycle::set_nspawn_flags` when -creating or updating containers. Each agent gets a deterministic IP -derived from its name so the address is reproducible across destroy/recreate. - -### Prerequisites before flipping on - -- All agents must have `hyperhive.web.useUnixSocket = true`. Agents that - still bind TCP on `0.0.0.0:` will be reachable at their bridge IP - from other agents on the same subnet — defeating the isolation goal. The - gateway routes via unix sockets so gateway reach is unaffected. - -### Migration behaviour - -Containers are destroyed and re-created when the flag flips. Agent state -under `/agents//state/` is bind-mounted and survives; the container -rootfs is recreated cleanly from the nix store. - ## Cross-references - `docs/gateway.md` — vhost map + the gateway container's other duties diff --git a/nix/modules/hive-network.nix b/nix/modules/hive-network.nix index 66725bc6..e304230b 100644 --- a/nix/modules/hive-network.nix +++ b/nix/modules/hive-network.nix @@ -15,22 +15,18 @@ in options.services.hyperhive.network = { enable = lib.mkOption { type = lib.types.bool; - default = config.services.hyperhive.enable; - defaultText = lib.literalExpression "config.services.hyperhive.enable"; - example = false; + default = false; + example = true; description = '' Stand up the hive-internal bridge + dnsmasq resolver. - Defaults to `config.services.hyperhive.enable` so it comes - on automatically with the rest of hyperhive. Requires - `services.hyperhive.domain` to be set — the dnsmasq resolver - is authoritative for `` and its sub-domains. - When enabled: a bridge interface (`bridgeName`) appears on the - host with `bridgeIp` assigned, and the hive-gateway container - runs a dnsmasq listening on that IP for `` + + Off by default while v1 phases in. When enabled: + a bridge interface (`bridgeName`) appears on the host with + `bridgeIp` assigned, and the hive-gateway container runs a + dnsmasq listening on that IP for `` + sub-domains. Agent containers still default to shared host - netns — the endpoint is up but only used once - `isolateContainers = true` flips containers to private netns - + veth peers on this bridge. + netns at v1 — the endpoint is up but only used once #14 + lands and flips containers to a private netns + veth peer + on this bridge. ''; }; @@ -91,138 +87,52 @@ in regardless of upstream choice. ''; }; - - isolateContainers = lib.mkOption { - type = lib.types.bool; - default = false; - example = true; - description = '' - Flip agent containers from shared host netns to private netns - (#14). When true, each agent container gets a dedicated veth - pair attached to `bridgeName` and a deterministic IP from - the bridge subnet. The bridge (already up when `enable = true`) - becomes the sole routed path between the host and agent - containers. - - The host-side nix effect (this option) is: - - Sets `HIVE_NETWORK_ISOLATION=1` in the c0re service env so - the Rust lifecycle knows to pass `--private-network` + - bridge settings when creating/updating containers. - - Enables IP forwarding + NAT so agents can reach the internet - through the host. - - Adds a firewall rule DROP'ing traffic from the bridge subnet - to the host's loopback addresses — defence-in-depth so a - compromised agent can't reach the c0re dashboard (already - bound to 127.0.0.1) or other host-loopback services even if - the routing table somehow leaks. - - Allows HTTP/HTTPS (80/443) traffic from the bridge subnet to - the host so agents can reach the gateway container (shared - host netns, proxies the operator's per-agent UI). - - **Prerequisite**: all agents must have - `hyperhive.web.useUnixSocket = true` before enabling isolation. - Agents that still bind TCP on `0.0.0.0:` will be - reachable at their bridge IP from other agents on the same - subnet — defeating the isolation goal. The gateway routes via - unix sockets so gateway reach still works regardless. - - **Migration**: containers are destroyed and re-created when - the network isolation flag flips. Operator state under - `/agents//state/` is bind-mounted and survives; the - container rootfs (nix store paths) is recreated cleanly. - - **Rust counterpart**: `hive-c0re` reads `HIVE_NETWORK_ISOLATION` - and `HIVE_NETWORK_BRIDGE` from its service env and uses them in - `lifecycle::set_nspawn_flags` to configure `PRIVATE_NETWORK`, - `LOCAL_ADDRESS`, and `HOST_BRIDGE` in each container's - `nixos-containers/.conf`. See `docs/network.md` for the - full design. - ''; - }; }; - config = lib.mkMerge [ - (lib.mkIf cfg.enable { - assertions = [ - { - assertion = config.services.hyperhive.domain != null; - message = '' - services.hyperhive.network.enable = true requires - services.hyperhive.domain to be set — the resolver needs a - domain to be authoritative for. Either pin a hostname - (`services.hyperhive.domain = "example.com";`) or set - `services.hyperhive.network.enable = false` explicitly. - ''; - } - { - assertion = config.services.hyperhive.gateway.enable; - message = '' - services.hyperhive.network.enable = true requires - services.hyperhive.gateway.enable = true — the dnsmasq - resolver runs inside the hive-gateway container (single - front-door for both DNS and HTTP). Enable the gateway or - set `services.hyperhive.network.enable = false` explicitly. - ''; - } - ]; + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = config.services.hyperhive.domain != null; + message = '' + services.hyperhive.network.enable = true requires + services.hyperhive.domain to be set — the resolver needs a + domain to be authoritative for. Either pin a hostname + (`services.hyperhive.domain = "example.com";`) or leave + `network.enable` at its default of false. + ''; + } + { + assertion = config.services.hyperhive.gateway.enable; + message = '' + services.hyperhive.network.enable = true requires + services.hyperhive.gateway.enable = true — the dnsmasq + resolver runs inside the hive-gateway container (single + front-door for both DNS and HTTP). Enable the gateway or + leave `network.enable` at its default of false. + ''; + } + ]; - # Virtual bridge — veth pairs attach when isolateContainers flips on. - networking.bridges.${cfg.bridgeName}.interfaces = [ ]; + # Bridge interface on the host. Empty interfaces list = purely + # virtual bridge (no slave NICs attached). Per-agent veth pairs + # will join this bridge once #14 lands; at v1 it stands alone. + networking.bridges.${cfg.bridgeName}.interfaces = [ ]; - # Bridge IP — dnsmasq (in the gateway container) binds here. - networking.interfaces.${cfg.bridgeName}.ipv4.addresses = [ - { - address = cfg.bridgeIp; - prefixLength = cfg.bridgePrefixLength; - } - ]; + # Host-side IP assignment on the bridge. This is what dnsmasq + # (inside the gateway container, shared host netns) binds on. + networking.interfaces.${cfg.bridgeName}.ipv4.addresses = [ + { + address = cfg.bridgeIp; + prefixLength = cfg.bridgePrefixLength; + } + ]; - # DNS only on the bridge interface — no external amplification surface. - networking.firewall.interfaces.${cfg.bridgeName} = { - allowedUDPPorts = [ 53 ]; - allowedTCPPorts = [ 53 ]; - }; - }) - - # Guard: fires unconditionally on isolateContainers so the assertion - # is not silently swallowed when enable=false. - (lib.mkIf cfg.isolateContainers { - assertions = [ - { - assertion = cfg.enable; - message = '' - services.hyperhive.network.isolateContainers = true requires - services.hyperhive.network.enable = true (the bridge and - resolver must be running before isolation is flipped on). - ''; - } - ]; - }) - - # Container isolation overlay — see docs/network.md#container-isolation. - (lib.mkIf (cfg.enable && cfg.isolateContainers) { - - # Agents route internet traffic via the bridge; NAT masquerades their RFC-1918 IPs. - boot.kernel.sysctl."net.ipv4.ip_forward" = 1; - networking.nat = { - enable = true; - internalInterfaces = [ cfg.bridgeName ]; - }; - - # Defence-in-depth: DROP bridge→loopback so compromised agents can't - # reach host-loopback services even via routing table leaks. - networking.firewall.extraInputRules = '' - ip saddr ${cfg.bridgeIp}/${toString cfg.bridgePrefixLength} ip daddr 127.0.0.0/8 drop - ''; - - # Tells hive-c0re to pass PRIVATE_NETWORK + bridge settings to each - # container. HIVE_NETWORK_SUBNET is host-bridge IP/prefix, not canonical - # network address — the Rust side normalises before subnet arithmetic. - systemd.services.hive-c0re.environment = { - HIVE_NETWORK_ISOLATION = "1"; - HIVE_NETWORK_BRIDGE = cfg.bridgeName; - HIVE_NETWORK_SUBNET = "${cfg.bridgeIp}/${toString cfg.bridgePrefixLength}"; - }; - }) - ]; + # Open the resolver port in the host firewall for traffic from + # the bridge subnet only. Other interfaces stay closed — + # external DNS-amplification surface is not exposed. + networking.firewall.interfaces.${cfg.bridgeName} = { + allowedUDPPorts = [ 53 ]; + allowedTCPPorts = [ 53 ]; + }; + }; }