diff --git a/docs/network.md b/docs/network.md index 7a14be0d..e87e5802 100644 --- a/docs/network.md +++ b/docs/network.md @@ -121,12 +121,12 @@ address arithmetic. ### What the Rust side does `hive-c0re` reads `HIVE_NETWORK_ISOLATION` and passes -`PRIVATE_NETWORK=1`, `LOCAL_ADDRESS=` (empty), `HOST_ADDRESS=`, -and `HOST_BRIDGE=` via `lifecycle::set_nspawn_flags` when -creating or updating containers. `LOCAL_ADDRESS` is left empty so the -container's dhcpcd acquires an address from the bridge dnsmasq pool -(`networking.useDHCP = true` in `harness-base.nix`). This applies uniformly -to all containers — agents and service containers alike. +`PRIVATE_NETWORK=1`, `LOCAL_ADDRESS=`, +`HOST_ADDRESS=`, and `HOST_BRIDGE=` via +`lifecycle::set_nspawn_flags` when creating or updating containers. Each +agent gets a deterministic IP derived from its name so the address is +reproducible across destroy/recreate. This applies uniformly to all +containers — no special case. `HOST_ADDRESS` is the bridge gateway IP (the address part of `HIVE_NETWORK_SUBNET`, via `lifecycle::bridge_gateway_ip` — taken verbatim diff --git a/hive-c0re/src/lifecycle/host_config.rs b/hive-c0re/src/lifecycle/host_config.rs index 1610ce3b..73210e53 100644 --- a/hive-c0re/src/lifecycle/host_config.rs +++ b/hive-c0re/src/lifecycle/host_config.rs @@ -10,7 +10,7 @@ use hive_sh4re::priv_proto::{BindMount, CredentialMount}; use crate::coordinator::{AgentPaths, HiveEnv}; use super::{ - AGENT_PREFIX, CONTAINER_RUNTIME_MOUNT, CONTAINER_SHARED_MOUNT, agent_uid_gid, + AGENT_PREFIX, CONTAINER_RUNTIME_MOUNT, CONTAINER_SHARED_MOUNT, agent_network_ip, agent_uid_gid, bridge_gateway_ip, container_claude_mount, container_name, validate, }; @@ -294,6 +294,21 @@ async fn set_nspawn_flags( let bridge = std::env::var("HIVE_NETWORK_BRIDGE").unwrap_or_default(); let subnet = std::env::var("HIVE_NETWORK_SUBNET").unwrap_or_default(); if isolate && !bridge.is_empty() && !subnet.is_empty() { + let Some(agent_ip) = agent_network_ip(agent_name, &subnet) else { + tracing::warn!( + %agent_name, %subnet, + "HIVE_NETWORK_SUBNET is set but could not derive a valid IP for agent \ + (bad CIDR? prefix too narrow?); skipping PRIVATE_NETWORK write to \ + avoid misconfigured isolation" + ); + return crate::priv_client::write_nspawn_flags( + container, + &binds, + None, + &load_creds, + ) + .await; + }; let Some(gateway_ip) = bridge_gateway_ip(&subnet) else { tracing::warn!( %agent_name, %subnet, @@ -310,10 +325,14 @@ async fn set_nspawn_flags( .await; }; tracing::info!( - %agent_name, %gateway_ip, %bridge, - "network isolation: PRIVATE_NETWORK=1 (DHCP)" + %agent_name, %agent_ip, %gateway_ip, %bridge, + "network isolation: PRIVATE_NETWORK=1" ); - Some(hive_sh4re::priv_proto::NetworkIsolation { bridge, gateway_ip }) + Some(hive_sh4re::priv_proto::NetworkIsolation { + agent_ip, + bridge, + gateway_ip, + }) } else { None } diff --git a/hive-c0re/src/lifecycle/mod.rs b/hive-c0re/src/lifecycle/mod.rs index f9c59695..ff0b9946 100644 --- a/hive-c0re/src/lifecycle/mod.rs +++ b/hive-c0re/src/lifecycle/mod.rs @@ -66,7 +66,8 @@ pub const CONTAINER_SHARED_MOUNT: &str = "/shared"; const WEB_PORT_BASE: u16 = 8100; const WEB_PORT_RANGE: u16 = 900; -/// FNV-1a hash of a string — used by `agent_web_port`. +/// FNV-1a hash of a string — shared by `agent_web_port` and +/// `agent_network_ip` so the derivation rule is identical. fn fnv1a(s: &str) -> u32 { let mut hash: u32 = 2_166_136_261; for b in s.bytes() { @@ -89,6 +90,60 @@ pub fn agent_web_port(name: &str) -> u16 { WEB_PORT_BASE + u16::try_from(fnv1a(name) % u32::from(WEB_PORT_RANGE)).unwrap_or(0) } +/// Deterministic IPv4 address for an agent inside an isolated subnet. +/// +/// Parses `subnet_cidr` as `/` (e.g. +/// `"10.42.0.0/24"`), then computes: +/// +/// ```text +/// host_count = 2^(32 - prefix_len) +/// usable = host_count - 3 // skip .0 (network), .1 (gateway), .255 (broadcast) +/// offset = FNV-1a(name) % usable + 2 // .2 is the first agent slot +/// agent_ip = network_base_u32 + offset +/// ``` +/// +/// Returns `None` when `subnet_cidr` can't be parsed (invalid format, +/// prefix out of range, etc.) so callers can fall back gracefully. +/// Collisions are possible (birthday paradox) and the operator resolves +/// them by renaming an agent, same as for port collisions. +#[must_use] +pub fn agent_network_ip(name: &str, subnet_cidr: &str) -> Option { + let (ip_str, prefix_str) = subnet_cidr.split_once('/')?; + let prefix_len: u32 = prefix_str.parse().ok()?; + if prefix_len > 30 { + // /31 and /32 have no room for agents; /30 has 1 usable slot. + // /0 (the other extreme) is handled further down: host_count + // overflows checked_shl(32) → 0 → usable = 0 → None. + return None; + } + // Parse dotted-decimal IPv4. + let octets: Vec = ip_str + .split('.') + .map(|o| o.parse::().ok()) + .collect::>>()?; + if octets.len() != 4 { + return None; + } + let base_u32 = u32::from_be_bytes([octets[0], octets[1], octets[2], octets[3]]); + // Mask off host bits to get the true network address. + let mask = if prefix_len == 0 { + 0u32 + } else { + !0u32 << (32 - prefix_len) + }; + let network_base = base_u32 & mask; + let host_count: u32 = 1u32.checked_shl(32 - prefix_len).unwrap_or(0); + // `.0` = network, `.1` = bridge gateway, last = broadcast → 3 reserved. + let usable = host_count.saturating_sub(3); + if usable == 0 { + return None; + } + let offset = fnv1a(name) % usable + 2; // +2: skip .0 and .1 + let ip_u32 = network_base + offset; + let [a, b, c, d] = ip_u32.to_be_bytes(); + Some(format!("{a}.{b}.{c}.{d}")) +} + /// Extract the bridge gateway IP from `HIVE_NETWORK_SUBNET`. /// /// `HIVE_NETWORK_SUBNET` carries the host-side bridge address verbatim @@ -108,8 +163,9 @@ pub fn agent_web_port(name: &str) -> u16 { pub fn bridge_gateway_ip(subnet_cidr: &str) -> Option { let (ip_str, prefix_str) = subnet_cidr.split_once('/')?; // Validate the prefix is a sane IPv4 CIDR length and the address is - // dotted-decimal IPv4 so a malformed `HIVE_NETWORK_SUBNET` can't - // smuggle a bogus HOST_ADDRESS into the nspawn conf. + // dotted-decimal IPv4 — same shape `agent_network_ip` accepts — so a + // malformed `HIVE_NETWORK_SUBNET` can't smuggle a bogus HOST_ADDRESS + // into the nspawn conf. let prefix_len: u32 = prefix_str.parse().ok()?; if prefix_len > 32 { return None; diff --git a/hive-c0re/src/lifecycle/tests.rs b/hive-c0re/src/lifecycle/tests.rs index 761ae999..af77f5bd 100644 --- a/hive-c0re/src/lifecycle/tests.rs +++ b/hive-c0re/src/lifecycle/tests.rs @@ -37,6 +37,43 @@ async fn setup_proposed_seeds_flake_nix() { assert!(tracked.contains("flake.nix"), "flake.nix not committed"); } +#[test] +fn agent_network_ip_is_in_subnet() { + // Default subnet 10.42.0.0/24 — agents get .2 to .254. + let ip = agent_network_ip("alice", "10.42.0.0/24").expect("should produce an IP"); + let octets: Vec = ip.split('.').map(|o| o.parse().unwrap()).collect(); + assert_eq!(&octets[..3], &[10, 42, 0], "wrong /24 prefix"); + assert!( + octets[3] >= 2 && octets[3] <= 254, + "host byte {}", + octets[3] + ); +} + +#[test] +fn agent_network_ip_stable() { + // Same name + subnet must always produce the same IP. + let a = agent_network_ip("damocles", "10.42.0.0/24"); + let b = agent_network_ip("damocles", "10.42.0.0/24"); + assert_eq!(a, b); +} + +#[test] +fn agent_network_ip_different_agents() { + // Different agent names very likely produce different IPs (not guaranteed, + // but for these two names the hashes don't collide). + let alice = agent_network_ip("alice", "10.42.0.0/24").unwrap(); + let bob = agent_network_ip("bob", "10.42.0.0/24").unwrap(); + assert_ne!(alice, bob, "alice and bob collide — rename one"); +} + +#[test] +fn agent_network_ip_different_subnet() { + let ip = agent_network_ip("alice", "192.168.5.0/24").expect("should produce an IP"); + let octets: Vec = ip.split('.').map(|o| o.parse().unwrap()).collect(); + assert_eq!(&octets[..3], &[192, 168, 5]); +} + #[test] fn bridge_gateway_ip_extracts_verbatim_address() { // HIVE_NETWORK_SUBNET carries the bridge IP verbatim, not the @@ -65,6 +102,31 @@ fn bridge_gateway_ip_rejects_bad_input() { assert!(bridge_gateway_ip("10.42.0/24").is_none()); // 3 octets } +#[test] +fn agent_network_ip_rejects_bad_input() { + assert!(agent_network_ip("alice", "notanip/24").is_none()); + assert!(agent_network_ip("alice", "10.0.0.0/33").is_none()); // prefix > 32 + assert!(agent_network_ip("alice", "10.0.0.0/31").is_none()); // too small + assert!(agent_network_ip("alice", "10.0.0.0").is_none()); // no prefix +} + +#[test] +fn agent_network_ip_normalizes_bridge_ip_subnet() { + // HIVE_NETWORK_SUBNET carries the bridge IP (10.42.0.1/24), not + // canonical network (10.42.0.0/24). Both must produce the same result + // after host-bit masking. + let from_bridge = agent_network_ip("alice", "10.42.0.1/24"); + let from_canonical = agent_network_ip("alice", "10.42.0.0/24"); + assert_eq!( + from_bridge, from_canonical, + "bridge-IP and canonical-network form should normalize to the same result" + ); + // Result must still be in .2-.254. + let ip = from_bridge.unwrap(); + let last: u8 = ip.rsplit('.').next().unwrap().parse().unwrap(); + assert!((2..=254).contains(&last), "host byte {last}"); +} + /// `setup_proposed` is idempotent: calling it on an existing repo is a /// no-op (the fresh guard skips all writes). #[tokio::test] diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index 991ee147..2977f8e9 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -1487,11 +1487,7 @@ fn write_nspawn_flags( // setup is skipped, so this only affects the container's route — // exactly what we want. let _ = writeln!(out, "HOST_ADDRESS={}", iso.gateway_ip); - // LOCAL_ADDRESS is intentionally empty: agent containers receive their - // IP dynamically via DHCP from the bridge dnsmasq pool. HOST_ADDRESS - // (the gateway IP) is still written so nixos-container's container-side - // init installs a default route before the DHCP lease arrives. - out.push_str("LOCAL_ADDRESS=\n"); + let _ = writeln!(out, "LOCAL_ADDRESS={}", iso.agent_ip); out.push_str("HOST_ADDRESS6=\n"); out.push_str("LOCAL_ADDRESS6=\n"); let _ = writeln!(out, "HOST_BRIDGE={}", iso.bridge); diff --git a/hive-sh4re/src/priv_proto.rs b/hive-sh4re/src/priv_proto.rs index 50101a45..bb4361e4 100644 --- a/hive-sh4re/src/priv_proto.rs +++ b/hive-sh4re/src/priv_proto.rs @@ -208,21 +208,21 @@ pub struct CredentialMount { /// Network isolation parameters for `WriteNspawnFlags`. When `Some`, /// hive-priv writes `PRIVATE_NETWORK=1` + veth bridge wiring instead -/// of the default `PRIVATE_NETWORK=0`. Containers receive their IP -/// dynamically via DHCP from the bridge dnsmasq pool (`networking.useDHCP` -/// in `harness-base.nix`); no static address is pre-assigned here. +/// of the default `PRIVATE_NETWORK=0`. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct NetworkIsolation { + /// Static IP address to assign to this container on the bridge subnet. + pub agent_ip: String, /// Host bridge interface name (e.g. `hive0`). pub bridge: String, /// Bridge gateway IP (the host-side bridge address, e.g. `10.42.0.1`). /// Written as `HOST_ADDRESS=` in the nspawn conf so nixos-container's - /// container-side setup installs a default route (`default via `) - /// before DHCP completes: without it the container has no route off - /// the bridge subnet until the DHCP lease arrives. The same IP runs the - /// hive dnsmasq resolver, so it's also written into the container's - /// `/etc/resolv.conf` (see the isolated-DNS oneshot in `harness-base.nix`, - /// gated on the marker hive-priv drops). + /// container-side setup installs a default route (`default via `): + /// without it the container comes up with an address but no route off + /// the bridge subnet — no internet, no `api.anthropic.com`. The same IP + /// runs the hive dnsmasq resolver, so it's also written into the + /// container's `/etc/resolv.conf` (see the isolated-DNS oneshot in + /// `harness-base.nix`, gated on the marker hive-priv drops). pub gateway_ip: String, } diff --git a/nix/modules/hive-ci.nix b/nix/modules/hive-ci.nix index 15251720..986d468b 100644 --- a/nix/modules/hive-ci.nix +++ b/nix/modules/hive-ci.nix @@ -11,6 +11,19 @@ let networkCfg = config.services.hyperhive.network; tlsCfg = config.services.hyperhive.tls; + # Static bridge address for the hive-ci container. The hive bridge has + # NO DHCP server: agent containers get deterministic static IPs + # (`lifecycle::agent_network_ip` hashes each name across .2..broadcast-1) + # and dnsmasq on the bridge is DNS-only. hive-ci is the one service + # container on the bridge, so it needs a static address too — `useDHCP` + # here only hangs the boot waiting for a lease nothing serves. Reserve the + # top host address of the (default /24) subnet; a clash with an agent that + # happens to hash here is the same rename-to-resolve case as any + # agent/agent IP collision. Operators on a non-/24 bridge (or with + # `bridgeIp` set to the top address) should pick a free host address. + ciBridgeOctets = lib.splitString "." networkCfg.bridgeIp; + ciBridgeIp = "${lib.elemAt ciBridgeOctets 0}.${lib.elemAt ciBridgeOctets 1}.${lib.elemAt ciBridgeOctets 2}.254"; + # Self-signed TLS is the gateway default (no operator cert / ACME). When # active, forgejo's ROOT_URL is `https://forge.` and the leaf is # signed by the host hive CA — so the runner's Node-based actions (e.g. @@ -409,11 +422,22 @@ in # registries, etc.). The bridge→loopback DROP rule does not # affect traffic destined for the bridge IP itself. networking.nameservers = [ networkCfg.bridgeIp ]; - # Bridge-attached via privateNetwork=true + hostBridge. The - # gateway's dnsmasq serves a DHCP pool covering all usable bridge - # addresses (see dhcp-range in hive-gateway.nix) — agents and - # service containers alike receive IPs dynamically. - networking.interfaces.eth0.useDHCP = true; + # With privateNetwork=true + hostBridge the container's veth + # (eth0) is bridge-attached. There is no DHCP server on the hive + # bridge (dnsmasq is DNS-only; agents use static IPs), so assign a + # static address + default route via the bridge gateway rather than + # DHCP — `useDHCP` here just hangs boot on a lease that never + # arrives. See `ciBridgeIp` above. + networking.interfaces.eth0.ipv4.addresses = [ + { + address = ciBridgeIp; + prefixLength = networkCfg.bridgePrefixLength; + } + ]; + networking.defaultGateway = { + address = networkCfg.bridgeIp; + interface = "eth0"; + }; # nspawn containers can't create user-namespaces, so nix # sandboxing always fails. Fall back to unsandboxed builds. diff --git a/nix/modules/hive-gateway.nix b/nix/modules/hive-gateway.nix index b3f13f13..f459d9f4 100644 --- a/nix/modules/hive-gateway.nix +++ b/nix/modules/hive-gateway.nix @@ -11,36 +11,6 @@ let forgeCfg = config.services.hyperhive.forge; networkCfg = config.services.hyperhive.network; - # DHCP pool covering all usable host addresses on the bridge subnet. - # All containers (agents and service containers such as hive-ci) receive - # their IPs dynamically; there are no hash-derived static assignments. - # Range: .2 to .(hostCount-2) — skipping .0 (network), .1 (gateway/host - # bridge), and the broadcast address. - # - # IPv4 helpers — nix integers are 64-bit so all /0-/32 values are safe. - ipToInt = - ip: - builtins.foldl' (acc: x: acc * 256 + x) 0 ( - map lib.strings.toIntBase10 (lib.strings.splitString "." ip) - ); - intToIp = - n: - let - a = n / 16777216; - b = (n - a * 16777216) / 65536; - c = (n - a * 16777216 - b * 65536) / 256; - d = n - a * 16777216 - b * 65536 - c * 256; - in - "${toString a}.${toString b}.${toString c}.${toString d}"; - # 2^n via recursion (nix has no pow builtin). - pow2 = n: if n == 0 then 1 else 2 * (pow2 (n - 1)); - hostCount = pow2 (32 - networkCfg.bridgePrefixLength); - # Mask off host bits to get the network base address. - networkBase = builtins.bitAnd (ipToInt networkCfg.bridgeIp) (4294967295 - hostCount + 1); - # DHCP range: .2 (first usable after gateway) to .(hostCount-2) (last usable). - dhcpStart = intToIp (networkBase + 2); # skip .0 (network) and .1 (gateway) - dhcpEnd = intToIp (networkBase + hostCount - 2); # last usable = broadcast - 1 - # Dashboard SPA dist, static-served by nginx below. Read in OUTER scope so # `config` is the host's (inside the container block it'd be the container's). dashboardDist = "${config.services.hyperhive.c0re.servedFrontend}/dashboard"; @@ -1029,13 +999,6 @@ in ++ lib.optional ( matrixCfg.enable && matrixCfg.gatewayHost != null ) "/${matrixCfg.gatewayHost}/${networkCfg.bridgeIp}"; - # DHCP pool covering all usable host addresses on the bridge subnet. - # Range is computed from bridgeIp/bridgePrefixLength at eval time: - # .2 (first after gateway) to .(hostCount-2) (last usable before - # broadcast). All containers — agents and service containers alike — - # receive their IPs dynamically from this pool. - dhcp-range = "${dhcpStart},${dhcpEnd},1h"; - dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases"; }; }; }; diff --git a/nix/templates/harness-base.nix b/nix/templates/harness-base.nix index 10d4d242..84bc6045 100644 --- a/nix/templates/harness-base.nix +++ b/nix/templates/harness-base.nix @@ -1695,10 +1695,6 @@ in # disabling dhcpcd itself, so the veth still gets its address); then # the hyperhive-isolated-dns oneshot owns resolv.conf. (Same "take # resolvconf out of the loop" approach the matrix container uses.) - # All agent containers receive their bridge IP via DHCP from the hive - # dnsmasq pool (see hive-gateway.nix). useDHCP runs dhcpcd on every - # interface (just eth0 in practice — the nspawn bridge veth). - networking.useDHCP = true; networking.resolvconf.enable = false; networking.dhcpcd.extraConfig = "nohook resolv.conf";