From a3796890f5e39b445b1ac009503796292f1de4fb Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 13 Jul 2026 10:31:44 +0200 Subject: [PATCH] feat(#2363): DHCP pool for bridge service containers Add a DHCP pool to the gateway's dnsmasq so bridge-attached service containers (hive-ci and future equivalents) get their addresses from a proper DHCP server instead of a brittle static derivation. gateway (hive-gateway.nix): - Add IPv4 arithmetic helpers (ipToInt, intToIp, pow2) to compute the DHCP pool range at nix eval time from bridgeIp + bridgePrefixLength. - Reserve the last dhcpPoolSize (14) usable host addresses as the DHCP pool (e.g. .241-.254 on a /24 with 10.42.0.0 network). - Add dhcp-range and dhcp-leasefile to the dnsmasq settings block. The pool is active whenever services.hyperhive.network.enable is true. hive-ci (hive-ci.nix): - Remove the ciBridgeIp / ciBridgeOctets static derivation and the brittle top-of-/24 comment block. - Switch networking.interfaces.eth0 to useDHCP = true so hive-ci gets its address from the gateway DHCP pool. lifecycle (mod.rs, tests.rs): - Add DHCP_POOL_SIZE = 14 constant (must stay in sync with dhcpPoolSize in hive-gateway.nix). - Remap agents whose FNV-1a hash falls in the DHCP pool into the agent-only window [2, dhcp_start - 1]. Only the rare agent whose name hashes into the pool is affected; all others keep their IPs. - Update and extend tests: agent range is now .2-.240 on /24; add agent_network_ip_never_in_dhcp_pool covering 18 agent names. --- hive-c0re/src/lifecycle/mod.rs | 32 ++++++++++++++++++++-- hive-c0re/src/lifecycle/tests.rs | 47 ++++++++++++++++++++++++++++---- nix/modules/hive-ci.nix | 34 ++++------------------- nix/modules/hive-gateway.nix | 37 +++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 36 deletions(-) diff --git a/hive-c0re/src/lifecycle/mod.rs b/hive-c0re/src/lifecycle/mod.rs index ff0b9946..45f4d641 100644 --- a/hive-c0re/src/lifecycle/mod.rs +++ b/hive-c0re/src/lifecycle/mod.rs @@ -90,6 +90,12 @@ pub fn agent_web_port(name: &str) -> u16 { WEB_PORT_BASE + u16::try_from(fnv1a(name) % u32::from(WEB_PORT_RANGE)).unwrap_or(0) } +/// Number of IP addresses at the top of each subnet reserved for the DHCP +/// pool (bridge-attached service containers such as hive-ci). Agents are +/// excluded from this range by the remap in `agent_network_ip`. Must stay +/// in sync with `dhcpPoolSize` in `nix/modules/hive-gateway.nix`. +const DHCP_POOL_SIZE: u32 = 14; + /// Deterministic IPv4 address for an agent inside an isolated subnet. /// /// Parses `subnet_cidr` as `/` (e.g. @@ -97,11 +103,18 @@ pub fn agent_web_port(name: &str) -> u16 { /// /// ```text /// host_count = 2^(32 - prefix_len) -/// usable = host_count - 3 // skip .0 (network), .1 (gateway), .255 (broadcast) +/// usable = host_count - 3 // skip .0 (network), .1 (gateway), last (broadcast) /// offset = FNV-1a(name) % usable + 2 // .2 is the first agent slot /// agent_ip = network_base_u32 + offset /// ``` /// +/// The last `DHCP_POOL_SIZE` usable host addresses are reserved for the +/// bridge DHCP pool (service containers). When the primary hash falls in +/// that range the offset is remapped into the agent-only window `[2, +/// dhcp_start - 1]` so no agent is ever assigned a DHCP-pool address. +/// Only the rare agent whose name hashes into the pool is affected; all +/// other agents keep their original IPs. +/// /// Returns `None` when `subnet_cidr` can't be parsed (invalid format, /// prefix out of range, etc.) so callers can fall back gracefully. /// Collisions are possible (birthday paradox) and the operator resolves @@ -138,7 +151,22 @@ pub fn agent_network_ip(name: &str, subnet_cidr: &str) -> Option { if usable == 0 { return None; } - let offset = fnv1a(name) % usable + 2; // +2: skip .0 and .1 + let mut offset = fnv1a(name) % usable + 2; // +2: skip .0 and .1 + + // Remap agents whose primary hash falls in the DHCP pool (the last + // DHCP_POOL_SIZE usable addresses). Only has an effect when the + // subnet is large enough to hold both agent slots AND a pool. + if usable > DHCP_POOL_SIZE { + // First offset that belongs to the DHCP pool. + let dhcp_start = usable + 2 - DHCP_POOL_SIZE; + if offset >= dhcp_start { + // Secondary hash into the agent-only window [2, dhcp_start - 1]. + // Result is guaranteed < dhcp_start (no overlap with pool). + let agent_only = dhcp_start - 2; + offset = fnv1a(name) % agent_only + 2; + } + } + let ip_u32 = network_base + offset; let [a, b, c, d] = ip_u32.to_be_bytes(); Some(format!("{a}.{b}.{c}.{d}")) diff --git a/hive-c0re/src/lifecycle/tests.rs b/hive-c0re/src/lifecycle/tests.rs index af77f5bd..1da16580 100644 --- a/hive-c0re/src/lifecycle/tests.rs +++ b/hive-c0re/src/lifecycle/tests.rs @@ -39,17 +39,54 @@ async fn setup_proposed_seeds_flake_nix() { #[test] fn agent_network_ip_is_in_subnet() { - // Default subnet 10.42.0.0/24 — agents get .2 to .254. + // Default subnet 10.42.0.0/24 — agents get .2 to .240 (last 14 are DHCP pool). let ip = agent_network_ip("alice", "10.42.0.0/24").expect("should produce an IP"); let octets: Vec = ip.split('.').map(|o| o.parse().unwrap()).collect(); assert_eq!(&octets[..3], &[10, 42, 0], "wrong /24 prefix"); + // usable=253, dhcp_start=241 → agent range is .2-.240 assert!( - octets[3] >= 2 && octets[3] <= 254, - "host byte {}", + octets[3] >= 2 && octets[3] <= 240, + "host byte {} — expected in agent-only range [2,240]", octets[3] ); } +#[test] +fn agent_network_ip_never_in_dhcp_pool() { + // No agent should be assigned an address in the DHCP pool + // (.241-.254 on a /24 with DHCP_POOL_SIZE=14). + let subnet = "10.42.0.0/24"; + let names = [ + "alice", + "bob", + "carol", + "damocles", + "iris", + "argus", + "atlas", + "ruth", + "dmatrix", + "bitburner", + "lexis", + "sock", + "triage", + "janet", + "eve", + "frank", + "grace", + "heidi", + ]; + for name in names { + let ip = agent_network_ip(name, subnet).unwrap_or_else(|| panic!("{name} returned None")); + let last: u8 = ip.rsplit('.').next().unwrap().parse().unwrap(); + // dhcp_start_offset = usable(253) + 2 - DHCP_POOL_SIZE(14) = 241 + assert!( + last < 241, + "{name} got .{last} — inside the DHCP pool [.241-.254]" + ); + } +} + #[test] fn agent_network_ip_stable() { // Same name + subnet must always produce the same IP. @@ -121,10 +158,10 @@ fn agent_network_ip_normalizes_bridge_ip_subnet() { from_bridge, from_canonical, "bridge-IP and canonical-network form should normalize to the same result" ); - // Result must still be in .2-.254. + // Result must be in .2-.240 (DHCP pool .241-.254 is excluded). let ip = from_bridge.unwrap(); let last: u8 = ip.rsplit('.').next().unwrap().parse().unwrap(); - assert!((2..=254).contains(&last), "host byte {last}"); + assert!((2..=240).contains(&last), "host byte {last}"); } /// `setup_proposed` is idempotent: calling it on an existing repo is a diff --git a/nix/modules/hive-ci.nix b/nix/modules/hive-ci.nix index 986d468b..f9805c8a 100644 --- a/nix/modules/hive-ci.nix +++ b/nix/modules/hive-ci.nix @@ -11,19 +11,6 @@ let networkCfg = config.services.hyperhive.network; tlsCfg = config.services.hyperhive.tls; - # Static bridge address for the hive-ci container. The hive bridge has - # NO DHCP server: agent containers get deterministic static IPs - # (`lifecycle::agent_network_ip` hashes each name across .2..broadcast-1) - # and dnsmasq on the bridge is DNS-only. hive-ci is the one service - # container on the bridge, so it needs a static address too — `useDHCP` - # here only hangs the boot waiting for a lease nothing serves. Reserve the - # top host address of the (default /24) subnet; a clash with an agent that - # happens to hash here is the same rename-to-resolve case as any - # agent/agent IP collision. Operators on a non-/24 bridge (or with - # `bridgeIp` set to the top address) should pick a free host address. - ciBridgeOctets = lib.splitString "." networkCfg.bridgeIp; - ciBridgeIp = "${lib.elemAt ciBridgeOctets 0}.${lib.elemAt ciBridgeOctets 1}.${lib.elemAt ciBridgeOctets 2}.254"; - # Self-signed TLS is the gateway default (no operator cert / ACME). When # active, forgejo's ROOT_URL is `https://forge.` and the leaf is # signed by the host hive CA — so the runner's Node-based actions (e.g. @@ -422,22 +409,11 @@ in # registries, etc.). The bridge→loopback DROP rule does not # affect traffic destined for the bridge IP itself. networking.nameservers = [ networkCfg.bridgeIp ]; - # With privateNetwork=true + hostBridge the container's veth - # (eth0) is bridge-attached. There is no DHCP server on the hive - # bridge (dnsmasq is DNS-only; agents use static IPs), so assign a - # static address + default route via the bridge gateway rather than - # DHCP — `useDHCP` here just hangs boot on a lease that never - # arrives. See `ciBridgeIp` above. - networking.interfaces.eth0.ipv4.addresses = [ - { - address = ciBridgeIp; - prefixLength = networkCfg.bridgePrefixLength; - } - ]; - networking.defaultGateway = { - address = networkCfg.bridgeIp; - interface = "eth0"; - }; + # Bridge-attached via privateNetwork=true + hostBridge. The + # gateway's dnsmasq now serves a DHCP pool for service containers + # (see dhcp-range in hive-gateway.nix). hive-ci gets its address + # from that pool; no static address needed. + networking.interfaces.eth0.useDHCP = true; # nspawn containers can't create user-namespaces, so nix # sandboxing always fails. Fall back to unsandboxed builds. diff --git a/nix/modules/hive-gateway.nix b/nix/modules/hive-gateway.nix index f459d9f4..2e22cf02 100644 --- a/nix/modules/hive-gateway.nix +++ b/nix/modules/hive-gateway.nix @@ -11,6 +11,37 @@ let forgeCfg = config.services.hyperhive.forge; networkCfg = config.services.hyperhive.network; + # DHCP pool for bridge-attached service containers (hive-ci, etc.). + # Occupies the last dhcpPoolSize usable addresses of the subnet + # (e.g. .241-.254 on a /24). Agent containers use deterministic static + # IPs (lifecycle::agent_network_ip) and are excluded from this range + # by a remap in the Rust code. Must stay in sync with DHCP_POOL_SIZE + # in hive-c0re/src/lifecycle/mod.rs. + dhcpPoolSize = 14; + # IPv4 helpers — nix integers are 64-bit so all /0-/32 values are safe. + ipToInt = + ip: + builtins.foldl' (acc: x: acc * 256 + x) 0 ( + map lib.strings.toIntBase10 (lib.strings.splitString "." ip) + ); + intToIp = + n: + let + a = n / 16777216; + b = (n - a * 16777216) / 65536; + c = (n - a * 16777216 - b * 65536) / 256; + d = n - a * 16777216 - b * 65536 - c * 256; + in + "${toString a}.${toString b}.${toString c}.${toString d}"; + # 2^n via recursion (nix has no pow builtin). + pow2 = n: if n == 0 then 1 else 2 * (pow2 (n - 1)); + hostCount = pow2 (32 - networkCfg.bridgePrefixLength); + # Mask off host bits to get the network base address. + networkBase = builtins.bitAnd (ipToInt networkCfg.bridgeIp) (4294967295 - hostCount + 1); + # DHCP range: last dhcpPoolSize usable host addresses (broadcast - 1 down). + dhcpEnd = intToIp (networkBase + hostCount - 2); # last usable = broadcast - 1 + dhcpStart = intToIp (networkBase + hostCount - 1 - dhcpPoolSize); # dhcpEnd - poolSize + 1 + # Dashboard SPA dist, static-served by nginx below. Read in OUTER scope so # `config` is the host's (inside the container block it'd be the container's). dashboardDist = "${config.services.hyperhive.c0re.servedFrontend}/dashboard"; @@ -999,6 +1030,12 @@ in ++ lib.optional ( matrixCfg.enable && matrixCfg.gatewayHost != null ) "/${matrixCfg.gatewayHost}/${networkCfg.bridgeIp}"; + # DHCP pool for bridge-attached service containers (hive-ci, etc.). + # Range is computed from the bridgeIp/bridgePrefixLength at eval + # time; the last dhcpPoolSize usable host addresses are reserved. + # Agent containers are excluded by agent_network_ip's DHCP remap. + dhcp-range = "${dhcpStart},${dhcpEnd},1h"; + dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases"; }; }; };