vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat(#2363): full-DHCP for all agents — drop static agent_network_ip

Browse source 
All agent containers now receive their bridge IP dynamically via DHCP
from the dnsmasq pool instead of a hash-derived static address:

- nix/templates/harness-base.nix: networking.useDHCP = true
- nix/modules/hive-gateway.nix: expand DHCP pool to full usable range
  (.2 to .254 on /24) — was last-14-IPs-only
- hive-sh4re/src/priv_proto.rs: remove agent_ip from NetworkIsolation
- hive-c0re/src/lifecycle/mod.rs: drop agent_network_ip + DHCP_POOL_SIZE
- hive-c0re/src/lifecycle/host_config.rs: remove agent_network_ip call
- hive-priv/src/main.rs: LOCAL_ADDRESS= empty (DHCP assigns IP);
  HOST_ADDRESS still set so nixos-container installs default route
  before the DHCP lease arrives
- nix/dhcp-pool-size: deleted (no longer needed)

The nix/dhcp-pool-size single-source-of-truth file and all associated
Rust/Nix dual-constant plumbing are gone — there is no static map.
bridge_gateway_ip() is retained (still needed for HOST_ADDRESS).

Closes #2363
This commit is contained in:
atlas 2026-07-13 11:18:42 +02:00 • committed by mara
commit 4cdbbafc44
8 changed files with 37 additions and 244 deletions
Download patch file Download diff file Expand all files Collapse all files

27
hive-c0re/src/lifecycle/host_config.rs
View file

@ -10,7 +10,7 @@ use hive_sh4re::priv_proto::{BindMount, CredentialMount};
use crate::coordinator::{AgentPaths, HiveEnv}; use crate::coordinator::{AgentPaths, HiveEnv};
use super::{ use super::{
AGENT_PREFIX, CONTAINER_RUNTIME_MOUNT, CONTAINER_SHARED_MOUNT, agent_network_ip, agent_uid_gid, AGENT_PREFIX, CONTAINER_RUNTIME_MOUNT, CONTAINER_SHARED_MOUNT, agent_uid_gid,
bridge_gateway_ip, container_claude_mount, container_name, validate, bridge_gateway_ip, container_claude_mount, container_name, validate,
}; };
@ -294,21 +294,6 @@ async fn set_nspawn_flags(
let bridge = std::env::var("HIVE_NETWORK_BRIDGE").unwrap_or_default(); let bridge = std::env::var("HIVE_NETWORK_BRIDGE").unwrap_or_default();
let subnet = std::env::var("HIVE_NETWORK_SUBNET").unwrap_or_default(); let subnet = std::env::var("HIVE_NETWORK_SUBNET").unwrap_or_default();
if isolate && !bridge.is_empty() && !subnet.is_empty() { if isolate && !bridge.is_empty() && !subnet.is_empty() {
let Some(agent_ip) = agent_network_ip(agent_name, &subnet) else {
tracing::warn!(
%agent_name, %subnet,
"HIVE_NETWORK_SUBNET is set but could not derive a valid IP for agent \
(bad CIDR? prefix too narrow?); skipping PRIVATE_NETWORK write to \
avoid misconfigured isolation"
);
return crate::priv_client::write_nspawn_flags(
container,
&binds,
None,
&load_creds,
)
.await;
};
let Some(gateway_ip) = bridge_gateway_ip(&subnet) else { let Some(gateway_ip) = bridge_gateway_ip(&subnet) else {
tracing::warn!( tracing::warn!(
%agent_name, %subnet, %agent_name, %subnet,
@ -325,14 +310,10 @@ async fn set_nspawn_flags(
.await; .await;
}; };
tracing::info!( tracing::info!(
%agent_name, %agent_ip, %gateway_ip, %bridge, %agent_name, %gateway_ip, %bridge,
"network isolation: PRIVATE_NETWORK=1" "network isolation: PRIVATE_NETWORK=1 (DHCP)"
); );
Some(hive_sh4re::priv_proto::NetworkIsolation { Some(hive_sh4re::priv_proto::NetworkIsolation { bridge, gateway_ip })
agent_ip,
bridge,
gateway_ip,
})
} else { } else {
None None
} }

97
hive-c0re/src/lifecycle/mod.rs
View file

@ -66,8 +66,7 @@ pub const CONTAINER_SHARED_MOUNT: &str = "/shared";
const WEB_PORT_BASE: u16 = 8100; const WEB_PORT_BASE: u16 = 8100;
const WEB_PORT_RANGE: u16 = 900; const WEB_PORT_RANGE: u16 = 900;
/// FNV-1a hash of a string — shared by `agent_web_port` and /// FNV-1a hash of a string — used by `agent_web_port`.
/// `agent_network_ip` so the derivation rule is identical.
fn fnv1a(s: &str) -> u32 { fn fnv1a(s: &str) -> u32 {
let mut hash: u32 = 2_166_136_261; let mut hash: u32 = 2_166_136_261;
for b in s.bytes() { for b in s.bytes() {
@ -90,95 +89,6 @@ pub fn agent_web_port(name: &str) -> u16 {
WEB_PORT_BASE + u16::try_from(fnv1a(name) % u32::from(WEB_PORT_RANGE)).unwrap_or(0) WEB_PORT_BASE + u16::try_from(fnv1a(name) % u32::from(WEB_PORT_RANGE)).unwrap_or(0)
} }
/// Number of IP addresses at the top of each subnet reserved for the DHCP
/// pool (bridge-attached service containers such as hive-ci). Agents are
/// excluded from this range by subtracting it from the usable count before
/// hashing, so agents only ever land in [2, usable - DHCP_POOL_SIZE + 1].
///
/// Single source of truth: `nix/dhcp-pool-size` (one integer, shared with
/// `nix/modules/hive-gateway.nix` which reads the same file via
/// `builtins.readFile`). Parsed at compile time via `include_bytes!`.
const DHCP_POOL_SIZE: u32 = {
let bytes = include_bytes!("../../../nix/dhcp-pool-size");
let mut n: u32 = 0;
let mut i = 0;
while i < bytes.len() {
let b = bytes[i];
if b'0' <= b && b <= b'9' {
n = n * 10 + (b - b'0') as u32;
}
i += 1;
}
n
};
/// Deterministic IPv4 address for an agent inside an isolated subnet.
///
/// Parses `subnet_cidr` as `<network_ip>/<prefix_len>` (e.g.
/// `"10.42.0.0/24"`), then computes:
///
/// ```text
/// host_count = 2^(32 - prefix_len)
/// usable = host_count - 3 // skip .0 (network), .1 (gateway), last (broadcast)
/// agent_slots = usable - DHCP_POOL_SIZE // top of range reserved for bridge DHCP pool
/// offset = FNV-1a(name) % agent_slots + 2 // .2 is the first agent slot
/// agent_ip = network_base_u32 + offset
/// ```
///
/// The last `DHCP_POOL_SIZE` usable host addresses are reserved for the
/// bridge DHCP pool (service containers such as hive-ci) and are never
/// assigned to agents. Agent IPs are stable as long as the agent name and
/// subnet don't change; a change to either (or to `DHCP_POOL_SIZE`) will
/// re-address agents, which is fine because nothing outside the container
/// depends on a specific agent IP — they reconnect on next rebuild.
///
/// Returns `None` when `subnet_cidr` can't be parsed (invalid format,
/// prefix out of range, subnet too small for both agents and DHCP pool,
/// etc.) so callers can fall back gracefully.
/// Collisions are possible (birthday paradox) and the operator resolves
/// them by renaming an agent, same as for port collisions.
#[must_use]
pub fn agent_network_ip(name: &str, subnet_cidr: &str) -> Option<String> {
let (ip_str, prefix_str) = subnet_cidr.split_once('/')?;
let prefix_len: u32 = prefix_str.parse().ok()?;
if prefix_len > 30 {
// /31 and /32 have no room for agents; /30 has 1 usable slot.
// /0 (the other extreme) is handled further down: host_count
// overflows checked_shl(32) → 0 → usable = 0 → None.
return None;
}
// Parse dotted-decimal IPv4.
let octets: Vec<u8> = ip_str
.split('.')
.map(|o| o.parse::<u8>().ok())
.collect::<Option<Vec<_>>>()?;
if octets.len() != 4 {
return None;
}
let base_u32 = u32::from_be_bytes([octets[0], octets[1], octets[2], octets[3]]);
// Mask off host bits to get the true network address.
let mask = if prefix_len == 0 {
0u32
} else {
!0u32 << (32 - prefix_len)
};
let network_base = base_u32 & mask;
let host_count: u32 = 1u32.checked_shl(32 - prefix_len).unwrap_or(0);
// `.0` = network, `.1` = bridge gateway, last = broadcast → 3 reserved.
let usable = host_count.saturating_sub(3);
// Reserve the last DHCP_POOL_SIZE usable addresses for the bridge DHCP
// pool. Agents only hash into the remaining agent-only window.
let agent_slots = usable.saturating_sub(DHCP_POOL_SIZE);
if agent_slots == 0 {
return None;
}
let offset = fnv1a(name) % agent_slots + 2; // +2: skip .0 and .1
let ip_u32 = network_base + offset;
let [a, b, c, d] = ip_u32.to_be_bytes();
Some(format!("{a}.{b}.{c}.{d}"))
}
/// Extract the bridge gateway IP from `HIVE_NETWORK_SUBNET`. /// Extract the bridge gateway IP from `HIVE_NETWORK_SUBNET`.
/// ///
/// `HIVE_NETWORK_SUBNET` carries the host-side bridge address verbatim /// `HIVE_NETWORK_SUBNET` carries the host-side bridge address verbatim
@ -198,9 +108,8 @@ pub fn agent_network_ip(name: &str, subnet_cidr: &str) -> Option<String> {
pub fn bridge_gateway_ip(subnet_cidr: &str) -> Option<String> { pub fn bridge_gateway_ip(subnet_cidr: &str) -> Option<String> {
let (ip_str, prefix_str) = subnet_cidr.split_once('/')?; let (ip_str, prefix_str) = subnet_cidr.split_once('/')?;
// Validate the prefix is a sane IPv4 CIDR length and the address is // Validate the prefix is a sane IPv4 CIDR length and the address is
// dotted-decimal IPv4 — same shape `agent_network_ip` accepts — so a // dotted-decimal IPv4 so a malformed `HIVE_NETWORK_SUBNET` can't
// malformed `HIVE_NETWORK_SUBNET` can't smuggle a bogus HOST_ADDRESS // smuggle a bogus HOST_ADDRESS into the nspawn conf.
// into the nspawn conf.
let prefix_len: u32 = prefix_str.parse().ok()?; let prefix_len: u32 = prefix_str.parse().ok()?;
if prefix_len > 32 { if prefix_len > 32 {
return None; return None;

99
hive-c0re/src/lifecycle/tests.rs
View file

@ -37,80 +37,6 @@ async fn setup_proposed_seeds_flake_nix() {
assert!(tracked.contains("flake.nix"), "flake.nix not committed"); assert!(tracked.contains("flake.nix"), "flake.nix not committed");
} }
#[test]
fn agent_network_ip_is_in_subnet() {
// Default subnet 10.42.0.0/24 — agents get .2 to .240 (last 14 are DHCP pool).
// agent_slots = usable(253) - DHCP_POOL_SIZE(14) = 239 → offsets in [2, 240].
let ip = agent_network_ip("alice", "10.42.0.0/24").expect("should produce an IP");
let octets: Vec<u8> = ip.split('.').map(|o| o.parse().unwrap()).collect();
assert_eq!(&octets[..3], &[10, 42, 0], "wrong /24 prefix");
assert!(
octets[3] >= 2 && octets[3] <= 240,
"host byte {} — expected in agent-only range [2,240]",
octets[3]
);
}
#[test]
fn agent_network_ip_never_in_dhcp_pool() {
// No agent should be assigned an address in the DHCP pool
// (.241-.254 on a /24 with DHCP_POOL_SIZE=14).
// agent_slots = usable(253) - DHCP_POOL_SIZE(14) = 239 → offsets in [2, 240].
let subnet = "10.42.0.0/24";
let names = [
"alice",
"bob",
"carol",
"damocles",
"iris",
"argus",
"atlas",
"ruth",
"dmatrix",
"bitburner",
"lexis",
"sock",
"triage",
"janet",
"eve",
"frank",
"grace",
"heidi",
];
for name in names {
let ip = agent_network_ip(name, subnet).unwrap_or_else(|| panic!("{name} returned None"));
let last: u8 = ip.rsplit('.').next().unwrap().parse().unwrap();
assert!(
last < 241,
"{name} got .{last} — inside the DHCP pool [.241-.254]"
);
}
}
#[test]
fn agent_network_ip_stable() {
// Same name + subnet must always produce the same IP.
let a = agent_network_ip("damocles", "10.42.0.0/24");
let b = agent_network_ip("damocles", "10.42.0.0/24");
assert_eq!(a, b);
}
#[test]
fn agent_network_ip_different_agents() {
// Different agent names very likely produce different IPs (not guaranteed,
// but for these two names the hashes don't collide).
let alice = agent_network_ip("alice", "10.42.0.0/24").unwrap();
let bob = agent_network_ip("bob", "10.42.0.0/24").unwrap();
assert_ne!(alice, bob, "alice and bob collide — rename one");
}
#[test]
fn agent_network_ip_different_subnet() {
let ip = agent_network_ip("alice", "192.168.5.0/24").expect("should produce an IP");
let octets: Vec<u8> = ip.split('.').map(|o| o.parse().unwrap()).collect();
assert_eq!(&octets[..3], &[192, 168, 5]);
}
#[test] #[test]
fn bridge_gateway_ip_extracts_verbatim_address() { fn bridge_gateway_ip_extracts_verbatim_address() {
// HIVE_NETWORK_SUBNET carries the bridge IP verbatim, not the // HIVE_NETWORK_SUBNET carries the bridge IP verbatim, not the
@ -139,31 +65,6 @@ fn bridge_gateway_ip_rejects_bad_input() {
assert!(bridge_gateway_ip("10.42.0/24").is_none()); // 3 octets assert!(bridge_gateway_ip("10.42.0/24").is_none()); // 3 octets
} }
#[test]
fn agent_network_ip_rejects_bad_input() {
assert!(agent_network_ip("alice", "notanip/24").is_none());
assert!(agent_network_ip("alice", "10.0.0.0/33").is_none()); // prefix > 32
assert!(agent_network_ip("alice", "10.0.0.0/31").is_none()); // too small
assert!(agent_network_ip("alice", "10.0.0.0").is_none()); // no prefix
}
#[test]
fn agent_network_ip_normalizes_bridge_ip_subnet() {
// HIVE_NETWORK_SUBNET carries the bridge IP (10.42.0.1/24), not
// canonical network (10.42.0.0/24). Both must produce the same result
// after host-bit masking.
let from_bridge = agent_network_ip("alice", "10.42.0.1/24");
let from_canonical = agent_network_ip("alice", "10.42.0.0/24");
assert_eq!(
from_bridge, from_canonical,
"bridge-IP and canonical-network form should normalize to the same result"
);
// Result must be in .2-.240 (DHCP pool .241-.254 is excluded).
let ip = from_bridge.unwrap();
let last: u8 = ip.rsplit('.').next().unwrap().parse().unwrap();
assert!((2..=240).contains(&last), "host byte {last}");
}
/// `setup_proposed` is idempotent: calling it on an existing repo is a /// `setup_proposed` is idempotent: calling it on an existing repo is a
/// no-op (the fresh guard skips all writes). /// no-op (the fresh guard skips all writes).
#[tokio::test] #[tokio::test]

6
hive-priv/src/main.rs
View file

@ -1487,7 +1487,11 @@ fn write_nspawn_flags(
// setup is skipped, so this only affects the container's route — // setup is skipped, so this only affects the container's route —
// exactly what we want. // exactly what we want.
let _ = writeln!(out, "HOST_ADDRESS={}", iso.gateway_ip); let _ = writeln!(out, "HOST_ADDRESS={}", iso.gateway_ip);
let _ = writeln!(out, "LOCAL_ADDRESS={}", iso.agent_ip); // LOCAL_ADDRESS is intentionally empty: agent containers receive their
// IP dynamically via DHCP from the bridge dnsmasq pool. HOST_ADDRESS
// (the gateway IP) is still written so nixos-container's container-side
// init installs a default route before the DHCP lease arrives.
out.push_str("LOCAL_ADDRESS=\n");
out.push_str("HOST_ADDRESS6=\n"); out.push_str("HOST_ADDRESS6=\n");
out.push_str("LOCAL_ADDRESS6=\n"); out.push_str("LOCAL_ADDRESS6=\n");
let _ = writeln!(out, "HOST_BRIDGE={}", iso.bridge); let _ = writeln!(out, "HOST_BRIDGE={}", iso.bridge);

18
hive-sh4re/src/priv_proto.rs
View file

@ -208,21 +208,21 @@ pub struct CredentialMount {
/// Network isolation parameters for `WriteNspawnFlags`. When `Some`, /// Network isolation parameters for `WriteNspawnFlags`. When `Some`,
/// hive-priv writes `PRIVATE_NETWORK=1` + veth bridge wiring instead /// hive-priv writes `PRIVATE_NETWORK=1` + veth bridge wiring instead
/// of the default `PRIVATE_NETWORK=0`. /// of the default `PRIVATE_NETWORK=0`. Containers receive their IP
/// dynamically via DHCP from the bridge dnsmasq pool (`networking.useDHCP`
/// in `harness-base.nix`); no static address is pre-assigned here.
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NetworkIsolation { pub struct NetworkIsolation {
/// Static IP address to assign to this container on the bridge subnet.
pub agent_ip: String,
/// Host bridge interface name (e.g. `hive0`). /// Host bridge interface name (e.g. `hive0`).
pub bridge: String, pub bridge: String,
/// Bridge gateway IP (the host-side bridge address, e.g. `10.42.0.1`). /// Bridge gateway IP (the host-side bridge address, e.g. `10.42.0.1`).
/// Written as `HOST_ADDRESS=` in the nspawn conf so nixos-container's /// Written as `HOST_ADDRESS=` in the nspawn conf so nixos-container's
/// container-side setup installs a default route (`default via <gw>`): /// container-side setup installs a default route (`default via <gw>`)
/// without it the container comes up with an address but no route off /// before DHCP completes: without it the container has no route off
/// the bridge subnet — no internet, no `api.anthropic.com`. The same IP /// the bridge subnet until the DHCP lease arrives. The same IP runs the
/// runs the hive dnsmasq resolver, so it's also written into the /// hive dnsmasq resolver, so it's also written into the container's
/// container's `/etc/resolv.conf` (see the isolated-DNS oneshot in /// `/etc/resolv.conf` (see the isolated-DNS oneshot in `harness-base.nix`,
/// `harness-base.nix`, gated on the marker hive-priv drops). /// gated on the marker hive-priv drops).
pub gateway_ip: String, pub gateway_ip: String,
} }

1
nix/dhcp-pool-size
View file

@ -1 +0,0 @@
14

29
nix/modules/hive-gateway.nix
View file

@ -11,17 +11,12 @@ let
forgeCfg = config.services.hyperhive.forge; forgeCfg = config.services.hyperhive.forge;
networkCfg = config.services.hyperhive.network; networkCfg = config.services.hyperhive.network;
# DHCP pool for bridge-attached service containers (hive-ci, etc.). # DHCP pool covering all usable host addresses on the bridge subnet.
# Occupies the last dhcpPoolSize usable addresses of the subnet # All containers (agents and service containers such as hive-ci) receive
# (e.g. .241-.254 on a /24). Agent containers use hash-derived static # their IPs dynamically; there are no hash-derived static assignments.
# IPs (lifecycle::agent_network_ip) and are excluded from this range # Range: .2 to .(hostCount-2) — skipping .0 (network), .1 (gateway/host
# by subtracting dhcpPoolSize from the usable count before hashing # bridge), and the broadcast address.
# (agent_slots = usable - dhcpPoolSize), so agents only ever land in
# [.2, .(usable-dhcpPoolSize+1)] by construction.
# #
# Single source of truth: `nix/dhcp-pool-size` (one integer).
# Rust reads it at compile time via `include_bytes!` in lifecycle/mod.rs.
dhcpPoolSize = lib.strings.toIntBase10 (lib.strings.trim (builtins.readFile ../dhcp-pool-size));
# IPv4 helpers — nix integers are 64-bit so all /0-/32 values are safe. # IPv4 helpers — nix integers are 64-bit so all /0-/32 values are safe.
ipToInt = ipToInt =
ip: ip:
@ -42,9 +37,9 @@ let
hostCount = pow2 (32 - networkCfg.bridgePrefixLength); hostCount = pow2 (32 - networkCfg.bridgePrefixLength);
# Mask off host bits to get the network base address. # Mask off host bits to get the network base address.
networkBase = builtins.bitAnd (ipToInt networkCfg.bridgeIp) (4294967295 - hostCount + 1); networkBase = builtins.bitAnd (ipToInt networkCfg.bridgeIp) (4294967295 - hostCount + 1);
# DHCP range: last dhcpPoolSize usable host addresses (broadcast - 1 down). # DHCP range: .2 (first usable after gateway) to .(hostCount-2) (last usable).
dhcpStart = intToIp (networkBase + 2); # skip .0 (network) and .1 (gateway)
dhcpEnd = intToIp (networkBase + hostCount - 2); # last usable = broadcast - 1 dhcpEnd = intToIp (networkBase + hostCount - 2); # last usable = broadcast - 1
dhcpStart = intToIp (networkBase + hostCount - 1 - dhcpPoolSize); # dhcpEnd - poolSize + 1
# Dashboard SPA dist, static-served by nginx below. Read in OUTER scope so # Dashboard SPA dist, static-served by nginx below. Read in OUTER scope so
# `config` is the host's (inside the container block it'd be the container's). # `config` is the host's (inside the container block it'd be the container's).
@ -1034,11 +1029,11 @@ in
++ lib.optional ( ++ lib.optional (
matrixCfg.enable && matrixCfg.gatewayHost != null matrixCfg.enable && matrixCfg.gatewayHost != null
) "/${matrixCfg.gatewayHost}/${networkCfg.bridgeIp}"; ) "/${matrixCfg.gatewayHost}/${networkCfg.bridgeIp}";
# DHCP pool for bridge-attached service containers (hive-ci, etc.). # DHCP pool covering all usable host addresses on the bridge subnet.
# Range is computed from the bridgeIp/bridgePrefixLength at eval # Range is computed from bridgeIp/bridgePrefixLength at eval time:
# time; the last dhcpPoolSize usable host addresses are reserved. # .2 (first after gateway) to .(hostCount-2) (last usable before
# Agent containers hash into agent_slots = usable - dhcpPoolSize # broadcast). All containers — agents and service containers alike —
# so they never land here (excluded by construction, not remapping). # receive their IPs dynamically from this pool.
dhcp-range = "${dhcpStart},${dhcpEnd},1h"; dhcp-range = "${dhcpStart},${dhcpEnd},1h";
dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases"; dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases";
}; };

4
nix/templates/harness-base.nix
View file

@ -1695,6 +1695,10 @@ in
# disabling dhcpcd itself, so the veth still gets its address); then # disabling dhcpcd itself, so the veth still gets its address); then
# the hyperhive-isolated-dns oneshot owns resolv.conf. (Same "take # the hyperhive-isolated-dns oneshot owns resolv.conf. (Same "take
# resolvconf out of the loop" approach the matrix container uses.) # resolvconf out of the loop" approach the matrix container uses.)
# All agent containers receive their bridge IP via DHCP from the hive
# dnsmasq pool (see hive-gateway.nix). useDHCP runs dhcpcd on every
# interface (just eth0 in practice — the nspawn bridge veth).
networking.useDHCP = true;
networking.resolvconf.enable = false; networking.resolvconf.enable = false;
networking.dhcpcd.extraConfig = "nohook resolv.conf"; networking.dhcpcd.extraConfig = "nohook resolv.conf";