feat(#2363): DHCP pool for bridge service containers
Add a DHCP pool to the gateway's dnsmasq so bridge-attached service containers (hive-ci and future equivalents) get their addresses from a proper DHCP server instead of a brittle static derivation. gateway (hive-gateway.nix): - Add IPv4 arithmetic helpers (ipToInt, intToIp, pow2) to compute the DHCP pool range at nix eval time from bridgeIp + bridgePrefixLength. - Reserve the last dhcpPoolSize (14) usable host addresses as the DHCP pool (e.g. .241-.254 on a /24 with 10.42.0.0 network). - Add dhcp-range and dhcp-leasefile to the dnsmasq settings block. The pool is active whenever services.hyperhive.network.enable is true. hive-ci (hive-ci.nix): - Remove the ciBridgeIp / ciBridgeOctets static derivation and the brittle top-of-/24 comment block. - Switch networking.interfaces.eth0 to useDHCP = true so hive-ci gets its address from the gateway DHCP pool. lifecycle (mod.rs, tests.rs): - Add DHCP_POOL_SIZE = 14 constant (must stay in sync with dhcpPoolSize in hive-gateway.nix). - Remap agents whose FNV-1a hash falls in the DHCP pool into the agent-only window [2, dhcp_start - 1]. Only the rare agent whose name hashes into the pool is affected; all others keep their IPs. - Update and extend tests: agent range is now .2-.240 on /24; add agent_network_ip_never_in_dhcp_pool covering 18 agent names.
This commit is contained in:
atlas
mara
parent
16f69ca890
commit
a3796890f5
4 changed files with 114 additions and 36 deletions
|
|
@ -90,6 +90,12 @@ pub fn agent_web_port(name: &str) -> u16 {
|
|||
WEB_PORT_BASE + u16::try_from(fnv1a(name) % u32::from(WEB_PORT_RANGE)).unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Number of IP addresses at the top of each subnet reserved for the DHCP
|
||||
/// pool (bridge-attached service containers such as hive-ci). Agents are
|
||||
/// excluded from this range by the remap in `agent_network_ip`. Must stay
|
||||
/// in sync with `dhcpPoolSize` in `nix/modules/hive-gateway.nix`.
|
||||
const DHCP_POOL_SIZE: u32 = 14;
|
||||
|
||||
/// Deterministic IPv4 address for an agent inside an isolated subnet.
|
||||
///
|
||||
/// Parses `subnet_cidr` as `<network_ip>/<prefix_len>` (e.g.
|
||||
|
|
@ -97,11 +103,18 @@ pub fn agent_web_port(name: &str) -> u16 {
|
|||
///
|
||||
/// ```text
|
||||
/// host_count = 2^(32 - prefix_len)
|
||||
/// usable = host_count - 3 // skip .0 (network), .1 (gateway), .255 (broadcast)
|
||||
/// usable = host_count - 3 // skip .0 (network), .1 (gateway), last (broadcast)
|
||||
/// offset = FNV-1a(name) % usable + 2 // .2 is the first agent slot
|
||||
/// agent_ip = network_base_u32 + offset
|
||||
/// ```
|
||||
///
|
||||
/// The last `DHCP_POOL_SIZE` usable host addresses are reserved for the
|
||||
/// bridge DHCP pool (service containers). When the primary hash falls in
|
||||
/// that range the offset is remapped into the agent-only window `[2,
|
||||
/// dhcp_start - 1]` so no agent is ever assigned a DHCP-pool address.
|
||||
/// Only the rare agent whose name hashes into the pool is affected; all
|
||||
/// other agents keep their original IPs.
|
||||
///
|
||||
/// Returns `None` when `subnet_cidr` can't be parsed (invalid format,
|
||||
/// prefix out of range, etc.) so callers can fall back gracefully.
|
||||
/// Collisions are possible (birthday paradox) and the operator resolves
|
||||
|
|
@ -138,7 +151,22 @@ pub fn agent_network_ip(name: &str, subnet_cidr: &str) -> Option<String> {
|
|||
if usable == 0 {
|
||||
return None;
|
||||
}
|
||||
let offset = fnv1a(name) % usable + 2; // +2: skip .0 and .1
|
||||
let mut offset = fnv1a(name) % usable + 2; // +2: skip .0 and .1
|
||||
|
||||
// Remap agents whose primary hash falls in the DHCP pool (the last
|
||||
// DHCP_POOL_SIZE usable addresses). Only has an effect when the
|
||||
// subnet is large enough to hold both agent slots AND a pool.
|
||||
if usable > DHCP_POOL_SIZE {
|
||||
// First offset that belongs to the DHCP pool.
|
||||
let dhcp_start = usable + 2 - DHCP_POOL_SIZE;
|
||||
if offset >= dhcp_start {
|
||||
// Secondary hash into the agent-only window [2, dhcp_start - 1].
|
||||
// Result is guaranteed < dhcp_start (no overlap with pool).
|
||||
let agent_only = dhcp_start - 2;
|
||||
offset = fnv1a(name) % agent_only + 2;
|
||||
}
|
||||
}
|
||||
|
||||
let ip_u32 = network_base + offset;
|
||||
let [a, b, c, d] = ip_u32.to_be_bytes();
|
||||
Some(format!("{a}.{b}.{c}.{d}"))
|
||||
|
|
|
|||
|
|
@ -39,17 +39,54 @@ async fn setup_proposed_seeds_flake_nix() {
|
|||
|
||||
#[test]
|
||||
fn agent_network_ip_is_in_subnet() {
|
||||
// Default subnet 10.42.0.0/24 — agents get .2 to .254.
|
||||
// Default subnet 10.42.0.0/24 — agents get .2 to .240 (last 14 are DHCP pool).
|
||||
let ip = agent_network_ip("alice", "10.42.0.0/24").expect("should produce an IP");
|
||||
let octets: Vec<u8> = ip.split('.').map(|o| o.parse().unwrap()).collect();
|
||||
assert_eq!(&octets[..3], &[10, 42, 0], "wrong /24 prefix");
|
||||
// usable=253, dhcp_start=241 → agent range is .2-.240
|
||||
assert!(
|
||||
octets[3] >= 2 && octets[3] <= 254,
|
||||
"host byte {}",
|
||||
octets[3] >= 2 && octets[3] <= 240,
|
||||
"host byte {} — expected in agent-only range [2,240]",
|
||||
octets[3]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agent_network_ip_never_in_dhcp_pool() {
|
||||
// No agent should be assigned an address in the DHCP pool
|
||||
// (.241-.254 on a /24 with DHCP_POOL_SIZE=14).
|
||||
let subnet = "10.42.0.0/24";
|
||||
let names = [
|
||||
"alice",
|
||||
"bob",
|
||||
"carol",
|
||||
"damocles",
|
||||
"iris",
|
||||
"argus",
|
||||
"atlas",
|
||||
"ruth",
|
||||
"dmatrix",
|
||||
"bitburner",
|
||||
"lexis",
|
||||
"sock",
|
||||
"triage",
|
||||
"janet",
|
||||
"eve",
|
||||
"frank",
|
||||
"grace",
|
||||
"heidi",
|
||||
];
|
||||
for name in names {
|
||||
let ip = agent_network_ip(name, subnet).unwrap_or_else(|| panic!("{name} returned None"));
|
||||
let last: u8 = ip.rsplit('.').next().unwrap().parse().unwrap();
|
||||
// dhcp_start_offset = usable(253) + 2 - DHCP_POOL_SIZE(14) = 241
|
||||
assert!(
|
||||
last < 241,
|
||||
"{name} got .{last} — inside the DHCP pool [.241-.254]"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agent_network_ip_stable() {
|
||||
// Same name + subnet must always produce the same IP.
|
||||
|
|
@ -121,10 +158,10 @@ fn agent_network_ip_normalizes_bridge_ip_subnet() {
|
|||
from_bridge, from_canonical,
|
||||
"bridge-IP and canonical-network form should normalize to the same result"
|
||||
);
|
||||
// Result must still be in .2-.254.
|
||||
// Result must be in .2-.240 (DHCP pool .241-.254 is excluded).
|
||||
let ip = from_bridge.unwrap();
|
||||
let last: u8 = ip.rsplit('.').next().unwrap().parse().unwrap();
|
||||
assert!((2..=254).contains(&last), "host byte {last}");
|
||||
assert!((2..=240).contains(&last), "host byte {last}");
|
||||
}
|
||||
|
||||
/// `setup_proposed` is idempotent: calling it on an existing repo is a
|
||||
|
|
|
|||
|
|
@ -11,19 +11,6 @@ let
|
|||
networkCfg = config.services.hyperhive.network;
|
||||
tlsCfg = config.services.hyperhive.tls;
|
||||
|
||||
# Static bridge address for the hive-ci container. The hive bridge has
|
||||
# NO DHCP server: agent containers get deterministic static IPs
|
||||
# (`lifecycle::agent_network_ip` hashes each name across .2..broadcast-1)
|
||||
# and dnsmasq on the bridge is DNS-only. hive-ci is the one service
|
||||
# container on the bridge, so it needs a static address too — `useDHCP`
|
||||
# here only hangs the boot waiting for a lease nothing serves. Reserve the
|
||||
# top host address of the (default /24) subnet; a clash with an agent that
|
||||
# happens to hash here is the same rename-to-resolve case as any
|
||||
# agent/agent IP collision. Operators on a non-/24 bridge (or with
|
||||
# `bridgeIp` set to the top address) should pick a free host address.
|
||||
ciBridgeOctets = lib.splitString "." networkCfg.bridgeIp;
|
||||
ciBridgeIp = "${lib.elemAt ciBridgeOctets 0}.${lib.elemAt ciBridgeOctets 1}.${lib.elemAt ciBridgeOctets 2}.254";
|
||||
|
||||
# Self-signed TLS is the gateway default (no operator cert / ACME). When
|
||||
# active, forgejo's ROOT_URL is `https://forge.<domain>` and the leaf is
|
||||
# signed by the host hive CA — so the runner's Node-based actions (e.g.
|
||||
|
|
@ -422,22 +409,11 @@ in
|
|||
# registries, etc.). The bridge→loopback DROP rule does not
|
||||
# affect traffic destined for the bridge IP itself.
|
||||
networking.nameservers = [ networkCfg.bridgeIp ];
|
||||
# With privateNetwork=true + hostBridge the container's veth
|
||||
# (eth0) is bridge-attached. There is no DHCP server on the hive
|
||||
# bridge (dnsmasq is DNS-only; agents use static IPs), so assign a
|
||||
# static address + default route via the bridge gateway rather than
|
||||
# DHCP — `useDHCP` here just hangs boot on a lease that never
|
||||
# arrives. See `ciBridgeIp` above.
|
||||
networking.interfaces.eth0.ipv4.addresses = [
|
||||
{
|
||||
address = ciBridgeIp;
|
||||
prefixLength = networkCfg.bridgePrefixLength;
|
||||
}
|
||||
];
|
||||
networking.defaultGateway = {
|
||||
address = networkCfg.bridgeIp;
|
||||
interface = "eth0";
|
||||
};
|
||||
# Bridge-attached via privateNetwork=true + hostBridge. The
|
||||
# gateway's dnsmasq now serves a DHCP pool for service containers
|
||||
# (see dhcp-range in hive-gateway.nix). hive-ci gets its address
|
||||
# from that pool; no static address needed.
|
||||
networking.interfaces.eth0.useDHCP = true;
|
||||
|
||||
# nspawn containers can't create user-namespaces, so nix
|
||||
# sandboxing always fails. Fall back to unsandboxed builds.
|
||||
|
|
|
|||
|
|
@ -11,6 +11,37 @@ let
|
|||
forgeCfg = config.services.hyperhive.forge;
|
||||
networkCfg = config.services.hyperhive.network;
|
||||
|
||||
# DHCP pool for bridge-attached service containers (hive-ci, etc.).
|
||||
# Occupies the last dhcpPoolSize usable addresses of the subnet
|
||||
# (e.g. .241-.254 on a /24). Agent containers use deterministic static
|
||||
# IPs (lifecycle::agent_network_ip) and are excluded from this range
|
||||
# by a remap in the Rust code. Must stay in sync with DHCP_POOL_SIZE
|
||||
# in hive-c0re/src/lifecycle/mod.rs.
|
||||
dhcpPoolSize = 14;
|
||||
# IPv4 helpers — nix integers are 64-bit so all /0-/32 values are safe.
|
||||
ipToInt =
|
||||
ip:
|
||||
builtins.foldl' (acc: x: acc * 256 + x) 0 (
|
||||
map lib.strings.toIntBase10 (lib.strings.splitString "." ip)
|
||||
);
|
||||
intToIp =
|
||||
n:
|
||||
let
|
||||
a = n / 16777216;
|
||||
b = (n - a * 16777216) / 65536;
|
||||
c = (n - a * 16777216 - b * 65536) / 256;
|
||||
d = n - a * 16777216 - b * 65536 - c * 256;
|
||||
in
|
||||
"${toString a}.${toString b}.${toString c}.${toString d}";
|
||||
# 2^n via recursion (nix has no pow builtin).
|
||||
pow2 = n: if n == 0 then 1 else 2 * (pow2 (n - 1));
|
||||
hostCount = pow2 (32 - networkCfg.bridgePrefixLength);
|
||||
# Mask off host bits to get the network base address.
|
||||
networkBase = builtins.bitAnd (ipToInt networkCfg.bridgeIp) (4294967295 - hostCount + 1);
|
||||
# DHCP range: last dhcpPoolSize usable host addresses (broadcast - 1 down).
|
||||
dhcpEnd = intToIp (networkBase + hostCount - 2); # last usable = broadcast - 1
|
||||
dhcpStart = intToIp (networkBase + hostCount - 1 - dhcpPoolSize); # dhcpEnd - poolSize + 1
|
||||
|
||||
# Dashboard SPA dist, static-served by nginx below. Read in OUTER scope so
|
||||
# `config` is the host's (inside the container block it'd be the container's).
|
||||
dashboardDist = "${config.services.hyperhive.c0re.servedFrontend}/dashboard";
|
||||
|
|
@ -999,6 +1030,12 @@ in
|
|||
++ lib.optional (
|
||||
matrixCfg.enable && matrixCfg.gatewayHost != null
|
||||
) "/${matrixCfg.gatewayHost}/${networkCfg.bridgeIp}";
|
||||
# DHCP pool for bridge-attached service containers (hive-ci, etc.).
|
||||
# Range is computed from the bridgeIp/bridgePrefixLength at eval
|
||||
# time; the last dhcpPoolSize usable host addresses are reserved.
|
||||
# Agent containers are excluded by agent_network_ip's DHCP remap.
|
||||
dhcp-range = "${dhcpStart},${dhcpEnd},1h";
|
||||
dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in a new issue