feat(#2363): DHCP pool for bridge service containers
Add a DHCP pool to the gateway's dnsmasq so bridge-attached service containers (hive-ci and future equivalents) get their addresses from a proper DHCP server instead of a brittle static derivation. gateway (hive-gateway.nix): - Add IPv4 arithmetic helpers (ipToInt, intToIp, pow2) to compute the DHCP pool range at nix eval time from bridgeIp + bridgePrefixLength. - Reserve the last dhcpPoolSize (14) usable host addresses as the DHCP pool (e.g. .241-.254 on a /24 with 10.42.0.0 network). - Add dhcp-range and dhcp-leasefile to the dnsmasq settings block. The pool is active whenever services.hyperhive.network.enable is true. hive-ci (hive-ci.nix): - Remove the ciBridgeIp / ciBridgeOctets static derivation and the brittle top-of-/24 comment block. - Switch networking.interfaces.eth0 to useDHCP = true so hive-ci gets its address from the gateway DHCP pool. lifecycle (mod.rs, tests.rs): - Add DHCP_POOL_SIZE = 14 constant (must stay in sync with dhcpPoolSize in hive-gateway.nix). - Remap agents whose FNV-1a hash falls in the DHCP pool into the agent-only window [2, dhcp_start - 1]. Only the rare agent whose name hashes into the pool is affected; all others keep their IPs. - Update and extend tests: agent range is now .2-.240 on /24; add agent_network_ip_never_in_dhcp_pool covering 18 agent names.
This commit is contained in:
parent
16f69ca890
commit
a3796890f5
4 changed files with 114 additions and 36 deletions
|
|
@ -90,6 +90,12 @@ pub fn agent_web_port(name: &str) -> u16 {
|
||||||
WEB_PORT_BASE + u16::try_from(fnv1a(name) % u32::from(WEB_PORT_RANGE)).unwrap_or(0)
|
WEB_PORT_BASE + u16::try_from(fnv1a(name) % u32::from(WEB_PORT_RANGE)).unwrap_or(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Number of IP addresses at the top of each subnet reserved for the DHCP
|
||||||
|
/// pool (bridge-attached service containers such as hive-ci). Agents are
|
||||||
|
/// excluded from this range by the remap in `agent_network_ip`. Must stay
|
||||||
|
/// in sync with `dhcpPoolSize` in `nix/modules/hive-gateway.nix`.
|
||||||
|
const DHCP_POOL_SIZE: u32 = 14;
|
||||||
|
|
||||||
/// Deterministic IPv4 address for an agent inside an isolated subnet.
|
/// Deterministic IPv4 address for an agent inside an isolated subnet.
|
||||||
///
|
///
|
||||||
/// Parses `subnet_cidr` as `<network_ip>/<prefix_len>` (e.g.
|
/// Parses `subnet_cidr` as `<network_ip>/<prefix_len>` (e.g.
|
||||||
|
|
@ -97,11 +103,18 @@ pub fn agent_web_port(name: &str) -> u16 {
|
||||||
///
|
///
|
||||||
/// ```text
|
/// ```text
|
||||||
/// host_count = 2^(32 - prefix_len)
|
/// host_count = 2^(32 - prefix_len)
|
||||||
/// usable = host_count - 3 // skip .0 (network), .1 (gateway), .255 (broadcast)
|
/// usable = host_count - 3 // skip .0 (network), .1 (gateway), last (broadcast)
|
||||||
/// offset = FNV-1a(name) % usable + 2 // .2 is the first agent slot
|
/// offset = FNV-1a(name) % usable + 2 // .2 is the first agent slot
|
||||||
/// agent_ip = network_base_u32 + offset
|
/// agent_ip = network_base_u32 + offset
|
||||||
/// ```
|
/// ```
|
||||||
///
|
///
|
||||||
|
/// The last `DHCP_POOL_SIZE` usable host addresses are reserved for the
|
||||||
|
/// bridge DHCP pool (service containers). When the primary hash falls in
|
||||||
|
/// that range the offset is remapped into the agent-only window `[2,
|
||||||
|
/// dhcp_start - 1]` so no agent is ever assigned a DHCP-pool address.
|
||||||
|
/// Only the rare agent whose name hashes into the pool is affected; all
|
||||||
|
/// other agents keep their original IPs.
|
||||||
|
///
|
||||||
/// Returns `None` when `subnet_cidr` can't be parsed (invalid format,
|
/// Returns `None` when `subnet_cidr` can't be parsed (invalid format,
|
||||||
/// prefix out of range, etc.) so callers can fall back gracefully.
|
/// prefix out of range, etc.) so callers can fall back gracefully.
|
||||||
/// Collisions are possible (birthday paradox) and the operator resolves
|
/// Collisions are possible (birthday paradox) and the operator resolves
|
||||||
|
|
@ -138,7 +151,22 @@ pub fn agent_network_ip(name: &str, subnet_cidr: &str) -> Option<String> {
|
||||||
if usable == 0 {
|
if usable == 0 {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
let offset = fnv1a(name) % usable + 2; // +2: skip .0 and .1
|
let mut offset = fnv1a(name) % usable + 2; // +2: skip .0 and .1
|
||||||
|
|
||||||
|
// Remap agents whose primary hash falls in the DHCP pool (the last
|
||||||
|
// DHCP_POOL_SIZE usable addresses). Only has an effect when the
|
||||||
|
// subnet is large enough to hold both agent slots AND a pool.
|
||||||
|
if usable > DHCP_POOL_SIZE {
|
||||||
|
// First offset that belongs to the DHCP pool.
|
||||||
|
let dhcp_start = usable + 2 - DHCP_POOL_SIZE;
|
||||||
|
if offset >= dhcp_start {
|
||||||
|
// Secondary hash into the agent-only window [2, dhcp_start - 1].
|
||||||
|
// Result is guaranteed < dhcp_start (no overlap with pool).
|
||||||
|
let agent_only = dhcp_start - 2;
|
||||||
|
offset = fnv1a(name) % agent_only + 2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let ip_u32 = network_base + offset;
|
let ip_u32 = network_base + offset;
|
||||||
let [a, b, c, d] = ip_u32.to_be_bytes();
|
let [a, b, c, d] = ip_u32.to_be_bytes();
|
||||||
Some(format!("{a}.{b}.{c}.{d}"))
|
Some(format!("{a}.{b}.{c}.{d}"))
|
||||||
|
|
|
||||||
|
|
@ -39,17 +39,54 @@ async fn setup_proposed_seeds_flake_nix() {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn agent_network_ip_is_in_subnet() {
|
fn agent_network_ip_is_in_subnet() {
|
||||||
// Default subnet 10.42.0.0/24 — agents get .2 to .254.
|
// Default subnet 10.42.0.0/24 — agents get .2 to .240 (last 14 are DHCP pool).
|
||||||
let ip = agent_network_ip("alice", "10.42.0.0/24").expect("should produce an IP");
|
let ip = agent_network_ip("alice", "10.42.0.0/24").expect("should produce an IP");
|
||||||
let octets: Vec<u8> = ip.split('.').map(|o| o.parse().unwrap()).collect();
|
let octets: Vec<u8> = ip.split('.').map(|o| o.parse().unwrap()).collect();
|
||||||
assert_eq!(&octets[..3], &[10, 42, 0], "wrong /24 prefix");
|
assert_eq!(&octets[..3], &[10, 42, 0], "wrong /24 prefix");
|
||||||
|
// usable=253, dhcp_start=241 → agent range is .2-.240
|
||||||
assert!(
|
assert!(
|
||||||
octets[3] >= 2 && octets[3] <= 254,
|
octets[3] >= 2 && octets[3] <= 240,
|
||||||
"host byte {}",
|
"host byte {} — expected in agent-only range [2,240]",
|
||||||
octets[3]
|
octets[3]
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn agent_network_ip_never_in_dhcp_pool() {
|
||||||
|
// No agent should be assigned an address in the DHCP pool
|
||||||
|
// (.241-.254 on a /24 with DHCP_POOL_SIZE=14).
|
||||||
|
let subnet = "10.42.0.0/24";
|
||||||
|
let names = [
|
||||||
|
"alice",
|
||||||
|
"bob",
|
||||||
|
"carol",
|
||||||
|
"damocles",
|
||||||
|
"iris",
|
||||||
|
"argus",
|
||||||
|
"atlas",
|
||||||
|
"ruth",
|
||||||
|
"dmatrix",
|
||||||
|
"bitburner",
|
||||||
|
"lexis",
|
||||||
|
"sock",
|
||||||
|
"triage",
|
||||||
|
"janet",
|
||||||
|
"eve",
|
||||||
|
"frank",
|
||||||
|
"grace",
|
||||||
|
"heidi",
|
||||||
|
];
|
||||||
|
for name in names {
|
||||||
|
let ip = agent_network_ip(name, subnet).unwrap_or_else(|| panic!("{name} returned None"));
|
||||||
|
let last: u8 = ip.rsplit('.').next().unwrap().parse().unwrap();
|
||||||
|
// dhcp_start_offset = usable(253) + 2 - DHCP_POOL_SIZE(14) = 241
|
||||||
|
assert!(
|
||||||
|
last < 241,
|
||||||
|
"{name} got .{last} — inside the DHCP pool [.241-.254]"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn agent_network_ip_stable() {
|
fn agent_network_ip_stable() {
|
||||||
// Same name + subnet must always produce the same IP.
|
// Same name + subnet must always produce the same IP.
|
||||||
|
|
@ -121,10 +158,10 @@ fn agent_network_ip_normalizes_bridge_ip_subnet() {
|
||||||
from_bridge, from_canonical,
|
from_bridge, from_canonical,
|
||||||
"bridge-IP and canonical-network form should normalize to the same result"
|
"bridge-IP and canonical-network form should normalize to the same result"
|
||||||
);
|
);
|
||||||
// Result must still be in .2-.254.
|
// Result must be in .2-.240 (DHCP pool .241-.254 is excluded).
|
||||||
let ip = from_bridge.unwrap();
|
let ip = from_bridge.unwrap();
|
||||||
let last: u8 = ip.rsplit('.').next().unwrap().parse().unwrap();
|
let last: u8 = ip.rsplit('.').next().unwrap().parse().unwrap();
|
||||||
assert!((2..=254).contains(&last), "host byte {last}");
|
assert!((2..=240).contains(&last), "host byte {last}");
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `setup_proposed` is idempotent: calling it on an existing repo is a
|
/// `setup_proposed` is idempotent: calling it on an existing repo is a
|
||||||
|
|
|
||||||
|
|
@ -11,19 +11,6 @@ let
|
||||||
networkCfg = config.services.hyperhive.network;
|
networkCfg = config.services.hyperhive.network;
|
||||||
tlsCfg = config.services.hyperhive.tls;
|
tlsCfg = config.services.hyperhive.tls;
|
||||||
|
|
||||||
# Static bridge address for the hive-ci container. The hive bridge has
|
|
||||||
# NO DHCP server: agent containers get deterministic static IPs
|
|
||||||
# (`lifecycle::agent_network_ip` hashes each name across .2..broadcast-1)
|
|
||||||
# and dnsmasq on the bridge is DNS-only. hive-ci is the one service
|
|
||||||
# container on the bridge, so it needs a static address too — `useDHCP`
|
|
||||||
# here only hangs the boot waiting for a lease nothing serves. Reserve the
|
|
||||||
# top host address of the (default /24) subnet; a clash with an agent that
|
|
||||||
# happens to hash here is the same rename-to-resolve case as any
|
|
||||||
# agent/agent IP collision. Operators on a non-/24 bridge (or with
|
|
||||||
# `bridgeIp` set to the top address) should pick a free host address.
|
|
||||||
ciBridgeOctets = lib.splitString "." networkCfg.bridgeIp;
|
|
||||||
ciBridgeIp = "${lib.elemAt ciBridgeOctets 0}.${lib.elemAt ciBridgeOctets 1}.${lib.elemAt ciBridgeOctets 2}.254";
|
|
||||||
|
|
||||||
# Self-signed TLS is the gateway default (no operator cert / ACME). When
|
# Self-signed TLS is the gateway default (no operator cert / ACME). When
|
||||||
# active, forgejo's ROOT_URL is `https://forge.<domain>` and the leaf is
|
# active, forgejo's ROOT_URL is `https://forge.<domain>` and the leaf is
|
||||||
# signed by the host hive CA — so the runner's Node-based actions (e.g.
|
# signed by the host hive CA — so the runner's Node-based actions (e.g.
|
||||||
|
|
@ -422,22 +409,11 @@ in
|
||||||
# registries, etc.). The bridge→loopback DROP rule does not
|
# registries, etc.). The bridge→loopback DROP rule does not
|
||||||
# affect traffic destined for the bridge IP itself.
|
# affect traffic destined for the bridge IP itself.
|
||||||
networking.nameservers = [ networkCfg.bridgeIp ];
|
networking.nameservers = [ networkCfg.bridgeIp ];
|
||||||
# With privateNetwork=true + hostBridge the container's veth
|
# Bridge-attached via privateNetwork=true + hostBridge. The
|
||||||
# (eth0) is bridge-attached. There is no DHCP server on the hive
|
# gateway's dnsmasq now serves a DHCP pool for service containers
|
||||||
# bridge (dnsmasq is DNS-only; agents use static IPs), so assign a
|
# (see dhcp-range in hive-gateway.nix). hive-ci gets its address
|
||||||
# static address + default route via the bridge gateway rather than
|
# from that pool; no static address needed.
|
||||||
# DHCP — `useDHCP` here just hangs boot on a lease that never
|
networking.interfaces.eth0.useDHCP = true;
|
||||||
# arrives. See `ciBridgeIp` above.
|
|
||||||
networking.interfaces.eth0.ipv4.addresses = [
|
|
||||||
{
|
|
||||||
address = ciBridgeIp;
|
|
||||||
prefixLength = networkCfg.bridgePrefixLength;
|
|
||||||
}
|
|
||||||
];
|
|
||||||
networking.defaultGateway = {
|
|
||||||
address = networkCfg.bridgeIp;
|
|
||||||
interface = "eth0";
|
|
||||||
};
|
|
||||||
|
|
||||||
# nspawn containers can't create user-namespaces, so nix
|
# nspawn containers can't create user-namespaces, so nix
|
||||||
# sandboxing always fails. Fall back to unsandboxed builds.
|
# sandboxing always fails. Fall back to unsandboxed builds.
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,37 @@ let
|
||||||
forgeCfg = config.services.hyperhive.forge;
|
forgeCfg = config.services.hyperhive.forge;
|
||||||
networkCfg = config.services.hyperhive.network;
|
networkCfg = config.services.hyperhive.network;
|
||||||
|
|
||||||
|
# DHCP pool for bridge-attached service containers (hive-ci, etc.).
|
||||||
|
# Occupies the last dhcpPoolSize usable addresses of the subnet
|
||||||
|
# (e.g. .241-.254 on a /24). Agent containers use deterministic static
|
||||||
|
# IPs (lifecycle::agent_network_ip) and are excluded from this range
|
||||||
|
# by a remap in the Rust code. Must stay in sync with DHCP_POOL_SIZE
|
||||||
|
# in hive-c0re/src/lifecycle/mod.rs.
|
||||||
|
dhcpPoolSize = 14;
|
||||||
|
# IPv4 helpers — nix integers are 64-bit so all /0-/32 values are safe.
|
||||||
|
ipToInt =
|
||||||
|
ip:
|
||||||
|
builtins.foldl' (acc: x: acc * 256 + x) 0 (
|
||||||
|
map lib.strings.toIntBase10 (lib.strings.splitString "." ip)
|
||||||
|
);
|
||||||
|
intToIp =
|
||||||
|
n:
|
||||||
|
let
|
||||||
|
a = n / 16777216;
|
||||||
|
b = (n - a * 16777216) / 65536;
|
||||||
|
c = (n - a * 16777216 - b * 65536) / 256;
|
||||||
|
d = n - a * 16777216 - b * 65536 - c * 256;
|
||||||
|
in
|
||||||
|
"${toString a}.${toString b}.${toString c}.${toString d}";
|
||||||
|
# 2^n via recursion (nix has no pow builtin).
|
||||||
|
pow2 = n: if n == 0 then 1 else 2 * (pow2 (n - 1));
|
||||||
|
hostCount = pow2 (32 - networkCfg.bridgePrefixLength);
|
||||||
|
# Mask off host bits to get the network base address.
|
||||||
|
networkBase = builtins.bitAnd (ipToInt networkCfg.bridgeIp) (4294967295 - hostCount + 1);
|
||||||
|
# DHCP range: last dhcpPoolSize usable host addresses (broadcast - 1 down).
|
||||||
|
dhcpEnd = intToIp (networkBase + hostCount - 2); # last usable = broadcast - 1
|
||||||
|
dhcpStart = intToIp (networkBase + hostCount - 1 - dhcpPoolSize); # dhcpEnd - poolSize + 1
|
||||||
|
|
||||||
# Dashboard SPA dist, static-served by nginx below. Read in OUTER scope so
|
# Dashboard SPA dist, static-served by nginx below. Read in OUTER scope so
|
||||||
# `config` is the host's (inside the container block it'd be the container's).
|
# `config` is the host's (inside the container block it'd be the container's).
|
||||||
dashboardDist = "${config.services.hyperhive.c0re.servedFrontend}/dashboard";
|
dashboardDist = "${config.services.hyperhive.c0re.servedFrontend}/dashboard";
|
||||||
|
|
@ -999,6 +1030,12 @@ in
|
||||||
++ lib.optional (
|
++ lib.optional (
|
||||||
matrixCfg.enable && matrixCfg.gatewayHost != null
|
matrixCfg.enable && matrixCfg.gatewayHost != null
|
||||||
) "/${matrixCfg.gatewayHost}/${networkCfg.bridgeIp}";
|
) "/${matrixCfg.gatewayHost}/${networkCfg.bridgeIp}";
|
||||||
|
# DHCP pool for bridge-attached service containers (hive-ci, etc.).
|
||||||
|
# Range is computed from the bridgeIp/bridgePrefixLength at eval
|
||||||
|
# time; the last dhcpPoolSize usable host addresses are reserved.
|
||||||
|
# Agent containers are excluded by agent_network_ip's DHCP remap.
|
||||||
|
dhcp-range = "${dhcpStart},${dhcpEnd},1h";
|
||||||
|
dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases";
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue