feat(#14): network isolation rust side — PRIVATE_NETWORK + veth wiring in set_nspawn_flags
This commit is contained in:
parent
ed50b858c5
commit
3bb07b1fde
4 changed files with 185 additions and 24 deletions
|
|
@ -55,6 +55,17 @@ const WEB_PORT_RANGE: u16 = 900;
|
||||||
const DEFAULT_MEMORY_MAX: &str = "2G";
|
const DEFAULT_MEMORY_MAX: &str = "2G";
|
||||||
const DEFAULT_CPU_QUOTA: &str = "50%";
|
const DEFAULT_CPU_QUOTA: &str = "50%";
|
||||||
|
|
||||||
|
/// FNV-1a hash of a string — shared by `agent_web_port` and
|
||||||
|
/// `agent_network_ip` so the derivation rule is identical.
|
||||||
|
fn fnv1a(s: &str) -> u32 {
|
||||||
|
let mut hash: u32 = 2_166_136_261;
|
||||||
|
for b in s.bytes() {
|
||||||
|
hash ^= u32::from(b);
|
||||||
|
hash = hash.wrapping_mul(16_777_619);
|
||||||
|
}
|
||||||
|
hash
|
||||||
|
}
|
||||||
|
|
||||||
/// Per-agent web UI port — `WEB_PORT_BASE + FNV-1a(name) %
|
/// Per-agent web UI port — `WEB_PORT_BASE + FNV-1a(name) %
|
||||||
/// WEB_PORT_RANGE` for every agent including the manager. The port
|
/// WEB_PORT_RANGE` for every agent including the manager. The port
|
||||||
/// allocation rule reads the same for every name; collisions are
|
/// allocation rule reads the same for every name; collisions are
|
||||||
|
|
@ -64,13 +75,56 @@ const DEFAULT_CPU_QUOTA: &str = "50%";
|
||||||
/// no state-file dance.
|
/// no state-file dance.
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn agent_web_port(name: &str) -> u16 {
|
pub fn agent_web_port(name: &str) -> u16 {
|
||||||
let mut hash: u32 = 2_166_136_261;
|
|
||||||
for b in name.bytes() {
|
|
||||||
hash ^= u32::from(b);
|
|
||||||
hash = hash.wrapping_mul(16_777_619);
|
|
||||||
}
|
|
||||||
// Modulo of a u32 by a u16's value is guaranteed < u16::MAX, so try_from never fails.
|
// Modulo of a u32 by a u16's value is guaranteed < u16::MAX, so try_from never fails.
|
||||||
WEB_PORT_BASE + u16::try_from(hash % u32::from(WEB_PORT_RANGE)).unwrap_or(0)
|
WEB_PORT_BASE + u16::try_from(fnv1a(name) % u32::from(WEB_PORT_RANGE)).unwrap_or(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Deterministic IPv4 address for an agent inside an isolated subnet.
|
||||||
|
///
|
||||||
|
/// Parses `subnet_cidr` as `<network_ip>/<prefix_len>` (e.g.
|
||||||
|
/// `"10.42.0.0/24"`), then computes:
|
||||||
|
///
|
||||||
|
/// ```text
|
||||||
|
/// host_count = 2^(32 - prefix_len)
|
||||||
|
/// usable = host_count - 3 // skip .0 (network), .1 (gateway), .255 (broadcast)
|
||||||
|
/// offset = FNV-1a(name) % usable + 2 // .2 is the first agent slot
|
||||||
|
/// agent_ip = network_base_u32 + offset
|
||||||
|
/// ```
|
||||||
|
///
|
||||||
|
/// Returns `None` when `subnet_cidr` can't be parsed (invalid format,
|
||||||
|
/// prefix out of range, etc.) so callers can fall back gracefully.
|
||||||
|
/// Collisions are possible (birthday paradox) and the operator resolves
|
||||||
|
/// them by renaming an agent, same as for port collisions.
|
||||||
|
#[must_use]
|
||||||
|
pub fn agent_network_ip(name: &str, subnet_cidr: &str) -> Option<String> {
|
||||||
|
let (ip_str, prefix_str) = subnet_cidr.split_once('/')?;
|
||||||
|
let prefix_len: u32 = prefix_str.parse().ok()?;
|
||||||
|
if prefix_len > 30 {
|
||||||
|
// /31 and /32 have no room for agents; /30 has 1 usable slot.
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
// Parse dotted-decimal IPv4.
|
||||||
|
let octets: Vec<u8> = ip_str
|
||||||
|
.split('.')
|
||||||
|
.map(|o| o.parse::<u8>().ok())
|
||||||
|
.collect::<Option<Vec<_>>>()?;
|
||||||
|
if octets.len() != 4 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let base_u32 = u32::from_be_bytes([octets[0], octets[1], octets[2], octets[3]]);
|
||||||
|
// Mask off host bits to get the true network address.
|
||||||
|
let mask = if prefix_len == 0 { 0u32 } else { !0u32 << (32 - prefix_len) };
|
||||||
|
let network_base = base_u32 & mask;
|
||||||
|
let host_count: u32 = 1u32.checked_shl(32 - prefix_len).unwrap_or(0);
|
||||||
|
// `.0` = network, `.1` = bridge gateway, last = broadcast → 3 reserved.
|
||||||
|
let usable = host_count.saturating_sub(3);
|
||||||
|
if usable == 0 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let offset = fnv1a(name) % usable + 2; // +2: skip .0 and .1
|
||||||
|
let ip_u32 = network_base + offset;
|
||||||
|
let [a, b, c, d] = ip_u32.to_be_bytes();
|
||||||
|
Some(format!("{a}.{b}.{c}.{d}"))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use]
|
#[must_use]
|
||||||
|
|
@ -1120,8 +1174,34 @@ async fn set_nspawn_flags(
|
||||||
}
|
}
|
||||||
binds.push(BindMount { host_path: socket_dir.to_string_lossy().into_owned(), container_path: socket_dir.to_string_lossy().into_owned(), read_only: false });
|
binds.push(BindMount { host_path: socket_dir.to_string_lossy().into_owned(), container_path: socket_dir.to_string_lossy().into_owned(), read_only: false });
|
||||||
|
|
||||||
|
// Network isolation: when HIVE_NETWORK_ISOLATION=1 is set (by the
|
||||||
|
// hive-network.nix module's `isolateContainers` option), flip the
|
||||||
|
// container to a private network namespace with a veth pair attached
|
||||||
|
// to the host bridge. Applies to all containers including the manager
|
||||||
|
// (all hive-c0re<->agent comms go through bind-mounted UDS, not TCP).
|
||||||
|
let isolation = {
|
||||||
|
let isolate = std::env::var("HIVE_NETWORK_ISOLATION").ok().as_deref() == Some("1");
|
||||||
|
let bridge = std::env::var("HIVE_NETWORK_BRIDGE").unwrap_or_default();
|
||||||
|
let subnet = std::env::var("HIVE_NETWORK_SUBNET").unwrap_or_default();
|
||||||
|
if isolate && !bridge.is_empty() && !subnet.is_empty() {
|
||||||
|
let Some(agent_ip) = agent_network_ip(agent_name, &subnet) else {
|
||||||
|
tracing::warn!(
|
||||||
|
%agent_name, %subnet,
|
||||||
|
"HIVE_NETWORK_SUBNET is set but could not derive a valid IP for agent \
|
||||||
|
(bad CIDR? prefix too narrow?); skipping PRIVATE_NETWORK write to \
|
||||||
|
avoid misconfigured isolation"
|
||||||
|
);
|
||||||
|
return crate::priv_client::write_nspawn_flags(container, &binds, None).await;
|
||||||
|
};
|
||||||
|
tracing::info!(%agent_name, %agent_ip, %bridge, "network isolation: PRIVATE_NETWORK=1");
|
||||||
|
Some(hive_sh4re::priv_proto::NetworkIsolation { agent_ip, bridge })
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
// Delegate the actual conf-file rewrite to hive-priv (runs as root).
|
// Delegate the actual conf-file rewrite to hive-priv (runs as root).
|
||||||
crate::priv_client::write_nspawn_flags(container, &binds).await
|
crate::priv_client::write_nspawn_flags(container, &binds, isolation).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Execute a container operation via hive-priv and integrate with
|
/// Execute a container operation via hive-priv and integrate with
|
||||||
|
|
@ -1256,6 +1336,47 @@ mod tests {
|
||||||
assert!(tracked.contains("flake.nix"), "flake.nix not committed");
|
assert!(tracked.contains("flake.nix"), "flake.nix not committed");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn agent_network_ip_is_in_subnet() {
|
||||||
|
// Default subnet 10.42.0.0/24 — agents get .2 to .254.
|
||||||
|
let ip = agent_network_ip("alice", "10.42.0.0/24").expect("should produce an IP");
|
||||||
|
let octets: Vec<u8> = ip.split('.').map(|o| o.parse().unwrap()).collect();
|
||||||
|
assert_eq!(&octets[..3], &[10, 42, 0], "wrong /24 prefix");
|
||||||
|
assert!(octets[3] >= 2 && octets[3] <= 254, "host byte {}", octets[3]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn agent_network_ip_stable() {
|
||||||
|
// Same name + subnet must always produce the same IP.
|
||||||
|
let a = agent_network_ip("damocles", "10.42.0.0/24");
|
||||||
|
let b = agent_network_ip("damocles", "10.42.0.0/24");
|
||||||
|
assert_eq!(a, b);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn agent_network_ip_different_agents() {
|
||||||
|
// Different agent names very likely produce different IPs (not guaranteed,
|
||||||
|
// but for these two names the hashes don't collide).
|
||||||
|
let alice = agent_network_ip("alice", "10.42.0.0/24").unwrap();
|
||||||
|
let bob = agent_network_ip("bob", "10.42.0.0/24").unwrap();
|
||||||
|
assert_ne!(alice, bob, "alice and bob collide — rename one");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn agent_network_ip_different_subnet() {
|
||||||
|
let ip = agent_network_ip("alice", "192.168.5.0/24").expect("should produce an IP");
|
||||||
|
let octets: Vec<u8> = ip.split('.').map(|o| o.parse().unwrap()).collect();
|
||||||
|
assert_eq!(&octets[..3], &[192, 168, 5]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn agent_network_ip_rejects_bad_input() {
|
||||||
|
assert!(agent_network_ip("alice", "notanip/24").is_none());
|
||||||
|
assert!(agent_network_ip("alice", "10.0.0.0/33").is_none()); // prefix > 32
|
||||||
|
assert!(agent_network_ip("alice", "10.0.0.0/31").is_none()); // too small
|
||||||
|
assert!(agent_network_ip("alice", "10.0.0.0").is_none()); // no prefix
|
||||||
|
}
|
||||||
|
|
||||||
/// `setup_proposed` is idempotent: calling it on an existing repo is a
|
/// `setup_proposed` is idempotent: calling it on an existing repo is a
|
||||||
/// no-op (the fresh guard skips all writes).
|
/// no-op (the fresh guard skips all writes).
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,9 @@
|
||||||
//! a persistent connection.
|
//! a persistent connection.
|
||||||
|
|
||||||
use anyhow::{Context as _, Result, bail};
|
use anyhow::{Context as _, Result, bail};
|
||||||
use hive_sh4re::priv_proto::{BindMount, JournalOutput, PRIV_SOCK, PrivRequest, PrivResponse};
|
use hive_sh4re::priv_proto::{
|
||||||
|
BindMount, JournalOutput, NetworkIsolation, PRIV_SOCK, PrivRequest, PrivResponse,
|
||||||
|
};
|
||||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||||
use tokio::net::UnixStream;
|
use tokio::net::UnixStream;
|
||||||
|
|
||||||
|
|
@ -112,10 +114,15 @@ pub async fn read_container_journal(
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn write_nspawn_flags(container: &str, binds: &[BindMount]) -> Result<()> {
|
pub async fn write_nspawn_flags(
|
||||||
|
container: &str,
|
||||||
|
binds: &[BindMount],
|
||||||
|
isolation: Option<NetworkIsolation>,
|
||||||
|
) -> Result<()> {
|
||||||
ok(call(&PrivRequest::WriteNspawnFlags {
|
ok(call(&PrivRequest::WriteNspawnFlags {
|
||||||
container: container.to_owned(),
|
container: container.to_owned(),
|
||||||
binds: binds.to_vec(),
|
binds: binds.to_vec(),
|
||||||
|
isolation,
|
||||||
})
|
})
|
||||||
.await?)
|
.await?)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -21,8 +21,8 @@ use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use anyhow::{Context as _, Result, bail};
|
use anyhow::{Context as _, Result, bail};
|
||||||
use hive_sh4re::priv_proto::{
|
use hive_sh4re::priv_proto::{
|
||||||
AGENT_PREFIX, BindMount, JournalOutput, MANAGER_NAME, META_DIR, PRIV_SOCK, PrivRequest,
|
AGENT_PREFIX, BindMount, JournalOutput, MANAGER_NAME, META_DIR, NetworkIsolation, PRIV_SOCK,
|
||||||
PrivResponse, SIBLING_CONTAINERS,
|
PrivRequest, PrivResponse, SIBLING_CONTAINERS,
|
||||||
};
|
};
|
||||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||||
use tokio::net::{UnixListener, UnixStream};
|
use tokio::net::{UnixListener, UnixStream};
|
||||||
|
|
@ -207,13 +207,14 @@ async fn exec(req: PrivRequest) -> Result<(String, String)> {
|
||||||
PrivRequest::WriteNspawnFlags {
|
PrivRequest::WriteNspawnFlags {
|
||||||
ref container,
|
ref container,
|
||||||
ref binds,
|
ref binds,
|
||||||
|
ref isolation,
|
||||||
} => {
|
} => {
|
||||||
validate_container_system_name(container)?;
|
validate_container_system_name(container)?;
|
||||||
for bind in binds {
|
for bind in binds {
|
||||||
validate_bind_path(&bind.host_path)?;
|
validate_bind_path(&bind.host_path)?;
|
||||||
validate_bind_path(&bind.container_path)?;
|
validate_bind_path(&bind.container_path)?;
|
||||||
}
|
}
|
||||||
write_nspawn_flags(container, binds)?;
|
write_nspawn_flags(container, binds, isolation.as_ref())?;
|
||||||
Ok((String::new(), String::new()))
|
Ok((String::new(), String::new()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -477,9 +478,15 @@ fn validate_bind_path(path: &str) -> Result<()> {
|
||||||
|
|
||||||
/// Update `/etc/nixos-containers/<container>.conf`: strips network-isolation
|
/// Update `/etc/nixos-containers/<container>.conf`: strips network-isolation
|
||||||
/// vars (`PRIVATE_NETWORK`, `HOST_ADDRESS*`, `LOCAL_ADDRESS*`, `HOST_BRIDGE`,
|
/// vars (`PRIVATE_NETWORK`, `HOST_ADDRESS*`, `LOCAL_ADDRESS*`, `HOST_BRIDGE`,
|
||||||
/// `EXTRA_NSPAWN_FLAGS`), forces `PRIVATE_NETWORK=0` and blank network vars,
|
/// Update `/etc/nixos-containers/<container>.conf`: strip old network vars,
|
||||||
/// then appends `EXTRA_NSPAWN_FLAGS="<flags>"`.
|
/// write network isolation settings, then append `EXTRA_NSPAWN_FLAGS`.
|
||||||
fn write_nspawn_flags(container: &str, binds: &[BindMount]) -> Result<()> {
|
/// When `isolation` is `Some`, writes `PRIVATE_NETWORK=1` + veth wiring;
|
||||||
|
/// when `None`, writes `PRIVATE_NETWORK=0`.
|
||||||
|
fn write_nspawn_flags(
|
||||||
|
container: &str,
|
||||||
|
binds: &[BindMount],
|
||||||
|
isolation: Option<&NetworkIsolation>,
|
||||||
|
) -> Result<()> {
|
||||||
let path = format!("/etc/nixos-containers/{container}.conf");
|
let path = format!("/etc/nixos-containers/{container}.conf");
|
||||||
let original = std::fs::read_to_string(&path).with_context(|| format!("read {path}"))?;
|
let original = std::fs::read_to_string(&path).with_context(|| format!("read {path}"))?;
|
||||||
let lines: Vec<&str> = original
|
let lines: Vec<&str> = original
|
||||||
|
|
@ -499,12 +506,21 @@ fn write_nspawn_flags(container: &str, binds: &[BindMount]) -> Result<()> {
|
||||||
if !out.is_empty() {
|
if !out.is_empty() {
|
||||||
out.push('\n');
|
out.push('\n');
|
||||||
}
|
}
|
||||||
out.push_str("PRIVATE_NETWORK=0\n");
|
if let Some(iso) = isolation {
|
||||||
out.push_str("HOST_ADDRESS=\n");
|
out.push_str("PRIVATE_NETWORK=1\n");
|
||||||
out.push_str("LOCAL_ADDRESS=\n");
|
out.push_str("HOST_ADDRESS=\n");
|
||||||
out.push_str("HOST_ADDRESS6=\n");
|
out.push_str(&format!("LOCAL_ADDRESS={}\n", iso.agent_ip));
|
||||||
out.push_str("LOCAL_ADDRESS6=\n");
|
out.push_str("HOST_ADDRESS6=\n");
|
||||||
out.push_str("HOST_BRIDGE=\n");
|
out.push_str("LOCAL_ADDRESS6=\n");
|
||||||
|
out.push_str(&format!("HOST_BRIDGE={}\n", iso.bridge));
|
||||||
|
} else {
|
||||||
|
out.push_str("PRIVATE_NETWORK=0\n");
|
||||||
|
out.push_str("HOST_ADDRESS=\n");
|
||||||
|
out.push_str("LOCAL_ADDRESS=\n");
|
||||||
|
out.push_str("HOST_ADDRESS6=\n");
|
||||||
|
out.push_str("LOCAL_ADDRESS6=\n");
|
||||||
|
out.push_str("HOST_BRIDGE=\n");
|
||||||
|
}
|
||||||
let flags: Vec<String> = binds
|
let flags: Vec<String> = binds
|
||||||
.iter()
|
.iter()
|
||||||
.map(|b| {
|
.map(|b| {
|
||||||
|
|
|
||||||
|
|
@ -55,6 +55,17 @@ pub struct BindMount {
|
||||||
pub read_only: bool,
|
pub read_only: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Network isolation parameters for `WriteNspawnFlags`. When `Some`,
|
||||||
|
/// hive-priv writes `PRIVATE_NETWORK=1` + veth bridge wiring instead
|
||||||
|
/// of the default `PRIVATE_NETWORK=0`.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct NetworkIsolation {
|
||||||
|
/// Static IP address to assign to this container on the bridge subnet.
|
||||||
|
pub agent_ip: String,
|
||||||
|
/// Host bridge interface name (e.g. `hive0`).
|
||||||
|
pub bridge: String,
|
||||||
|
}
|
||||||
|
|
||||||
/// A request to the privileged helper.
|
/// A request to the privileged helper.
|
||||||
///
|
///
|
||||||
/// Wire format: one JSON object per line over `/run/hive/priv.sock`.
|
/// Wire format: one JSON object per line over `/run/hive/priv.sock`.
|
||||||
|
|
@ -128,12 +139,18 @@ pub enum PrivRequest {
|
||||||
},
|
},
|
||||||
|
|
||||||
// --- Config file writes ---
|
// --- Config file writes ---
|
||||||
/// Update `/etc/nixos-containers/<container>.conf`: strip network-isolation
|
/// Update `/etc/nixos-containers/<container>.conf`: strip old network-isolation
|
||||||
/// vars, force `PRIVATE_NETWORK=0`, and set `EXTRA_NSPAWN_FLAGS` from the
|
/// vars, write `PRIVATE_NETWORK` + bridge settings, and set `EXTRA_NSPAWN_FLAGS`
|
||||||
/// provided bind-mount list. Written by `lifecycle::set_nspawn_flags`.
|
/// from the provided bind-mount list. Written by `lifecycle::set_nspawn_flags`.
|
||||||
|
/// When `isolation` is `Some`, writes `PRIVATE_NETWORK=1` + veth wiring;
|
||||||
|
/// when `None`, writes `PRIVATE_NETWORK=0`.
|
||||||
WriteNspawnFlags {
|
WriteNspawnFlags {
|
||||||
container: String,
|
container: String,
|
||||||
binds: Vec<BindMount>,
|
binds: Vec<BindMount>,
|
||||||
|
/// `None` = host netns (PRIVATE_NETWORK=0). `Some` = private netns with
|
||||||
|
/// veth on the specified bridge (PRIVATE_NETWORK=1).
|
||||||
|
#[serde(default)]
|
||||||
|
isolation: Option<NetworkIsolation>,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// Write `/run/systemd/system/container@<container>.service.d/hyperhive-limits.conf`
|
/// Write `/run/systemd/system/container@<container>.service.d/hyperhive-limits.conf`
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue