vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(network): wire default route + bridge DNS for isolated containers

Browse source 
When isolateContainers=true, claude (and all egress) broke in every
container: agents came up with an IP but no way off the bridge subnet.

Two container-side gaps, both confirmed against nixpkgs
nixos-containers.nix:

1. No default route. hive-priv wrote HOST_ADDRESS= empty in the nspawn
   conf. nixos-container's container-side setup only installs
   `ip route add default via $HOST_ADDRESS` when HOST_ADDRESS is
   non-empty, so the container had an address but no gateway -> nothing
   off-subnet (incl. api.anthropic.com) was reachable. Fix: write
   HOST_ADDRESS=<bridge-ip>. In bridge mode the host-side address/route
   setup is skipped, so this only affects the container's default route.

2. No usable resolver. nixos-container copies the host's /etc/resolv.conf
   into the container at every start; the host resolver (e.g. 127.0.0.53)
   is unreachable from a private netns and isn't authoritative for the
   hive's own zones. Fix: hive-priv drops a marker carrying the gateway
   IP only when isolated, and a new harness-base oneshot
   (hyperhive-isolated-dns) rewrites resolv.conf to point at the bridge
   dnsmasq. Inert in shared-netns mode (no marker), so the shared
   container toplevel does the right thing in both modes.

The gateway IP is the address part of HIVE_NETWORK_SUBNET (the bridge IP
verbatim, honouring a non-.1 operator override), via a new validated
bridge_gateway_ip() helper with unit tests.

Unblocks defaulting isolation on.
This commit is contained in:
atlas 2026-06-10 20:52:12 +02:00 • committed by mara
commit d993ad2c47
4 changed files with 177 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

84
hive-c0re/src/lifecycle.rs
View file

@ -130,6 +130,42 @@ pub fn agent_network_ip(name: &str, subnet_cidr: &str) -> Option<String> {
Some(format!("{a}.{b}.{c}.{d}"))
}
/// Extract the bridge gateway IP from `HIVE_NETWORK_SUBNET`.
///
/// `HIVE_NETWORK_SUBNET` carries the host-side bridge address verbatim
/// (e.g. `10.42.0.1/24`), **not** the canonical network address — see
/// the note in `set_nspawn_flags` + `docs/network.md`. The IP part is
/// therefore the bridge IP itself: the host end of the bridge, the
/// default-route target for isolated containers, and the address the
/// hive dnsmasq resolver binds. Returns the dotted-decimal IP with the
/// `/<prefix>` stripped, or `None` if the input isn't a valid
/// `<ipv4>/<prefix>` pair.
///
/// Deliberately returns the operator-configured address verbatim rather
/// than deriving `network + 1`: an operator who sets `bridgeIp` to a
/// non-`.1` host address (e.g. `10.42.0.254`) runs the bridge + resolver
/// there, so that — not `.1` — is the real gateway.
#[must_use]
pub fn bridge_gateway_ip(subnet_cidr: &str) -> Option<String> {
let (ip_str, prefix_str) = subnet_cidr.split_once('/')?;
// Validate the prefix is a sane IPv4 CIDR length and the address is
// dotted-decimal IPv4 — same shape `agent_network_ip` accepts — so a
// malformed `HIVE_NETWORK_SUBNET` can't smuggle a bogus HOST_ADDRESS
// into the nspawn conf.
let prefix_len: u32 = prefix_str.parse().ok()?;
if prefix_len > 32 {
return None;
}
let octets: Vec<u8> = ip_str
.split('.')
.map(|o| o.parse::<u8>().ok())
.collect::<Option<Vec<_>>>()?;
if octets.len() != 4 {
return None;
}
Some(ip_str.to_owned())
}
#[must_use]
pub fn container_name(name: &str) -> String {
format!("{AGENT_PREFIX}{name}")
@ -1309,8 +1345,24 @@ async fn set_nspawn_flags(
);
return crate::priv_client::write_nspawn_flags(container, &binds, None).await;
};
tracing::info!(%agent_name, %agent_ip, %bridge, "network isolation: PRIVATE_NETWORK=1");
Some(hive_sh4re::priv_proto::NetworkIsolation { agent_ip, bridge })
let Some(gateway_ip) = bridge_gateway_ip(&subnet) else {
tracing::warn!(
%agent_name, %subnet,
"HIVE_NETWORK_SUBNET is set but the bridge gateway IP is unparseable; \
skipping PRIVATE_NETWORK write to avoid an isolated container with no \
default route or resolver"
);
return crate::priv_client::write_nspawn_flags(container, &binds, None).await;
};
tracing::info!(
%agent_name, %agent_ip, %gateway_ip, %bridge,
"network isolation: PRIVATE_NETWORK=1"
);
Some(hive_sh4re::priv_proto::NetworkIsolation {
agent_ip,
bridge,
gateway_ip,
})
} else {
None
}
@ -1546,6 +1598,34 @@ mod tests {
assert_eq!(&octets[..3], &[192, 168, 5]);
}
#[test]
fn bridge_gateway_ip_extracts_verbatim_address() {
// HIVE_NETWORK_SUBNET carries the bridge IP verbatim, not the
// canonical network — the gateway is the address before the `/`.
assert_eq!(
bridge_gateway_ip("10.42.0.1/24").as_deref(),
Some("10.42.0.1")
);
// Non-`.1` operator override: the gateway is wherever the bridge is.
assert_eq!(
bridge_gateway_ip("10.42.0.254/24").as_deref(),
Some("10.42.0.254")
);
assert_eq!(
bridge_gateway_ip("172.30.0.1/16").as_deref(),
Some("172.30.0.1")
);
}
#[test]
fn bridge_gateway_ip_rejects_bad_input() {
assert!(bridge_gateway_ip("notanip/24").is_none());
assert!(bridge_gateway_ip("10.42.0.1").is_none()); // no prefix
assert!(bridge_gateway_ip("10.42.0.1/33").is_none()); // prefix > 32
assert!(bridge_gateway_ip("10.42.0.999/24").is_none()); // octet > 255
assert!(bridge_gateway_ip("10.42.0/24").is_none()); // 3 octets
}
#[test]
fn agent_network_ip_rejects_bad_input() {
assert!(agent_network_ip("alice", "notanip/24").is_none());

48
hive-priv/src/main.rs
View file

@ -857,7 +857,15 @@ fn write_nspawn_flags(
}
if let Some(iso) = isolation {
out.push_str("PRIVATE_NETWORK=1\n");
out.push_str("HOST_ADDRESS=\n");
// HOST_ADDRESS = the bridge gateway IP. nixos-container's
// container-side setup only installs a default route
// (`ip route add default via $HOST_ADDRESS`) when HOST_ADDRESS is
// non-empty; leaving it blank gave the container an address but no
// route off the bridge subnet (no internet, no api.anthropic.com).
// In bridge mode (HOST_BRIDGE set) the host-side address/route
// setup is skipped, so this only affects the container's route —
// exactly what we want.
let _ = writeln!(out, "HOST_ADDRESS={}", iso.gateway_ip);
let _ = writeln!(out, "LOCAL_ADDRESS={}", iso.agent_ip);
out.push_str("HOST_ADDRESS6=\n");
out.push_str("LOCAL_ADDRESS6=\n");
@ -879,5 +887,41 @@ fn write_nspawn_flags(
.collect();
let flags_joined = flags.join(" ");
let _ = writeln!(out, "EXTRA_NSPAWN_FLAGS=\"{flags_joined}\"");
std::fs::write(&path, out).with_context(|| format!("write {path}"))
std::fs::write(&path, out).with_context(|| format!("write {path}"))?;
// DNS marker for the in-container resolver oneshot. nixos-container
// copies the *host's* /etc/resolv.conf into the container at every
// start (its host resolver — e.g. 127.0.0.53 — is unreachable from a
// private netns, and isn't authoritative for the hive's own zones
// anyway). The `hyperhive-isolated-dns` oneshot in harness-base.nix
// rewrites resolv.conf to point at the bridge resolver, but only when
// this marker exists; it carries the gateway IP so the container
// doesn't have to re-derive it. Written on isolate, removed otherwise,
// so the same shared container toplevel behaves correctly in both modes.
write_bridge_dns_marker(container, isolation)?;
Ok(())
}
/// Path to the in-container DNS marker (the container's own `/etc`).
fn bridge_dns_marker_path(container: &str) -> String {
format!("/var/lib/nixos-containers/{container}/etc/hyperhive-bridge-dns")
}
/// Write (isolated) or remove (host-netns) the bridge-DNS marker the
/// `hyperhive-isolated-dns` oneshot keys off. The marker file contains
/// just the gateway IP. Best-effort on removal (absence is the goal).
fn write_bridge_dns_marker(container: &str, isolation: Option<&NetworkIsolation>) -> Result<()> {
let path = bridge_dns_marker_path(container);
match isolation {
Some(iso) => {
std::fs::write(&path, format!("{}\n", iso.gateway_ip))
.with_context(|| format!("write bridge-DNS marker {path}"))?;
}
None => match std::fs::remove_file(&path) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(e).with_context(|| format!("remove bridge-DNS marker {path}")),
},
}
Ok(())
}

9
hive-sh4re/src/priv_proto.rs
View file

@ -105,6 +105,15 @@ pub struct NetworkIsolation {
pub agent_ip: String,
/// Host bridge interface name (e.g. `hive0`).
pub bridge: String,
/// Bridge gateway IP (the host-side bridge address, e.g. `10.42.0.1`).
/// Written as `HOST_ADDRESS=` in the nspawn conf so nixos-container's
/// container-side setup installs a default route (`default via <gw>`):
/// without it the container comes up with an address but no route off
/// the bridge subnet — no internet, no `api.anthropic.com`. The same IP
/// runs the hive dnsmasq resolver, so it's also written into the
/// container's `/etc/resolv.conf` (see the isolated-DNS oneshot in
/// `harness-base.nix`, gated on the marker hive-priv drops).
pub gateway_ip: String,
}
/// A request to the privileged helper.

40
nix/templates/harness-base.nix
View file

@ -1025,6 +1025,46 @@ in
# One-shot: tea config.yml from the seeded forge token. Shape
# contract (always exit 0, no set -e, skip-silently, re-runnable):
# docs/conventions.md::Best-effort oneshot services.
# Point resolv.conf at the hive bridge resolver when the container is
# network-isolated. nixos-container copies the *host's* /etc/resolv.conf
# into the container at every start — but the host resolver (e.g.
# 127.0.0.53) is unreachable from a private netns and isn't
# authoritative for the hive's own zones (forge.<domain> etc.). The
# bridge dnsmasq (gateway IP) is. hive-priv drops the marker
# `/etc/hyperhive-bridge-dns` (containing the gateway IP) only when
# isolation is on, so this oneshot is inert in shared-netns mode — the
# same shared container toplevel does the right thing in both modes.
# Ordered before the first DNS consumer (tea-login) and the network
# targets so name resolution works for the very first turn.
systemd.services.hyperhive-isolated-dns = {
description = "point resolv.conf at the hive bridge resolver (isolated containers)";
wantedBy = [ "multi-user.target" ];
after = [ "local-fs.target" ];
before = [
"network-online.target"
"tea-login.service"
];
unitConfig.ConditionPathExists = "/etc/hyperhive-bridge-dns";
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
path = [ pkgs.coreutils ];
script = ''
set -eu
gw=$(tr -d '[:space:]' < /etc/hyperhive-bridge-dns)
if [ -z "$gw" ]; then
echo "hyperhive-isolated-dns: empty marker; leaving resolv.conf as-is"
exit 0
fi
# resolv.conf is a regular file copied from the host by
# nixos-container; replace it (rm first in case it's a symlink).
rm -f /etc/resolv.conf
printf 'nameserver %s\n' "$gw" > /etc/resolv.conf
echo "hyperhive-isolated-dns: resolv.conf -> nameserver $gw"
'';
};
systemd.services.tea-login = {
description = "configure tea CLI from hive-forge token (best-effort)";
wantedBy = [ "multi-user.target" ];