vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

run opted-in agents in a user namespace with a dedicated data owner

Browse source
This commit is contained in:
atlas 2026-11-01 13:50:53 +01:00 • committed by mara
commit bfdf603d12
9 changed files with 828 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

105
hive-priv/src/container_config.rs
View file

@ -50,6 +50,9 @@ pub(crate) fn handle_write_nspawn_flags(
);
false
});
// After the salvage, so the mount points land in the root the start will
// snapshot.
let userns = crate::userns::activate(container, agent, binds)?;
write_nspawn_flags(
container,
binds,
@ -57,6 +60,7 @@ pub(crate) fn handle_write_nspawn_flags(
load_credentials,
&hive_facts,
ephemeral,
userns,
)?;
Ok((String::new(), String::new()))
}
@ -364,6 +368,7 @@ fn write_nspawn_flags(
load_credentials: &[CredentialMount],
hive_facts: &str,
ephemeral: bool,
userns: bool,
) -> Result<()> {
use std::fmt::Write as _;
use std::os::unix::fs::MetadataExt as _;
@ -371,21 +376,13 @@ fn write_nspawn_flags(
let original = std::fs::read_to_string(&path).with_context(|| format!("read {path}"))?;
let lines: Vec<&str> = original
.lines()
.filter(|line| {
let t = line.trim_start();
!t.starts_with("EXTRA_NSPAWN_FLAGS=")
&& !t.starts_with("PRIVATE_NETWORK=")
&& !t.starts_with("HOST_ADDRESS=")
&& !t.starts_with("LOCAL_ADDRESS=")
&& !t.starts_with("HOST_ADDRESS6=")
&& !t.starts_with("LOCAL_ADDRESS6=")
&& !t.starts_with("HOST_BRIDGE=")
})
.filter(|line| !is_owned_var(line))
.collect();
let mut out = lines.join("\n");
if !out.is_empty() {
out.push('\n');
}
out.push_str(private_users_line(userns));
{
let iso = isolation;
out.push_str("PRIVATE_NETWORK=1\n");
@ -406,7 +403,7 @@ fn write_nspawn_flags(
out.push_str("LOCAL_ADDRESS6=\n");
let _ = writeln!(out, "HOST_BRIDGE={}", iso.bridge);
}
let flags_joined = nspawn_flags(binds, load_credentials, ephemeral).join(" ");
let flags_joined = nspawn_flags(binds, load_credentials, ephemeral, userns).join(" ");
let _ = writeln!(out, "EXTRA_NSPAWN_FLAGS=\"{flags_joined}\"");
// Keeps the owner and mode `nixos-container` created the file with; this
// helper only rewrites its lines.
@ -432,24 +429,57 @@ fn write_nspawn_flags(
Ok(())
}
/// Whether a line of the container conf is one `write_nspawn_flags` writes
/// itself, and so drops before writing it again. Dropped for every container,
/// not only those it writes it for: a `PRIVATE_USERS=pick` left behind would
/// keep a container namespaced after it left the list.
fn is_owned_var(line: &str) -> bool {
let t = line.trim_start();
[
"EXTRA_NSPAWN_FLAGS=",
"PRIVATE_NETWORK=",
"PRIVATE_USERS=",
"HOST_ADDRESS=",
"LOCAL_ADDRESS=",
"HOST_ADDRESS6=",
"LOCAL_ADDRESS6=",
"HOST_BRIDGE=",
]
.iter()
.any(|var| t.starts_with(var))
}
/// The conf line that makes nixos-container pass `--private-users=pick`.
fn private_users_line(userns: bool) -> &'static str {
if userns { "PRIVATE_USERS=pick\n" } else { "" }
}
/// The flags `EXTRA_NSPAWN_FLAGS` carries, in the order nspawn needs them.
///
/// `--ephemeral` runs the container on a snapshot of its root and drops the
/// snapshot at stop; `crate::rootfs` has made the root fit for that before
/// the flag is asked for. The git overlays and credentials follow the binds
/// they refer to.
///
/// `userns` mounts every bind `owneridmap` and idmaps the root instead of
/// chowning it, so a different `pick` range never walks the tree.
fn nspawn_flags(
binds: &[BindMount],
load_credentials: &[CredentialMount],
ephemeral: bool,
userns: bool,
) -> Vec<String> {
let mut flags: Vec<String> = ephemeral
.then(|| "--ephemeral".to_owned())
.into_iter()
.collect();
if userns {
flags.push("--private-users-ownership=map".to_owned());
}
let idmap = if userns { ":owneridmap" } else { "" };
flags.extend(binds.iter().map(|b| {
let flag = if b.read_only { "--bind-ro" } else { "--bind" };
format!("{flag}={}:{}", b.host_path, b.container_path)
format!("{flag}={}:{}{idmap}", b.host_path, b.container_path)
}));
flags.extend(git_overlay_flags(binds));
// Credential forwarding: nspawn loads each host secret into the
@ -621,9 +651,9 @@ pub(crate) fn write_container_etc_file_in(rootfs: &Path, name: &str, contents: &
mod tests {
use super::{
BindMount, HiveFacts, NetworkIsolation, git_overlay_flags, handle_write_nspawn_flags,
hive_priv_exe, limits_dropin_body, nspawn_flags, publish_file, remove_service_dropin,
render_hive_facts, rootfs_check_command, validate_container_system_name,
write_bridge_dns_marker_in, write_resource_limits,
hive_priv_exe, is_owned_var, limits_dropin_body, nspawn_flags, private_users_line,
publish_file, remove_service_dropin, render_hive_facts, rootfs_check_command,
validate_container_system_name, write_bridge_dns_marker_in, write_resource_limits,
};
use crate::shared::test_support::{entries, scratch};
@ -1072,14 +1102,55 @@ mod tests {
#[test]
fn ephemeral_is_passed_only_when_asked() {
let binds = [bind("/agents/atlas/work")];
let with = nspawn_flags(&binds, &[], true);
let with = nspawn_flags(&binds, &[], true, false);
assert_eq!(with[0], "--ephemeral", "{with:?}");
assert!(with.iter().any(|f| f.starts_with("--bind-ro=")), "{with:?}");
let without = nspawn_flags(&binds, &[], false);
let without = nspawn_flags(&binds, &[], false, false);
assert!(!without.iter().any(|f| f == "--ephemeral"), "{without:?}");
assert_eq!(without.len() + 1, with.len());
}
/// Every bind, read-only ones included, is mounted `owneridmap` for a
/// namespaced agent and none is for any other; the root is idmapped, not
/// chowned. The `false` arm is the control that the lookup finds the binds.
#[test]
fn binds_are_owneridmap_only_for_a_namespaced_agent() {
let binds = [bind("/knowledge"), bind("/agents/a/config")];
let plain = nspawn_flags(&binds, &[], false, false);
assert!(
plain.iter().any(|f| f.starts_with("--bind-ro=")),
"{plain:?}"
);
assert!(
plain
.iter()
.all(|f| !f.contains("idmap") && !f.contains("ownership")),
"{plain:?}"
);
let userns = nspawn_flags(&binds, &[], false, true);
assert!(
userns.contains(&"--private-users-ownership=map".to_owned()),
"{userns:?}"
);
let mounts: Vec<&String> = userns.iter().filter(|f| f.starts_with("--bind")).collect();
assert_eq!(mounts.len(), 2, "{userns:?}");
assert!(
mounts.iter().all(|f| f.ends_with(":owneridmap")),
"{userns:?}"
);
}
/// `PRIVATE_USERS=` is dropped from the conf whatever the agent now is, so
/// leaving the list cannot leave the line behind. `AUTO_START=` is the
/// control: a line that is not ours stays.
#[test]
fn the_private_users_line_is_ours_to_drop_and_to_write() {
assert!(is_owned_var("PRIVATE_USERS=pick"));
assert!(!is_owned_var("AUTO_START=1"));
assert_eq!(private_users_line(true), "PRIVATE_USERS=pick\n");
assert_eq!(private_users_line(false), "");
}
/// The swarm's forge and matrix containers are not the hive's: every
/// request that names a container as a string refuses them before
/// touching the filesystem. Control: an agent container name passes.

9
hive-priv/src/files.rs
View file

@ -13,7 +13,7 @@ use hive_priv_sock::{AGENT_STATE_ROOT, PAUSED_MARKER_FILE};
use crate::shared::{StagedFile, ensure_plain_filename, open_dir, validate_agent_name};
/// Root of the per-agent unix-socket dirs on the host.
const SOCKET_DIR_ROOT: &str = "/run/hive-agent";
pub(crate) const SOCKET_DIR_ROOT: &str = "/run/hive-agent";
/// `SetAgentPaused`.
pub(crate) fn exec_set_agent_paused(agent_name: &str, paused: bool) -> Result<(String, String)> {
@ -155,6 +155,13 @@ pub(crate) fn remove_legacy_tmpfiles() -> Result<()> {
/// `EnsureAgentSocketDir` — create `/run/hive-agent/<name>` if missing.
pub(crate) fn ensure_agent_socket_dir(name: &str) -> Result<(String, String)> {
ensure_socket_dir_in(Path::new(SOCKET_DIR_ROOT), name)?;
// A namespaced agent's activation cannot chown the dir through the id
// mapping, so the data owner, which is the agent user inside, gets it here.
if let Some(owner) = crate::userns::owner_for_agent(name)? {
let path = Path::new(SOCKET_DIR_ROOT).join(name);
std::os::unix::fs::lchown(&path, Some(owner), Some(owner))
.with_context(|| format!("chown {}", path.display()))?;
}
Ok((String::new(), String::new()))
}

1
hive-priv/src/main.rs
View file

@ -37,6 +37,7 @@ mod rootfs;
mod salvage_io;
mod shared;
mod storage;
mod userns;
use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd, RawFd};

4
hive-priv/src/rootfs.rs
View file

@ -512,7 +512,7 @@ fn carry_nix_state(
/// owned by `uid:gid`), or `None` when `base` or an existing component is not a
/// real directory. The container could have replaced any of them by a link, and
/// this runs as root.
fn own_dir(base: &Path, parts: &[&str], uid: u32, gid: u32) -> Result<Option<PathBuf>> {
pub(crate) fn own_dir(base: &Path, parts: &[&str], uid: u32, gid: u32) -> Result<Option<PathBuf>> {
let mut path = base.to_path_buf();
if !std::fs::symlink_metadata(&path).is_ok_and(|m| m.is_dir()) {
return Ok(None);
@ -711,7 +711,7 @@ fn is_regular(path: &Path) -> bool {
/// `systemctl is-active` answers `inactive` or `failed` for a unit that is not
/// running; anything else (`active`, `activating`, `deactivating`) means the
/// root is, or is about to be, in use.
fn unit_is_stopped(container: &str) -> Result<bool> {
pub(crate) fn unit_is_stopped(container: &str) -> Result<bool> {
let out = std::process::Command::new("systemctl")
.args(["is-active", &format!("container@{container}.service")])
.output()

496
hive-priv/src/userns.rs Normal file
View file

@ -0,0 +1,496 @@
//! Agents that run in their own user namespace.
//!
//! The operator lists them in `services.hyperhive.userns.agents` and names the
//! one host uid:gid that owns their data on disk (`dataOwner`); the unit's
//! environment carries both, so only root decides who is namespaced. A listed
//! agent starts with `--private-users=pick`, a root idmapped rather than
//! chowned, and every bind mounted `owneridmap`: the owner of the bind's
//! target inside the container, the agent user, is the data owner on disk.
//!
//! Getting there moves the agent's data once. [`activate`] chowns its `state`,
//! `harness`, `work` and `claude` dirs to the data owner, never touching
//! anything else, and records that in a marker only root can write; the next
//! call sees the marker and does nothing. `owneridmap` needs the target
//! directory to be owned by the agent when nspawn mounts, and nspawn creates a
//! missing one as root in a throwaway snapshot, so the mount points are made
//! in the template root too. The agent's socket dir lives on a tmpfs and goes
//! to the data owner at every start instead ([`owner_for_agent`]).
use std::os::unix::fs::{MetadataExt as _, lchown};
use std::path::{Path, PathBuf};
use anyhow::{Context as _, Result, bail};
use hive_priv_sock::{AGENT_STATE_ROOT, BindMount};
use crate::files::SOCKET_DIR_ROOT;
const AGENTS_ENV: &str = "HIVE_PRIV_USERNS_AGENTS";
const DATA_OWNER_ENV: &str = "HIVE_PRIV_USERNS_DATA_OWNER";
/// One `<owner> <uid> <gid>` file per agent: the data owner its data was moved
/// to, and the agent's own ids inside the container.
const MARKER_ROOT: &str = "/var/lib/hive-priv/userns";
/// NixOS gives a normal user, which the agent is, an id from 1000 up.
const MIN_AGENT_UID: u32 = 1000;
/// The dirs under `<AGENT_STATE_ROOT>/<agent>` the agent writes into. `config`
/// is not one of them: hive-core writes it and the agent only reads it.
const DATA_DIRS: [&str; 4] = ["state", "harness", "work", "claude"];
/// Where one call looks and writes, and what it was told.
pub(crate) struct Env<'a> {
pub(crate) state_root: &'a Path,
/// The agent's socket dir, a bind source like the data dirs.
pub(crate) socket_dir: &'a Path,
pub(crate) marker_root: &'a Path,
pub(crate) rootfs: &'a Path,
/// The data owner when the agent is listed, else `None`.
pub(crate) owner: Option<u32>,
pub(crate) stopped: bool,
/// Lowest uid taken for an agent's own: below it a dir belongs to root or
/// to a system user such as hive-core.
pub(crate) min_uid: u32,
pub(crate) chown: &'a dyn Fn(&Path, u32, u32) -> std::io::Result<()>,
}
/// The data owner for `agent` under the unit's environment: `None` when the
/// agent is not listed.
///
/// # Errors
/// The agent is listed and the data owner is missing or not a plain uid.
pub(crate) fn owner_for(
agents: Option<&str>,
owner: Option<&str>,
agent: &str,
) -> Result<Option<u32>> {
if !agents.is_some_and(|list| list.split_whitespace().any(|a| a == agent)) {
return Ok(None);
}
let owner = owner.with_context(|| {
format!("{agent} is listed in {AGENTS_ENV}, but {DATA_OWNER_ENV} is not set")
})?;
let owner = owner
.trim()
.parse()
.with_context(|| format!("{DATA_OWNER_ENV}={owner:?} is not a uid"))?;
Ok(Some(owner))
}
/// [`owner_for`] under the unit's environment.
///
/// # Errors
/// As [`owner_for`].
pub(crate) fn owner_for_agent(agent: &str) -> Result<Option<u32>> {
owner_for(
std::env::var(AGENTS_ENV).ok().as_deref(),
std::env::var(DATA_OWNER_ENV).ok().as_deref(),
agent,
)
}
/// Make `agent`'s next start of `container` namespaced when it is listed, and
/// say whether it is. `binds` are the ones the start will mount.
///
/// A container that is running, and whose data has not been moved yet, keeps
/// its current start: the move waits for a stopped container.
///
/// # Errors
/// The environment is inconsistent, the data owner changed since the data
/// moved, the agent was dropped from the list after it moved, or a chown or
/// mkdir fails.
pub(crate) fn activate(container: &str, agent: &str, binds: &[BindMount]) -> Result<bool> {
let owner = owner_for_agent(agent)?;
let rootfs = crate::container_config::container_rootfs(container);
let stopped = crate::rootfs::unit_is_stopped(container)?;
activate_in(
&Env {
state_root: Path::new(AGENT_STATE_ROOT),
socket_dir: &Path::new(SOCKET_DIR_ROOT).join(agent),
marker_root: Path::new(MARKER_ROOT),
rootfs: &rootfs,
owner,
stopped,
min_uid: MIN_AGENT_UID,
chown: &|path, uid, gid| lchown(path, Some(uid), Some(gid)),
},
agent,
binds,
)
}
/// [`activate`] over an explicit [`Env`].
pub(crate) fn activate_in(env: &Env<'_>, agent: &str, binds: &[BindMount]) -> Result<bool> {
crate::shared::validate_agent_name(agent)?;
let marker = env.marker_root.join(agent);
let moved = read_marker(&marker)?;
let Some(owner) = env.owner else {
if moved.is_some() {
bail!(
"{agent}: its data was moved to the userns data owner and it is no longer \
listed; list it again, or chown its dirs back and remove {}",
marker.display()
);
}
return Ok(false);
};
let ids = match moved {
Some((was, uid, gid)) if was == owner => (uid, gid),
Some((was, ..)) => {
bail!("{agent}: its data is owned by {was}, not the configured data owner {owner}")
}
None if !env.stopped => return Ok(false),
None => {
let ids = move_data(env, agent, owner)?;
write_marker(&marker, owner, ids)?;
ids
}
};
if env.stopped {
mount_points(env, agent, binds, ids)?;
}
Ok(true)
}
/// `<owner> <uid> <gid>` from the marker, `None` when there is none.
fn read_marker(path: &Path) -> Result<Option<(u32, u32, u32)>> {
let text = match std::fs::read_to_string(path) {
Ok(text) => text,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(e) => return Err(e).with_context(|| format!("read {}", path.display())),
};
let ids: Vec<u32> = text
.split_whitespace()
.map(str::parse)
.collect::<Result<_, _>>()
.with_context(|| format!("parse {}", path.display()))?;
let [owner, uid, gid] = ids[..] else {
bail!("{} does not hold `<owner> <uid> <gid>`", path.display());
};
Ok(Some((owner, uid, gid)))
}
fn write_marker(path: &Path, owner: u32, (uid, gid): (u32, u32)) -> Result<()> {
if let Some(dir) = path.parent() {
std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
}
std::fs::write(path, format!("{owner} {uid} {gid}\n"))
.with_context(|| format!("write {}", path.display()))
}
/// Chown the agent's data dirs to `owner:owner`; returns the agent's own uid
/// and gid, read from its `state` dir before the chown.
///
/// Only the [`DATA_DIRS`] under `<state_root>/<agent>` are touched, each only
/// if it is a real directory. The walk does not follow links or cross into
/// another filesystem, and the container is stopped while it runs.
fn move_data(env: &Env<'_>, agent: &str, owner: u32) -> Result<(u32, u32)> {
let base = env.state_root.join(agent);
let state = base.join("state");
let meta =
std::fs::symlink_metadata(&state).with_context(|| format!("stat {}", state.display()))?;
let (uid, gid) = (meta.uid(), meta.gid());
if !meta.is_dir() || uid < env.min_uid || uid == owner {
bail!(
"{agent}: {} is not a directory owned by a container user (uid {uid}); start the \
agent once without the namespace so its user exists",
state.display()
);
}
for sub in DATA_DIRS {
let dir = base.join(sub);
match std::fs::symlink_metadata(&dir) {
Ok(m) if m.is_dir() => chown_tree(env, &dir, m.dev(), owner)?,
Ok(_) => bail!("{} is not a directory", dir.display()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(e).with_context(|| format!("stat {}", dir.display())),
}
}
Ok((uid, gid))
}
/// Chown everything under `top`, `top` included, to `owner:owner`.
fn chown_tree(env: &Env<'_>, top: &Path, dev: u64, owner: u32) -> Result<()> {
let mut dirs: Vec<PathBuf> = vec![top.to_path_buf()];
while let Some(dir) = dirs.pop() {
(env.chown)(&dir, owner, owner).with_context(|| format!("chown {}", dir.display()))?;
for entry in std::fs::read_dir(&dir).with_context(|| format!("read {}", dir.display()))? {
let entry = entry?;
let meta = entry.metadata()?;
if meta.dev() != dev {
continue;
}
if meta.is_dir() {
dirs.push(entry.path());
} else {
(env.chown)(&entry.path(), owner, owner)
.with_context(|| format!("chown {}", entry.path().display()))?;
}
}
}
Ok(())
}
/// Create, in the template root, the mount point of each bind whose source is
/// one of the agent's data dirs or its socket dir, owned by the agent's ids
/// inside the container.
fn mount_points(
env: &Env<'_>,
agent: &str,
binds: &[BindMount],
(uid, gid): (u32, u32),
) -> Result<()> {
let sources: Vec<PathBuf> = DATA_DIRS
.iter()
.map(|sub| env.state_root.join(agent).join(sub))
.chain([env.socket_dir.to_path_buf()])
.collect();
for bind in binds.iter().filter(|b| {
sources
.iter()
.any(|s| s.as_os_str() == b.host_path.as_str())
}) {
let parts: Vec<&str> = bind
.container_path
.split('/')
.filter(|p| !p.is_empty())
.collect();
let Some(dir) = crate::rootfs::own_dir(env.rootfs, &parts, uid, gid)? else {
bail!(
"{}: a link or file where the mount point {} goes",
env.rootfs.display(),
bind.container_path
);
};
(env.chown)(&dir, uid, gid).with_context(|| format!("chown {}", dir.display()))?;
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::{Env, activate_in, owner_for};
use crate::shared::test_support::scratch;
use hive_priv_sock::BindMount;
use std::cell::RefCell;
use std::os::unix::fs::MetadataExt as _;
use std::path::{Path, PathBuf};
/// Not the uid the tests run as, so a chown is a change.
const OWNER: u32 = 60000;
fn write(path: &Path) {
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(path, "x").unwrap();
}
/// `<root>/agents/a` with data in each dir, `config` beside them, a link
/// out of `work`, and `outside/o` next to the tree. Returns the root.
fn tree() -> PathBuf {
let root = scratch();
let a = root.join("agents/a");
for f in ["state/f", "harness/d/g", "work/x", "claude/c", "config/cfg"] {
write(&a.join(f));
}
write(&root.join("outside/o"));
std::os::unix::fs::symlink(root.join("outside"), a.join("work/link")).unwrap();
root
}
/// Run [`activate_in`] on `root` and return its answer with every path it
/// asked to chown, relative to `root`.
fn run(
root: &Path,
owner: Option<u32>,
stopped: bool,
agent: &str,
binds: &[BindMount],
) -> (anyhow::Result<bool>, Vec<String>) {
let seen = RefCell::new(Vec::new());
let chown = |p: &Path, _: u32, _: u32| {
seen.borrow_mut()
.push(p.strip_prefix(root).unwrap().display().to_string());
Ok(())
};
let (state_root, rootfs) = (root.join("agents"), root.join("rootfs"));
std::fs::create_dir_all(&rootfs).unwrap();
let env = Env {
state_root: &state_root,
socket_dir: &root.join("run/a"),
marker_root: &root.join("marker"),
rootfs: &rootfs,
owner,
stopped,
min_uid: 0,
chown: &chown,
};
let answer = activate_in(&env, agent, binds);
(answer, seen.into_inner())
}
fn bind(root: &Path, sub: &str, target: &str) -> BindMount {
BindMount {
host_path: root.join("agents/a").join(sub).display().to_string(),
container_path: target.to_owned(),
read_only: false,
}
}
#[test]
fn only_listed_agents_have_an_owner() {
let owner = |agents, owner| owner_for(agents, owner, "a");
assert_eq!(owner(Some("b a"), Some("60000")).unwrap(), Some(60000));
assert_eq!(owner(Some("b c"), Some("60000")).unwrap(), None);
assert_eq!(owner(None, None).unwrap(), None);
assert!(owner(Some("a"), None).is_err(), "listed without an owner");
assert!(owner(Some("a"), Some("root")).is_err(), "not a uid");
}
/// The move chowns the four data dirs and what is in them and nothing
/// else: not `config`, not what a link in `work` points at.
#[test]
fn the_move_stays_inside_the_agents_data_dirs() {
let root = tree();
let (answer, mut seen) = run(&root, Some(OWNER), true, "a", &[]);
assert!(answer.unwrap());
seen.sort();
let want = [
"agents/a/claude",
"agents/a/claude/c",
"agents/a/harness",
"agents/a/harness/d",
"agents/a/harness/d/g",
"agents/a/state",
"agents/a/state/f",
"agents/a/work",
"agents/a/work/link",
"agents/a/work/x",
];
assert_eq!(seen, want);
assert!(root.join("marker/a").exists());
}
/// Once the marker is there a second call chowns nothing. The first call
/// in `the_move_stays_inside_the_agents_data_dirs` is the control that
/// the same setup does chown.
#[test]
fn the_move_runs_once() {
let root = tree();
assert!(run(&root, Some(OWNER), true, "a", &[]).0.unwrap());
let (answer, seen) = run(&root, Some(OWNER), true, "a", &[]);
assert!(answer.unwrap());
assert!(seen.is_empty(), "{seen:?}");
}
/// A running container's data is left alone and the start stays as it is.
#[test]
fn a_running_container_is_not_moved() {
let root = tree();
let (answer, seen) = run(&root, Some(OWNER), false, "a", &[]);
assert!(!answer.unwrap());
assert!(seen.is_empty(), "{seen:?}");
assert!(!root.join("marker/a").exists());
}
#[test]
fn an_unlisted_agent_is_left_alone_and_a_moved_one_is_refused() {
let root = tree();
let (answer, seen) = run(&root, None, true, "a", &[]);
assert!(!answer.unwrap() && seen.is_empty());
assert!(
run(&root, Some(OWNER), true, "a", &[]).0.unwrap(),
"control"
);
assert!(run(&root, None, true, "a", &[]).0.is_err(), "left the list");
assert!(
run(&root, Some(OWNER + 1), true, "a", &[]).0.is_err(),
"data owner changed"
);
}
/// A data dir that is a link is refused before anything past it is touched.
#[test]
fn a_linked_data_dir_is_refused() {
let root = tree();
let work = root.join("agents/a/work");
std::fs::remove_dir_all(&work).unwrap();
std::os::unix::fs::symlink(root.join("outside"), &work).unwrap();
let (answer, seen) = run(&root, Some(OWNER), true, "a", &[]);
assert!(answer.is_err());
assert!(seen.iter().all(|p| !p.starts_with("outside")), "{seen:?}");
assert!(!root.join("marker/a").exists());
}
#[test]
fn a_path_is_not_an_agent_name() {
let root = tree();
let (answer, seen) = run(&root, Some(OWNER), true, "../outside", &[]);
assert!(answer.is_err() && seen.is_empty());
}
/// The agent's own uid is read off its `state` dir, and root's or the
/// data owner's is refused.
#[test]
fn the_state_dir_has_to_be_owned_by_a_container_user() {
let root = tree();
let euid = std::fs::metadata(root.join("agents/a/state"))
.unwrap()
.uid();
let seen = RefCell::new(Vec::new());
let chown = |p: &Path, _: u32, _: u32| {
seen.borrow_mut().push(p.to_path_buf());
Ok(())
};
let (state_root, rootfs) = (root.join("agents"), root.join("rootfs"));
let mut env = Env {
state_root: &state_root,
socket_dir: &root.join("run/a"),
marker_root: &root.join("marker"),
rootfs: &rootfs,
owner: Some(OWNER),
stopped: true,
min_uid: euid + 1,
chown: &chown,
};
assert!(activate_in(&env, "a", &[]).is_err(), "below the minimum");
assert!(seen.borrow().is_empty());
env.min_uid = euid;
env.owner = Some(euid);
assert!(activate_in(&env, "a", &[]).is_err(), "already the owner");
env.owner = Some(OWNER);
assert!(activate_in(&env, "a", &[]).unwrap(), "control");
}
/// A mount point is made for a bind of the agent's own data dir, and for
/// no other bind; a link in the way is refused.
#[test]
fn mount_points_are_made_for_the_agents_data_binds_only() {
let root = tree();
let binds = [
bind(&root, "state", "/agents/a/state"),
BindMount {
host_path: root.join("outside").display().to_string(),
container_path: "/agents/a/stray".to_owned(),
read_only: false,
},
BindMount {
host_path: root.join("run/a").display().to_string(),
container_path: "/run/hive-agent/a".to_owned(),
read_only: false,
},
];
let (answer, seen) = run(&root, Some(OWNER), true, "a", &binds);
assert!(answer.unwrap());
assert!(root.join("rootfs/agents/a/state").is_dir());
assert!(root.join("rootfs/run/hive-agent/a").is_dir(), "socket dir");
assert!(!root.join("rootfs/agents/a/stray").exists());
assert!(
seen.contains(&"rootfs/agents/a/state".to_owned()),
"{seen:?}"
);
std::fs::remove_dir_all(root.join("rootfs/agents")).unwrap();
std::os::unix::fs::symlink(root.join("outside"), root.join("rootfs/agents")).unwrap();
assert!(run(&root, Some(OWNER), true, "a", &binds).0.is_err());
}
}

54
nix/host-modules/hive-priv.nix
View file

@ -10,6 +10,8 @@
let
controllerCfg = config.services.hyperhive.deploy.hive-controller;
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
userns = config.services.hyperhive.userns;
dataOwnerProblems = import ./lib/userns-data-owner.nix { inherit lib; };
in
{
# The platform containers run their warn-only root check by this path
@ -24,12 +26,60 @@ in
}
];
options.services.hyperhive.userns = {
agents = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
example = [ "alice" ];
description = ''
Agents that run in their own user namespace (`--private-users=pick`,
root idmapped). Every hive bind into such a container is mounted
`owneridmap`, so its data on disk belongs to `dataOwner`, which the
agent user is inside. Listing an agent moves its `state`, `harness`,
`work` and `claude` dirs to `dataOwner` once, at its next start with
the container stopped. Take the agent off the list only after chowning
those dirs back and removing `/var/lib/hive-priv/userns/<agent>`;
until then it will not start.
'';
};
dataOwner = lib.mkOption {
type = lib.types.ints.u32;
default = 60000;
description = ''
The host uid and gid that own a namespaced agent's data on disk. One
number for both: nspawn copies the uid map for gids, so the gid
cannot differ. It must be no host user or group, outside the normal and
system users (below 30000) and the dynamic users (61184 and above), and
below the range `pick` allocates from (524288 and above), so no
container's range covers it. Changing it after an agent has moved is
refused at that agent's next start.
'';
};
};
config = lib.mkIf config.services.hyperhive.deploy.hive-controller.enable {
# The parent of every agent socket dir. hive-priv is its only writer
# (`EnsureAgentSocketDir`), and its unit lists it in `ReadWritePaths`,
# which fails the unit when the path is missing.
systemd.tmpfiles.rules = [ "d /run/hive-agent 0755 root root - -" ];
assertions = [
(
let
problems = dataOwnerProblems {
owner = userns.dataOwner;
uids = lib.filter (uid: uid != null) (lib.mapAttrsToList (_: u: u.uid) config.users.users);
gids = lib.filter (gid: gid != null) (lib.mapAttrsToList (_: g: g.gid) config.users.groups);
};
in
{
assertion = userns.agents == [ ] || problems == [ ];
message = "services.hyperhive.userns.dataOwner: ${lib.concatStringsSep "; " problems}";
}
)
];
# Socket unit for hive-priv — the narrow root helper that executes
# privileged operations on behalf of hive-c0re. Systemd creates and
# holds `/run/hive/priv.sock` before the first connection arrives.
@ -93,6 +143,10 @@ in
# routing so the closure `RealiseClosure` downloads is in the same
# store the container is then pointed at.
NIX_REMOTE = "daemon";
# Who is namespaced, and the owner their data moves to: set by the
# operator in the unit, so only root decides.
HIVE_PRIV_USERNS_AGENTS = lib.concatStringsSep " " userns.agents;
HIVE_PRIV_USERNS_DATA_OWNER = toString userns.dataOwner;
};
serviceConfig = {
ExecStart = "${controllerCfg.package}/bin/hive-priv";

17
nix/host-modules/lib/userns-data-owner.nix Normal file
View file

@ -0,0 +1,17 @@
# What is wrong with a uid:gid chosen as the on-disk owner of a user-namespaced
# agent's data. Empty when it is fine.
{ lib }:
{
owner,
# Every uid and gid the host defines, from `users.users` and `users.groups`.
uids,
gids,
}:
lib.optional (
owner < 30000
) "${toString owner} is in the range NixOS gives normal users and system users (below 30000)"
++
lib.optional (owner >= 61184)
"${toString owner} is in the range systemd allocates dynamic users from (61184 and above), or is `nobody`"
++ lib.optional (builtins.elem owner uids) "${toString owner} is the uid of a user the host defines"
++ lib.optional (builtins.elem owner gids) "${toString owner} is the gid of a group the host defines"

52
nix/module-eval/core-toggle.nix
View file

@ -137,6 +137,58 @@ let
remainsAfterExit = u: u.serviceConfig.RemainAfterExit or false;
cases = [
{
# The control is the default on a host that defines the ids below:
# it gets past every arm, so each refusal is the arm's own.
name = "the userns data owner is refused where it collides with a host id or a reserved range";
ok =
let
problems =
owner:
import ../host-modules/lib/userns-data-owner.nix { inherit lib; } {
inherit owner;
uids = [
0
60001
];
gids = [
100
60002
];
};
in
problems 60000 == [ ]
&&
map (o: builtins.length (problems o)) [
1000
29999
61184
65534
60001
60002
] == [
1
1
1
1
1
1
];
}
{
# Read back from the rendered unit and assertion list: a misspelt
# variable or a dropped assertion fails here.
name = "hive-priv carries the userns agents and data owner, and the owner assertion passes";
ok =
let
e = allLocal.systemd.services.hive-priv.environment;
in
e.HIVE_PRIV_USERNS_AGENTS == ""
&& e.HIVE_PRIV_USERNS_DATA_OWNER == "60000"
&& lib.any (
a: lib.hasPrefix "services.hyperhive.userns.dataOwner" a.message && a.assertion
) allLocal.assertions;
}
{
# The domain is the stub's swarm domain, which both fixtures share.
name = "the forge's publicUrl default does not consult the central toggle";

111
nix/vm-tests/agent-ephemeral-rootfs.nix
View file

@ -18,6 +18,12 @@
# leaves the link's target untouched, and the start goes through once the
# link is gone.
#
# - an agent listed in `services.hyperhive.userns.agents` starts in its own
# user namespace with state it already had, owned by uid 1000 on disk: the
# data moves once to `dataOwner`, the agent user reads and writes it as
# itself, the unlisted agent's conf is untouched, and a second
# `WriteNspawnFlags` moves nothing.
#
# The agent's journal forwarder itself (an OTLP push to the hive's collector) is
# not booted here; the module-eval suite covers its wiring.
{ pkgs, hivePriv }:
@ -29,6 +35,28 @@ let
bridge = "hive0";
gateway = "10.42.0.1";
# An agent in its own user namespace. Its user is pinned to 1000:1000: the
# id map copies the uid map for gids, so its group has to share its uid.
nsAgent = "juno";
nsMachine = "h-${nsAgent}";
nsDir = "/var/lib/hyperhive/agents/${nsAgent}";
nsSocketDir = "/run/hive-agent/${nsAgent}";
dataOwner = "60000";
nsBinds =
map (dir: {
host_path = "${nsDir}/${dir}";
container_path =
if dir == "claude" then "/home/${nsAgent}/.claude" else "/agents/${nsAgent}/${dir}";
read_only = dir == "config";
}) (hostDirs ++ [ "config" ])
++ [
{
host_path = nsSocketDir;
container_path = nsSocketDir;
read_only = false;
}
];
# Host side of what hive-c0re passes `WriteNspawnFlags` for an agent.
binds = map (dir: {
host_path = "${agentDir}/${dir}";
@ -90,7 +118,15 @@ pkgs.testers.runNixOSTest {
prefixLength = 24;
}
];
systemd.tmpfiles.rules = map (dir: "d ${agentDir}/${dir} 0755 root root - -") hostDirs;
services.hyperhive.userns.agents = [ nsAgent ];
systemd.tmpfiles.rules =
map (dir: "d ${agentDir}/${dir} 0755 root root - -") hostDirs
# What the namespaced agent has from before: dirs its user owned.
++ map (dir: "d ${nsDir}/${dir} 0755 1000 1000 - -") hostDirs
++ [
"d ${nsDir}/config 0755 root root - -"
"f ${nsDir}/config/cfg 0644 root root - from-config"
];
environment.systemPackages = [ pkgs.socat ];
containers.${machine} = {
@ -106,6 +142,21 @@ pkgs.testers.runNixOSTest {
services.journald.storage = "persistent";
};
};
containers.${nsMachine} = {
autoStart = false;
privateNetwork = true;
hostBridge = bridge;
config = {
imports = [ ../agent-modules/user.nix ];
system.stateVersion = "25.11";
services.hyperhive.agent.user = {
name = nsAgent;
uid = 1000;
gid = 1000;
};
};
};
};
testScript = ''
@ -260,5 +311,63 @@ pkgs.testers.runNixOSTest {
host.succeed(f"systemctl stop container@{MACHINE}")
host.succeed(f"rm {ROOT}/root && mkdir -m 0700 {ROOT}/root")
boot()
with subtest("a listed agent starts in its own user namespace with the state it had"):
NS = "${nsMachine}"
NS_AGENT = "${nsAgent}"
NS_DIR = "${nsDir}"
NS_BINDS = json.loads(${builtins.toJSON (builtins.toJSON nsBinds)})
host.succeed(f"echo before > {NS_DIR}/state/pre.txt && chown 1000:1000 {NS_DIR}/state/pre.txt")
host.succeed(f"stat -c %u:%g {NS_DIR}/state/pre.txt | grep -x 1000:1000")
priv_request(host, {"op": "ensure_agent_socket_dir", "name": NS_AGENT})
write = {
"op": "write_nspawn_flags",
"container": NS,
"binds": NS_BINDS,
"isolation": {"bridge": "${bridge}", "gateway_ip": "${gateway}"},
}
priv_request(host, write)
ns_conf = host.succeed(f"cat /etc/nixos-containers/{NS}.conf")
assert "PRIVATE_USERS=pick" in ns_conf, ns_conf
assert "--private-users-ownership=map" in ns_conf, ns_conf
# Every bind, the read-only config included, is mounted owneridmap.
assert ns_conf.count(":owneridmap") == len(NS_BINDS), ns_conf
# The unlisted agent's conf has none of it: the control that the lookups
# above find these strings where they are.
assert "owneridmap" not in conf() and "PRIVATE_USERS" not in conf(), conf()
# The data moved to the data owner, the config dir stayed root's.
for sub in ["state", "harness", "work", "claude"]:
host.succeed(f"stat -c %u:%g {NS_DIR}/{sub} | grep -x ${dataOwner}:${dataOwner}")
host.succeed(f"stat -c %u:%g {NS_DIR}/state/pre.txt | grep -x ${dataOwner}:${dataOwner}")
host.succeed(f"stat -c %u:%g {NS_DIR}/config/cfg | grep -x 0:0")
host.succeed(f"stat -c %u:%g ${nsSocketDir} | grep -x ${dataOwner}:${dataOwner}")
boot_ns = lambda: (
host.succeed(f"nixos-container start {NS}"),
host.wait_until_succeeds(f"nixos-container run {NS} -- systemctl is-active multi-user.target"),
)
boot_ns()
def as_ns(script):
return host.succeed(f"nixos-container run {NS} -- runuser -l {NS_AGENT} -c '{script}'").strip()
# Inside, the data is the agent's: read, written and listed as itself.
assert as_ns(f"cat /agents/{NS_AGENT}/state/pre.txt") == "before"
assert as_ns(f"stat -c %u:%g /agents/{NS_AGENT}/state/pre.txt") == "1000:1000"
as_ns(f"echo after > /agents/{NS_AGENT}/state/post.txt")
as_ns(f"touch /run/hive-agent/{NS_AGENT}/web.sock")
assert as_ns(f"cat /agents/{NS_AGENT}/config/cfg") == "from-config"
host.fail(f"nixos-container run {NS} -- runuser -l {NS_AGENT} -c 'touch /agents/{NS_AGENT}/config/new'")
# On disk the new file belongs to the data owner, not to a picked range.
host.succeed(f"stat -c %u:%g {NS_DIR}/state/post.txt | grep -x ${dataOwner}:${dataOwner}")
host.succeed(f"stat -c %u:%g ${nsSocketDir}/web.sock | grep -x ${dataOwner}:${dataOwner}")
host.succeed(f"nixos-container run {NS} -- cat /proc/self/uid_map | grep -v ' 4294967295$'")
# The move runs once: a file a host process leaves owned by 1000 after it
# is not chowned by the next request, and the agent restarts.
host.succeed(f"nixos-container stop {NS}")
host.succeed(f"echo late > {NS_DIR}/state/late.txt && chown 1000:1000 {NS_DIR}/state/late.txt")
priv_request(host, write)
host.succeed(f"stat -c %u:%g {NS_DIR}/state/late.txt | grep -x 1000:1000")
boot_ns()
'';
}