diff --git a/hive-priv/src/container_config.rs b/hive-priv/src/container_config.rs index fb685a3b..8a9090b3 100644 --- a/hive-priv/src/container_config.rs +++ b/hive-priv/src/container_config.rs @@ -50,6 +50,9 @@ pub(crate) fn handle_write_nspawn_flags( ); false }); + // After the salvage, so the mount points land in the root the start will + // snapshot. + let userns = crate::userns::activate(container, agent, binds)?; write_nspawn_flags( container, binds, @@ -57,6 +60,7 @@ pub(crate) fn handle_write_nspawn_flags( load_credentials, &hive_facts, ephemeral, + userns, )?; Ok((String::new(), String::new())) } @@ -364,6 +368,7 @@ fn write_nspawn_flags( load_credentials: &[CredentialMount], hive_facts: &str, ephemeral: bool, + userns: bool, ) -> Result<()> { use std::fmt::Write as _; use std::os::unix::fs::MetadataExt as _; @@ -371,21 +376,13 @@ fn write_nspawn_flags( let original = std::fs::read_to_string(&path).with_context(|| format!("read {path}"))?; let lines: Vec<&str> = original .lines() - .filter(|line| { - let t = line.trim_start(); - !t.starts_with("EXTRA_NSPAWN_FLAGS=") - && !t.starts_with("PRIVATE_NETWORK=") - && !t.starts_with("HOST_ADDRESS=") - && !t.starts_with("LOCAL_ADDRESS=") - && !t.starts_with("HOST_ADDRESS6=") - && !t.starts_with("LOCAL_ADDRESS6=") - && !t.starts_with("HOST_BRIDGE=") - }) + .filter(|line| !is_owned_var(line)) .collect(); let mut out = lines.join("\n"); if !out.is_empty() { out.push('\n'); } + out.push_str(private_users_line(userns)); { let iso = isolation; out.push_str("PRIVATE_NETWORK=1\n"); @@ -406,7 +403,7 @@ fn write_nspawn_flags( out.push_str("LOCAL_ADDRESS6=\n"); let _ = writeln!(out, "HOST_BRIDGE={}", iso.bridge); } - let flags_joined = nspawn_flags(binds, load_credentials, ephemeral).join(" "); + let flags_joined = nspawn_flags(binds, load_credentials, ephemeral, userns).join(" "); let _ = writeln!(out, "EXTRA_NSPAWN_FLAGS=\"{flags_joined}\""); // Keeps the owner and mode `nixos-container` created the file with; this // helper only rewrites its lines. @@ -432,24 +429,57 @@ fn write_nspawn_flags( Ok(()) } +/// Whether a line of the container conf is one `write_nspawn_flags` writes +/// itself, and so drops before writing it again. Dropped for every container, +/// not only those it writes it for: a `PRIVATE_USERS=pick` left behind would +/// keep a container namespaced after it left the list. +fn is_owned_var(line: &str) -> bool { + let t = line.trim_start(); + [ + "EXTRA_NSPAWN_FLAGS=", + "PRIVATE_NETWORK=", + "PRIVATE_USERS=", + "HOST_ADDRESS=", + "LOCAL_ADDRESS=", + "HOST_ADDRESS6=", + "LOCAL_ADDRESS6=", + "HOST_BRIDGE=", + ] + .iter() + .any(|var| t.starts_with(var)) +} + +/// The conf line that makes nixos-container pass `--private-users=pick`. +fn private_users_line(userns: bool) -> &'static str { + if userns { "PRIVATE_USERS=pick\n" } else { "" } +} + /// The flags `EXTRA_NSPAWN_FLAGS` carries, in the order nspawn needs them. /// /// `--ephemeral` runs the container on a snapshot of its root and drops the /// snapshot at stop; `crate::rootfs` has made the root fit for that before /// the flag is asked for. The git overlays and credentials follow the binds /// they refer to. +/// +/// `userns` mounts every bind `owneridmap` and idmaps the root instead of +/// chowning it, so a different `pick` range never walks the tree. fn nspawn_flags( binds: &[BindMount], load_credentials: &[CredentialMount], ephemeral: bool, + userns: bool, ) -> Vec { let mut flags: Vec = ephemeral .then(|| "--ephemeral".to_owned()) .into_iter() .collect(); + if userns { + flags.push("--private-users-ownership=map".to_owned()); + } + let idmap = if userns { ":owneridmap" } else { "" }; flags.extend(binds.iter().map(|b| { let flag = if b.read_only { "--bind-ro" } else { "--bind" }; - format!("{flag}={}:{}", b.host_path, b.container_path) + format!("{flag}={}:{}{idmap}", b.host_path, b.container_path) })); flags.extend(git_overlay_flags(binds)); // Credential forwarding: nspawn loads each host secret into the @@ -621,9 +651,9 @@ pub(crate) fn write_container_etc_file_in(rootfs: &Path, name: &str, contents: & mod tests { use super::{ BindMount, HiveFacts, NetworkIsolation, git_overlay_flags, handle_write_nspawn_flags, - hive_priv_exe, limits_dropin_body, nspawn_flags, publish_file, remove_service_dropin, - render_hive_facts, rootfs_check_command, validate_container_system_name, - write_bridge_dns_marker_in, write_resource_limits, + hive_priv_exe, is_owned_var, limits_dropin_body, nspawn_flags, private_users_line, + publish_file, remove_service_dropin, render_hive_facts, rootfs_check_command, + validate_container_system_name, write_bridge_dns_marker_in, write_resource_limits, }; use crate::shared::test_support::{entries, scratch}; @@ -1072,14 +1102,55 @@ mod tests { #[test] fn ephemeral_is_passed_only_when_asked() { let binds = [bind("/agents/atlas/work")]; - let with = nspawn_flags(&binds, &[], true); + let with = nspawn_flags(&binds, &[], true, false); assert_eq!(with[0], "--ephemeral", "{with:?}"); assert!(with.iter().any(|f| f.starts_with("--bind-ro=")), "{with:?}"); - let without = nspawn_flags(&binds, &[], false); + let without = nspawn_flags(&binds, &[], false, false); assert!(!without.iter().any(|f| f == "--ephemeral"), "{without:?}"); assert_eq!(without.len() + 1, with.len()); } + /// Every bind, read-only ones included, is mounted `owneridmap` for a + /// namespaced agent and none is for any other; the root is idmapped, not + /// chowned. The `false` arm is the control that the lookup finds the binds. + #[test] + fn binds_are_owneridmap_only_for_a_namespaced_agent() { + let binds = [bind("/knowledge"), bind("/agents/a/config")]; + let plain = nspawn_flags(&binds, &[], false, false); + assert!( + plain.iter().any(|f| f.starts_with("--bind-ro=")), + "{plain:?}" + ); + assert!( + plain + .iter() + .all(|f| !f.contains("idmap") && !f.contains("ownership")), + "{plain:?}" + ); + let userns = nspawn_flags(&binds, &[], false, true); + assert!( + userns.contains(&"--private-users-ownership=map".to_owned()), + "{userns:?}" + ); + let mounts: Vec<&String> = userns.iter().filter(|f| f.starts_with("--bind")).collect(); + assert_eq!(mounts.len(), 2, "{userns:?}"); + assert!( + mounts.iter().all(|f| f.ends_with(":owneridmap")), + "{userns:?}" + ); + } + + /// `PRIVATE_USERS=` is dropped from the conf whatever the agent now is, so + /// leaving the list cannot leave the line behind. `AUTO_START=` is the + /// control: a line that is not ours stays. + #[test] + fn the_private_users_line_is_ours_to_drop_and_to_write() { + assert!(is_owned_var("PRIVATE_USERS=pick")); + assert!(!is_owned_var("AUTO_START=1")); + assert_eq!(private_users_line(true), "PRIVATE_USERS=pick\n"); + assert_eq!(private_users_line(false), ""); + } + /// The swarm's forge and matrix containers are not the hive's: every /// request that names a container as a string refuses them before /// touching the filesystem. Control: an agent container name passes. diff --git a/hive-priv/src/files.rs b/hive-priv/src/files.rs index fb9a3b01..b5c810b0 100644 --- a/hive-priv/src/files.rs +++ b/hive-priv/src/files.rs @@ -13,7 +13,7 @@ use hive_priv_sock::{AGENT_STATE_ROOT, PAUSED_MARKER_FILE}; use crate::shared::{StagedFile, ensure_plain_filename, open_dir, validate_agent_name}; /// Root of the per-agent unix-socket dirs on the host. -const SOCKET_DIR_ROOT: &str = "/run/hive-agent"; +pub(crate) const SOCKET_DIR_ROOT: &str = "/run/hive-agent"; /// `SetAgentPaused`. pub(crate) fn exec_set_agent_paused(agent_name: &str, paused: bool) -> Result<(String, String)> { @@ -155,6 +155,13 @@ pub(crate) fn remove_legacy_tmpfiles() -> Result<()> { /// `EnsureAgentSocketDir` — create `/run/hive-agent/` if missing. pub(crate) fn ensure_agent_socket_dir(name: &str) -> Result<(String, String)> { ensure_socket_dir_in(Path::new(SOCKET_DIR_ROOT), name)?; + // A namespaced agent's activation cannot chown the dir through the id + // mapping, so the data owner, which is the agent user inside, gets it here. + if let Some(owner) = crate::userns::owner_for_agent(name)? { + let path = Path::new(SOCKET_DIR_ROOT).join(name); + std::os::unix::fs::lchown(&path, Some(owner), Some(owner)) + .with_context(|| format!("chown {}", path.display()))?; + } Ok((String::new(), String::new())) } diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index 10971f6a..00639c90 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -37,6 +37,7 @@ mod rootfs; mod salvage_io; mod shared; mod storage; +mod userns; use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd, RawFd}; diff --git a/hive-priv/src/rootfs.rs b/hive-priv/src/rootfs.rs index 25079f94..8c211dfc 100644 --- a/hive-priv/src/rootfs.rs +++ b/hive-priv/src/rootfs.rs @@ -512,7 +512,7 @@ fn carry_nix_state( /// owned by `uid:gid`), or `None` when `base` or an existing component is not a /// real directory. The container could have replaced any of them by a link, and /// this runs as root. -fn own_dir(base: &Path, parts: &[&str], uid: u32, gid: u32) -> Result> { +pub(crate) fn own_dir(base: &Path, parts: &[&str], uid: u32, gid: u32) -> Result> { let mut path = base.to_path_buf(); if !std::fs::symlink_metadata(&path).is_ok_and(|m| m.is_dir()) { return Ok(None); @@ -711,7 +711,7 @@ fn is_regular(path: &Path) -> bool { /// `systemctl is-active` answers `inactive` or `failed` for a unit that is not /// running; anything else (`active`, `activating`, `deactivating`) means the /// root is, or is about to be, in use. -fn unit_is_stopped(container: &str) -> Result { +pub(crate) fn unit_is_stopped(container: &str) -> Result { let out = std::process::Command::new("systemctl") .args(["is-active", &format!("container@{container}.service")]) .output() diff --git a/hive-priv/src/userns.rs b/hive-priv/src/userns.rs new file mode 100644 index 00000000..66c8a03b --- /dev/null +++ b/hive-priv/src/userns.rs @@ -0,0 +1,496 @@ +//! Agents that run in their own user namespace. +//! +//! The operator lists them in `services.hyperhive.userns.agents` and names the +//! one host uid:gid that owns their data on disk (`dataOwner`); the unit's +//! environment carries both, so only root decides who is namespaced. A listed +//! agent starts with `--private-users=pick`, a root idmapped rather than +//! chowned, and every bind mounted `owneridmap`: the owner of the bind's +//! target inside the container, the agent user, is the data owner on disk. +//! +//! Getting there moves the agent's data once. [`activate`] chowns its `state`, +//! `harness`, `work` and `claude` dirs to the data owner, never touching +//! anything else, and records that in a marker only root can write; the next +//! call sees the marker and does nothing. `owneridmap` needs the target +//! directory to be owned by the agent when nspawn mounts, and nspawn creates a +//! missing one as root in a throwaway snapshot, so the mount points are made +//! in the template root too. The agent's socket dir lives on a tmpfs and goes +//! to the data owner at every start instead ([`owner_for_agent`]). + +use std::os::unix::fs::{MetadataExt as _, lchown}; +use std::path::{Path, PathBuf}; + +use anyhow::{Context as _, Result, bail}; +use hive_priv_sock::{AGENT_STATE_ROOT, BindMount}; + +use crate::files::SOCKET_DIR_ROOT; + +const AGENTS_ENV: &str = "HIVE_PRIV_USERNS_AGENTS"; +const DATA_OWNER_ENV: &str = "HIVE_PRIV_USERNS_DATA_OWNER"; + +/// One ` ` file per agent: the data owner its data was moved +/// to, and the agent's own ids inside the container. +const MARKER_ROOT: &str = "/var/lib/hive-priv/userns"; + +/// NixOS gives a normal user, which the agent is, an id from 1000 up. +const MIN_AGENT_UID: u32 = 1000; + +/// The dirs under `/` the agent writes into. `config` +/// is not one of them: hive-core writes it and the agent only reads it. +const DATA_DIRS: [&str; 4] = ["state", "harness", "work", "claude"]; + +/// Where one call looks and writes, and what it was told. +pub(crate) struct Env<'a> { + pub(crate) state_root: &'a Path, + /// The agent's socket dir, a bind source like the data dirs. + pub(crate) socket_dir: &'a Path, + pub(crate) marker_root: &'a Path, + pub(crate) rootfs: &'a Path, + /// The data owner when the agent is listed, else `None`. + pub(crate) owner: Option, + pub(crate) stopped: bool, + /// Lowest uid taken for an agent's own: below it a dir belongs to root or + /// to a system user such as hive-core. + pub(crate) min_uid: u32, + pub(crate) chown: &'a dyn Fn(&Path, u32, u32) -> std::io::Result<()>, +} + +/// The data owner for `agent` under the unit's environment: `None` when the +/// agent is not listed. +/// +/// # Errors +/// The agent is listed and the data owner is missing or not a plain uid. +pub(crate) fn owner_for( + agents: Option<&str>, + owner: Option<&str>, + agent: &str, +) -> Result> { + if !agents.is_some_and(|list| list.split_whitespace().any(|a| a == agent)) { + return Ok(None); + } + let owner = owner.with_context(|| { + format!("{agent} is listed in {AGENTS_ENV}, but {DATA_OWNER_ENV} is not set") + })?; + let owner = owner + .trim() + .parse() + .with_context(|| format!("{DATA_OWNER_ENV}={owner:?} is not a uid"))?; + Ok(Some(owner)) +} + +/// [`owner_for`] under the unit's environment. +/// +/// # Errors +/// As [`owner_for`]. +pub(crate) fn owner_for_agent(agent: &str) -> Result> { + owner_for( + std::env::var(AGENTS_ENV).ok().as_deref(), + std::env::var(DATA_OWNER_ENV).ok().as_deref(), + agent, + ) +} + +/// Make `agent`'s next start of `container` namespaced when it is listed, and +/// say whether it is. `binds` are the ones the start will mount. +/// +/// A container that is running, and whose data has not been moved yet, keeps +/// its current start: the move waits for a stopped container. +/// +/// # Errors +/// The environment is inconsistent, the data owner changed since the data +/// moved, the agent was dropped from the list after it moved, or a chown or +/// mkdir fails. +pub(crate) fn activate(container: &str, agent: &str, binds: &[BindMount]) -> Result { + let owner = owner_for_agent(agent)?; + let rootfs = crate::container_config::container_rootfs(container); + let stopped = crate::rootfs::unit_is_stopped(container)?; + activate_in( + &Env { + state_root: Path::new(AGENT_STATE_ROOT), + socket_dir: &Path::new(SOCKET_DIR_ROOT).join(agent), + marker_root: Path::new(MARKER_ROOT), + rootfs: &rootfs, + owner, + stopped, + min_uid: MIN_AGENT_UID, + chown: &|path, uid, gid| lchown(path, Some(uid), Some(gid)), + }, + agent, + binds, + ) +} + +/// [`activate`] over an explicit [`Env`]. +pub(crate) fn activate_in(env: &Env<'_>, agent: &str, binds: &[BindMount]) -> Result { + crate::shared::validate_agent_name(agent)?; + let marker = env.marker_root.join(agent); + let moved = read_marker(&marker)?; + let Some(owner) = env.owner else { + if moved.is_some() { + bail!( + "{agent}: its data was moved to the userns data owner and it is no longer \ + listed; list it again, or chown its dirs back and remove {}", + marker.display() + ); + } + return Ok(false); + }; + let ids = match moved { + Some((was, uid, gid)) if was == owner => (uid, gid), + Some((was, ..)) => { + bail!("{agent}: its data is owned by {was}, not the configured data owner {owner}") + } + None if !env.stopped => return Ok(false), + None => { + let ids = move_data(env, agent, owner)?; + write_marker(&marker, owner, ids)?; + ids + } + }; + if env.stopped { + mount_points(env, agent, binds, ids)?; + } + Ok(true) +} + +/// ` ` from the marker, `None` when there is none. +fn read_marker(path: &Path) -> Result> { + let text = match std::fs::read_to_string(path) { + Ok(text) => text, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e).with_context(|| format!("read {}", path.display())), + }; + let ids: Vec = text + .split_whitespace() + .map(str::parse) + .collect::>() + .with_context(|| format!("parse {}", path.display()))?; + let [owner, uid, gid] = ids[..] else { + bail!("{} does not hold ` `", path.display()); + }; + Ok(Some((owner, uid, gid))) +} + +fn write_marker(path: &Path, owner: u32, (uid, gid): (u32, u32)) -> Result<()> { + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?; + } + std::fs::write(path, format!("{owner} {uid} {gid}\n")) + .with_context(|| format!("write {}", path.display())) +} + +/// Chown the agent's data dirs to `owner:owner`; returns the agent's own uid +/// and gid, read from its `state` dir before the chown. +/// +/// Only the [`DATA_DIRS`] under `/` are touched, each only +/// if it is a real directory. The walk does not follow links or cross into +/// another filesystem, and the container is stopped while it runs. +fn move_data(env: &Env<'_>, agent: &str, owner: u32) -> Result<(u32, u32)> { + let base = env.state_root.join(agent); + let state = base.join("state"); + let meta = + std::fs::symlink_metadata(&state).with_context(|| format!("stat {}", state.display()))?; + let (uid, gid) = (meta.uid(), meta.gid()); + if !meta.is_dir() || uid < env.min_uid || uid == owner { + bail!( + "{agent}: {} is not a directory owned by a container user (uid {uid}); start the \ + agent once without the namespace so its user exists", + state.display() + ); + } + for sub in DATA_DIRS { + let dir = base.join(sub); + match std::fs::symlink_metadata(&dir) { + Ok(m) if m.is_dir() => chown_tree(env, &dir, m.dev(), owner)?, + Ok(_) => bail!("{} is not a directory", dir.display()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e).with_context(|| format!("stat {}", dir.display())), + } + } + Ok((uid, gid)) +} + +/// Chown everything under `top`, `top` included, to `owner:owner`. +fn chown_tree(env: &Env<'_>, top: &Path, dev: u64, owner: u32) -> Result<()> { + let mut dirs: Vec = vec![top.to_path_buf()]; + while let Some(dir) = dirs.pop() { + (env.chown)(&dir, owner, owner).with_context(|| format!("chown {}", dir.display()))?; + for entry in std::fs::read_dir(&dir).with_context(|| format!("read {}", dir.display()))? { + let entry = entry?; + let meta = entry.metadata()?; + if meta.dev() != dev { + continue; + } + if meta.is_dir() { + dirs.push(entry.path()); + } else { + (env.chown)(&entry.path(), owner, owner) + .with_context(|| format!("chown {}", entry.path().display()))?; + } + } + } + Ok(()) +} + +/// Create, in the template root, the mount point of each bind whose source is +/// one of the agent's data dirs or its socket dir, owned by the agent's ids +/// inside the container. +fn mount_points( + env: &Env<'_>, + agent: &str, + binds: &[BindMount], + (uid, gid): (u32, u32), +) -> Result<()> { + let sources: Vec = DATA_DIRS + .iter() + .map(|sub| env.state_root.join(agent).join(sub)) + .chain([env.socket_dir.to_path_buf()]) + .collect(); + for bind in binds.iter().filter(|b| { + sources + .iter() + .any(|s| s.as_os_str() == b.host_path.as_str()) + }) { + let parts: Vec<&str> = bind + .container_path + .split('/') + .filter(|p| !p.is_empty()) + .collect(); + let Some(dir) = crate::rootfs::own_dir(env.rootfs, &parts, uid, gid)? else { + bail!( + "{}: a link or file where the mount point {} goes", + env.rootfs.display(), + bind.container_path + ); + }; + (env.chown)(&dir, uid, gid).with_context(|| format!("chown {}", dir.display()))?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{Env, activate_in, owner_for}; + use crate::shared::test_support::scratch; + use hive_priv_sock::BindMount; + use std::cell::RefCell; + use std::os::unix::fs::MetadataExt as _; + use std::path::{Path, PathBuf}; + + /// Not the uid the tests run as, so a chown is a change. + const OWNER: u32 = 60000; + + fn write(path: &Path) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, "x").unwrap(); + } + + /// `/agents/a` with data in each dir, `config` beside them, a link + /// out of `work`, and `outside/o` next to the tree. Returns the root. + fn tree() -> PathBuf { + let root = scratch(); + let a = root.join("agents/a"); + for f in ["state/f", "harness/d/g", "work/x", "claude/c", "config/cfg"] { + write(&a.join(f)); + } + write(&root.join("outside/o")); + std::os::unix::fs::symlink(root.join("outside"), a.join("work/link")).unwrap(); + root + } + + /// Run [`activate_in`] on `root` and return its answer with every path it + /// asked to chown, relative to `root`. + fn run( + root: &Path, + owner: Option, + stopped: bool, + agent: &str, + binds: &[BindMount], + ) -> (anyhow::Result, Vec) { + let seen = RefCell::new(Vec::new()); + let chown = |p: &Path, _: u32, _: u32| { + seen.borrow_mut() + .push(p.strip_prefix(root).unwrap().display().to_string()); + Ok(()) + }; + let (state_root, rootfs) = (root.join("agents"), root.join("rootfs")); + std::fs::create_dir_all(&rootfs).unwrap(); + let env = Env { + state_root: &state_root, + socket_dir: &root.join("run/a"), + marker_root: &root.join("marker"), + rootfs: &rootfs, + owner, + stopped, + min_uid: 0, + chown: &chown, + }; + let answer = activate_in(&env, agent, binds); + (answer, seen.into_inner()) + } + + fn bind(root: &Path, sub: &str, target: &str) -> BindMount { + BindMount { + host_path: root.join("agents/a").join(sub).display().to_string(), + container_path: target.to_owned(), + read_only: false, + } + } + + #[test] + fn only_listed_agents_have_an_owner() { + let owner = |agents, owner| owner_for(agents, owner, "a"); + assert_eq!(owner(Some("b a"), Some("60000")).unwrap(), Some(60000)); + assert_eq!(owner(Some("b c"), Some("60000")).unwrap(), None); + assert_eq!(owner(None, None).unwrap(), None); + assert!(owner(Some("a"), None).is_err(), "listed without an owner"); + assert!(owner(Some("a"), Some("root")).is_err(), "not a uid"); + } + + /// The move chowns the four data dirs and what is in them and nothing + /// else: not `config`, not what a link in `work` points at. + #[test] + fn the_move_stays_inside_the_agents_data_dirs() { + let root = tree(); + let (answer, mut seen) = run(&root, Some(OWNER), true, "a", &[]); + assert!(answer.unwrap()); + seen.sort(); + let want = [ + "agents/a/claude", + "agents/a/claude/c", + "agents/a/harness", + "agents/a/harness/d", + "agents/a/harness/d/g", + "agents/a/state", + "agents/a/state/f", + "agents/a/work", + "agents/a/work/link", + "agents/a/work/x", + ]; + assert_eq!(seen, want); + assert!(root.join("marker/a").exists()); + } + + /// Once the marker is there a second call chowns nothing. The first call + /// in `the_move_stays_inside_the_agents_data_dirs` is the control that + /// the same setup does chown. + #[test] + fn the_move_runs_once() { + let root = tree(); + assert!(run(&root, Some(OWNER), true, "a", &[]).0.unwrap()); + let (answer, seen) = run(&root, Some(OWNER), true, "a", &[]); + assert!(answer.unwrap()); + assert!(seen.is_empty(), "{seen:?}"); + } + + /// A running container's data is left alone and the start stays as it is. + #[test] + fn a_running_container_is_not_moved() { + let root = tree(); + let (answer, seen) = run(&root, Some(OWNER), false, "a", &[]); + assert!(!answer.unwrap()); + assert!(seen.is_empty(), "{seen:?}"); + assert!(!root.join("marker/a").exists()); + } + + #[test] + fn an_unlisted_agent_is_left_alone_and_a_moved_one_is_refused() { + let root = tree(); + let (answer, seen) = run(&root, None, true, "a", &[]); + assert!(!answer.unwrap() && seen.is_empty()); + assert!( + run(&root, Some(OWNER), true, "a", &[]).0.unwrap(), + "control" + ); + assert!(run(&root, None, true, "a", &[]).0.is_err(), "left the list"); + assert!( + run(&root, Some(OWNER + 1), true, "a", &[]).0.is_err(), + "data owner changed" + ); + } + + /// A data dir that is a link is refused before anything past it is touched. + #[test] + fn a_linked_data_dir_is_refused() { + let root = tree(); + let work = root.join("agents/a/work"); + std::fs::remove_dir_all(&work).unwrap(); + std::os::unix::fs::symlink(root.join("outside"), &work).unwrap(); + let (answer, seen) = run(&root, Some(OWNER), true, "a", &[]); + assert!(answer.is_err()); + assert!(seen.iter().all(|p| !p.starts_with("outside")), "{seen:?}"); + assert!(!root.join("marker/a").exists()); + } + + #[test] + fn a_path_is_not_an_agent_name() { + let root = tree(); + let (answer, seen) = run(&root, Some(OWNER), true, "../outside", &[]); + assert!(answer.is_err() && seen.is_empty()); + } + + /// The agent's own uid is read off its `state` dir, and root's or the + /// data owner's is refused. + #[test] + fn the_state_dir_has_to_be_owned_by_a_container_user() { + let root = tree(); + let euid = std::fs::metadata(root.join("agents/a/state")) + .unwrap() + .uid(); + let seen = RefCell::new(Vec::new()); + let chown = |p: &Path, _: u32, _: u32| { + seen.borrow_mut().push(p.to_path_buf()); + Ok(()) + }; + let (state_root, rootfs) = (root.join("agents"), root.join("rootfs")); + let mut env = Env { + state_root: &state_root, + socket_dir: &root.join("run/a"), + marker_root: &root.join("marker"), + rootfs: &rootfs, + owner: Some(OWNER), + stopped: true, + min_uid: euid + 1, + chown: &chown, + }; + assert!(activate_in(&env, "a", &[]).is_err(), "below the minimum"); + assert!(seen.borrow().is_empty()); + env.min_uid = euid; + env.owner = Some(euid); + assert!(activate_in(&env, "a", &[]).is_err(), "already the owner"); + env.owner = Some(OWNER); + assert!(activate_in(&env, "a", &[]).unwrap(), "control"); + } + + /// A mount point is made for a bind of the agent's own data dir, and for + /// no other bind; a link in the way is refused. + #[test] + fn mount_points_are_made_for_the_agents_data_binds_only() { + let root = tree(); + let binds = [ + bind(&root, "state", "/agents/a/state"), + BindMount { + host_path: root.join("outside").display().to_string(), + container_path: "/agents/a/stray".to_owned(), + read_only: false, + }, + BindMount { + host_path: root.join("run/a").display().to_string(), + container_path: "/run/hive-agent/a".to_owned(), + read_only: false, + }, + ]; + let (answer, seen) = run(&root, Some(OWNER), true, "a", &binds); + assert!(answer.unwrap()); + assert!(root.join("rootfs/agents/a/state").is_dir()); + assert!(root.join("rootfs/run/hive-agent/a").is_dir(), "socket dir"); + assert!(!root.join("rootfs/agents/a/stray").exists()); + assert!( + seen.contains(&"rootfs/agents/a/state".to_owned()), + "{seen:?}" + ); + + std::fs::remove_dir_all(root.join("rootfs/agents")).unwrap(); + std::os::unix::fs::symlink(root.join("outside"), root.join("rootfs/agents")).unwrap(); + assert!(run(&root, Some(OWNER), true, "a", &binds).0.is_err()); + } +} diff --git a/nix/host-modules/hive-priv.nix b/nix/host-modules/hive-priv.nix index 2a526147..235eb978 100644 --- a/nix/host-modules/hive-priv.nix +++ b/nix/host-modules/hive-priv.nix @@ -10,6 +10,8 @@ let controllerCfg = config.services.hyperhive.deploy.hive-controller; rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; + userns = config.services.hyperhive.userns; + dataOwnerProblems = import ./lib/userns-data-owner.nix { inherit lib; }; in { # The platform containers run their warn-only root check by this path @@ -24,12 +26,60 @@ in } ]; + options.services.hyperhive.userns = { + agents = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ "alice" ]; + description = '' + Agents that run in their own user namespace (`--private-users=pick`, + root idmapped). Every hive bind into such a container is mounted + `owneridmap`, so its data on disk belongs to `dataOwner`, which the + agent user is inside. Listing an agent moves its `state`, `harness`, + `work` and `claude` dirs to `dataOwner` once, at its next start with + the container stopped. Take the agent off the list only after chowning + those dirs back and removing `/var/lib/hive-priv/userns/`; + until then it will not start. + ''; + }; + + dataOwner = lib.mkOption { + type = lib.types.ints.u32; + default = 60000; + description = '' + The host uid and gid that own a namespaced agent's data on disk. One + number for both: nspawn copies the uid map for gids, so the gid + cannot differ. It must be no host user or group, outside the normal and + system users (below 30000) and the dynamic users (61184 and above), and + below the range `pick` allocates from (524288 and above), so no + container's range covers it. Changing it after an agent has moved is + refused at that agent's next start. + ''; + }; + }; + config = lib.mkIf config.services.hyperhive.deploy.hive-controller.enable { # The parent of every agent socket dir. hive-priv is its only writer # (`EnsureAgentSocketDir`), and its unit lists it in `ReadWritePaths`, # which fails the unit when the path is missing. systemd.tmpfiles.rules = [ "d /run/hive-agent 0755 root root - -" ]; + assertions = [ + ( + let + problems = dataOwnerProblems { + owner = userns.dataOwner; + uids = lib.filter (uid: uid != null) (lib.mapAttrsToList (_: u: u.uid) config.users.users); + gids = lib.filter (gid: gid != null) (lib.mapAttrsToList (_: g: g.gid) config.users.groups); + }; + in + { + assertion = userns.agents == [ ] || problems == [ ]; + message = "services.hyperhive.userns.dataOwner: ${lib.concatStringsSep "; " problems}"; + } + ) + ]; + # Socket unit for hive-priv — the narrow root helper that executes # privileged operations on behalf of hive-c0re. Systemd creates and # holds `/run/hive/priv.sock` before the first connection arrives. @@ -93,6 +143,10 @@ in # routing so the closure `RealiseClosure` downloads is in the same # store the container is then pointed at. NIX_REMOTE = "daemon"; + # Who is namespaced, and the owner their data moves to: set by the + # operator in the unit, so only root decides. + HIVE_PRIV_USERNS_AGENTS = lib.concatStringsSep " " userns.agents; + HIVE_PRIV_USERNS_DATA_OWNER = toString userns.dataOwner; }; serviceConfig = { ExecStart = "${controllerCfg.package}/bin/hive-priv"; diff --git a/nix/host-modules/lib/userns-data-owner.nix b/nix/host-modules/lib/userns-data-owner.nix new file mode 100644 index 00000000..03be83c2 --- /dev/null +++ b/nix/host-modules/lib/userns-data-owner.nix @@ -0,0 +1,17 @@ +# What is wrong with a uid:gid chosen as the on-disk owner of a user-namespaced +# agent's data. Empty when it is fine. +{ lib }: +{ + owner, + # Every uid and gid the host defines, from `users.users` and `users.groups`. + uids, + gids, +}: +lib.optional ( + owner < 30000 +) "${toString owner} is in the range NixOS gives normal users and system users (below 30000)" +++ + lib.optional (owner >= 61184) + "${toString owner} is in the range systemd allocates dynamic users from (61184 and above), or is `nobody`" +++ lib.optional (builtins.elem owner uids) "${toString owner} is the uid of a user the host defines" +++ lib.optional (builtins.elem owner gids) "${toString owner} is the gid of a group the host defines" diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 471b3598..3a64eec8 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -137,6 +137,58 @@ let remainsAfterExit = u: u.serviceConfig.RemainAfterExit or false; cases = [ + { + # The control is the default on a host that defines the ids below: + # it gets past every arm, so each refusal is the arm's own. + name = "the userns data owner is refused where it collides with a host id or a reserved range"; + ok = + let + problems = + owner: + import ../host-modules/lib/userns-data-owner.nix { inherit lib; } { + inherit owner; + uids = [ + 0 + 60001 + ]; + gids = [ + 100 + 60002 + ]; + }; + in + problems 60000 == [ ] + && + map (o: builtins.length (problems o)) [ + 1000 + 29999 + 61184 + 65534 + 60001 + 60002 + ] == [ + 1 + 1 + 1 + 1 + 1 + 1 + ]; + } + { + # Read back from the rendered unit and assertion list: a misspelt + # variable or a dropped assertion fails here. + name = "hive-priv carries the userns agents and data owner, and the owner assertion passes"; + ok = + let + e = allLocal.systemd.services.hive-priv.environment; + in + e.HIVE_PRIV_USERNS_AGENTS == "" + && e.HIVE_PRIV_USERNS_DATA_OWNER == "60000" + && lib.any ( + a: lib.hasPrefix "services.hyperhive.userns.dataOwner" a.message && a.assertion + ) allLocal.assertions; + } { # The domain is the stub's swarm domain, which both fixtures share. name = "the forge's publicUrl default does not consult the central toggle"; diff --git a/nix/vm-tests/agent-ephemeral-rootfs.nix b/nix/vm-tests/agent-ephemeral-rootfs.nix index d16620a6..7cbd717c 100644 --- a/nix/vm-tests/agent-ephemeral-rootfs.nix +++ b/nix/vm-tests/agent-ephemeral-rootfs.nix @@ -18,6 +18,12 @@ # leaves the link's target untouched, and the start goes through once the # link is gone. # +# - an agent listed in `services.hyperhive.userns.agents` starts in its own +# user namespace with state it already had, owned by uid 1000 on disk: the +# data moves once to `dataOwner`, the agent user reads and writes it as +# itself, the unlisted agent's conf is untouched, and a second +# `WriteNspawnFlags` moves nothing. +# # The agent's journal forwarder itself (an OTLP push to the hive's collector) is # not booted here; the module-eval suite covers its wiring. { pkgs, hivePriv }: @@ -29,6 +35,28 @@ let bridge = "hive0"; gateway = "10.42.0.1"; + # An agent in its own user namespace. Its user is pinned to 1000:1000: the + # id map copies the uid map for gids, so its group has to share its uid. + nsAgent = "juno"; + nsMachine = "h-${nsAgent}"; + nsDir = "/var/lib/hyperhive/agents/${nsAgent}"; + nsSocketDir = "/run/hive-agent/${nsAgent}"; + dataOwner = "60000"; + nsBinds = + map (dir: { + host_path = "${nsDir}/${dir}"; + container_path = + if dir == "claude" then "/home/${nsAgent}/.claude" else "/agents/${nsAgent}/${dir}"; + read_only = dir == "config"; + }) (hostDirs ++ [ "config" ]) + ++ [ + { + host_path = nsSocketDir; + container_path = nsSocketDir; + read_only = false; + } + ]; + # Host side of what hive-c0re passes `WriteNspawnFlags` for an agent. binds = map (dir: { host_path = "${agentDir}/${dir}"; @@ -90,7 +118,15 @@ pkgs.testers.runNixOSTest { prefixLength = 24; } ]; - systemd.tmpfiles.rules = map (dir: "d ${agentDir}/${dir} 0755 root root - -") hostDirs; + services.hyperhive.userns.agents = [ nsAgent ]; + systemd.tmpfiles.rules = + map (dir: "d ${agentDir}/${dir} 0755 root root - -") hostDirs + # What the namespaced agent has from before: dirs its user owned. + ++ map (dir: "d ${nsDir}/${dir} 0755 1000 1000 - -") hostDirs + ++ [ + "d ${nsDir}/config 0755 root root - -" + "f ${nsDir}/config/cfg 0644 root root - from-config" + ]; environment.systemPackages = [ pkgs.socat ]; containers.${machine} = { @@ -106,6 +142,21 @@ pkgs.testers.runNixOSTest { services.journald.storage = "persistent"; }; }; + + containers.${nsMachine} = { + autoStart = false; + privateNetwork = true; + hostBridge = bridge; + config = { + imports = [ ../agent-modules/user.nix ]; + system.stateVersion = "25.11"; + services.hyperhive.agent.user = { + name = nsAgent; + uid = 1000; + gid = 1000; + }; + }; + }; }; testScript = '' @@ -260,5 +311,63 @@ pkgs.testers.runNixOSTest { host.succeed(f"systemctl stop container@{MACHINE}") host.succeed(f"rm {ROOT}/root && mkdir -m 0700 {ROOT}/root") boot() + + with subtest("a listed agent starts in its own user namespace with the state it had"): + NS = "${nsMachine}" + NS_AGENT = "${nsAgent}" + NS_DIR = "${nsDir}" + NS_BINDS = json.loads(${builtins.toJSON (builtins.toJSON nsBinds)}) + host.succeed(f"echo before > {NS_DIR}/state/pre.txt && chown 1000:1000 {NS_DIR}/state/pre.txt") + host.succeed(f"stat -c %u:%g {NS_DIR}/state/pre.txt | grep -x 1000:1000") + priv_request(host, {"op": "ensure_agent_socket_dir", "name": NS_AGENT}) + write = { + "op": "write_nspawn_flags", + "container": NS, + "binds": NS_BINDS, + "isolation": {"bridge": "${bridge}", "gateway_ip": "${gateway}"}, + } + priv_request(host, write) + ns_conf = host.succeed(f"cat /etc/nixos-containers/{NS}.conf") + assert "PRIVATE_USERS=pick" in ns_conf, ns_conf + assert "--private-users-ownership=map" in ns_conf, ns_conf + # Every bind, the read-only config included, is mounted owneridmap. + assert ns_conf.count(":owneridmap") == len(NS_BINDS), ns_conf + # The unlisted agent's conf has none of it: the control that the lookups + # above find these strings where they are. + assert "owneridmap" not in conf() and "PRIVATE_USERS" not in conf(), conf() + + # The data moved to the data owner, the config dir stayed root's. + for sub in ["state", "harness", "work", "claude"]: + host.succeed(f"stat -c %u:%g {NS_DIR}/{sub} | grep -x ${dataOwner}:${dataOwner}") + host.succeed(f"stat -c %u:%g {NS_DIR}/state/pre.txt | grep -x ${dataOwner}:${dataOwner}") + host.succeed(f"stat -c %u:%g {NS_DIR}/config/cfg | grep -x 0:0") + host.succeed(f"stat -c %u:%g ${nsSocketDir} | grep -x ${dataOwner}:${dataOwner}") + + boot_ns = lambda: ( + host.succeed(f"nixos-container start {NS}"), + host.wait_until_succeeds(f"nixos-container run {NS} -- systemctl is-active multi-user.target"), + ) + boot_ns() + def as_ns(script): + return host.succeed(f"nixos-container run {NS} -- runuser -l {NS_AGENT} -c '{script}'").strip() + # Inside, the data is the agent's: read, written and listed as itself. + assert as_ns(f"cat /agents/{NS_AGENT}/state/pre.txt") == "before" + assert as_ns(f"stat -c %u:%g /agents/{NS_AGENT}/state/pre.txt") == "1000:1000" + as_ns(f"echo after > /agents/{NS_AGENT}/state/post.txt") + as_ns(f"touch /run/hive-agent/{NS_AGENT}/web.sock") + assert as_ns(f"cat /agents/{NS_AGENT}/config/cfg") == "from-config" + host.fail(f"nixos-container run {NS} -- runuser -l {NS_AGENT} -c 'touch /agents/{NS_AGENT}/config/new'") + # On disk the new file belongs to the data owner, not to a picked range. + host.succeed(f"stat -c %u:%g {NS_DIR}/state/post.txt | grep -x ${dataOwner}:${dataOwner}") + host.succeed(f"stat -c %u:%g ${nsSocketDir}/web.sock | grep -x ${dataOwner}:${dataOwner}") + host.succeed(f"nixos-container run {NS} -- cat /proc/self/uid_map | grep -v ' 4294967295$'") + + # The move runs once: a file a host process leaves owned by 1000 after it + # is not chowned by the next request, and the agent restarts. + host.succeed(f"nixos-container stop {NS}") + host.succeed(f"echo late > {NS_DIR}/state/late.txt && chown 1000:1000 {NS_DIR}/state/late.txt") + priv_request(host, write) + host.succeed(f"stat -c %u:%g {NS_DIR}/state/late.txt | grep -x 1000:1000") + boot_ns() ''; }