diff --git a/docs/agent-lifecycle/persistence.md b/docs/agent-lifecycle/persistence.md index 66d71e9f..a271868a 100644 --- a/docs/agent-lifecycle/persistence.md +++ b/docs/agent-lifecycle/persistence.md @@ -488,6 +488,11 @@ hive-c0re can't read, or none, leaves the agent as it is. user. hive-priv mounts the agent's `state/`, `harness/`, `work/` and `claude/` dirs and its socket dir with `owneridmap`, and every other bind too (config, `/knowledge`, the runtime dir), whose owner shows as root inside. +- **`/nix/store` has to be on ext4, btrfs or xfs.** nspawn mounts the store + into a namespaced container idmapped, which other filesystems (overlayfs, + tmpfs, older zfs) refuse. `WriteNspawnFlags` for an opted-in agent checks + the `statfs` type of `/nix/store` and fails with the type and the fix, + before it moves anything. Check a host with `findmnt -no FSTYPE /nix/store`. - **The container starts with `--private-users=pick --private-users-ownership=map`.** nspawn idmaps the root at mount time, so a different range for `pick` never walks or chowns it. diff --git a/hive-priv/src/userns.rs b/hive-priv/src/userns.rs index 42be68b8..dde13745 100644 --- a/hive-priv/src/userns.rs +++ b/hive-priv/src/userns.rs @@ -43,6 +43,17 @@ const MARKER_ROOT: &str = "/var/lib/hive-priv/userns"; /// NixOS gives a normal user, which the agent is, an id from 1000 up. const MIN_AGENT_UID: u32 = 1000; +/// The store nspawn bind-mounts into every container; a namespaced container +/// mounts it idmapped. +const STORE: &str = "/nix/store"; + +/// The type of `statfs`'s `f_type`, which differs by target. +type FsType = libc::__fsword_t; + +/// `f_type` of the filesystems that support idmapped mounts and that a store +/// can sit on: ext4, btrfs and xfs. +const IDMAP_FS: [FsType; 3] = [0xEF53, 0x9123_683E, 0x5846_5342]; + /// The dirs under `/` the agent writes into. `config` /// is not one of them: hive-core writes it and the agent only reads it. `state` /// stays first: [`move_data`] opens it by name and opens the rest from `[1..]`. @@ -64,6 +75,8 @@ pub(crate) struct Env<'a> { /// Whether an agent's gid has to equal its uid: the id map copies the uid /// map for gids, so a group of another number has no mapping. pub(crate) same_gid: bool, + /// `f_type` of the filesystem under [`STORE`]. + pub(crate) store_fs: &'a dyn Fn() -> std::io::Result, /// Chown a mount point in the root template. pub(crate) chown: &'a dyn Fn(&Path, u32, u32) -> std::io::Result<()>, /// Chown the entry `name` of an open directory, following no link; `"."` @@ -84,6 +97,35 @@ fn chown_at(dir: BorrowedFd<'_>, name: &CStr, uid: u32, gid: u32) -> std::io::Re )?) } +/// `statfs(2)` `f_type` of `path`. +fn fs_type(path: &Path) -> std::io::Result { + use std::os::unix::ffi::OsStrExt as _; + let c_path = std::ffi::CString::new(path.as_os_str().as_bytes()) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; + // SAFETY: `c_path` is a valid NUL-terminated string that outlives the + // call; `statfs` only writes into the zero-initialised `buf`. + let mut buf: libc::statfs = unsafe { std::mem::zeroed() }; + if unsafe { libc::statfs(c_path.as_ptr(), &raw mut buf) } != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(buf.f_type) +} + +/// Refuse a store on a filesystem that cannot be idmapped: nspawn clones a +/// namespaced container's `/nix/store` bind with an idmap, which the kernel +/// refuses there, so the container would not boot. +fn check_store(agent: &str, f_type: std::io::Result) -> Result<()> { + let f_type = f_type.with_context(|| format!("{agent}: statfs {STORE}"))?; + if IDMAP_FS.contains(&f_type) { + return Ok(()); + } + bail!( + "{agent} sets privateUsers, but {STORE} is on a filesystem (f_type {f_type:#x}) that \ + cannot be idmapped; move {STORE} to ext4, btrfs or xfs, or leave privateUsers off for \ + this agent" + ) +} + /// The data owner when `agent` is to run in a user namespace, else `None`. /// /// `requested` is what hive-c0re sent from the closure record. `None` there @@ -154,6 +196,7 @@ pub(crate) fn activate( stopped, min_uid: MIN_AGENT_UID, same_gid: true, + store_fs: &|| fs_type(Path::new(STORE)), chown: &|path, uid, gid| lchown(path, Some(uid), Some(gid)), chown_at: &chown_at, }, @@ -178,6 +221,7 @@ pub(crate) fn activate_in(env: &Env<'_>, agent: &str, binds: &[BindMount]) -> Re } return Ok(false); }; + check_store(agent, (env.store_fs)())?; let ids = match moved { Some((was, uid, gid)) if was == owner => (uid, gid), Some((was, ..)) => { @@ -413,7 +457,7 @@ fn mount_points( #[cfg(test)] mod tests { - use super::{Env, activate_in, check_ids, owner_for}; + use super::{Env, activate_in, check_ids, check_store, owner_for}; use crate::shared::test_support::scratch; use hive_priv_sock::BindMount; use std::cell::RefCell; @@ -502,6 +546,7 @@ mod tests { stopped, min_uid, same_gid, + store_fs: &|| Ok(0xEF53), chown: &chown, chown_at: &chown_at, }; @@ -635,6 +680,55 @@ mod tests { ); } + /// An idmap-capable store passes, any other is refused with its type and + /// the fix before anything moves, and an agent without a namespace never + /// looks. + #[test] + fn the_store_has_to_be_on_a_filesystem_that_can_be_idmapped() { + for t in [0xEF53, 0x9123_683E, 0x5846_5342] { + check_store("a", Ok(t)).unwrap(); + } + // overlayfs and tmpfs + for t in [0x794c_7630, 0x0102_1994] { + let msg = format!("{:#}", check_store("a", Ok(t)).unwrap_err()); + assert!( + msg.contains(&format!("{t:#x}")) && msg.contains("ext4, btrfs or xfs"), + "{msg}" + ); + } + assert!(check_store("a", Err(std::io::Error::other("x"))).is_err()); + + let root = tree(); + let looked = std::cell::Cell::new(false); + let tmpfs = || { + looked.set(true); + Ok(0x0102_1994) + }; + let (state_root, rootfs) = (root.join("agents"), root.join("rootfs")); + std::fs::create_dir_all(&rootfs).unwrap(); + let chown = |_: &Path, _: u32, _: u32| Ok(()); + let chown_at = |_: std::os::fd::BorrowedFd<'_>, _: &std::ffi::CStr, _: u32, _: u32| Ok(()); + let mut env = Env { + state_root: &state_root, + socket_dir: &root.join("run/a"), + marker_root: &root.join("marker"), + rootfs: &rootfs, + owner: Some(OWNER), + stopped: true, + min_uid: 0, + same_gid: false, + store_fs: &tmpfs, + chown: &chown, + chown_at: &chown_at, + }; + assert!(activate_in(&env, "a", &[]).is_err(), "opted in, bad store"); + assert!(!root.join("marker/a").exists(), "refused before the move"); + env.owner = None; + looked.set(false); + assert!(!activate_in(&env, "a", &[]).unwrap()); + assert!(!looked.get(), "an unnamespaced agent does not look"); + } + /// A data dir that is a link is refused before anything past it is touched. #[test] fn a_linked_data_dir_is_refused() { diff --git a/nix/vm-tests/agent-ephemeral-rootfs.nix b/nix/vm-tests/agent-ephemeral-rootfs.nix index 9373f1f3..ff6689d1 100644 --- a/nix/vm-tests/agent-ephemeral-rootfs.nix +++ b/nix/vm-tests/agent-ephemeral-rootfs.nix @@ -337,6 +337,10 @@ pkgs.testers.runNixOSTest { NS_AGENT = "${nsAgent}" NS_DIR = "${nsDir}" NS_BINDS = json.loads(${builtins.toJSON (builtins.toJSON nsBinds)}) + # nspawn idmaps the /nix/store bind under a user namespace, and hive-priv + # refuses an opted-in agent whose store is on overlayfs; the image under + # it passes both. The remaining steps write nothing to the store. + host.succeed("mount --bind /nix/.ro-store /nix/store") host.succeed(f"echo before > {NS_DIR}/state/pre.txt && chown 1000:1000 {NS_DIR}/state/pre.txt") host.succeed(f"stat -c %u:%g {NS_DIR}/state/pre.txt | grep -x 1000:1000") priv_request(host, {"op": "ensure_agent_socket_dir", "name": NS_AGENT, "private_users": True}) @@ -368,10 +372,6 @@ pkgs.testers.runNixOSTest { host.succeed(f"stat -c %u:%g {NS_DIR}/config/cfg | grep -x 0:0") host.succeed("stat -c %u:%g ${nsSocketDir} | grep -x ${dataOwner}:${dataOwner}") - # nspawn idmaps the /nix/store bind under a user namespace, which an - # overlayfs store refuses; the image under it accepts it. The remaining - # steps write nothing to the store. - host.succeed("mount --bind /nix/.ro-store /nix/store") boot_ns = lambda: ( host.succeed(f"nixos-container start {NS}"), host.wait_until_succeeds(f"nixos-container run {NS} -- systemctl is-active multi-user.target"),