refuse a namespaced agent whose /nix/store cannot be idmapped

Refs #5888

docs/agent-lifecycle/persistence.md

@@ -488,6 +488,11 @@ hive-c0re can't read, or none, leaves the agent as it is.
   user. hive-priv mounts the agent's `state/`, `harness/`, `work/` and
   `claude/` dirs and its socket dir with `owneridmap`, and every other bind too
   (config, `/knowledge`, the runtime dir), whose owner shows as root inside.
+- **`/nix/store` has to be on ext4, btrfs or xfs.** nspawn mounts the store
+  into a namespaced container idmapped, which other filesystems (overlayfs,
+  tmpfs, older zfs) refuse. `WriteNspawnFlags` for an opted-in agent checks
+  the `statfs` type of `/nix/store` and fails with the type and the fix,
+  before it moves anything. Check a host with `findmnt -no FSTYPE /nix/store`.
 - **The container starts with `--private-users=pick --private-users-ownership=map`.**
   nspawn idmaps the root at mount time, so a different range for `pick` never
   walks or chowns it.

hive-priv/src/userns.rs

@@ -43,6 +43,17 @@ const MARKER_ROOT: &str = "/var/lib/hive-priv/userns";
 /// NixOS gives a normal user, which the agent is, an id from 1000 up.
 const MIN_AGENT_UID: u32 = 1000;
 
+/// The store nspawn bind-mounts into every container; a namespaced container
+/// mounts it idmapped.
+const STORE: &str = "/nix/store";
+
+/// The type of `statfs`'s `f_type`, which differs by target.
+type FsType = libc::__fsword_t;
+
+/// `f_type` of the filesystems that support idmapped mounts and that a store
+/// can sit on: ext4, btrfs and xfs.
+const IDMAP_FS: [FsType; 3] = [0xEF53, 0x9123_683E, 0x5846_5342];
+
 /// The dirs under `<AGENT_STATE_ROOT>/<agent>` the agent writes into. `config`
 /// is not one of them: hive-core writes it and the agent only reads it. `state`
 /// stays first: [`move_data`] opens it by name and opens the rest from `[1..]`.
@@ -64,6 +75,8 @@ pub(crate) struct Env<'a> {
     /// Whether an agent's gid has to equal its uid: the id map copies the uid
     /// map for gids, so a group of another number has no mapping.
     pub(crate) same_gid: bool,
+    /// `f_type` of the filesystem under [`STORE`].
+    pub(crate) store_fs: &'a dyn Fn() -> std::io::Result<FsType>,
     /// Chown a mount point in the root template.
     pub(crate) chown: &'a dyn Fn(&Path, u32, u32) -> std::io::Result<()>,
     /// Chown the entry `name` of an open directory, following no link; `"."`
@@ -84,6 +97,35 @@ fn chown_at(dir: BorrowedFd<'_>, name: &CStr, uid: u32, gid: u32) -> std::io::Re
     )?)
 }
 
+/// `statfs(2)` `f_type` of `path`.
+fn fs_type(path: &Path) -> std::io::Result<FsType> {
+    use std::os::unix::ffi::OsStrExt as _;
+    let c_path = std::ffi::CString::new(path.as_os_str().as_bytes())
+        .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
+    // SAFETY: `c_path` is a valid NUL-terminated string that outlives the
+    // call; `statfs` only writes into the zero-initialised `buf`.
+    let mut buf: libc::statfs = unsafe { std::mem::zeroed() };
+    if unsafe { libc::statfs(c_path.as_ptr(), &raw mut buf) } != 0 {
+        return Err(std::io::Error::last_os_error());
+    }
+    Ok(buf.f_type)
+}
+
+/// Refuse a store on a filesystem that cannot be idmapped: nspawn clones a
+/// namespaced container's `/nix/store` bind with an idmap, which the kernel
+/// refuses there, so the container would not boot.
+fn check_store(agent: &str, f_type: std::io::Result<FsType>) -> Result<()> {
+    let f_type = f_type.with_context(|| format!("{agent}: statfs {STORE}"))?;
+    if IDMAP_FS.contains(&f_type) {
+        return Ok(());
+    }
+    bail!(
+        "{agent} sets privateUsers, but {STORE} is on a filesystem (f_type {f_type:#x}) that \
+         cannot be idmapped; move {STORE} to ext4, btrfs or xfs, or leave privateUsers off for \
+         this agent"
+    )
+}
+
 /// The data owner when `agent` is to run in a user namespace, else `None`.
 ///
 /// `requested` is what hive-c0re sent from the closure record. `None` there
@@ -154,6 +196,7 @@ pub(crate) fn activate(
             stopped,
             min_uid: MIN_AGENT_UID,
             same_gid: true,
+            store_fs: &|| fs_type(Path::new(STORE)),
             chown: &|path, uid, gid| lchown(path, Some(uid), Some(gid)),
             chown_at: &chown_at,
         },
@@ -178,6 +221,7 @@ pub(crate) fn activate_in(env: &Env<'_>, agent: &str, binds: &[BindMount]) -> Re
         }
         return Ok(false);
     };
+    check_store(agent, (env.store_fs)())?;
     let ids = match moved {
         Some((was, uid, gid)) if was == owner => (uid, gid),
         Some((was, ..)) => {
@@ -413,7 +457,7 @@ fn mount_points(
 
 #[cfg(test)]
 mod tests {
-    use super::{Env, activate_in, check_ids, owner_for};
+    use super::{Env, activate_in, check_ids, check_store, owner_for};
     use crate::shared::test_support::scratch;
     use hive_priv_sock::BindMount;
     use std::cell::RefCell;
@@ -502,6 +546,7 @@ mod tests {
             stopped,
             min_uid,
             same_gid,
+            store_fs: &|| Ok(0xEF53),
             chown: &chown,
             chown_at: &chown_at,
         };
@@ -635,6 +680,55 @@ mod tests {
         );
     }
 
+    /// An idmap-capable store passes, any other is refused with its type and
+    /// the fix before anything moves, and an agent without a namespace never
+    /// looks.
+    #[test]
+    fn the_store_has_to_be_on_a_filesystem_that_can_be_idmapped() {
+        for t in [0xEF53, 0x9123_683E, 0x5846_5342] {
+            check_store("a", Ok(t)).unwrap();
+        }
+        // overlayfs and tmpfs
+        for t in [0x794c_7630, 0x0102_1994] {
+            let msg = format!("{:#}", check_store("a", Ok(t)).unwrap_err());
+            assert!(
+                msg.contains(&format!("{t:#x}")) && msg.contains("ext4, btrfs or xfs"),
+                "{msg}"
+            );
+        }
+        assert!(check_store("a", Err(std::io::Error::other("x"))).is_err());
+
+        let root = tree();
+        let looked = std::cell::Cell::new(false);
+        let tmpfs = || {
+            looked.set(true);
+            Ok(0x0102_1994)
+        };
+        let (state_root, rootfs) = (root.join("agents"), root.join("rootfs"));
+        std::fs::create_dir_all(&rootfs).unwrap();
+        let chown = |_: &Path, _: u32, _: u32| Ok(());
+        let chown_at = |_: std::os::fd::BorrowedFd<'_>, _: &std::ffi::CStr, _: u32, _: u32| Ok(());
+        let mut env = Env {
+            state_root: &state_root,
+            socket_dir: &root.join("run/a"),
+            marker_root: &root.join("marker"),
+            rootfs: &rootfs,
+            owner: Some(OWNER),
+            stopped: true,
+            min_uid: 0,
+            same_gid: false,
+            store_fs: &tmpfs,
+            chown: &chown,
+            chown_at: &chown_at,
+        };
+        assert!(activate_in(&env, "a", &[]).is_err(), "opted in, bad store");
+        assert!(!root.join("marker/a").exists(), "refused before the move");
+        env.owner = None;
+        looked.set(false);
+        assert!(!activate_in(&env, "a", &[]).unwrap());
+        assert!(!looked.get(), "an unnamespaced agent does not look");
+    }
+
     /// A data dir that is a link is refused before anything past it is touched.
     #[test]
     fn a_linked_data_dir_is_refused() {

nix/vm-tests/agent-ephemeral-rootfs.nix

@@ -337,6 +337,10 @@ pkgs.testers.runNixOSTest {
         NS_AGENT = "${nsAgent}"
         NS_DIR = "${nsDir}"
         NS_BINDS = json.loads(${builtins.toJSON (builtins.toJSON nsBinds)})
+        # nspawn idmaps the /nix/store bind under a user namespace, and hive-priv
+        # refuses an opted-in agent whose store is on overlayfs; the image under
+        # it passes both. The remaining steps write nothing to the store.
+        host.succeed("mount --bind /nix/.ro-store /nix/store")
         host.succeed(f"echo before > {NS_DIR}/state/pre.txt && chown 1000:1000 {NS_DIR}/state/pre.txt")
         host.succeed(f"stat -c %u:%g {NS_DIR}/state/pre.txt | grep -x 1000:1000")
         priv_request(host, {"op": "ensure_agent_socket_dir", "name": NS_AGENT, "private_users": True})
@@ -368,10 +372,6 @@ pkgs.testers.runNixOSTest {
         host.succeed(f"stat -c %u:%g {NS_DIR}/config/cfg | grep -x 0:0")
         host.succeed("stat -c %u:%g ${nsSocketDir} | grep -x ${dataOwner}:${dataOwner}")
 
-        # nspawn idmaps the /nix/store bind under a user namespace, which an
-        # overlayfs store refuses; the image under it accepts it. The remaining
-        # steps write nothing to the store.
-        host.succeed("mount --bind /nix/.ro-store /nix/store")
         boot_ns = lambda: (
             host.succeed(f"nixos-container start {NS}"),
             host.wait_until_succeeds(f"nixos-container run {NS} -- systemctl is-active multi-user.target"),