harden the ephemeral-root salvage: clone, check space, lock

- clone each file with FICLONE, copy the data only where that fails
- skip the salvage when the copy would leave too little free space
- hold a per-container flock and one copy at a time across containers
- run WriteNspawnFlags on the blocking pool

Cargo.lock

@@ -1990,6 +1990,7 @@ dependencies = [
  "anyhow",
  "hive-priv-sock",
  "libc",
+ "rustix",
  "serde_json",
  "tokio",
  "tracing",

docs/agent-lifecycle/persistence.md

@@ -414,6 +414,29 @@ which is the only gcroots path bound into a container, so only root writes it.
 Destroying the container removes the drop-in and then its root; the build stays
 until the next collection. A drop-in written by an older hive-priv has no
 `ExecStartPre=` and no root, and the container starts as before.
+
+**Disk use.** The copy shares blocks with the old root where it can: hive-priv
+clones each file (`FICLONE`), which on btrfs or xfs allocates metadata only, and
+copies the data of a file the filesystem refuses to clone. Before it creates
+anything, hive-priv sizes the root and compares it with the free space under
+`work/`. A salvage whose files clone (tested with one file of the root) must
+leave 2 GiB free. One that has to copy data must leave 15 % of the filesystem
+free. When the free space is short, hive-priv skips the salvage, deletes
+nothing, logs `salvage skipped: <container> need <size> free <size>`, and the
+container starts persistent; the next start tries again. hive-priv checks a file that fails to
+clone against the 2 GiB reserve again before it copies the data. After a
+salvage, hive-priv logs `salvage: <container> <size> reflinked=<n>/<m> files`.
+
+**Concurrency.** hive-priv holds an `flock` on
+`/run/hive-priv/salvage-<container>.lock` (the unit's `RuntimeDirectory`, which
+no container can write) for the whole `WriteNspawnFlags`, so a second request for
+the same container waits and then finds the finished template. Across
+containers, one salvage copies at a time, so the free space each one checks
+excludes any copy still in flight. The request runs on the blocking
+thread pool, so a long copy holds no async worker. hive-c0re's per-agent lease
+already serialises the requests it sends; the lock covers hive-priv's own
+guarantee.
+
 The journal still ships. The agent's collector (`nix/agent-modules/otel.nix`)
 reads the guest's own `/var/log/journal` and pushes it over OTLP, so no
 journal link to the host takes part; nspawn skips `--link-journal=try-guest`

hive-priv/Cargo.toml

@@ -11,6 +11,7 @@ workspace = true
 anyhow.workspace = true
 hive-priv-sock.workspace = true
 libc.workspace = true
+rustix.workspace = true
 serde_json.workspace = true
 tokio.workspace = true
 tracing.workspace = true

hive-priv/src/container_config.rs

@@ -61,6 +61,32 @@ pub(crate) fn handle_write_nspawn_flags(
     Ok((String::new(), String::new()))
 }
 
+/// [`handle_write_nspawn_flags`] on the blocking pool. The root salvage inside
+/// it copies files for as long as the root is big, and would hold a runtime
+/// worker for that time.
+///
+/// # Errors
+/// As [`handle_write_nspawn_flags`], or when the task fails to join.
+pub(crate) async fn handle_write_nspawn_flags_blocking(
+    container: String,
+    binds: Vec<BindMount>,
+    isolation: NetworkIsolation,
+    load_credentials: Vec<CredentialMount>,
+    hive_facts: HiveFacts,
+) -> Result<(String, String)> {
+    tokio::task::spawn_blocking(move || {
+        handle_write_nspawn_flags(
+            &container,
+            &binds,
+            &isolation,
+            &load_credentials,
+            &hive_facts,
+        )
+    })
+    .await
+    .context("write nspawn flags task")?
+}
+
 /// Validate a system-level container name (`h-` prefix for all agents,
 /// the manager included) and return the logical agent name after it.
 /// Anything else is refused, the swarm's forge and matrix containers

hive-priv/src/main.rs

@@ -20,8 +20,9 @@
 //! dispatch table. Each op family's handlers and validation live in their
 //! own module: [`container`] (lifecycle), [`container_config`] (nspawn flags,
 //! systemd drop-ins), [`rootfs`] (the salvage of a container's root before it
-//! turns ephemeral), [`prestart`] (the check of a container's root that runs
+//! turns ephemeral, with [`salvage_io`] for its space, reflink and lock calls),
-//! before each start, as a sub-command), [`storage`] (btrfs subvolumes, quotas, snapshots, disk
+//! [`prestart`] (the check of a container's root that runs before each start,
+//! as a sub-command), [`storage`] (btrfs subvolumes, quotas, snapshots, disk
 //! measurement), [`files`] (pause marker, socket dir, legacy tmpfiles),
 //! [`infra`] (gateway nginx, infra containers) and [`closure`] (downloads from
 //! the swarm cache). Helpers more than one family uses are in [`shared`].
@@ -33,6 +34,7 @@ mod files;
 mod infra;
 mod prestart;
 mod rootfs;
+mod salvage_io;
 mod shared;
 mod storage;
 
@@ -383,18 +385,21 @@ async fn exec(
         PrivRequest::ListContainers => container::container_run(&["list"]).await,
 
         PrivRequest::WriteNspawnFlags {
-            ref container,
-            ref binds,
-            ref isolation,
-            ref load_credentials,
-            ref hive_facts,
-        } => container_config::handle_write_nspawn_flags(
             container,
             binds,
             isolation,
             load_credentials,
             hive_facts,
-        ),
+        } => {
+            container_config::handle_write_nspawn_flags_blocking(
+                container,
+                binds,
+                isolation,
+                load_credentials,
+                hive_facts,
+            )
+            .await
+        }
 
         PrivRequest::EnsureAgentSocketDir { ref name } => files::ensure_agent_socket_dir(name),
 

hive-priv/src/rootfs.rs

@@ -16,14 +16,32 @@
 //!
 //! Only a stopped container is touched: a running one keeps whatever flag its
 //! root already warrants.
+//!
+//! Disk use. A file is cloned (`FICLONE`) when the filesystem can, which costs
+//! metadata only; otherwise its data is copied. Before creating anything,
+//! [`rotate`] compares what the copy needs with the free space under `work/` and
+//! returns [`Outcome::Skipped`] when less than [`COPY_FLOOR_PERCENT`] of the
+//! filesystem would stay free, or less than [`CLONE_RESERVE`] when a probe shows
+//! files clone. A file that fails to clone is checked against the reserve again.
+//!
+//! Concurrency. One salvage per container runs at a time, under an `flock` in
+//! [`LOCK_DIR`]; a second request waits, then sees the finished template. Across
+//! containers [`ONE_AT_A_TIME`] admits one copy at a time, so the free space a
+//! salvage reads excludes any copy still in flight.
 
-use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _, lchown, symlink};
+use std::fs::{File, OpenOptions};
+use std::io;
+use std::os::unix::fs::{
+    MetadataExt as _, OpenOptionsExt as _, PermissionsExt as _, lchown, symlink,
+};
 use std::path::{Path, PathBuf};
+use std::sync::{Mutex, PoisonError};
 
 use anyhow::{Context as _, Result, bail};
 use hive_priv_sock::AGENT_STATE_ROOT;
 
 use crate::container_config::{container_rootfs, write_container_etc_file_in};
+use crate::salvage_io::{Backend, ContainerLock, LOCK_DIR, ONE_AT_A_TIME, Os};
 
 /// The per-container nix profile bind on the host, which persists across starts.
 /// What nix keeps for the agent user is carried here; see [`carry_nix_state`].
@@ -54,6 +72,25 @@ const CACHE: [&str; 5] = [
     "var/lib/opentelemetry-collector",
 ];
 
+/// Share of the filesystem under `work/` that must stay free after a salvage
+/// that copies data. btrfs allocates metadata chunks from unallocated space, and
+/// every agent and host service writes to the same filesystem, so a full disk
+/// stops all of them.
+const COPY_FLOOR_PERCENT: u64 = 15;
+
+/// Free space a salvage must leave when its files clone, which allocate
+/// metadata only; also the margin kept when a single file has to be copied.
+const CLONE_RESERVE: u64 = 2 << 30;
+
+/// Metadata a cloned entry is assumed to cost.
+const ENTRY_COST: u64 = 8 << 10;
+
+/// Allocation unit a copied file is rounded up to.
+const BLOCK: u64 = 4096;
+
+/// The file in `work/` that tests whether cloning into it works.
+const CLONE_PROBE: &str = ".hive-clone-probe";
+
 /// What [`rotate`] did.
 #[derive(Debug, PartialEq, Eq)]
 pub(crate) enum Outcome {
@@ -61,7 +98,34 @@ pub(crate) enum Outcome {
     Clean,
     /// The root's content is copied to `salvage`, and the root itself kept at
     /// `old_root`.
-    Salvaged { salvage: PathBuf, old_root: PathBuf },
+    Salvaged {
+        salvage: PathBuf,
+        old_root: PathBuf,
+        stats: Stats,
+    },
+    /// The copy would leave too little free space: `need` bytes must be free,
+    /// `free` are. Nothing was created or moved.
+    Skipped { need: u64, free: u64 },
+}
+
+/// What a salvage copied.
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
+pub(crate) struct Stats {
+    /// Bytes in the regular files.
+    pub(crate) bytes: u64,
+    /// Regular files copied.
+    pub(crate) files: u64,
+    /// Of those, the files that share blocks with the old root.
+    pub(crate) cloned: u64,
+}
+
+/// The host a salvage runs against.
+pub(crate) struct Env<'a> {
+    pub(crate) backend: &'a dyn Backend,
+    /// Where the per-container lock files go.
+    pub(crate) lock_dir: &'a Path,
+    /// Taken by the salvage that is copying; see the module docs.
+    pub(crate) one_at_a_time: &'a Mutex<()>,
 }
 
 /// Whether `container` may be started with `--ephemeral`, salvaging its root
@@ -70,7 +134,8 @@ pub(crate) enum Outcome {
 /// # Errors
 /// Returns an error when the unit state cannot be read or the salvage fails.
 /// Nothing is deleted on that path, and the caller starts the container
-/// without the flag.
+/// without the flag. A salvage skipped for lack of space is not an error: it
+/// returns `false` and the next start tries again.
 pub(crate) fn prepare(container: &str, agent: &str) -> Result<bool> {
     let root = container_rootfs(container);
     if !unit_is_stopped(container)? {
@@ -78,47 +143,231 @@ pub(crate) fn prepare(container: &str, agent: &str) -> Result<bool> {
     }
     let work = Path::new(AGENT_STATE_ROOT).join(agent).join("work");
     let profiles = Path::new(PROFILES_ROOT).join(container);
-    match rotate(&root, agent, &work, &profiles, &today())? {
+    let env = Env {
+        backend: &Os,
+        lock_dir: Path::new(LOCK_DIR),
+        one_at_a_time: &ONE_AT_A_TIME,
+    };
+    let outcome = rotate(&root, agent, &work, &profiles, &today(), &env)?;
+    Ok(report(container, &outcome))
+}
+
+/// Log what [`rotate`] did, and say whether the container may start `--ephemeral`.
+fn report(container: &str, outcome: &Outcome) -> bool {
+    match outcome {
         Outcome::Clean => {}
-        Outcome::Salvaged { salvage, old_root } => tracing::warn!(
+        Outcome::Salvaged {
-            %container,
+            salvage,
-            salvage = %salvage.display(),
+            old_root,
-            old_root = %old_root.display(),
+            stats,
-            "root salvaged before the first ephemeral start; the old root is kept, remove it by hand"
+        } => {
-        ),
+            tracing::info!(
+                "salvage: {container} {} reflinked={}/{} files",
+                gib(stats.bytes),
+                stats.cloned,
+                stats.files
+            );
+            tracing::warn!(
+                %container,
+                salvage = %salvage.display(),
+                old_root = %old_root.display(),
+                "root salvaged before the first ephemeral start; the old root is kept, remove it by hand"
+            );
+        }
+        Outcome::Skipped { need, free } => {
+            tracing::warn!(
+                "salvage skipped: {container} need {} free {}",
+                gib(*need),
+                gib(*free)
+            );
+            return false;
+        }
     }
-    Ok(true)
+    true
+}
+
+/// `bytes` in GiB, to one decimal.
+fn gib(bytes: u64) -> String {
+    const GIB: u64 = 1 << 30;
+    format!("{}.{} GiB", bytes / GIB, bytes % GIB * 10 / GIB)
 }
 
 /// Salvage `root` if it holds content, then leave a marked template in its
 /// place. `work` is the agent's work dir on the host, `profiles` the host side of
 /// the container's `/nix/var/nix/profiles`; `date` is `YYYYMMDD`.
-fn rotate(root: &Path, agent: &str, work: &Path, profiles: &Path, date: &str) -> Result<Outcome> {
+///
+/// The container's lock is held throughout, so a second call for the same root
+/// waits and then finds the template the first one left.
+fn rotate(
+    root: &Path,
+    agent: &str,
+    work: &Path,
+    profiles: &Path,
+    date: &str,
+    env: &Env<'_>,
+) -> Result<Outcome> {
+    let name = root
+        .file_name()
+        .and_then(|n| n.to_str())
+        .with_context(|| format!("{} has no usable name", root.display()))?;
+    let _lock = ContainerLock::acquire(env.lock_dir, name)?;
     let outcome = if has_content(root)? {
-        let (uid, gid) = agent_owner(root, agent, work)?;
+        let _turn = env
-        let machine_id = read_machine_id(root);
+            .one_at_a_time
-        let salvage = fresh_dir(work, &format!("salvaged-rootfs-{date}"))?;
+            .lock()
-        lchown(&salvage, Some(uid), Some(gid))
+            .unwrap_or_else(PoisonError::into_inner);
-            .with_context(|| format!("chown {}", salvage.display()))?;
+        salvage(root, agent, work, profiles, date, env)?
-        copy_tree(root, &salvage, Path::new(""), uid, gid)
-            .with_context(|| format!("copy {} to {}", root.display(), salvage.display()))?;
-        carry_nix_state(root, agent, profiles, uid, gid)?;
-        let old_root = free_sibling(root, &format!("pre-ephemeral-{date}"))?;
-        std::fs::rename(root, &old_root)
-            .with_context(|| format!("move {} to {}", root.display(), old_root.display()))?;
-        std::fs::create_dir(root).with_context(|| format!("create {}", root.display()))?;
-        if let Some(id) = machine_id {
-            write_container_etc_file_in(root, "machine-id", &format!("{id}\n"))?;
-        }
-        Outcome::Salvaged { salvage, old_root }
     } else {
         Outcome::Clean
     };
+    if matches!(outcome, Outcome::Skipped { .. }) {
+        return Ok(outcome);
+    }
     ensure_machine_id(root)?;
     write_container_etc_file_in(root, MARKER, "")?;
     Ok(outcome)
 }
 
+/// The copy-and-move half of [`rotate`], for a root that has content.
+fn salvage(
+    root: &Path,
+    agent: &str,
+    work: &Path,
+    profiles: &Path,
+    date: &str,
+    env: &Env<'_>,
+) -> Result<Outcome> {
+    let (uid, gid) = agent_owner(root, agent, work)?;
+    let machine_id = read_machine_id(root);
+    if let Some((need, free)) = shortfall(env.backend, root, work)? {
+        return Ok(Outcome::Skipped { need, free });
+    }
+    let salvage = fresh_dir(work, &format!("salvaged-rootfs-{date}"))?;
+    lchown(&salvage, Some(uid), Some(gid))
+        .with_context(|| format!("chown {}", salvage.display()))?;
+    let mut copier = Copier::new(env.backend, uid, gid);
+    copier
+        .tree(root, &salvage, Path::new(""))
+        .with_context(|| format!("copy {} to {}", root.display(), salvage.display()))?;
+    carry_nix_state(root, agent, profiles, env.backend, uid, gid)?;
+    let old_root = free_sibling(root, &format!("pre-ephemeral-{date}"))?;
+    std::fs::rename(root, &old_root)
+        .with_context(|| format!("move {} to {}", root.display(), old_root.display()))?;
+    std::fs::create_dir(root).with_context(|| format!("create {}", root.display()))?;
+    if let Some(id) = machine_id {
+        write_container_etc_file_in(root, "machine-id", &format!("{id}\n"))?;
+    }
+    Ok(Outcome::Salvaged {
+        salvage,
+        old_root,
+        stats: copier.stats,
+    })
+}
+
+/// What [`measure`] found under a root.
+struct Tally {
+    /// Everything outside [`CACHE`], directories and links included.
+    entries: u64,
+    /// Regular-file bytes, each file rounded up to [`BLOCK`].
+    bytes: u64,
+    /// The first regular file with data, for the clone probe.
+    sample: Option<PathBuf>,
+}
+
+/// Size up what [`Copier::tree`] would copy from `root`.
+fn measure(root: &Path) -> Result<Tally> {
+    fn walk(dir: &Path, rel: &Path, tally: &mut Tally) -> Result<()> {
+        for entry in std::fs::read_dir(dir).with_context(|| format!("read {}", dir.display()))? {
+            let entry = entry.with_context(|| format!("read {}", dir.display()))?;
+            let rel = rel.join(entry.file_name());
+            if is_cache(&rel) {
+                continue;
+            }
+            let meta = entry
+                .metadata()
+                .with_context(|| format!("stat {}", entry.path().display()))?;
+            tally.entries += 1;
+            if meta.is_dir() {
+                walk(&entry.path(), &rel, tally)?;
+            } else if meta.is_file() {
+                tally.bytes = tally
+                    .bytes
+                    .saturating_add(meta.len().div_ceil(BLOCK) * BLOCK);
+                if meta.len() > 0 && tally.sample.is_none() {
+                    tally.sample = Some(entry.path());
+                }
+            }
+        }
+        Ok(())
+    }
+    let mut tally = Tally {
+        entries: 0,
+        bytes: 0,
+        sample: None,
+    };
+    walk(root, Path::new(""), &mut tally)?;
+    Ok(tally)
+}
+
+/// `Some((need, free))` when a salvage of `root` into `work` would leave less
+/// free space than it must: `need` bytes have to be free, `free` are.
+///
+/// Files that clone cost metadata only. Whether they do is probed with one
+/// file of the root ([`clones`]); when that fails the whole data is counted.
+fn shortfall(backend: &dyn Backend, root: &Path, work: &Path) -> Result<Option<(u64, u64)>> {
+    let tally = measure(root)?;
+    let space = backend
+        .space(work)
+        .with_context(|| format!("free space of {}", work.display()))?;
+    let metadata = tally.entries.saturating_mul(ENTRY_COST);
+    let need = if tally
+        .sample
+        .as_deref()
+        .is_none_or(|sample| clones(backend, sample, work))
+    {
+        metadata.saturating_add(CLONE_RESERVE)
+    } else {
+        let floor = space.total / 100 * COPY_FLOOR_PERCENT;
+        tally.bytes.saturating_add(metadata).saturating_add(floor)
+    };
+    Ok((space.free < need).then_some((need, space.free)))
+}
+
+/// Whether `sample` can be cloned into a file in `work`. The probe file is
+/// created exclusively and removed again; if one is already there the answer
+/// is no.
+fn clones(backend: &dyn Backend, sample: &Path, work: &Path) -> bool {
+    let probe = work.join(CLONE_PROBE);
+    let Ok(src) = open_nofollow(sample) else {
+        return false;
+    };
+    let Ok(dst) = create_new(&probe) else {
+        return false;
+    };
+    let cloned = backend.clone_file(&src, &dst).is_ok();
+    drop(dst);
+    std::fs::remove_file(&probe).ok();
+    cloned
+}
+
+/// `path` opened for reading; a link at `path` is an error.
+fn open_nofollow(path: &Path) -> io::Result<File> {
+    OpenOptions::new()
+        .read(true)
+        .custom_flags(libc::O_NOFOLLOW)
+        .open(path)
+}
+
+/// `path` created for writing, mode 0600; an existing entry is an error.
+fn create_new(path: &Path) -> io::Result<File> {
+    OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .mode(0o600)
+        .custom_flags(libc::O_NOFOLLOW)
+        .open(path)
+}
+
 /// Whether anything but directories, [`TEMPLATE_FILES`] and [`CACHE`] is under
 /// `root`. A missing root has none.
 fn has_content(root: &Path) -> Result<bool> {
@@ -198,7 +447,14 @@ fn real_dir(base: &Path, parts: &[&str]) -> Option<PathBuf> {
 /// GC roots that name them are still valid after the switch. A target that
 /// holds anything is left alone; so is the whole copy when a component of the
 /// target path is not a real directory, as the container can write `profiles`.
-fn carry_nix_state(root: &Path, agent: &str, profiles: &Path, uid: u32, gid: u32) -> Result<()> {
+fn carry_nix_state(
+    root: &Path,
+    agent: &str,
+    profiles: &Path,
+    backend: &dyn Backend,
+    uid: u32,
+    gid: u32,
+) -> Result<()> {
     let Some(src) = real_dir(root, &["home", agent, ".local", "state", "nix"]) else {
         return Ok(());
     };
@@ -218,7 +474,8 @@ fn carry_nix_state(root: &Path, agent: &str, profiles: &Path, uid: u32, gid: u32
     {
         return Ok(());
     }
-    copy_tree(&src, &target, Path::new(""), uid, gid)
+    Copier::new(backend, uid, gid)
+        .tree(&src, &target, Path::new(""))
         .with_context(|| format!("copy {} to {}", src.display(), target.display()))
 }
 
@@ -287,49 +544,101 @@ fn free_sibling(root: &Path, suffix: &str) -> Result<PathBuf> {
     unreachable!("the loop above returns or continues")
 }
 
-/// Copy everything under `src` except [`CACHE`] into `dst`, owned by
+/// Copies a tree for [`salvage`] and keeps count.
-/// `uid:gid`, with modes and mtimes kept. Sockets, fifos and device nodes are
+struct Copier<'a> {
-/// skipped; hard links become separate copies.
+    backend: &'a dyn Backend,
-fn copy_tree(src: &Path, dst: &Path, rel: &Path, uid: u32, gid: u32) -> Result<()> {
+    uid: u32,
-    let dir = src.join(rel);
+    gid: u32,
-    for entry in std::fs::read_dir(&dir).with_context(|| format!("read {}", dir.display()))? {
+    stats: Stats,
-        let entry = entry.with_context(|| format!("read {}", dir.display()))?;
+}
-        let rel = rel.join(entry.file_name());
+
-        if is_cache(&rel) {
+impl<'a> Copier<'a> {
-            continue;
+    fn new(backend: &'a dyn Backend, uid: u32, gid: u32) -> Self {
-        }
+        Self {
-        let from = src.join(&rel);
+            backend,
-        let to = dst.join(&rel);
+            uid,
-        let meta =
+            gid,
-            std::fs::symlink_metadata(&from).with_context(|| format!("stat {}", from.display()))?;
+            stats: Stats::default(),
-        let kind = meta.file_type();
-        if kind.is_dir() {
-            std::fs::create_dir(&to).with_context(|| format!("create {}", to.display()))?;
-            copy_tree(src, dst, &rel, uid, gid)?;
-            lchown(&to, Some(uid), Some(gid)).with_context(|| format!("chown {}", to.display()))?;
-            set_mode(&to, &meta)?;
-        } else if kind.is_symlink() {
-            let target = std::fs::read_link(&from)
-                .with_context(|| format!("read link {}", from.display()))?;
-            symlink(&target, &to).with_context(|| format!("link {}", to.display()))?;
-            lchown(&to, Some(uid), Some(gid)).with_context(|| format!("chown {}", to.display()))?;
-        } else if kind.is_file() {
-            std::fs::copy(&from, &to)
-                .with_context(|| format!("copy {} to {}", from.display(), to.display()))?;
-            if let Ok(mtime) = meta.modified() {
-                std::fs::File::options()
-                    .write(true)
-                    .open(&to)
-                    .and_then(|f| f.set_modified(mtime))
-                    .with_context(|| format!("set mtime of {}", to.display()))?;
-            }
-            lchown(&to, Some(uid), Some(gid)).with_context(|| format!("chown {}", to.display()))?;
-            set_mode(&to, &meta)?;
-        } else {
-            tracing::info!(path = %from.display(), "salvage skips a socket, fifo or device node");
         }
     }
-    Ok(())
+
+    /// Copy everything under `src` except [`CACHE`] into `dst`, owned by
+    /// `uid:gid`, with modes and mtimes kept. Sockets, fifos and device nodes
+    /// are skipped; hard links become separate copies.
+    fn tree(&mut self, src: &Path, dst: &Path, rel: &Path) -> Result<()> {
+        let (uid, gid) = (self.uid, self.gid);
+        let dir = src.join(rel);
+        for entry in std::fs::read_dir(&dir).with_context(|| format!("read {}", dir.display()))? {
+            let entry = entry.with_context(|| format!("read {}", dir.display()))?;
+            let rel = rel.join(entry.file_name());
+            if is_cache(&rel) {
+                continue;
+            }
+            let from = src.join(&rel);
+            let to = dst.join(&rel);
+            let meta = std::fs::symlink_metadata(&from)
+                .with_context(|| format!("stat {}", from.display()))?;
+            let kind = meta.file_type();
+            if kind.is_dir() {
+                std::fs::create_dir(&to).with_context(|| format!("create {}", to.display()))?;
+                self.tree(src, dst, &rel)?;
+                lchown(&to, Some(uid), Some(gid))
+                    .with_context(|| format!("chown {}", to.display()))?;
+                set_mode(&to, &meta)?;
+            } else if kind.is_symlink() {
+                let target = std::fs::read_link(&from)
+                    .with_context(|| format!("read link {}", from.display()))?;
+                symlink(&target, &to).with_context(|| format!("link {}", to.display()))?;
+                lchown(&to, Some(uid), Some(gid))
+                    .with_context(|| format!("chown {}", to.display()))?;
+            } else if kind.is_file() {
+                self.file(&from, &to, &meta)?;
+            } else {
+                tracing::info!(path = %from.display(), "salvage skips a socket, fifo or device node");
+            }
+        }
+        Ok(())
+    }
+
+    /// One regular file: cloned when the filesystem allows it, else copied.
+    fn file(&mut self, from: &Path, to: &Path, meta: &std::fs::Metadata) -> Result<()> {
+        let mut src = open_nofollow(from).with_context(|| format!("open {}", from.display()))?;
+        let mut dst = create_new(to).with_context(|| format!("create {}", to.display()))?;
+        self.stats.files += 1;
+        self.stats.bytes = self.stats.bytes.saturating_add(meta.len());
+        if self.backend.clone_file(&src, &dst).is_ok() {
+            self.stats.cloned += 1;
+        } else {
+            self.check_room(to, meta.len())?;
+            io::copy(&mut src, &mut dst)
+                .with_context(|| format!("copy {} to {}", from.display(), to.display()))?;
+        }
+        if let Ok(mtime) = meta.modified() {
+            dst.set_modified(mtime)
+                .with_context(|| format!("set mtime of {}", to.display()))?;
+        }
+        lchown(to, Some(self.uid), Some(self.gid))
+            .with_context(|| format!("chown {}", to.display()))?;
+        set_mode(to, meta)
+    }
+
+    /// Fail when copying `len` bytes to `to` would leave less than
+    /// [`CLONE_RESERVE`] free.
+    fn check_room(&self, to: &Path, len: u64) -> Result<()> {
+        let dir = to.parent().unwrap_or(to);
+        let space = self
+            .backend
+            .space(dir)
+            .with_context(|| format!("free space of {}", dir.display()))?;
+        if space.free < len.saturating_add(CLONE_RESERVE) {
+            bail!(
+                "{} is {len} bytes and the filesystem has {} free; at least {CLONE_RESERVE} must stay",
+                to.display(),
+                space.free
+            );
+        }
+        Ok(())
+    }
 }
 
 /// Give `path` the permission bits of `meta`, which `chown` may have cleared.
@@ -406,14 +715,108 @@ fn date_of(secs: u64) -> String {
 
 #[cfg(test)]
 mod tests {
+    use std::fs::File;
+    use std::io;
     use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _, lchown, symlink};
     use std::path::{Path, PathBuf};
+    use std::sync::Mutex;
+    use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
+    use std::time::Duration;
 
-    use super::{MARKER, Outcome, date_of, has_content, rotate};
+    use anyhow::Result;
+
+    use super::{
+        CLONE_RESERVE, Env, MARKER, Outcome, Stats, date_of, gib, has_content, report, rotate,
+    };
+    use crate::salvage_io::{Backend, FsSpace, Os};
     use crate::shared::test_support::scratch;
 
     const DATE: &str = "20261028";
 
+    /// How [`Fake`] answers a clone request.
+    #[derive(Clone, Copy)]
+    enum CloneMode {
+        /// Succeeds, by copying the bytes.
+        Copy,
+        /// Fails with this errno.
+        Fail(i32),
+        /// Succeeds for the first request (the probe), fails for the rest.
+        FirstOnly,
+        /// Defers to the real `ioctl(FICLONE)`.
+        Real,
+    }
+
+    /// A [`Backend`] with the free space and the clone result set by the test.
+    struct Fake {
+        free: u64,
+        total: u64,
+        mode: CloneMode,
+        delay: Duration,
+        calls: AtomicU32,
+        started: AtomicBool,
+        active: AtomicU32,
+        peak: AtomicU32,
+    }
+
+    impl Fake {
+        /// Plenty of space; every clone fails with `EXDEV`, so files are copied.
+        fn new() -> Self {
+            Self {
+                free: 1 << 50,
+                total: 1 << 51,
+                mode: CloneMode::Fail(libc::EXDEV),
+                delay: Duration::ZERO,
+                calls: AtomicU32::new(0),
+                started: AtomicBool::new(false),
+                active: AtomicU32::new(0),
+                peak: AtomicU32::new(0),
+            }
+        }
+
+        fn space(mut self, free: u64, total: u64) -> Self {
+            self.free = free;
+            self.total = total;
+            self
+        }
+
+        fn mode(mut self, mode: CloneMode) -> Self {
+            self.mode = mode;
+            self
+        }
+
+        fn delay(mut self, delay: Duration) -> Self {
+            self.delay = delay;
+            self
+        }
+    }
+
+    impl Backend for Fake {
+        fn space(&self, _path: &Path) -> io::Result<FsSpace> {
+            Ok(FsSpace {
+                free: self.free,
+                total: self.total,
+            })
+        }
+
+        fn clone_file(&self, src: &File, dst: &File) -> io::Result<()> {
+            let call = self.calls.fetch_add(1, Ordering::SeqCst);
+            self.started.store(true, Ordering::SeqCst);
+            let active = self.active.fetch_add(1, Ordering::SeqCst) + 1;
+            self.peak.fetch_max(active, Ordering::SeqCst);
+            std::thread::sleep(self.delay);
+            let copy = || io::copy(&mut &*src, &mut &*dst).map(drop);
+            let result = match self.mode {
+                CloneMode::Copy => copy(),
+                CloneMode::Fail(errno) => Err(io::Error::from_raw_os_error(errno)),
+                CloneMode::FirstOnly if call == 0 => copy(),
+                CloneMode::FirstOnly => Err(io::Error::from_raw_os_error(libc::EXDEV)),
+                CloneMode::Real => Os.clone_file(src, dst),
+            };
+            self.active.fetch_sub(1, Ordering::SeqCst);
+            result
+        }
+    }
+
     /// A stopped agent's old root and work dir under one scratch dir. The
     /// "agent" is the test's own user, the one it can chown to.
     struct Fixture {
@@ -421,6 +824,8 @@ mod tests {
         root: PathBuf,
         work: PathBuf,
         profiles: PathBuf,
+        fake: Fake,
+        serial: Mutex<()>,
     }
 
     impl Fixture {
@@ -437,17 +842,38 @@ mod tests {
                 root,
                 work,
                 profiles,
+                fake: Fake::new(),
+                serial: Mutex::new(()),
             }
         }
 
+        fn with(fake: Fake) -> Self {
+            let mut f = Self::new();
+            f.fake = fake;
+            f
+        }
+
         fn put(&self, rel: &str, body: &str) {
             let path = self.root.join(rel);
             std::fs::create_dir_all(path.parent().unwrap()).unwrap();
             std::fs::write(path, body).unwrap();
         }
 
+        fn try_rotate_in(&self, backend: &dyn Backend, serial: &Mutex<()>) -> Result<Outcome> {
+            let env = Env {
+                backend,
+                lock_dir: &self.top.join("locks"),
+                one_at_a_time: serial,
+            };
+            rotate(&self.root, "iris", &self.work, &self.profiles, DATE, &env)
+        }
+
+        fn try_rotate(&self) -> Result<Outcome> {
+            self.try_rotate_in(&self.fake, &self.serial)
+        }
+
         fn rotate(&self) -> Outcome {
-            rotate(&self.root, "iris", &self.work, &self.profiles, DATE).unwrap()
+            self.try_rotate().unwrap()
         }
 
         /// Where the container finds nix's per-user state, on the host.
@@ -483,7 +909,10 @@ mod tests {
         f.put("home/iris/notes.txt", "keep me");
         f.put("etc/hostname", "iris");
 
-        let Outcome::Salvaged { salvage, old_root } = f.rotate() else {
+        let Outcome::Salvaged {
+            salvage, old_root, ..
+        } = f.rotate()
+        else {
             panic!("the first call must salvage");
         };
         assert_eq!(salvage, f.salvage());
@@ -552,7 +981,10 @@ mod tests {
         ] {
             f.put(real, "real");
         }
-        let Outcome::Salvaged { salvage, old_root } = f.rotate() else {
+        let Outcome::Salvaged {
+            salvage, old_root, ..
+        } = f.rotate()
+        else {
             panic!("must salvage");
         };
         for cache in [
@@ -590,10 +1022,7 @@ mod tests {
         assert!(f.root.join("etc").join(MARKER).is_file());
         // The same call with one real file makes the control: it salvages.
         f.put("home/iris/real", "x");
-        assert!(matches!(
+        assert!(matches!(f.rotate(), Outcome::Salvaged { .. }));
-            rotate(&f.root, "iris", &f.work, &f.profiles, DATE).unwrap(),
-            Outcome::Salvaged { .. }
-        ));
     }
 
     /// Nothing is removed: the old root keeps every file, and a failed
@@ -619,7 +1048,7 @@ mod tests {
         g.put("home/iris/file", "data");
         std::fs::remove_dir(&g.work).unwrap();
         std::fs::write(&g.work, "not a dir").unwrap();
-        assert!(rotate(&g.root, "iris", &g.work, &g.profiles, DATE).is_err());
+        assert!(g.try_rotate().is_err());
         assert_eq!(
             std::fs::read_to_string(g.root.join("home/iris/file")).unwrap(),
             "data"
@@ -666,7 +1095,10 @@ mod tests {
             lchown(f.root.join("home/iris"), None, Some(other)).unwrap();
         }
 
-        let Outcome::Salvaged { salvage, old_root } = f.rotate() else {
+        let Outcome::Salvaged {
+            salvage, old_root, ..
+        } = f.rotate()
+        else {
             panic!("must salvage");
         };
         let me = std::fs::metadata(old_root.join("home/iris")).unwrap();
@@ -833,6 +1265,219 @@ mod tests {
         }
     }
 
+    const GIB: u64 = 1 << 30;
+
+    /// A copy that would leave less than 15 % of the filesystem free is skipped
+    /// and changes nothing; the same root with the space for it is salvaged.
+    #[test]
+    fn a_copy_that_leaves_too_little_free_is_skipped() {
+        // A 100 GiB filesystem keeps 15 GiB free; the root needs a little more.
+        let f = Fixture::with(Fake::new().space(15 * GIB, 100 * GIB));
+        f.put("home/iris/data", "x");
+        let skipped = f.rotate();
+        let Outcome::Skipped { need, free } = skipped else {
+            panic!("a copy into 15 GiB free must be skipped, got {skipped:?}");
+        };
+        assert_eq!(free, 15 * GIB);
+        assert!(need > 15 * GIB, "need {need}");
+        assert!(
+            f.work.read_dir().unwrap().next().is_none(),
+            "work/ was touched"
+        );
+        assert!(f.root.join("home/iris/data").is_file());
+        assert!(!f.root.join("etc").join(MARKER).exists());
+        assert!(
+            !report("h-iris", &skipped),
+            "a skipped salvage must start persistent"
+        );
+
+        // Control: a little more room, the same root.
+        let g = Fixture::with(Fake::new().space(15 * GIB + (1 << 20), 100 * GIB));
+        g.put("home/iris/data", "x");
+        let salvaged = g.rotate();
+        assert!(matches!(salvaged, Outcome::Salvaged { .. }), "{salvaged:?}");
+        assert!(report("h-iris", &salvaged));
+        assert!(report("h-iris", &Outcome::Clean));
+    }
+
+    /// Files that clone cost metadata, so the 15 % floor does not apply: the
+    /// same root and space that skip when files must be copied go through when
+    /// the probe clones.
+    #[test]
+    fn files_that_clone_need_only_metadata() {
+        let copying = Fixture::with(Fake::new().space(5 * GIB, 100 * GIB));
+        copying.put("home/iris/data", "x");
+        assert!(matches!(copying.rotate(), Outcome::Skipped { .. }));
+
+        let cloning = Fixture::with(Fake::new().space(5 * GIB, 100 * GIB).mode(CloneMode::Copy));
+        cloning.put("home/iris/data", "x");
+        let Outcome::Salvaged { stats, .. } = cloning.rotate() else {
+            panic!("cloning files need no more than the reserve");
+        };
+        assert_eq!(
+            stats,
+            Stats {
+                bytes: 1,
+                files: 1,
+                cloned: 1
+            }
+        );
+
+        // Not even metadata fits below the reserve.
+        let full = Fixture::with(
+            Fake::new()
+                .space(CLONE_RESERVE - 1, 100 * GIB)
+                .mode(CloneMode::Copy),
+        );
+        full.put("home/iris/data", "x");
+        assert!(matches!(full.rotate(), Outcome::Skipped { .. }));
+    }
+
+    /// A clone that fails (another filesystem, one without reflinks) leaves the
+    /// file to be copied, whatever the errno; the control counts a clone.
+    #[test]
+    fn a_failed_clone_falls_back_to_a_copy() {
+        for errno in [libc::EXDEV, libc::EOPNOTSUPP, libc::EINVAL] {
+            let f = Fixture::with(Fake::new().mode(CloneMode::Fail(errno)));
+            f.put("home/iris/a", "alpha");
+            f.put("home/iris/b", "beta");
+            let Outcome::Salvaged { salvage, stats, .. } = f.rotate() else {
+                panic!("errno {errno}: must salvage");
+            };
+            assert_eq!(
+                std::fs::read_to_string(salvage.join("home/iris/a")).unwrap(),
+                "alpha"
+            );
+            assert_eq!(
+                std::fs::read_to_string(salvage.join("home/iris/b")).unwrap(),
+                "beta"
+            );
+            assert_eq!(
+                stats,
+                Stats {
+                    bytes: 9,
+                    files: 2,
+                    cloned: 0
+                },
+                "errno {errno}"
+            );
+        }
+        let f = Fixture::with(Fake::new().mode(CloneMode::Copy));
+        f.put("home/iris/a", "alpha");
+        let Outcome::Salvaged { salvage, stats, .. } = f.rotate() else {
+            panic!("must salvage");
+        };
+        assert_eq!(stats.cloned, 1);
+        assert_eq!(
+            std::fs::read_to_string(salvage.join("home/iris/a")).unwrap(),
+            "alpha"
+        );
+    }
+
+    /// The real `ioctl(FICLONE)`: whether the scratch filesystem has reflinks or
+    /// not, the salvage holds the same bytes.
+    #[test]
+    fn the_real_clone_call_leaves_a_faithful_copy() {
+        let f = Fixture::with(Fake::new().mode(CloneMode::Real));
+        let body = "0123456789".repeat(10_000);
+        f.put("home/iris/big", &body);
+        f.put("home/iris/empty", "");
+        let Outcome::Salvaged { salvage, stats, .. } = f.rotate() else {
+            panic!("must salvage");
+        };
+        assert_eq!(
+            std::fs::read_to_string(salvage.join("home/iris/big")).unwrap(),
+            body
+        );
+        assert_eq!(stats.files, 2);
+        assert!(stats.cloned <= stats.files);
+    }
+
+    /// A file that cannot clone, in a salvage whose probe did, is copied only
+    /// while the reserve stays free.
+    #[test]
+    fn a_file_that_must_be_copied_keeps_the_reserve_free() {
+        let body = "x".repeat(1 << 20);
+        let tight = Fixture::with(
+            Fake::new()
+                .space(CLONE_RESERVE + (512 << 10), 100 * GIB)
+                .mode(CloneMode::FirstOnly),
+        );
+        tight.put("home/iris/big", &body);
+        let err = tight.try_rotate().unwrap_err();
+        assert!(format!("{err:#}").contains("must stay"), "{err:#}");
+        assert!(
+            tight.root.join("home/iris/big").is_file(),
+            "root was touched"
+        );
+        assert!(!tight.root.join("etc").join(MARKER).exists());
+
+        let roomy = Fixture::with(
+            Fake::new()
+                .space(CLONE_RESERVE + (8 << 20), 100 * GIB)
+                .mode(CloneMode::FirstOnly),
+        );
+        roomy.put("home/iris/big", &body);
+        let Outcome::Salvaged { stats, .. } = roomy.rotate() else {
+            panic!("1 MiB fits above the reserve");
+        };
+        assert_eq!(stats.cloned, 0);
+    }
+
+    /// A second request for a root that is being salvaged waits, then finds
+    /// the finished template: one salvage directory, one moved-aside root.
+    #[test]
+    fn two_requests_for_one_root_salvage_it_once() {
+        let f = Fixture::with(
+            Fake::new()
+                .mode(CloneMode::Copy)
+                .delay(Duration::from_millis(200)),
+        );
+        f.put("home/iris/a", "1");
+        let (first, second) = std::thread::scope(|s| {
+            let first = s.spawn(|| f.rotate());
+            while !f.fake.started.load(Ordering::SeqCst) {
+                std::thread::sleep(Duration::from_millis(5));
+            }
+            let second = s.spawn(|| f.rotate());
+            (first.join().unwrap(), second.join().unwrap())
+        });
+        assert!(matches!(first, Outcome::Salvaged { .. }), "{first:?}");
+        assert_eq!(second, Outcome::Clean);
+        assert_eq!(Fixture::names(&f.work), [format!("salvaged-rootfs-{DATE}")]);
+        let moved: Vec<String> = Fixture::names(&f.top)
+            .into_iter()
+            .filter(|n| n.starts_with("h-iris.pre-ephemeral-"))
+            .collect();
+        assert_eq!(moved.len(), 1, "{moved:?}");
+    }
+
+    /// Salvages of different containers copy one after the other.
+    #[test]
+    fn salvages_of_different_containers_copy_one_at_a_time() {
+        let a = Fixture::new();
+        let b = Fixture::new();
+        a.put("home/iris/a", "1");
+        b.put("home/iris/b", "2");
+        let fake = Fake::new()
+            .mode(CloneMode::Copy)
+            .delay(Duration::from_millis(100));
+        let serial = Mutex::new(());
+        std::thread::scope(|s| {
+            s.spawn(|| a.try_rotate_in(&fake, &serial).unwrap());
+            s.spawn(|| b.try_rotate_in(&fake, &serial).unwrap());
+        });
+        assert!(fake.calls.load(Ordering::SeqCst) >= 4, "both salvages ran");
+        assert_eq!(fake.peak.load(Ordering::SeqCst), 1);
+    }
+
+    #[test]
+    fn sizes_are_shown_in_gib() {
+        assert_eq!(gib(0), "0.0 GiB");
+        assert_eq!(gib(3 * GIB / 2), "1.5 GiB");
+        assert_eq!(gib(87 * GIB), "87.0 GiB");
+    }
+
     #[test]
     fn dates() {
         assert_eq!(date_of(0), "19700101");

hive-priv/src/salvage_io.rs

@@ -0,0 +1,84 @@
+//! What the root salvage ([`crate::rootfs`]) asks of the host: free space,
+//! reflinks, and the locks that keep two salvages apart.
+//!
+//! The space and reflink calls sit behind [`Backend`] so a test can set the
+//! free space and make a clone fail.
+
+use std::fs::{File, OpenOptions};
+use std::io;
+use std::os::unix::fs::OpenOptionsExt as _;
+use std::path::Path;
+use std::sync::Mutex;
+
+use anyhow::{Context as _, Result};
+
+/// Where the per-container lock files live. A root-owned runtime directory
+/// (`RuntimeDirectory=` of `hive-priv.service`), so no container can write it.
+pub(crate) const LOCK_DIR: &str = "/run/hive-priv";
+
+/// Held by the one salvage that is copying data at any moment.
+pub(crate) static ONE_AT_A_TIME: Mutex<()> = Mutex::new(());
+
+/// Size and free space of a filesystem, in bytes. `free` is what an
+/// unprivileged writer could still use.
+#[derive(Debug, Clone, Copy)]
+pub(crate) struct FsSpace {
+    pub(crate) free: u64,
+    pub(crate) total: u64,
+}
+
+/// The host calls the salvage makes.
+pub(crate) trait Backend {
+    /// Space on the filesystem holding `path`.
+    fn space(&self, path: &Path) -> io::Result<FsSpace>;
+
+    /// Make `dst`, an empty file, share `src`'s data blocks. An error leaves
+    /// `dst` as it was: the filesystem cannot clone, or `src` and `dst` are on
+    /// different ones.
+    fn clone_file(&self, src: &File, dst: &File) -> io::Result<()>;
+}
+
+/// The real filesystem: `statvfs(2)` and `ioctl(FICLONE)`.
+pub(crate) struct Os;
+
+impl Backend for Os {
+    fn space(&self, path: &Path) -> io::Result<FsSpace> {
+        let st = rustix::fs::statvfs(path)?;
+        Ok(FsSpace {
+            free: st.f_bavail.saturating_mul(st.f_frsize),
+            total: st.f_blocks.saturating_mul(st.f_frsize),
+        })
+    }
+
+    fn clone_file(&self, src: &File, dst: &File) -> io::Result<()> {
+        Ok(rustix::fs::ioctl_ficlone(dst, src)?)
+    }
+}
+
+/// An exclusive `flock(2)` on a container's lock file, released on drop.
+pub(crate) struct ContainerLock {
+    _file: File,
+}
+
+impl ContainerLock {
+    /// Wait for the lock of `name` in `dir`, creating the file if needed.
+    ///
+    /// # Errors
+    /// Returns an error when the directory or file cannot be created or the
+    /// lock call fails.
+    pub(crate) fn acquire(dir: &Path, name: &str) -> Result<Self> {
+        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
+        let path = dir.join(format!("salvage-{name}.lock"));
+        let file = OpenOptions::new()
+            .write(true)
+            .create(true)
+            .truncate(false)
+            .mode(0o600)
+            .custom_flags(libc::O_NOFOLLOW)
+            .open(&path)
+            .with_context(|| format!("open {}", path.display()))?;
+        file.lock()
+            .with_context(|| format!("lock {}", path.display()))?;
+        Ok(Self { _file: file })
+    }
+}

nix/host-modules/hive-priv.nix

@@ -146,6 +146,10 @@ in
         ];
         # Writable HOME for nix's caches (see environment.HOME above).
         StateDirectory = "hive-priv";
+        # /run/hive-priv: the per-container salvage lock files. Root-owned and
+        # outside every path a container can write.
+        RuntimeDirectory = "hive-priv";
+        RuntimeDirectoryMode = "0700";
         # With ProtectSystem=strict the root filesystem is read-only inside
         # hive-priv. When `nixos-container create/update` invokes nix, nix
         # creates a temporary result symlink in its working directory. Without