//! The agent container's non-persistent root.
//!
//! Agent containers start with `systemd-nspawn --ephemeral`: nspawn runs the
//! container on a throwaway snapshot of `/var/lib/nixos-containers/<c>`, so
//! anything written outside the binds is gone at stop. [`prepare`] runs ahead
//! of every `WriteNspawnFlags` and decides whether the flag may be passed.
//!
//! A root that holds anything beyond what the start path itself writes is
//! first salvaged. Everything outside [`CACHE`] is copied into
//! `<agent>/work/salvaged-rootfs-<YYYYMMDD>/`, owned by the agent, the old
//! root is renamed to `<root>.pre-ephemeral-<YYYYMMDD>` (never deleted), and a
//! fresh template root takes its place. The template is what nspawn snapshots
//! on every start, so it must stay small and free of the previous contents.
//! The template carries [`MARKER`]; it is written last, so a crash anywhere
//! earlier leaves the old root in place and the next call tries again.
//!
//! Only a stopped container is touched: a running one keeps whatever flag its
//! root already warrants.
//!
//! Disk use. A file is cloned (`FICLONE`) when the filesystem can, which costs
//! metadata only; otherwise its data is copied. Before creating anything,
//! [`rotate`] compares what the copy needs with the free space under `work/` and
//! returns [`Outcome::Skipped`] when less than [`COPY_FLOOR_PERCENT`] of the
//! filesystem would stay free, or less than [`CLONE_RESERVE`] when a probe shows
//! files clone. A file that fails to clone is checked against the reserve again.
//!
//! Concurrency. One salvage per container runs at a time, under an `flock` in
//! [`LOCK_DIR`]; a second request waits, then sees the finished template. Across
//! containers [`ONE_AT_A_TIME`] admits one copy at a time, so the free space a
//! salvage reads excludes any copy still in flight.

use std::fs::{File, OpenOptions};
use std::io;
use std::os::unix::fs::{
    MetadataExt as _, OpenOptionsExt as _, PermissionsExt as _, lchown, symlink,
};
use std::path::{Path, PathBuf};
use std::sync::{Mutex, PoisonError};

use anyhow::{Context as _, Result, bail};
use hive_priv_sock::AGENT_STATE_ROOT;

use crate::container_config::{container_rootfs, write_container_etc_file_in};
use crate::salvage_io::{Backend, ContainerLock, LOCK_DIR, ONE_AT_A_TIME, Os};

/// The per-container nix profile bind on the host, which persists across starts.
/// What nix keeps for the agent user is carried here; see [`carry_nix_state`].
const PROFILES_ROOT: &str = "/nix/var/nix/profiles/per-container";

/// Written into a template root once it is ready to be snapshotted.
const MARKER: &str = "hyperhive-ephemeral-root";

/// Files in `etc/` that the start path writes into every root, the template
/// included. They are not content: the root counts as clean with only these.
const TEMPLATE_FILES: [&str; 6] = [
    MARKER,
    "hyperhive-bridge-dns",
    "hyperhive-hive-facts",
    "machine-id",
    "os-release",
    "resolv.conf",
];

/// Paths relative to the root that the salvage skips: the guest journal, its
/// forwarder's read cursor, weston's state and scratch space. The first start
/// of a container rebuilds all of them.
const CACHE: [&str; 5] = [
    "tmp",
    "var/cache",
    "var/log/journal",
    "var/lib/weston",
    "var/lib/opentelemetry-collector",
];

/// Share of the filesystem under `work/` that must stay free after a salvage
/// that copies data. btrfs allocates metadata chunks from unallocated space, and
/// every agent and host service writes to the same filesystem, so a full disk
/// stops all of them.
const COPY_FLOOR_PERCENT: u64 = 15;

/// Free space a salvage must leave when its files clone, which allocate
/// metadata only; also the margin kept when a single file has to be copied.
const CLONE_RESERVE: u64 = 2 << 30;

/// Metadata a cloned entry is assumed to cost.
const ENTRY_COST: u64 = 8 << 10;

/// Allocation unit a copied file is rounded up to.
const BLOCK: u64 = 4096;

/// The file in `work/` that tests whether cloning into it works.
const CLONE_PROBE: &str = ".hive-clone-probe";

/// What [`rotate`] did.
#[derive(Debug, PartialEq, Eq)]
pub(crate) enum Outcome {
    /// Nothing outside [`CACHE`] was in the root; nothing was moved.
    Clean,
    /// The root's content is copied to `salvage`, and the root itself kept at
    /// `old_root`.
    Salvaged {
        salvage: PathBuf,
        old_root: PathBuf,
        stats: Stats,
    },
    /// The copy would leave too little free space: `need` bytes must be free,
    /// `free` are. Nothing was created or moved.
    Skipped { need: u64, free: u64 },
}

/// What a salvage copied.
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub(crate) struct Stats {
    /// Bytes in the regular files.
    pub(crate) bytes: u64,
    /// Regular files copied.
    pub(crate) files: u64,
    /// Of those, the files that share blocks with the old root.
    pub(crate) cloned: u64,
}

/// The host a salvage runs against.
pub(crate) struct Env<'a> {
    pub(crate) backend: &'a dyn Backend,
    /// Where the per-container lock files go.
    pub(crate) lock_dir: &'a Path,
    /// Taken by the salvage that is copying; see the module docs.
    pub(crate) one_at_a_time: &'a Mutex<()>,
}

/// Whether `container` may be started with `--ephemeral`, salvaging its root
/// first when it holds content and the container is stopped.
///
/// # Errors
/// Returns an error when the unit state cannot be read or the salvage fails.
/// Nothing is deleted on that path, and the caller starts the container
/// without the flag. A salvage skipped for lack of space is not an error: it
/// returns `false` and the next start tries again.
pub(crate) fn prepare(container: &str, agent: &str) -> Result<bool> {
    let root = container_rootfs(container);
    if !unit_is_stopped(container)? {
        return Ok(is_regular(&root.join("etc").join(MARKER)));
    }
    let work = Path::new(AGENT_STATE_ROOT).join(agent).join("work");
    let profiles = Path::new(PROFILES_ROOT).join(container);
    let env = Env {
        backend: &Os,
        lock_dir: Path::new(LOCK_DIR),
        one_at_a_time: &ONE_AT_A_TIME,
    };
    let outcome = rotate(&root, agent, &work, &profiles, &today(), &env)?;
    Ok(report(container, &outcome))
}

/// Log what [`rotate`] did, and say whether the container may start `--ephemeral`.
fn report(container: &str, outcome: &Outcome) -> bool {
    match outcome {
        Outcome::Clean => {}
        Outcome::Salvaged {
            salvage,
            old_root,
            stats,
        } => {
            tracing::info!(
                "salvage: {container} {} reflinked={}/{} files",
                gib(stats.bytes),
                stats.cloned,
                stats.files
            );
            tracing::warn!(
                %container,
                salvage = %salvage.display(),
                old_root = %old_root.display(),
                "root salvaged before the first ephemeral start; the old root is kept, remove it by hand"
            );
        }
        Outcome::Skipped { need, free } => {
            tracing::warn!(
                "salvage skipped: {container} need {} free {}",
                gib(*need),
                gib(*free)
            );
            return false;
        }
    }
    true
}

/// `bytes` in GiB, to one decimal.
fn gib(bytes: u64) -> String {
    const GIB: u64 = 1 << 30;
    format!("{}.{} GiB", bytes / GIB, bytes % GIB * 10 / GIB)
}

/// Salvage `root` if it holds content, then leave a marked template in its
/// place. `work` is the agent's work dir on the host, `profiles` the host side of
/// the container's `/nix/var/nix/profiles`; `date` is `YYYYMMDD`.
///
/// The container's lock is held throughout, so a second call for the same root
/// waits and then finds the template the first one left.
fn rotate(
    root: &Path,
    agent: &str,
    work: &Path,
    profiles: &Path,
    date: &str,
    env: &Env<'_>,
) -> Result<Outcome> {
    let name = root
        .file_name()
        .and_then(|n| n.to_str())
        .with_context(|| format!("{} has no usable name", root.display()))?;
    let _lock = ContainerLock::acquire(env.lock_dir, name)?;
    let outcome = if has_content(root)? {
        let _turn = env
            .one_at_a_time
            .lock()
            .unwrap_or_else(PoisonError::into_inner);
        salvage(root, agent, work, profiles, date, env)?
    } else {
        Outcome::Clean
    };
    if matches!(outcome, Outcome::Skipped { .. }) {
        return Ok(outcome);
    }
    ensure_machine_id(root)?;
    write_container_etc_file_in(root, MARKER, "")?;
    Ok(outcome)
}

/// The copy-and-move half of [`rotate`], for a root that has content.
fn salvage(
    root: &Path,
    agent: &str,
    work: &Path,
    profiles: &Path,
    date: &str,
    env: &Env<'_>,
) -> Result<Outcome> {
    let (uid, gid) = agent_owner(root, agent, work)?;
    let machine_id = read_machine_id(root);
    if let Some((need, free)) = shortfall(env.backend, root, work)? {
        return Ok(Outcome::Skipped { need, free });
    }
    let salvage = fresh_dir(work, &format!("salvaged-rootfs-{date}"))?;
    let copied = copy_and_move_aside(root, &salvage, agent, profiles, date, env, (uid, gid));
    let (old_root, stats) = copied.inspect_err(|_| discard_partial(&salvage))?;
    std::fs::create_dir(root).with_context(|| format!("create {}", root.display()))?;
    if let Some(id) = machine_id {
        write_container_etc_file_in(root, "machine-id", &format!("{id}\n"))?;
    }
    Ok(Outcome::Salvaged {
        salvage,
        old_root,
        stats,
    })
}

/// Copy `root` into the fresh `salvage` dir, then rename `root` aside; returns
/// where it went. Up to the rename the old root is untouched, so a failure
/// anywhere in here leaves it where it was.
fn copy_and_move_aside(
    root: &Path,
    salvage: &Path,
    agent: &str,
    profiles: &Path,
    date: &str,
    env: &Env<'_>,
    (uid, gid): (u32, u32),
) -> Result<(PathBuf, Stats)> {
    lchown(salvage, Some(uid), Some(gid))
        .with_context(|| format!("chown {}", salvage.display()))?;
    let mut copier = Copier::new(env.backend, uid, gid);
    copier
        .tree(root, salvage, Path::new(""))
        .with_context(|| format!("copy {} to {}", root.display(), salvage.display()))?;
    carry_nix_state(root, agent, profiles, env.backend, uid, gid)?;
    let old_root = free_sibling(root, &format!("pre-ephemeral-{date}"))?;
    std::fs::rename(root, &old_root)
        .with_context(|| format!("move {} to {}", root.display(), old_root.display()))?;
    Ok((old_root, copier.stats))
}

/// Remove the directory a failed salvage was filling: the one this attempt
/// created, never an earlier salvage. `remove_dir_all` does not follow links.
fn discard_partial(salvage: &Path) {
    if let Err(e) = std::fs::remove_dir_all(salvage) {
        tracing::warn!(
            salvage = %salvage.display(),
            error = %e,
            "could not remove the partial salvage; remove it by hand"
        );
    }
}

/// What [`measure`] found under a root.
struct Tally {
    /// Everything outside [`CACHE`], directories and links included.
    entries: u64,
    /// Regular-file bytes, each file rounded up to [`BLOCK`].
    bytes: u64,
    /// The first regular file with data, for the clone probe.
    sample: Option<PathBuf>,
}

/// Size up what [`Copier::tree`] would copy from `root`.
fn measure(root: &Path) -> Result<Tally> {
    fn walk(dir: &Path, rel: &Path, tally: &mut Tally) -> Result<()> {
        for entry in std::fs::read_dir(dir).with_context(|| format!("read {}", dir.display()))? {
            let entry = entry.with_context(|| format!("read {}", dir.display()))?;
            let rel = rel.join(entry.file_name());
            if is_cache(&rel) {
                continue;
            }
            let meta = entry
                .metadata()
                .with_context(|| format!("stat {}", entry.path().display()))?;
            tally.entries += 1;
            if meta.is_dir() {
                walk(&entry.path(), &rel, tally)?;
            } else if meta.is_file() {
                tally.bytes = tally
                    .bytes
                    .saturating_add(meta.len().div_ceil(BLOCK) * BLOCK);
                if meta.len() > 0 && tally.sample.is_none() {
                    tally.sample = Some(entry.path());
                }
            }
        }
        Ok(())
    }
    let mut tally = Tally {
        entries: 0,
        bytes: 0,
        sample: None,
    };
    walk(root, Path::new(""), &mut tally)?;
    Ok(tally)
}

/// `Some((need, free))` when a salvage of `root` into `work` would leave less
/// free space than it must: `need` bytes have to be free, `free` are.
///
/// Files that clone cost metadata only. Whether they do is probed with one
/// file of the root ([`clones`]); when that fails the whole data is counted.
fn shortfall(backend: &dyn Backend, root: &Path, work: &Path) -> Result<Option<(u64, u64)>> {
    let tally = measure(root)?;
    let space = backend
        .space(work)
        .with_context(|| format!("free space of {}", work.display()))?;
    let metadata = tally.entries.saturating_mul(ENTRY_COST);
    let need = if tally
        .sample
        .as_deref()
        .is_none_or(|sample| clones(backend, sample, work))
    {
        metadata.saturating_add(CLONE_RESERVE)
    } else {
        let floor = space.total / 100 * COPY_FLOOR_PERCENT;
        tally.bytes.saturating_add(metadata).saturating_add(floor)
    };
    Ok((space.free < need).then_some((need, space.free)))
}

/// Whether `sample` can be cloned into a file in `work`. The probe file is
/// created exclusively and removed again; if one is already there the answer
/// is no.
fn clones(backend: &dyn Backend, sample: &Path, work: &Path) -> bool {
    let probe = work.join(CLONE_PROBE);
    let Ok(src) = open_nofollow(sample) else {
        return false;
    };
    let Ok(dst) = create_new(&probe) else {
        return false;
    };
    let cloned = backend.clone_file(&src, &dst).is_ok();
    drop(dst);
    std::fs::remove_file(&probe).ok();
    cloned
}

/// `path` opened for reading; a link at `path` is an error.
fn open_nofollow(path: &Path) -> io::Result<File> {
    OpenOptions::new()
        .read(true)
        .custom_flags(libc::O_NOFOLLOW)
        .open(path)
}

/// `path` created for writing, mode 0600; an existing entry is an error.
fn create_new(path: &Path) -> io::Result<File> {
    OpenOptions::new()
        .write(true)
        .create_new(true)
        .mode(0o600)
        .custom_flags(libc::O_NOFOLLOW)
        .open(path)
}

/// Whether anything but directories, [`TEMPLATE_FILES`] and [`CACHE`] is under
/// `root`. A missing root has none.
fn has_content(root: &Path) -> Result<bool> {
    fn walk(dir: &Path, rel: &Path) -> Result<bool> {
        let entries = match std::fs::read_dir(dir) {
            Ok(entries) => entries,
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false),
            Err(e) => return Err(e).with_context(|| format!("read {}", dir.display())),
        };
        for entry in entries {
            let entry = entry.with_context(|| format!("read {}", dir.display()))?;
            let rel = rel.join(entry.file_name());
            if is_cache(&rel) {
                continue;
            }
            let kind = entry
                .file_type()
                .with_context(|| format!("stat {}", entry.path().display()))?;
            let is_template = TEMPLATE_FILES
                .iter()
                .any(|name| rel == Path::new("etc").join(name));
            if kind.is_dir() {
                if walk(&entry.path(), &rel)? {
                    return Ok(true);
                }
            } else if !is_template {
                return Ok(true);
            }
        }
        Ok(false)
    }
    walk(root, Path::new(""))
}

fn is_cache(rel: &Path) -> bool {
    CACHE.iter().any(|c| rel == Path::new(c))
}

/// The uid and gid the salvage is handed to: the owner of the agent's home in
/// the old root, else of its work dir. Never root.
fn agent_owner(root: &Path, agent: &str, work: &Path) -> Result<(u32, u32)> {
    let home = real_dir(root, &["home", agent]);
    for dir in [home, real_dir(work, &[])].into_iter().flatten() {
        if let Ok(meta) = std::fs::symlink_metadata(&dir)
            && meta.uid() != 0
        {
            return Ok((meta.uid(), meta.gid()));
        }
    }
    bail!(
        "agent {agent}: neither its home in {} nor {} has a non-root owner",
        root.display(),
        work.display()
    )
}

/// `base` joined with `parts`, if every component is a real directory. The
/// container could have made any of them a link, and this runs as root.
fn real_dir(base: &Path, parts: &[&str]) -> Option<PathBuf> {
    let mut path = base.to_path_buf();
    if !std::fs::symlink_metadata(&path).ok()?.is_dir() {
        return None;
    }
    for part in parts {
        path.push(part);
        if !std::fs::symlink_metadata(&path).ok()?.is_dir() {
            return None;
        }
    }
    Some(path)
}

/// nix keeps a non-root user's profile under `~/.local/state/nix`, which the
/// agent's tmpfiles rules (`nix/agent-modules/user.nix`) point at
/// `per-user/<agent>` in `profiles`. A root that was lived in has the real
/// directory there: copy it across, so the agent's installed packages and the
/// GC roots that name them are still valid after the switch. A target that
/// holds anything is left alone; so is the whole copy when a component of the
/// target path is not a real directory, as the container can write `profiles`.
fn carry_nix_state(
    root: &Path,
    agent: &str,
    profiles: &Path,
    backend: &dyn Backend,
    uid: u32,
    gid: u32,
) -> Result<()> {
    let Some(src) = real_dir(root, &["home", agent, ".local", "state", "nix"]) else {
        return Ok(());
    };
    let Some(target) = own_dir(profiles, &["per-user", agent], uid, gid)? else {
        tracing::warn!(
            profiles = %profiles.display(),
            "a link or file where the nix state goes; the agent's nix state is in the salvage only"
        );
        return Ok(());
    };
    // The tmpfiles rules create `target` empty on a boot of the new module;
    // anything in it is the agent's newer state.
    if std::fs::read_dir(&target)
        .with_context(|| format!("read {}", target.display()))?
        .next()
        .is_some()
    {
        return Ok(());
    }
    Copier::new(backend, uid, gid)
        .tree(&src, &target, Path::new(""))
        .with_context(|| format!("copy {} to {}", src.display(), target.display()))
}

/// `base` joined with `parts`, creating each missing component (the last one
/// owned by `uid:gid`), or `None` when `base` or an existing component is not a
/// real directory. The container could have replaced any of them by a link, and
/// this runs as root.
pub(crate) fn own_dir(base: &Path, parts: &[&str], uid: u32, gid: u32) -> Result<Option<PathBuf>> {
    let mut path = base.to_path_buf();
    if !std::fs::symlink_metadata(&path).is_ok_and(|m| m.is_dir()) {
        return Ok(None);
    }
    for (i, part) in parts.iter().enumerate() {
        path.push(part);
        match std::fs::symlink_metadata(&path) {
            Ok(meta) if meta.is_dir() => {}
            Ok(_) => return Ok(None),
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
                std::fs::create_dir(&path).with_context(|| format!("create {}", path.display()))?;
                if i + 1 == parts.len() {
                    lchown(&path, Some(uid), Some(gid))
                        .with_context(|| format!("chown {}", path.display()))?;
                }
            }
            Err(e) => return Err(e).with_context(|| format!("stat {}", path.display())),
        }
    }
    Ok(Some(path))
}

/// Create `parent/base`, or `parent/base-2`, ... when taken. Created, never
/// reused: `mkdir` does not follow a symlink an agent left under that name.
fn fresh_dir(parent: &Path, base: &str) -> Result<PathBuf> {
    for n in 1.. {
        let path = if n == 1 {
            parent.join(base)
        } else {
            parent.join(format!("{base}-{n}"))
        };
        match std::fs::create_dir(&path) {
            Ok(()) => return Ok(path),
            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {}
            Err(e) => return Err(e).with_context(|| format!("create {}", path.display())),
        }
    }
    unreachable!("the loop above returns or continues")
}

/// `<root>.<suffix>`, or `<root>.<suffix>-2`, ... the first name nothing uses.
fn free_sibling(root: &Path, suffix: &str) -> Result<PathBuf> {
    let name = root
        .file_name()
        .and_then(|n| n.to_str())
        .with_context(|| format!("{} has no usable name", root.display()))?;
    for n in 1.. {
        let candidate = if n == 1 {
            format!("{name}.{suffix}")
        } else {
            format!("{name}.{suffix}-{n}")
        };
        let path = root.with_file_name(candidate);
        if std::fs::symlink_metadata(&path).is_err() {
            return Ok(path);
        }
    }
    unreachable!("the loop above returns or continues")
}

/// Copies a tree for [`salvage`] and keeps count.
struct Copier<'a> {
    backend: &'a dyn Backend,
    uid: u32,
    gid: u32,
    stats: Stats,
}

impl<'a> Copier<'a> {
    fn new(backend: &'a dyn Backend, uid: u32, gid: u32) -> Self {
        Self {
            backend,
            uid,
            gid,
            stats: Stats::default(),
        }
    }

    /// Copy everything under `src` except [`CACHE`] into `dst`, owned by
    /// `uid:gid`, with modes and mtimes kept. Sockets, fifos and device nodes
    /// are skipped; hard links become separate copies.
    fn tree(&mut self, src: &Path, dst: &Path, rel: &Path) -> Result<()> {
        let (uid, gid) = (self.uid, self.gid);
        let dir = src.join(rel);
        for entry in std::fs::read_dir(&dir).with_context(|| format!("read {}", dir.display()))? {
            let entry = entry.with_context(|| format!("read {}", dir.display()))?;
            let rel = rel.join(entry.file_name());
            if is_cache(&rel) {
                continue;
            }
            let from = src.join(&rel);
            let to = dst.join(&rel);
            let meta = std::fs::symlink_metadata(&from)
                .with_context(|| format!("stat {}", from.display()))?;
            let kind = meta.file_type();
            if kind.is_dir() {
                std::fs::create_dir(&to).with_context(|| format!("create {}", to.display()))?;
                self.tree(src, dst, &rel)?;
                lchown(&to, Some(uid), Some(gid))
                    .with_context(|| format!("chown {}", to.display()))?;
                set_mode(&to, &meta)?;
            } else if kind.is_symlink() {
                let target = std::fs::read_link(&from)
                    .with_context(|| format!("read link {}", from.display()))?;
                symlink(&target, &to).with_context(|| format!("link {}", to.display()))?;
                lchown(&to, Some(uid), Some(gid))
                    .with_context(|| format!("chown {}", to.display()))?;
            } else if kind.is_file() {
                self.file(&from, &to, &meta)?;
            } else {
                tracing::info!(path = %from.display(), "salvage skips a socket, fifo or device node");
            }
        }
        Ok(())
    }

    /// One regular file: cloned when the filesystem allows it, else copied.
    fn file(&mut self, from: &Path, to: &Path, meta: &std::fs::Metadata) -> Result<()> {
        let mut src = open_nofollow(from).with_context(|| format!("open {}", from.display()))?;
        let mut dst = create_new(to).with_context(|| format!("create {}", to.display()))?;
        self.stats.files += 1;
        self.stats.bytes = self.stats.bytes.saturating_add(meta.len());
        if self.backend.clone_file(&src, &dst).is_ok() {
            self.stats.cloned += 1;
        } else {
            self.check_room(to, meta.len())?;
            io::copy(&mut src, &mut dst)
                .with_context(|| format!("copy {} to {}", from.display(), to.display()))?;
        }
        if let Ok(mtime) = meta.modified() {
            dst.set_modified(mtime)
                .with_context(|| format!("set mtime of {}", to.display()))?;
        }
        lchown(to, Some(self.uid), Some(self.gid))
            .with_context(|| format!("chown {}", to.display()))?;
        set_mode(to, meta)
    }

    /// Fail when copying `len` bytes to `to` would leave less than
    /// [`CLONE_RESERVE`] free.
    fn check_room(&self, to: &Path, len: u64) -> Result<()> {
        let dir = to.parent().unwrap_or(to);
        let space = self
            .backend
            .space(dir)
            .with_context(|| format!("free space of {}", dir.display()))?;
        if space.free < len.saturating_add(CLONE_RESERVE) {
            bail!(
                "{} is {len} bytes and the filesystem has {} free; at least {CLONE_RESERVE} must stay",
                to.display(),
                space.free
            );
        }
        Ok(())
    }
}

/// Give `path` the permission bits of `meta`, which `chown` may have cleared.
fn set_mode(path: &Path, meta: &std::fs::Metadata) -> Result<()> {
    std::fs::set_permissions(path, std::fs::Permissions::from_mode(meta.mode() & 0o7777))
        .with_context(|| format!("chmod {}", path.display()))
}

/// The root's machine id when `etc/machine-id` is a regular file holding one.
/// The container may have replaced `etc` or the file by a link, so neither is
/// followed.
fn read_machine_id(root: &Path) -> Option<String> {
    let etc = std::fs::symlink_metadata(root.join("etc")).ok()?;
    let path = root.join("etc/machine-id");
    if !etc.is_dir() || !is_regular(&path) {
        return None;
    }
    let text = std::fs::read_to_string(path).ok()?;
    let id = text.trim_end_matches('\n');
    (id.len() == 32 && id.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')))
        .then(|| id.to_owned())
}

/// Give the template a machine id when it has none: with an empty
/// `etc/machine-id` the guest makes a new one on every boot, and its journal
/// directory and DHCP identity change with it.
fn ensure_machine_id(root: &Path) -> Result<()> {
    if read_machine_id(root).is_some() {
        return Ok(());
    }
    let uuid = std::fs::read_to_string("/proc/sys/kernel/random/uuid")
        .context("read a random uuid for the machine id")?;
    let id: String = uuid.trim().chars().filter(|c| *c != '-').collect();
    write_container_etc_file_in(root, "machine-id", &format!("{id}\n"))
}

fn is_regular(path: &Path) -> bool {
    std::fs::symlink_metadata(path).is_ok_and(|m| m.is_file())
}

/// `systemctl is-active` answers `inactive` or `failed` for a unit that is not
/// running; anything else (`active`, `activating`, `deactivating`) means the
/// root is, or is about to be, in use.
pub(crate) fn unit_is_stopped(container: &str) -> Result<bool> {
    let out = std::process::Command::new("systemctl")
        .args(["is-active", &format!("container@{container}.service")])
        .output()
        .context("invoke systemctl is-active")?;
    let state = String::from_utf8_lossy(&out.stdout);
    Ok(matches!(state.trim(), "inactive" | "failed"))
}

/// Today in UTC as `YYYYMMDD`.
fn today() -> String {
    let secs = std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .map_or(0, |d| d.as_secs());
    date_of(secs)
}

/// `YYYYMMDD` of a unix time, by Howard Hinnant's days-to-civil algorithm.
fn date_of(secs: u64) -> String {
    let z = i64::try_from(secs / 86_400).unwrap_or(0) + 719_468;
    let era = z.div_euclid(146_097);
    let doe = z.rem_euclid(146_097);
    let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
    let mp = (5 * doy + 2) / 153;
    let day = doy - (153 * mp + 2) / 5 + 1;
    let month = if mp < 10 { mp + 3 } else { mp - 9 };
    let year = yoe + era * 400 + i64::from(month <= 2);
    format!("{year:04}{month:02}{day:02}")
}

#[cfg(test)]
mod tests {
    use std::fs::File;
    use std::io;
    use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _, lchown, symlink};
    use std::path::{Path, PathBuf};
    use std::sync::Mutex;
    use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
    use std::time::Duration;

    use anyhow::Result;

    use super::{
        CLONE_RESERVE, Env, MARKER, Outcome, Stats, date_of, gib, has_content, report, rotate,
    };
    use crate::salvage_io::{Backend, FsSpace, Os};
    use crate::shared::test_support::scratch;

    const DATE: &str = "20261028";

    /// How [`Fake`] answers a clone request.
    #[derive(Clone, Copy)]
    enum CloneMode {
        /// Succeeds, by copying the bytes.
        Copy,
        /// Fails with this errno.
        Fail(i32),
        /// Succeeds for the first request (the probe), fails for the rest.
        FirstOnly,
        /// Defers to the real `ioctl(FICLONE)`.
        Real,
    }

    /// A [`Backend`] with the free space and the clone result set by the test.
    struct Fake {
        free: u64,
        total: u64,
        mode: CloneMode,
        delay: Duration,
        space_calls: AtomicU32,
        space_fails_from: u32,
        calls: AtomicU32,
        started: AtomicBool,
        active: AtomicU32,
        peak: AtomicU32,
    }

    impl Fake {
        /// Plenty of space; every clone fails with `EXDEV`, so files are copied.
        fn new() -> Self {
            Self {
                free: 1 << 50,
                total: 1 << 51,
                mode: CloneMode::Fail(libc::EXDEV),
                delay: Duration::ZERO,
                space_calls: AtomicU32::new(0),
                space_fails_from: u32::MAX,
                calls: AtomicU32::new(0),
                started: AtomicBool::new(false),
                active: AtomicU32::new(0),
                peak: AtomicU32::new(0),
            }
        }

        fn space(mut self, free: u64, total: u64) -> Self {
            self.free = free;
            self.total = total;
            self
        }

        fn mode(mut self, mode: CloneMode) -> Self {
            self.mode = mode;
            self
        }

        /// Make the `n`th and later free-space reads fail, counting from 0.
        fn space_fails_from(mut self, n: u32) -> Self {
            self.space_fails_from = n;
            self
        }

        fn delay(mut self, delay: Duration) -> Self {
            self.delay = delay;
            self
        }
    }

    impl Backend for Fake {
        fn space(&self, _path: &Path) -> io::Result<FsSpace> {
            if self.space_calls.fetch_add(1, Ordering::SeqCst) >= self.space_fails_from {
                return Err(io::Error::from_raw_os_error(libc::EIO));
            }
            Ok(FsSpace {
                free: self.free,
                total: self.total,
            })
        }

        fn clone_file(&self, src: &File, dst: &File) -> io::Result<()> {
            let call = self.calls.fetch_add(1, Ordering::SeqCst);
            self.started.store(true, Ordering::SeqCst);
            let active = self.active.fetch_add(1, Ordering::SeqCst) + 1;
            self.peak.fetch_max(active, Ordering::SeqCst);
            std::thread::sleep(self.delay);
            let copy = || io::copy(&mut &*src, &mut &*dst).map(drop);
            let result = match self.mode {
                CloneMode::Copy => copy(),
                CloneMode::Fail(errno) => Err(io::Error::from_raw_os_error(errno)),
                CloneMode::FirstOnly if call == 0 => copy(),
                CloneMode::FirstOnly => Err(io::Error::from_raw_os_error(libc::EXDEV)),
                CloneMode::Real => Os.clone_file(src, dst),
            };
            self.active.fetch_sub(1, Ordering::SeqCst);
            result
        }
    }

    /// A stopped agent's old root and work dir under one scratch dir. The
    /// "agent" is the test's own user, the one it can chown to.
    struct Fixture {
        top: PathBuf,
        root: PathBuf,
        work: PathBuf,
        profiles: PathBuf,
        fake: Fake,
        serial: Mutex<()>,
    }

    impl Fixture {
        fn new() -> Self {
            let top = scratch();
            let root = top.join("h-iris");
            let work = top.join("work");
            let profiles = top.join("profiles");
            std::fs::create_dir_all(&profiles).unwrap();
            std::fs::create_dir_all(root.join("home/iris")).unwrap();
            std::fs::create_dir_all(&work).unwrap();
            Self {
                top,
                root,
                work,
                profiles,
                fake: Fake::new(),
                serial: Mutex::new(()),
            }
        }

        fn with(fake: Fake) -> Self {
            let mut f = Self::new();
            f.fake = fake;
            f
        }

        fn put(&self, rel: &str, body: &str) {
            let path = self.root.join(rel);
            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
            std::fs::write(path, body).unwrap();
        }

        fn try_rotate_in(&self, backend: &dyn Backend, serial: &Mutex<()>) -> Result<Outcome> {
            let env = Env {
                backend,
                lock_dir: &self.top.join("locks"),
                one_at_a_time: serial,
            };
            rotate(&self.root, "iris", &self.work, &self.profiles, DATE, &env)
        }

        fn try_rotate(&self) -> Result<Outcome> {
            self.try_rotate_in(&self.fake, &self.serial)
        }

        fn rotate(&self) -> Outcome {
            self.try_rotate().unwrap()
        }

        /// Where the container finds nix's per-user state, on the host.
        fn profile(&self) -> PathBuf {
            self.profiles.join("per-user/iris")
        }

        fn salvage(&self) -> PathBuf {
            self.work.join(format!("salvaged-rootfs-{DATE}"))
        }

        fn names(dir: &Path) -> Vec<String> {
            let mut v: Vec<String> = std::fs::read_dir(dir)
                .unwrap()
                .map(|e| e.unwrap().file_name().into_string().unwrap())
                .collect();
            v.sort();
            v
        }
    }

    impl Drop for Fixture {
        fn drop(&mut self) {
            std::fs::remove_dir_all(&self.top).ok();
        }
    }

    /// The first call copies the agent's files and moves the old root aside;
    /// the second finds a clean template and does nothing.
    #[test]
    fn salvages_once() {
        let f = Fixture::new();
        f.put("home/iris/notes.txt", "keep me");
        f.put("etc/hostname", "iris");

        let Outcome::Salvaged {
            salvage, old_root, ..
        } = f.rotate()
        else {
            panic!("the first call must salvage");
        };
        assert_eq!(salvage, f.salvage());
        assert_eq!(
            std::fs::read_to_string(salvage.join("home/iris/notes.txt")).unwrap(),
            "keep me"
        );
        assert_eq!(
            std::fs::read_to_string(old_root.join("home/iris/notes.txt")).unwrap(),
            "keep me"
        );
        assert!(f.root.join("etc").join(MARKER).is_file());
        assert!(
            !f.root.join("home/iris").exists(),
            "template holds the old home"
        );

        assert_eq!(f.rotate(), Outcome::Clean);
        assert_eq!(
            Fixture::names(&f.work),
            [format!("salvaged-rootfs-{DATE}")],
            "a second salvage dir appeared"
        );
    }

    /// Content that appears in the template after the marker (a run by a
    /// build without the flag) is salvaged again, into its own directory.
    #[test]
    fn content_after_the_marker_is_salvaged_again() {
        let f = Fixture::new();
        f.put("home/iris/a", "1");
        f.rotate();
        f.put("home/iris/b", "2");
        let Outcome::Salvaged { salvage, .. } = f.rotate() else {
            panic!("late content must be salvaged");
        };
        assert_eq!(salvage, f.work.join(format!("salvaged-rootfs-{DATE}-2")));
        assert!(salvage.join("home/iris/b").is_file());
        assert!(
            f.salvage().join("home/iris/a").is_file(),
            "first salvage kept"
        );
    }

    /// The journal, `/tmp`, `/var/cache`, weston and the otel cursor stay out
    /// of the salvage; a sibling of each does not. The root's own copies are
    /// untouched.
    #[test]
    fn cache_paths_are_skipped_and_their_siblings_are_not() {
        let f = Fixture::new();
        f.put("home/iris/keep", "x");
        for cache in [
            "tmp/scratch",
            "var/cache/nix/x",
            "var/log/journal/id/system.journal",
            "var/lib/weston/state",
            "var/lib/opentelemetry-collector/cursor",
        ] {
            f.put(cache, "cache");
        }
        for real in [
            "var/log/other.log",
            "var/lib/other/db",
            "var/tmp/keep",
            "root/.bash_history",
        ] {
            f.put(real, "real");
        }
        let Outcome::Salvaged {
            salvage, old_root, ..
        } = f.rotate()
        else {
            panic!("must salvage");
        };
        for cache in [
            "tmp",
            "var/cache",
            "var/log/journal",
            "var/lib/weston",
            "var/lib/opentelemetry-collector",
        ] {
            assert!(!salvage.join(cache).exists(), "{cache} was copied");
            assert!(old_root.join(cache).exists(), "{cache} left the old root");
        }
        for real in [
            "var/log/other.log",
            "var/lib/other/db",
            "var/tmp/keep",
            "root/.bash_history",
            "home/iris/keep",
        ] {
            assert!(salvage.join(real).is_file(), "{real} was not copied");
        }
    }

    /// A root with only the cache paths and the start path's own files has
    /// nothing to salvage: no directory in `work/`, no move.
    #[test]
    fn a_root_without_content_is_not_salvaged() {
        let f = Fixture::new();
        f.put("var/cache/x", "c");
        f.put("etc/hyperhive-hive-facts", "A=b\n");
        f.put("etc/resolv.conf", "nameserver 1.1.1.1\n");
        assert_eq!(f.rotate(), Outcome::Clean);
        assert!(f.work.read_dir().unwrap().next().is_none());
        assert!(f.root.join("var/cache/x").is_file(), "root was touched");
        assert!(f.root.join("etc").join(MARKER).is_file());
        // The same call with one real file makes the control: it salvages.
        f.put("home/iris/real", "x");
        assert!(matches!(f.rotate(), Outcome::Salvaged { .. }));
    }

    /// Nothing is removed: the old root keeps every file, and a failed
    /// salvage leaves the root where it was.
    #[test]
    fn nothing_is_deleted() {
        let f = Fixture::new();
        f.put("home/iris/deep/er/file", "data");
        f.put("etc/passwd", "iris");
        let Outcome::Salvaged { old_root, .. } = f.rotate() else {
            panic!("must salvage");
        };
        assert_eq!(
            std::fs::read_to_string(old_root.join("home/iris/deep/er/file")).unwrap(),
            "data"
        );
        assert_eq!(
            std::fs::read_to_string(old_root.join("etc/passwd")).unwrap(),
            "iris"
        );

        let g = Fixture::new();
        g.put("home/iris/file", "data");
        std::fs::remove_dir(&g.work).unwrap();
        std::fs::write(&g.work, "not a dir").unwrap();
        assert!(g.try_rotate().is_err());
        assert_eq!(
            std::fs::read_to_string(g.root.join("home/iris/file")).unwrap(),
            "data"
        );
        assert!(!g.root.join("etc").join(MARKER).exists());
    }

    /// A group of the test user other than `primary`, if it has one.
    fn other_group(primary: u32) -> Option<u32> {
        let mut groups = [0 as libc::gid_t; 64];
        // SAFETY: `groups` has room for the 64 ids it declares.
        let n = unsafe { libc::getgroups(64, groups.as_mut_ptr()) };
        let n = usize::try_from(n).ok()?;
        groups[..n].iter().copied().find(|g| *g != primary)
    }

    /// Modes, mtimes, symlinks and the owner reach the salvage. The owner is
    /// the test user, who can chown only to itself: the check pins that the
    /// requested id lands on files, directories and links alike.
    #[test]
    fn modes_links_and_owner_are_kept() {
        let f = Fixture::new();
        f.put("home/iris/run.sh", "#!/bin/sh\n");
        std::fs::set_permissions(
            f.root.join("home/iris/run.sh"),
            std::fs::Permissions::from_mode(0o750),
        )
        .unwrap();
        std::fs::set_permissions(
            f.root.join("home/iris"),
            std::fs::Permissions::from_mode(0o700),
        )
        .unwrap();
        symlink("run.sh", f.root.join("home/iris/link")).unwrap();
        let mtime = std::fs::metadata(f.root.join("home/iris/run.sh"))
            .unwrap()
            .mtime();
        // An unprivileged user can chown a file to any group it is in, so a
        // second group gives the owner something to differ from the group new
        // files get. Without one the gid check below cannot tell a chown from
        // none.
        let primary = std::fs::metadata(&f.work).unwrap().gid();
        if let Some(other) = other_group(primary) {
            lchown(f.root.join("home/iris"), None, Some(other)).unwrap();
        }

        let Outcome::Salvaged {
            salvage, old_root, ..
        } = f.rotate()
        else {
            panic!("must salvage");
        };
        let me = std::fs::metadata(old_root.join("home/iris")).unwrap();
        let file = std::fs::symlink_metadata(salvage.join("home/iris/run.sh")).unwrap();
        assert_eq!(file.mode() & 0o7777, 0o750);
        assert_eq!(file.mtime(), mtime);
        assert_eq!(
            std::fs::symlink_metadata(salvage.join("home/iris"))
                .unwrap()
                .mode()
                & 0o7777,
            0o700
        );
        assert_eq!(
            std::fs::read_link(salvage.join("home/iris/link")).unwrap(),
            Path::new("run.sh")
        );
        for rel in [
            "",
            "home",
            "home/iris",
            "home/iris/run.sh",
            "home/iris/link",
        ] {
            let meta = std::fs::symlink_metadata(salvage.join(rel)).unwrap();
            assert_eq!((meta.uid(), meta.gid()), (me.uid(), me.gid()), "{rel:?}");
        }
    }

    /// The template keeps the old root's machine id; one with none gets a
    /// fresh id that the next call keeps.
    #[test]
    fn the_template_has_a_stable_machine_id() {
        let f = Fixture::new();
        f.put("home/iris/a", "1");
        f.put("etc/machine-id", "0123456789abcdef0123456789abcdef\n");
        f.rotate();
        let id = std::fs::read_to_string(f.root.join("etc/machine-id")).unwrap();
        assert_eq!(id, "0123456789abcdef0123456789abcdef\n");

        let g = Fixture::new();
        g.put("etc/machine-id", "");
        g.rotate();
        let fresh = std::fs::read_to_string(g.root.join("etc/machine-id")).unwrap();
        assert_eq!(fresh.trim().len(), 32, "{fresh:?}");
        g.rotate();
        assert_eq!(
            std::fs::read_to_string(g.root.join("etc/machine-id")).unwrap(),
            fresh
        );
    }

    /// `etc` replaced by a link to a directory holding a machine-id is not
    /// followed.
    #[test]
    fn a_linked_etc_is_not_followed() {
        let f = Fixture::new();
        let outside = f.top.join("outside");
        std::fs::create_dir_all(&outside).unwrap();
        std::fs::write(
            outside.join("machine-id"),
            "0123456789abcdef0123456789abcdef\n",
        )
        .unwrap();
        symlink(&outside, f.root.join("etc")).unwrap();
        f.put("home/iris/a", "1");
        f.rotate();
        let id = std::fs::read_to_string(f.root.join("etc/machine-id")).unwrap();
        assert_ne!(id, "0123456789abcdef0123456789abcdef\n");
    }

    /// A root that is gone has nothing to salvage and gets a template.
    #[test]
    fn a_missing_root_gets_a_template() {
        let f = Fixture::new();
        std::fs::remove_dir_all(&f.root).unwrap();
        assert!(!has_content(&f.root).unwrap());
        assert_eq!(f.rotate(), Outcome::Clean);
        assert!(f.root.join("etc").join(MARKER).is_file());
    }

    /// What nix keeps for the agent user is copied to where the container finds
    /// it next time, with its links intact; the old root keeps its own copy. A
    /// directory already there is not overwritten, and a linked `.local` is not
    /// followed.
    #[test]
    fn the_nix_profile_is_carried_over() {
        let f = Fixture::new();
        f.put("home/iris/.local/state/nix/profiles/profile-1-link", "gen");
        symlink(
            "profile-1-link",
            f.root.join("home/iris/.local/state/nix/profiles/profile"),
        )
        .unwrap();
        let Outcome::Salvaged { old_root, .. } = f.rotate() else {
            panic!("must salvage");
        };
        let link = f.profile().join("profiles/profile");
        assert_eq!(
            std::fs::read_link(&link).unwrap(),
            Path::new("profile-1-link")
        );
        assert_eq!(std::fs::read_to_string(&link).unwrap(), "gen");
        assert!(
            old_root
                .join("home/iris/.local/state/nix/profiles/profile-1-link")
                .is_file()
        );
        assert_eq!(
            std::fs::metadata(f.profile()).unwrap().uid(),
            std::fs::metadata(&f.work).unwrap().uid()
        );

        // An existing target stays as it is.
        let g = Fixture::new();
        g.put("home/iris/.local/state/nix/profiles/profile-1-link", "old");
        std::fs::create_dir_all(g.profile()).unwrap();
        std::fs::write(g.profile().join("mine"), "new").unwrap();
        g.rotate();
        assert_eq!(Fixture::names(&g.profile()), ["mine"]);

        // An empty one, as the agent's tmpfiles rules leave it, is filled.
        let e = Fixture::new();
        e.put("home/iris/.local/state/nix/profiles/profile-1-link", "old");
        std::fs::create_dir_all(e.profile()).unwrap();
        e.rotate();
        assert_eq!(Fixture::names(&e.profile()), ["profiles"]);

        // A `.local` that is a link to a directory with the same layout.
        let h = Fixture::new();
        let outside = h.top.join("outside");
        std::fs::create_dir_all(outside.join("state/nix")).unwrap();
        std::fs::write(outside.join("state/nix/secret"), "host").unwrap();
        symlink(&outside, h.root.join("home/iris/.local")).unwrap();
        h.put("home/iris/other", "x");
        h.rotate();
        assert!(!h.profile().exists(), "followed a linked .local");
    }

    /// `per-user` or the agent's directory in it replaced by a link, as the
    /// container can do: nothing lands where the link points, and the salvage
    /// itself still completes.
    #[test]
    fn a_linked_profile_dir_is_not_followed() {
        for linked in ["per-user", "per-user/iris"] {
            let f = Fixture::new();
            f.put("home/iris/.local/state/nix/profiles/profile-1-link", "gen");
            let outside = f.top.join("outside");
            std::fs::create_dir_all(&outside).unwrap();
            if linked == "per-user/iris" {
                std::fs::create_dir_all(f.profiles.join("per-user")).unwrap();
            }
            symlink(&outside, f.profiles.join(linked)).unwrap();
            assert!(matches!(f.rotate(), Outcome::Salvaged { .. }), "{linked}");
            assert!(
                outside.read_dir().unwrap().next().is_none(),
                "{linked}: wrote through the link"
            );
            assert!(
                f.salvage()
                    .join("home/iris/.local/state/nix/profiles/profile-1-link")
                    .is_file()
            );
        }
    }

    const GIB: u64 = 1 << 30;

    /// A copy that would leave less than 15 % of the filesystem free is skipped
    /// and changes nothing; the same root with the space for it is salvaged.
    #[test]
    fn a_copy_that_leaves_too_little_free_is_skipped() {
        // A 100 GiB filesystem keeps 15 GiB free; the root needs a little more.
        let f = Fixture::with(Fake::new().space(15 * GIB, 100 * GIB));
        f.put("home/iris/data", "x");
        let skipped = f.rotate();
        let Outcome::Skipped { need, free } = skipped else {
            panic!("a copy into 15 GiB free must be skipped, got {skipped:?}");
        };
        assert_eq!(free, 15 * GIB);
        assert!(need > 15 * GIB, "need {need}");
        assert!(
            f.work.read_dir().unwrap().next().is_none(),
            "work/ was touched"
        );
        assert!(f.root.join("home/iris/data").is_file());
        assert!(!f.root.join("etc").join(MARKER).exists());
        assert!(
            !report("h-iris", &skipped),
            "a skipped salvage must start persistent"
        );

        // Control: a little more room, the same root.
        let g = Fixture::with(Fake::new().space(15 * GIB + (1 << 20), 100 * GIB));
        g.put("home/iris/data", "x");
        let salvaged = g.rotate();
        assert!(matches!(salvaged, Outcome::Salvaged { .. }), "{salvaged:?}");
        assert!(report("h-iris", &salvaged));
        assert!(report("h-iris", &Outcome::Clean));
    }

    /// Files that clone cost metadata, so the 15 % floor does not apply: the
    /// same root and space that skip when files must be copied go through when
    /// the probe clones.
    #[test]
    fn files_that_clone_need_only_metadata() {
        let copying = Fixture::with(Fake::new().space(5 * GIB, 100 * GIB));
        copying.put("home/iris/data", "x");
        assert!(matches!(copying.rotate(), Outcome::Skipped { .. }));

        let cloning = Fixture::with(Fake::new().space(5 * GIB, 100 * GIB).mode(CloneMode::Copy));
        cloning.put("home/iris/data", "x");
        let Outcome::Salvaged { stats, .. } = cloning.rotate() else {
            panic!("cloning files need no more than the reserve");
        };
        assert_eq!(
            stats,
            Stats {
                bytes: 1,
                files: 1,
                cloned: 1
            }
        );

        // Not even metadata fits below the reserve.
        let full = Fixture::with(
            Fake::new()
                .space(CLONE_RESERVE - 1, 100 * GIB)
                .mode(CloneMode::Copy),
        );
        full.put("home/iris/data", "x");
        assert!(matches!(full.rotate(), Outcome::Skipped { .. }));
    }

    /// A clone that fails (another filesystem, one without reflinks) leaves the
    /// file to be copied, whatever the errno; the control counts a clone.
    #[test]
    fn a_failed_clone_falls_back_to_a_copy() {
        for errno in [libc::EXDEV, libc::EOPNOTSUPP, libc::EINVAL] {
            let f = Fixture::with(Fake::new().mode(CloneMode::Fail(errno)));
            f.put("home/iris/a", "alpha");
            f.put("home/iris/b", "beta");
            let Outcome::Salvaged { salvage, stats, .. } = f.rotate() else {
                panic!("errno {errno}: must salvage");
            };
            assert_eq!(
                std::fs::read_to_string(salvage.join("home/iris/a")).unwrap(),
                "alpha"
            );
            assert_eq!(
                std::fs::read_to_string(salvage.join("home/iris/b")).unwrap(),
                "beta"
            );
            assert_eq!(
                stats,
                Stats {
                    bytes: 9,
                    files: 2,
                    cloned: 0
                },
                "errno {errno}"
            );
        }
        let f = Fixture::with(Fake::new().mode(CloneMode::Copy));
        f.put("home/iris/a", "alpha");
        let Outcome::Salvaged { salvage, stats, .. } = f.rotate() else {
            panic!("must salvage");
        };
        assert_eq!(stats.cloned, 1);
        assert_eq!(
            std::fs::read_to_string(salvage.join("home/iris/a")).unwrap(),
            "alpha"
        );
    }

    /// The real `ioctl(FICLONE)`: whether the scratch filesystem has reflinks or
    /// not, the salvage holds the same bytes.
    #[test]
    fn the_real_clone_call_leaves_a_faithful_copy() {
        let f = Fixture::with(Fake::new().mode(CloneMode::Real));
        let body = "0123456789".repeat(10_000);
        f.put("home/iris/big", &body);
        f.put("home/iris/empty", "");
        let Outcome::Salvaged { salvage, stats, .. } = f.rotate() else {
            panic!("must salvage");
        };
        assert_eq!(
            std::fs::read_to_string(salvage.join("home/iris/big")).unwrap(),
            body
        );
        assert_eq!(stats.files, 2);
        assert!(stats.cloned <= stats.files);
    }

    /// A file that cannot clone, in a salvage whose probe did, is copied only
    /// while the reserve stays free.
    #[test]
    fn a_file_that_must_be_copied_keeps_the_reserve_free() {
        let body = "x".repeat(1 << 20);
        let tight = Fixture::with(
            Fake::new()
                .space(CLONE_RESERVE + (512 << 10), 100 * GIB)
                .mode(CloneMode::FirstOnly),
        );
        tight.put("home/iris/big", &body);
        let err = tight.try_rotate().unwrap_err();
        assert!(format!("{err:#}").contains("must stay"), "{err:#}");
        assert!(
            tight.root.join("home/iris/big").is_file(),
            "root was touched"
        );
        assert!(!tight.root.join("etc").join(MARKER).exists());

        let roomy = Fixture::with(
            Fake::new()
                .space(CLONE_RESERVE + (8 << 20), 100 * GIB)
                .mode(CloneMode::FirstOnly),
        );
        roomy.put("home/iris/big", &body);
        let Outcome::Salvaged { stats, .. } = roomy.rotate() else {
            panic!("1 MiB fits above the reserve");
        };
        assert_eq!(stats.cloned, 0);
    }

    /// A salvage that fails after it created its directory removes that
    /// directory and nothing else: an earlier salvage stays, the old root is
    /// as it was, and no marker is written. Once without room for a copy, once
    /// with a free-space read that fails.
    #[test]
    fn a_failed_salvage_removes_its_partial_directory() {
        let body = "x".repeat(1 << 20);
        let cases = [
            (
                "no room for a file",
                Fake::new()
                    .space(CLONE_RESERVE + (512 << 10), 100 * GIB)
                    .mode(CloneMode::FirstOnly),
            ),
            ("free space unreadable", Fake::new().space_fails_from(1)),
        ];
        for (what, fake) in cases {
            let f = Fixture::with(fake);
            f.put("home/iris/big", &body);
            f.put("home/iris/small", "s");
            // An earlier salvage of the same day, which must survive.
            let earlier = f.salvage();
            std::fs::create_dir_all(earlier.join("home")).unwrap();
            std::fs::write(earlier.join("home/old"), "kept").unwrap();

            assert!(f.try_rotate().is_err(), "{what}: must fail");
            assert_eq!(
                Fixture::names(&f.work),
                [format!("salvaged-rootfs-{DATE}")],
                "{what}: a partial directory is left"
            );
            assert_eq!(
                std::fs::read_to_string(earlier.join("home/old")).unwrap(),
                "kept"
            );
            assert!(
                f.root.join("home/iris/big").is_file(),
                "{what}: root was touched"
            );
            assert!(f.root.join("home/iris/small").is_file(), "{what}");
            assert!(!f.root.join("etc").join(MARKER).exists(), "{what}");
            assert!(
                Fixture::names(&f.top)
                    .iter()
                    .all(|n| !n.contains("pre-ephemeral")),
                "{what}: the root was moved"
            );
        }

        // Control: the same root with room and a working space read succeeds.
        let ok = Fixture::with(Fake::new());
        ok.put("home/iris/big", &body);
        assert!(matches!(ok.rotate(), Outcome::Salvaged { .. }));
        assert!(ok.salvage().join("home/iris/big").is_file());
    }

    /// A second request for a root that is being salvaged waits, then finds
    /// the finished template: one salvage directory, one moved-aside root.
    #[test]
    fn two_requests_for_one_root_salvage_it_once() {
        let f = Fixture::with(
            Fake::new()
                .mode(CloneMode::Copy)
                .delay(Duration::from_millis(200)),
        );
        f.put("home/iris/a", "1");
        let (first, second) = std::thread::scope(|s| {
            let first = s.spawn(|| f.rotate());
            while !f.fake.started.load(Ordering::SeqCst) {
                std::thread::sleep(Duration::from_millis(5));
            }
            let second = s.spawn(|| f.rotate());
            (first.join().unwrap(), second.join().unwrap())
        });
        assert!(matches!(first, Outcome::Salvaged { .. }), "{first:?}");
        assert_eq!(second, Outcome::Clean);
        assert_eq!(Fixture::names(&f.work), [format!("salvaged-rootfs-{DATE}")]);
        let moved: Vec<String> = Fixture::names(&f.top)
            .into_iter()
            .filter(|n| n.starts_with("h-iris.pre-ephemeral-"))
            .collect();
        assert_eq!(moved.len(), 1, "{moved:?}");
    }

    /// Salvages of different containers copy one after the other.
    #[test]
    fn salvages_of_different_containers_copy_one_at_a_time() {
        let a = Fixture::new();
        let b = Fixture::new();
        a.put("home/iris/a", "1");
        b.put("home/iris/b", "2");
        let fake = Fake::new()
            .mode(CloneMode::Copy)
            .delay(Duration::from_millis(100));
        let serial = Mutex::new(());
        std::thread::scope(|s| {
            s.spawn(|| a.try_rotate_in(&fake, &serial).unwrap());
            s.spawn(|| b.try_rotate_in(&fake, &serial).unwrap());
        });
        assert!(fake.calls.load(Ordering::SeqCst) >= 4, "both salvages ran");
        assert_eq!(fake.peak.load(Ordering::SeqCst), 1);
    }

    #[test]
    fn sizes_are_shown_in_gib() {
        assert_eq!(gib(0), "0.0 GiB");
        assert_eq!(gib(3 * GIB / 2), "1.5 GiB");
        assert_eq!(gib(87 * GIB), "87.0 GiB");
    }

    #[test]
    fn dates() {
        assert_eq!(date_of(0), "19700101");
        assert_eq!(date_of(951_782_400), "20000229");
        assert_eq!(date_of(1_793_145_600), "20261028");
    }
}