From 836b15a46e702a4d45d483de113e4782a90c7205 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 21:37:27 +0100 Subject: [PATCH] harden the ephemeral-root salvage: clone, check space, lock - clone each file with FICLONE, copy the data only where that fails - skip the salvage when the copy would leave too little free space - hold a per-container flock and one copy at a time across containers - run WriteNspawnFlags on the blocking pool --- Cargo.lock | 1 + docs/agent-lifecycle/persistence.md | 23 + hive-priv/Cargo.toml | 1 + hive-priv/src/container_config.rs | 26 + hive-priv/src/main.rs | 23 +- hive-priv/src/rootfs.rs | 807 +++++++++++++++++++++++++--- hive-priv/src/salvage_io.rs | 84 +++ nix/host-modules/hive-priv.nix | 4 + 8 files changed, 879 insertions(+), 90 deletions(-) create mode 100644 hive-priv/src/salvage_io.rs diff --git a/Cargo.lock b/Cargo.lock index d2363cf5..8565cce9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1990,6 +1990,7 @@ dependencies = [ "anyhow", "hive-priv-sock", "libc", + "rustix", "serde_json", "tokio", "tracing", diff --git a/docs/agent-lifecycle/persistence.md b/docs/agent-lifecycle/persistence.md index 85d6c563..bcc95e6a 100644 --- a/docs/agent-lifecycle/persistence.md +++ b/docs/agent-lifecycle/persistence.md @@ -414,6 +414,29 @@ which is the only gcroots path bound into a container, so only root writes it. Destroying the container removes the drop-in and then its root; the build stays until the next collection. A drop-in written by an older hive-priv has no `ExecStartPre=` and no root, and the container starts as before. + +**Disk use.** The copy shares blocks with the old root where it can: hive-priv +clones each file (`FICLONE`), which on btrfs or xfs allocates metadata only, and +copies the data of a file the filesystem refuses to clone. Before it creates +anything, hive-priv sizes the root and compares it with the free space under +`work/`. A salvage whose files clone (tested with one file of the root) must +leave 2 GiB free. One that has to copy data must leave 15 % of the filesystem +free. When the free space is short, hive-priv skips the salvage, deletes +nothing, logs `salvage skipped: need free `, and the +container starts persistent; the next start tries again. hive-priv checks a file that fails to +clone against the 2 GiB reserve again before it copies the data. After a +salvage, hive-priv logs `salvage: reflinked=/ files`. + +**Concurrency.** hive-priv holds an `flock` on +`/run/hive-priv/salvage-.lock` (the unit's `RuntimeDirectory`, which +no container can write) for the whole `WriteNspawnFlags`, so a second request for +the same container waits and then finds the finished template. Across +containers, one salvage copies at a time, so the free space each one checks +excludes any copy still in flight. The request runs on the blocking +thread pool, so a long copy holds no async worker. hive-c0re's per-agent lease +already serialises the requests it sends; the lock covers hive-priv's own +guarantee. + The journal still ships. The agent's collector (`nix/agent-modules/otel.nix`) reads the guest's own `/var/log/journal` and pushes it over OTLP, so no journal link to the host takes part; nspawn skips `--link-journal=try-guest` diff --git a/hive-priv/Cargo.toml b/hive-priv/Cargo.toml index a322d0f9..09148524 100644 --- a/hive-priv/Cargo.toml +++ b/hive-priv/Cargo.toml @@ -11,6 +11,7 @@ workspace = true anyhow.workspace = true hive-priv-sock.workspace = true libc.workspace = true +rustix.workspace = true serde_json.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/hive-priv/src/container_config.rs b/hive-priv/src/container_config.rs index 8d88a26a..197747c7 100644 --- a/hive-priv/src/container_config.rs +++ b/hive-priv/src/container_config.rs @@ -61,6 +61,32 @@ pub(crate) fn handle_write_nspawn_flags( Ok((String::new(), String::new())) } +/// [`handle_write_nspawn_flags`] on the blocking pool. The root salvage inside +/// it copies files for as long as the root is big, and would hold a runtime +/// worker for that time. +/// +/// # Errors +/// As [`handle_write_nspawn_flags`], or when the task fails to join. +pub(crate) async fn handle_write_nspawn_flags_blocking( + container: String, + binds: Vec, + isolation: NetworkIsolation, + load_credentials: Vec, + hive_facts: HiveFacts, +) -> Result<(String, String)> { + tokio::task::spawn_blocking(move || { + handle_write_nspawn_flags( + &container, + &binds, + &isolation, + &load_credentials, + &hive_facts, + ) + }) + .await + .context("write nspawn flags task")? +} + /// Validate a system-level container name (`h-` prefix for all agents, /// the manager included) and return the logical agent name after it. /// Anything else is refused, the swarm's forge and matrix containers diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index 2fb357a8..48ca9f7b 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -20,8 +20,9 @@ //! dispatch table. Each op family's handlers and validation live in their //! own module: [`container`] (lifecycle), [`container_config`] (nspawn flags, //! systemd drop-ins), [`rootfs`] (the salvage of a container's root before it -//! turns ephemeral), [`prestart`] (the check of a container's root that runs -//! before each start, as a sub-command), [`storage`] (btrfs subvolumes, quotas, snapshots, disk +//! turns ephemeral, with [`salvage_io`] for its space, reflink and lock calls), +//! [`prestart`] (the check of a container's root that runs before each start, +//! as a sub-command), [`storage`] (btrfs subvolumes, quotas, snapshots, disk //! measurement), [`files`] (pause marker, socket dir, legacy tmpfiles), //! [`infra`] (gateway nginx, infra containers) and [`closure`] (downloads from //! the swarm cache). Helpers more than one family uses are in [`shared`]. @@ -33,6 +34,7 @@ mod files; mod infra; mod prestart; mod rootfs; +mod salvage_io; mod shared; mod storage; @@ -383,18 +385,21 @@ async fn exec( PrivRequest::ListContainers => container::container_run(&["list"]).await, PrivRequest::WriteNspawnFlags { - ref container, - ref binds, - ref isolation, - ref load_credentials, - ref hive_facts, - } => container_config::handle_write_nspawn_flags( container, binds, isolation, load_credentials, hive_facts, - ), + } => { + container_config::handle_write_nspawn_flags_blocking( + container, + binds, + isolation, + load_credentials, + hive_facts, + ) + .await + } PrivRequest::EnsureAgentSocketDir { ref name } => files::ensure_agent_socket_dir(name), diff --git a/hive-priv/src/rootfs.rs b/hive-priv/src/rootfs.rs index 870c3ef7..56d7f62e 100644 --- a/hive-priv/src/rootfs.rs +++ b/hive-priv/src/rootfs.rs @@ -16,14 +16,32 @@ //! //! Only a stopped container is touched: a running one keeps whatever flag its //! root already warrants. +//! +//! Disk use. A file is cloned (`FICLONE`) when the filesystem can, which costs +//! metadata only; otherwise its data is copied. Before creating anything, +//! [`rotate`] compares what the copy needs with the free space under `work/` and +//! returns [`Outcome::Skipped`] when less than [`COPY_FLOOR_PERCENT`] of the +//! filesystem would stay free, or less than [`CLONE_RESERVE`] when a probe shows +//! files clone. A file that fails to clone is checked against the reserve again. +//! +//! Concurrency. One salvage per container runs at a time, under an `flock` in +//! [`LOCK_DIR`]; a second request waits, then sees the finished template. Across +//! containers [`ONE_AT_A_TIME`] admits one copy at a time, so the free space a +//! salvage reads excludes any copy still in flight. -use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _, lchown, symlink}; +use std::fs::{File, OpenOptions}; +use std::io; +use std::os::unix::fs::{ + MetadataExt as _, OpenOptionsExt as _, PermissionsExt as _, lchown, symlink, +}; use std::path::{Path, PathBuf}; +use std::sync::{Mutex, PoisonError}; use anyhow::{Context as _, Result, bail}; use hive_priv_sock::AGENT_STATE_ROOT; use crate::container_config::{container_rootfs, write_container_etc_file_in}; +use crate::salvage_io::{Backend, ContainerLock, LOCK_DIR, ONE_AT_A_TIME, Os}; /// The per-container nix profile bind on the host, which persists across starts. /// What nix keeps for the agent user is carried here; see [`carry_nix_state`]. @@ -54,6 +72,25 @@ const CACHE: [&str; 5] = [ "var/lib/opentelemetry-collector", ]; +/// Share of the filesystem under `work/` that must stay free after a salvage +/// that copies data. btrfs allocates metadata chunks from unallocated space, and +/// every agent and host service writes to the same filesystem, so a full disk +/// stops all of them. +const COPY_FLOOR_PERCENT: u64 = 15; + +/// Free space a salvage must leave when its files clone, which allocate +/// metadata only; also the margin kept when a single file has to be copied. +const CLONE_RESERVE: u64 = 2 << 30; + +/// Metadata a cloned entry is assumed to cost. +const ENTRY_COST: u64 = 8 << 10; + +/// Allocation unit a copied file is rounded up to. +const BLOCK: u64 = 4096; + +/// The file in `work/` that tests whether cloning into it works. +const CLONE_PROBE: &str = ".hive-clone-probe"; + /// What [`rotate`] did. #[derive(Debug, PartialEq, Eq)] pub(crate) enum Outcome { @@ -61,7 +98,34 @@ pub(crate) enum Outcome { Clean, /// The root's content is copied to `salvage`, and the root itself kept at /// `old_root`. - Salvaged { salvage: PathBuf, old_root: PathBuf }, + Salvaged { + salvage: PathBuf, + old_root: PathBuf, + stats: Stats, + }, + /// The copy would leave too little free space: `need` bytes must be free, + /// `free` are. Nothing was created or moved. + Skipped { need: u64, free: u64 }, +} + +/// What a salvage copied. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Stats { + /// Bytes in the regular files. + pub(crate) bytes: u64, + /// Regular files copied. + pub(crate) files: u64, + /// Of those, the files that share blocks with the old root. + pub(crate) cloned: u64, +} + +/// The host a salvage runs against. +pub(crate) struct Env<'a> { + pub(crate) backend: &'a dyn Backend, + /// Where the per-container lock files go. + pub(crate) lock_dir: &'a Path, + /// Taken by the salvage that is copying; see the module docs. + pub(crate) one_at_a_time: &'a Mutex<()>, } /// Whether `container` may be started with `--ephemeral`, salvaging its root @@ -70,7 +134,8 @@ pub(crate) enum Outcome { /// # Errors /// Returns an error when the unit state cannot be read or the salvage fails. /// Nothing is deleted on that path, and the caller starts the container -/// without the flag. +/// without the flag. A salvage skipped for lack of space is not an error: it +/// returns `false` and the next start tries again. pub(crate) fn prepare(container: &str, agent: &str) -> Result { let root = container_rootfs(container); if !unit_is_stopped(container)? { @@ -78,47 +143,231 @@ pub(crate) fn prepare(container: &str, agent: &str) -> Result { } let work = Path::new(AGENT_STATE_ROOT).join(agent).join("work"); let profiles = Path::new(PROFILES_ROOT).join(container); - match rotate(&root, agent, &work, &profiles, &today())? { + let env = Env { + backend: &Os, + lock_dir: Path::new(LOCK_DIR), + one_at_a_time: &ONE_AT_A_TIME, + }; + let outcome = rotate(&root, agent, &work, &profiles, &today(), &env)?; + Ok(report(container, &outcome)) +} + +/// Log what [`rotate`] did, and say whether the container may start `--ephemeral`. +fn report(container: &str, outcome: &Outcome) -> bool { + match outcome { Outcome::Clean => {} - Outcome::Salvaged { salvage, old_root } => tracing::warn!( - %container, - salvage = %salvage.display(), - old_root = %old_root.display(), - "root salvaged before the first ephemeral start; the old root is kept, remove it by hand" - ), + Outcome::Salvaged { + salvage, + old_root, + stats, + } => { + tracing::info!( + "salvage: {container} {} reflinked={}/{} files", + gib(stats.bytes), + stats.cloned, + stats.files + ); + tracing::warn!( + %container, + salvage = %salvage.display(), + old_root = %old_root.display(), + "root salvaged before the first ephemeral start; the old root is kept, remove it by hand" + ); + } + Outcome::Skipped { need, free } => { + tracing::warn!( + "salvage skipped: {container} need {} free {}", + gib(*need), + gib(*free) + ); + return false; + } } - Ok(true) + true +} + +/// `bytes` in GiB, to one decimal. +fn gib(bytes: u64) -> String { + const GIB: u64 = 1 << 30; + format!("{}.{} GiB", bytes / GIB, bytes % GIB * 10 / GIB) } /// Salvage `root` if it holds content, then leave a marked template in its /// place. `work` is the agent's work dir on the host, `profiles` the host side of /// the container's `/nix/var/nix/profiles`; `date` is `YYYYMMDD`. -fn rotate(root: &Path, agent: &str, work: &Path, profiles: &Path, date: &str) -> Result { +/// +/// The container's lock is held throughout, so a second call for the same root +/// waits and then finds the template the first one left. +fn rotate( + root: &Path, + agent: &str, + work: &Path, + profiles: &Path, + date: &str, + env: &Env<'_>, +) -> Result { + let name = root + .file_name() + .and_then(|n| n.to_str()) + .with_context(|| format!("{} has no usable name", root.display()))?; + let _lock = ContainerLock::acquire(env.lock_dir, name)?; let outcome = if has_content(root)? { - let (uid, gid) = agent_owner(root, agent, work)?; - let machine_id = read_machine_id(root); - let salvage = fresh_dir(work, &format!("salvaged-rootfs-{date}"))?; - lchown(&salvage, Some(uid), Some(gid)) - .with_context(|| format!("chown {}", salvage.display()))?; - copy_tree(root, &salvage, Path::new(""), uid, gid) - .with_context(|| format!("copy {} to {}", root.display(), salvage.display()))?; - carry_nix_state(root, agent, profiles, uid, gid)?; - let old_root = free_sibling(root, &format!("pre-ephemeral-{date}"))?; - std::fs::rename(root, &old_root) - .with_context(|| format!("move {} to {}", root.display(), old_root.display()))?; - std::fs::create_dir(root).with_context(|| format!("create {}", root.display()))?; - if let Some(id) = machine_id { - write_container_etc_file_in(root, "machine-id", &format!("{id}\n"))?; - } - Outcome::Salvaged { salvage, old_root } + let _turn = env + .one_at_a_time + .lock() + .unwrap_or_else(PoisonError::into_inner); + salvage(root, agent, work, profiles, date, env)? } else { Outcome::Clean }; + if matches!(outcome, Outcome::Skipped { .. }) { + return Ok(outcome); + } ensure_machine_id(root)?; write_container_etc_file_in(root, MARKER, "")?; Ok(outcome) } +/// The copy-and-move half of [`rotate`], for a root that has content. +fn salvage( + root: &Path, + agent: &str, + work: &Path, + profiles: &Path, + date: &str, + env: &Env<'_>, +) -> Result { + let (uid, gid) = agent_owner(root, agent, work)?; + let machine_id = read_machine_id(root); + if let Some((need, free)) = shortfall(env.backend, root, work)? { + return Ok(Outcome::Skipped { need, free }); + } + let salvage = fresh_dir(work, &format!("salvaged-rootfs-{date}"))?; + lchown(&salvage, Some(uid), Some(gid)) + .with_context(|| format!("chown {}", salvage.display()))?; + let mut copier = Copier::new(env.backend, uid, gid); + copier + .tree(root, &salvage, Path::new("")) + .with_context(|| format!("copy {} to {}", root.display(), salvage.display()))?; + carry_nix_state(root, agent, profiles, env.backend, uid, gid)?; + let old_root = free_sibling(root, &format!("pre-ephemeral-{date}"))?; + std::fs::rename(root, &old_root) + .with_context(|| format!("move {} to {}", root.display(), old_root.display()))?; + std::fs::create_dir(root).with_context(|| format!("create {}", root.display()))?; + if let Some(id) = machine_id { + write_container_etc_file_in(root, "machine-id", &format!("{id}\n"))?; + } + Ok(Outcome::Salvaged { + salvage, + old_root, + stats: copier.stats, + }) +} + +/// What [`measure`] found under a root. +struct Tally { + /// Everything outside [`CACHE`], directories and links included. + entries: u64, + /// Regular-file bytes, each file rounded up to [`BLOCK`]. + bytes: u64, + /// The first regular file with data, for the clone probe. + sample: Option, +} + +/// Size up what [`Copier::tree`] would copy from `root`. +fn measure(root: &Path) -> Result { + fn walk(dir: &Path, rel: &Path, tally: &mut Tally) -> Result<()> { + for entry in std::fs::read_dir(dir).with_context(|| format!("read {}", dir.display()))? { + let entry = entry.with_context(|| format!("read {}", dir.display()))?; + let rel = rel.join(entry.file_name()); + if is_cache(&rel) { + continue; + } + let meta = entry + .metadata() + .with_context(|| format!("stat {}", entry.path().display()))?; + tally.entries += 1; + if meta.is_dir() { + walk(&entry.path(), &rel, tally)?; + } else if meta.is_file() { + tally.bytes = tally + .bytes + .saturating_add(meta.len().div_ceil(BLOCK) * BLOCK); + if meta.len() > 0 && tally.sample.is_none() { + tally.sample = Some(entry.path()); + } + } + } + Ok(()) + } + let mut tally = Tally { + entries: 0, + bytes: 0, + sample: None, + }; + walk(root, Path::new(""), &mut tally)?; + Ok(tally) +} + +/// `Some((need, free))` when a salvage of `root` into `work` would leave less +/// free space than it must: `need` bytes have to be free, `free` are. +/// +/// Files that clone cost metadata only. Whether they do is probed with one +/// file of the root ([`clones`]); when that fails the whole data is counted. +fn shortfall(backend: &dyn Backend, root: &Path, work: &Path) -> Result> { + let tally = measure(root)?; + let space = backend + .space(work) + .with_context(|| format!("free space of {}", work.display()))?; + let metadata = tally.entries.saturating_mul(ENTRY_COST); + let need = if tally + .sample + .as_deref() + .is_none_or(|sample| clones(backend, sample, work)) + { + metadata.saturating_add(CLONE_RESERVE) + } else { + let floor = space.total / 100 * COPY_FLOOR_PERCENT; + tally.bytes.saturating_add(metadata).saturating_add(floor) + }; + Ok((space.free < need).then_some((need, space.free))) +} + +/// Whether `sample` can be cloned into a file in `work`. The probe file is +/// created exclusively and removed again; if one is already there the answer +/// is no. +fn clones(backend: &dyn Backend, sample: &Path, work: &Path) -> bool { + let probe = work.join(CLONE_PROBE); + let Ok(src) = open_nofollow(sample) else { + return false; + }; + let Ok(dst) = create_new(&probe) else { + return false; + }; + let cloned = backend.clone_file(&src, &dst).is_ok(); + drop(dst); + std::fs::remove_file(&probe).ok(); + cloned +} + +/// `path` opened for reading; a link at `path` is an error. +fn open_nofollow(path: &Path) -> io::Result { + OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path) +} + +/// `path` created for writing, mode 0600; an existing entry is an error. +fn create_new(path: &Path) -> io::Result { + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW) + .open(path) +} + /// Whether anything but directories, [`TEMPLATE_FILES`] and [`CACHE`] is under /// `root`. A missing root has none. fn has_content(root: &Path) -> Result { @@ -198,7 +447,14 @@ fn real_dir(base: &Path, parts: &[&str]) -> Option { /// GC roots that name them are still valid after the switch. A target that /// holds anything is left alone; so is the whole copy when a component of the /// target path is not a real directory, as the container can write `profiles`. -fn carry_nix_state(root: &Path, agent: &str, profiles: &Path, uid: u32, gid: u32) -> Result<()> { +fn carry_nix_state( + root: &Path, + agent: &str, + profiles: &Path, + backend: &dyn Backend, + uid: u32, + gid: u32, +) -> Result<()> { let Some(src) = real_dir(root, &["home", agent, ".local", "state", "nix"]) else { return Ok(()); }; @@ -218,7 +474,8 @@ fn carry_nix_state(root: &Path, agent: &str, profiles: &Path, uid: u32, gid: u32 { return Ok(()); } - copy_tree(&src, &target, Path::new(""), uid, gid) + Copier::new(backend, uid, gid) + .tree(&src, &target, Path::new("")) .with_context(|| format!("copy {} to {}", src.display(), target.display())) } @@ -287,49 +544,101 @@ fn free_sibling(root: &Path, suffix: &str) -> Result { unreachable!("the loop above returns or continues") } -/// Copy everything under `src` except [`CACHE`] into `dst`, owned by -/// `uid:gid`, with modes and mtimes kept. Sockets, fifos and device nodes are -/// skipped; hard links become separate copies. -fn copy_tree(src: &Path, dst: &Path, rel: &Path, uid: u32, gid: u32) -> Result<()> { - let dir = src.join(rel); - for entry in std::fs::read_dir(&dir).with_context(|| format!("read {}", dir.display()))? { - let entry = entry.with_context(|| format!("read {}", dir.display()))?; - let rel = rel.join(entry.file_name()); - if is_cache(&rel) { - continue; - } - let from = src.join(&rel); - let to = dst.join(&rel); - let meta = - std::fs::symlink_metadata(&from).with_context(|| format!("stat {}", from.display()))?; - let kind = meta.file_type(); - if kind.is_dir() { - std::fs::create_dir(&to).with_context(|| format!("create {}", to.display()))?; - copy_tree(src, dst, &rel, uid, gid)?; - lchown(&to, Some(uid), Some(gid)).with_context(|| format!("chown {}", to.display()))?; - set_mode(&to, &meta)?; - } else if kind.is_symlink() { - let target = std::fs::read_link(&from) - .with_context(|| format!("read link {}", from.display()))?; - symlink(&target, &to).with_context(|| format!("link {}", to.display()))?; - lchown(&to, Some(uid), Some(gid)).with_context(|| format!("chown {}", to.display()))?; - } else if kind.is_file() { - std::fs::copy(&from, &to) - .with_context(|| format!("copy {} to {}", from.display(), to.display()))?; - if let Ok(mtime) = meta.modified() { - std::fs::File::options() - .write(true) - .open(&to) - .and_then(|f| f.set_modified(mtime)) - .with_context(|| format!("set mtime of {}", to.display()))?; - } - lchown(&to, Some(uid), Some(gid)).with_context(|| format!("chown {}", to.display()))?; - set_mode(&to, &meta)?; - } else { - tracing::info!(path = %from.display(), "salvage skips a socket, fifo or device node"); +/// Copies a tree for [`salvage`] and keeps count. +struct Copier<'a> { + backend: &'a dyn Backend, + uid: u32, + gid: u32, + stats: Stats, +} + +impl<'a> Copier<'a> { + fn new(backend: &'a dyn Backend, uid: u32, gid: u32) -> Self { + Self { + backend, + uid, + gid, + stats: Stats::default(), } } - Ok(()) + + /// Copy everything under `src` except [`CACHE`] into `dst`, owned by + /// `uid:gid`, with modes and mtimes kept. Sockets, fifos and device nodes + /// are skipped; hard links become separate copies. + fn tree(&mut self, src: &Path, dst: &Path, rel: &Path) -> Result<()> { + let (uid, gid) = (self.uid, self.gid); + let dir = src.join(rel); + for entry in std::fs::read_dir(&dir).with_context(|| format!("read {}", dir.display()))? { + let entry = entry.with_context(|| format!("read {}", dir.display()))?; + let rel = rel.join(entry.file_name()); + if is_cache(&rel) { + continue; + } + let from = src.join(&rel); + let to = dst.join(&rel); + let meta = std::fs::symlink_metadata(&from) + .with_context(|| format!("stat {}", from.display()))?; + let kind = meta.file_type(); + if kind.is_dir() { + std::fs::create_dir(&to).with_context(|| format!("create {}", to.display()))?; + self.tree(src, dst, &rel)?; + lchown(&to, Some(uid), Some(gid)) + .with_context(|| format!("chown {}", to.display()))?; + set_mode(&to, &meta)?; + } else if kind.is_symlink() { + let target = std::fs::read_link(&from) + .with_context(|| format!("read link {}", from.display()))?; + symlink(&target, &to).with_context(|| format!("link {}", to.display()))?; + lchown(&to, Some(uid), Some(gid)) + .with_context(|| format!("chown {}", to.display()))?; + } else if kind.is_file() { + self.file(&from, &to, &meta)?; + } else { + tracing::info!(path = %from.display(), "salvage skips a socket, fifo or device node"); + } + } + Ok(()) + } + + /// One regular file: cloned when the filesystem allows it, else copied. + fn file(&mut self, from: &Path, to: &Path, meta: &std::fs::Metadata) -> Result<()> { + let mut src = open_nofollow(from).with_context(|| format!("open {}", from.display()))?; + let mut dst = create_new(to).with_context(|| format!("create {}", to.display()))?; + self.stats.files += 1; + self.stats.bytes = self.stats.bytes.saturating_add(meta.len()); + if self.backend.clone_file(&src, &dst).is_ok() { + self.stats.cloned += 1; + } else { + self.check_room(to, meta.len())?; + io::copy(&mut src, &mut dst) + .with_context(|| format!("copy {} to {}", from.display(), to.display()))?; + } + if let Ok(mtime) = meta.modified() { + dst.set_modified(mtime) + .with_context(|| format!("set mtime of {}", to.display()))?; + } + lchown(to, Some(self.uid), Some(self.gid)) + .with_context(|| format!("chown {}", to.display()))?; + set_mode(to, meta) + } + + /// Fail when copying `len` bytes to `to` would leave less than + /// [`CLONE_RESERVE`] free. + fn check_room(&self, to: &Path, len: u64) -> Result<()> { + let dir = to.parent().unwrap_or(to); + let space = self + .backend + .space(dir) + .with_context(|| format!("free space of {}", dir.display()))?; + if space.free < len.saturating_add(CLONE_RESERVE) { + bail!( + "{} is {len} bytes and the filesystem has {} free; at least {CLONE_RESERVE} must stay", + to.display(), + space.free + ); + } + Ok(()) + } } /// Give `path` the permission bits of `meta`, which `chown` may have cleared. @@ -406,14 +715,108 @@ fn date_of(secs: u64) -> String { #[cfg(test)] mod tests { + use std::fs::File; + use std::io; use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _, lchown, symlink}; use std::path::{Path, PathBuf}; + use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + use std::time::Duration; - use super::{MARKER, Outcome, date_of, has_content, rotate}; + use anyhow::Result; + + use super::{ + CLONE_RESERVE, Env, MARKER, Outcome, Stats, date_of, gib, has_content, report, rotate, + }; + use crate::salvage_io::{Backend, FsSpace, Os}; use crate::shared::test_support::scratch; const DATE: &str = "20261028"; + /// How [`Fake`] answers a clone request. + #[derive(Clone, Copy)] + enum CloneMode { + /// Succeeds, by copying the bytes. + Copy, + /// Fails with this errno. + Fail(i32), + /// Succeeds for the first request (the probe), fails for the rest. + FirstOnly, + /// Defers to the real `ioctl(FICLONE)`. + Real, + } + + /// A [`Backend`] with the free space and the clone result set by the test. + struct Fake { + free: u64, + total: u64, + mode: CloneMode, + delay: Duration, + calls: AtomicU32, + started: AtomicBool, + active: AtomicU32, + peak: AtomicU32, + } + + impl Fake { + /// Plenty of space; every clone fails with `EXDEV`, so files are copied. + fn new() -> Self { + Self { + free: 1 << 50, + total: 1 << 51, + mode: CloneMode::Fail(libc::EXDEV), + delay: Duration::ZERO, + calls: AtomicU32::new(0), + started: AtomicBool::new(false), + active: AtomicU32::new(0), + peak: AtomicU32::new(0), + } + } + + fn space(mut self, free: u64, total: u64) -> Self { + self.free = free; + self.total = total; + self + } + + fn mode(mut self, mode: CloneMode) -> Self { + self.mode = mode; + self + } + + fn delay(mut self, delay: Duration) -> Self { + self.delay = delay; + self + } + } + + impl Backend for Fake { + fn space(&self, _path: &Path) -> io::Result { + Ok(FsSpace { + free: self.free, + total: self.total, + }) + } + + fn clone_file(&self, src: &File, dst: &File) -> io::Result<()> { + let call = self.calls.fetch_add(1, Ordering::SeqCst); + self.started.store(true, Ordering::SeqCst); + let active = self.active.fetch_add(1, Ordering::SeqCst) + 1; + self.peak.fetch_max(active, Ordering::SeqCst); + std::thread::sleep(self.delay); + let copy = || io::copy(&mut &*src, &mut &*dst).map(drop); + let result = match self.mode { + CloneMode::Copy => copy(), + CloneMode::Fail(errno) => Err(io::Error::from_raw_os_error(errno)), + CloneMode::FirstOnly if call == 0 => copy(), + CloneMode::FirstOnly => Err(io::Error::from_raw_os_error(libc::EXDEV)), + CloneMode::Real => Os.clone_file(src, dst), + }; + self.active.fetch_sub(1, Ordering::SeqCst); + result + } + } + /// A stopped agent's old root and work dir under one scratch dir. The /// "agent" is the test's own user, the one it can chown to. struct Fixture { @@ -421,6 +824,8 @@ mod tests { root: PathBuf, work: PathBuf, profiles: PathBuf, + fake: Fake, + serial: Mutex<()>, } impl Fixture { @@ -437,17 +842,38 @@ mod tests { root, work, profiles, + fake: Fake::new(), + serial: Mutex::new(()), } } + fn with(fake: Fake) -> Self { + let mut f = Self::new(); + f.fake = fake; + f + } + fn put(&self, rel: &str, body: &str) { let path = self.root.join(rel); std::fs::create_dir_all(path.parent().unwrap()).unwrap(); std::fs::write(path, body).unwrap(); } + fn try_rotate_in(&self, backend: &dyn Backend, serial: &Mutex<()>) -> Result { + let env = Env { + backend, + lock_dir: &self.top.join("locks"), + one_at_a_time: serial, + }; + rotate(&self.root, "iris", &self.work, &self.profiles, DATE, &env) + } + + fn try_rotate(&self) -> Result { + self.try_rotate_in(&self.fake, &self.serial) + } + fn rotate(&self) -> Outcome { - rotate(&self.root, "iris", &self.work, &self.profiles, DATE).unwrap() + self.try_rotate().unwrap() } /// Where the container finds nix's per-user state, on the host. @@ -483,7 +909,10 @@ mod tests { f.put("home/iris/notes.txt", "keep me"); f.put("etc/hostname", "iris"); - let Outcome::Salvaged { salvage, old_root } = f.rotate() else { + let Outcome::Salvaged { + salvage, old_root, .. + } = f.rotate() + else { panic!("the first call must salvage"); }; assert_eq!(salvage, f.salvage()); @@ -552,7 +981,10 @@ mod tests { ] { f.put(real, "real"); } - let Outcome::Salvaged { salvage, old_root } = f.rotate() else { + let Outcome::Salvaged { + salvage, old_root, .. + } = f.rotate() + else { panic!("must salvage"); }; for cache in [ @@ -590,10 +1022,7 @@ mod tests { assert!(f.root.join("etc").join(MARKER).is_file()); // The same call with one real file makes the control: it salvages. f.put("home/iris/real", "x"); - assert!(matches!( - rotate(&f.root, "iris", &f.work, &f.profiles, DATE).unwrap(), - Outcome::Salvaged { .. } - )); + assert!(matches!(f.rotate(), Outcome::Salvaged { .. })); } /// Nothing is removed: the old root keeps every file, and a failed @@ -619,7 +1048,7 @@ mod tests { g.put("home/iris/file", "data"); std::fs::remove_dir(&g.work).unwrap(); std::fs::write(&g.work, "not a dir").unwrap(); - assert!(rotate(&g.root, "iris", &g.work, &g.profiles, DATE).is_err()); + assert!(g.try_rotate().is_err()); assert_eq!( std::fs::read_to_string(g.root.join("home/iris/file")).unwrap(), "data" @@ -666,7 +1095,10 @@ mod tests { lchown(f.root.join("home/iris"), None, Some(other)).unwrap(); } - let Outcome::Salvaged { salvage, old_root } = f.rotate() else { + let Outcome::Salvaged { + salvage, old_root, .. + } = f.rotate() + else { panic!("must salvage"); }; let me = std::fs::metadata(old_root.join("home/iris")).unwrap(); @@ -833,6 +1265,219 @@ mod tests { } } + const GIB: u64 = 1 << 30; + + /// A copy that would leave less than 15 % of the filesystem free is skipped + /// and changes nothing; the same root with the space for it is salvaged. + #[test] + fn a_copy_that_leaves_too_little_free_is_skipped() { + // A 100 GiB filesystem keeps 15 GiB free; the root needs a little more. + let f = Fixture::with(Fake::new().space(15 * GIB, 100 * GIB)); + f.put("home/iris/data", "x"); + let skipped = f.rotate(); + let Outcome::Skipped { need, free } = skipped else { + panic!("a copy into 15 GiB free must be skipped, got {skipped:?}"); + }; + assert_eq!(free, 15 * GIB); + assert!(need > 15 * GIB, "need {need}"); + assert!( + f.work.read_dir().unwrap().next().is_none(), + "work/ was touched" + ); + assert!(f.root.join("home/iris/data").is_file()); + assert!(!f.root.join("etc").join(MARKER).exists()); + assert!( + !report("h-iris", &skipped), + "a skipped salvage must start persistent" + ); + + // Control: a little more room, the same root. + let g = Fixture::with(Fake::new().space(15 * GIB + (1 << 20), 100 * GIB)); + g.put("home/iris/data", "x"); + let salvaged = g.rotate(); + assert!(matches!(salvaged, Outcome::Salvaged { .. }), "{salvaged:?}"); + assert!(report("h-iris", &salvaged)); + assert!(report("h-iris", &Outcome::Clean)); + } + + /// Files that clone cost metadata, so the 15 % floor does not apply: the + /// same root and space that skip when files must be copied go through when + /// the probe clones. + #[test] + fn files_that_clone_need_only_metadata() { + let copying = Fixture::with(Fake::new().space(5 * GIB, 100 * GIB)); + copying.put("home/iris/data", "x"); + assert!(matches!(copying.rotate(), Outcome::Skipped { .. })); + + let cloning = Fixture::with(Fake::new().space(5 * GIB, 100 * GIB).mode(CloneMode::Copy)); + cloning.put("home/iris/data", "x"); + let Outcome::Salvaged { stats, .. } = cloning.rotate() else { + panic!("cloning files need no more than the reserve"); + }; + assert_eq!( + stats, + Stats { + bytes: 1, + files: 1, + cloned: 1 + } + ); + + // Not even metadata fits below the reserve. + let full = Fixture::with( + Fake::new() + .space(CLONE_RESERVE - 1, 100 * GIB) + .mode(CloneMode::Copy), + ); + full.put("home/iris/data", "x"); + assert!(matches!(full.rotate(), Outcome::Skipped { .. })); + } + + /// A clone that fails (another filesystem, one without reflinks) leaves the + /// file to be copied, whatever the errno; the control counts a clone. + #[test] + fn a_failed_clone_falls_back_to_a_copy() { + for errno in [libc::EXDEV, libc::EOPNOTSUPP, libc::EINVAL] { + let f = Fixture::with(Fake::new().mode(CloneMode::Fail(errno))); + f.put("home/iris/a", "alpha"); + f.put("home/iris/b", "beta"); + let Outcome::Salvaged { salvage, stats, .. } = f.rotate() else { + panic!("errno {errno}: must salvage"); + }; + assert_eq!( + std::fs::read_to_string(salvage.join("home/iris/a")).unwrap(), + "alpha" + ); + assert_eq!( + std::fs::read_to_string(salvage.join("home/iris/b")).unwrap(), + "beta" + ); + assert_eq!( + stats, + Stats { + bytes: 9, + files: 2, + cloned: 0 + }, + "errno {errno}" + ); + } + let f = Fixture::with(Fake::new().mode(CloneMode::Copy)); + f.put("home/iris/a", "alpha"); + let Outcome::Salvaged { salvage, stats, .. } = f.rotate() else { + panic!("must salvage"); + }; + assert_eq!(stats.cloned, 1); + assert_eq!( + std::fs::read_to_string(salvage.join("home/iris/a")).unwrap(), + "alpha" + ); + } + + /// The real `ioctl(FICLONE)`: whether the scratch filesystem has reflinks or + /// not, the salvage holds the same bytes. + #[test] + fn the_real_clone_call_leaves_a_faithful_copy() { + let f = Fixture::with(Fake::new().mode(CloneMode::Real)); + let body = "0123456789".repeat(10_000); + f.put("home/iris/big", &body); + f.put("home/iris/empty", ""); + let Outcome::Salvaged { salvage, stats, .. } = f.rotate() else { + panic!("must salvage"); + }; + assert_eq!( + std::fs::read_to_string(salvage.join("home/iris/big")).unwrap(), + body + ); + assert_eq!(stats.files, 2); + assert!(stats.cloned <= stats.files); + } + + /// A file that cannot clone, in a salvage whose probe did, is copied only + /// while the reserve stays free. + #[test] + fn a_file_that_must_be_copied_keeps_the_reserve_free() { + let body = "x".repeat(1 << 20); + let tight = Fixture::with( + Fake::new() + .space(CLONE_RESERVE + (512 << 10), 100 * GIB) + .mode(CloneMode::FirstOnly), + ); + tight.put("home/iris/big", &body); + let err = tight.try_rotate().unwrap_err(); + assert!(format!("{err:#}").contains("must stay"), "{err:#}"); + assert!( + tight.root.join("home/iris/big").is_file(), + "root was touched" + ); + assert!(!tight.root.join("etc").join(MARKER).exists()); + + let roomy = Fixture::with( + Fake::new() + .space(CLONE_RESERVE + (8 << 20), 100 * GIB) + .mode(CloneMode::FirstOnly), + ); + roomy.put("home/iris/big", &body); + let Outcome::Salvaged { stats, .. } = roomy.rotate() else { + panic!("1 MiB fits above the reserve"); + }; + assert_eq!(stats.cloned, 0); + } + + /// A second request for a root that is being salvaged waits, then finds + /// the finished template: one salvage directory, one moved-aside root. + #[test] + fn two_requests_for_one_root_salvage_it_once() { + let f = Fixture::with( + Fake::new() + .mode(CloneMode::Copy) + .delay(Duration::from_millis(200)), + ); + f.put("home/iris/a", "1"); + let (first, second) = std::thread::scope(|s| { + let first = s.spawn(|| f.rotate()); + while !f.fake.started.load(Ordering::SeqCst) { + std::thread::sleep(Duration::from_millis(5)); + } + let second = s.spawn(|| f.rotate()); + (first.join().unwrap(), second.join().unwrap()) + }); + assert!(matches!(first, Outcome::Salvaged { .. }), "{first:?}"); + assert_eq!(second, Outcome::Clean); + assert_eq!(Fixture::names(&f.work), [format!("salvaged-rootfs-{DATE}")]); + let moved: Vec = Fixture::names(&f.top) + .into_iter() + .filter(|n| n.starts_with("h-iris.pre-ephemeral-")) + .collect(); + assert_eq!(moved.len(), 1, "{moved:?}"); + } + + /// Salvages of different containers copy one after the other. + #[test] + fn salvages_of_different_containers_copy_one_at_a_time() { + let a = Fixture::new(); + let b = Fixture::new(); + a.put("home/iris/a", "1"); + b.put("home/iris/b", "2"); + let fake = Fake::new() + .mode(CloneMode::Copy) + .delay(Duration::from_millis(100)); + let serial = Mutex::new(()); + std::thread::scope(|s| { + s.spawn(|| a.try_rotate_in(&fake, &serial).unwrap()); + s.spawn(|| b.try_rotate_in(&fake, &serial).unwrap()); + }); + assert!(fake.calls.load(Ordering::SeqCst) >= 4, "both salvages ran"); + assert_eq!(fake.peak.load(Ordering::SeqCst), 1); + } + + #[test] + fn sizes_are_shown_in_gib() { + assert_eq!(gib(0), "0.0 GiB"); + assert_eq!(gib(3 * GIB / 2), "1.5 GiB"); + assert_eq!(gib(87 * GIB), "87.0 GiB"); + } + #[test] fn dates() { assert_eq!(date_of(0), "19700101"); diff --git a/hive-priv/src/salvage_io.rs b/hive-priv/src/salvage_io.rs new file mode 100644 index 00000000..c11ca0a1 --- /dev/null +++ b/hive-priv/src/salvage_io.rs @@ -0,0 +1,84 @@ +//! What the root salvage ([`crate::rootfs`]) asks of the host: free space, +//! reflinks, and the locks that keep two salvages apart. +//! +//! The space and reflink calls sit behind [`Backend`] so a test can set the +//! free space and make a clone fail. + +use std::fs::{File, OpenOptions}; +use std::io; +use std::os::unix::fs::OpenOptionsExt as _; +use std::path::Path; +use std::sync::Mutex; + +use anyhow::{Context as _, Result}; + +/// Where the per-container lock files live. A root-owned runtime directory +/// (`RuntimeDirectory=` of `hive-priv.service`), so no container can write it. +pub(crate) const LOCK_DIR: &str = "/run/hive-priv"; + +/// Held by the one salvage that is copying data at any moment. +pub(crate) static ONE_AT_A_TIME: Mutex<()> = Mutex::new(()); + +/// Size and free space of a filesystem, in bytes. `free` is what an +/// unprivileged writer could still use. +#[derive(Debug, Clone, Copy)] +pub(crate) struct FsSpace { + pub(crate) free: u64, + pub(crate) total: u64, +} + +/// The host calls the salvage makes. +pub(crate) trait Backend { + /// Space on the filesystem holding `path`. + fn space(&self, path: &Path) -> io::Result; + + /// Make `dst`, an empty file, share `src`'s data blocks. An error leaves + /// `dst` as it was: the filesystem cannot clone, or `src` and `dst` are on + /// different ones. + fn clone_file(&self, src: &File, dst: &File) -> io::Result<()>; +} + +/// The real filesystem: `statvfs(2)` and `ioctl(FICLONE)`. +pub(crate) struct Os; + +impl Backend for Os { + fn space(&self, path: &Path) -> io::Result { + let st = rustix::fs::statvfs(path)?; + Ok(FsSpace { + free: st.f_bavail.saturating_mul(st.f_frsize), + total: st.f_blocks.saturating_mul(st.f_frsize), + }) + } + + fn clone_file(&self, src: &File, dst: &File) -> io::Result<()> { + Ok(rustix::fs::ioctl_ficlone(dst, src)?) + } +} + +/// An exclusive `flock(2)` on a container's lock file, released on drop. +pub(crate) struct ContainerLock { + _file: File, +} + +impl ContainerLock { + /// Wait for the lock of `name` in `dir`, creating the file if needed. + /// + /// # Errors + /// Returns an error when the directory or file cannot be created or the + /// lock call fails. + pub(crate) fn acquire(dir: &Path, name: &str) -> Result { + std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?; + let path = dir.join(format!("salvage-{name}.lock")); + let file = OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW) + .open(&path) + .with_context(|| format!("open {}", path.display()))?; + file.lock() + .with_context(|| format!("lock {}", path.display()))?; + Ok(Self { _file: file }) + } +} diff --git a/nix/host-modules/hive-priv.nix b/nix/host-modules/hive-priv.nix index 54359c45..0774b9c5 100644 --- a/nix/host-modules/hive-priv.nix +++ b/nix/host-modules/hive-priv.nix @@ -146,6 +146,10 @@ in ]; # Writable HOME for nix's caches (see environment.HOME above). StateDirectory = "hive-priv"; + # /run/hive-priv: the per-container salvage lock files. Root-owned and + # outside every path a container can write. + RuntimeDirectory = "hive-priv"; + RuntimeDirectoryMode = "0700"; # With ProtectSystem=strict the root filesystem is read-only inside # hive-priv. When `nixos-container create/update` invokes nix, nix # creates a temporary result symlink in its working directory. Without