diff --git a/Cargo.lock b/Cargo.lock index 8565cce9f..d2363cf51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1990,7 +1990,6 @@ dependencies = [ "anyhow", "hive-priv-sock", "libc", - "rustix", "serde_json", "tokio", "tracing", diff --git a/docs/agent-lifecycle/persistence.md b/docs/agent-lifecycle/persistence.md index 25be4475b..85d6c5634 100644 --- a/docs/agent-lifecycle/persistence.md +++ b/docs/agent-lifecycle/persistence.md @@ -414,30 +414,6 @@ which is the only gcroots path bound into a container, so only root writes it. Destroying the container removes the drop-in and then its root; the build stays until the next collection. A drop-in written by an older hive-priv has no `ExecStartPre=` and no root, and the container starts as before. - -**Disk use.** The copy shares blocks with the old root where it can: hive-priv -clones each file (`FICLONE`), which on btrfs or xfs allocates metadata only, and -copies the data of a file the filesystem refuses to clone. Before it creates -anything, hive-priv sizes the root and compares it with the free space under -`work/`. A salvage whose files clone (tested with one file of the root) must -leave 2 GiB free. One that has to copy data must leave 15 % of the filesystem -free. When the free space is short, hive-priv skips the salvage, deletes -nothing, logs `salvage skipped: need free `, and the -container starts persistent; the next start tries again. hive-priv checks a file -that fails to clone against the 2 GiB reserve again before it copies the data. A -salvage that fails partway removes the directory it created, and leaves the root -and any earlier salvage as they were. After a salvage, hive-priv logs `salvage: reflinked=/ files`. - -**Concurrency.** hive-priv holds an `flock` on -`/run/hive-priv/salvage-.lock` (the unit's `RuntimeDirectory`, which -no container can write) for the whole `WriteNspawnFlags`, so a second request for -the same container waits and then finds the finished template. Across -containers, one salvage copies at a time, so the free space each one checks -excludes any copy still in flight. The request runs on the blocking -thread pool, so a long copy holds no async worker. hive-c0re's per-agent lease -already serialises the requests it sends; the lock covers hive-priv's own -guarantee. - The journal still ships. The agent's collector (`nix/agent-modules/otel.nix`) reads the guest's own `/var/log/journal` and pushes it over OTLP, so no journal link to the host takes part; nspawn skips `--link-journal=try-guest` diff --git a/hive-priv/Cargo.toml b/hive-priv/Cargo.toml index 091485241..a322d0f95 100644 --- a/hive-priv/Cargo.toml +++ b/hive-priv/Cargo.toml @@ -11,7 +11,6 @@ workspace = true anyhow.workspace = true hive-priv-sock.workspace = true libc.workspace = true -rustix.workspace = true serde_json.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/hive-priv/src/container_config.rs b/hive-priv/src/container_config.rs index 197747c70..8d88a26a7 100644 --- a/hive-priv/src/container_config.rs +++ b/hive-priv/src/container_config.rs @@ -61,32 +61,6 @@ pub(crate) fn handle_write_nspawn_flags( Ok((String::new(), String::new())) } -/// [`handle_write_nspawn_flags`] on the blocking pool. The root salvage inside -/// it copies files for as long as the root is big, and would hold a runtime -/// worker for that time. -/// -/// # Errors -/// As [`handle_write_nspawn_flags`], or when the task fails to join. -pub(crate) async fn handle_write_nspawn_flags_blocking( - container: String, - binds: Vec, - isolation: NetworkIsolation, - load_credentials: Vec, - hive_facts: HiveFacts, -) -> Result<(String, String)> { - tokio::task::spawn_blocking(move || { - handle_write_nspawn_flags( - &container, - &binds, - &isolation, - &load_credentials, - &hive_facts, - ) - }) - .await - .context("write nspawn flags task")? -} - /// Validate a system-level container name (`h-` prefix for all agents, /// the manager included) and return the logical agent name after it. /// Anything else is refused, the swarm's forge and matrix containers diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index 48ca9f7b2..2fb357a8a 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -20,9 +20,8 @@ //! dispatch table. Each op family's handlers and validation live in their //! own module: [`container`] (lifecycle), [`container_config`] (nspawn flags, //! systemd drop-ins), [`rootfs`] (the salvage of a container's root before it -//! turns ephemeral, with [`salvage_io`] for its space, reflink and lock calls), -//! [`prestart`] (the check of a container's root that runs before each start, -//! as a sub-command), [`storage`] (btrfs subvolumes, quotas, snapshots, disk +//! turns ephemeral), [`prestart`] (the check of a container's root that runs +//! before each start, as a sub-command), [`storage`] (btrfs subvolumes, quotas, snapshots, disk //! measurement), [`files`] (pause marker, socket dir, legacy tmpfiles), //! [`infra`] (gateway nginx, infra containers) and [`closure`] (downloads from //! the swarm cache). Helpers more than one family uses are in [`shared`]. @@ -34,7 +33,6 @@ mod files; mod infra; mod prestart; mod rootfs; -mod salvage_io; mod shared; mod storage; @@ -385,21 +383,18 @@ async fn exec( PrivRequest::ListContainers => container::container_run(&["list"]).await, PrivRequest::WriteNspawnFlags { + ref container, + ref binds, + ref isolation, + ref load_credentials, + ref hive_facts, + } => container_config::handle_write_nspawn_flags( container, binds, isolation, load_credentials, hive_facts, - } => { - container_config::handle_write_nspawn_flags_blocking( - container, - binds, - isolation, - load_credentials, - hive_facts, - ) - .await - } + ), PrivRequest::EnsureAgentSocketDir { ref name } => files::ensure_agent_socket_dir(name), diff --git a/hive-priv/src/rootfs.rs b/hive-priv/src/rootfs.rs index 25079f945..870c3ef70 100644 --- a/hive-priv/src/rootfs.rs +++ b/hive-priv/src/rootfs.rs @@ -16,32 +16,14 @@ //! //! Only a stopped container is touched: a running one keeps whatever flag its //! root already warrants. -//! -//! Disk use. A file is cloned (`FICLONE`) when the filesystem can, which costs -//! metadata only; otherwise its data is copied. Before creating anything, -//! [`rotate`] compares what the copy needs with the free space under `work/` and -//! returns [`Outcome::Skipped`] when less than [`COPY_FLOOR_PERCENT`] of the -//! filesystem would stay free, or less than [`CLONE_RESERVE`] when a probe shows -//! files clone. A file that fails to clone is checked against the reserve again. -//! -//! Concurrency. One salvage per container runs at a time, under an `flock` in -//! [`LOCK_DIR`]; a second request waits, then sees the finished template. Across -//! containers [`ONE_AT_A_TIME`] admits one copy at a time, so the free space a -//! salvage reads excludes any copy still in flight. -use std::fs::{File, OpenOptions}; -use std::io; -use std::os::unix::fs::{ - MetadataExt as _, OpenOptionsExt as _, PermissionsExt as _, lchown, symlink, -}; +use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _, lchown, symlink}; use std::path::{Path, PathBuf}; -use std::sync::{Mutex, PoisonError}; use anyhow::{Context as _, Result, bail}; use hive_priv_sock::AGENT_STATE_ROOT; use crate::container_config::{container_rootfs, write_container_etc_file_in}; -use crate::salvage_io::{Backend, ContainerLock, LOCK_DIR, ONE_AT_A_TIME, Os}; /// The per-container nix profile bind on the host, which persists across starts. /// What nix keeps for the agent user is carried here; see [`carry_nix_state`]. @@ -72,25 +54,6 @@ const CACHE: [&str; 5] = [ "var/lib/opentelemetry-collector", ]; -/// Share of the filesystem under `work/` that must stay free after a salvage -/// that copies data. btrfs allocates metadata chunks from unallocated space, and -/// every agent and host service writes to the same filesystem, so a full disk -/// stops all of them. -const COPY_FLOOR_PERCENT: u64 = 15; - -/// Free space a salvage must leave when its files clone, which allocate -/// metadata only; also the margin kept when a single file has to be copied. -const CLONE_RESERVE: u64 = 2 << 30; - -/// Metadata a cloned entry is assumed to cost. -const ENTRY_COST: u64 = 8 << 10; - -/// Allocation unit a copied file is rounded up to. -const BLOCK: u64 = 4096; - -/// The file in `work/` that tests whether cloning into it works. -const CLONE_PROBE: &str = ".hive-clone-probe"; - /// What [`rotate`] did. #[derive(Debug, PartialEq, Eq)] pub(crate) enum Outcome { @@ -98,34 +61,7 @@ pub(crate) enum Outcome { Clean, /// The root's content is copied to `salvage`, and the root itself kept at /// `old_root`. - Salvaged { - salvage: PathBuf, - old_root: PathBuf, - stats: Stats, - }, - /// The copy would leave too little free space: `need` bytes must be free, - /// `free` are. Nothing was created or moved. - Skipped { need: u64, free: u64 }, -} - -/// What a salvage copied. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub(crate) struct Stats { - /// Bytes in the regular files. - pub(crate) bytes: u64, - /// Regular files copied. - pub(crate) files: u64, - /// Of those, the files that share blocks with the old root. - pub(crate) cloned: u64, -} - -/// The host a salvage runs against. -pub(crate) struct Env<'a> { - pub(crate) backend: &'a dyn Backend, - /// Where the per-container lock files go. - pub(crate) lock_dir: &'a Path, - /// Taken by the salvage that is copying; see the module docs. - pub(crate) one_at_a_time: &'a Mutex<()>, + Salvaged { salvage: PathBuf, old_root: PathBuf }, } /// Whether `container` may be started with `--ephemeral`, salvaging its root @@ -134,8 +70,7 @@ pub(crate) struct Env<'a> { /// # Errors /// Returns an error when the unit state cannot be read or the salvage fails. /// Nothing is deleted on that path, and the caller starts the container -/// without the flag. A salvage skipped for lack of space is not an error: it -/// returns `false` and the next start tries again. +/// without the flag. pub(crate) fn prepare(container: &str, agent: &str) -> Result { let root = container_rootfs(container); if !unit_is_stopped(container)? { @@ -143,260 +78,47 @@ pub(crate) fn prepare(container: &str, agent: &str) -> Result { } let work = Path::new(AGENT_STATE_ROOT).join(agent).join("work"); let profiles = Path::new(PROFILES_ROOT).join(container); - let env = Env { - backend: &Os, - lock_dir: Path::new(LOCK_DIR), - one_at_a_time: &ONE_AT_A_TIME, - }; - let outcome = rotate(&root, agent, &work, &profiles, &today(), &env)?; - Ok(report(container, &outcome)) -} - -/// Log what [`rotate`] did, and say whether the container may start `--ephemeral`. -fn report(container: &str, outcome: &Outcome) -> bool { - match outcome { + match rotate(&root, agent, &work, &profiles, &today())? { Outcome::Clean => {} - Outcome::Salvaged { - salvage, - old_root, - stats, - } => { - tracing::info!( - "salvage: {container} {} reflinked={}/{} files", - gib(stats.bytes), - stats.cloned, - stats.files - ); - tracing::warn!( - %container, - salvage = %salvage.display(), - old_root = %old_root.display(), - "root salvaged before the first ephemeral start; the old root is kept, remove it by hand" - ); - } - Outcome::Skipped { need, free } => { - tracing::warn!( - "salvage skipped: {container} need {} free {}", - gib(*need), - gib(*free) - ); - return false; - } + Outcome::Salvaged { salvage, old_root } => tracing::warn!( + %container, + salvage = %salvage.display(), + old_root = %old_root.display(), + "root salvaged before the first ephemeral start; the old root is kept, remove it by hand" + ), } - true -} - -/// `bytes` in GiB, to one decimal. -fn gib(bytes: u64) -> String { - const GIB: u64 = 1 << 30; - format!("{}.{} GiB", bytes / GIB, bytes % GIB * 10 / GIB) + Ok(true) } /// Salvage `root` if it holds content, then leave a marked template in its /// place. `work` is the agent's work dir on the host, `profiles` the host side of /// the container's `/nix/var/nix/profiles`; `date` is `YYYYMMDD`. -/// -/// The container's lock is held throughout, so a second call for the same root -/// waits and then finds the template the first one left. -fn rotate( - root: &Path, - agent: &str, - work: &Path, - profiles: &Path, - date: &str, - env: &Env<'_>, -) -> Result { - let name = root - .file_name() - .and_then(|n| n.to_str()) - .with_context(|| format!("{} has no usable name", root.display()))?; - let _lock = ContainerLock::acquire(env.lock_dir, name)?; +fn rotate(root: &Path, agent: &str, work: &Path, profiles: &Path, date: &str) -> Result { let outcome = if has_content(root)? { - let _turn = env - .one_at_a_time - .lock() - .unwrap_or_else(PoisonError::into_inner); - salvage(root, agent, work, profiles, date, env)? + let (uid, gid) = agent_owner(root, agent, work)?; + let machine_id = read_machine_id(root); + let salvage = fresh_dir(work, &format!("salvaged-rootfs-{date}"))?; + lchown(&salvage, Some(uid), Some(gid)) + .with_context(|| format!("chown {}", salvage.display()))?; + copy_tree(root, &salvage, Path::new(""), uid, gid) + .with_context(|| format!("copy {} to {}", root.display(), salvage.display()))?; + carry_nix_state(root, agent, profiles, uid, gid)?; + let old_root = free_sibling(root, &format!("pre-ephemeral-{date}"))?; + std::fs::rename(root, &old_root) + .with_context(|| format!("move {} to {}", root.display(), old_root.display()))?; + std::fs::create_dir(root).with_context(|| format!("create {}", root.display()))?; + if let Some(id) = machine_id { + write_container_etc_file_in(root, "machine-id", &format!("{id}\n"))?; + } + Outcome::Salvaged { salvage, old_root } } else { Outcome::Clean }; - if matches!(outcome, Outcome::Skipped { .. }) { - return Ok(outcome); - } ensure_machine_id(root)?; write_container_etc_file_in(root, MARKER, "")?; Ok(outcome) } -/// The copy-and-move half of [`rotate`], for a root that has content. -fn salvage( - root: &Path, - agent: &str, - work: &Path, - profiles: &Path, - date: &str, - env: &Env<'_>, -) -> Result { - let (uid, gid) = agent_owner(root, agent, work)?; - let machine_id = read_machine_id(root); - if let Some((need, free)) = shortfall(env.backend, root, work)? { - return Ok(Outcome::Skipped { need, free }); - } - let salvage = fresh_dir(work, &format!("salvaged-rootfs-{date}"))?; - let copied = copy_and_move_aside(root, &salvage, agent, profiles, date, env, (uid, gid)); - let (old_root, stats) = copied.inspect_err(|_| discard_partial(&salvage))?; - std::fs::create_dir(root).with_context(|| format!("create {}", root.display()))?; - if let Some(id) = machine_id { - write_container_etc_file_in(root, "machine-id", &format!("{id}\n"))?; - } - Ok(Outcome::Salvaged { - salvage, - old_root, - stats, - }) -} - -/// Copy `root` into the fresh `salvage` dir, then rename `root` aside; returns -/// where it went. Up to the rename the old root is untouched, so a failure -/// anywhere in here leaves it where it was. -fn copy_and_move_aside( - root: &Path, - salvage: &Path, - agent: &str, - profiles: &Path, - date: &str, - env: &Env<'_>, - (uid, gid): (u32, u32), -) -> Result<(PathBuf, Stats)> { - lchown(salvage, Some(uid), Some(gid)) - .with_context(|| format!("chown {}", salvage.display()))?; - let mut copier = Copier::new(env.backend, uid, gid); - copier - .tree(root, salvage, Path::new("")) - .with_context(|| format!("copy {} to {}", root.display(), salvage.display()))?; - carry_nix_state(root, agent, profiles, env.backend, uid, gid)?; - let old_root = free_sibling(root, &format!("pre-ephemeral-{date}"))?; - std::fs::rename(root, &old_root) - .with_context(|| format!("move {} to {}", root.display(), old_root.display()))?; - Ok((old_root, copier.stats)) -} - -/// Remove the directory a failed salvage was filling: the one this attempt -/// created, never an earlier salvage. `remove_dir_all` does not follow links. -fn discard_partial(salvage: &Path) { - if let Err(e) = std::fs::remove_dir_all(salvage) { - tracing::warn!( - salvage = %salvage.display(), - error = %e, - "could not remove the partial salvage; remove it by hand" - ); - } -} - -/// What [`measure`] found under a root. -struct Tally { - /// Everything outside [`CACHE`], directories and links included. - entries: u64, - /// Regular-file bytes, each file rounded up to [`BLOCK`]. - bytes: u64, - /// The first regular file with data, for the clone probe. - sample: Option, -} - -/// Size up what [`Copier::tree`] would copy from `root`. -fn measure(root: &Path) -> Result { - fn walk(dir: &Path, rel: &Path, tally: &mut Tally) -> Result<()> { - for entry in std::fs::read_dir(dir).with_context(|| format!("read {}", dir.display()))? { - let entry = entry.with_context(|| format!("read {}", dir.display()))?; - let rel = rel.join(entry.file_name()); - if is_cache(&rel) { - continue; - } - let meta = entry - .metadata() - .with_context(|| format!("stat {}", entry.path().display()))?; - tally.entries += 1; - if meta.is_dir() { - walk(&entry.path(), &rel, tally)?; - } else if meta.is_file() { - tally.bytes = tally - .bytes - .saturating_add(meta.len().div_ceil(BLOCK) * BLOCK); - if meta.len() > 0 && tally.sample.is_none() { - tally.sample = Some(entry.path()); - } - } - } - Ok(()) - } - let mut tally = Tally { - entries: 0, - bytes: 0, - sample: None, - }; - walk(root, Path::new(""), &mut tally)?; - Ok(tally) -} - -/// `Some((need, free))` when a salvage of `root` into `work` would leave less -/// free space than it must: `need` bytes have to be free, `free` are. -/// -/// Files that clone cost metadata only. Whether they do is probed with one -/// file of the root ([`clones`]); when that fails the whole data is counted. -fn shortfall(backend: &dyn Backend, root: &Path, work: &Path) -> Result> { - let tally = measure(root)?; - let space = backend - .space(work) - .with_context(|| format!("free space of {}", work.display()))?; - let metadata = tally.entries.saturating_mul(ENTRY_COST); - let need = if tally - .sample - .as_deref() - .is_none_or(|sample| clones(backend, sample, work)) - { - metadata.saturating_add(CLONE_RESERVE) - } else { - let floor = space.total / 100 * COPY_FLOOR_PERCENT; - tally.bytes.saturating_add(metadata).saturating_add(floor) - }; - Ok((space.free < need).then_some((need, space.free))) -} - -/// Whether `sample` can be cloned into a file in `work`. The probe file is -/// created exclusively and removed again; if one is already there the answer -/// is no. -fn clones(backend: &dyn Backend, sample: &Path, work: &Path) -> bool { - let probe = work.join(CLONE_PROBE); - let Ok(src) = open_nofollow(sample) else { - return false; - }; - let Ok(dst) = create_new(&probe) else { - return false; - }; - let cloned = backend.clone_file(&src, &dst).is_ok(); - drop(dst); - std::fs::remove_file(&probe).ok(); - cloned -} - -/// `path` opened for reading; a link at `path` is an error. -fn open_nofollow(path: &Path) -> io::Result { - OpenOptions::new() - .read(true) - .custom_flags(libc::O_NOFOLLOW) - .open(path) -} - -/// `path` created for writing, mode 0600; an existing entry is an error. -fn create_new(path: &Path) -> io::Result { - OpenOptions::new() - .write(true) - .create_new(true) - .mode(0o600) - .custom_flags(libc::O_NOFOLLOW) - .open(path) -} - /// Whether anything but directories, [`TEMPLATE_FILES`] and [`CACHE`] is under /// `root`. A missing root has none. fn has_content(root: &Path) -> Result { @@ -476,14 +198,7 @@ fn real_dir(base: &Path, parts: &[&str]) -> Option { /// GC roots that name them are still valid after the switch. A target that /// holds anything is left alone; so is the whole copy when a component of the /// target path is not a real directory, as the container can write `profiles`. -fn carry_nix_state( - root: &Path, - agent: &str, - profiles: &Path, - backend: &dyn Backend, - uid: u32, - gid: u32, -) -> Result<()> { +fn carry_nix_state(root: &Path, agent: &str, profiles: &Path, uid: u32, gid: u32) -> Result<()> { let Some(src) = real_dir(root, &["home", agent, ".local", "state", "nix"]) else { return Ok(()); }; @@ -503,8 +218,7 @@ fn carry_nix_state( { return Ok(()); } - Copier::new(backend, uid, gid) - .tree(&src, &target, Path::new("")) + copy_tree(&src, &target, Path::new(""), uid, gid) .with_context(|| format!("copy {} to {}", src.display(), target.display())) } @@ -573,101 +287,49 @@ fn free_sibling(root: &Path, suffix: &str) -> Result { unreachable!("the loop above returns or continues") } -/// Copies a tree for [`salvage`] and keeps count. -struct Copier<'a> { - backend: &'a dyn Backend, - uid: u32, - gid: u32, - stats: Stats, -} - -impl<'a> Copier<'a> { - fn new(backend: &'a dyn Backend, uid: u32, gid: u32) -> Self { - Self { - backend, - uid, - gid, - stats: Stats::default(), +/// Copy everything under `src` except [`CACHE`] into `dst`, owned by +/// `uid:gid`, with modes and mtimes kept. Sockets, fifos and device nodes are +/// skipped; hard links become separate copies. +fn copy_tree(src: &Path, dst: &Path, rel: &Path, uid: u32, gid: u32) -> Result<()> { + let dir = src.join(rel); + for entry in std::fs::read_dir(&dir).with_context(|| format!("read {}", dir.display()))? { + let entry = entry.with_context(|| format!("read {}", dir.display()))?; + let rel = rel.join(entry.file_name()); + if is_cache(&rel) { + continue; } - } - - /// Copy everything under `src` except [`CACHE`] into `dst`, owned by - /// `uid:gid`, with modes and mtimes kept. Sockets, fifos and device nodes - /// are skipped; hard links become separate copies. - fn tree(&mut self, src: &Path, dst: &Path, rel: &Path) -> Result<()> { - let (uid, gid) = (self.uid, self.gid); - let dir = src.join(rel); - for entry in std::fs::read_dir(&dir).with_context(|| format!("read {}", dir.display()))? { - let entry = entry.with_context(|| format!("read {}", dir.display()))?; - let rel = rel.join(entry.file_name()); - if is_cache(&rel) { - continue; - } - let from = src.join(&rel); - let to = dst.join(&rel); - let meta = std::fs::symlink_metadata(&from) - .with_context(|| format!("stat {}", from.display()))?; - let kind = meta.file_type(); - if kind.is_dir() { - std::fs::create_dir(&to).with_context(|| format!("create {}", to.display()))?; - self.tree(src, dst, &rel)?; - lchown(&to, Some(uid), Some(gid)) - .with_context(|| format!("chown {}", to.display()))?; - set_mode(&to, &meta)?; - } else if kind.is_symlink() { - let target = std::fs::read_link(&from) - .with_context(|| format!("read link {}", from.display()))?; - symlink(&target, &to).with_context(|| format!("link {}", to.display()))?; - lchown(&to, Some(uid), Some(gid)) - .with_context(|| format!("chown {}", to.display()))?; - } else if kind.is_file() { - self.file(&from, &to, &meta)?; - } else { - tracing::info!(path = %from.display(), "salvage skips a socket, fifo or device node"); - } - } - Ok(()) - } - - /// One regular file: cloned when the filesystem allows it, else copied. - fn file(&mut self, from: &Path, to: &Path, meta: &std::fs::Metadata) -> Result<()> { - let mut src = open_nofollow(from).with_context(|| format!("open {}", from.display()))?; - let mut dst = create_new(to).with_context(|| format!("create {}", to.display()))?; - self.stats.files += 1; - self.stats.bytes = self.stats.bytes.saturating_add(meta.len()); - if self.backend.clone_file(&src, &dst).is_ok() { - self.stats.cloned += 1; - } else { - self.check_room(to, meta.len())?; - io::copy(&mut src, &mut dst) + let from = src.join(&rel); + let to = dst.join(&rel); + let meta = + std::fs::symlink_metadata(&from).with_context(|| format!("stat {}", from.display()))?; + let kind = meta.file_type(); + if kind.is_dir() { + std::fs::create_dir(&to).with_context(|| format!("create {}", to.display()))?; + copy_tree(src, dst, &rel, uid, gid)?; + lchown(&to, Some(uid), Some(gid)).with_context(|| format!("chown {}", to.display()))?; + set_mode(&to, &meta)?; + } else if kind.is_symlink() { + let target = std::fs::read_link(&from) + .with_context(|| format!("read link {}", from.display()))?; + symlink(&target, &to).with_context(|| format!("link {}", to.display()))?; + lchown(&to, Some(uid), Some(gid)).with_context(|| format!("chown {}", to.display()))?; + } else if kind.is_file() { + std::fs::copy(&from, &to) .with_context(|| format!("copy {} to {}", from.display(), to.display()))?; + if let Ok(mtime) = meta.modified() { + std::fs::File::options() + .write(true) + .open(&to) + .and_then(|f| f.set_modified(mtime)) + .with_context(|| format!("set mtime of {}", to.display()))?; + } + lchown(&to, Some(uid), Some(gid)).with_context(|| format!("chown {}", to.display()))?; + set_mode(&to, &meta)?; + } else { + tracing::info!(path = %from.display(), "salvage skips a socket, fifo or device node"); } - if let Ok(mtime) = meta.modified() { - dst.set_modified(mtime) - .with_context(|| format!("set mtime of {}", to.display()))?; - } - lchown(to, Some(self.uid), Some(self.gid)) - .with_context(|| format!("chown {}", to.display()))?; - set_mode(to, meta) - } - - /// Fail when copying `len` bytes to `to` would leave less than - /// [`CLONE_RESERVE`] free. - fn check_room(&self, to: &Path, len: u64) -> Result<()> { - let dir = to.parent().unwrap_or(to); - let space = self - .backend - .space(dir) - .with_context(|| format!("free space of {}", dir.display()))?; - if space.free < len.saturating_add(CLONE_RESERVE) { - bail!( - "{} is {len} bytes and the filesystem has {} free; at least {CLONE_RESERVE} must stay", - to.display(), - space.free - ); - } - Ok(()) } + Ok(()) } /// Give `path` the permission bits of `meta`, which `chown` may have cleared. @@ -744,121 +406,14 @@ fn date_of(secs: u64) -> String { #[cfg(test)] mod tests { - use std::fs::File; - use std::io; use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _, lchown, symlink}; use std::path::{Path, PathBuf}; - use std::sync::Mutex; - use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; - use std::time::Duration; - use anyhow::Result; - - use super::{ - CLONE_RESERVE, Env, MARKER, Outcome, Stats, date_of, gib, has_content, report, rotate, - }; - use crate::salvage_io::{Backend, FsSpace, Os}; + use super::{MARKER, Outcome, date_of, has_content, rotate}; use crate::shared::test_support::scratch; const DATE: &str = "20261028"; - /// How [`Fake`] answers a clone request. - #[derive(Clone, Copy)] - enum CloneMode { - /// Succeeds, by copying the bytes. - Copy, - /// Fails with this errno. - Fail(i32), - /// Succeeds for the first request (the probe), fails for the rest. - FirstOnly, - /// Defers to the real `ioctl(FICLONE)`. - Real, - } - - /// A [`Backend`] with the free space and the clone result set by the test. - struct Fake { - free: u64, - total: u64, - mode: CloneMode, - delay: Duration, - space_calls: AtomicU32, - space_fails_from: u32, - calls: AtomicU32, - started: AtomicBool, - active: AtomicU32, - peak: AtomicU32, - } - - impl Fake { - /// Plenty of space; every clone fails with `EXDEV`, so files are copied. - fn new() -> Self { - Self { - free: 1 << 50, - total: 1 << 51, - mode: CloneMode::Fail(libc::EXDEV), - delay: Duration::ZERO, - space_calls: AtomicU32::new(0), - space_fails_from: u32::MAX, - calls: AtomicU32::new(0), - started: AtomicBool::new(false), - active: AtomicU32::new(0), - peak: AtomicU32::new(0), - } - } - - fn space(mut self, free: u64, total: u64) -> Self { - self.free = free; - self.total = total; - self - } - - fn mode(mut self, mode: CloneMode) -> Self { - self.mode = mode; - self - } - - /// Make the `n`th and later free-space reads fail, counting from 0. - fn space_fails_from(mut self, n: u32) -> Self { - self.space_fails_from = n; - self - } - - fn delay(mut self, delay: Duration) -> Self { - self.delay = delay; - self - } - } - - impl Backend for Fake { - fn space(&self, _path: &Path) -> io::Result { - if self.space_calls.fetch_add(1, Ordering::SeqCst) >= self.space_fails_from { - return Err(io::Error::from_raw_os_error(libc::EIO)); - } - Ok(FsSpace { - free: self.free, - total: self.total, - }) - } - - fn clone_file(&self, src: &File, dst: &File) -> io::Result<()> { - let call = self.calls.fetch_add(1, Ordering::SeqCst); - self.started.store(true, Ordering::SeqCst); - let active = self.active.fetch_add(1, Ordering::SeqCst) + 1; - self.peak.fetch_max(active, Ordering::SeqCst); - std::thread::sleep(self.delay); - let copy = || io::copy(&mut &*src, &mut &*dst).map(drop); - let result = match self.mode { - CloneMode::Copy => copy(), - CloneMode::Fail(errno) => Err(io::Error::from_raw_os_error(errno)), - CloneMode::FirstOnly if call == 0 => copy(), - CloneMode::FirstOnly => Err(io::Error::from_raw_os_error(libc::EXDEV)), - CloneMode::Real => Os.clone_file(src, dst), - }; - self.active.fetch_sub(1, Ordering::SeqCst); - result - } - } - /// A stopped agent's old root and work dir under one scratch dir. The /// "agent" is the test's own user, the one it can chown to. struct Fixture { @@ -866,8 +421,6 @@ mod tests { root: PathBuf, work: PathBuf, profiles: PathBuf, - fake: Fake, - serial: Mutex<()>, } impl Fixture { @@ -884,38 +437,17 @@ mod tests { root, work, profiles, - fake: Fake::new(), - serial: Mutex::new(()), } } - fn with(fake: Fake) -> Self { - let mut f = Self::new(); - f.fake = fake; - f - } - fn put(&self, rel: &str, body: &str) { let path = self.root.join(rel); std::fs::create_dir_all(path.parent().unwrap()).unwrap(); std::fs::write(path, body).unwrap(); } - fn try_rotate_in(&self, backend: &dyn Backend, serial: &Mutex<()>) -> Result { - let env = Env { - backend, - lock_dir: &self.top.join("locks"), - one_at_a_time: serial, - }; - rotate(&self.root, "iris", &self.work, &self.profiles, DATE, &env) - } - - fn try_rotate(&self) -> Result { - self.try_rotate_in(&self.fake, &self.serial) - } - fn rotate(&self) -> Outcome { - self.try_rotate().unwrap() + rotate(&self.root, "iris", &self.work, &self.profiles, DATE).unwrap() } /// Where the container finds nix's per-user state, on the host. @@ -951,10 +483,7 @@ mod tests { f.put("home/iris/notes.txt", "keep me"); f.put("etc/hostname", "iris"); - let Outcome::Salvaged { - salvage, old_root, .. - } = f.rotate() - else { + let Outcome::Salvaged { salvage, old_root } = f.rotate() else { panic!("the first call must salvage"); }; assert_eq!(salvage, f.salvage()); @@ -1023,10 +552,7 @@ mod tests { ] { f.put(real, "real"); } - let Outcome::Salvaged { - salvage, old_root, .. - } = f.rotate() - else { + let Outcome::Salvaged { salvage, old_root } = f.rotate() else { panic!("must salvage"); }; for cache in [ @@ -1064,7 +590,10 @@ mod tests { assert!(f.root.join("etc").join(MARKER).is_file()); // The same call with one real file makes the control: it salvages. f.put("home/iris/real", "x"); - assert!(matches!(f.rotate(), Outcome::Salvaged { .. })); + assert!(matches!( + rotate(&f.root, "iris", &f.work, &f.profiles, DATE).unwrap(), + Outcome::Salvaged { .. } + )); } /// Nothing is removed: the old root keeps every file, and a failed @@ -1090,7 +619,7 @@ mod tests { g.put("home/iris/file", "data"); std::fs::remove_dir(&g.work).unwrap(); std::fs::write(&g.work, "not a dir").unwrap(); - assert!(g.try_rotate().is_err()); + assert!(rotate(&g.root, "iris", &g.work, &g.profiles, DATE).is_err()); assert_eq!( std::fs::read_to_string(g.root.join("home/iris/file")).unwrap(), "data" @@ -1137,10 +666,7 @@ mod tests { lchown(f.root.join("home/iris"), None, Some(other)).unwrap(); } - let Outcome::Salvaged { - salvage, old_root, .. - } = f.rotate() - else { + let Outcome::Salvaged { salvage, old_root } = f.rotate() else { panic!("must salvage"); }; let me = std::fs::metadata(old_root.join("home/iris")).unwrap(); @@ -1307,275 +833,6 @@ mod tests { } } - const GIB: u64 = 1 << 30; - - /// A copy that would leave less than 15 % of the filesystem free is skipped - /// and changes nothing; the same root with the space for it is salvaged. - #[test] - fn a_copy_that_leaves_too_little_free_is_skipped() { - // A 100 GiB filesystem keeps 15 GiB free; the root needs a little more. - let f = Fixture::with(Fake::new().space(15 * GIB, 100 * GIB)); - f.put("home/iris/data", "x"); - let skipped = f.rotate(); - let Outcome::Skipped { need, free } = skipped else { - panic!("a copy into 15 GiB free must be skipped, got {skipped:?}"); - }; - assert_eq!(free, 15 * GIB); - assert!(need > 15 * GIB, "need {need}"); - assert!( - f.work.read_dir().unwrap().next().is_none(), - "work/ was touched" - ); - assert!(f.root.join("home/iris/data").is_file()); - assert!(!f.root.join("etc").join(MARKER).exists()); - assert!( - !report("h-iris", &skipped), - "a skipped salvage must start persistent" - ); - - // Control: a little more room, the same root. - let g = Fixture::with(Fake::new().space(15 * GIB + (1 << 20), 100 * GIB)); - g.put("home/iris/data", "x"); - let salvaged = g.rotate(); - assert!(matches!(salvaged, Outcome::Salvaged { .. }), "{salvaged:?}"); - assert!(report("h-iris", &salvaged)); - assert!(report("h-iris", &Outcome::Clean)); - } - - /// Files that clone cost metadata, so the 15 % floor does not apply: the - /// same root and space that skip when files must be copied go through when - /// the probe clones. - #[test] - fn files_that_clone_need_only_metadata() { - let copying = Fixture::with(Fake::new().space(5 * GIB, 100 * GIB)); - copying.put("home/iris/data", "x"); - assert!(matches!(copying.rotate(), Outcome::Skipped { .. })); - - let cloning = Fixture::with(Fake::new().space(5 * GIB, 100 * GIB).mode(CloneMode::Copy)); - cloning.put("home/iris/data", "x"); - let Outcome::Salvaged { stats, .. } = cloning.rotate() else { - panic!("cloning files need no more than the reserve"); - }; - assert_eq!( - stats, - Stats { - bytes: 1, - files: 1, - cloned: 1 - } - ); - - // Not even metadata fits below the reserve. - let full = Fixture::with( - Fake::new() - .space(CLONE_RESERVE - 1, 100 * GIB) - .mode(CloneMode::Copy), - ); - full.put("home/iris/data", "x"); - assert!(matches!(full.rotate(), Outcome::Skipped { .. })); - } - - /// A clone that fails (another filesystem, one without reflinks) leaves the - /// file to be copied, whatever the errno; the control counts a clone. - #[test] - fn a_failed_clone_falls_back_to_a_copy() { - for errno in [libc::EXDEV, libc::EOPNOTSUPP, libc::EINVAL] { - let f = Fixture::with(Fake::new().mode(CloneMode::Fail(errno))); - f.put("home/iris/a", "alpha"); - f.put("home/iris/b", "beta"); - let Outcome::Salvaged { salvage, stats, .. } = f.rotate() else { - panic!("errno {errno}: must salvage"); - }; - assert_eq!( - std::fs::read_to_string(salvage.join("home/iris/a")).unwrap(), - "alpha" - ); - assert_eq!( - std::fs::read_to_string(salvage.join("home/iris/b")).unwrap(), - "beta" - ); - assert_eq!( - stats, - Stats { - bytes: 9, - files: 2, - cloned: 0 - }, - "errno {errno}" - ); - } - let f = Fixture::with(Fake::new().mode(CloneMode::Copy)); - f.put("home/iris/a", "alpha"); - let Outcome::Salvaged { salvage, stats, .. } = f.rotate() else { - panic!("must salvage"); - }; - assert_eq!(stats.cloned, 1); - assert_eq!( - std::fs::read_to_string(salvage.join("home/iris/a")).unwrap(), - "alpha" - ); - } - - /// The real `ioctl(FICLONE)`: whether the scratch filesystem has reflinks or - /// not, the salvage holds the same bytes. - #[test] - fn the_real_clone_call_leaves_a_faithful_copy() { - let f = Fixture::with(Fake::new().mode(CloneMode::Real)); - let body = "0123456789".repeat(10_000); - f.put("home/iris/big", &body); - f.put("home/iris/empty", ""); - let Outcome::Salvaged { salvage, stats, .. } = f.rotate() else { - panic!("must salvage"); - }; - assert_eq!( - std::fs::read_to_string(salvage.join("home/iris/big")).unwrap(), - body - ); - assert_eq!(stats.files, 2); - assert!(stats.cloned <= stats.files); - } - - /// A file that cannot clone, in a salvage whose probe did, is copied only - /// while the reserve stays free. - #[test] - fn a_file_that_must_be_copied_keeps_the_reserve_free() { - let body = "x".repeat(1 << 20); - let tight = Fixture::with( - Fake::new() - .space(CLONE_RESERVE + (512 << 10), 100 * GIB) - .mode(CloneMode::FirstOnly), - ); - tight.put("home/iris/big", &body); - let err = tight.try_rotate().unwrap_err(); - assert!(format!("{err:#}").contains("must stay"), "{err:#}"); - assert!( - tight.root.join("home/iris/big").is_file(), - "root was touched" - ); - assert!(!tight.root.join("etc").join(MARKER).exists()); - - let roomy = Fixture::with( - Fake::new() - .space(CLONE_RESERVE + (8 << 20), 100 * GIB) - .mode(CloneMode::FirstOnly), - ); - roomy.put("home/iris/big", &body); - let Outcome::Salvaged { stats, .. } = roomy.rotate() else { - panic!("1 MiB fits above the reserve"); - }; - assert_eq!(stats.cloned, 0); - } - - /// A salvage that fails after it created its directory removes that - /// directory and nothing else: an earlier salvage stays, the old root is - /// as it was, and no marker is written. Once without room for a copy, once - /// with a free-space read that fails. - #[test] - fn a_failed_salvage_removes_its_partial_directory() { - let body = "x".repeat(1 << 20); - let cases = [ - ( - "no room for a file", - Fake::new() - .space(CLONE_RESERVE + (512 << 10), 100 * GIB) - .mode(CloneMode::FirstOnly), - ), - ("free space unreadable", Fake::new().space_fails_from(1)), - ]; - for (what, fake) in cases { - let f = Fixture::with(fake); - f.put("home/iris/big", &body); - f.put("home/iris/small", "s"); - // An earlier salvage of the same day, which must survive. - let earlier = f.salvage(); - std::fs::create_dir_all(earlier.join("home")).unwrap(); - std::fs::write(earlier.join("home/old"), "kept").unwrap(); - - assert!(f.try_rotate().is_err(), "{what}: must fail"); - assert_eq!( - Fixture::names(&f.work), - [format!("salvaged-rootfs-{DATE}")], - "{what}: a partial directory is left" - ); - assert_eq!( - std::fs::read_to_string(earlier.join("home/old")).unwrap(), - "kept" - ); - assert!( - f.root.join("home/iris/big").is_file(), - "{what}: root was touched" - ); - assert!(f.root.join("home/iris/small").is_file(), "{what}"); - assert!(!f.root.join("etc").join(MARKER).exists(), "{what}"); - assert!( - Fixture::names(&f.top) - .iter() - .all(|n| !n.contains("pre-ephemeral")), - "{what}: the root was moved" - ); - } - - // Control: the same root with room and a working space read succeeds. - let ok = Fixture::with(Fake::new()); - ok.put("home/iris/big", &body); - assert!(matches!(ok.rotate(), Outcome::Salvaged { .. })); - assert!(ok.salvage().join("home/iris/big").is_file()); - } - - /// A second request for a root that is being salvaged waits, then finds - /// the finished template: one salvage directory, one moved-aside root. - #[test] - fn two_requests_for_one_root_salvage_it_once() { - let f = Fixture::with( - Fake::new() - .mode(CloneMode::Copy) - .delay(Duration::from_millis(200)), - ); - f.put("home/iris/a", "1"); - let (first, second) = std::thread::scope(|s| { - let first = s.spawn(|| f.rotate()); - while !f.fake.started.load(Ordering::SeqCst) { - std::thread::sleep(Duration::from_millis(5)); - } - let second = s.spawn(|| f.rotate()); - (first.join().unwrap(), second.join().unwrap()) - }); - assert!(matches!(first, Outcome::Salvaged { .. }), "{first:?}"); - assert_eq!(second, Outcome::Clean); - assert_eq!(Fixture::names(&f.work), [format!("salvaged-rootfs-{DATE}")]); - let moved: Vec = Fixture::names(&f.top) - .into_iter() - .filter(|n| n.starts_with("h-iris.pre-ephemeral-")) - .collect(); - assert_eq!(moved.len(), 1, "{moved:?}"); - } - - /// Salvages of different containers copy one after the other. - #[test] - fn salvages_of_different_containers_copy_one_at_a_time() { - let a = Fixture::new(); - let b = Fixture::new(); - a.put("home/iris/a", "1"); - b.put("home/iris/b", "2"); - let fake = Fake::new() - .mode(CloneMode::Copy) - .delay(Duration::from_millis(100)); - let serial = Mutex::new(()); - std::thread::scope(|s| { - s.spawn(|| a.try_rotate_in(&fake, &serial).unwrap()); - s.spawn(|| b.try_rotate_in(&fake, &serial).unwrap()); - }); - assert!(fake.calls.load(Ordering::SeqCst) >= 4, "both salvages ran"); - assert_eq!(fake.peak.load(Ordering::SeqCst), 1); - } - - #[test] - fn sizes_are_shown_in_gib() { - assert_eq!(gib(0), "0.0 GiB"); - assert_eq!(gib(3 * GIB / 2), "1.5 GiB"); - assert_eq!(gib(87 * GIB), "87.0 GiB"); - } - #[test] fn dates() { assert_eq!(date_of(0), "19700101"); diff --git a/hive-priv/src/salvage_io.rs b/hive-priv/src/salvage_io.rs deleted file mode 100644 index c11ca0a17..000000000 --- a/hive-priv/src/salvage_io.rs +++ /dev/null @@ -1,84 +0,0 @@ -//! What the root salvage ([`crate::rootfs`]) asks of the host: free space, -//! reflinks, and the locks that keep two salvages apart. -//! -//! The space and reflink calls sit behind [`Backend`] so a test can set the -//! free space and make a clone fail. - -use std::fs::{File, OpenOptions}; -use std::io; -use std::os::unix::fs::OpenOptionsExt as _; -use std::path::Path; -use std::sync::Mutex; - -use anyhow::{Context as _, Result}; - -/// Where the per-container lock files live. A root-owned runtime directory -/// (`RuntimeDirectory=` of `hive-priv.service`), so no container can write it. -pub(crate) const LOCK_DIR: &str = "/run/hive-priv"; - -/// Held by the one salvage that is copying data at any moment. -pub(crate) static ONE_AT_A_TIME: Mutex<()> = Mutex::new(()); - -/// Size and free space of a filesystem, in bytes. `free` is what an -/// unprivileged writer could still use. -#[derive(Debug, Clone, Copy)] -pub(crate) struct FsSpace { - pub(crate) free: u64, - pub(crate) total: u64, -} - -/// The host calls the salvage makes. -pub(crate) trait Backend { - /// Space on the filesystem holding `path`. - fn space(&self, path: &Path) -> io::Result; - - /// Make `dst`, an empty file, share `src`'s data blocks. An error leaves - /// `dst` as it was: the filesystem cannot clone, or `src` and `dst` are on - /// different ones. - fn clone_file(&self, src: &File, dst: &File) -> io::Result<()>; -} - -/// The real filesystem: `statvfs(2)` and `ioctl(FICLONE)`. -pub(crate) struct Os; - -impl Backend for Os { - fn space(&self, path: &Path) -> io::Result { - let st = rustix::fs::statvfs(path)?; - Ok(FsSpace { - free: st.f_bavail.saturating_mul(st.f_frsize), - total: st.f_blocks.saturating_mul(st.f_frsize), - }) - } - - fn clone_file(&self, src: &File, dst: &File) -> io::Result<()> { - Ok(rustix::fs::ioctl_ficlone(dst, src)?) - } -} - -/// An exclusive `flock(2)` on a container's lock file, released on drop. -pub(crate) struct ContainerLock { - _file: File, -} - -impl ContainerLock { - /// Wait for the lock of `name` in `dir`, creating the file if needed. - /// - /// # Errors - /// Returns an error when the directory or file cannot be created or the - /// lock call fails. - pub(crate) fn acquire(dir: &Path, name: &str) -> Result { - std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?; - let path = dir.join(format!("salvage-{name}.lock")); - let file = OpenOptions::new() - .write(true) - .create(true) - .truncate(false) - .mode(0o600) - .custom_flags(libc::O_NOFOLLOW) - .open(&path) - .with_context(|| format!("open {}", path.display()))?; - file.lock() - .with_context(|| format!("lock {}", path.display()))?; - Ok(Self { _file: file }) - } -} diff --git a/nix/host-modules/hive-priv.nix b/nix/host-modules/hive-priv.nix index 0774b9c5c..54359c452 100644 --- a/nix/host-modules/hive-priv.nix +++ b/nix/host-modules/hive-priv.nix @@ -146,10 +146,6 @@ in ]; # Writable HOME for nix's caches (see environment.HOME above). StateDirectory = "hive-priv"; - # /run/hive-priv: the per-container salvage lock files. Root-owned and - # outside every path a container can write. - RuntimeDirectory = "hive-priv"; - RuntimeDirectoryMode = "0700"; # With ProtectSystem=strict the root filesystem is read-only inside # hive-priv. When `nixos-container create/update` invokes nix, nix # creates a temporary result symlink in its working directory. Without diff --git a/nix/vm-tests/agent-ephemeral-rootfs.nix b/nix/vm-tests/agent-ephemeral-rootfs.nix index 74a651aa3..3088a7065 100644 --- a/nix/vm-tests/agent-ephemeral-rootfs.nix +++ b/nix/vm-tests/agent-ephemeral-rootfs.nix @@ -78,9 +78,6 @@ pkgs.testers.runNixOSTest { pkgs.hello pkgs.tree ]; - # The salvage skips when a copy would leave under 15% of the disk free, and - # this disk cannot clone files; the default 1 GiB leaves too little room. - virtualisation.diskSize = 4096; # The agent bridge hive-priv names in the container's conf. networking.bridges.${bridge}.interfaces = [ ]; networking.interfaces.${bridge}.ipv4.addresses = [