86e80250bb
- a failure between creating the dir and the rename removes that dir only - give the ephemeral-rootfs vm test a 4 GiB disk so the space check passes
1585 lines
57 KiB
Rust
1585 lines
57 KiB
Rust
//! The agent container's non-persistent root.
|
|
//!
|
|
//! Agent containers start with `systemd-nspawn --ephemeral`: nspawn runs the
|
|
//! container on a throwaway snapshot of `/var/lib/nixos-containers/<c>`, so
|
|
//! anything written outside the binds is gone at stop. [`prepare`] runs ahead
|
|
//! of every `WriteNspawnFlags` and decides whether the flag may be passed.
|
|
//!
|
|
//! A root that holds anything beyond what the start path itself writes is
|
|
//! first salvaged. Everything outside [`CACHE`] is copied into
|
|
//! `<agent>/work/salvaged-rootfs-<YYYYMMDD>/`, owned by the agent, the old
|
|
//! root is renamed to `<root>.pre-ephemeral-<YYYYMMDD>` (never deleted), and a
|
|
//! fresh template root takes its place. The template is what nspawn snapshots
|
|
//! on every start, so it must stay small and free of the previous contents.
|
|
//! The template carries [`MARKER`]; it is written last, so a crash anywhere
|
|
//! earlier leaves the old root in place and the next call tries again.
|
|
//!
|
|
//! Only a stopped container is touched: a running one keeps whatever flag its
|
|
//! root already warrants.
|
|
//!
|
|
//! Disk use. A file is cloned (`FICLONE`) when the filesystem can, which costs
|
|
//! metadata only; otherwise its data is copied. Before creating anything,
|
|
//! [`rotate`] compares what the copy needs with the free space under `work/` and
|
|
//! returns [`Outcome::Skipped`] when less than [`COPY_FLOOR_PERCENT`] of the
|
|
//! filesystem would stay free, or less than [`CLONE_RESERVE`] when a probe shows
|
|
//! files clone. A file that fails to clone is checked against the reserve again.
|
|
//!
|
|
//! Concurrency. One salvage per container runs at a time, under an `flock` in
|
|
//! [`LOCK_DIR`]; a second request waits, then sees the finished template. Across
|
|
//! containers [`ONE_AT_A_TIME`] admits one copy at a time, so the free space a
|
|
//! salvage reads excludes any copy still in flight.
|
|
|
|
use std::fs::{File, OpenOptions};
|
|
use std::io;
|
|
use std::os::unix::fs::{
|
|
MetadataExt as _, OpenOptionsExt as _, PermissionsExt as _, lchown, symlink,
|
|
};
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::{Mutex, PoisonError};
|
|
|
|
use anyhow::{Context as _, Result, bail};
|
|
use hive_priv_sock::AGENT_STATE_ROOT;
|
|
|
|
use crate::container_config::{container_rootfs, write_container_etc_file_in};
|
|
use crate::salvage_io::{Backend, ContainerLock, LOCK_DIR, ONE_AT_A_TIME, Os};
|
|
|
|
/// The per-container nix profile bind on the host, which persists across starts.
|
|
/// What nix keeps for the agent user is carried here; see [`carry_nix_state`].
|
|
const PROFILES_ROOT: &str = "/nix/var/nix/profiles/per-container";
|
|
|
|
/// Written into a template root once it is ready to be snapshotted.
|
|
const MARKER: &str = "hyperhive-ephemeral-root";
|
|
|
|
/// Files in `etc/` that the start path writes into every root, the template
|
|
/// included. They are not content: the root counts as clean with only these.
|
|
const TEMPLATE_FILES: [&str; 6] = [
|
|
MARKER,
|
|
"hyperhive-bridge-dns",
|
|
"hyperhive-hive-facts",
|
|
"machine-id",
|
|
"os-release",
|
|
"resolv.conf",
|
|
];
|
|
|
|
/// Paths relative to the root that the salvage skips: the guest journal, its
|
|
/// forwarder's read cursor, weston's state and scratch space. The first start
|
|
/// of a container rebuilds all of them.
|
|
const CACHE: [&str; 5] = [
|
|
"tmp",
|
|
"var/cache",
|
|
"var/log/journal",
|
|
"var/lib/weston",
|
|
"var/lib/opentelemetry-collector",
|
|
];
|
|
|
|
/// Share of the filesystem under `work/` that must stay free after a salvage
|
|
/// that copies data. btrfs allocates metadata chunks from unallocated space, and
|
|
/// every agent and host service writes to the same filesystem, so a full disk
|
|
/// stops all of them.
|
|
const COPY_FLOOR_PERCENT: u64 = 15;
|
|
|
|
/// Free space a salvage must leave when its files clone, which allocate
|
|
/// metadata only; also the margin kept when a single file has to be copied.
|
|
const CLONE_RESERVE: u64 = 2 << 30;
|
|
|
|
/// Metadata a cloned entry is assumed to cost.
|
|
const ENTRY_COST: u64 = 8 << 10;
|
|
|
|
/// Allocation unit a copied file is rounded up to.
|
|
const BLOCK: u64 = 4096;
|
|
|
|
/// The file in `work/` that tests whether cloning into it works.
|
|
const CLONE_PROBE: &str = ".hive-clone-probe";
|
|
|
|
/// What [`rotate`] did.
|
|
#[derive(Debug, PartialEq, Eq)]
|
|
pub(crate) enum Outcome {
|
|
/// Nothing outside [`CACHE`] was in the root; nothing was moved.
|
|
Clean,
|
|
/// The root's content is copied to `salvage`, and the root itself kept at
|
|
/// `old_root`.
|
|
Salvaged {
|
|
salvage: PathBuf,
|
|
old_root: PathBuf,
|
|
stats: Stats,
|
|
},
|
|
/// The copy would leave too little free space: `need` bytes must be free,
|
|
/// `free` are. Nothing was created or moved.
|
|
Skipped { need: u64, free: u64 },
|
|
}
|
|
|
|
/// What a salvage copied.
|
|
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
|
pub(crate) struct Stats {
|
|
/// Bytes in the regular files.
|
|
pub(crate) bytes: u64,
|
|
/// Regular files copied.
|
|
pub(crate) files: u64,
|
|
/// Of those, the files that share blocks with the old root.
|
|
pub(crate) cloned: u64,
|
|
}
|
|
|
|
/// The host a salvage runs against.
|
|
pub(crate) struct Env<'a> {
|
|
pub(crate) backend: &'a dyn Backend,
|
|
/// Where the per-container lock files go.
|
|
pub(crate) lock_dir: &'a Path,
|
|
/// Taken by the salvage that is copying; see the module docs.
|
|
pub(crate) one_at_a_time: &'a Mutex<()>,
|
|
}
|
|
|
|
/// Whether `container` may be started with `--ephemeral`, salvaging its root
|
|
/// first when it holds content and the container is stopped.
|
|
///
|
|
/// # Errors
|
|
/// Returns an error when the unit state cannot be read or the salvage fails.
|
|
/// Nothing is deleted on that path, and the caller starts the container
|
|
/// without the flag. A salvage skipped for lack of space is not an error: it
|
|
/// returns `false` and the next start tries again.
|
|
pub(crate) fn prepare(container: &str, agent: &str) -> Result<bool> {
|
|
let root = container_rootfs(container);
|
|
if !unit_is_stopped(container)? {
|
|
return Ok(is_regular(&root.join("etc").join(MARKER)));
|
|
}
|
|
let work = Path::new(AGENT_STATE_ROOT).join(agent).join("work");
|
|
let profiles = Path::new(PROFILES_ROOT).join(container);
|
|
let env = Env {
|
|
backend: &Os,
|
|
lock_dir: Path::new(LOCK_DIR),
|
|
one_at_a_time: &ONE_AT_A_TIME,
|
|
};
|
|
let outcome = rotate(&root, agent, &work, &profiles, &today(), &env)?;
|
|
Ok(report(container, &outcome))
|
|
}
|
|
|
|
/// Log what [`rotate`] did, and say whether the container may start `--ephemeral`.
|
|
fn report(container: &str, outcome: &Outcome) -> bool {
|
|
match outcome {
|
|
Outcome::Clean => {}
|
|
Outcome::Salvaged {
|
|
salvage,
|
|
old_root,
|
|
stats,
|
|
} => {
|
|
tracing::info!(
|
|
"salvage: {container} {} reflinked={}/{} files",
|
|
gib(stats.bytes),
|
|
stats.cloned,
|
|
stats.files
|
|
);
|
|
tracing::warn!(
|
|
%container,
|
|
salvage = %salvage.display(),
|
|
old_root = %old_root.display(),
|
|
"root salvaged before the first ephemeral start; the old root is kept, remove it by hand"
|
|
);
|
|
}
|
|
Outcome::Skipped { need, free } => {
|
|
tracing::warn!(
|
|
"salvage skipped: {container} need {} free {}",
|
|
gib(*need),
|
|
gib(*free)
|
|
);
|
|
return false;
|
|
}
|
|
}
|
|
true
|
|
}
|
|
|
|
/// `bytes` in GiB, to one decimal.
|
|
fn gib(bytes: u64) -> String {
|
|
const GIB: u64 = 1 << 30;
|
|
format!("{}.{} GiB", bytes / GIB, bytes % GIB * 10 / GIB)
|
|
}
|
|
|
|
/// Salvage `root` if it holds content, then leave a marked template in its
|
|
/// place. `work` is the agent's work dir on the host, `profiles` the host side of
|
|
/// the container's `/nix/var/nix/profiles`; `date` is `YYYYMMDD`.
|
|
///
|
|
/// The container's lock is held throughout, so a second call for the same root
|
|
/// waits and then finds the template the first one left.
|
|
fn rotate(
|
|
root: &Path,
|
|
agent: &str,
|
|
work: &Path,
|
|
profiles: &Path,
|
|
date: &str,
|
|
env: &Env<'_>,
|
|
) -> Result<Outcome> {
|
|
let name = root
|
|
.file_name()
|
|
.and_then(|n| n.to_str())
|
|
.with_context(|| format!("{} has no usable name", root.display()))?;
|
|
let _lock = ContainerLock::acquire(env.lock_dir, name)?;
|
|
let outcome = if has_content(root)? {
|
|
let _turn = env
|
|
.one_at_a_time
|
|
.lock()
|
|
.unwrap_or_else(PoisonError::into_inner);
|
|
salvage(root, agent, work, profiles, date, env)?
|
|
} else {
|
|
Outcome::Clean
|
|
};
|
|
if matches!(outcome, Outcome::Skipped { .. }) {
|
|
return Ok(outcome);
|
|
}
|
|
ensure_machine_id(root)?;
|
|
write_container_etc_file_in(root, MARKER, "")?;
|
|
Ok(outcome)
|
|
}
|
|
|
|
/// The copy-and-move half of [`rotate`], for a root that has content.
|
|
fn salvage(
|
|
root: &Path,
|
|
agent: &str,
|
|
work: &Path,
|
|
profiles: &Path,
|
|
date: &str,
|
|
env: &Env<'_>,
|
|
) -> Result<Outcome> {
|
|
let (uid, gid) = agent_owner(root, agent, work)?;
|
|
let machine_id = read_machine_id(root);
|
|
if let Some((need, free)) = shortfall(env.backend, root, work)? {
|
|
return Ok(Outcome::Skipped { need, free });
|
|
}
|
|
let salvage = fresh_dir(work, &format!("salvaged-rootfs-{date}"))?;
|
|
let copied = copy_and_move_aside(root, &salvage, agent, profiles, date, env, (uid, gid));
|
|
let (old_root, stats) = copied.inspect_err(|_| discard_partial(&salvage))?;
|
|
std::fs::create_dir(root).with_context(|| format!("create {}", root.display()))?;
|
|
if let Some(id) = machine_id {
|
|
write_container_etc_file_in(root, "machine-id", &format!("{id}\n"))?;
|
|
}
|
|
Ok(Outcome::Salvaged {
|
|
salvage,
|
|
old_root,
|
|
stats,
|
|
})
|
|
}
|
|
|
|
/// Copy `root` into the fresh `salvage` dir, then rename `root` aside; returns
|
|
/// where it went. Up to the rename the old root is untouched, so a failure
|
|
/// anywhere in here leaves it where it was.
|
|
fn copy_and_move_aside(
|
|
root: &Path,
|
|
salvage: &Path,
|
|
agent: &str,
|
|
profiles: &Path,
|
|
date: &str,
|
|
env: &Env<'_>,
|
|
(uid, gid): (u32, u32),
|
|
) -> Result<(PathBuf, Stats)> {
|
|
lchown(salvage, Some(uid), Some(gid))
|
|
.with_context(|| format!("chown {}", salvage.display()))?;
|
|
let mut copier = Copier::new(env.backend, uid, gid);
|
|
copier
|
|
.tree(root, salvage, Path::new(""))
|
|
.with_context(|| format!("copy {} to {}", root.display(), salvage.display()))?;
|
|
carry_nix_state(root, agent, profiles, env.backend, uid, gid)?;
|
|
let old_root = free_sibling(root, &format!("pre-ephemeral-{date}"))?;
|
|
std::fs::rename(root, &old_root)
|
|
.with_context(|| format!("move {} to {}", root.display(), old_root.display()))?;
|
|
Ok((old_root, copier.stats))
|
|
}
|
|
|
|
/// Remove the directory a failed salvage was filling: the one this attempt
|
|
/// created, never an earlier salvage. `remove_dir_all` does not follow links.
|
|
fn discard_partial(salvage: &Path) {
|
|
if let Err(e) = std::fs::remove_dir_all(salvage) {
|
|
tracing::warn!(
|
|
salvage = %salvage.display(),
|
|
error = %e,
|
|
"could not remove the partial salvage; remove it by hand"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// What [`measure`] found under a root.
|
|
struct Tally {
|
|
/// Everything outside [`CACHE`], directories and links included.
|
|
entries: u64,
|
|
/// Regular-file bytes, each file rounded up to [`BLOCK`].
|
|
bytes: u64,
|
|
/// The first regular file with data, for the clone probe.
|
|
sample: Option<PathBuf>,
|
|
}
|
|
|
|
/// Size up what [`Copier::tree`] would copy from `root`.
|
|
fn measure(root: &Path) -> Result<Tally> {
|
|
fn walk(dir: &Path, rel: &Path, tally: &mut Tally) -> Result<()> {
|
|
for entry in std::fs::read_dir(dir).with_context(|| format!("read {}", dir.display()))? {
|
|
let entry = entry.with_context(|| format!("read {}", dir.display()))?;
|
|
let rel = rel.join(entry.file_name());
|
|
if is_cache(&rel) {
|
|
continue;
|
|
}
|
|
let meta = entry
|
|
.metadata()
|
|
.with_context(|| format!("stat {}", entry.path().display()))?;
|
|
tally.entries += 1;
|
|
if meta.is_dir() {
|
|
walk(&entry.path(), &rel, tally)?;
|
|
} else if meta.is_file() {
|
|
tally.bytes = tally
|
|
.bytes
|
|
.saturating_add(meta.len().div_ceil(BLOCK) * BLOCK);
|
|
if meta.len() > 0 && tally.sample.is_none() {
|
|
tally.sample = Some(entry.path());
|
|
}
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
let mut tally = Tally {
|
|
entries: 0,
|
|
bytes: 0,
|
|
sample: None,
|
|
};
|
|
walk(root, Path::new(""), &mut tally)?;
|
|
Ok(tally)
|
|
}
|
|
|
|
/// `Some((need, free))` when a salvage of `root` into `work` would leave less
|
|
/// free space than it must: `need` bytes have to be free, `free` are.
|
|
///
|
|
/// Files that clone cost metadata only. Whether they do is probed with one
|
|
/// file of the root ([`clones`]); when that fails the whole data is counted.
|
|
fn shortfall(backend: &dyn Backend, root: &Path, work: &Path) -> Result<Option<(u64, u64)>> {
|
|
let tally = measure(root)?;
|
|
let space = backend
|
|
.space(work)
|
|
.with_context(|| format!("free space of {}", work.display()))?;
|
|
let metadata = tally.entries.saturating_mul(ENTRY_COST);
|
|
let need = if tally
|
|
.sample
|
|
.as_deref()
|
|
.is_none_or(|sample| clones(backend, sample, work))
|
|
{
|
|
metadata.saturating_add(CLONE_RESERVE)
|
|
} else {
|
|
let floor = space.total / 100 * COPY_FLOOR_PERCENT;
|
|
tally.bytes.saturating_add(metadata).saturating_add(floor)
|
|
};
|
|
Ok((space.free < need).then_some((need, space.free)))
|
|
}
|
|
|
|
/// Whether `sample` can be cloned into a file in `work`. The probe file is
|
|
/// created exclusively and removed again; if one is already there the answer
|
|
/// is no.
|
|
fn clones(backend: &dyn Backend, sample: &Path, work: &Path) -> bool {
|
|
let probe = work.join(CLONE_PROBE);
|
|
let Ok(src) = open_nofollow(sample) else {
|
|
return false;
|
|
};
|
|
let Ok(dst) = create_new(&probe) else {
|
|
return false;
|
|
};
|
|
let cloned = backend.clone_file(&src, &dst).is_ok();
|
|
drop(dst);
|
|
std::fs::remove_file(&probe).ok();
|
|
cloned
|
|
}
|
|
|
|
/// `path` opened for reading; a link at `path` is an error.
|
|
fn open_nofollow(path: &Path) -> io::Result<File> {
|
|
OpenOptions::new()
|
|
.read(true)
|
|
.custom_flags(libc::O_NOFOLLOW)
|
|
.open(path)
|
|
}
|
|
|
|
/// `path` created for writing, mode 0600; an existing entry is an error.
|
|
fn create_new(path: &Path) -> io::Result<File> {
|
|
OpenOptions::new()
|
|
.write(true)
|
|
.create_new(true)
|
|
.mode(0o600)
|
|
.custom_flags(libc::O_NOFOLLOW)
|
|
.open(path)
|
|
}
|
|
|
|
/// Whether anything but directories, [`TEMPLATE_FILES`] and [`CACHE`] is under
|
|
/// `root`. A missing root has none.
|
|
fn has_content(root: &Path) -> Result<bool> {
|
|
fn walk(dir: &Path, rel: &Path) -> Result<bool> {
|
|
let entries = match std::fs::read_dir(dir) {
|
|
Ok(entries) => entries,
|
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false),
|
|
Err(e) => return Err(e).with_context(|| format!("read {}", dir.display())),
|
|
};
|
|
for entry in entries {
|
|
let entry = entry.with_context(|| format!("read {}", dir.display()))?;
|
|
let rel = rel.join(entry.file_name());
|
|
if is_cache(&rel) {
|
|
continue;
|
|
}
|
|
let kind = entry
|
|
.file_type()
|
|
.with_context(|| format!("stat {}", entry.path().display()))?;
|
|
let is_template = TEMPLATE_FILES
|
|
.iter()
|
|
.any(|name| rel == Path::new("etc").join(name));
|
|
if kind.is_dir() {
|
|
if walk(&entry.path(), &rel)? {
|
|
return Ok(true);
|
|
}
|
|
} else if !is_template {
|
|
return Ok(true);
|
|
}
|
|
}
|
|
Ok(false)
|
|
}
|
|
walk(root, Path::new(""))
|
|
}
|
|
|
|
fn is_cache(rel: &Path) -> bool {
|
|
CACHE.iter().any(|c| rel == Path::new(c))
|
|
}
|
|
|
|
/// The uid and gid the salvage is handed to: the owner of the agent's home in
|
|
/// the old root, else of its work dir. Never root.
|
|
fn agent_owner(root: &Path, agent: &str, work: &Path) -> Result<(u32, u32)> {
|
|
let home = real_dir(root, &["home", agent]);
|
|
for dir in [home, real_dir(work, &[])].into_iter().flatten() {
|
|
if let Ok(meta) = std::fs::symlink_metadata(&dir)
|
|
&& meta.uid() != 0
|
|
{
|
|
return Ok((meta.uid(), meta.gid()));
|
|
}
|
|
}
|
|
bail!(
|
|
"agent {agent}: neither its home in {} nor {} has a non-root owner",
|
|
root.display(),
|
|
work.display()
|
|
)
|
|
}
|
|
|
|
/// `base` joined with `parts`, if every component is a real directory. The
|
|
/// container could have made any of them a link, and this runs as root.
|
|
fn real_dir(base: &Path, parts: &[&str]) -> Option<PathBuf> {
|
|
let mut path = base.to_path_buf();
|
|
if !std::fs::symlink_metadata(&path).ok()?.is_dir() {
|
|
return None;
|
|
}
|
|
for part in parts {
|
|
path.push(part);
|
|
if !std::fs::symlink_metadata(&path).ok()?.is_dir() {
|
|
return None;
|
|
}
|
|
}
|
|
Some(path)
|
|
}
|
|
|
|
/// nix keeps a non-root user's profile under `~/.local/state/nix`, which the
|
|
/// agent's tmpfiles rules (`nix/agent-modules/user.nix`) point at
|
|
/// `per-user/<agent>` in `profiles`. A root that was lived in has the real
|
|
/// directory there: copy it across, so the agent's installed packages and the
|
|
/// GC roots that name them are still valid after the switch. A target that
|
|
/// holds anything is left alone; so is the whole copy when a component of the
|
|
/// target path is not a real directory, as the container can write `profiles`.
|
|
fn carry_nix_state(
|
|
root: &Path,
|
|
agent: &str,
|
|
profiles: &Path,
|
|
backend: &dyn Backend,
|
|
uid: u32,
|
|
gid: u32,
|
|
) -> Result<()> {
|
|
let Some(src) = real_dir(root, &["home", agent, ".local", "state", "nix"]) else {
|
|
return Ok(());
|
|
};
|
|
let Some(target) = own_dir(profiles, &["per-user", agent], uid, gid)? else {
|
|
tracing::warn!(
|
|
profiles = %profiles.display(),
|
|
"a link or file where the nix state goes; the agent's nix state is in the salvage only"
|
|
);
|
|
return Ok(());
|
|
};
|
|
// The tmpfiles rules create `target` empty on a boot of the new module;
|
|
// anything in it is the agent's newer state.
|
|
if std::fs::read_dir(&target)
|
|
.with_context(|| format!("read {}", target.display()))?
|
|
.next()
|
|
.is_some()
|
|
{
|
|
return Ok(());
|
|
}
|
|
Copier::new(backend, uid, gid)
|
|
.tree(&src, &target, Path::new(""))
|
|
.with_context(|| format!("copy {} to {}", src.display(), target.display()))
|
|
}
|
|
|
|
/// `base` joined with `parts`, creating each missing component (the last one
|
|
/// owned by `uid:gid`), or `None` when `base` or an existing component is not a
|
|
/// real directory. The container could have replaced any of them by a link, and
|
|
/// this runs as root.
|
|
fn own_dir(base: &Path, parts: &[&str], uid: u32, gid: u32) -> Result<Option<PathBuf>> {
|
|
let mut path = base.to_path_buf();
|
|
if !std::fs::symlink_metadata(&path).is_ok_and(|m| m.is_dir()) {
|
|
return Ok(None);
|
|
}
|
|
for (i, part) in parts.iter().enumerate() {
|
|
path.push(part);
|
|
match std::fs::symlink_metadata(&path) {
|
|
Ok(meta) if meta.is_dir() => {}
|
|
Ok(_) => return Ok(None),
|
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
|
|
std::fs::create_dir(&path).with_context(|| format!("create {}", path.display()))?;
|
|
if i + 1 == parts.len() {
|
|
lchown(&path, Some(uid), Some(gid))
|
|
.with_context(|| format!("chown {}", path.display()))?;
|
|
}
|
|
}
|
|
Err(e) => return Err(e).with_context(|| format!("stat {}", path.display())),
|
|
}
|
|
}
|
|
Ok(Some(path))
|
|
}
|
|
|
|
/// Create `parent/base`, or `parent/base-2`, ... when taken. Created, never
|
|
/// reused: `mkdir` does not follow a symlink an agent left under that name.
|
|
fn fresh_dir(parent: &Path, base: &str) -> Result<PathBuf> {
|
|
for n in 1.. {
|
|
let path = if n == 1 {
|
|
parent.join(base)
|
|
} else {
|
|
parent.join(format!("{base}-{n}"))
|
|
};
|
|
match std::fs::create_dir(&path) {
|
|
Ok(()) => return Ok(path),
|
|
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {}
|
|
Err(e) => return Err(e).with_context(|| format!("create {}", path.display())),
|
|
}
|
|
}
|
|
unreachable!("the loop above returns or continues")
|
|
}
|
|
|
|
/// `<root>.<suffix>`, or `<root>.<suffix>-2`, ... the first name nothing uses.
|
|
fn free_sibling(root: &Path, suffix: &str) -> Result<PathBuf> {
|
|
let name = root
|
|
.file_name()
|
|
.and_then(|n| n.to_str())
|
|
.with_context(|| format!("{} has no usable name", root.display()))?;
|
|
for n in 1.. {
|
|
let candidate = if n == 1 {
|
|
format!("{name}.{suffix}")
|
|
} else {
|
|
format!("{name}.{suffix}-{n}")
|
|
};
|
|
let path = root.with_file_name(candidate);
|
|
if std::fs::symlink_metadata(&path).is_err() {
|
|
return Ok(path);
|
|
}
|
|
}
|
|
unreachable!("the loop above returns or continues")
|
|
}
|
|
|
|
/// Copies a tree for [`salvage`] and keeps count.
|
|
struct Copier<'a> {
|
|
backend: &'a dyn Backend,
|
|
uid: u32,
|
|
gid: u32,
|
|
stats: Stats,
|
|
}
|
|
|
|
impl<'a> Copier<'a> {
|
|
fn new(backend: &'a dyn Backend, uid: u32, gid: u32) -> Self {
|
|
Self {
|
|
backend,
|
|
uid,
|
|
gid,
|
|
stats: Stats::default(),
|
|
}
|
|
}
|
|
|
|
/// Copy everything under `src` except [`CACHE`] into `dst`, owned by
|
|
/// `uid:gid`, with modes and mtimes kept. Sockets, fifos and device nodes
|
|
/// are skipped; hard links become separate copies.
|
|
fn tree(&mut self, src: &Path, dst: &Path, rel: &Path) -> Result<()> {
|
|
let (uid, gid) = (self.uid, self.gid);
|
|
let dir = src.join(rel);
|
|
for entry in std::fs::read_dir(&dir).with_context(|| format!("read {}", dir.display()))? {
|
|
let entry = entry.with_context(|| format!("read {}", dir.display()))?;
|
|
let rel = rel.join(entry.file_name());
|
|
if is_cache(&rel) {
|
|
continue;
|
|
}
|
|
let from = src.join(&rel);
|
|
let to = dst.join(&rel);
|
|
let meta = std::fs::symlink_metadata(&from)
|
|
.with_context(|| format!("stat {}", from.display()))?;
|
|
let kind = meta.file_type();
|
|
if kind.is_dir() {
|
|
std::fs::create_dir(&to).with_context(|| format!("create {}", to.display()))?;
|
|
self.tree(src, dst, &rel)?;
|
|
lchown(&to, Some(uid), Some(gid))
|
|
.with_context(|| format!("chown {}", to.display()))?;
|
|
set_mode(&to, &meta)?;
|
|
} else if kind.is_symlink() {
|
|
let target = std::fs::read_link(&from)
|
|
.with_context(|| format!("read link {}", from.display()))?;
|
|
symlink(&target, &to).with_context(|| format!("link {}", to.display()))?;
|
|
lchown(&to, Some(uid), Some(gid))
|
|
.with_context(|| format!("chown {}", to.display()))?;
|
|
} else if kind.is_file() {
|
|
self.file(&from, &to, &meta)?;
|
|
} else {
|
|
tracing::info!(path = %from.display(), "salvage skips a socket, fifo or device node");
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// One regular file: cloned when the filesystem allows it, else copied.
|
|
fn file(&mut self, from: &Path, to: &Path, meta: &std::fs::Metadata) -> Result<()> {
|
|
let mut src = open_nofollow(from).with_context(|| format!("open {}", from.display()))?;
|
|
let mut dst = create_new(to).with_context(|| format!("create {}", to.display()))?;
|
|
self.stats.files += 1;
|
|
self.stats.bytes = self.stats.bytes.saturating_add(meta.len());
|
|
if self.backend.clone_file(&src, &dst).is_ok() {
|
|
self.stats.cloned += 1;
|
|
} else {
|
|
self.check_room(to, meta.len())?;
|
|
io::copy(&mut src, &mut dst)
|
|
.with_context(|| format!("copy {} to {}", from.display(), to.display()))?;
|
|
}
|
|
if let Ok(mtime) = meta.modified() {
|
|
dst.set_modified(mtime)
|
|
.with_context(|| format!("set mtime of {}", to.display()))?;
|
|
}
|
|
lchown(to, Some(self.uid), Some(self.gid))
|
|
.with_context(|| format!("chown {}", to.display()))?;
|
|
set_mode(to, meta)
|
|
}
|
|
|
|
/// Fail when copying `len` bytes to `to` would leave less than
|
|
/// [`CLONE_RESERVE`] free.
|
|
fn check_room(&self, to: &Path, len: u64) -> Result<()> {
|
|
let dir = to.parent().unwrap_or(to);
|
|
let space = self
|
|
.backend
|
|
.space(dir)
|
|
.with_context(|| format!("free space of {}", dir.display()))?;
|
|
if space.free < len.saturating_add(CLONE_RESERVE) {
|
|
bail!(
|
|
"{} is {len} bytes and the filesystem has {} free; at least {CLONE_RESERVE} must stay",
|
|
to.display(),
|
|
space.free
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
/// Give `path` the permission bits of `meta`, which `chown` may have cleared.
|
|
fn set_mode(path: &Path, meta: &std::fs::Metadata) -> Result<()> {
|
|
std::fs::set_permissions(path, std::fs::Permissions::from_mode(meta.mode() & 0o7777))
|
|
.with_context(|| format!("chmod {}", path.display()))
|
|
}
|
|
|
|
/// The root's machine id when `etc/machine-id` is a regular file holding one.
|
|
/// The container may have replaced `etc` or the file by a link, so neither is
|
|
/// followed.
|
|
fn read_machine_id(root: &Path) -> Option<String> {
|
|
let etc = std::fs::symlink_metadata(root.join("etc")).ok()?;
|
|
let path = root.join("etc/machine-id");
|
|
if !etc.is_dir() || !is_regular(&path) {
|
|
return None;
|
|
}
|
|
let text = std::fs::read_to_string(path).ok()?;
|
|
let id = text.trim_end_matches('\n');
|
|
(id.len() == 32 && id.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')))
|
|
.then(|| id.to_owned())
|
|
}
|
|
|
|
/// Give the template a machine id when it has none: with an empty
|
|
/// `etc/machine-id` the guest makes a new one on every boot, and its journal
|
|
/// directory and DHCP identity change with it.
|
|
fn ensure_machine_id(root: &Path) -> Result<()> {
|
|
if read_machine_id(root).is_some() {
|
|
return Ok(());
|
|
}
|
|
let uuid = std::fs::read_to_string("/proc/sys/kernel/random/uuid")
|
|
.context("read a random uuid for the machine id")?;
|
|
let id: String = uuid.trim().chars().filter(|c| *c != '-').collect();
|
|
write_container_etc_file_in(root, "machine-id", &format!("{id}\n"))
|
|
}
|
|
|
|
fn is_regular(path: &Path) -> bool {
|
|
std::fs::symlink_metadata(path).is_ok_and(|m| m.is_file())
|
|
}
|
|
|
|
/// `systemctl is-active` answers `inactive` or `failed` for a unit that is not
|
|
/// running; anything else (`active`, `activating`, `deactivating`) means the
|
|
/// root is, or is about to be, in use.
|
|
fn unit_is_stopped(container: &str) -> Result<bool> {
|
|
let out = std::process::Command::new("systemctl")
|
|
.args(["is-active", &format!("container@{container}.service")])
|
|
.output()
|
|
.context("invoke systemctl is-active")?;
|
|
let state = String::from_utf8_lossy(&out.stdout);
|
|
Ok(matches!(state.trim(), "inactive" | "failed"))
|
|
}
|
|
|
|
/// Today in UTC as `YYYYMMDD`.
|
|
fn today() -> String {
|
|
let secs = std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.map_or(0, |d| d.as_secs());
|
|
date_of(secs)
|
|
}
|
|
|
|
/// `YYYYMMDD` of a unix time, by Howard Hinnant's days-to-civil algorithm.
|
|
fn date_of(secs: u64) -> String {
|
|
let z = i64::try_from(secs / 86_400).unwrap_or(0) + 719_468;
|
|
let era = z.div_euclid(146_097);
|
|
let doe = z.rem_euclid(146_097);
|
|
let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
|
|
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
|
let mp = (5 * doy + 2) / 153;
|
|
let day = doy - (153 * mp + 2) / 5 + 1;
|
|
let month = if mp < 10 { mp + 3 } else { mp - 9 };
|
|
let year = yoe + era * 400 + i64::from(month <= 2);
|
|
format!("{year:04}{month:02}{day:02}")
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use std::fs::File;
|
|
use std::io;
|
|
use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _, lchown, symlink};
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::Mutex;
|
|
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
|
use std::time::Duration;
|
|
|
|
use anyhow::Result;
|
|
|
|
use super::{
|
|
CLONE_RESERVE, Env, MARKER, Outcome, Stats, date_of, gib, has_content, report, rotate,
|
|
};
|
|
use crate::salvage_io::{Backend, FsSpace, Os};
|
|
use crate::shared::test_support::scratch;
|
|
|
|
const DATE: &str = "20261028";
|
|
|
|
/// How [`Fake`] answers a clone request.
|
|
#[derive(Clone, Copy)]
|
|
enum CloneMode {
|
|
/// Succeeds, by copying the bytes.
|
|
Copy,
|
|
/// Fails with this errno.
|
|
Fail(i32),
|
|
/// Succeeds for the first request (the probe), fails for the rest.
|
|
FirstOnly,
|
|
/// Defers to the real `ioctl(FICLONE)`.
|
|
Real,
|
|
}
|
|
|
|
/// A [`Backend`] with the free space and the clone result set by the test.
|
|
struct Fake {
|
|
free: u64,
|
|
total: u64,
|
|
mode: CloneMode,
|
|
delay: Duration,
|
|
space_calls: AtomicU32,
|
|
space_fails_from: u32,
|
|
calls: AtomicU32,
|
|
started: AtomicBool,
|
|
active: AtomicU32,
|
|
peak: AtomicU32,
|
|
}
|
|
|
|
impl Fake {
|
|
/// Plenty of space; every clone fails with `EXDEV`, so files are copied.
|
|
fn new() -> Self {
|
|
Self {
|
|
free: 1 << 50,
|
|
total: 1 << 51,
|
|
mode: CloneMode::Fail(libc::EXDEV),
|
|
delay: Duration::ZERO,
|
|
space_calls: AtomicU32::new(0),
|
|
space_fails_from: u32::MAX,
|
|
calls: AtomicU32::new(0),
|
|
started: AtomicBool::new(false),
|
|
active: AtomicU32::new(0),
|
|
peak: AtomicU32::new(0),
|
|
}
|
|
}
|
|
|
|
fn space(mut self, free: u64, total: u64) -> Self {
|
|
self.free = free;
|
|
self.total = total;
|
|
self
|
|
}
|
|
|
|
fn mode(mut self, mode: CloneMode) -> Self {
|
|
self.mode = mode;
|
|
self
|
|
}
|
|
|
|
/// Make the `n`th and later free-space reads fail, counting from 0.
|
|
fn space_fails_from(mut self, n: u32) -> Self {
|
|
self.space_fails_from = n;
|
|
self
|
|
}
|
|
|
|
fn delay(mut self, delay: Duration) -> Self {
|
|
self.delay = delay;
|
|
self
|
|
}
|
|
}
|
|
|
|
impl Backend for Fake {
|
|
fn space(&self, _path: &Path) -> io::Result<FsSpace> {
|
|
if self.space_calls.fetch_add(1, Ordering::SeqCst) >= self.space_fails_from {
|
|
return Err(io::Error::from_raw_os_error(libc::EIO));
|
|
}
|
|
Ok(FsSpace {
|
|
free: self.free,
|
|
total: self.total,
|
|
})
|
|
}
|
|
|
|
fn clone_file(&self, src: &File, dst: &File) -> io::Result<()> {
|
|
let call = self.calls.fetch_add(1, Ordering::SeqCst);
|
|
self.started.store(true, Ordering::SeqCst);
|
|
let active = self.active.fetch_add(1, Ordering::SeqCst) + 1;
|
|
self.peak.fetch_max(active, Ordering::SeqCst);
|
|
std::thread::sleep(self.delay);
|
|
let copy = || io::copy(&mut &*src, &mut &*dst).map(drop);
|
|
let result = match self.mode {
|
|
CloneMode::Copy => copy(),
|
|
CloneMode::Fail(errno) => Err(io::Error::from_raw_os_error(errno)),
|
|
CloneMode::FirstOnly if call == 0 => copy(),
|
|
CloneMode::FirstOnly => Err(io::Error::from_raw_os_error(libc::EXDEV)),
|
|
CloneMode::Real => Os.clone_file(src, dst),
|
|
};
|
|
self.active.fetch_sub(1, Ordering::SeqCst);
|
|
result
|
|
}
|
|
}
|
|
|
|
/// A stopped agent's old root and work dir under one scratch dir. The
|
|
/// "agent" is the test's own user, the one it can chown to.
|
|
struct Fixture {
|
|
top: PathBuf,
|
|
root: PathBuf,
|
|
work: PathBuf,
|
|
profiles: PathBuf,
|
|
fake: Fake,
|
|
serial: Mutex<()>,
|
|
}
|
|
|
|
impl Fixture {
|
|
fn new() -> Self {
|
|
let top = scratch();
|
|
let root = top.join("h-iris");
|
|
let work = top.join("work");
|
|
let profiles = top.join("profiles");
|
|
std::fs::create_dir_all(&profiles).unwrap();
|
|
std::fs::create_dir_all(root.join("home/iris")).unwrap();
|
|
std::fs::create_dir_all(&work).unwrap();
|
|
Self {
|
|
top,
|
|
root,
|
|
work,
|
|
profiles,
|
|
fake: Fake::new(),
|
|
serial: Mutex::new(()),
|
|
}
|
|
}
|
|
|
|
fn with(fake: Fake) -> Self {
|
|
let mut f = Self::new();
|
|
f.fake = fake;
|
|
f
|
|
}
|
|
|
|
fn put(&self, rel: &str, body: &str) {
|
|
let path = self.root.join(rel);
|
|
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
|
|
std::fs::write(path, body).unwrap();
|
|
}
|
|
|
|
fn try_rotate_in(&self, backend: &dyn Backend, serial: &Mutex<()>) -> Result<Outcome> {
|
|
let env = Env {
|
|
backend,
|
|
lock_dir: &self.top.join("locks"),
|
|
one_at_a_time: serial,
|
|
};
|
|
rotate(&self.root, "iris", &self.work, &self.profiles, DATE, &env)
|
|
}
|
|
|
|
fn try_rotate(&self) -> Result<Outcome> {
|
|
self.try_rotate_in(&self.fake, &self.serial)
|
|
}
|
|
|
|
fn rotate(&self) -> Outcome {
|
|
self.try_rotate().unwrap()
|
|
}
|
|
|
|
/// Where the container finds nix's per-user state, on the host.
|
|
fn profile(&self) -> PathBuf {
|
|
self.profiles.join("per-user/iris")
|
|
}
|
|
|
|
fn salvage(&self) -> PathBuf {
|
|
self.work.join(format!("salvaged-rootfs-{DATE}"))
|
|
}
|
|
|
|
fn names(dir: &Path) -> Vec<String> {
|
|
let mut v: Vec<String> = std::fs::read_dir(dir)
|
|
.unwrap()
|
|
.map(|e| e.unwrap().file_name().into_string().unwrap())
|
|
.collect();
|
|
v.sort();
|
|
v
|
|
}
|
|
}
|
|
|
|
impl Drop for Fixture {
|
|
fn drop(&mut self) {
|
|
std::fs::remove_dir_all(&self.top).ok();
|
|
}
|
|
}
|
|
|
|
/// The first call copies the agent's files and moves the old root aside;
|
|
/// the second finds a clean template and does nothing.
|
|
#[test]
|
|
fn salvages_once() {
|
|
let f = Fixture::new();
|
|
f.put("home/iris/notes.txt", "keep me");
|
|
f.put("etc/hostname", "iris");
|
|
|
|
let Outcome::Salvaged {
|
|
salvage, old_root, ..
|
|
} = f.rotate()
|
|
else {
|
|
panic!("the first call must salvage");
|
|
};
|
|
assert_eq!(salvage, f.salvage());
|
|
assert_eq!(
|
|
std::fs::read_to_string(salvage.join("home/iris/notes.txt")).unwrap(),
|
|
"keep me"
|
|
);
|
|
assert_eq!(
|
|
std::fs::read_to_string(old_root.join("home/iris/notes.txt")).unwrap(),
|
|
"keep me"
|
|
);
|
|
assert!(f.root.join("etc").join(MARKER).is_file());
|
|
assert!(
|
|
!f.root.join("home/iris").exists(),
|
|
"template holds the old home"
|
|
);
|
|
|
|
assert_eq!(f.rotate(), Outcome::Clean);
|
|
assert_eq!(
|
|
Fixture::names(&f.work),
|
|
[format!("salvaged-rootfs-{DATE}")],
|
|
"a second salvage dir appeared"
|
|
);
|
|
}
|
|
|
|
/// Content that appears in the template after the marker (a run by a
|
|
/// build without the flag) is salvaged again, into its own directory.
|
|
#[test]
|
|
fn content_after_the_marker_is_salvaged_again() {
|
|
let f = Fixture::new();
|
|
f.put("home/iris/a", "1");
|
|
f.rotate();
|
|
f.put("home/iris/b", "2");
|
|
let Outcome::Salvaged { salvage, .. } = f.rotate() else {
|
|
panic!("late content must be salvaged");
|
|
};
|
|
assert_eq!(salvage, f.work.join(format!("salvaged-rootfs-{DATE}-2")));
|
|
assert!(salvage.join("home/iris/b").is_file());
|
|
assert!(
|
|
f.salvage().join("home/iris/a").is_file(),
|
|
"first salvage kept"
|
|
);
|
|
}
|
|
|
|
/// The journal, `/tmp`, `/var/cache`, weston and the otel cursor stay out
|
|
/// of the salvage; a sibling of each does not. The root's own copies are
|
|
/// untouched.
|
|
#[test]
|
|
fn cache_paths_are_skipped_and_their_siblings_are_not() {
|
|
let f = Fixture::new();
|
|
f.put("home/iris/keep", "x");
|
|
for cache in [
|
|
"tmp/scratch",
|
|
"var/cache/nix/x",
|
|
"var/log/journal/id/system.journal",
|
|
"var/lib/weston/state",
|
|
"var/lib/opentelemetry-collector/cursor",
|
|
] {
|
|
f.put(cache, "cache");
|
|
}
|
|
for real in [
|
|
"var/log/other.log",
|
|
"var/lib/other/db",
|
|
"var/tmp/keep",
|
|
"root/.bash_history",
|
|
] {
|
|
f.put(real, "real");
|
|
}
|
|
let Outcome::Salvaged {
|
|
salvage, old_root, ..
|
|
} = f.rotate()
|
|
else {
|
|
panic!("must salvage");
|
|
};
|
|
for cache in [
|
|
"tmp",
|
|
"var/cache",
|
|
"var/log/journal",
|
|
"var/lib/weston",
|
|
"var/lib/opentelemetry-collector",
|
|
] {
|
|
assert!(!salvage.join(cache).exists(), "{cache} was copied");
|
|
assert!(old_root.join(cache).exists(), "{cache} left the old root");
|
|
}
|
|
for real in [
|
|
"var/log/other.log",
|
|
"var/lib/other/db",
|
|
"var/tmp/keep",
|
|
"root/.bash_history",
|
|
"home/iris/keep",
|
|
] {
|
|
assert!(salvage.join(real).is_file(), "{real} was not copied");
|
|
}
|
|
}
|
|
|
|
/// A root with only the cache paths and the start path's own files has
|
|
/// nothing to salvage: no directory in `work/`, no move.
|
|
#[test]
|
|
fn a_root_without_content_is_not_salvaged() {
|
|
let f = Fixture::new();
|
|
f.put("var/cache/x", "c");
|
|
f.put("etc/hyperhive-hive-facts", "A=b\n");
|
|
f.put("etc/resolv.conf", "nameserver 1.1.1.1\n");
|
|
assert_eq!(f.rotate(), Outcome::Clean);
|
|
assert!(f.work.read_dir().unwrap().next().is_none());
|
|
assert!(f.root.join("var/cache/x").is_file(), "root was touched");
|
|
assert!(f.root.join("etc").join(MARKER).is_file());
|
|
// The same call with one real file makes the control: it salvages.
|
|
f.put("home/iris/real", "x");
|
|
assert!(matches!(f.rotate(), Outcome::Salvaged { .. }));
|
|
}
|
|
|
|
/// Nothing is removed: the old root keeps every file, and a failed
|
|
/// salvage leaves the root where it was.
|
|
#[test]
|
|
fn nothing_is_deleted() {
|
|
let f = Fixture::new();
|
|
f.put("home/iris/deep/er/file", "data");
|
|
f.put("etc/passwd", "iris");
|
|
let Outcome::Salvaged { old_root, .. } = f.rotate() else {
|
|
panic!("must salvage");
|
|
};
|
|
assert_eq!(
|
|
std::fs::read_to_string(old_root.join("home/iris/deep/er/file")).unwrap(),
|
|
"data"
|
|
);
|
|
assert_eq!(
|
|
std::fs::read_to_string(old_root.join("etc/passwd")).unwrap(),
|
|
"iris"
|
|
);
|
|
|
|
let g = Fixture::new();
|
|
g.put("home/iris/file", "data");
|
|
std::fs::remove_dir(&g.work).unwrap();
|
|
std::fs::write(&g.work, "not a dir").unwrap();
|
|
assert!(g.try_rotate().is_err());
|
|
assert_eq!(
|
|
std::fs::read_to_string(g.root.join("home/iris/file")).unwrap(),
|
|
"data"
|
|
);
|
|
assert!(!g.root.join("etc").join(MARKER).exists());
|
|
}
|
|
|
|
/// A group of the test user other than `primary`, if it has one.
|
|
fn other_group(primary: u32) -> Option<u32> {
|
|
let mut groups = [0 as libc::gid_t; 64];
|
|
// SAFETY: `groups` has room for the 64 ids it declares.
|
|
let n = unsafe { libc::getgroups(64, groups.as_mut_ptr()) };
|
|
let n = usize::try_from(n).ok()?;
|
|
groups[..n].iter().copied().find(|g| *g != primary)
|
|
}
|
|
|
|
/// Modes, mtimes, symlinks and the owner reach the salvage. The owner is
|
|
/// the test user, who can chown only to itself: the check pins that the
|
|
/// requested id lands on files, directories and links alike.
|
|
#[test]
|
|
fn modes_links_and_owner_are_kept() {
|
|
let f = Fixture::new();
|
|
f.put("home/iris/run.sh", "#!/bin/sh\n");
|
|
std::fs::set_permissions(
|
|
f.root.join("home/iris/run.sh"),
|
|
std::fs::Permissions::from_mode(0o750),
|
|
)
|
|
.unwrap();
|
|
std::fs::set_permissions(
|
|
f.root.join("home/iris"),
|
|
std::fs::Permissions::from_mode(0o700),
|
|
)
|
|
.unwrap();
|
|
symlink("run.sh", f.root.join("home/iris/link")).unwrap();
|
|
let mtime = std::fs::metadata(f.root.join("home/iris/run.sh"))
|
|
.unwrap()
|
|
.mtime();
|
|
// An unprivileged user can chown a file to any group it is in, so a
|
|
// second group gives the owner something to differ from the group new
|
|
// files get. Without one the gid check below cannot tell a chown from
|
|
// none.
|
|
let primary = std::fs::metadata(&f.work).unwrap().gid();
|
|
if let Some(other) = other_group(primary) {
|
|
lchown(f.root.join("home/iris"), None, Some(other)).unwrap();
|
|
}
|
|
|
|
let Outcome::Salvaged {
|
|
salvage, old_root, ..
|
|
} = f.rotate()
|
|
else {
|
|
panic!("must salvage");
|
|
};
|
|
let me = std::fs::metadata(old_root.join("home/iris")).unwrap();
|
|
let file = std::fs::symlink_metadata(salvage.join("home/iris/run.sh")).unwrap();
|
|
assert_eq!(file.mode() & 0o7777, 0o750);
|
|
assert_eq!(file.mtime(), mtime);
|
|
assert_eq!(
|
|
std::fs::symlink_metadata(salvage.join("home/iris"))
|
|
.unwrap()
|
|
.mode()
|
|
& 0o7777,
|
|
0o700
|
|
);
|
|
assert_eq!(
|
|
std::fs::read_link(salvage.join("home/iris/link")).unwrap(),
|
|
Path::new("run.sh")
|
|
);
|
|
for rel in [
|
|
"",
|
|
"home",
|
|
"home/iris",
|
|
"home/iris/run.sh",
|
|
"home/iris/link",
|
|
] {
|
|
let meta = std::fs::symlink_metadata(salvage.join(rel)).unwrap();
|
|
assert_eq!((meta.uid(), meta.gid()), (me.uid(), me.gid()), "{rel:?}");
|
|
}
|
|
}
|
|
|
|
/// The template keeps the old root's machine id; one with none gets a
|
|
/// fresh id that the next call keeps.
|
|
#[test]
|
|
fn the_template_has_a_stable_machine_id() {
|
|
let f = Fixture::new();
|
|
f.put("home/iris/a", "1");
|
|
f.put("etc/machine-id", "0123456789abcdef0123456789abcdef\n");
|
|
f.rotate();
|
|
let id = std::fs::read_to_string(f.root.join("etc/machine-id")).unwrap();
|
|
assert_eq!(id, "0123456789abcdef0123456789abcdef\n");
|
|
|
|
let g = Fixture::new();
|
|
g.put("etc/machine-id", "");
|
|
g.rotate();
|
|
let fresh = std::fs::read_to_string(g.root.join("etc/machine-id")).unwrap();
|
|
assert_eq!(fresh.trim().len(), 32, "{fresh:?}");
|
|
g.rotate();
|
|
assert_eq!(
|
|
std::fs::read_to_string(g.root.join("etc/machine-id")).unwrap(),
|
|
fresh
|
|
);
|
|
}
|
|
|
|
/// `etc` replaced by a link to a directory holding a machine-id is not
|
|
/// followed.
|
|
#[test]
|
|
fn a_linked_etc_is_not_followed() {
|
|
let f = Fixture::new();
|
|
let outside = f.top.join("outside");
|
|
std::fs::create_dir_all(&outside).unwrap();
|
|
std::fs::write(
|
|
outside.join("machine-id"),
|
|
"0123456789abcdef0123456789abcdef\n",
|
|
)
|
|
.unwrap();
|
|
symlink(&outside, f.root.join("etc")).unwrap();
|
|
f.put("home/iris/a", "1");
|
|
f.rotate();
|
|
let id = std::fs::read_to_string(f.root.join("etc/machine-id")).unwrap();
|
|
assert_ne!(id, "0123456789abcdef0123456789abcdef\n");
|
|
}
|
|
|
|
/// A root that is gone has nothing to salvage and gets a template.
|
|
#[test]
|
|
fn a_missing_root_gets_a_template() {
|
|
let f = Fixture::new();
|
|
std::fs::remove_dir_all(&f.root).unwrap();
|
|
assert!(!has_content(&f.root).unwrap());
|
|
assert_eq!(f.rotate(), Outcome::Clean);
|
|
assert!(f.root.join("etc").join(MARKER).is_file());
|
|
}
|
|
|
|
/// What nix keeps for the agent user is copied to where the container finds
|
|
/// it next time, with its links intact; the old root keeps its own copy. A
|
|
/// directory already there is not overwritten, and a linked `.local` is not
|
|
/// followed.
|
|
#[test]
|
|
fn the_nix_profile_is_carried_over() {
|
|
let f = Fixture::new();
|
|
f.put("home/iris/.local/state/nix/profiles/profile-1-link", "gen");
|
|
symlink(
|
|
"profile-1-link",
|
|
f.root.join("home/iris/.local/state/nix/profiles/profile"),
|
|
)
|
|
.unwrap();
|
|
let Outcome::Salvaged { old_root, .. } = f.rotate() else {
|
|
panic!("must salvage");
|
|
};
|
|
let link = f.profile().join("profiles/profile");
|
|
assert_eq!(
|
|
std::fs::read_link(&link).unwrap(),
|
|
Path::new("profile-1-link")
|
|
);
|
|
assert_eq!(std::fs::read_to_string(&link).unwrap(), "gen");
|
|
assert!(
|
|
old_root
|
|
.join("home/iris/.local/state/nix/profiles/profile-1-link")
|
|
.is_file()
|
|
);
|
|
assert_eq!(
|
|
std::fs::metadata(f.profile()).unwrap().uid(),
|
|
std::fs::metadata(&f.work).unwrap().uid()
|
|
);
|
|
|
|
// An existing target stays as it is.
|
|
let g = Fixture::new();
|
|
g.put("home/iris/.local/state/nix/profiles/profile-1-link", "old");
|
|
std::fs::create_dir_all(g.profile()).unwrap();
|
|
std::fs::write(g.profile().join("mine"), "new").unwrap();
|
|
g.rotate();
|
|
assert_eq!(Fixture::names(&g.profile()), ["mine"]);
|
|
|
|
// An empty one, as the agent's tmpfiles rules leave it, is filled.
|
|
let e = Fixture::new();
|
|
e.put("home/iris/.local/state/nix/profiles/profile-1-link", "old");
|
|
std::fs::create_dir_all(e.profile()).unwrap();
|
|
e.rotate();
|
|
assert_eq!(Fixture::names(&e.profile()), ["profiles"]);
|
|
|
|
// A `.local` that is a link to a directory with the same layout.
|
|
let h = Fixture::new();
|
|
let outside = h.top.join("outside");
|
|
std::fs::create_dir_all(outside.join("state/nix")).unwrap();
|
|
std::fs::write(outside.join("state/nix/secret"), "host").unwrap();
|
|
symlink(&outside, h.root.join("home/iris/.local")).unwrap();
|
|
h.put("home/iris/other", "x");
|
|
h.rotate();
|
|
assert!(!h.profile().exists(), "followed a linked .local");
|
|
}
|
|
|
|
/// `per-user` or the agent's directory in it replaced by a link, as the
|
|
/// container can do: nothing lands where the link points, and the salvage
|
|
/// itself still completes.
|
|
#[test]
|
|
fn a_linked_profile_dir_is_not_followed() {
|
|
for linked in ["per-user", "per-user/iris"] {
|
|
let f = Fixture::new();
|
|
f.put("home/iris/.local/state/nix/profiles/profile-1-link", "gen");
|
|
let outside = f.top.join("outside");
|
|
std::fs::create_dir_all(&outside).unwrap();
|
|
if linked == "per-user/iris" {
|
|
std::fs::create_dir_all(f.profiles.join("per-user")).unwrap();
|
|
}
|
|
symlink(&outside, f.profiles.join(linked)).unwrap();
|
|
assert!(matches!(f.rotate(), Outcome::Salvaged { .. }), "{linked}");
|
|
assert!(
|
|
outside.read_dir().unwrap().next().is_none(),
|
|
"{linked}: wrote through the link"
|
|
);
|
|
assert!(
|
|
f.salvage()
|
|
.join("home/iris/.local/state/nix/profiles/profile-1-link")
|
|
.is_file()
|
|
);
|
|
}
|
|
}
|
|
|
|
const GIB: u64 = 1 << 30;
|
|
|
|
/// A copy that would leave less than 15 % of the filesystem free is skipped
|
|
/// and changes nothing; the same root with the space for it is salvaged.
|
|
#[test]
|
|
fn a_copy_that_leaves_too_little_free_is_skipped() {
|
|
// A 100 GiB filesystem keeps 15 GiB free; the root needs a little more.
|
|
let f = Fixture::with(Fake::new().space(15 * GIB, 100 * GIB));
|
|
f.put("home/iris/data", "x");
|
|
let skipped = f.rotate();
|
|
let Outcome::Skipped { need, free } = skipped else {
|
|
panic!("a copy into 15 GiB free must be skipped, got {skipped:?}");
|
|
};
|
|
assert_eq!(free, 15 * GIB);
|
|
assert!(need > 15 * GIB, "need {need}");
|
|
assert!(
|
|
f.work.read_dir().unwrap().next().is_none(),
|
|
"work/ was touched"
|
|
);
|
|
assert!(f.root.join("home/iris/data").is_file());
|
|
assert!(!f.root.join("etc").join(MARKER).exists());
|
|
assert!(
|
|
!report("h-iris", &skipped),
|
|
"a skipped salvage must start persistent"
|
|
);
|
|
|
|
// Control: a little more room, the same root.
|
|
let g = Fixture::with(Fake::new().space(15 * GIB + (1 << 20), 100 * GIB));
|
|
g.put("home/iris/data", "x");
|
|
let salvaged = g.rotate();
|
|
assert!(matches!(salvaged, Outcome::Salvaged { .. }), "{salvaged:?}");
|
|
assert!(report("h-iris", &salvaged));
|
|
assert!(report("h-iris", &Outcome::Clean));
|
|
}
|
|
|
|
/// Files that clone cost metadata, so the 15 % floor does not apply: the
|
|
/// same root and space that skip when files must be copied go through when
|
|
/// the probe clones.
|
|
#[test]
|
|
fn files_that_clone_need_only_metadata() {
|
|
let copying = Fixture::with(Fake::new().space(5 * GIB, 100 * GIB));
|
|
copying.put("home/iris/data", "x");
|
|
assert!(matches!(copying.rotate(), Outcome::Skipped { .. }));
|
|
|
|
let cloning = Fixture::with(Fake::new().space(5 * GIB, 100 * GIB).mode(CloneMode::Copy));
|
|
cloning.put("home/iris/data", "x");
|
|
let Outcome::Salvaged { stats, .. } = cloning.rotate() else {
|
|
panic!("cloning files need no more than the reserve");
|
|
};
|
|
assert_eq!(
|
|
stats,
|
|
Stats {
|
|
bytes: 1,
|
|
files: 1,
|
|
cloned: 1
|
|
}
|
|
);
|
|
|
|
// Not even metadata fits below the reserve.
|
|
let full = Fixture::with(
|
|
Fake::new()
|
|
.space(CLONE_RESERVE - 1, 100 * GIB)
|
|
.mode(CloneMode::Copy),
|
|
);
|
|
full.put("home/iris/data", "x");
|
|
assert!(matches!(full.rotate(), Outcome::Skipped { .. }));
|
|
}
|
|
|
|
/// A clone that fails (another filesystem, one without reflinks) leaves the
|
|
/// file to be copied, whatever the errno; the control counts a clone.
|
|
#[test]
|
|
fn a_failed_clone_falls_back_to_a_copy() {
|
|
for errno in [libc::EXDEV, libc::EOPNOTSUPP, libc::EINVAL] {
|
|
let f = Fixture::with(Fake::new().mode(CloneMode::Fail(errno)));
|
|
f.put("home/iris/a", "alpha");
|
|
f.put("home/iris/b", "beta");
|
|
let Outcome::Salvaged { salvage, stats, .. } = f.rotate() else {
|
|
panic!("errno {errno}: must salvage");
|
|
};
|
|
assert_eq!(
|
|
std::fs::read_to_string(salvage.join("home/iris/a")).unwrap(),
|
|
"alpha"
|
|
);
|
|
assert_eq!(
|
|
std::fs::read_to_string(salvage.join("home/iris/b")).unwrap(),
|
|
"beta"
|
|
);
|
|
assert_eq!(
|
|
stats,
|
|
Stats {
|
|
bytes: 9,
|
|
files: 2,
|
|
cloned: 0
|
|
},
|
|
"errno {errno}"
|
|
);
|
|
}
|
|
let f = Fixture::with(Fake::new().mode(CloneMode::Copy));
|
|
f.put("home/iris/a", "alpha");
|
|
let Outcome::Salvaged { salvage, stats, .. } = f.rotate() else {
|
|
panic!("must salvage");
|
|
};
|
|
assert_eq!(stats.cloned, 1);
|
|
assert_eq!(
|
|
std::fs::read_to_string(salvage.join("home/iris/a")).unwrap(),
|
|
"alpha"
|
|
);
|
|
}
|
|
|
|
/// The real `ioctl(FICLONE)`: whether the scratch filesystem has reflinks or
|
|
/// not, the salvage holds the same bytes.
|
|
#[test]
|
|
fn the_real_clone_call_leaves_a_faithful_copy() {
|
|
let f = Fixture::with(Fake::new().mode(CloneMode::Real));
|
|
let body = "0123456789".repeat(10_000);
|
|
f.put("home/iris/big", &body);
|
|
f.put("home/iris/empty", "");
|
|
let Outcome::Salvaged { salvage, stats, .. } = f.rotate() else {
|
|
panic!("must salvage");
|
|
};
|
|
assert_eq!(
|
|
std::fs::read_to_string(salvage.join("home/iris/big")).unwrap(),
|
|
body
|
|
);
|
|
assert_eq!(stats.files, 2);
|
|
assert!(stats.cloned <= stats.files);
|
|
}
|
|
|
|
/// A file that cannot clone, in a salvage whose probe did, is copied only
|
|
/// while the reserve stays free.
|
|
#[test]
|
|
fn a_file_that_must_be_copied_keeps_the_reserve_free() {
|
|
let body = "x".repeat(1 << 20);
|
|
let tight = Fixture::with(
|
|
Fake::new()
|
|
.space(CLONE_RESERVE + (512 << 10), 100 * GIB)
|
|
.mode(CloneMode::FirstOnly),
|
|
);
|
|
tight.put("home/iris/big", &body);
|
|
let err = tight.try_rotate().unwrap_err();
|
|
assert!(format!("{err:#}").contains("must stay"), "{err:#}");
|
|
assert!(
|
|
tight.root.join("home/iris/big").is_file(),
|
|
"root was touched"
|
|
);
|
|
assert!(!tight.root.join("etc").join(MARKER).exists());
|
|
|
|
let roomy = Fixture::with(
|
|
Fake::new()
|
|
.space(CLONE_RESERVE + (8 << 20), 100 * GIB)
|
|
.mode(CloneMode::FirstOnly),
|
|
);
|
|
roomy.put("home/iris/big", &body);
|
|
let Outcome::Salvaged { stats, .. } = roomy.rotate() else {
|
|
panic!("1 MiB fits above the reserve");
|
|
};
|
|
assert_eq!(stats.cloned, 0);
|
|
}
|
|
|
|
/// A salvage that fails after it created its directory removes that
|
|
/// directory and nothing else: an earlier salvage stays, the old root is
|
|
/// as it was, and no marker is written. Once without room for a copy, once
|
|
/// with a free-space read that fails.
|
|
#[test]
|
|
fn a_failed_salvage_removes_its_partial_directory() {
|
|
let body = "x".repeat(1 << 20);
|
|
let cases = [
|
|
(
|
|
"no room for a file",
|
|
Fake::new()
|
|
.space(CLONE_RESERVE + (512 << 10), 100 * GIB)
|
|
.mode(CloneMode::FirstOnly),
|
|
),
|
|
("free space unreadable", Fake::new().space_fails_from(1)),
|
|
];
|
|
for (what, fake) in cases {
|
|
let f = Fixture::with(fake);
|
|
f.put("home/iris/big", &body);
|
|
f.put("home/iris/small", "s");
|
|
// An earlier salvage of the same day, which must survive.
|
|
let earlier = f.salvage();
|
|
std::fs::create_dir_all(earlier.join("home")).unwrap();
|
|
std::fs::write(earlier.join("home/old"), "kept").unwrap();
|
|
|
|
assert!(f.try_rotate().is_err(), "{what}: must fail");
|
|
assert_eq!(
|
|
Fixture::names(&f.work),
|
|
[format!("salvaged-rootfs-{DATE}")],
|
|
"{what}: a partial directory is left"
|
|
);
|
|
assert_eq!(
|
|
std::fs::read_to_string(earlier.join("home/old")).unwrap(),
|
|
"kept"
|
|
);
|
|
assert!(
|
|
f.root.join("home/iris/big").is_file(),
|
|
"{what}: root was touched"
|
|
);
|
|
assert!(f.root.join("home/iris/small").is_file(), "{what}");
|
|
assert!(!f.root.join("etc").join(MARKER).exists(), "{what}");
|
|
assert!(
|
|
Fixture::names(&f.top)
|
|
.iter()
|
|
.all(|n| !n.contains("pre-ephemeral")),
|
|
"{what}: the root was moved"
|
|
);
|
|
}
|
|
|
|
// Control: the same root with room and a working space read succeeds.
|
|
let ok = Fixture::with(Fake::new());
|
|
ok.put("home/iris/big", &body);
|
|
assert!(matches!(ok.rotate(), Outcome::Salvaged { .. }));
|
|
assert!(ok.salvage().join("home/iris/big").is_file());
|
|
}
|
|
|
|
/// A second request for a root that is being salvaged waits, then finds
|
|
/// the finished template: one salvage directory, one moved-aside root.
|
|
#[test]
|
|
fn two_requests_for_one_root_salvage_it_once() {
|
|
let f = Fixture::with(
|
|
Fake::new()
|
|
.mode(CloneMode::Copy)
|
|
.delay(Duration::from_millis(200)),
|
|
);
|
|
f.put("home/iris/a", "1");
|
|
let (first, second) = std::thread::scope(|s| {
|
|
let first = s.spawn(|| f.rotate());
|
|
while !f.fake.started.load(Ordering::SeqCst) {
|
|
std::thread::sleep(Duration::from_millis(5));
|
|
}
|
|
let second = s.spawn(|| f.rotate());
|
|
(first.join().unwrap(), second.join().unwrap())
|
|
});
|
|
assert!(matches!(first, Outcome::Salvaged { .. }), "{first:?}");
|
|
assert_eq!(second, Outcome::Clean);
|
|
assert_eq!(Fixture::names(&f.work), [format!("salvaged-rootfs-{DATE}")]);
|
|
let moved: Vec<String> = Fixture::names(&f.top)
|
|
.into_iter()
|
|
.filter(|n| n.starts_with("h-iris.pre-ephemeral-"))
|
|
.collect();
|
|
assert_eq!(moved.len(), 1, "{moved:?}");
|
|
}
|
|
|
|
/// Salvages of different containers copy one after the other.
|
|
#[test]
|
|
fn salvages_of_different_containers_copy_one_at_a_time() {
|
|
let a = Fixture::new();
|
|
let b = Fixture::new();
|
|
a.put("home/iris/a", "1");
|
|
b.put("home/iris/b", "2");
|
|
let fake = Fake::new()
|
|
.mode(CloneMode::Copy)
|
|
.delay(Duration::from_millis(100));
|
|
let serial = Mutex::new(());
|
|
std::thread::scope(|s| {
|
|
s.spawn(|| a.try_rotate_in(&fake, &serial).unwrap());
|
|
s.spawn(|| b.try_rotate_in(&fake, &serial).unwrap());
|
|
});
|
|
assert!(fake.calls.load(Ordering::SeqCst) >= 4, "both salvages ran");
|
|
assert_eq!(fake.peak.load(Ordering::SeqCst), 1);
|
|
}
|
|
|
|
#[test]
|
|
fn sizes_are_shown_in_gib() {
|
|
assert_eq!(gib(0), "0.0 GiB");
|
|
assert_eq!(gib(3 * GIB / 2), "1.5 GiB");
|
|
assert_eq!(gib(87 * GIB), "87.0 GiB");
|
|
}
|
|
|
|
#[test]
|
|
fn dates() {
|
|
assert_eq!(date_of(0), "19700101");
|
|
assert_eq!(date_of(951_782_400), "20000229");
|
|
assert_eq!(date_of(1_793_145_600), "20261028");
|
|
}
|
|
}
|