Watch
0
0
Fork
You've already forked hyperhive
0

run agent containers on an ephemeral root, salvaging the old one first

hive-priv copies a stopped container's root to the agent's work dir and
renames it aside before passing nspawn --ephemeral; the agent's nix state
moves to the persistent profiles bind. Adds the agent-ephemeral-rootfs vm
test and a module-eval case for the nix state redirect.
This commit is contained in:
atlas 2026-10-28 19:15:26 +01:00 • committed by mara
commit 81439fa710
3 changed files with 1238 additions and 39 deletions

View file

@ -27,7 +27,7 @@ pub(crate) fn handle_write_nspawn_flags(
load_credentials: &[CredentialMount],
hive_facts: &HiveFacts,
) -> Result<(String, String)> {
validate_container_system_name(container)?;
let agent = validate_container_system_name(container)?;
for bind in binds {
validate_bind_path(&bind.host_path)?;
validate_bind_path(&bind.container_path)?;
@ -40,7 +40,24 @@ pub(crate) fn handle_write_nspawn_flags(
validate_bind_path(&cred.host_path)?;
}
let hive_facts = render_hive_facts(hive_facts)?;
write_nspawn_flags(container, binds, isolation, load_credentials, &hive_facts)?;
// A root that cannot be salvaged keeps its persistent start: the flag is
// only ever passed for a root the salvage has dealt with.
let ephemeral = crate::rootfs::prepare(container, agent).unwrap_or_else(|e| {
tracing::error!(
%container,
error = %format!("{e:#}"),
"root salvage failed; the container starts with its persistent root"
);
false
});
write_nspawn_flags(
container,
binds,
isolation,
load_credentials,
&hive_facts,
ephemeral,
)?;
Ok((String::new(), String::new()))
}
@ -267,6 +284,7 @@ fn write_nspawn_flags(
isolation: &NetworkIsolation,
load_credentials: &[CredentialMount],
hive_facts: &str,
ephemeral: bool,
) -> Result<()> {
use std::fmt::Write as _;
use std::os::unix::fs::MetadataExt as _;
@ -309,25 +327,7 @@ fn write_nspawn_flags(
out.push_str("LOCAL_ADDRESS6=\n");
let _ = writeln!(out, "HOST_BRIDGE={}", iso.bridge);
}
let mut flags: Vec<String> = binds
.iter()
.map(|b| {
let flag = if b.read_only { "--bind-ro" } else { "--bind" };
format!("{flag}={}:{}", b.host_path, b.container_path)
})
.collect();
flags.extend(git_overlay_flags(binds));
// Credential forwarding: nspawn loads each host secret into the
// container's credential store under `<name>`; inner units inherit it
// via `LoadCredential=<name>`. Validated (name charset + bind-path
// rules) in handle_write_nspawn_flags above.
for cred in load_credentials {
flags.push(format!(
"--load-credential={}:{}",
cred.name, cred.host_path
));
}
let flags_joined = flags.join(" ");
let flags_joined = nspawn_flags(binds, load_credentials, ephemeral).join(" ");
let _ = writeln!(out, "EXTRA_NSPAWN_FLAGS=\"{flags_joined}\"");
// Keeps the owner and mode `nixos-container` created the file with; this
// helper only rewrites its lines.
@ -353,6 +353,38 @@ fn write_nspawn_flags(
Ok(())
}
/// The flags `EXTRA_NSPAWN_FLAGS` carries, in the order nspawn needs them.
///
/// `--ephemeral` runs the container on a snapshot of its root and drops the
/// snapshot at stop; `crate::rootfs` has made the root fit for that before
/// the flag is asked for. The git overlays and credentials follow the binds
/// they refer to.
fn nspawn_flags(
binds: &[BindMount],
load_credentials: &[CredentialMount],
ephemeral: bool,
) -> Vec<String> {
let mut flags: Vec<String> = ephemeral
.then(|| "--ephemeral".to_owned())
.into_iter()
.collect();
flags.extend(binds.iter().map(|b| {
let flag = if b.read_only { "--bind-ro" } else { "--bind" };
format!("{flag}={}:{}", b.host_path, b.container_path)
}));
flags.extend(git_overlay_flags(binds));
// Credential forwarding: nspawn loads each host secret into the
// container's credential store under `<name>`; inner units inherit it
// via `LoadCredential=<name>`. Validated (name charset + bind-path
// rules) in handle_write_nspawn_flags.
flags.extend(
load_credentials
.iter()
.map(|cred| format!("--load-credential={}:{}", cred.name, cred.host_path)),
);
flags
}
/// Filename of the in-container DNS marker, in the container's own `/etc`.
const BRIDGE_DNS_MARKER: &str = "hyperhive-bridge-dns";
@ -360,7 +392,7 @@ const BRIDGE_DNS_MARKER: &str = "hyperhive-bridge-dns";
const HIVE_FACTS_FILE: &str = "hyperhive-hive-facts";
/// The container's `/` on the host.
fn container_rootfs(container: &str) -> PathBuf {
pub(crate) fn container_rootfs(container: &str) -> PathBuf {
PathBuf::from(format!("/var/lib/nixos-containers/{container}"))
}
@ -469,7 +501,7 @@ fn write_bridge_dns_marker_in(rootfs: &Path, gateway_ip: &str) -> Result<()> {
/// materialises the rootfs on the first start), so this creates it first; the
/// dir and the file persist through that start. Mode 0644: nothing written
/// here is a secret.
fn write_container_etc_file_in(rootfs: &Path, name: &str, contents: &str) -> Result<()> {
pub(crate) fn write_container_etc_file_in(rootfs: &Path, name: &str, contents: &str) -> Result<()> {
use std::os::unix::fs::OpenOptionsExt as _;
let etc_path = rootfs.join("etc");
@ -510,7 +542,7 @@ fn write_container_etc_file_in(rootfs: &Path, name: &str, contents: &str) -> Res
mod tests {
use super::{
BindMount, HiveFacts, NetworkIsolation, git_overlay_flags, handle_write_nspawn_flags,
limits_dropin_body, publish_file, remove_service_dropin, render_hive_facts,
limits_dropin_body, nspawn_flags, publish_file, remove_service_dropin, render_hive_facts,
validate_container_system_name, write_bridge_dns_marker_in, write_resource_limits,
};
use crate::shared::test_support::{entries, scratch};
@ -909,6 +941,20 @@ mod tests {
}
}
/// `--ephemeral` is asked for, not assumed: present exactly when the root has
/// been dealt with, ahead of the binds it applies to. The binds are the
/// control that the lookup finds what is there.
#[test]
fn ephemeral_is_passed_only_when_asked() {
let binds = [bind("/agents/atlas/work")];
let with = nspawn_flags(&binds, &[], true);
assert_eq!(with[0], "--ephemeral", "{with:?}");
assert!(with.iter().any(|f| f.starts_with("--bind-ro=")), "{with:?}");
let without = nspawn_flags(&binds, &[], false);
assert!(!without.iter().any(|f| f == "--ephemeral"), "{without:?}");
assert_eq!(without.len() + 1, with.len());
}
/// The swarm's forge and matrix containers are not the hive's: every
/// request that names a container as a string refuses them before
/// touching the filesystem. Control: an agent container name passes.

View file

@ -19,16 +19,18 @@
//! **Layout**: this file holds the socket loop and the one-arm-per-op
//! dispatch table. Each op family's handlers and validation live in their
//! own module: [`container`] (lifecycle), [`container_config`] (nspawn flags,
//! systemd drop-ins), [`storage`] (btrfs subvolumes, quotas, snapshots, disk
//! measurement), [`files`] (pause marker, socket dir, legacy tmpfiles), [`infra`] (gateway nginx,
//! infra containers) and [`closure`] (downloads from the swarm cache). Helpers
//! more than one family uses are in [`shared`].
//! systemd drop-ins), [`rootfs`] (the salvage of a container's root before it
//! turns ephemeral), [`storage`] (btrfs subvolumes, quotas, snapshots, disk
//! measurement), [`files`] (pause marker, socket dir, legacy tmpfiles),
//! [`infra`] (gateway nginx, infra containers) and [`closure`] (downloads from
//! the swarm cache). Helpers more than one family uses are in [`shared`].
mod closure;
mod container;
mod container_config;
mod files;
mod infra;
mod rootfs;
mod shared;
mod storage;

842
hive-priv/src/rootfs.rs Normal file
View file

@ -0,0 +1,842 @@
//! The agent container's non-persistent root.
//!
//! Agent containers start with `systemd-nspawn --ephemeral`: nspawn runs the
//! container on a throwaway snapshot of `/var/lib/nixos-containers/<c>`, so
//! anything written outside the binds is gone at stop. [`prepare`] runs ahead
//! of every `WriteNspawnFlags` and decides whether the flag may be passed.
//!
//! A root that holds anything beyond what the start path itself writes is
//! first salvaged. Everything outside [`CACHE`] is copied into
//! `<agent>/work/salvaged-rootfs-<YYYYMMDD>/`, owned by the agent, the old
//! root is renamed to `<root>.pre-ephemeral-<YYYYMMDD>` (never deleted), and a
//! fresh template root takes its place. The template is what nspawn snapshots
//! on every start, so it must stay small and free of the previous contents.
//! The template carries [`MARKER`]; it is written last, so a crash anywhere
//! earlier leaves the old root in place and the next call tries again.
//!
//! Only a stopped container is touched: a running one keeps whatever flag its
//! root already warrants.
use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _, lchown, symlink};
use std::path::{Path, PathBuf};
use anyhow::{Context as _, Result, bail};
use hive_priv_sock::AGENT_STATE_ROOT;
use crate::container_config::{container_rootfs, write_container_etc_file_in};
/// The per-container nix profile bind on the host, which persists across starts.
/// What nix keeps for the agent user is carried here; see [`carry_nix_state`].
const PROFILES_ROOT: &str = "/nix/var/nix/profiles/per-container";
/// Written into a template root once it is ready to be snapshotted.
const MARKER: &str = "hyperhive-ephemeral-root";
/// Files in `etc/` that the start path writes into every root, the template
/// included. They are not content: the root counts as clean with only these.
const TEMPLATE_FILES: [&str; 6] = [
MARKER,
"hyperhive-bridge-dns",
"hyperhive-hive-facts",
"machine-id",
"os-release",
"resolv.conf",
];
/// Paths relative to the root that the salvage skips: the guest journal, its
/// forwarder's read cursor, weston's state and scratch space. The first start
/// of a container rebuilds all of them.
const CACHE: [&str; 5] = [
"tmp",
"var/cache",
"var/log/journal",
"var/lib/weston",
"var/lib/opentelemetry-collector",
];
/// What [`rotate`] did.
#[derive(Debug, PartialEq, Eq)]
pub(crate) enum Outcome {
/// Nothing outside [`CACHE`] was in the root; nothing was moved.
Clean,
/// The root's content is copied to `salvage`, and the root itself kept at
/// `old_root`.
Salvaged { salvage: PathBuf, old_root: PathBuf },
}
/// Whether `container` may be started with `--ephemeral`, salvaging its root
/// first when it holds content and the container is stopped.
///
/// # Errors
/// Returns an error when the unit state cannot be read or the salvage fails.
/// Nothing is deleted on that path, and the caller starts the container
/// without the flag.
pub(crate) fn prepare(container: &str, agent: &str) -> Result<bool> {
let root = container_rootfs(container);
if !unit_is_stopped(container)? {
return Ok(is_regular(&root.join("etc").join(MARKER)));
}
let work = Path::new(AGENT_STATE_ROOT).join(agent).join("work");
let profiles = Path::new(PROFILES_ROOT).join(container);
match rotate(&root, agent, &work, &profiles, &today())? {
Outcome::Clean => {}
Outcome::Salvaged { salvage, old_root } => tracing::warn!(
%container,
salvage = %salvage.display(),
old_root = %old_root.display(),
"root salvaged before the first ephemeral start; the old root is kept, remove it by hand"
),
}
Ok(true)
}
/// Salvage `root` if it holds content, then leave a marked template in its
/// place. `work` is the agent's work dir on the host, `profiles` the host side of
/// the container's `/nix/var/nix/profiles`; `date` is `YYYYMMDD`.
fn rotate(root: &Path, agent: &str, work: &Path, profiles: &Path, date: &str) -> Result<Outcome> {
let outcome = if has_content(root)? {
let (uid, gid) = agent_owner(root, agent, work)?;
let machine_id = read_machine_id(root);
let salvage = fresh_dir(work, &format!("salvaged-rootfs-{date}"))?;
lchown(&salvage, Some(uid), Some(gid))
.with_context(|| format!("chown {}", salvage.display()))?;
copy_tree(root, &salvage, Path::new(""), uid, gid)
.with_context(|| format!("copy {} to {}", root.display(), salvage.display()))?;
carry_nix_state(root, agent, profiles, uid, gid)?;
let old_root = free_sibling(root, &format!("pre-ephemeral-{date}"))?;
std::fs::rename(root, &old_root)
.with_context(|| format!("move {} to {}", root.display(), old_root.display()))?;
std::fs::create_dir(root).with_context(|| format!("create {}", root.display()))?;
if let Some(id) = machine_id {
write_container_etc_file_in(root, "machine-id", &format!("{id}\n"))?;
}
Outcome::Salvaged { salvage, old_root }
} else {
Outcome::Clean
};
ensure_machine_id(root)?;
write_container_etc_file_in(root, MARKER, "")?;
Ok(outcome)
}
/// Whether anything but directories, [`TEMPLATE_FILES`] and [`CACHE`] is under
/// `root`. A missing root has none.
fn has_content(root: &Path) -> Result<bool> {
fn walk(dir: &Path, rel: &Path) -> Result<bool> {
let entries = match std::fs::read_dir(dir) {
Ok(entries) => entries,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false),
Err(e) => return Err(e).with_context(|| format!("read {}", dir.display())),
};
for entry in entries {
let entry = entry.with_context(|| format!("read {}", dir.display()))?;
let rel = rel.join(entry.file_name());
if is_cache(&rel) {
continue;
}
let kind = entry
.file_type()
.with_context(|| format!("stat {}", entry.path().display()))?;
let is_template = TEMPLATE_FILES
.iter()
.any(|name| rel == Path::new("etc").join(name));
if kind.is_dir() {
if walk(&entry.path(), &rel)? {
return Ok(true);
}
} else if !is_template {
return Ok(true);
}
}
Ok(false)
}
walk(root, Path::new(""))
}
fn is_cache(rel: &Path) -> bool {
CACHE.iter().any(|c| rel == Path::new(c))
}
/// The uid and gid the salvage is handed to: the owner of the agent's home in
/// the old root, else of its work dir. Never root.
fn agent_owner(root: &Path, agent: &str, work: &Path) -> Result<(u32, u32)> {
let home = real_dir(root, &["home", agent]);
for dir in [home, real_dir(work, &[])].into_iter().flatten() {
if let Ok(meta) = std::fs::symlink_metadata(&dir)
&& meta.uid() != 0
{
return Ok((meta.uid(), meta.gid()));
}
}
bail!(
"agent {agent}: neither its home in {} nor {} has a non-root owner",
root.display(),
work.display()
)
}
/// `base` joined with `parts`, if every component is a real directory. The
/// container could have made any of them a link, and this runs as root.
fn real_dir(base: &Path, parts: &[&str]) -> Option<PathBuf> {
let mut path = base.to_path_buf();
if !std::fs::symlink_metadata(&path).ok()?.is_dir() {
return None;
}
for part in parts {
path.push(part);
if !std::fs::symlink_metadata(&path).ok()?.is_dir() {
return None;
}
}
Some(path)
}
/// nix keeps a non-root user's profile under `~/.local/state/nix`, which the
/// agent's tmpfiles rules (`nix/agent-modules/user.nix`) point at
/// `per-user/<agent>` in `profiles`. A root that was lived in has the real
/// directory there: copy it across, so the agent's installed packages and the
/// GC roots that name them are still valid after the switch. A target that
/// holds anything is left alone; so is the whole copy when a component of the
/// target path is not a real directory, as the container can write `profiles`.
fn carry_nix_state(root: &Path, agent: &str, profiles: &Path, uid: u32, gid: u32) -> Result<()> {
let Some(src) = real_dir(root, &["home", agent, ".local", "state", "nix"]) else {
return Ok(());
};
let Some(target) = own_dir(profiles, &["per-user", agent], uid, gid)? else {
tracing::warn!(
profiles = %profiles.display(),
"a link or file where the nix state goes; the agent's nix state is in the salvage only"
);
return Ok(());
};
// The tmpfiles rules create `target` empty on a boot of the new module;
// anything in it is the agent's newer state.
if std::fs::read_dir(&target)
.with_context(|| format!("read {}", target.display()))?
.next()
.is_some()
{
return Ok(());
}
copy_tree(&src, &target, Path::new(""), uid, gid)
.with_context(|| format!("copy {} to {}", src.display(), target.display()))
}
/// `base` joined with `parts`, creating each missing component (the last one
/// owned by `uid:gid`), or `None` when `base` or an existing component is not a
/// real directory. The container could have replaced any of them by a link, and
/// this runs as root.
fn own_dir(base: &Path, parts: &[&str], uid: u32, gid: u32) -> Result<Option<PathBuf>> {
let mut path = base.to_path_buf();
if !std::fs::symlink_metadata(&path).is_ok_and(|m| m.is_dir()) {
return Ok(None);
}
for (i, part) in parts.iter().enumerate() {
path.push(part);
match std::fs::symlink_metadata(&path) {
Ok(meta) if meta.is_dir() => {}
Ok(_) => return Ok(None),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
std::fs::create_dir(&path).with_context(|| format!("create {}", path.display()))?;
if i + 1 == parts.len() {
lchown(&path, Some(uid), Some(gid))
.with_context(|| format!("chown {}", path.display()))?;
}
}
Err(e) => return Err(e).with_context(|| format!("stat {}", path.display())),
}
}
Ok(Some(path))
}
/// Create `parent/base`, or `parent/base-2`, ... when taken. Created, never
/// reused: `mkdir` does not follow a symlink an agent left under that name.
fn fresh_dir(parent: &Path, base: &str) -> Result<PathBuf> {
for n in 1.. {
let path = if n == 1 {
parent.join(base)
} else {
parent.join(format!("{base}-{n}"))
};
match std::fs::create_dir(&path) {
Ok(()) => return Ok(path),
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {}
Err(e) => return Err(e).with_context(|| format!("create {}", path.display())),
}
}
unreachable!("the loop above returns or continues")
}
/// `<root>.<suffix>`, or `<root>.<suffix>-2`, ... the first name nothing uses.
fn free_sibling(root: &Path, suffix: &str) -> Result<PathBuf> {
let name = root
.file_name()
.and_then(|n| n.to_str())
.with_context(|| format!("{} has no usable name", root.display()))?;
for n in 1.. {
let candidate = if n == 1 {
format!("{name}.{suffix}")
} else {
format!("{name}.{suffix}-{n}")
};
let path = root.with_file_name(candidate);
if std::fs::symlink_metadata(&path).is_err() {
return Ok(path);
}
}
unreachable!("the loop above returns or continues")
}
/// Copy everything under `src` except [`CACHE`] into `dst`, owned by
/// `uid:gid`, with modes and mtimes kept. Sockets, fifos and device nodes are
/// skipped; hard links become separate copies.
fn copy_tree(src: &Path, dst: &Path, rel: &Path, uid: u32, gid: u32) -> Result<()> {
let dir = src.join(rel);
for entry in std::fs::read_dir(&dir).with_context(|| format!("read {}", dir.display()))? {
let entry = entry.with_context(|| format!("read {}", dir.display()))?;
let rel = rel.join(entry.file_name());
if is_cache(&rel) {
continue;
}
let from = src.join(&rel);
let to = dst.join(&rel);
let meta =
std::fs::symlink_metadata(&from).with_context(|| format!("stat {}", from.display()))?;
let kind = meta.file_type();
if kind.is_dir() {
std::fs::create_dir(&to).with_context(|| format!("create {}", to.display()))?;
copy_tree(src, dst, &rel, uid, gid)?;
lchown(&to, Some(uid), Some(gid)).with_context(|| format!("chown {}", to.display()))?;
set_mode(&to, &meta)?;
} else if kind.is_symlink() {
let target = std::fs::read_link(&from)
.with_context(|| format!("read link {}", from.display()))?;
symlink(&target, &to).with_context(|| format!("link {}", to.display()))?;
lchown(&to, Some(uid), Some(gid)).with_context(|| format!("chown {}", to.display()))?;
} else if kind.is_file() {
std::fs::copy(&from, &to)
.with_context(|| format!("copy {} to {}", from.display(), to.display()))?;
if let Ok(mtime) = meta.modified() {
std::fs::File::options()
.write(true)
.open(&to)
.and_then(|f| f.set_modified(mtime))
.with_context(|| format!("set mtime of {}", to.display()))?;
}
lchown(&to, Some(uid), Some(gid)).with_context(|| format!("chown {}", to.display()))?;
set_mode(&to, &meta)?;
} else {
tracing::info!(path = %from.display(), "salvage skips a socket, fifo or device node");
}
}
Ok(())
}
/// Give `path` the permission bits of `meta`, which `chown` may have cleared.
fn set_mode(path: &Path, meta: &std::fs::Metadata) -> Result<()> {
std::fs::set_permissions(path, std::fs::Permissions::from_mode(meta.mode() & 0o7777))
.with_context(|| format!("chmod {}", path.display()))
}
/// The root's machine id when `etc/machine-id` is a regular file holding one.
/// The container may have replaced `etc` or the file by a link, so neither is
/// followed.
fn read_machine_id(root: &Path) -> Option<String> {
let etc = std::fs::symlink_metadata(root.join("etc")).ok()?;
let path = root.join("etc/machine-id");
if !etc.is_dir() || !is_regular(&path) {
return None;
}
let text = std::fs::read_to_string(path).ok()?;
let id = text.trim_end_matches('\n');
(id.len() == 32 && id.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')))
.then(|| id.to_owned())
}
/// Give the template a machine id when it has none: with an empty
/// `etc/machine-id` the guest makes a new one on every boot, and its journal
/// directory and DHCP identity change with it.
fn ensure_machine_id(root: &Path) -> Result<()> {
if read_machine_id(root).is_some() {
return Ok(());
}
let uuid = std::fs::read_to_string("/proc/sys/kernel/random/uuid")
.context("read a random uuid for the machine id")?;
let id: String = uuid.trim().chars().filter(|c| *c != '-').collect();
write_container_etc_file_in(root, "machine-id", &format!("{id}\n"))
}
fn is_regular(path: &Path) -> bool {
std::fs::symlink_metadata(path).is_ok_and(|m| m.is_file())
}
/// `systemctl is-active` answers `inactive` or `failed` for a unit that is not
/// running; anything else (`active`, `activating`, `deactivating`) means the
/// root is, or is about to be, in use.
fn unit_is_stopped(container: &str) -> Result<bool> {
let out = std::process::Command::new("systemctl")
.args(["is-active", &format!("container@{container}.service")])
.output()
.context("invoke systemctl is-active")?;
let state = String::from_utf8_lossy(&out.stdout);
Ok(matches!(state.trim(), "inactive" | "failed"))
}
/// Today in UTC as `YYYYMMDD`.
fn today() -> String {
let secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_secs());
date_of(secs)
}
/// `YYYYMMDD` of a unix time, by Howard Hinnant's days-to-civil algorithm.
fn date_of(secs: u64) -> String {
let z = i64::try_from(secs / 86_400).unwrap_or(0) + 719_468;
let era = z.div_euclid(146_097);
let doe = z.rem_euclid(146_097);
let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let day = doy - (153 * mp + 2) / 5 + 1;
let month = if mp < 10 { mp + 3 } else { mp - 9 };
let year = yoe + era * 400 + i64::from(month <= 2);
format!("{year:04}{month:02}{day:02}")
}
#[cfg(test)]
mod tests {
use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _, lchown, symlink};
use std::path::{Path, PathBuf};
use super::{MARKER, Outcome, date_of, has_content, rotate};
use crate::shared::test_support::scratch;
const DATE: &str = "20261028";
/// A stopped agent's old root and work dir under one scratch dir. The
/// "agent" is the test's own user, the one it can chown to.
struct Fixture {
top: PathBuf,
root: PathBuf,
work: PathBuf,
profiles: PathBuf,
}
impl Fixture {
fn new() -> Self {
let top = scratch();
let root = top.join("h-iris");
let work = top.join("work");
let profiles = top.join("profiles");
std::fs::create_dir_all(&profiles).unwrap();
std::fs::create_dir_all(root.join("home/iris")).unwrap();
std::fs::create_dir_all(&work).unwrap();
Self {
top,
root,
work,
profiles,
}
}
fn put(&self, rel: &str, body: &str) {
let path = self.root.join(rel);
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(path, body).unwrap();
}
fn rotate(&self) -> Outcome {
rotate(&self.root, "iris", &self.work, &self.profiles, DATE).unwrap()
}
/// Where the container finds nix's per-user state, on the host.
fn profile(&self) -> PathBuf {
self.profiles.join("per-user/iris")
}
fn salvage(&self) -> PathBuf {
self.work.join(format!("salvaged-rootfs-{DATE}"))
}
fn names(dir: &Path) -> Vec<String> {
let mut v: Vec<String> = std::fs::read_dir(dir)
.unwrap()
.map(|e| e.unwrap().file_name().into_string().unwrap())
.collect();
v.sort();
v
}
}
impl Drop for Fixture {
fn drop(&mut self) {
std::fs::remove_dir_all(&self.top).ok();
}
}
/// The first call copies the agent's files and moves the old root aside;
/// the second finds a clean template and does nothing.
#[test]
fn salvages_once() {
let f = Fixture::new();
f.put("home/iris/notes.txt", "keep me");
f.put("etc/hostname", "iris");
let Outcome::Salvaged { salvage, old_root } = f.rotate() else {
panic!("the first call must salvage");
};
assert_eq!(salvage, f.salvage());
assert_eq!(
std::fs::read_to_string(salvage.join("home/iris/notes.txt")).unwrap(),
"keep me"
);
assert_eq!(
std::fs::read_to_string(old_root.join("home/iris/notes.txt")).unwrap(),
"keep me"
);
assert!(f.root.join("etc").join(MARKER).is_file());
assert!(
!f.root.join("home/iris").exists(),
"template holds the old home"
);
assert_eq!(f.rotate(), Outcome::Clean);
assert_eq!(
Fixture::names(&f.work),
[format!("salvaged-rootfs-{DATE}")],
"a second salvage dir appeared"
);
}
/// Content that appears in the template after the marker (a run by a
/// build without the flag) is salvaged again, into its own directory.
#[test]
fn content_after_the_marker_is_salvaged_again() {
let f = Fixture::new();
f.put("home/iris/a", "1");
f.rotate();
f.put("home/iris/b", "2");
let Outcome::Salvaged { salvage, .. } = f.rotate() else {
panic!("late content must be salvaged");
};
assert_eq!(salvage, f.work.join(format!("salvaged-rootfs-{DATE}-2")));
assert!(salvage.join("home/iris/b").is_file());
assert!(
f.salvage().join("home/iris/a").is_file(),
"first salvage kept"
);
}
/// The journal, `/tmp`, `/var/cache`, weston and the otel cursor stay out
/// of the salvage; a sibling of each does not. The root's own copies are
/// untouched.
#[test]
fn cache_paths_are_skipped_and_their_siblings_are_not() {
let f = Fixture::new();
f.put("home/iris/keep", "x");
for cache in [
"tmp/scratch",
"var/cache/nix/x",
"var/log/journal/id/system.journal",
"var/lib/weston/state",
"var/lib/opentelemetry-collector/cursor",
] {
f.put(cache, "cache");
}
for real in [
"var/log/other.log",
"var/lib/other/db",
"var/tmp/keep",
"root/.bash_history",
] {
f.put(real, "real");
}
let Outcome::Salvaged { salvage, old_root } = f.rotate() else {
panic!("must salvage");
};
for cache in [
"tmp",
"var/cache",
"var/log/journal",
"var/lib/weston",
"var/lib/opentelemetry-collector",
] {
assert!(!salvage.join(cache).exists(), "{cache} was copied");
assert!(old_root.join(cache).exists(), "{cache} left the old root");
}
for real in [
"var/log/other.log",
"var/lib/other/db",
"var/tmp/keep",
"root/.bash_history",
"home/iris/keep",
] {
assert!(salvage.join(real).is_file(), "{real} was not copied");
}
}
/// A root with only the cache paths and the start path's own files has
/// nothing to salvage: no directory in `work/`, no move.
#[test]
fn a_root_without_content_is_not_salvaged() {
let f = Fixture::new();
f.put("var/cache/x", "c");
f.put("etc/hyperhive-hive-facts", "A=b\n");
f.put("etc/resolv.conf", "nameserver 1.1.1.1\n");
assert_eq!(f.rotate(), Outcome::Clean);
assert!(f.work.read_dir().unwrap().next().is_none());
assert!(f.root.join("var/cache/x").is_file(), "root was touched");
assert!(f.root.join("etc").join(MARKER).is_file());
// The same call with one real file makes the control: it salvages.
f.put("home/iris/real", "x");
assert!(matches!(
rotate(&f.root, "iris", &f.work, &f.profiles, DATE).unwrap(),
Outcome::Salvaged { .. }
));
}
/// Nothing is removed: the old root keeps every file, and a failed
/// salvage leaves the root where it was.
#[test]
fn nothing_is_deleted() {
let f = Fixture::new();
f.put("home/iris/deep/er/file", "data");
f.put("etc/passwd", "iris");
let Outcome::Salvaged { old_root, .. } = f.rotate() else {
panic!("must salvage");
};
assert_eq!(
std::fs::read_to_string(old_root.join("home/iris/deep/er/file")).unwrap(),
"data"
);
assert_eq!(
std::fs::read_to_string(old_root.join("etc/passwd")).unwrap(),
"iris"
);
let g = Fixture::new();
g.put("home/iris/file", "data");
std::fs::remove_dir(&g.work).unwrap();
std::fs::write(&g.work, "not a dir").unwrap();
assert!(rotate(&g.root, "iris", &g.work, &g.profiles, DATE).is_err());
assert_eq!(
std::fs::read_to_string(g.root.join("home/iris/file")).unwrap(),
"data"
);
assert!(!g.root.join("etc").join(MARKER).exists());
}
/// A group of the test user other than `primary`, if it has one.
fn other_group(primary: u32) -> Option<u32> {
let mut groups = [0 as libc::gid_t; 64];
// SAFETY: `groups` has room for the 64 ids it declares.
let n = unsafe { libc::getgroups(64, groups.as_mut_ptr()) };
let n = usize::try_from(n).ok()?;
groups[..n].iter().copied().find(|g| *g != primary)
}
/// Modes, mtimes, symlinks and the owner reach the salvage. The owner is
/// the test user, who can chown only to itself: the check pins that the
/// requested id lands on files, directories and links alike.
#[test]
fn modes_links_and_owner_are_kept() {
let f = Fixture::new();
f.put("home/iris/run.sh", "#!/bin/sh\n");
std::fs::set_permissions(
f.root.join("home/iris/run.sh"),
std::fs::Permissions::from_mode(0o750),
)
.unwrap();
std::fs::set_permissions(
f.root.join("home/iris"),
std::fs::Permissions::from_mode(0o700),
)
.unwrap();
symlink("run.sh", f.root.join("home/iris/link")).unwrap();
let mtime = std::fs::metadata(f.root.join("home/iris/run.sh"))
.unwrap()
.mtime();
// An unprivileged user can chown a file to any group it is in, so a
// second group gives the owner something to differ from the group new
// files get. Without one the gid check below cannot tell a chown from
// none.
let primary = std::fs::metadata(&f.work).unwrap().gid();
if let Some(other) = other_group(primary) {
lchown(f.root.join("home/iris"), None, Some(other)).unwrap();
}
let Outcome::Salvaged { salvage, old_root } = f.rotate() else {
panic!("must salvage");
};
let me = std::fs::metadata(old_root.join("home/iris")).unwrap();
let file = std::fs::symlink_metadata(salvage.join("home/iris/run.sh")).unwrap();
assert_eq!(file.mode() & 0o7777, 0o750);
assert_eq!(file.mtime(), mtime);
assert_eq!(
std::fs::symlink_metadata(salvage.join("home/iris"))
.unwrap()
.mode()
& 0o7777,
0o700
);
assert_eq!(
std::fs::read_link(salvage.join("home/iris/link")).unwrap(),
Path::new("run.sh")
);
for rel in [
"",
"home",
"home/iris",
"home/iris/run.sh",
"home/iris/link",
] {
let meta = std::fs::symlink_metadata(salvage.join(rel)).unwrap();
assert_eq!((meta.uid(), meta.gid()), (me.uid(), me.gid()), "{rel:?}");
}
}
/// The template keeps the old root's machine id; one with none gets a
/// fresh id that the next call keeps.
#[test]
fn the_template_has_a_stable_machine_id() {
let f = Fixture::new();
f.put("home/iris/a", "1");
f.put("etc/machine-id", "0123456789abcdef0123456789abcdef\n");
f.rotate();
let id = std::fs::read_to_string(f.root.join("etc/machine-id")).unwrap();
assert_eq!(id, "0123456789abcdef0123456789abcdef\n");
let g = Fixture::new();
g.put("etc/machine-id", "");
g.rotate();
let fresh = std::fs::read_to_string(g.root.join("etc/machine-id")).unwrap();
assert_eq!(fresh.trim().len(), 32, "{fresh:?}");
g.rotate();
assert_eq!(
std::fs::read_to_string(g.root.join("etc/machine-id")).unwrap(),
fresh
);
}
/// `etc` replaced by a link to a directory holding a machine-id is not
/// followed.
#[test]
fn a_linked_etc_is_not_followed() {
let f = Fixture::new();
let outside = f.top.join("outside");
std::fs::create_dir_all(&outside).unwrap();
std::fs::write(
outside.join("machine-id"),
"0123456789abcdef0123456789abcdef\n",
)
.unwrap();
symlink(&outside, f.root.join("etc")).unwrap();
f.put("home/iris/a", "1");
f.rotate();
let id = std::fs::read_to_string(f.root.join("etc/machine-id")).unwrap();
assert_ne!(id, "0123456789abcdef0123456789abcdef\n");
}
/// A root that is gone has nothing to salvage and gets a template.
#[test]
fn a_missing_root_gets_a_template() {
let f = Fixture::new();
std::fs::remove_dir_all(&f.root).unwrap();
assert!(!has_content(&f.root).unwrap());
assert_eq!(f.rotate(), Outcome::Clean);
assert!(f.root.join("etc").join(MARKER).is_file());
}
/// What nix keeps for the agent user is copied to where the container finds
/// it next time, with its links intact; the old root keeps its own copy. A
/// directory already there is not overwritten, and a linked `.local` is not
/// followed.
#[test]
fn the_nix_profile_is_carried_over() {
let f = Fixture::new();
f.put("home/iris/.local/state/nix/profiles/profile-1-link", "gen");
symlink(
"profile-1-link",
f.root.join("home/iris/.local/state/nix/profiles/profile"),
)
.unwrap();
let Outcome::Salvaged { old_root, .. } = f.rotate() else {
panic!("must salvage");
};
let link = f.profile().join("profiles/profile");
assert_eq!(
std::fs::read_link(&link).unwrap(),
Path::new("profile-1-link")
);
assert_eq!(std::fs::read_to_string(&link).unwrap(), "gen");
assert!(
old_root
.join("home/iris/.local/state/nix/profiles/profile-1-link")
.is_file()
);
assert_eq!(
std::fs::metadata(f.profile()).unwrap().uid(),
std::fs::metadata(&f.work).unwrap().uid()
);
// An existing target stays as it is.
let g = Fixture::new();
g.put("home/iris/.local/state/nix/profiles/profile-1-link", "old");
std::fs::create_dir_all(g.profile()).unwrap();
std::fs::write(g.profile().join("mine"), "new").unwrap();
g.rotate();
assert_eq!(Fixture::names(&g.profile()), ["mine"]);
// An empty one, as the agent's tmpfiles rules leave it, is filled.
let e = Fixture::new();
e.put("home/iris/.local/state/nix/profiles/profile-1-link", "old");
std::fs::create_dir_all(e.profile()).unwrap();
e.rotate();
assert_eq!(Fixture::names(&e.profile()), ["profiles"]);
// A `.local` that is a link to a directory with the same layout.
let h = Fixture::new();
let outside = h.top.join("outside");
std::fs::create_dir_all(outside.join("state/nix")).unwrap();
std::fs::write(outside.join("state/nix/secret"), "host").unwrap();
symlink(&outside, h.root.join("home/iris/.local")).unwrap();
h.put("home/iris/other", "x");
h.rotate();
assert!(!h.profile().exists(), "followed a linked .local");
}
/// `per-user` or the agent's directory in it replaced by a link, as the
/// container can do: nothing lands where the link points, and the salvage
/// itself still completes.
#[test]
fn a_linked_profile_dir_is_not_followed() {
for linked in ["per-user", "per-user/iris"] {
let f = Fixture::new();
f.put("home/iris/.local/state/nix/profiles/profile-1-link", "gen");
let outside = f.top.join("outside");
std::fs::create_dir_all(&outside).unwrap();
if linked == "per-user/iris" {
std::fs::create_dir_all(f.profiles.join("per-user")).unwrap();
}
symlink(&outside, f.profiles.join(linked)).unwrap();
assert!(matches!(f.rotate(), Outcome::Salvaged { .. }), "{linked}");
assert!(
outside.read_dir().unwrap().next().is_none(),
"{linked}: wrote through the link"
);
assert!(
f.salvage()
.join("home/iris/.local/state/nix/profiles/profile-1-link")
.is_file()
);
}
}
#[test]
fn dates() {
assert_eq!(date_of(0), "19700101");
assert_eq!(date_of(951_782_400), "20000229");
assert_eq!(date_of(1_793_145_600), "20261028");
}
}