vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

check every container's root before its start, through the container@ template

Browse source
This commit is contained in:
atlas 2026-10-28 23:15:03 +01:00
commit c191975aab
1 changed files with 179 additions and 339 deletions
Download patch file Download diff file Expand all files Collapse all files

247
hive-priv/src/prestart.rs
View file

@ -3,33 +3,28 @@
//! nixpkgs' `container@.service` start script runs as host root against the
//! container's own root, which is persistent and writable by the container.
//! `mkdir -p`, `chmod`, `touch` and `cp --remove-destination` there follow a
//! link the container planted at any of the names the script touches. The
//! per-container drop-in ([`crate::container_config`]) puts
//! `hive-priv check-rootfs <container>` ahead of that script as an
//! `ExecStartPre=`; a non-zero exit fails the start. The container is stopped
//! link the container planted at any of the names the script touches. A
//! drop-in of the `container@` template
//! (`nix/host-modules/container-rootfs-check.nix`) runs
//! `hive-priv check-rootfs %i` as an `ExecStartPre=` ahead of that script for
//! every instance; a non-zero exit fails the start. The container is stopped
//! while this runs, so nothing changes under the walk.
//!
//! Every component is looked at with `symlink_metadata`: a link is reported,
//! never followed. Nothing is deleted or repaired, so whatever is there stays
//! for the operator.
//!
//! The drop-in names the running hive-priv binary, which nothing else roots
//! once the system has moved on to a newer build. [`pin_binary`] keeps a GC
//! root on it for as long as the drop-in exists, and [`unpin_binary`] drops
//! the root with the drop-in.
//!
//! The entry point is [`run`]; [`check`] holds the rules.
use std::fs::{FileType, Metadata};
use std::os::unix::fs::{DirBuilderExt as _, symlink};
use std::path::{Path, PathBuf};
use anyhow::{Context as _, Result, bail};
use crate::container_config::{container_rootfs, validate_container_system_name};
use crate::shared::partial_name;
use crate::container_config::container_rootfs;
use crate::shared::validate_name_chars;
/// The sub-command the drop-in's `ExecStartPre=` names.
/// The sub-command the template drop-in's `ExecStartPre=` names.
pub(crate) const SUBCOMMAND: &str = "check-rootfs";
/// The per-container nix profile directories on the host.
@ -64,74 +59,6 @@ const REGULAR_ONLY: [&str; 2] = ["etc/machine-id", "etc/resolv.conf"];
/// `/etc/static` once it has booted.
const OS_RELEASE: &str = "etc/os-release";
/// Where hive-priv roots the build named by each container's `ExecStartPre=`:
/// `<container>` is a symlink to that build's store path. The directory is
/// under the host's `/nix/var/nix/gcroots`, not `per-container`, which is the
/// only part of it a container is given (`--bind=.../gcroots/per-container/<c>`
/// in the nixpkgs start script), so a container can't reach it.
const GCROOTS: &str = "/nix/var/nix/gcroots/hyperhive-check-rootfs";
/// Root the store path of `exe` for `container`, replacing the root a previous
/// write left, so the build a drop-in names outlives a garbage collection for
/// as long as the drop-in does. One link per container, so they don't pile up.
pub(crate) fn pin_binary(container: &str, exe: &Path) -> Result<()> {
pin_binary_in(Path::new(GCROOTS), Path::new(STORE), container, exe)
}
/// Drop the root [`pin_binary`] left for `container`, once its drop-in is gone.
/// A name that holds anything but a link is left alone and reported.
pub(crate) fn unpin_binary(container: &str) -> Result<()> {
unpin_binary_in(Path::new(GCROOTS), container)
}
/// [`pin_binary`] with the roots directory and the store as parameters.
///
/// A binary outside `store` can't be collected, so it gets no root. The link
/// is made under a temporary name and renamed over `<container>`: `rename`
/// replaces whatever link is there without following it.
fn pin_binary_in(roots: &Path, store: &Path, container: &str, exe: &Path) -> Result<()> {
let Some(entry) = exe
.strip_prefix(store)
.ok()
.and_then(|rest| rest.components().next())
else {
return Ok(());
};
let target = store.join(entry);
std::fs::DirBuilder::new()
.recursive(true)
.mode(0o755)
.create(roots)
.with_context(|| format!("create {}", roots.display()))?;
if !lstat(roots)?.is_some_and(|m| m.file_type().is_dir()) {
bail!("{} is not a real directory", roots.display());
}
let tmp = roots.join(partial_name(container));
symlink(&target, &tmp).with_context(|| format!("create {}", tmp.display()))?;
let root = roots.join(container);
std::fs::rename(&tmp, &root).map_err(|e| {
// Best effort: a leftover temp is a root too, but a harmless one.
let _ = std::fs::remove_file(&tmp);
anyhow::Error::new(e).context(format!("replace {}", root.display()))
})
}
/// [`unpin_binary`] with the roots directory as a parameter.
fn unpin_binary_in(roots: &Path, container: &str) -> Result<()> {
let root = roots.join(container);
match lstat(&root)? {
None => Ok(()),
Some(meta) if meta.file_type().is_symlink() => {
std::fs::remove_file(&root).with_context(|| format!("remove {}", root.display()))
}
Some(meta) => bail!(
"{} is {}, not a link: left alone",
root.display(),
kind(meta.file_type())
),
}
}
/// Handle `hive-priv <subcommand> ...` when the first argument names one.
/// `None` means it is not a sub-command and the daemon starts.
///
@ -156,12 +83,21 @@ pub(crate) fn run(args: &[String]) -> Option<i32> {
/// [`check`] on the host paths of `container`.
fn check_container(container: &str) -> Result<()> {
validate_container_system_name(container)?;
check(
&container_rootfs(container),
&Path::new(PROFILES_ROOT).join(container).join("system"),
Path::new(STORE),
)
let (root, system) = container_paths(container)?;
check(&root, &system, Path::new(STORE))
}
/// The container's root and its profile's `system` link on the host.
///
/// The unit instance supplies the name and it selects both paths, so anything
/// but a plain container name is refused: lowercase ascii, digits and
/// hyphens. That holds no `/` and is never `.` or `..`.
fn container_paths(container: &str) -> Result<(PathBuf, PathBuf)> {
validate_name_chars(container)?;
Ok((
container_rootfs(container),
Path::new(PROFILES_ROOT).join(container).join("system"),
))
}
/// Check the paths the start script touches as root, returning an error that
@ -285,8 +221,8 @@ mod tests {
use std::os::unix::fs::symlink;
use std::path::PathBuf;
use super::{check, pin_binary_in, run, unpin_binary_in};
use crate::shared::test_support::{entries, scratch};
use super::{check, container_paths, run};
use crate::shared::test_support::scratch;
/// A scratch container: root, the profile link's path and a store with one
/// toplevel in it. Nothing is linked yet.
@ -514,110 +450,53 @@ mod tests {
assert!(target.is_dir());
}
/// A scratch store holding one build with a binary in it, a roots dir and
/// the binary's path.
fn pin_fixture() -> (PathBuf, PathBuf, PathBuf, PathBuf) {
let base = scratch();
let store = base.join("store");
let build = store.join("abc-hive-priv");
std::fs::create_dir_all(build.join("bin")).unwrap();
let exe = build.join("bin/hive-priv");
std::fs::write(&exe, "").unwrap();
(base.join("roots"), store, build, exe)
#[test]
fn a_plain_container_name_selects_its_root_and_profile() {
// Agent and non-agent names alike.
for name in ["h-iris", "hive-forge", "swarm-review-classifier"] {
let (root, system) = container_paths(name).unwrap();
assert_eq!(
root,
PathBuf::from(format!("/var/lib/nixos-containers/{name}"))
);
assert_eq!(
system,
PathBuf::from(format!("/nix/var/nix/profiles/per-container/{name}/system"))
);
}
}
#[test]
fn the_root_points_at_the_store_path_the_drop_in_names() {
let (roots, store, build, exe) = pin_fixture();
pin_binary_in(&roots, &store, "h-iris", &exe).unwrap();
let root = roots.join("h-iris");
assert!(std::fs::symlink_metadata(&root).unwrap().is_symlink());
// The build, not the binary inside it.
assert_eq!(std::fs::read_link(&root).unwrap(), build);
assert_eq!(std::fs::read_dir(&roots).unwrap().count(), 1);
fn a_name_that_is_not_a_plain_container_name_is_refused() {
for name in [
"",
".",
"..",
"../h-iris",
"a/b",
"/etc",
"h-iris/",
"H-iris",
"a.b",
"a b",
] {
assert!(container_paths(name).is_err(), "{name:?} was accepted");
}
}
#[test]
fn a_rewrite_replaces_the_root_and_leaves_no_temp() {
let (roots, store, build, exe) = pin_fixture();
let newer = store.join("def-hive-priv");
std::fs::create_dir_all(newer.join("bin")).unwrap();
pin_binary_in(&roots, &store, "h-iris", &exe).unwrap();
pin_binary_in(&roots, &store, "h-iris", &newer.join("bin/hive-priv")).unwrap();
pin_binary_in(&roots, &store, "h-bob", &exe).unwrap();
assert_eq!(std::fs::read_link(roots.join("h-iris")).unwrap(), newer);
assert_eq!(std::fs::read_link(roots.join("h-bob")).unwrap(), build);
assert_eq!(entries(&roots), ["h-bob", "h-iris"]);
}
#[test]
fn a_link_planted_at_the_root_name_is_replaced_not_followed() {
let (roots, store, build, exe) = pin_fixture();
let victim = victim();
std::fs::write(victim.join("keep"), "data").unwrap();
std::fs::create_dir_all(&roots).unwrap();
symlink(&victim, roots.join("h-iris")).unwrap();
pin_binary_in(&roots, &store, "h-iris", &exe).unwrap();
assert_eq!(std::fs::read_link(roots.join("h-iris")).unwrap(), build);
assert_eq!(
std::fs::read_to_string(victim.join("keep")).unwrap(),
"data"
);
assert_eq!(std::fs::read_dir(&victim).unwrap().count(), 1);
}
#[test]
fn a_roots_dir_that_is_a_link_is_refused() {
let (roots, store, _, exe) = pin_fixture();
let victim = victim();
symlink(&victim, &roots).unwrap();
let msg = format!(
"{:#}",
pin_binary_in(&roots, &store, "h-iris", &exe).unwrap_err()
);
assert!(msg.contains("is not a real directory"), "{msg}");
assert_eq!(std::fs::read_dir(&victim).unwrap().count(), 0);
}
#[test]
fn a_binary_outside_the_store_gets_no_root() {
let (roots, store, _, _) = pin_fixture();
let elsewhere = scratch().join("hive-priv");
pin_binary_in(&roots, &store, "h-iris", &elsewhere).unwrap();
assert!(!roots.exists());
}
#[test]
fn unpinning_removes_the_container_root_and_nothing_else() {
let (roots, store, build, exe) = pin_fixture();
pin_binary_in(&roots, &store, "h-iris", &exe).unwrap();
pin_binary_in(&roots, &store, "h-bob", &exe).unwrap();
unpin_binary_in(&roots, "h-iris").unwrap();
assert_eq!(entries(&roots), ["h-bob"]);
// Already gone: fine. The build it named is untouched.
unpin_binary_in(&roots, "h-iris").unwrap();
assert!(build.join("bin/hive-priv").is_file());
}
#[test]
fn unpinning_leaves_a_name_that_is_not_a_link_alone() {
let (roots, _, _, _) = pin_fixture();
std::fs::create_dir_all(roots.join("h-iris")).unwrap();
std::fs::write(roots.join("h-iris/keep"), "").unwrap();
let msg = format!("{:#}", unpin_binary_in(&roots, "h-iris").unwrap_err());
assert!(msg.contains("left alone"), "{msg}");
assert!(roots.join("h-iris/keep").exists());
}
#[test]
fn only_the_check_rootfs_sub_command_is_taken() {
assert_eq!(run(&[]), None);
assert_eq!(run(&["serve".to_string()]), None);
// Wrong arity and a name that is not an agent container are refused,
// whatever is on disk.
// Wrong arity and a name that selects a path outside the containers
// directory are refused, whatever is on disk.
assert_eq!(run(&["check-rootfs".to_string()]), Some(1));
assert_eq!(
run(&["check-rootfs".to_string(), "not-hive".to_string()]),
Some(1)
);
for name in ["", "..", "a/b"] {
assert_eq!(
run(&["check-rootfs".to_string(), name.to_string()]),
Some(1),
"{name:?}"
);
}
}
}