vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

check every container's root before its start, through the container@ template

Browse source
This commit is contained in:
atlas 2026-10-28 23:15:03 +01:00
commit c191975aab
3 changed files with 179 additions and 339 deletions
Download patch file Download diff file Expand all files Collapse all files

117
hive-priv/src/container_config.rs
View file

@ -91,7 +91,7 @@ pub(crate) async fn handle_write_nspawn_flags_blocking(
/// the manager included) and return the logical agent name after it.
/// Anything else is refused, the swarm's forge and matrix containers
/// included: they belong to the host that runs them, not to a hive.
pub(crate) fn validate_container_system_name(name: &str) -> Result<&str> {
fn validate_container_system_name(name: &str) -> Result<&str> {
let Some(suffix) = name.strip_prefix(AGENT_PREFIX) else {
bail!("container name {name:?} is not managed by hive");
};
@ -100,14 +100,13 @@ pub(crate) fn validate_container_system_name(name: &str) -> Result<&str> {
}
/// `RemoveServiceDropin` — remove the container service's drop-in dir
/// if present (idempotent), then the GC root of the build it named.
/// if present (idempotent).
pub(crate) fn remove_service_dropin(container: &str) -> Result<(String, String)> {
validate_container_system_name(container)?;
let dir = format!("/run/systemd/system/container@{container}.service.d");
if Path::new(&dir).exists() {
std::fs::remove_dir_all(&dir).with_context(|| format!("remove {dir}"))?;
}
crate::prestart::unpin_binary(container)?;
Ok((String::new(), String::new()))
}
@ -115,13 +114,6 @@ pub(crate) fn remove_service_dropin(container: &str) -> Result<(String, String)>
/// container service's drop-in dir, together with a
/// `ConditionPathIsDirectory=` guard on the agent's MCP runtime dir.
///
/// The drop-in also carries the `ExecStartPre=` that runs
/// `hive-priv check-rootfs <container>` ([`crate::prestart`]) ahead of
/// nixpkgs' start script. It names this process's own binary, so the drop-in
/// and the binary that implements the check are always the same build. The
/// build is rooted against garbage collection for as long as the drop-in
/// names it ([`crate::prestart::pin_binary`]).
///
/// Two different kinds of setting land in the same file. `MemoryMax=` /
/// `CPUQuota=` are hard caps that throttle even on an idle host;
/// `CPUWeight=` / `IOWeight=` are cgroup v2 relative shares that only
@ -147,19 +139,7 @@ pub(crate) fn write_resource_limits(
let dir = format!("/run/systemd/system/container@{container}.service.d");
std::fs::create_dir_all(&dir).with_context(|| format!("create {dir}"))?;
let path = format!("{dir}/hyperhive-limits.conf");
let exe = hive_priv_exe()?;
// Rooted before the drop-in names it, so the drop-in never names a build a
// collection can take.
crate::prestart::pin_binary(container, Path::new(&exe))?;
let rootfs_check = rootfs_check_command(&exe, container);
let content = limits_dropin_body(
&runtime_dir,
&rootfs_check,
memory_max,
cpu_quota,
cpu_weight,
io_weight,
);
let content = limits_dropin_body(&runtime_dir, memory_max, cpu_quota, cpu_weight, io_weight);
// Published whole: a `daemon-reload` between truncate and write would
// otherwise load the unit with its limits and start guard missing.
publish_file(Path::new(&path), content.as_bytes(), 0o644, None)?;
@ -195,10 +175,6 @@ const RESTART_SEC: u32 = 5;
/// Render the body of `hyperhive-limits.conf`.
///
/// `rootfs_check` is the command line of the pre-start root check, the first
/// `[Service]` line: it runs before the nixpkgs start script and a non-zero
/// exit fails the start.
///
/// `[Unit]`: the condition is checked at start time — it skips (not fails)
/// the unit when the MCP socket dir is absent, avoiding restart loops —
/// plus the bounded start limit (see the constants above; `StartLimit*` are
@ -210,7 +186,6 @@ const RESTART_SEC: u32 = 5;
/// existed — or one whose nix option is `null` — renders no weight lines.
fn limits_dropin_body(
runtime_dir: &str,
rootfs_check: &str,
memory_max: &str,
cpu_quota: &str,
cpu_weight: Option<u32>,
@ -228,7 +203,6 @@ fn limits_dropin_body(
StartLimitBurst={START_LIMIT_BURST}\n\
\n\
[Service]\n\
ExecStartPre={rootfs_check}\n\
RestartSec={RESTART_SEC}\n\
MemoryMax={memory_max}\n\
CPUQuota={cpu_quota}\n\
@ -236,36 +210,6 @@ fn limits_dropin_body(
)
}
/// This binary's absolute path, for the drop-in's `ExecStartPre=`.
///
/// The path is the running binary's, not a fixed one, so a drop-in never names
/// a build without the sub-command. systemd expands `%` and `$` in command
/// lines and splits on spaces, so a path with any of them is refused rather
/// than escaped; store paths never have them.
fn hive_priv_exe() -> Result<String> {
let exe = std::env::current_exe().context("resolve the hive-priv binary path")?;
let exe = exe
.to_str()
.filter(|p| {
p.starts_with('/')
&& p.bytes()
.all(|b| b.is_ascii_alphanumeric() || b"/._+-".contains(&b))
})
.with_context(|| {
format!(
"hive-priv binary path {} is not safe in a unit file",
exe.display()
)
})?;
Ok(exe.to_owned())
}
/// The `ExecStartPre=` command of the pre-start root check for `container`:
/// the binary `exe`, then the sub-command and the name.
fn rootfs_check_command(exe: &str, container: &str) -> String {
format!("{exe} {} {container}", crate::prestart::SUBCOMMAND)
}
/// `DaemonReload` — `systemctl daemon-reload` on the host.
pub(crate) async fn daemon_reload() -> Result<(String, String)> {
let out = Command::new("systemctl")
@ -624,15 +568,11 @@ pub(crate) fn write_container_etc_file_in(rootfs: &Path, name: &str, contents: &
mod tests {
use super::{
BindMount, HiveFacts, NetworkIsolation, git_overlay_flags, handle_write_nspawn_flags,
hive_priv_exe, limits_dropin_body, nspawn_flags, publish_file, remove_service_dropin,
render_hive_facts, rootfs_check_command, validate_container_system_name,
write_bridge_dns_marker_in, write_resource_limits,
limits_dropin_body, nspawn_flags, publish_file, remove_service_dropin, render_hive_facts,
validate_container_system_name, write_bridge_dns_marker_in, write_resource_limits,
};
use crate::shared::test_support::{entries, scratch};
/// A stand-in for the pre-start check's command line.
const CHECK: &str = "/store/hive-priv/bin/hive-priv check-rootfs h-iris";
fn bind(container_path: &str) -> BindMount {
BindMount {
host_path: "/var/lib/hyperhive/whatever".to_owned(),
@ -690,21 +630,13 @@ mod tests {
#[test]
fn dropin_body_is_pinned_exactly() {
assert_eq!(
limits_dropin_body(
"/run/hyperhive/agents/iris",
CHECK,
"4G",
"200%",
None,
None
),
limits_dropin_body("/run/hyperhive/agents/iris", "4G", "200%", None, None),
"[Unit]\n\
ConditionPathIsDirectory=/run/hyperhive/agents/iris\n\
StartLimitIntervalSec=600\n\
StartLimitBurst=3\n\
\n\
[Service]\n\
ExecStartPre=/store/hive-priv/bin/hive-priv check-rootfs h-iris\n\
RestartSec=5\n\
MemoryMax=4G\n\
CPUQuota=200%\n"
@ -722,7 +654,7 @@ mod tests {
("512M", "50%", Some(10), Some(10)),
("infinity", "infinity", Some(10_000), None),
] {
let body = limits_dropin_body("/rt/x", CHECK, mem, cpu, cw, iw);
let body = limits_dropin_body("/rt/x", mem, cpu, cw, iw);
let unit = body
.split("[Service]")
.next()
@ -742,55 +674,26 @@ mod tests {
/// independently when `None`.
#[test]
fn weights_are_emitted_only_when_set() {
let both = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), Some(80));
let both = limits_dropin_body("/rt/x", "4G", "200%", Some(80), Some(80));
assert!(
both.ends_with("CPUQuota=200%\nCPUWeight=80\nIOWeight=80\n"),
"{both}"
);
let cpu_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), None);
let cpu_only = limits_dropin_body("/rt/x", "4G", "200%", Some(80), None);
assert!(
cpu_only.ends_with("CPUQuota=200%\nCPUWeight=80\n"),
"{cpu_only}"
);
assert!(!cpu_only.contains("IOWeight"), "{cpu_only}");
let io_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", None, Some(80));
let io_only = limits_dropin_body("/rt/x", "4G", "200%", None, Some(80));
assert!(
io_only.ends_with("CPUQuota=200%\nIOWeight=80\n"),
"{io_only}"
);
assert!(!io_only.contains("CPUWeight"), "{io_only}");
}
/// The pre-start check is the first `[Service]` line, whatever else the
/// caller sets, and a drop-in without it is what an old hive-priv wrote.
#[test]
fn the_rootfs_check_leads_the_service_section_whatever_the_caller_passes() {
for (cw, iw) in [(None, None), (Some(80), Some(80))] {
let body = limits_dropin_body("/rt/x", CHECK, "4G", "200%", cw, iw);
let service = body
.split("[Service]\n")
.nth(1)
.expect("the body always has a [Service] section");
assert_eq!(
service.lines().next(),
Some("ExecStartPre=/store/hive-priv/bin/hive-priv check-rootfs h-iris")
);
assert_eq!(body.matches("ExecStartPre=").count(), 1, "{body}");
}
}
/// The command names an absolute path to this very binary, the
/// sub-command that binary implements and the container.
#[test]
fn the_check_command_names_this_binary_the_subcommand_and_the_container() {
let exe = std::env::current_exe().unwrap();
let cmd = rootfs_check_command(&hive_priv_exe().unwrap(), "h-iris");
assert_eq!(cmd, format!("{} check-rootfs h-iris", exe.display()));
assert!(cmd.starts_with('/'), "{cmd}");
}
/// The mode lands exactly as asked, whatever the umask would have made
/// of it (0666 under the usual 022 would otherwise come out 0644), and
/// the requested owner is on the file.

247
hive-priv/src/prestart.rs
View file

@ -3,33 +3,28 @@
//! nixpkgs' `container@.service` start script runs as host root against the
//! container's own root, which is persistent and writable by the container.
//! `mkdir -p`, `chmod`, `touch` and `cp --remove-destination` there follow a
//! link the container planted at any of the names the script touches. The
//! per-container drop-in ([`crate::container_config`]) puts
//! `hive-priv check-rootfs <container>` ahead of that script as an
//! `ExecStartPre=`; a non-zero exit fails the start. The container is stopped
//! link the container planted at any of the names the script touches. A
//! drop-in of the `container@` template
//! (`nix/host-modules/container-rootfs-check.nix`) runs
//! `hive-priv check-rootfs %i` as an `ExecStartPre=` ahead of that script for
//! every instance; a non-zero exit fails the start. The container is stopped
//! while this runs, so nothing changes under the walk.
//!
//! Every component is looked at with `symlink_metadata`: a link is reported,
//! never followed. Nothing is deleted or repaired, so whatever is there stays
//! for the operator.
//!
//! The drop-in names the running hive-priv binary, which nothing else roots
//! once the system has moved on to a newer build. [`pin_binary`] keeps a GC
//! root on it for as long as the drop-in exists, and [`unpin_binary`] drops
//! the root with the drop-in.
//!
//! The entry point is [`run`]; [`check`] holds the rules.
use std::fs::{FileType, Metadata};
use std::os::unix::fs::{DirBuilderExt as _, symlink};
use std::path::{Path, PathBuf};
use anyhow::{Context as _, Result, bail};
use crate::container_config::{container_rootfs, validate_container_system_name};
use crate::shared::partial_name;
use crate::container_config::container_rootfs;
use crate::shared::validate_name_chars;
/// The sub-command the drop-in's `ExecStartPre=` names.
/// The sub-command the template drop-in's `ExecStartPre=` names.
pub(crate) const SUBCOMMAND: &str = "check-rootfs";
/// The per-container nix profile directories on the host.
@ -64,74 +59,6 @@ const REGULAR_ONLY: [&str; 2] = ["etc/machine-id", "etc/resolv.conf"];
/// `/etc/static` once it has booted.
const OS_RELEASE: &str = "etc/os-release";
/// Where hive-priv roots the build named by each container's `ExecStartPre=`:
/// `<container>` is a symlink to that build's store path. The directory is
/// under the host's `/nix/var/nix/gcroots`, not `per-container`, which is the
/// only part of it a container is given (`--bind=.../gcroots/per-container/<c>`
/// in the nixpkgs start script), so a container can't reach it.
const GCROOTS: &str = "/nix/var/nix/gcroots/hyperhive-check-rootfs";
/// Root the store path of `exe` for `container`, replacing the root a previous
/// write left, so the build a drop-in names outlives a garbage collection for
/// as long as the drop-in does. One link per container, so they don't pile up.
pub(crate) fn pin_binary(container: &str, exe: &Path) -> Result<()> {
pin_binary_in(Path::new(GCROOTS), Path::new(STORE), container, exe)
}
/// Drop the root [`pin_binary`] left for `container`, once its drop-in is gone.
/// A name that holds anything but a link is left alone and reported.
pub(crate) fn unpin_binary(container: &str) -> Result<()> {
unpin_binary_in(Path::new(GCROOTS), container)
}
/// [`pin_binary`] with the roots directory and the store as parameters.
///
/// A binary outside `store` can't be collected, so it gets no root. The link
/// is made under a temporary name and renamed over `<container>`: `rename`
/// replaces whatever link is there without following it.
fn pin_binary_in(roots: &Path, store: &Path, container: &str, exe: &Path) -> Result<()> {
let Some(entry) = exe
.strip_prefix(store)
.ok()
.and_then(|rest| rest.components().next())
else {
return Ok(());
};
let target = store.join(entry);
std::fs::DirBuilder::new()
.recursive(true)
.mode(0o755)
.create(roots)
.with_context(|| format!("create {}", roots.display()))?;
if !lstat(roots)?.is_some_and(|m| m.file_type().is_dir()) {
bail!("{} is not a real directory", roots.display());
}
let tmp = roots.join(partial_name(container));
symlink(&target, &tmp).with_context(|| format!("create {}", tmp.display()))?;
let root = roots.join(container);
std::fs::rename(&tmp, &root).map_err(|e| {
// Best effort: a leftover temp is a root too, but a harmless one.
let _ = std::fs::remove_file(&tmp);
anyhow::Error::new(e).context(format!("replace {}", root.display()))
})
}
/// [`unpin_binary`] with the roots directory as a parameter.
fn unpin_binary_in(roots: &Path, container: &str) -> Result<()> {
let root = roots.join(container);
match lstat(&root)? {
None => Ok(()),
Some(meta) if meta.file_type().is_symlink() => {
std::fs::remove_file(&root).with_context(|| format!("remove {}", root.display()))
}
Some(meta) => bail!(
"{} is {}, not a link: left alone",
root.display(),
kind(meta.file_type())
),
}
}
/// Handle `hive-priv <subcommand> ...` when the first argument names one.
/// `None` means it is not a sub-command and the daemon starts.
///
@ -156,12 +83,21 @@ pub(crate) fn run(args: &[String]) -> Option<i32> {
/// [`check`] on the host paths of `container`.
fn check_container(container: &str) -> Result<()> {
validate_container_system_name(container)?;
check(
&container_rootfs(container),
&Path::new(PROFILES_ROOT).join(container).join("system"),
Path::new(STORE),
)
let (root, system) = container_paths(container)?;
check(&root, &system, Path::new(STORE))
}
/// The container's root and its profile's `system` link on the host.
///
/// The unit instance supplies the name and it selects both paths, so anything
/// but a plain container name is refused: lowercase ascii, digits and
/// hyphens. That holds no `/` and is never `.` or `..`.
fn container_paths(container: &str) -> Result<(PathBuf, PathBuf)> {
validate_name_chars(container)?;
Ok((
container_rootfs(container),
Path::new(PROFILES_ROOT).join(container).join("system"),
))
}
/// Check the paths the start script touches as root, returning an error that
@ -285,8 +221,8 @@ mod tests {
use std::os::unix::fs::symlink;
use std::path::PathBuf;
use super::{check, pin_binary_in, run, unpin_binary_in};
use crate::shared::test_support::{entries, scratch};
use super::{check, container_paths, run};
use crate::shared::test_support::scratch;
/// A scratch container: root, the profile link's path and a store with one
/// toplevel in it. Nothing is linked yet.
@ -514,110 +450,53 @@ mod tests {
assert!(target.is_dir());
}
/// A scratch store holding one build with a binary in it, a roots dir and
/// the binary's path.
fn pin_fixture() -> (PathBuf, PathBuf, PathBuf, PathBuf) {
let base = scratch();
let store = base.join("store");
let build = store.join("abc-hive-priv");
std::fs::create_dir_all(build.join("bin")).unwrap();
let exe = build.join("bin/hive-priv");
std::fs::write(&exe, "").unwrap();
(base.join("roots"), store, build, exe)
#[test]
fn a_plain_container_name_selects_its_root_and_profile() {
// Agent and non-agent names alike.
for name in ["h-iris", "hive-forge", "swarm-review-classifier"] {
let (root, system) = container_paths(name).unwrap();
assert_eq!(
root,
PathBuf::from(format!("/var/lib/nixos-containers/{name}"))
);
assert_eq!(
system,
PathBuf::from(format!("/nix/var/nix/profiles/per-container/{name}/system"))
);
}
}
#[test]
fn the_root_points_at_the_store_path_the_drop_in_names() {
let (roots, store, build, exe) = pin_fixture();
pin_binary_in(&roots, &store, "h-iris", &exe).unwrap();
let root = roots.join("h-iris");
assert!(std::fs::symlink_metadata(&root).unwrap().is_symlink());
// The build, not the binary inside it.
assert_eq!(std::fs::read_link(&root).unwrap(), build);
assert_eq!(std::fs::read_dir(&roots).unwrap().count(), 1);
fn a_name_that_is_not_a_plain_container_name_is_refused() {
for name in [
"",
".",
"..",
"../h-iris",
"a/b",
"/etc",
"h-iris/",
"H-iris",
"a.b",
"a b",
] {
assert!(container_paths(name).is_err(), "{name:?} was accepted");
}
}
#[test]
fn a_rewrite_replaces_the_root_and_leaves_no_temp() {
let (roots, store, build, exe) = pin_fixture();
let newer = store.join("def-hive-priv");
std::fs::create_dir_all(newer.join("bin")).unwrap();
pin_binary_in(&roots, &store, "h-iris", &exe).unwrap();
pin_binary_in(&roots, &store, "h-iris", &newer.join("bin/hive-priv")).unwrap();
pin_binary_in(&roots, &store, "h-bob", &exe).unwrap();
assert_eq!(std::fs::read_link(roots.join("h-iris")).unwrap(), newer);
assert_eq!(std::fs::read_link(roots.join("h-bob")).unwrap(), build);
assert_eq!(entries(&roots), ["h-bob", "h-iris"]);
}
#[test]
fn a_link_planted_at_the_root_name_is_replaced_not_followed() {
let (roots, store, build, exe) = pin_fixture();
let victim = victim();
std::fs::write(victim.join("keep"), "data").unwrap();
std::fs::create_dir_all(&roots).unwrap();
symlink(&victim, roots.join("h-iris")).unwrap();
pin_binary_in(&roots, &store, "h-iris", &exe).unwrap();
assert_eq!(std::fs::read_link(roots.join("h-iris")).unwrap(), build);
assert_eq!(
std::fs::read_to_string(victim.join("keep")).unwrap(),
"data"
);
assert_eq!(std::fs::read_dir(&victim).unwrap().count(), 1);
}
#[test]
fn a_roots_dir_that_is_a_link_is_refused() {
let (roots, store, _, exe) = pin_fixture();
let victim = victim();
symlink(&victim, &roots).unwrap();
let msg = format!(
"{:#}",
pin_binary_in(&roots, &store, "h-iris", &exe).unwrap_err()
);
assert!(msg.contains("is not a real directory"), "{msg}");
assert_eq!(std::fs::read_dir(&victim).unwrap().count(), 0);
}
#[test]
fn a_binary_outside_the_store_gets_no_root() {
let (roots, store, _, _) = pin_fixture();
let elsewhere = scratch().join("hive-priv");
pin_binary_in(&roots, &store, "h-iris", &elsewhere).unwrap();
assert!(!roots.exists());
}
#[test]
fn unpinning_removes_the_container_root_and_nothing_else() {
let (roots, store, build, exe) = pin_fixture();
pin_binary_in(&roots, &store, "h-iris", &exe).unwrap();
pin_binary_in(&roots, &store, "h-bob", &exe).unwrap();
unpin_binary_in(&roots, "h-iris").unwrap();
assert_eq!(entries(&roots), ["h-bob"]);
// Already gone: fine. The build it named is untouched.
unpin_binary_in(&roots, "h-iris").unwrap();
assert!(build.join("bin/hive-priv").is_file());
}
#[test]
fn unpinning_leaves_a_name_that_is_not_a_link_alone() {
let (roots, _, _, _) = pin_fixture();
std::fs::create_dir_all(roots.join("h-iris")).unwrap();
std::fs::write(roots.join("h-iris/keep"), "").unwrap();
let msg = format!("{:#}", unpin_binary_in(&roots, "h-iris").unwrap_err());
assert!(msg.contains("left alone"), "{msg}");
assert!(roots.join("h-iris/keep").exists());
}
#[test]
fn only_the_check_rootfs_sub_command_is_taken() {
assert_eq!(run(&[]), None);
assert_eq!(run(&["serve".to_string()]), None);
// Wrong arity and a name that is not an agent container are refused,
// whatever is on disk.
// Wrong arity and a name that selects a path outside the containers
// directory are refused, whatever is on disk.
assert_eq!(run(&["check-rootfs".to_string()]), Some(1));
assert_eq!(
run(&["check-rootfs".to_string(), "not-hive".to_string()]),
Some(1)
);
for name in ["", "..", "a/b"] {
assert_eq!(
run(&["check-rootfs".to_string(), name.to_string()]),
Some(1),
"{name:?}"
);
}
}
}

2
hive-priv/src/shared.rs
View file

@ -75,7 +75,7 @@ pub(crate) fn ensure_plain_filename(who: &str, filename: &str) -> Result<()> {
/// The leading dot and the `.partial` suffix are load-bearing: a reader
/// globbing on a published name's prefix, or systemd reading `*.conf` out of
/// `tmpfiles.d` and drop-in dirs, cannot pick up a half-written temp.
pub(crate) fn partial_name(filename: &str) -> String {
fn partial_name(filename: &str) -> String {
use std::sync::atomic::{AtomicU64, Ordering};
static SEQ: AtomicU64 = AtomicU64::new(0);
let n = SEQ.fetch_add(1, Ordering::Relaxed);