vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

check the container root before each start

Browse source
This commit is contained in:
atlas 2026-10-28 20:57:27 +01:00 • committed by mara
commit a44a7256a7
1 changed files with 624 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

100
hive-priv/src/container_config.rs
View file

@ -65,7 +65,7 @@ pub(crate) fn handle_write_nspawn_flags(
/// the manager included) and return the logical agent name after it.
/// Anything else is refused, the swarm's forge and matrix containers
/// included: they belong to the host that runs them, not to a hive.
fn validate_container_system_name(name: &str) -> Result<&str> {
pub(crate) fn validate_container_system_name(name: &str) -> Result<&str> {
let Some(suffix) = name.strip_prefix(AGENT_PREFIX) else {
bail!("container name {name:?} is not managed by hive");
};
@ -88,6 +88,11 @@ pub(crate) fn remove_service_dropin(container: &str) -> Result<(String, String)>
/// container service's drop-in dir, together with a
/// `ConditionPathIsDirectory=` guard on the agent's MCP runtime dir.
///
/// The drop-in also carries the `ExecStartPre=` that runs
/// `hive-priv check-rootfs <container>` ([`crate::prestart`]) ahead of
/// nixpkgs' start script. It names this process's own binary, so the drop-in
/// and the binary that implements the check are always the same build.
///
/// Two different kinds of setting land in the same file. `MemoryMax=` /
/// `CPUQuota=` are hard caps that throttle even on an idle host;
/// `CPUWeight=` / `IOWeight=` are cgroup v2 relative shares that only
@ -113,7 +118,15 @@ pub(crate) fn write_resource_limits(
let dir = format!("/run/systemd/system/container@{container}.service.d");
std::fs::create_dir_all(&dir).with_context(|| format!("create {dir}"))?;
let path = format!("{dir}/hyperhive-limits.conf");
let content = limits_dropin_body(&runtime_dir, memory_max, cpu_quota, cpu_weight, io_weight);
let rootfs_check = rootfs_check_command(container)?;
let content = limits_dropin_body(
&runtime_dir,
&rootfs_check,
memory_max,
cpu_quota,
cpu_weight,
io_weight,
);
// Published whole: a `daemon-reload` between truncate and write would
// otherwise load the unit with its limits and start guard missing.
publish_file(Path::new(&path), content.as_bytes(), 0o644, None)?;
@ -149,6 +162,10 @@ const RESTART_SEC: u32 = 5;
/// Render the body of `hyperhive-limits.conf`.
///
/// `rootfs_check` is the command line of the pre-start root check, the first
/// `[Service]` line: it runs before the nixpkgs start script and a non-zero
/// exit fails the start.
///
/// `[Unit]`: the condition is checked at start time — it skips (not fails)
/// the unit when the MCP socket dir is absent, avoiding restart loops —
/// plus the bounded start limit (see the constants above; `StartLimit*` are
@ -160,6 +177,7 @@ const RESTART_SEC: u32 = 5;
/// existed — or one whose nix option is `null` — renders no weight lines.
fn limits_dropin_body(
runtime_dir: &str,
rootfs_check: &str,
memory_max: &str,
cpu_quota: &str,
cpu_weight: Option<u32>,
@ -177,6 +195,7 @@ fn limits_dropin_body(
StartLimitBurst={START_LIMIT_BURST}\n\
\n\
[Service]\n\
ExecStartPre={rootfs_check}\n\
RestartSec={RESTART_SEC}\n\
MemoryMax={memory_max}\n\
CPUQuota={cpu_quota}\n\
@ -184,6 +203,31 @@ fn limits_dropin_body(
)
}
/// The `ExecStartPre=` command of the pre-start root check for `container`:
/// this binary's absolute path, then the sub-command and the name.
///
/// The path is the running binary's, not a fixed one, so a drop-in never names
/// a build without the sub-command. systemd expands `%` and `$` in command
/// lines and splits on spaces, so a path with any of them is refused rather
/// than escaped; store paths never have them.
fn rootfs_check_command(container: &str) -> Result<String> {
let exe = std::env::current_exe().context("resolve the hive-priv binary path")?;
let exe = exe
.to_str()
.filter(|p| {
p.starts_with('/')
&& p.bytes()
.all(|b| b.is_ascii_alphanumeric() || b"/._+-".contains(&b))
})
.with_context(|| {
format!(
"hive-priv binary path {} is not safe in a unit file",
exe.display()
)
})?;
Ok(format!("{exe} {} {container}", crate::prestart::SUBCOMMAND))
}
/// `DaemonReload` — `systemctl daemon-reload` on the host.
pub(crate) async fn daemon_reload() -> Result<(String, String)> {
let out = Command::new("systemctl")
@ -543,10 +587,14 @@ mod tests {
use super::{
BindMount, HiveFacts, NetworkIsolation, git_overlay_flags, handle_write_nspawn_flags,
limits_dropin_body, nspawn_flags, publish_file, remove_service_dropin, render_hive_facts,
validate_container_system_name, write_bridge_dns_marker_in, write_resource_limits,
rootfs_check_command, validate_container_system_name, write_bridge_dns_marker_in,
write_resource_limits,
};
use crate::shared::test_support::{entries, scratch};
/// A stand-in for the pre-start check's command line.
const CHECK: &str = "/store/hive-priv/bin/hive-priv check-rootfs h-iris";
fn bind(container_path: &str) -> BindMount {
BindMount {
host_path: "/var/lib/hyperhive/whatever".to_owned(),
@ -604,13 +652,21 @@ mod tests {
#[test]
fn dropin_body_is_pinned_exactly() {
assert_eq!(
limits_dropin_body("/run/hyperhive/agents/iris", "4G", "200%", None, None),
limits_dropin_body(
"/run/hyperhive/agents/iris",
CHECK,
"4G",
"200%",
None,
None
),
"[Unit]\n\
ConditionPathIsDirectory=/run/hyperhive/agents/iris\n\
StartLimitIntervalSec=600\n\
StartLimitBurst=3\n\
\n\
[Service]\n\
ExecStartPre=/store/hive-priv/bin/hive-priv check-rootfs h-iris\n\
RestartSec=5\n\
MemoryMax=4G\n\
CPUQuota=200%\n"
@ -628,7 +684,7 @@ mod tests {
("512M", "50%", Some(10), Some(10)),
("infinity", "infinity", Some(10_000), None),
] {
let body = limits_dropin_body("/rt/x", mem, cpu, cw, iw);
let body = limits_dropin_body("/rt/x", CHECK, mem, cpu, cw, iw);
let unit = body
.split("[Service]")
.next()
@ -648,20 +704,20 @@ mod tests {
/// independently when `None`.
#[test]
fn weights_are_emitted_only_when_set() {
let both = limits_dropin_body("/rt/x", "4G", "200%", Some(80), Some(80));
let both = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), Some(80));
assert!(
both.ends_with("CPUQuota=200%\nCPUWeight=80\nIOWeight=80\n"),
"{both}"
);
let cpu_only = limits_dropin_body("/rt/x", "4G", "200%", Some(80), None);
let cpu_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), None);
assert!(
cpu_only.ends_with("CPUQuota=200%\nCPUWeight=80\n"),
"{cpu_only}"
);
assert!(!cpu_only.contains("IOWeight"), "{cpu_only}");
let io_only = limits_dropin_body("/rt/x", "4G", "200%", None, Some(80));
let io_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", None, Some(80));
assert!(
io_only.ends_with("CPUQuota=200%\nIOWeight=80\n"),
"{io_only}"
@ -669,6 +725,34 @@ mod tests {
assert!(!io_only.contains("CPUWeight"), "{io_only}");
}
/// The pre-start check is the first `[Service]` line, whatever else the
/// caller sets, and a drop-in without it is what an old hive-priv wrote.
#[test]
fn the_rootfs_check_leads_the_service_section_whatever_the_caller_passes() {
for (cw, iw) in [(None, None), (Some(80), Some(80))] {
let body = limits_dropin_body("/rt/x", CHECK, "4G", "200%", cw, iw);
let service = body
.split("[Service]\n")
.nth(1)
.expect("the body always has a [Service] section");
assert_eq!(
service.lines().next(),
Some("ExecStartPre=/store/hive-priv/bin/hive-priv check-rootfs h-iris")
);
assert_eq!(body.matches("ExecStartPre=").count(), 1, "{body}");
}
}
/// The command names an absolute path to this very binary, the
/// sub-command that binary implements and the container.
#[test]
fn the_check_command_names_this_binary_the_subcommand_and_the_container() {
let exe = std::env::current_exe().unwrap();
let cmd = rootfs_check_command("h-iris").unwrap();
assert_eq!(cmd, format!("{} check-rootfs h-iris", exe.display()));
assert!(cmd.starts_with('/'), "{cmd}");
}
/// The mode lands exactly as asked, whatever the umask would have made
/// of it (0666 under the usual 022 would otherwise come out 0644), and
/// the requested owner is on the file.