mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

budgeted agents throttle at 90% of the cap; alert on repeated oom kills

Browse source
This commit is contained in:
atlas 2026-11-11 02:38:53 +01:00
commit 11393bc538
2 changed files with 201 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

51
hive-priv/src/container_config.rs
View file

@ -156,6 +156,7 @@ pub(crate) fn write_resource_limits(
cpu_quota: &str,
cpu_weight: Option<u32>,
io_weight: Option<u32>,
memory_high: Option<u64>,
) -> Result<(String, String)> {
let logical = validate_container_system_name(container)?;
validate_memory_max(memory_max)?;
@ -176,6 +177,7 @@ pub(crate) fn write_resource_limits(
cpu_quota,
cpu_weight,
io_weight,
memory_high,
);
// Published whole: a `daemon-reload` between truncate and write would
// otherwise load the unit with its limits and start guard missing.
@ -226,6 +228,10 @@ const RESTART_SEC: u32 = 5;
/// entirely, so a request from a hive-c0re built before the weights
/// existed — or one whose nix option is `null` — renders no weight lines.
///
/// `memory_high` is the byte count of `MemoryHigh=`: the throttle point below
/// the cap, where the kernel slows the container and reclaims from it. `None`
/// omits the line.
///
/// A finite `MemoryMax=` comes with `MemorySwapMax=0`: at the budget the
/// kernel kills inside the container instead of paging it out to host swap.
/// `MemoryMax=infinity` is no budget and leaves swap alone.
@ -236,12 +242,14 @@ fn limits_dropin_body(
cpu_quota: &str,
cpu_weight: Option<u32>,
io_weight: Option<u32>,
memory_high: Option<u64>,
) -> String {
// Built as two possibly-empty lines rather than pushed onto the
// string: `format!` appended to a `String` trips clippy::pedantic's
// `format_push_string`, and a `write!` would need an unwrap.
let cpu_weight_line = cpu_weight.map_or_else(String::new, |w| format!("CPUWeight={w}\n"));
let io_weight_line = io_weight.map_or_else(String::new, |w| format!("IOWeight={w}\n"));
let high_line = memory_high.map_or_else(String::new, |b| format!("MemoryHigh={b}\n"));
let swap_line = if memory_max == "infinity" {
""
} else {
@ -257,6 +265,7 @@ fn limits_dropin_body(
ExecStartPre={rootfs_check}\n\
RestartSec={RESTART_SEC}\n\
MemoryMax={memory_max}\n\
{high_line}\
{swap_line}\
CPUQuota={cpu_quota}\n\
{cpu_weight_line}{io_weight_line}"
@ -892,6 +901,7 @@ mod tests {
"4G",
"200%",
None,
None,
None
),
"[Unit]\n\
@ -919,7 +929,7 @@ mod tests {
("512M", "50%", Some(10), Some(10)),
("infinity", "infinity", Some(10_000), None),
] {
let body = limits_dropin_body("/rt/x", CHECK, mem, cpu, cw, iw);
let body = limits_dropin_body("/rt/x", CHECK, mem, cpu, cw, iw, None);
let unit = body
.split("[Service]")
.next()
@ -940,38 +950,59 @@ mod tests {
#[test]
fn swap_is_off_exactly_when_the_memory_is_budgeted() {
for mem in ["4G", "512M", "50%", "8589934592"] {
let body = limits_dropin_body("/rt/x", CHECK, mem, "200%", None, None);
let body = limits_dropin_body("/rt/x", CHECK, mem, "200%", None, None, None);
assert!(
body.contains(&format!("\nMemoryMax={mem}\nMemorySwapMax=0\n")),
"no swap line under a budget of {mem}: {body}"
);
assert_eq!(body.matches("MemorySwapMax=").count(), 1, "{body}");
}
let unbudgeted = limits_dropin_body("/rt/x", CHECK, "infinity", "200%", None, None);
let unbudgeted = limits_dropin_body("/rt/x", CHECK, "infinity", "200%", None, None, None);
assert!(
unbudgeted.contains("\nMemoryMax=infinity\n"),
"{unbudgeted}"
);
assert!(!unbudgeted.contains("MemorySwapMax"), "{unbudgeted}");
}
/// `MemoryHigh=` is the byte count the caller sends, between `MemoryMax=`
/// and the swap line, and absent when none is sent.
#[test]
fn memory_high_is_rendered_only_when_sent() {
let with = limits_dropin_body(
"/rt/x",
CHECK,
"4G",
"200%",
None,
None,
Some(3_865_470_566),
);
assert!(
with.contains("\nMemoryMax=4G\nMemoryHigh=3865470566\nMemorySwapMax=0\n"),
"{with}"
);
let without = limits_dropin_body("/rt/x", CHECK, "4G", "200%", None, None, None);
assert!(!without.contains("MemoryHigh"), "{without}");
}
/// Weights are appended to the `[Service]` section, each omitted
/// independently when `None`.
#[test]
fn weights_are_emitted_only_when_set() {
let both = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), Some(80));
let both = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), Some(80), None);
assert!(
both.ends_with("CPUQuota=200%\nCPUWeight=80\nIOWeight=80\n"),
"{both}"
);
let cpu_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), None);
let cpu_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), None, None);
assert!(
cpu_only.ends_with("CPUQuota=200%\nCPUWeight=80\n"),
"{cpu_only}"
);
assert!(!cpu_only.contains("IOWeight"), "{cpu_only}");
let io_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", None, Some(80));
let io_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", None, Some(80), None);
assert!(
io_only.ends_with("CPUQuota=200%\nIOWeight=80\n"),
"{io_only}"
@ -984,7 +1015,7 @@ mod tests {
#[test]
fn the_rootfs_check_leads_the_service_section_whatever_the_caller_passes() {
for (cw, iw) in [(None, None), (Some(80), Some(80))] {
let body = limits_dropin_body("/rt/x", CHECK, "4G", "200%", cw, iw);
let body = limits_dropin_body("/rt/x", CHECK, "4G", "200%", cw, iw, None);
let service = body
.split("[Service]\n")
.nth(1)
@ -1405,7 +1436,7 @@ mod tests {
let errs = [
handle_write_nspawn_flags(name, &[], &isolation, &[], &facts, None, None)
.unwrap_err(),
write_resource_limits(name, "4G", "200%", None, None).unwrap_err(),
write_resource_limits(name, "4G", "200%", None, None, None).unwrap_err(),
remove_service_dropin(name).unwrap_err(),
];
for e in errs {
@ -1471,7 +1502,7 @@ mod tests {
"$(id)",
"%t",
] {
let err = write_resource_limits(PROBE, bad, "200%", None, None).unwrap_err();
let err = write_resource_limits(PROBE, bad, "200%", None, None, None).unwrap_err();
assert!(
err.to_string().contains("invalid memory_max"),
"{bad:?}: {err:#}"
@ -1491,7 +1522,7 @@ mod tests {
"4G",
"%t",
] {
let err = write_resource_limits(PROBE, "4G", bad, None, None).unwrap_err();
let err = write_resource_limits(PROBE, "4G", bad, None, None, None).unwrap_err();
assert!(
err.to_string().contains("invalid cpu_quota"),
"{bad:?}: {err:#}"

8
hive-priv/src/main.rs
View file

@ -257,8 +257,14 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String,
ref cpu_quota,
cpu_weight,
io_weight,
memory_high,
} => container_config::write_resource_limits(
container, memory_max, cpu_quota, cpu_weight, io_weight,
container,
memory_max,
cpu_quota,
cpu_weight,
io_weight,
memory_high,
),
PrivRequest::RemoveServiceDropin { ref container } => {