mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

budgeted agent containers get no swap; export oom kills and swap use

Browse source
This commit is contained in:
atlas 2026-11-11 01:58:30 +01:00
commit 4eb27046a0
3 changed files with 131 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

41
hive-c0re/src/stats/container_stats.rs
View file

@ -64,6 +64,13 @@ pub struct ContainerResource {
/// High-water memory usage since container start (`memory.peak`),
/// bytes. `None` if the kernel doesn't expose `memory.peak`.
pub mem_peak_bytes: Option<u64>,
/// Processes the kernel OOM-killed inside the container since it started
/// (`memory.events` `oom_kill`, cumulative, counting every descendant
/// cgroup). `None` if the file or the key is missing, never 0.
pub oom_kills: Option<u64>,
/// Memory currently swapped out (`memory.swap.current`), bytes. `None`
/// if the kernel doesn't expose it.
pub swap_current_bytes: Option<u64>,
/// Memory ceiling (`memory.max`), bytes. `None` when unlimited
/// (the file reads `max`).
pub mem_max_bytes: Option<u64>,
@ -168,6 +175,19 @@ fn read_mem_max(path: &std::path::Path) -> Option<u64> {
}
}
/// `oom_kill` from a cgroup's `memory.events`, the count of processes the
/// kernel killed for memory in the cgroup and everything below it.
fn read_oom_kills(dir: &std::path::Path) -> Option<u64> {
oom_kills_in(&std::fs::read_to_string(dir.join("memory.events")).ok()?)
}
/// The `oom_kill` value in `memory.events` text (`key value` lines).
fn oom_kills_in(events: &str) -> Option<u64> {
events
.lines()
.find_map(|line| line.strip_prefix("oom_kill ")?.trim().parse::<u64>().ok())
}
/// Parse `usage_usec` (cumulative CPU time, microseconds) from a
/// scope's `cpu.stat`.
fn read_usage_usec(dir: &std::path::Path) -> Option<u64> {
@ -279,6 +299,8 @@ pub async fn gather() -> Vec<ContainerResource> {
cpu_time_usec: usage_now,
mem_current_bytes: read_u64(&dir.join("memory.current")).unwrap_or(0),
mem_peak_bytes: read_u64(&dir.join("memory.peak")),
oom_kills: read_oom_kills(dir),
swap_current_bytes: read_u64(&dir.join("memory.swap.current")),
mem_max_bytes: read_mem_max(&dir.join("memory.max")),
disk_bytes: agent.as_ref().and_then(|a| disk.get(a).copied().flatten()),
});
@ -338,6 +360,25 @@ mod tests {
assert_eq!(running_candidates(slice.path(), &[]).len(), 1);
}
const EVENTS: &str = "low 0\nhigh 0\nmax 12\noom 5\noom_kill 3\noom_group_kill 1\n";
#[test]
fn oom_kills_reads_the_oom_kill_key_not_its_neighbours() {
assert_eq!(oom_kills_in(EVENTS), Some(3));
assert_eq!(oom_kills_in("oom_kill 0\n"), Some(0));
// Neighbouring keys with the same prefix or value are not it.
assert_eq!(oom_kills_in("oom 5\noom_group_kill 1\n"), None);
assert_eq!(oom_kills_in("oom_kill x\n"), None);
}
#[test]
fn a_missing_memory_events_is_no_sample_not_zero() {
let dir = tempfile::tempdir().expect("tempdir");
assert_eq!(read_oom_kills(dir.path()), None);
// Control: the same reader finds the count once the file exists.
std::fs::write(dir.path().join("memory.events"), EVENTS).expect("write");
assert_eq!(read_oom_kills(dir.path()), Some(3));
}
#[test]
fn failed_disk_measurement_is_unknown_not_zero() {
let name = hive_types::Ident::parse("iris").expect("valid ident");

76
hive-c0re/src/stats/otel_metrics.rs
View file

@ -138,6 +138,59 @@ fn build_provider(
Ok(provider)
}
/// The custom (`hyperhive.`) memory series: peak, OOM kills and swap.
fn register_memory_series(
meter: &opentelemetry::metrics::Meter,
snapshot: &Arc<Mutex<Vec<ContainerResource>>>,
) {
let snap = snapshot.clone();
meter
.u64_observable_gauge("hyperhive.container.memory.peak")
.with_unit("By")
.with_callback(move |obs| {
if let Ok(g) = snap.lock() {
for c in g.iter() {
if let Some(peak) = c.mem_peak_bytes {
obs.observe(peak, &attrs(c));
}
}
}
})
.build();
// Processes the kernel OOM-killed in the container (cumulative). An agent
// with a RAM budget has no swap, so reaching the budget ends in a kill.
let snap = snapshot.clone();
meter
.u64_observable_counter("hyperhive.container.memory.oom_kills")
.with_unit("{process}")
.with_callback(move |obs| {
if let Ok(g) = snap.lock() {
for c in g.iter() {
if let Some(kills) = c.oom_kills {
obs.observe(kills, &attrs(c));
}
}
}
})
.build();
let snap = snapshot.clone();
meter
.u64_observable_gauge("hyperhive.container.memory.swap.usage")
.with_unit("By")
.with_callback(move |obs| {
if let Ok(g) = snap.lock() {
for c in g.iter() {
if let Some(swap) = c.swap_current_bytes {
obs.observe(swap, &attrs(c));
}
}
}
})
.build();
}
/// Register the observable instruments. Each callback reads the shared
/// snapshot and reports one data point per agent. Instruments are held by the
/// meter/provider (kept alive in `PROVIDER`).
@ -192,23 +245,10 @@ fn register_instruments(provider: &SdkMeterProvider, snapshot: Arc<Mutex<Vec<Con
})
.build();
// Custom (`hyperhive.`) metrics semconv doesn't standardise: memory peak,
// on-disk footprint, and the instantaneous cpu percent.
let snap = snapshot.clone();
meter
.u64_observable_gauge("hyperhive.container.memory.peak")
.with_unit("By")
.with_callback(move |obs| {
if let Ok(g) = snap.lock() {
for c in g.iter() {
if let Some(peak) = c.mem_peak_bytes {
obs.observe(peak, &attrs(c));
}
}
}
})
.build();
register_memory_series(&meter, &snapshot);
// Custom (`hyperhive.`) metrics semconv doesn't standardise: on-disk
// footprint and the instantaneous cpu percent.
let snap = snapshot.clone();
meter
.u64_observable_gauge("hyperhive.container.storage.usage")
@ -587,6 +627,8 @@ mod tests {
cpu_time_usec: None,
mem_current_bytes: 0,
mem_peak_bytes: None,
oom_kills: None,
swap_current_bytes: None,
mem_max_bytes: None,
disk_bytes: None,
};
@ -631,6 +673,8 @@ mod tests {
cpu_time_usec: None,
mem_current_bytes: 0,
mem_peak_bytes: None,
oom_kills: None,
swap_current_bytes: None,
mem_max_bytes: None,
disk_bytes: None,
};

30
hive-priv/src/container_config.rs
View file

@ -225,6 +225,10 @@ const RESTART_SEC: u32 = 5;
/// weights. A weight of `None` means "not configured" and omits its line
/// entirely, so a request from a hive-c0re built before the weights
/// existed — or one whose nix option is `null` — renders no weight lines.
///
/// A finite `MemoryMax=` comes with `MemorySwapMax=0`: at the budget the
/// kernel kills inside the container instead of paging it out to host swap.
/// `MemoryMax=infinity` is no budget and leaves swap alone.
fn limits_dropin_body(
runtime_dir: &str,
rootfs_check: &str,
@ -238,6 +242,11 @@ fn limits_dropin_body(
// `format_push_string`, and a `write!` would need an unwrap.
let cpu_weight_line = cpu_weight.map_or_else(String::new, |w| format!("CPUWeight={w}\n"));
let io_weight_line = io_weight.map_or_else(String::new, |w| format!("IOWeight={w}\n"));
let swap_line = if memory_max == "infinity" {
""
} else {
"MemorySwapMax=0\n"
};
format!(
"[Unit]\n\
ConditionPathIsDirectory={runtime_dir}\n\
@ -248,6 +257,7 @@ fn limits_dropin_body(
ExecStartPre={rootfs_check}\n\
RestartSec={RESTART_SEC}\n\
MemoryMax={memory_max}\n\
{swap_line}\
CPUQuota={cpu_quota}\n\
{cpu_weight_line}{io_weight_line}"
)
@ -893,6 +903,7 @@ mod tests {
ExecStartPre=/store/hive-priv/bin/hive-priv check-rootfs h-iris\n\
RestartSec=5\n\
MemoryMax=4G\n\
MemorySwapMax=0\n\
CPUQuota=200%\n"
);
}
@ -924,6 +935,25 @@ mod tests {
}
}
/// A budgeted container gets no swap; an unbudgeted one keeps it. The
/// finite arm is the presence control for the infinity arm's absence.
#[test]
fn swap_is_off_exactly_when_the_memory_is_budgeted() {
for mem in ["4G", "512M", "50%", "8589934592"] {
let body = limits_dropin_body("/rt/x", CHECK, mem, "200%", None, None);
assert!(
body.contains(&format!("\nMemoryMax={mem}\nMemorySwapMax=0\n")),
"no swap line under a budget of {mem}: {body}"
);
assert_eq!(body.matches("MemorySwapMax=").count(), 1, "{body}");
}
let unbudgeted = limits_dropin_body("/rt/x", CHECK, "infinity", "200%", None, None);
assert!(
unbudgeted.contains("\nMemoryMax=infinity\n"),
"{unbudgeted}"
);
assert!(!unbudgeted.contains("MemorySwapMax"), "{unbudgeted}");
}
/// Weights are appended to the `[Service]` section, each omitted
/// independently when `None`.
#[test]