vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

agents: always write the memory cap, infinity for none

Browse source 
The cap file was removed for a cap of `infinity` or a RAM percentage,
so the boot check warned on every start of such an agent, for a
legitimate config. The file is now written on every start:

- a size: its byte count, as before;
- `infinity`: the literal `infinity`. The generators set no JSC heap
  ceiling and no subagent MemoryHigh=, and the boot check is quiet;
- a percentage: resolved on the host the way systemd resolves
  MemoryMax=N% for the container@ unit. That is permyriad (at most two
  decimals, at most 100%) of sysconf(_SC_PHYS_PAGES) *
  sysconf(_SC_PAGESIZE), rounded down to a page;
- a value systemd ignores (zero, over 100%, more than two decimals,
  unparseable) leaves the unit uncapped, so it is `infinity`.

The file is missing only when hive-c0re can't tell the cap: the
limits file and the kept drop-in are both unreadable, or a percentage
meets an unknown host memory size. The boot check now warns only for
a missing file or a first line that is neither a byte count nor
`infinity`.

Part of #5081 (slice S0-2).
This commit is contained in:
atlas 2026-10-12 10:35:15 +02:00
commit eaca3c4eb3
5 changed files with 253 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

241
hive-c0re/src/lifecycle/host_config.rs
View file

@ -31,6 +31,7 @@ pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Re
&limits_dropin_path(&container),
&paths.agent,
hive,
HostMemory::read(),
)? {
set_resource_limits(
&container,
@ -54,10 +55,40 @@ fn limits_dropin_path(container: &str) -> PathBuf {
}
/// File in the agent's runtime dir, `/run/hive/` in the container, holding the
/// byte count of the `MemoryMax=` its drop-in carries.
/// `nix/agent-modules/memory-cap.nix` reads it at boot under this name.
/// `MemoryMax=` its drop-in carries: a byte count, or `infinity` for no cap.
/// `nix/agent-modules/memory-cap.nix` reads it at boot under this name, and
/// treats a missing file as a fault.
const MEMORY_CAP_FILE: &str = "memory-max-bytes";
/// The host's physical memory and page size, the two numbers systemd resolves
/// a `MemoryMax=` percentage against.
#[derive(Clone, Copy, Debug)]
struct HostMemory {
bytes: u64,
page: u64,
}
impl HostMemory {
/// `sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE)`, as systemd's
/// `physical_memory()` computes it. systemd also caps that by the root
/// cgroup's `memory.max`, which the host's root cgroup does not have.
fn read() -> Option<Self> {
// SAFETY: `sysconf(3)` only reads system configuration.
let (pages, page) = unsafe {
(
libc::sysconf(libc::_SC_PHYS_PAGES),
libc::sysconf(libc::_SC_PAGESIZE),
)
};
let pages = u64::try_from(pages).ok().filter(|&p| p > 0)?;
let page = u64::try_from(page).ok().filter(|&p| p > 0)?;
Some(Self {
bytes: pages.checked_mul(page)?,
page,
})
}
}
/// [`limits_to_apply`], plus handing the memory cap to the agent through
/// [`MEMORY_CAP_FILE`] in `runtime_dir`. The container cannot read the cap
/// systemd enforces on it, so this file is how it learns the value it sizes
@ -69,20 +100,22 @@ fn stage_limits(
dropin_path: &Path,
runtime_dir: &Path,
hive: &HiveEnv,
host: Option<HostMemory>,
) -> Result<Option<(String, String)>> {
let limits = limits_to_apply(agent_name, limits_path, dropin_path, hive);
let memory_max = match &limits {
Some((_, memory_max)) => memory_max.clone(),
Some((_, memory_max)) => Some(memory_max.clone()),
None => dropin_memory_max(agent_name, dropin_path),
};
write_memory_cap(runtime_dir, agent_name, &memory_max)?;
let cap = memory_max.and_then(|m| memory_cap_value(agent_name, &m, host));
write_memory_cap(runtime_dir, cap.as_deref())?;
Ok(limits)
}
/// The `MemoryMax=` the drop-in at `path` applies: its last assignment, as
/// systemd reads it. `infinity`, systemd's default, when it has none, its last
/// one is empty, or the file can't be read.
fn dropin_memory_max(agent_name: &str, path: &Path) -> String {
/// systemd reads it, or `infinity`, systemd's default, when it has none or its
/// last one is empty. `None` when the file can't be read.
fn dropin_memory_max(agent_name: &str, path: &Path) -> Option<String> {
let body = match std::fs::read_to_string(path) {
Ok(body) => body,
Err(e) => {
@ -90,35 +123,90 @@ fn dropin_memory_max(agent_name: &str, path: &Path) -> String {
agent = %agent_name,
path = %path.display(),
error = ?e,
"limits drop-in unreadable; the agent gets no memory cap byte count"
"limits drop-in unreadable; the agent gets no memory cap"
);
return "infinity".to_owned();
return None;
}
};
body.lines()
let value = body
.lines()
.rev()
.find_map(|line| line.trim().strip_prefix("MemoryMax="))
.map(str::trim)
.filter(|v| !v.is_empty())
.unwrap_or("infinity")
.to_owned()
.unwrap_or("infinity");
Some(value.to_owned())
}
/// Write `memory_max` into [`MEMORY_CAP_FILE`] as a byte count, or remove the
/// file when the value has none: `infinity`, or a RAM percentage, which needs
/// the host's `MemTotal` to resolve. With no file the agent sets no heap or
/// subagent ceiling.
fn write_memory_cap(runtime_dir: &Path, agent_name: &str, memory_max: &str) -> Result<()> {
let path = runtime_dir.join(MEMORY_CAP_FILE);
let Some(bytes) = crate::resource_limits::parse_bytes(memory_max) else {
if memory_max.ends_with('%') {
/// What [`MEMORY_CAP_FILE`] says for `memory_max`, read the way systemd reads
/// `MemoryMax=`: a size is its byte count, and a percentage is that share of
/// `host`'s physical memory, rounded down to a page. A value systemd ignores
/// (zero, over 100%, more than two decimals, unparseable) leaves the unit
/// uncapped, so it is `infinity` here. `None` for a percentage when `host` is
/// unknown.
fn memory_cap_value(
agent_name: &str,
memory_max: &str,
host: Option<HostMemory>,
) -> Option<String> {
let bytes = if let Some(percent) = memory_max.strip_suffix('%') {
let Some(host) = host else {
tracing::warn!(
agent = agent_name,
memory_max,
"effective MemoryMax= is a RAM percentage; the agent gets no byte count, so \
no JSC heap ceiling and no subagent MemoryHigh="
"host physical memory unknown; the agent gets no memory cap"
);
return None;
};
permyriad(percent).and_then(|p| scale_physical_memory(host, p))
} else {
crate::resource_limits::parse_bytes(memory_max)
};
Some(
bytes
.filter(|&b| b > 0)
.map_or_else(|| "infinity".to_owned(), |b| b.to_string()),
)
}
/// `percent` (the part before `%`) in hundredths of a percent, with systemd's
/// `parse_permyriad` rules: digits, at most two decimals, at most 100%.
fn permyriad(percent: &str) -> Option<u64> {
let (int, frac) = percent.split_once('.').unwrap_or((percent, ""));
if int.is_empty() || !int.bytes().all(|b| b.is_ascii_digit()) {
return None;
}
let hundredths = match frac.as_bytes() {
[] if !percent.contains('.') => 0,
[d] if d.is_ascii_digit() => u64::from(d - b'0') * 10,
[d1, d2] if d1.is_ascii_digit() && d2.is_ascii_digit() => {
u64::from(d1 - b'0') * 10 + u64::from(d2 - b'0')
}
_ => return None,
};
let value = int
.parse::<u64>()
.ok()?
.checked_mul(100)?
.checked_add(hundredths)?;
(value <= 10_000).then_some(value)
}
/// systemd's `physical_memory_scale(permyriad, 10000)`: whole pages, rounded
/// down, and the exact total at 100%.
fn scale_physical_memory(host: HostMemory, permyriad: u64) -> Option<u64> {
if permyriad == 10_000 {
return Some(host.bytes);
}
let pages = u128::from(host.bytes / host.page) * u128::from(permyriad) / 10_000;
u64::try_from(pages * u128::from(host.page)).ok()
}
/// Write `cap` into [`MEMORY_CAP_FILE`], or remove the file when the cap is
/// unknown, which the agent reports at boot.
fn write_memory_cap(runtime_dir: &Path, cap: Option<&str>) -> Result<()> {
let path = runtime_dir.join(MEMORY_CAP_FILE);
let Some(cap) = cap else {
return match std::fs::remove_file(&path) {
Err(e) if e.kind() != std::io::ErrorKind::NotFound => {
Err(e).with_context(|| format!("remove {}", path.display()))
@ -129,8 +217,7 @@ fn write_memory_cap(runtime_dir: &Path, agent_name: &str, memory_max: &str) -> R
std::fs::create_dir_all(runtime_dir)
.with_context(|| format!("create agent runtime dir {}", runtime_dir.display()))?;
let tmp = runtime_dir.join(format!(".{MEMORY_CAP_FILE}.tmp"));
std::fs::write(&tmp, format!("{bytes}\n"))
.with_context(|| format!("write {}", tmp.display()))?;
std::fs::write(&tmp, format!("{cap}\n")).with_context(|| format!("write {}", tmp.display()))?;
std::fs::rename(&tmp, &path).with_context(|| format!("rename onto {}", path.display()))
}
@ -572,9 +659,10 @@ async fn set_nspawn_flags(
#[cfg(test)]
mod tests {
use super::{
BindMount, MEMORY_CAP_FILE, QUEUE_CLIENT_ID_CREDENTIAL, QUEUE_SECRET_CREDENTIAL,
bind_child_agent_dirs, bind_own_agent_dirs, holds_manage_root_agent, limits_to_apply,
queue_agent_credentials, stage_limits,
BindMount, HostMemory, MEMORY_CAP_FILE, QUEUE_CLIENT_ID_CREDENTIAL,
QUEUE_SECRET_CREDENTIAL, bind_child_agent_dirs, bind_own_agent_dirs,
holds_manage_root_agent, limits_to_apply, memory_cap_value, queue_agent_credentials,
stage_limits,
};
use crate::coordinator::HiveEnv;
@ -792,6 +880,12 @@ mod tests {
assert_eq!(applied, Some(("200%".to_owned(), "1G".to_owned())));
}
/// 4 000 003 pages of 4 KiB: odd, so rounding to a page shows.
const HOST: HostMemory = HostMemory {
bytes: 4_000_003 * 4096,
page: 4096,
};
/// Runs [`stage_limits`] against the files in `dir` and returns the
/// drop-in's `MemoryMax=` and the agent's copy of it, `None` when absent.
fn stage(dir: &tempfile::TempDir) -> (Option<String>, Option<String>) {
@ -802,6 +896,7 @@ mod tests {
&dir.path().join("hyperhive-limits.conf"),
&runtime,
&HiveEnv::default(),
Some(HOST),
)
.expect("stage limits");
let copy = std::fs::read_to_string(runtime.join(MEMORY_CAP_FILE)).ok();
@ -827,18 +922,63 @@ mod tests {
}
#[test]
fn a_cap_with_no_byte_count_removes_the_agents_copy() {
fn every_kind_of_cap_is_written_for_the_agent() {
let dir = tempfile::tempdir().expect("tempdir");
let limits = dir.path().join("resource-limits.json");
assert_eq!(stage(&dir).1, Some("4294967296\n".to_owned()));
for (memory_max, copy) in [
("infinity", "infinity\n"),
("50%", "8192004096\n"),
("8G", "8589934592\n"),
] {
std::fs::write(
&limits,
format!(r#"{{"sock": {{"memory_max": "{memory_max}"}}}}"#),
)
.expect("seed limits");
assert_eq!(
stage(&dir),
(Some(memory_max.to_owned()), Some(copy.to_owned())),
"{memory_max}"
);
}
}
std::fs::write(&limits, r#"{"sock": {"memory_max": "infinity"}}"#).expect("seed limits");
assert_eq!(stage(&dir), (Some("infinity".to_owned()), None));
#[test]
fn a_cap_is_read_the_way_systemd_reads_memory_max() {
for (memory_max, cap) in [
("4G", "4294967296"),
("infinity", "infinity"),
("50%", "8192004096"),
("33.33%", "5460787200"),
("12.5%", "2048000000"),
("100%", "16384012288"),
// systemd ignores these and the unit runs uncapped.
("0", "infinity"),
("0%", "infinity"),
("150%", "infinity"),
("12.345%", "infinity"),
("lots", "infinity"),
] {
assert_eq!(
memory_cap_value("sock", memory_max, Some(HOST)).as_deref(),
Some(cap),
"{memory_max}"
);
}
assert_eq!(memory_cap_value("sock", "50%", None), None);
assert_eq!(
memory_cap_value("sock", "8G", None).as_deref(),
Some("8589934592")
);
}
assert_eq!(stage(&dir).1, None, "an absent copy stays absent");
std::fs::write(&limits, r#"{"sock": {"memory_max": "50%"}}"#).expect("seed limits");
assert_eq!(stage(&dir), (Some("50%".to_owned()), None));
#[test]
fn the_host_memory_systemd_would_read_is_found() {
let host = HostMemory::read().expect("sysconf");
assert!(
host.page >= 4096 && host.bytes.is_multiple_of(host.page),
"{host:?}"
);
}
#[test]
@ -867,8 +1007,37 @@ mod tests {
std::fs::write(&dropin, "[Service]\nMemoryMax=2G\nMemoryMax=3G\n").expect("seed drop-in");
assert_eq!(stage(&dir), (None, Some("3221225472\n".to_owned())));
// No `MemoryMax=` is systemd's default, `infinity`: no byte count.
// No `MemoryMax=` is systemd's default.
std::fs::write(&dropin, "[Service]\nCPUQuota=200%\n").expect("seed drop-in");
assert_eq!(stage(&dir), (None, Some("infinity\n".to_owned())));
}
#[test]
fn an_unknown_cap_leaves_no_copy() {
let dir = tempfile::tempdir().expect("tempdir");
let limits = dir.path().join("resource-limits.json");
let dropin = dir.path().join("hyperhive-limits.conf");
assert_eq!(stage(&dir).1, Some("4294967296\n".to_owned()));
// Unreadable limits and an unreadable drop-in: nothing says what the
// cgroup enforces.
std::fs::write(&limits, TRUNCATED_LIMITS).expect("seed limits");
std::fs::create_dir(&dropin).expect("seed unreadable drop-in");
assert_eq!(stage(&dir), (None, None));
// A percentage with no host memory to resolve it against.
std::fs::write(&limits, r#"{"sock": {"memory_max": "50%"}}"#).expect("seed limits");
let runtime = dir.path().join("runtime");
let applied = stage_limits(
"sock",
&limits,
&dropin,
&runtime,
&HiveEnv::default(),
None,
)
.expect("stage limits");
assert_eq!(applied.map(|(_, mem)| mem), Some("50%".to_owned()));
assert!(!runtime.join(MEMORY_CAP_FILE).exists());
}
}