diff --git a/docs/agent-lifecycle/persistence.md b/docs/agent-lifecycle/persistence.md index c12d7aab..3ca1043c 100644 --- a/docs/agent-lifecycle/persistence.md +++ b/docs/agent-lifecycle/persistence.md @@ -404,7 +404,7 @@ Contents: `lifecycle::write_dropins` generates the systemd drop-in, **not** injected into the container — these are host-side caps on the container, so the capped party never sets them. It sees only the resulting memory cap, - as a byte count in its runtime dir (below). Fallback is per *field*: + as a byte count or `infinity` in its runtime dir (below). Fallback is per *field*: an absent file, absent agent, or absent field falls back to the hive-wide `services.hyperhive.c0re.agentCpuQuota` / `agentMemoryMax`, so an agent can override only its memory and still track the hive @@ -510,8 +510,9 @@ containers can't start. - `/run/hyperhive/agents//mcp.sock` — per-agent socket (bind-mounted into the container as `/run/hive/mcp.sock`). - `/run/hyperhive/agents//memory-max-bytes` — the agent's effective - `MemoryMax=` in bytes, written by `lifecycle::write_dropins` before every - start (absent when the cap is `infinity` or a RAM percentage). The + `MemoryMax=` as a byte count (a percentage resolved against the host's + physical memory) or `infinity`, written by `lifecycle::write_dropins` + before every start. Absent only when hive-c0re can't tell the cap. The container reads it at boot as `/run/hive/memory-max-bytes` (`nix/agent-modules/memory-cap.nix`). diff --git a/docs/scheduler/coordinator.md b/docs/scheduler/coordinator.md index 85b9e006..20d4dab3 100644 --- a/docs/scheduler/coordinator.md +++ b/docs/scheduler/coordinator.md @@ -453,17 +453,21 @@ rebuild, so changes take effect on the next lifecycle op without requiring a host rebuild. The container can't read its own cap, so `write_dropins` also writes the -effective `MemoryMax=` in bytes into the agent's runtime dir -(`/run/hive/memory-max-bytes` inside). At boot the container derives two -values from it: `BUN_JSC_forceRAMSize` (75%, claude-code's heap ceiling) -and `MemoryHigh=` on `hive-subagent-daemon` (two thirds). Neither is in the -agent's build, so a changed cap reaches them at the next container start -without a rebuild. A cap of `infinity` or a RAM percentage sets neither. -When hive-c0re keeps the last drop-in because it can't read the overrides -file, the file carries that drop-in's `MemoryMax=`. A missing or malformed -file sets neither value, and `hyperhive-memory-cap-check.service` logs a -warning naming the path in the container's journal at boot. The -container's `MemoryMax=` still applies. +effective `MemoryMax=` into the agent's runtime dir +(`/run/hive/memory-max-bytes` inside) before every start: a byte count, or +`infinity` for no cap. A percentage becomes bytes the way systemd resolves +it, as that share of the host's physical memory rounded down to a page. A +value systemd ignores (zero, over 100%) is `infinity`. When hive-c0re keeps +the last drop-in because it can't read the overrides file, the file carries +that drop-in's `MemoryMax=`. + +At boot the container derives two values from a byte count: +`BUN_JSC_forceRAMSize` (75%, claude-code's heap ceiling) and `MemoryHigh=` on +`hive-subagent-daemon` (two thirds). Neither is in the agent's build, so a +changed cap reaches them at the next container start without a rebuild. +`infinity` sets neither. A missing or malformed file sets neither as well, and +`hyperhive-memory-cap-check.service` logs a warning naming the path in the +container's journal. The container's `MemoryMax=` still applies. The same drop-in carries `CPUWeight=` / `IOWeight=` from `agentCpuWeight` / `agentIoWeight`. Those are a different kind of diff --git a/hive-c0re/src/lifecycle/host_config.rs b/hive-c0re/src/lifecycle/host_config.rs index a5e763eb..0552b1fd 100644 --- a/hive-c0re/src/lifecycle/host_config.rs +++ b/hive-c0re/src/lifecycle/host_config.rs @@ -31,6 +31,7 @@ pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Re &limits_dropin_path(&container), &paths.agent, hive, + HostMemory::read(), )? { set_resource_limits( &container, @@ -54,10 +55,40 @@ fn limits_dropin_path(container: &str) -> PathBuf { } /// File in the agent's runtime dir, `/run/hive/` in the container, holding the -/// byte count of the `MemoryMax=` its drop-in carries. -/// `nix/agent-modules/memory-cap.nix` reads it at boot under this name. +/// `MemoryMax=` its drop-in carries: a byte count, or `infinity` for no cap. +/// `nix/agent-modules/memory-cap.nix` reads it at boot under this name, and +/// treats a missing file as a fault. const MEMORY_CAP_FILE: &str = "memory-max-bytes"; +/// The host's physical memory and page size, the two numbers systemd resolves +/// a `MemoryMax=` percentage against. +#[derive(Clone, Copy, Debug)] +struct HostMemory { + bytes: u64, + page: u64, +} + +impl HostMemory { + /// `sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE)`, as systemd's + /// `physical_memory()` computes it. systemd also caps that by the root + /// cgroup's `memory.max`, which the host's root cgroup does not have. + fn read() -> Option { + // SAFETY: `sysconf(3)` only reads system configuration. + let (pages, page) = unsafe { + ( + libc::sysconf(libc::_SC_PHYS_PAGES), + libc::sysconf(libc::_SC_PAGESIZE), + ) + }; + let pages = u64::try_from(pages).ok().filter(|&p| p > 0)?; + let page = u64::try_from(page).ok().filter(|&p| p > 0)?; + Some(Self { + bytes: pages.checked_mul(page)?, + page, + }) + } +} + /// [`limits_to_apply`], plus handing the memory cap to the agent through /// [`MEMORY_CAP_FILE`] in `runtime_dir`. The container cannot read the cap /// systemd enforces on it, so this file is how it learns the value it sizes @@ -69,20 +100,22 @@ fn stage_limits( dropin_path: &Path, runtime_dir: &Path, hive: &HiveEnv, + host: Option, ) -> Result> { let limits = limits_to_apply(agent_name, limits_path, dropin_path, hive); let memory_max = match &limits { - Some((_, memory_max)) => memory_max.clone(), + Some((_, memory_max)) => Some(memory_max.clone()), None => dropin_memory_max(agent_name, dropin_path), }; - write_memory_cap(runtime_dir, agent_name, &memory_max)?; + let cap = memory_max.and_then(|m| memory_cap_value(agent_name, &m, host)); + write_memory_cap(runtime_dir, cap.as_deref())?; Ok(limits) } /// The `MemoryMax=` the drop-in at `path` applies: its last assignment, as -/// systemd reads it. `infinity`, systemd's default, when it has none, its last -/// one is empty, or the file can't be read. -fn dropin_memory_max(agent_name: &str, path: &Path) -> String { +/// systemd reads it, or `infinity`, systemd's default, when it has none or its +/// last one is empty. `None` when the file can't be read. +fn dropin_memory_max(agent_name: &str, path: &Path) -> Option { let body = match std::fs::read_to_string(path) { Ok(body) => body, Err(e) => { @@ -90,35 +123,90 @@ fn dropin_memory_max(agent_name: &str, path: &Path) -> String { agent = %agent_name, path = %path.display(), error = ?e, - "limits drop-in unreadable; the agent gets no memory cap byte count" + "limits drop-in unreadable; the agent gets no memory cap" ); - return "infinity".to_owned(); + return None; } }; - body.lines() + let value = body + .lines() .rev() .find_map(|line| line.trim().strip_prefix("MemoryMax=")) .map(str::trim) .filter(|v| !v.is_empty()) - .unwrap_or("infinity") - .to_owned() + .unwrap_or("infinity"); + Some(value.to_owned()) } -/// Write `memory_max` into [`MEMORY_CAP_FILE`] as a byte count, or remove the -/// file when the value has none: `infinity`, or a RAM percentage, which needs -/// the host's `MemTotal` to resolve. With no file the agent sets no heap or -/// subagent ceiling. -fn write_memory_cap(runtime_dir: &Path, agent_name: &str, memory_max: &str) -> Result<()> { - let path = runtime_dir.join(MEMORY_CAP_FILE); - let Some(bytes) = crate::resource_limits::parse_bytes(memory_max) else { - if memory_max.ends_with('%') { +/// What [`MEMORY_CAP_FILE`] says for `memory_max`, read the way systemd reads +/// `MemoryMax=`: a size is its byte count, and a percentage is that share of +/// `host`'s physical memory, rounded down to a page. A value systemd ignores +/// (zero, over 100%, more than two decimals, unparseable) leaves the unit +/// uncapped, so it is `infinity` here. `None` for a percentage when `host` is +/// unknown. +fn memory_cap_value( + agent_name: &str, + memory_max: &str, + host: Option, +) -> Option { + let bytes = if let Some(percent) = memory_max.strip_suffix('%') { + let Some(host) = host else { tracing::warn!( agent = agent_name, memory_max, - "effective MemoryMax= is a RAM percentage; the agent gets no byte count, so \ - no JSC heap ceiling and no subagent MemoryHigh=" + "host physical memory unknown; the agent gets no memory cap" ); + return None; + }; + permyriad(percent).and_then(|p| scale_physical_memory(host, p)) + } else { + crate::resource_limits::parse_bytes(memory_max) + }; + Some( + bytes + .filter(|&b| b > 0) + .map_or_else(|| "infinity".to_owned(), |b| b.to_string()), + ) +} + +/// `percent` (the part before `%`) in hundredths of a percent, with systemd's +/// `parse_permyriad` rules: digits, at most two decimals, at most 100%. +fn permyriad(percent: &str) -> Option { + let (int, frac) = percent.split_once('.').unwrap_or((percent, "")); + if int.is_empty() || !int.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let hundredths = match frac.as_bytes() { + [] if !percent.contains('.') => 0, + [d] if d.is_ascii_digit() => u64::from(d - b'0') * 10, + [d1, d2] if d1.is_ascii_digit() && d2.is_ascii_digit() => { + u64::from(d1 - b'0') * 10 + u64::from(d2 - b'0') } + _ => return None, + }; + let value = int + .parse::() + .ok()? + .checked_mul(100)? + .checked_add(hundredths)?; + (value <= 10_000).then_some(value) +} + +/// systemd's `physical_memory_scale(permyriad, 10000)`: whole pages, rounded +/// down, and the exact total at 100%. +fn scale_physical_memory(host: HostMemory, permyriad: u64) -> Option { + if permyriad == 10_000 { + return Some(host.bytes); + } + let pages = u128::from(host.bytes / host.page) * u128::from(permyriad) / 10_000; + u64::try_from(pages * u128::from(host.page)).ok() +} + +/// Write `cap` into [`MEMORY_CAP_FILE`], or remove the file when the cap is +/// unknown, which the agent reports at boot. +fn write_memory_cap(runtime_dir: &Path, cap: Option<&str>) -> Result<()> { + let path = runtime_dir.join(MEMORY_CAP_FILE); + let Some(cap) = cap else { return match std::fs::remove_file(&path) { Err(e) if e.kind() != std::io::ErrorKind::NotFound => { Err(e).with_context(|| format!("remove {}", path.display())) @@ -129,8 +217,7 @@ fn write_memory_cap(runtime_dir: &Path, agent_name: &str, memory_max: &str) -> R std::fs::create_dir_all(runtime_dir) .with_context(|| format!("create agent runtime dir {}", runtime_dir.display()))?; let tmp = runtime_dir.join(format!(".{MEMORY_CAP_FILE}.tmp")); - std::fs::write(&tmp, format!("{bytes}\n")) - .with_context(|| format!("write {}", tmp.display()))?; + std::fs::write(&tmp, format!("{cap}\n")).with_context(|| format!("write {}", tmp.display()))?; std::fs::rename(&tmp, &path).with_context(|| format!("rename onto {}", path.display())) } @@ -572,9 +659,10 @@ async fn set_nspawn_flags( #[cfg(test)] mod tests { use super::{ - BindMount, MEMORY_CAP_FILE, QUEUE_CLIENT_ID_CREDENTIAL, QUEUE_SECRET_CREDENTIAL, - bind_child_agent_dirs, bind_own_agent_dirs, holds_manage_root_agent, limits_to_apply, - queue_agent_credentials, stage_limits, + BindMount, HostMemory, MEMORY_CAP_FILE, QUEUE_CLIENT_ID_CREDENTIAL, + QUEUE_SECRET_CREDENTIAL, bind_child_agent_dirs, bind_own_agent_dirs, + holds_manage_root_agent, limits_to_apply, memory_cap_value, queue_agent_credentials, + stage_limits, }; use crate::coordinator::HiveEnv; @@ -792,6 +880,12 @@ mod tests { assert_eq!(applied, Some(("200%".to_owned(), "1G".to_owned()))); } + /// 4 000 003 pages of 4 KiB: odd, so rounding to a page shows. + const HOST: HostMemory = HostMemory { + bytes: 4_000_003 * 4096, + page: 4096, + }; + /// Runs [`stage_limits`] against the files in `dir` and returns the /// drop-in's `MemoryMax=` and the agent's copy of it, `None` when absent. fn stage(dir: &tempfile::TempDir) -> (Option, Option) { @@ -802,6 +896,7 @@ mod tests { &dir.path().join("hyperhive-limits.conf"), &runtime, &HiveEnv::default(), + Some(HOST), ) .expect("stage limits"); let copy = std::fs::read_to_string(runtime.join(MEMORY_CAP_FILE)).ok(); @@ -827,18 +922,63 @@ mod tests { } #[test] - fn a_cap_with_no_byte_count_removes_the_agents_copy() { + fn every_kind_of_cap_is_written_for_the_agent() { let dir = tempfile::tempdir().expect("tempdir"); let limits = dir.path().join("resource-limits.json"); - assert_eq!(stage(&dir).1, Some("4294967296\n".to_owned())); + for (memory_max, copy) in [ + ("infinity", "infinity\n"), + ("50%", "8192004096\n"), + ("8G", "8589934592\n"), + ] { + std::fs::write( + &limits, + format!(r#"{{"sock": {{"memory_max": "{memory_max}"}}}}"#), + ) + .expect("seed limits"); + assert_eq!( + stage(&dir), + (Some(memory_max.to_owned()), Some(copy.to_owned())), + "{memory_max}" + ); + } + } - std::fs::write(&limits, r#"{"sock": {"memory_max": "infinity"}}"#).expect("seed limits"); - assert_eq!(stage(&dir), (Some("infinity".to_owned()), None)); + #[test] + fn a_cap_is_read_the_way_systemd_reads_memory_max() { + for (memory_max, cap) in [ + ("4G", "4294967296"), + ("infinity", "infinity"), + ("50%", "8192004096"), + ("33.33%", "5460787200"), + ("12.5%", "2048000000"), + ("100%", "16384012288"), + // systemd ignores these and the unit runs uncapped. + ("0", "infinity"), + ("0%", "infinity"), + ("150%", "infinity"), + ("12.345%", "infinity"), + ("lots", "infinity"), + ] { + assert_eq!( + memory_cap_value("sock", memory_max, Some(HOST)).as_deref(), + Some(cap), + "{memory_max}" + ); + } + assert_eq!(memory_cap_value("sock", "50%", None), None); + assert_eq!( + memory_cap_value("sock", "8G", None).as_deref(), + Some("8589934592") + ); + } - assert_eq!(stage(&dir).1, None, "an absent copy stays absent"); - - std::fs::write(&limits, r#"{"sock": {"memory_max": "50%"}}"#).expect("seed limits"); - assert_eq!(stage(&dir), (Some("50%".to_owned()), None)); + #[test] + fn the_host_memory_systemd_would_read_is_found() { + let host = HostMemory::read().expect("sysconf"); + assert!( + host.page >= 4096 && host.bytes.is_multiple_of(host.page), + "{host:?}" + ); } #[test] @@ -867,8 +1007,37 @@ mod tests { std::fs::write(&dropin, "[Service]\nMemoryMax=2G\nMemoryMax=3G\n").expect("seed drop-in"); assert_eq!(stage(&dir), (None, Some("3221225472\n".to_owned()))); - // No `MemoryMax=` is systemd's default, `infinity`: no byte count. + // No `MemoryMax=` is systemd's default. std::fs::write(&dropin, "[Service]\nCPUQuota=200%\n").expect("seed drop-in"); + assert_eq!(stage(&dir), (None, Some("infinity\n".to_owned()))); + } + + #[test] + fn an_unknown_cap_leaves_no_copy() { + let dir = tempfile::tempdir().expect("tempdir"); + let limits = dir.path().join("resource-limits.json"); + let dropin = dir.path().join("hyperhive-limits.conf"); + assert_eq!(stage(&dir).1, Some("4294967296\n".to_owned())); + + // Unreadable limits and an unreadable drop-in: nothing says what the + // cgroup enforces. + std::fs::write(&limits, TRUNCATED_LIMITS).expect("seed limits"); + std::fs::create_dir(&dropin).expect("seed unreadable drop-in"); assert_eq!(stage(&dir), (None, None)); + + // A percentage with no host memory to resolve it against. + std::fs::write(&limits, r#"{"sock": {"memory_max": "50%"}}"#).expect("seed limits"); + let runtime = dir.path().join("runtime"); + let applied = stage_limits( + "sock", + &limits, + &dropin, + &runtime, + &HiveEnv::default(), + None, + ) + .expect("stage limits"); + assert_eq!(applied.map(|(_, mem)| mem), Some("50%".to_owned())); + assert!(!runtime.join(MEMORY_CAP_FILE).exists()); } } diff --git a/nix/agent-modules/memory-cap.nix b/nix/agent-modules/memory-cap.nix index 477bd023..b570d6a7 100644 --- a/nix/agent-modules/memory-cap.nix +++ b/nix/agent-modules/memory-cap.nix @@ -3,10 +3,11 @@ # # The cap is `MemoryMax=` on the host's `container@` unit, which nothing in # here can read: `/sys/fs/cgroup/memory.max` reads `max` at every level from -# inside. hive-c0re writes its byte count to `/run/hive/memory-max-bytes` -# before every start (`lifecycle::host_config`), and removes the file when the -# cap is `infinity` or a RAM percentage. Two values are derived from it, and -# neither is set without the file; `hyperhive-memory-cap-check` then logs a +# inside. Before every start hive-c0re (`lifecycle::host_config`) writes it to +# `/run/hive/memory-max-bytes` as a byte count, a percentage resolved against +# the host's RAM as systemd resolves it, or `infinity` for no cap. Two values +# are derived from a byte count, and neither is set for `infinity`. A missing +# or malformed file sets neither too, and `hyperhive-memory-cap-check` logs a # warning at boot: # # - `BUN_JSC_forceRAMSize`, 75% of the cap. Bun/JavaScriptCore, the runtime @@ -29,13 +30,15 @@ # `checks.module-eval-agent-memory`. { pkgs, ... }: let - # Sets `bytes`, or runs `noCap` when there is no file or its first line is - # not a plain byte count. A leading zero would read as octal, and 16 digits - # or more could overflow `bytes * 75`. Shell builtins only. + # Sets `bytes`. Exits for `infinity`, and runs `noCap` when there is no file + # or its first line is neither `infinity` nor a plain byte count. A leading + # zero would read as octal, and 16 digits or more could overflow + # `bytes * 75`. Shell builtins only. readCap = noCap: '' bytes= [ -r /run/hive/memory-max-bytes ] && read -r bytes < /run/hive/memory-max-bytes case "$bytes" in + infinity) exit 0 ;; "" | 0* | *[!0-9]* | ????????????????*) ${noCap} ;; esac ''; @@ -55,16 +58,16 @@ let ''; # The generators run before the container's journal starts and an nspawn - # container has no /dev/kmsg, so the boot warning for an unusable cap comes - # from a unit. `<4>` is the journal's warning priority. + # container has no /dev/kmsg, so the boot warning for a missing or malformed + # cap file comes from a unit. `<4>` is the journal's warning priority. check = pkgs.writeShellScript "hyperhive-memory-cap-check" '' ${readCap '' if [ -e /run/hive/memory-max-bytes ]; then - why="does not start with a plain byte count" + why="holds neither a byte count nor infinity" else why="is missing" fi - echo "<4>/run/hive/memory-max-bytes $why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies. hive-c0re writes no file for a cap of infinity or a RAM percentage." >&2 + echo "<4>/run/hive/memory-max-bytes $why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies." >&2 exit 0 ''} ''; diff --git a/nix/script-tests/agent-memory-cap.sh b/nix/script-tests/agent-memory-cap.sh index fc873486..f60ed37d 100644 --- a/nix/script-tests/agent-memory-cap.sh +++ b/nix/script-tests/agent-memory-cap.sh @@ -58,11 +58,11 @@ check() { fi # The warning goes to stderr with the journal's `<4>` priority prefix, and - # nothing goes anywhere for a usable cap. + # nothing goes anywhere for a byte count or `infinity`. got=$(env -i "$dir/check" 2>&1) local want_warn= if [ -n "$want_why" ]; then - want_warn="<4>$cap $want_why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies. hive-c0re writes no file for a cap of infinity or a RAM percentage." + want_warn="<4>$cap $want_why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies." fi if [ "$got" != "$want_warn" ]; then fail "boot check printed:"$'\n'"$got"$'\n'"wanted:"$'\n'"$want_warn" @@ -88,17 +88,26 @@ check "a line without a newline is kept" 3221225472 2863311530 printf '4294967296\nLD_PRELOAD=/x\n' >"$cap" check "only the first line is read" 3221225472 2863311530 +printf 'infinity\n' >"$cap" +check "infinity sets nothing and warns of nothing" "" "" "" + +printf 'infinity' >"$cap" +check "infinity without a newline" "" "" "" + +printf 'infinityx\n' >"$cap" +check "a word that only starts with infinity" "" "" "holds neither a byte count nor infinity" + printf '8G\n' >"$cap" -check "a size with a suffix is ignored" "" "" "does not start with a plain byte count" +check "a size with a suffix is ignored" "" "" "holds neither a byte count nor infinity" printf '0123\n' >"$cap" -check "a leading zero is ignored" "" "" "does not start with a plain byte count" +check "a leading zero is ignored" "" "" "holds neither a byte count nor infinity" printf '1234567890123456\n' >"$cap" -check "sixteen digits are ignored" "" "" "does not start with a plain byte count" +check "sixteen digits are ignored" "" "" "holds neither a byte count nor infinity" : >"$cap" -check "an empty file" "" "" "does not start with a plain byte count" +check "an empty file" "" "" "holds neither a byte count nor infinity" rm -f "$cap" check "no file" "" "" "is missing"