From 57955a6c53aaa0470b7c203aead0dd61cbe813fa Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 12 Oct 2026 10:10:11 +0200 Subject: [PATCH] agents: hand over the kept drop-in's cap, warn at boot without one When hive-c0re can't read resource-limits.json and a limits drop-in already exists, it keeps the drop-in, but it wrote no memory cap file. A container whose runtime dir had no file yet (hive-c0re upgraded, /run not cleared) then started with no heap ceiling and no subagent MemoryHigh=, silently. `stage_limits` now writes the file on every call: from the new limits, or from the last `MemoryMax=` in the kept drop-in (`infinity` when it has none or can't be read). A missing or malformed cap file at boot is no longer silent. The generators run before the container's journal and an nspawn container has no /dev/kmsg, so a oneshot `hyperhive-memory-cap-check.service` logs a warning naming the path. The agent then runs with no JSC heap ceiling and no subagent MemoryHigh=; the container's MemoryMax= still applies. module-eval-agent-memory gains a case pinning how the ceiling reaches claude: `hive-agent` and `hive-subagent-daemon` neither set nor unset BUN_JSC_forceRAMSize, both generators and the check unit are installed. Part of #5081 (slice S0-2). --- docs/scheduler/coordinator.md | 5 ++ hive-c0re/src/lifecycle/host_config.rs | 63 ++++++++++++++++++++++---- nix/agent-modules/memory-cap.nix | 45 ++++++++++++++---- nix/module-eval/agent-memory.nix | 20 ++++++++ nix/script-tests/agent-memory-cap.nix | 8 ++-- nix/script-tests/agent-memory-cap.sh | 35 +++++++++----- 6 files changed, 144 insertions(+), 32 deletions(-) diff --git a/docs/scheduler/coordinator.md b/docs/scheduler/coordinator.md index 3b87bcec..85b9e006 100644 --- a/docs/scheduler/coordinator.md +++ b/docs/scheduler/coordinator.md @@ -459,6 +459,11 @@ values from it: `BUN_JSC_forceRAMSize` (75%, claude-code's heap ceiling) and `MemoryHigh=` on `hive-subagent-daemon` (two thirds). Neither is in the agent's build, so a changed cap reaches them at the next container start without a rebuild. A cap of `infinity` or a RAM percentage sets neither. +When hive-c0re keeps the last drop-in because it can't read the overrides +file, the file carries that drop-in's `MemoryMax=`. A missing or malformed +file sets neither value, and `hyperhive-memory-cap-check.service` logs a +warning naming the path in the container's journal at boot. The +container's `MemoryMax=` still applies. The same drop-in carries `CPUWeight=` / `IOWeight=` from `agentCpuWeight` / `agentIoWeight`. Those are a different kind of diff --git a/hive-c0re/src/lifecycle/host_config.rs b/hive-c0re/src/lifecycle/host_config.rs index afb00da5..a5e763eb 100644 --- a/hive-c0re/src/lifecycle/host_config.rs +++ b/hive-c0re/src/lifecycle/host_config.rs @@ -61,8 +61,8 @@ const MEMORY_CAP_FILE: &str = "memory-max-bytes"; /// [`limits_to_apply`], plus handing the memory cap to the agent through /// [`MEMORY_CAP_FILE`] in `runtime_dir`. The container cannot read the cap /// systemd enforces on it, so this file is how it learns the value it sizes -/// its heap and its subagents against. When the drop-in is kept as it is, so is -/// the file. +/// its heap and its subagents against. It is written on every call, from the +/// `MemoryMax=` the agent will start under: the new one, or the kept drop-in's. fn stage_limits( agent_name: &str, limits_path: &Path, @@ -71,12 +71,39 @@ fn stage_limits( hive: &HiveEnv, ) -> Result> { let limits = limits_to_apply(agent_name, limits_path, dropin_path, hive); - if let Some((_, memory_max)) = &limits { - write_memory_cap(runtime_dir, agent_name, memory_max)?; - } + let memory_max = match &limits { + Some((_, memory_max)) => memory_max.clone(), + None => dropin_memory_max(agent_name, dropin_path), + }; + write_memory_cap(runtime_dir, agent_name, &memory_max)?; Ok(limits) } +/// The `MemoryMax=` the drop-in at `path` applies: its last assignment, as +/// systemd reads it. `infinity`, systemd's default, when it has none, its last +/// one is empty, or the file can't be read. +fn dropin_memory_max(agent_name: &str, path: &Path) -> String { + let body = match std::fs::read_to_string(path) { + Ok(body) => body, + Err(e) => { + tracing::warn!( + agent = %agent_name, + path = %path.display(), + error = ?e, + "limits drop-in unreadable; the agent gets no memory cap byte count" + ); + return "infinity".to_owned(); + } + }; + body.lines() + .rev() + .find_map(|line| line.trim().strip_prefix("MemoryMax=")) + .map(str::trim) + .filter(|v| !v.is_empty()) + .unwrap_or("infinity") + .to_owned() +} + /// Write `memory_max` into [`MEMORY_CAP_FILE`] as a byte count, or remove the /// file when the value has none: `infinity`, or a RAM percentage, which needs /// the host's `MemTotal` to resolve. With no file the agent sets no heap or @@ -815,15 +842,33 @@ mod tests { } #[test] - fn a_kept_dropin_keeps_the_agents_copy() { + fn a_kept_dropin_hands_the_agent_its_cap() { let dir = tempfile::tempdir().expect("tempdir"); let limits = dir.path().join("resource-limits.json"); + let dropin = dir.path().join("hyperhive-limits.conf"); + let copy = dir.path().join("runtime").join(MEMORY_CAP_FILE); std::fs::write(&limits, r#"{"sock": {"memory_max": "1G"}}"#).expect("seed limits"); assert_eq!(stage(&dir).1, Some("1073741824\n".to_owned())); + // Unreadable limits keep the drop-in; the copy follows the drop-in. std::fs::write(&limits, TRUNCATED_LIMITS).expect("seed limits"); - std::fs::write(dir.path().join("hyperhive-limits.conf"), "MemoryMax=1G\n") - .expect("seed drop-in"); - assert_eq!(stage(&dir), (None, Some("1073741824\n".to_owned()))); + std::fs::write( + &dropin, + "[Unit]\nConditionPathIsDirectory=/rt/sock\n\n[Service]\nRestartSec=5\nMemoryMax=2G\nCPUQuota=200%\n", + ) + .expect("seed drop-in"); + assert_eq!(stage(&dir), (None, Some("2147483648\n".to_owned()))); + + // A kept drop-in with no copy beside it, as after an upgrade. + std::fs::remove_file(©).expect("remove copy"); + assert_eq!(stage(&dir), (None, Some("2147483648\n".to_owned()))); + + // The last assignment is the one systemd applies. + std::fs::write(&dropin, "[Service]\nMemoryMax=2G\nMemoryMax=3G\n").expect("seed drop-in"); + assert_eq!(stage(&dir), (None, Some("3221225472\n".to_owned()))); + + // No `MemoryMax=` is systemd's default, `infinity`: no byte count. + std::fs::write(&dropin, "[Service]\nCPUQuota=200%\n").expect("seed drop-in"); + assert_eq!(stage(&dir), (None, None)); } } diff --git a/nix/agent-modules/memory-cap.nix b/nix/agent-modules/memory-cap.nix index 47c29968..477bd023 100644 --- a/nix/agent-modules/memory-cap.nix +++ b/nix/agent-modules/memory-cap.nix @@ -6,7 +6,8 @@ # inside. hive-c0re writes its byte count to `/run/hive/memory-max-bytes` # before every start (`lifecycle::host_config`), and removes the file when the # cap is `infinity` or a RAM percentage. Two values are derived from it, and -# neither is set without the file: +# neither is set without the file; `hyperhive-memory-cap-check` then logs a +# warning at boot: # # - `BUN_JSC_forceRAMSize`, 75% of the cap. Bun/JavaScriptCore, the runtime # under claude-code, sizes its heap ceiling off `/proc/meminfo`, which under @@ -15,6 +16,8 @@ # generator hands it to every unit of the container's systemd, `machinectl # shell` sessions (`hivectl choom`) included; `/etc/set-environment` exports # it in login shells that no unit starts (`nixos-container root-login`). +# `hive-agent` and `hive-subagent-daemon` spawn `claude` as a plain child, +# which inherits it, so neither unit may set or unset it. # - `MemoryHigh=` on `hive-subagent-daemon`, two thirds of the cap, as a # generator drop-in. The daemon spawns nested `claude` sessions as plain # children, so its cgroup already is the "all subagents" cgroup. `High`, not @@ -22,34 +25,50 @@ # last third stays free for the agent's own turn, but a subagent that needs # more still gets it when the container has memory to spare. # -# Tested by `checks.script-test-agent-memory-cap`. +# Tested by `checks.script-test-agent-memory-cap` and +# `checks.module-eval-agent-memory`. { pkgs, ... }: let - # Sets `bytes`, or exits when there is no file or its first line is not a - # plain byte count. A leading zero would read as octal, and 16 digits or - # more could overflow `bytes * 75`. Shell builtins only. - readCap = '' + # Sets `bytes`, or runs `noCap` when there is no file or its first line is + # not a plain byte count. A leading zero would read as octal, and 16 digits + # or more could overflow `bytes * 75`. Shell builtins only. + readCap = noCap: '' bytes= [ -r /run/hive/memory-max-bytes ] && read -r bytes < /run/hive/memory-max-bytes case "$bytes" in - "" | 0* | *[!0-9]* | ????????????????*) exit 0 ;; + "" | 0* | *[!0-9]* | ????????????????*) ${noCap} ;; esac ''; envGenerator = pkgs.writeShellScript "hyperhive-memory-cap-env" '' - ${readCap} + ${readCap "exit 0"} printf 'BUN_JSC_forceRAMSize=%s\n' "$((bytes * 75 / 100))" ''; # `$1` is the generator's normal-priority output dir. A drop-in there merges # with the unit's own file in /etc. unitGenerator = pkgs.writeShellScript "hyperhive-memory-cap" '' - ${readCap} + ${readCap "exit 0"} dir="$1/hive-subagent-daemon.service.d" ${pkgs.coreutils}/bin/mkdir -p "$dir" printf '[Service]\nMemoryHigh=%s\n' "$((bytes * 2 / 3))" > "$dir/50-memory-high.conf" ''; + # The generators run before the container's journal starts and an nspawn + # container has no /dev/kmsg, so the boot warning for an unusable cap comes + # from a unit. `<4>` is the journal's warning priority. + check = pkgs.writeShellScript "hyperhive-memory-cap-check" '' + ${readCap '' + if [ -e /run/hive/memory-max-bytes ]; then + why="does not start with a plain byte count" + else + why="is missing" + fi + echo "<4>/run/hive/memory-max-bytes $why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies. hive-c0re writes no file for a cap of infinity or a RAM percentage." >&2 + exit 0 + ''} + ''; + # Sourced by every shell's `/etc/set-environment`, so POSIX sh only. profile = pkgs.writeText "hyperhive-memory-cap.sh" '' __hyperhive_cap=$(${envGenerator}) @@ -63,6 +82,14 @@ in environment.etc."systemd/system-environment-generators/50-hyperhive-memory-cap".source = envGenerator; systemd.generators.hyperhive-memory-cap = unitGenerator; + systemd.services.hyperhive-memory-cap-check = { + description = "Warn when the container's memory cap is unusable"; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = check; + }; + }; environment.etc."hyperhive/memory-cap.sh".source = profile; environment.extraInit = ". /etc/hyperhive/memory-cap.sh"; } diff --git a/nix/module-eval/agent-memory.nix b/nix/module-eval/agent-memory.nix index fd81a954..639309c5 100644 --- a/nix/module-eval/agent-memory.nix +++ b/nix/module-eval/agent-memory.nix @@ -34,6 +34,26 @@ let name = "the subagent daemon's memory ceiling is not built into its unit"; ok = !((agentSubagentDaemon agentDefault).serviceConfig ? MemoryHigh); } + { + # claude's heap ceiling reaches it only as inherited environment: the + # environment generator fills the manager's, systemd hands that to every + # unit, and both units spawn claude as a plain child. A unit that sets + # or unsets the variable itself cuts that. Without the two generators the + # cap reaches nothing, and without the check unit nothing says so. + name = "claude's units take the heap ceiling from the memory cap read at boot"; + ok = + let + leavesIt = + unit: + !(unit.environment ? BUN_JSC_forceRAMSize) + && !(lib.elem "BUN_JSC_forceRAMSize" (lib.toList (unit.serviceConfig.UnsetEnvironment or [ ]))); + in + leavesIt (agentHarness agentDefault) + && leavesIt (agentSubagentDaemon agentDefault) + && agentDefault.environment.etc ? "systemd/system-environment-generators/50-hyperhive-memory-cap" + && agentDefault.systemd.generators ? hyperhive-memory-cap + && lib.elem "multi-user.target" agentDefault.systemd.services.hyperhive-memory-cap-check.wantedBy; + } { # The sign is the whole property, and it is easy to write backwards: # a HIGHER OOMScoreAdjust is a MORE likely victim. So the subagent diff --git a/nix/script-tests/agent-memory-cap.nix b/nix/script-tests/agent-memory-cap.nix index f347176a..7844673e 100644 --- a/nix/script-tests/agent-memory-cap.nix +++ b/nix/script-tests/agent-memory-cap.nix @@ -1,7 +1,8 @@ -# `checks.script-test-agent-memory-cap` — runs the three readers of +# `checks.script-test-agent-memory-cap` — runs the four readers of # `/run/hive/memory-max-bytes` in ../agent-modules/memory-cap.nix: the -# environment generator, the unit generator, and the login-shell snippet -# `/etc/set-environment` sources. The cases are in ./agent-memory-cap.sh. +# environment generator, the unit generator, the boot check that warns about +# an unusable cap, and the login-shell snippet `/etc/set-environment` sources. +# The cases are in ./agent-memory-cap.sh. # # What runs is the rendered scripts. The one edit is the cap file's path, moved # under the build directory, and the snippet's reference to the environment @@ -32,6 +33,7 @@ pkgs.runCommand "hyperhive-script-test-agent-memory-cap" ENV_GENERATOR = machine.environment.etc."systemd/system-environment-generators/50-hyperhive-memory-cap".source; UNIT_GENERATOR = machine.systemd.generators.hyperhive-memory-cap; + CHECK = machine.systemd.services.hyperhive-memory-cap-check.serviceConfig.ExecStart; PROFILE = machine.environment.etc."hyperhive/memory-cap.sh".source; EXTRA_INIT = machine.environment.extraInit; SHELLS = "${pkgs.dash}/bin/dash ${pkgs.bash}/bin/sh"; diff --git a/nix/script-tests/agent-memory-cap.sh b/nix/script-tests/agent-memory-cap.sh index 22b9e1f0..fc873486 100644 --- a/nix/script-tests/agent-memory-cap.sh +++ b/nix/script-tests/agent-memory-cap.sh @@ -1,7 +1,7 @@ # Cases for ./agent-memory-cap.nix. Each case writes a cap file (or none), runs -# both generators with a clean environment and compares what they produced, -# then sources the login-shell snippet under each POSIX shell and compares what -# a child process sees. +# both generators and the boot check with a clean environment and compares what +# they produced, then sources the login-shell snippet under each POSIX shell +# and compares what a child process sees. set -uo pipefail failures=0 @@ -16,12 +16,13 @@ dir=$(mktemp -d) cap=$dir/memory-max-bytes sed "s|/run/hive/memory-max-bytes|$cap|g" "$ENV_GENERATOR" >"$dir/env-generator" sed "s|/run/hive/memory-max-bytes|$cap|g" "$UNIT_GENERATOR" >"$dir/unit-generator" -chmod +x "$dir/env-generator" "$dir/unit-generator" +sed "s|/run/hive/memory-max-bytes|$cap|g" "$CHECK" >"$dir/check" +chmod +x "$dir/env-generator" "$dir/unit-generator" "$dir/check" sed "s|$ENV_GENERATOR|$dir/env-generator|g" "$PROFILE" >"$dir/profile" envbin=$(command -v env) case="setup" -for g in env-generator unit-generator; do +for g in env-generator unit-generator check; do if grep -q '/run/hive/memory-max-bytes' "$dir/$g"; then fail "$g still names /run/hive after the rewrite"; fi if ! grep -qF "$cap" "$dir/$g"; then fail "the rewrite did not reach $g's read"; fi done @@ -31,10 +32,11 @@ if ! printf '%s\n' "$EXTRA_INIT" | grep -qxF '. /etc/hyperhive/memory-cap.sh'; t fi # check +# check() { case=$1 count=$((count + 1)) - local want_env=${2:+BUN_JSC_forceRAMSize=$2} want_high=$3 got + local want_env=${2:+BUN_JSC_forceRAMSize=$2} want_high=$3 want_why=${4:-} got got=$(env -i "$dir/env-generator") if [ "$got" != "$want_env" ]; then fail "environment generator printed:"$'\n'"$got"$'\n'"wanted:"$'\n'"$want_env" @@ -55,6 +57,17 @@ check() { fi fi + # The warning goes to stderr with the journal's `<4>` priority prefix, and + # nothing goes anywhere for a usable cap. + got=$(env -i "$dir/check" 2>&1) + local want_warn= + if [ -n "$want_why" ]; then + want_warn="<4>$cap $want_why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies. hive-c0re writes no file for a cap of infinity or a RAM percentage." + fi + if [ "$got" != "$want_warn" ]; then + fail "boot check printed:"$'\n'"$got"$'\n'"wanted:"$'\n'"$want_warn" + fi + local sh for sh in $SHELLS; do # Only what the snippet exported reaches the child's environment. No PATH: @@ -76,19 +89,19 @@ printf '4294967296\nLD_PRELOAD=/x\n' >"$cap" check "only the first line is read" 3221225472 2863311530 printf '8G\n' >"$cap" -check "a size with a suffix is ignored" "" "" +check "a size with a suffix is ignored" "" "" "does not start with a plain byte count" printf '0123\n' >"$cap" -check "a leading zero is ignored" "" "" +check "a leading zero is ignored" "" "" "does not start with a plain byte count" printf '1234567890123456\n' >"$cap" -check "sixteen digits are ignored" "" "" +check "sixteen digits are ignored" "" "" "does not start with a plain byte count" : >"$cap" -check "an empty file" "" "" +check "an empty file" "" "" "does not start with a plain byte count" rm -f "$cap" -check "no file" "" "" +check "no file" "" "" "is missing" echo "$count cases, $failures failed" [ "$failures" -eq 0 ]