diff --git a/docs/4-implementation/scheduler/coordinator.md b/docs/4-implementation/scheduler/coordinator.md index f38d0c5e4..a74cba57f 100644 --- a/docs/4-implementation/scheduler/coordinator.md +++ b/docs/4-implementation/scheduler/coordinator.md @@ -462,7 +462,9 @@ the file carries that drop-in's `MemoryMax=`. At boot the container derives two values from a byte count: `BUN_JSC_forceRAMSize` (75%, claude-code's heap ceiling) and `MemoryHigh=` on -`hive-subagent-daemon` (two thirds). Neither is in the agent's build, so a +`hive-subagent-daemon` and on `hive-bash-daemon` (two thirds each; the bash +daemon's covers every bash task, so a build storm throttles before the +container's cap). Neither is in the agent's build, so a changed cap reaches them at the next container start without a rebuild. `infinity` sets neither. A missing or malformed file sets neither as well, and `hyperhive-memory-cap-check.service` logs a warning naming the path in the @@ -477,8 +479,8 @@ rest of the drop-in, they reach a running agent at its next spawn or rebuild. When the container reaches its cap, the kernel kills inside it, in this order: -1. a subagent run (`oom_score_adj` +1000); -2. a bash task (0); +1. a bash task (`oom_score_adj` +1000); +2. a subagent run (+900); 3. the harness and the agent's own `claude` (-500). Each subagent run and each bash task has its own cgroup with @@ -488,7 +490,7 @@ outside it. Inside the container these are and `/sys/fs/cgroup/system.slice/hive-bash-daemon.service/tasks//`; the daemon itself sits in `daemon/` beside them. A daemon that can't set these up logs one warning at start and runs everything in its own cgroup; subagent -runs still get +1000. +runs still get +900 and bash tasks +1000. The container stays up through these kills. Each one counts in the `hyperhive.container.memory.oom_kills` metric, shown as "OOM kills by agent" diff --git a/hive-agent-sock/src/run_cgroup.rs b/hive-agent-sock/src/run_cgroup.rs index a0b5a8333..746fec331 100644 --- a/hive-agent-sock/src/run_cgroup.rs +++ b/hive-agent-sock/src/run_cgroup.rs @@ -41,6 +41,16 @@ const ENTRY_PROGRAM_ENV: &str = "HIVE_RUN_ENTRY_PROGRAM"; const ENTRY_CGROUP_ENV: &str = "HIVE_RUN_ENTRY_CGROUP"; const ENTRY_OOM_SCORE_ADJ_ENV: &str = "HIVE_RUN_ENTRY_OOM_SCORE_ADJ"; +/// A bash task's `oom_score_adj`: the maximum, so under memory pressure the +/// kernel takes a bash task first. A runaway `cargo test` or `nix build` is a +/// task, and the subagent runs and the agent's own session are worth more. +pub const BASH_TASK_OOM_SCORE_ADJ: i32 = 1000; + +/// A subagent run's `oom_score_adj`: below a bash task, above everything else, so +/// the order is bash task, then subagent run, then the agent's own session (-500, +/// `nix/agent-modules/agent-service.nix`). +pub const SUBAGENT_RUN_OOM_SCORE_ADJ: i32 = 900; + /// How long a run's cgroup may stay populated after the run ended before /// removal is left to the next daemon start. const RELEASE_GRACE: Duration = Duration::from_secs(30); @@ -268,6 +278,50 @@ pub fn enter_in_pre_exec(procs: RawFd) -> io::Result<()> { Ok(()) } +/// Set the calling process's `oom_score_adj` to `adj`. Only `open(2)`, +/// `write(2)` and `close(2)` on a stack buffer, so it is safe between fork and +/// exec. Raising the score needs no privilege. +/// +/// # Errors +/// +/// The `open(2)` or `write(2)` failure. +pub fn set_oom_score_adj_in_pre_exec(adj: i32) -> io::Result<()> { + // Digits of `adj`, written backwards: no allocation, no `format!`. + let mut buf = [0_u8; 12]; + let mut at = buf.len(); + let mut n = adj.unsigned_abs(); + loop { + at -= 1; + buf[at] = b'0' + u8::try_from(n % 10).unwrap_or(0); + n /= 10; + if n == 0 { + break; + } + } + if adj < 0 { + at -= 1; + buf[at] = b'-'; + } + // SAFETY: a NUL-terminated static path, then a write of a stack buffer to + // the fd just opened, which is closed again. + unsafe { + let fd = libc::open( + c"/proc/self/oom_score_adj".as_ptr(), + libc::O_WRONLY | libc::O_CLOEXEC, + ); + if fd < 0 { + return Err(io::Error::last_os_error()); + } + let wrote = libc::write(fd, buf[at..].as_ptr().cast(), buf.len() - at); + let err = io::Error::last_os_error(); + libc::close(fd); + if wrote < 0 { + return Err(err); + } + } + Ok(()) +} + /// When this process was started by [`entry`]: enter the run's /// cgroup if it has one, set its `oom_score_adj`, and exec the run's program with this /// process's arguments; on failure print the reason and exit 127. Otherwise @@ -423,6 +477,39 @@ mod tests { assert_eq!(get(ENTRY_CGROUP_ENV), None); } + #[test] + fn the_kill_order_is_bash_task_then_subagent_run_then_the_session() { + // The agent's own session: `OOMScoreAdjust` in + // `nix/agent-modules/agent-service.nix`. + const SESSION: i32 = -500; + assert_eq!(BASH_TASK_OOM_SCORE_ADJ, 1000); + assert_eq!(SUBAGENT_RUN_OOM_SCORE_ADJ, 900); + assert!(BASH_TASK_OOM_SCORE_ADJ > SUBAGENT_RUN_OOM_SCORE_ADJ); + assert!(SUBAGENT_RUN_OOM_SCORE_ADJ > SESSION); + } + + #[test] + fn set_oom_score_adj_in_pre_exec_sets_the_score_the_child_reads_back() { + use std::process::Command; + // 0 is one digit, 7 one, 900 and 1000 are the two scores in use. + for adj in [0, 7, SUBAGENT_RUN_OOM_SCORE_ADJ, BASH_TASK_OOM_SCORE_ADJ] { + let mut cmd = Command::new("cat"); + cmd.arg("/proc/self/oom_score_adj"); + // SAFETY: the hook only calls the function under test, which is + // async-signal-safe. + unsafe { + cmd.pre_exec(move || set_oom_score_adj_in_pre_exec(adj)); + } + let out = cmd.output().expect("cat runs"); + assert!(out.status.success(), "{out:?}"); + assert_eq!( + String::from_utf8_lossy(&out.stdout).trim(), + adj.to_string(), + "adj {adj}" + ); + } + } + #[test] fn enter_in_pre_exec_writes_zero_to_the_fd() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/hive-bash-mcp/src/runner.rs b/hive-bash-mcp/src/runner.rs index 3797b9c35..66ba96848 100644 --- a/hive-bash-mcp/src/runner.rs +++ b/hive-bash-mcp/src/runner.rs @@ -786,7 +786,7 @@ async fn exec_cmd( .stdout(std::process::Stdio::from(stdout_file)) .stderr(std::process::Stdio::from(stderr_file)); let cgroup_procs = cgroup_procs.map(std::os::fd::AsRawFd::as_raw_fd); - // SAFETY: `setpgid`/`nice`/`write` are async-signal-safe and only touch + // SAFETY: `setpgid`/`nice`/`open`/`write` are async-signal-safe and only touch // the calling (child) process before exec — no allocations, no locks. // `setpgid(0, 0)` makes the child its own process-group leader so the // whole subtree can be signalled as a group (`kill(-pgid, …)`). @@ -796,6 +796,9 @@ async fn exec_cmd( cmd_builder.pre_exec(move || { libc::setpgid(0, 0); libc::nice(10); + // Before the cgroup, and a failure is not the task's: a container + // that denies the write still runs it, at the daemon's own score. + let _ = run_cgroup::set_oom_score_adj_in_pre_exec(run_cgroup::BASH_TASK_OOM_SCORE_ADJ); if let Some(procs) = cgroup_procs { run_cgroup::enter_in_pre_exec(procs)?; } @@ -1241,6 +1244,18 @@ mod tests { assert_eq!(out, "hi\n"); } + #[test] + fn a_task_and_what_it_forks_run_at_the_bash_task_score() { + let (outcome, out) = run_within( + "cat /proc/self/oom_score_adj; sh -c 'cat /proc/self/oom_score_adj'", + None, + Duration::from_secs(10), + ) + .unwrap(); + assert!(matches!(outcome, ExecOutcome::Exited(0))); + assert_eq!(out, "1000\n1000\n"); + } + #[test] fn a_backgrounded_child_does_not_hold_the_task_open() { let (outcome, out) = run_within("sleep 30 & echo started", None, Duration::from_secs(5)) diff --git a/hive-subagent-mcp/src/session.rs b/hive-subagent-mcp/src/session.rs index 37dd55be4..e3c5b82b2 100644 --- a/hive-subagent-mcp/src/session.rs +++ b/hive-subagent-mcp/src/session.rs @@ -2105,10 +2105,10 @@ fn acp_session_file(sessions: &Path, name: &str) -> PathBuf { sessions.join(format!("subagent-acp-session-{name}")) } -/// A subagent run's `oom_score_adj`: the maximum, so under memory pressure -/// the kernel takes a subagent run before a bash task (0) or the agent's -/// own session (-500, `nix/agent-modules/agent-service.nix`). -const RUN_OOM_SCORE_ADJ: i32 = 1000; +/// A subagent run's `oom_score_adj`: under memory pressure the kernel takes a +/// bash task (1000) first, then a subagent run, then the agent's own session +/// (-500, `nix/agent-modules/agent-service.nix`). +const RUN_OOM_SCORE_ADJ: i32 = run_cgroup::SUBAGENT_RUN_OOM_SCORE_ADJ; /// The ACP runtime `name`'s run drives, keeping its session id in /// [`acp_session_file`] and answering permission requests per @@ -2730,6 +2730,12 @@ fn turn_end_summary(name: &str, end: &TurnEnd, stop: Option<&StopReason>) -> Str mod tests { use super::*; + #[test] + fn a_subagent_run_is_killed_after_a_bash_task_and_before_the_session() { + assert_eq!(RUN_OOM_SCORE_ADJ, 900); + assert!(RUN_OOM_SCORE_ADJ < run_cgroup::BASH_TASK_OOM_SCORE_ADJ); + } + // `Cancel` can only be constructed from a real spawned process (no test // fixture in `hive_claude` for it), so these exercise the reservation // half of `State` directly — the actual TOCTOU-closure logic — rather diff --git a/nix/agent-modules/memory-cap.nix b/nix/agent-modules/memory-cap.nix index b570d6a7e..fa8631653 100644 --- a/nix/agent-modules/memory-cap.nix +++ b/nix/agent-modules/memory-cap.nix @@ -19,12 +19,15 @@ # it in login shells that no unit starts (`nixos-container root-login`). # `hive-agent` and `hive-subagent-daemon` spawn `claude` as a plain child, # which inherits it, so neither unit may set or unset it. -# - `MemoryHigh=` on `hive-subagent-daemon`, two thirds of the cap, as a -# generator drop-in. The daemon spawns nested `claude` sessions as plain -# children, so its cgroup already is the "all subagents" cgroup. `High`, not -# `Max`: past it the kernel reclaims hard and the subagents slow down, so the -# last third stays free for the agent's own turn, but a subagent that needs -# more still gets it when the container has memory to spare. +# - `MemoryHigh=` on `hive-subagent-daemon` and on `hive-bash-daemon`, two +# thirds of the cap each, as generator drop-ins. The subagent daemon spawns +# nested `claude` sessions as plain children, so its cgroup already is the +# "all subagents" cgroup. The bash daemon's cgroup holds every bash task +# (`tasks//`), so a `cargo` or `nix build` storm throttles itself there +# before it reaches the container's `MemoryMax=`. `High`, not `Max`: past it +# the kernel reclaims hard and the work slows down, so the last third stays +# free for the agent's own turn, but a task that needs more still gets it when +# the container has memory to spare. # # Tested by `checks.script-test-agent-memory-cap` and # `checks.module-eval-agent-memory`. @@ -52,9 +55,11 @@ let # with the unit's own file in /etc. unitGenerator = pkgs.writeShellScript "hyperhive-memory-cap" '' ${readCap "exit 0"} - dir="$1/hive-subagent-daemon.service.d" - ${pkgs.coreutils}/bin/mkdir -p "$dir" - printf '[Service]\nMemoryHigh=%s\n' "$((bytes * 2 / 3))" > "$dir/50-memory-high.conf" + for unit in hive-subagent-daemon hive-bash-daemon; do + dir="$1/$unit.service.d" + ${pkgs.coreutils}/bin/mkdir -p "$dir" + printf '[Service]\nMemoryHigh=%s\n' "$((bytes * 2 / 3))" > "$dir/50-memory-high.conf" + done ''; # The generators run before the container's journal starts and an nspawn @@ -67,7 +72,7 @@ let else why="is missing" fi - echo "<4>/run/hive/memory-max-bytes $why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies." >&2 + echo "<4>/run/hive/memory-max-bytes $why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon or hive-bash-daemon; the host's MemoryMax= on the container still applies." >&2 exit 0 ''} ''; diff --git a/nix/module-eval/agent-memory.nix b/nix/module-eval/agent-memory.nix index 16ade1dde..89dcd7cbd 100644 --- a/nix/module-eval/agent-memory.nix +++ b/nix/module-eval/agent-memory.nix @@ -31,8 +31,10 @@ let # The ceiling is sized at boot from the container's cap # (../agent-modules/memory-cap.nix). One in the unit would be the cap at # the last rebuild, and every agent would carry its own hive's number. - name = "the subagent daemon's memory ceiling is not built into its unit"; - ok = !((agentSubagentDaemon agentDefault).serviceConfig ? MemoryHigh); + name = "the subagent and bash daemons' memory ceiling is not built into their units"; + ok = + !((agentSubagentDaemon agentDefault).serviceConfig ? MemoryHigh) + && !((agentBashDaemon agentDefault).serviceConfig ? MemoryHigh); } { # claude's heap ceiling reaches it only as inherited environment: the @@ -55,11 +57,12 @@ let && lib.elem "multi-user.target" agentDefault.systemd.services.hyperhive-memory-cap-check.wantedBy; } { - # A HIGHER OOMScoreAdjust is a MORE likely victim. The subagent daemon - # puts each run at +1000 itself and bash tasks stay at 0, so the harness - # must sit below 0 for the agent's own session to go last. Neither - # daemon unit carries a value: it would make the idle daemon a likelier - # victim than a bash task, and a floor no run under it could go below. + # A HIGHER OOMScoreAdjust is a MORE likely victim. The bash daemon puts + # each task at +1000 and the subagent daemon each run at +900 itself + # (`hive_agent_sock::run_cgroup`), so the harness must sit below 0 for the + # agent's own session to go last. Neither daemon unit carries a value: it + # would make the idle daemon a likelier victim than a task, and a floor no + # run under it could go below. name = "the OOM killer takes the agent's own session last"; ok = (agentHarness agentDefault).serviceConfig.OOMScoreAdjust < 0 diff --git a/nix/script-tests/agent-memory-cap.sh b/nix/script-tests/agent-memory-cap.sh index f60ed37db..aa186eb73 100644 --- a/nix/script-tests/agent-memory-cap.sh +++ b/nix/script-tests/agent-memory-cap.sh @@ -46,15 +46,20 @@ check() { rm -rf "$out" mkdir -p "$out/normal" "$out/early" "$out/late" env -i "$dir/unit-generator" "$out/normal" "$out/early" "$out/late" - local dropin=$out/normal/hive-subagent-daemon.service.d/50-memory-high.conf if [ -z "$want_high" ]; then got=$(find "$out" -type f) if [ -n "$got" ]; then fail "unit generator wrote:"$'\n'"$got"; fi else - got=$(cat "$dropin" 2>&1) - if [ "$got" != "[Service]"$'\n'"MemoryHigh=$want_high" ]; then - fail "unit generator's drop-in:"$'\n'"$got" - fi + # The subagent daemon and the bash daemon each get one, and nothing else is written. + local unit + for unit in hive-subagent-daemon hive-bash-daemon; do + got=$(cat "$out/normal/$unit.service.d/50-memory-high.conf" 2>&1) + if [ "$got" != "[Service]"$'\n'"MemoryHigh=$want_high" ]; then + fail "unit generator's drop-in for $unit:"$'\n'"$got" + fi + done + got=$(find "$out" -type f | wc -l) + if [ "$got" -ne 2 ]; then fail "unit generator wrote $got files, wanted 2"; fi fi # The warning goes to stderr with the journal's `<4>` priority prefix, and @@ -62,7 +67,7 @@ check() { got=$(env -i "$dir/check" 2>&1) local want_warn= if [ -n "$want_why" ]; then - want_warn="<4>$cap $want_why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies." + want_warn="<4>$cap $want_why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon or hive-bash-daemon; the host's MemoryMax= on the container still applies." fi if [ "$got" != "$want_warn" ]; then fail "boot check printed:"$'\n'"$got"$'\n'"wanted:"$'\n'"$want_warn"