Watch
0
0
Fork
You've already forked hyperhive
0

bash tasks are the OOM killer's first victims: score 1000, subagent runs 900, MemoryHigh on the bash daemon

This commit is contained in:
maia 2026-11-18 20:27:14 +01:00 • committed by mara
commit 77f33a23d8
7 changed files with 155 additions and 32 deletions

View file

@ -462,7 +462,9 @@ the file carries that drop-in's `MemoryMax=`.
At boot the container derives two values from a byte count:
`BUN_JSC_forceRAMSize` (75%, claude-code's heap ceiling) and `MemoryHigh=` on
`hive-subagent-daemon` (two thirds). Neither is in the agent's build, so a
`hive-subagent-daemon` and on `hive-bash-daemon` (two thirds each; the bash
daemon's covers every bash task, so a build storm throttles before the
container's cap). Neither is in the agent's build, so a
changed cap reaches them at the next container start without a rebuild.
`infinity` sets neither. A missing or malformed file sets neither as well, and
`hyperhive-memory-cap-check.service` logs a warning naming the path in the
@ -477,8 +479,8 @@ rest of the drop-in, they reach a running agent at its next spawn or rebuild.
When the container reaches its cap, the kernel kills inside it, in this order:
1. a subagent run (`oom_score_adj` +1000);
2. a bash task (0);
1. a bash task (`oom_score_adj` +1000);
2. a subagent run (+900);
3. the harness and the agent's own `claude` (-500).
Each subagent run and each bash task has its own cgroup with
@ -488,7 +490,7 @@ outside it. Inside the container these are
and `/sys/fs/cgroup/system.slice/hive-bash-daemon.service/tasks/<id>/`; the
daemon itself sits in `daemon/` beside them. A daemon that can't set these
up logs one warning at start and runs everything in its own cgroup; subagent
runs still get +1000.
runs still get +900 and bash tasks +1000.
The container stays up through these kills. Each one counts in the
`hyperhive.container.memory.oom_kills` metric, shown as "OOM kills by agent"

View file

@ -41,6 +41,16 @@ const ENTRY_PROGRAM_ENV: &str = "HIVE_RUN_ENTRY_PROGRAM";
const ENTRY_CGROUP_ENV: &str = "HIVE_RUN_ENTRY_CGROUP";
const ENTRY_OOM_SCORE_ADJ_ENV: &str = "HIVE_RUN_ENTRY_OOM_SCORE_ADJ";
/// A bash task's `oom_score_adj`: the maximum, so under memory pressure the
/// kernel takes a bash task first. A runaway `cargo test` or `nix build` is a
/// task, and the subagent runs and the agent's own session are worth more.
pub const BASH_TASK_OOM_SCORE_ADJ: i32 = 1000;
/// A subagent run's `oom_score_adj`: below a bash task, above everything else, so
/// the order is bash task, then subagent run, then the agent's own session (-500,
/// `nix/agent-modules/agent-service.nix`).
pub const SUBAGENT_RUN_OOM_SCORE_ADJ: i32 = 900;
/// How long a run's cgroup may stay populated after the run ended before
/// removal is left to the next daemon start.
const RELEASE_GRACE: Duration = Duration::from_secs(30);
@ -268,6 +278,50 @@ pub fn enter_in_pre_exec(procs: RawFd) -> io::Result<()> {
Ok(())
}
/// Set the calling process's `oom_score_adj` to `adj`. Only `open(2)`,
/// `write(2)` and `close(2)` on a stack buffer, so it is safe between fork and
/// exec. Raising the score needs no privilege.
///
/// # Errors
///
/// The `open(2)` or `write(2)` failure.
pub fn set_oom_score_adj_in_pre_exec(adj: i32) -> io::Result<()> {
// Digits of `adj`, written backwards: no allocation, no `format!`.
let mut buf = [0_u8; 12];
let mut at = buf.len();
let mut n = adj.unsigned_abs();
loop {
at -= 1;
buf[at] = b'0' + u8::try_from(n % 10).unwrap_or(0);
n /= 10;
if n == 0 {
break;
}
}
if adj < 0 {
at -= 1;
buf[at] = b'-';
}
// SAFETY: a NUL-terminated static path, then a write of a stack buffer to
// the fd just opened, which is closed again.
unsafe {
let fd = libc::open(
c"/proc/self/oom_score_adj".as_ptr(),
libc::O_WRONLY | libc::O_CLOEXEC,
);
if fd < 0 {
return Err(io::Error::last_os_error());
}
let wrote = libc::write(fd, buf[at..].as_ptr().cast(), buf.len() - at);
let err = io::Error::last_os_error();
libc::close(fd);
if wrote < 0 {
return Err(err);
}
}
Ok(())
}
/// When this process was started by [`entry`]: enter the run's
/// cgroup if it has one, set its `oom_score_adj`, and exec the run's program with this
/// process's arguments; on failure print the reason and exit 127. Otherwise
@ -423,6 +477,39 @@ mod tests {
assert_eq!(get(ENTRY_CGROUP_ENV), None);
}
#[test]
fn the_kill_order_is_bash_task_then_subagent_run_then_the_session() {
// The agent's own session: `OOMScoreAdjust` in
// `nix/agent-modules/agent-service.nix`.
const SESSION: i32 = -500;
assert_eq!(BASH_TASK_OOM_SCORE_ADJ, 1000);
assert_eq!(SUBAGENT_RUN_OOM_SCORE_ADJ, 900);
assert!(BASH_TASK_OOM_SCORE_ADJ > SUBAGENT_RUN_OOM_SCORE_ADJ);
assert!(SUBAGENT_RUN_OOM_SCORE_ADJ > SESSION);
}
#[test]
fn set_oom_score_adj_in_pre_exec_sets_the_score_the_child_reads_back() {
use std::process::Command;
// 0 is one digit, 7 one, 900 and 1000 are the two scores in use.
for adj in [0, 7, SUBAGENT_RUN_OOM_SCORE_ADJ, BASH_TASK_OOM_SCORE_ADJ] {
let mut cmd = Command::new("cat");
cmd.arg("/proc/self/oom_score_adj");
// SAFETY: the hook only calls the function under test, which is
// async-signal-safe.
unsafe {
cmd.pre_exec(move || set_oom_score_adj_in_pre_exec(adj));
}
let out = cmd.output().expect("cat runs");
assert!(out.status.success(), "{out:?}");
assert_eq!(
String::from_utf8_lossy(&out.stdout).trim(),
adj.to_string(),
"adj {adj}"
);
}
}
#[test]
fn enter_in_pre_exec_writes_zero_to_the_fd() {
let dir = tempfile::tempdir().expect("tempdir");

View file

@ -786,7 +786,7 @@ async fn exec_cmd(
.stdout(std::process::Stdio::from(stdout_file))
.stderr(std::process::Stdio::from(stderr_file));
let cgroup_procs = cgroup_procs.map(std::os::fd::AsRawFd::as_raw_fd);
// SAFETY: `setpgid`/`nice`/`write` are async-signal-safe and only touch
// SAFETY: `setpgid`/`nice`/`open`/`write` are async-signal-safe and only touch
// the calling (child) process before exec — no allocations, no locks.
// `setpgid(0, 0)` makes the child its own process-group leader so the
// whole subtree can be signalled as a group (`kill(-pgid, …)`).
@ -796,6 +796,9 @@ async fn exec_cmd(
cmd_builder.pre_exec(move || {
libc::setpgid(0, 0);
libc::nice(10);
// Before the cgroup, and a failure is not the task's: a container
// that denies the write still runs it, at the daemon's own score.
let _ = run_cgroup::set_oom_score_adj_in_pre_exec(run_cgroup::BASH_TASK_OOM_SCORE_ADJ);
if let Some(procs) = cgroup_procs {
run_cgroup::enter_in_pre_exec(procs)?;
}
@ -1241,6 +1244,18 @@ mod tests {
assert_eq!(out, "hi\n");
}
#[test]
fn a_task_and_what_it_forks_run_at_the_bash_task_score() {
let (outcome, out) = run_within(
"cat /proc/self/oom_score_adj; sh -c 'cat /proc/self/oom_score_adj'",
None,
Duration::from_secs(10),
)
.unwrap();
assert!(matches!(outcome, ExecOutcome::Exited(0)));
assert_eq!(out, "1000\n1000\n");
}
#[test]
fn a_backgrounded_child_does_not_hold_the_task_open() {
let (outcome, out) = run_within("sleep 30 & echo started", None, Duration::from_secs(5))

View file

@ -2105,10 +2105,10 @@ fn acp_session_file(sessions: &Path, name: &str) -> PathBuf {
sessions.join(format!("subagent-acp-session-{name}"))
}
/// A subagent run's `oom_score_adj`: the maximum, so under memory pressure
/// the kernel takes a subagent run before a bash task (0) or the agent's
/// own session (-500, `nix/agent-modules/agent-service.nix`).
const RUN_OOM_SCORE_ADJ: i32 = 1000;
/// A subagent run's `oom_score_adj`: under memory pressure the kernel takes a
/// bash task (1000) first, then a subagent run, then the agent's own session
/// (-500, `nix/agent-modules/agent-service.nix`).
const RUN_OOM_SCORE_ADJ: i32 = run_cgroup::SUBAGENT_RUN_OOM_SCORE_ADJ;
/// The ACP runtime `name`'s run drives, keeping its session id in
/// [`acp_session_file`] and answering permission requests per
@ -2730,6 +2730,12 @@ fn turn_end_summary(name: &str, end: &TurnEnd, stop: Option<&StopReason>) -> Str
mod tests {
use super::*;
#[test]
fn a_subagent_run_is_killed_after_a_bash_task_and_before_the_session() {
assert_eq!(RUN_OOM_SCORE_ADJ, 900);
assert!(RUN_OOM_SCORE_ADJ < run_cgroup::BASH_TASK_OOM_SCORE_ADJ);
}
// `Cancel` can only be constructed from a real spawned process (no test
// fixture in `hive_claude` for it), so these exercise the reservation
// half of `State` directly — the actual TOCTOU-closure logic — rather

View file

@ -19,12 +19,15 @@
# it in login shells that no unit starts (`nixos-container root-login`).
# `hive-agent` and `hive-subagent-daemon` spawn `claude` as a plain child,
# which inherits it, so neither unit may set or unset it.
# - `MemoryHigh=` on `hive-subagent-daemon`, two thirds of the cap, as a
# generator drop-in. The daemon spawns nested `claude` sessions as plain
# children, so its cgroup already is the "all subagents" cgroup. `High`, not
# `Max`: past it the kernel reclaims hard and the subagents slow down, so the
# last third stays free for the agent's own turn, but a subagent that needs
# more still gets it when the container has memory to spare.
# - `MemoryHigh=` on `hive-subagent-daemon` and on `hive-bash-daemon`, two
# thirds of the cap each, as generator drop-ins. The subagent daemon spawns
# nested `claude` sessions as plain children, so its cgroup already is the
# "all subagents" cgroup. The bash daemon's cgroup holds every bash task
# (`tasks/<id>/`), so a `cargo` or `nix build` storm throttles itself there
# before it reaches the container's `MemoryMax=`. `High`, not `Max`: past it
# the kernel reclaims hard and the work slows down, so the last third stays
# free for the agent's own turn, but a task that needs more still gets it when
# the container has memory to spare.
#
# Tested by `checks.script-test-agent-memory-cap` and
# `checks.module-eval-agent-memory`.
@ -52,9 +55,11 @@ let
# with the unit's own file in /etc.
unitGenerator = pkgs.writeShellScript "hyperhive-memory-cap" ''
${readCap "exit 0"}
dir="$1/hive-subagent-daemon.service.d"
for unit in hive-subagent-daemon hive-bash-daemon; do
dir="$1/$unit.service.d"
${pkgs.coreutils}/bin/mkdir -p "$dir"
printf '[Service]\nMemoryHigh=%s\n' "$((bytes * 2 / 3))" > "$dir/50-memory-high.conf"
done
'';
# The generators run before the container's journal starts and an nspawn
@ -67,7 +72,7 @@ let
else
why="is missing"
fi
echo "<4>/run/hive/memory-max-bytes $why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies." >&2
echo "<4>/run/hive/memory-max-bytes $why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon or hive-bash-daemon; the host's MemoryMax= on the container still applies." >&2
exit 0
''}
'';

View file

@ -31,8 +31,10 @@ let
# The ceiling is sized at boot from the container's cap
# (../agent-modules/memory-cap.nix). One in the unit would be the cap at
# the last rebuild, and every agent would carry its own hive's number.
name = "the subagent daemon's memory ceiling is not built into its unit";
ok = !((agentSubagentDaemon agentDefault).serviceConfig ? MemoryHigh);
name = "the subagent and bash daemons' memory ceiling is not built into their units";
ok =
!((agentSubagentDaemon agentDefault).serviceConfig ? MemoryHigh)
&& !((agentBashDaemon agentDefault).serviceConfig ? MemoryHigh);
}
{
# claude's heap ceiling reaches it only as inherited environment: the
@ -55,11 +57,12 @@ let
&& lib.elem "multi-user.target" agentDefault.systemd.services.hyperhive-memory-cap-check.wantedBy;
}
{
# A HIGHER OOMScoreAdjust is a MORE likely victim. The subagent daemon
# puts each run at +1000 itself and bash tasks stay at 0, so the harness
# must sit below 0 for the agent's own session to go last. Neither
# daemon unit carries a value: it would make the idle daemon a likelier
# victim than a bash task, and a floor no run under it could go below.
# A HIGHER OOMScoreAdjust is a MORE likely victim. The bash daemon puts
# each task at +1000 and the subagent daemon each run at +900 itself
# (`hive_agent_sock::run_cgroup`), so the harness must sit below 0 for the
# agent's own session to go last. Neither daemon unit carries a value: it
# would make the idle daemon a likelier victim than a task, and a floor no
# run under it could go below.
name = "the OOM killer takes the agent's own session last";
ok =
(agentHarness agentDefault).serviceConfig.OOMScoreAdjust < 0

View file

@ -46,15 +46,20 @@ check() {
rm -rf "$out"
mkdir -p "$out/normal" "$out/early" "$out/late"
env -i "$dir/unit-generator" "$out/normal" "$out/early" "$out/late"
local dropin=$out/normal/hive-subagent-daemon.service.d/50-memory-high.conf
if [ -z "$want_high" ]; then
got=$(find "$out" -type f)
if [ -n "$got" ]; then fail "unit generator wrote:"$'\n'"$got"; fi
else
got=$(cat "$dropin" 2>&1)
# The subagent daemon and the bash daemon each get one, and nothing else is written.
local unit
for unit in hive-subagent-daemon hive-bash-daemon; do
got=$(cat "$out/normal/$unit.service.d/50-memory-high.conf" 2>&1)
if [ "$got" != "[Service]"$'\n'"MemoryHigh=$want_high" ]; then
fail "unit generator's drop-in:"$'\n'"$got"
fail "unit generator's drop-in for $unit:"$'\n'"$got"
fi
done
got=$(find "$out" -type f | wc -l)
if [ "$got" -ne 2 ]; then fail "unit generator wrote $got files, wanted 2"; fi
fi
# The warning goes to stderr with the journal's `<4>` priority prefix, and
@ -62,7 +67,7 @@ check() {
got=$(env -i "$dir/check" 2>&1)
local want_warn=
if [ -n "$want_why" ]; then
want_warn="<4>$cap $want_why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies."
want_warn="<4>$cap $want_why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon or hive-bash-daemon; the host's MemoryMax= on the container still applies."
fi
if [ "$got" != "$want_warn" ]; then
fail "boot check printed:"$'\n'"$got"$'\n'"wanted:"$'\n'"$want_warn"