Watch
0
0
Fork
You've already forked hyperhive
0

bash tasks are the OOM killer's first victims: score 1000, subagent runs 900, MemoryHigh on the bash daemon

This commit is contained in:
maia 2026-11-18 20:27:14 +01:00 • committed by mara
commit 77f33a23d8
3 changed files with 155 additions and 32 deletions

View file

@ -19,12 +19,15 @@
# it in login shells that no unit starts (`nixos-container root-login`).
# `hive-agent` and `hive-subagent-daemon` spawn `claude` as a plain child,
# which inherits it, so neither unit may set or unset it.
# - `MemoryHigh=` on `hive-subagent-daemon`, two thirds of the cap, as a
# generator drop-in. The daemon spawns nested `claude` sessions as plain
# children, so its cgroup already is the "all subagents" cgroup. `High`, not
# `Max`: past it the kernel reclaims hard and the subagents slow down, so the
# last third stays free for the agent's own turn, but a subagent that needs
# more still gets it when the container has memory to spare.
# - `MemoryHigh=` on `hive-subagent-daemon` and on `hive-bash-daemon`, two
# thirds of the cap each, as generator drop-ins. The subagent daemon spawns
# nested `claude` sessions as plain children, so its cgroup already is the
# "all subagents" cgroup. The bash daemon's cgroup holds every bash task
# (`tasks/<id>/`), so a `cargo` or `nix build` storm throttles itself there
# before it reaches the container's `MemoryMax=`. `High`, not `Max`: past it
# the kernel reclaims hard and the work slows down, so the last third stays
# free for the agent's own turn, but a task that needs more still gets it when
# the container has memory to spare.
#
# Tested by `checks.script-test-agent-memory-cap` and
# `checks.module-eval-agent-memory`.
@ -52,9 +55,11 @@ let
# with the unit's own file in /etc.
unitGenerator = pkgs.writeShellScript "hyperhive-memory-cap" ''
${readCap "exit 0"}
dir="$1/hive-subagent-daemon.service.d"
${pkgs.coreutils}/bin/mkdir -p "$dir"
printf '[Service]\nMemoryHigh=%s\n' "$((bytes * 2 / 3))" > "$dir/50-memory-high.conf"
for unit in hive-subagent-daemon hive-bash-daemon; do
dir="$1/$unit.service.d"
${pkgs.coreutils}/bin/mkdir -p "$dir"
printf '[Service]\nMemoryHigh=%s\n' "$((bytes * 2 / 3))" > "$dir/50-memory-high.conf"
done
'';
# The generators run before the container's journal starts and an nspawn
@ -67,7 +72,7 @@ let
else
why="is missing"
fi
echo "<4>/run/hive/memory-max-bytes $why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies." >&2
echo "<4>/run/hive/memory-max-bytes $why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon or hive-bash-daemon; the host's MemoryMax= on the container still applies." >&2
exit 0
''}
'';

View file

@ -31,8 +31,10 @@ let
# The ceiling is sized at boot from the container's cap
# (../agent-modules/memory-cap.nix). One in the unit would be the cap at
# the last rebuild, and every agent would carry its own hive's number.
name = "the subagent daemon's memory ceiling is not built into its unit";
ok = !((agentSubagentDaemon agentDefault).serviceConfig ? MemoryHigh);
name = "the subagent and bash daemons' memory ceiling is not built into their units";
ok =
!((agentSubagentDaemon agentDefault).serviceConfig ? MemoryHigh)
&& !((agentBashDaemon agentDefault).serviceConfig ? MemoryHigh);
}
{
# claude's heap ceiling reaches it only as inherited environment: the
@ -55,11 +57,12 @@ let
&& lib.elem "multi-user.target" agentDefault.systemd.services.hyperhive-memory-cap-check.wantedBy;
}
{
# A HIGHER OOMScoreAdjust is a MORE likely victim. The subagent daemon
# puts each run at +1000 itself and bash tasks stay at 0, so the harness
# must sit below 0 for the agent's own session to go last. Neither
# daemon unit carries a value: it would make the idle daemon a likelier
# victim than a bash task, and a floor no run under it could go below.
# A HIGHER OOMScoreAdjust is a MORE likely victim. The bash daemon puts
# each task at +1000 and the subagent daemon each run at +900 itself
# (`hive_agent_sock::run_cgroup`), so the harness must sit below 0 for the
# agent's own session to go last. Neither daemon unit carries a value: it
# would make the idle daemon a likelier victim than a task, and a floor no
# run under it could go below.
name = "the OOM killer takes the agent's own session last";
ok =
(agentHarness agentDefault).serviceConfig.OOMScoreAdjust < 0

View file

@ -46,15 +46,20 @@ check() {
rm -rf "$out"
mkdir -p "$out/normal" "$out/early" "$out/late"
env -i "$dir/unit-generator" "$out/normal" "$out/early" "$out/late"
local dropin=$out/normal/hive-subagent-daemon.service.d/50-memory-high.conf
if [ -z "$want_high" ]; then
got=$(find "$out" -type f)
if [ -n "$got" ]; then fail "unit generator wrote:"$'\n'"$got"; fi
else
got=$(cat "$dropin" 2>&1)
if [ "$got" != "[Service]"$'\n'"MemoryHigh=$want_high" ]; then
fail "unit generator's drop-in:"$'\n'"$got"
fi
# The subagent daemon and the bash daemon each get one, and nothing else is written.
local unit
for unit in hive-subagent-daemon hive-bash-daemon; do
got=$(cat "$out/normal/$unit.service.d/50-memory-high.conf" 2>&1)
if [ "$got" != "[Service]"$'\n'"MemoryHigh=$want_high" ]; then
fail "unit generator's drop-in for $unit:"$'\n'"$got"
fi
done
got=$(find "$out" -type f | wc -l)
if [ "$got" -ne 2 ]; then fail "unit generator wrote $got files, wanted 2"; fi
fi
# The warning goes to stderr with the journal's `<4>` priority prefix, and
@ -62,7 +67,7 @@ check() {
got=$(env -i "$dir/check" 2>&1)
local want_warn=
if [ -n "$want_why" ]; then
want_warn="<4>$cap $want_why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies."
want_warn="<4>$cap $want_why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon or hive-bash-daemon; the host's MemoryMax= on the container still applies."
fi
if [ "$got" != "$want_warn" ]; then
fail "boot check printed:"$'\n'"$got"$'\n'"wanted:"$'\n'"$want_warn"