bash tasks are the OOM killer's first victims: score 1000, subagent runs 900, MemoryHigh on the bash daemon
This commit is contained in:
parent
d4dc8b48b1
commit
77f33a23d8
3 changed files with 155 additions and 32 deletions
|
|
@ -19,12 +19,15 @@
|
|||
# it in login shells that no unit starts (`nixos-container root-login`).
|
||||
# `hive-agent` and `hive-subagent-daemon` spawn `claude` as a plain child,
|
||||
# which inherits it, so neither unit may set or unset it.
|
||||
# - `MemoryHigh=` on `hive-subagent-daemon`, two thirds of the cap, as a
|
||||
# generator drop-in. The daemon spawns nested `claude` sessions as plain
|
||||
# children, so its cgroup already is the "all subagents" cgroup. `High`, not
|
||||
# `Max`: past it the kernel reclaims hard and the subagents slow down, so the
|
||||
# last third stays free for the agent's own turn, but a subagent that needs
|
||||
# more still gets it when the container has memory to spare.
|
||||
# - `MemoryHigh=` on `hive-subagent-daemon` and on `hive-bash-daemon`, two
|
||||
# thirds of the cap each, as generator drop-ins. The subagent daemon spawns
|
||||
# nested `claude` sessions as plain children, so its cgroup already is the
|
||||
# "all subagents" cgroup. The bash daemon's cgroup holds every bash task
|
||||
# (`tasks/<id>/`), so a `cargo` or `nix build` storm throttles itself there
|
||||
# before it reaches the container's `MemoryMax=`. `High`, not `Max`: past it
|
||||
# the kernel reclaims hard and the work slows down, so the last third stays
|
||||
# free for the agent's own turn, but a task that needs more still gets it when
|
||||
# the container has memory to spare.
|
||||
#
|
||||
# Tested by `checks.script-test-agent-memory-cap` and
|
||||
# `checks.module-eval-agent-memory`.
|
||||
|
|
@ -52,9 +55,11 @@ let
|
|||
# with the unit's own file in /etc.
|
||||
unitGenerator = pkgs.writeShellScript "hyperhive-memory-cap" ''
|
||||
${readCap "exit 0"}
|
||||
dir="$1/hive-subagent-daemon.service.d"
|
||||
${pkgs.coreutils}/bin/mkdir -p "$dir"
|
||||
printf '[Service]\nMemoryHigh=%s\n' "$((bytes * 2 / 3))" > "$dir/50-memory-high.conf"
|
||||
for unit in hive-subagent-daemon hive-bash-daemon; do
|
||||
dir="$1/$unit.service.d"
|
||||
${pkgs.coreutils}/bin/mkdir -p "$dir"
|
||||
printf '[Service]\nMemoryHigh=%s\n' "$((bytes * 2 / 3))" > "$dir/50-memory-high.conf"
|
||||
done
|
||||
'';
|
||||
|
||||
# The generators run before the container's journal starts and an nspawn
|
||||
|
|
@ -67,7 +72,7 @@ let
|
|||
else
|
||||
why="is missing"
|
||||
fi
|
||||
echo "<4>/run/hive/memory-max-bytes $why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies." >&2
|
||||
echo "<4>/run/hive/memory-max-bytes $why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon or hive-bash-daemon; the host's MemoryMax= on the container still applies." >&2
|
||||
exit 0
|
||||
''}
|
||||
'';
|
||||
|
|
|
|||
|
|
@ -31,8 +31,10 @@ let
|
|||
# The ceiling is sized at boot from the container's cap
|
||||
# (../agent-modules/memory-cap.nix). One in the unit would be the cap at
|
||||
# the last rebuild, and every agent would carry its own hive's number.
|
||||
name = "the subagent daemon's memory ceiling is not built into its unit";
|
||||
ok = !((agentSubagentDaemon agentDefault).serviceConfig ? MemoryHigh);
|
||||
name = "the subagent and bash daemons' memory ceiling is not built into their units";
|
||||
ok =
|
||||
!((agentSubagentDaemon agentDefault).serviceConfig ? MemoryHigh)
|
||||
&& !((agentBashDaemon agentDefault).serviceConfig ? MemoryHigh);
|
||||
}
|
||||
{
|
||||
# claude's heap ceiling reaches it only as inherited environment: the
|
||||
|
|
@ -55,11 +57,12 @@ let
|
|||
&& lib.elem "multi-user.target" agentDefault.systemd.services.hyperhive-memory-cap-check.wantedBy;
|
||||
}
|
||||
{
|
||||
# A HIGHER OOMScoreAdjust is a MORE likely victim. The subagent daemon
|
||||
# puts each run at +1000 itself and bash tasks stay at 0, so the harness
|
||||
# must sit below 0 for the agent's own session to go last. Neither
|
||||
# daemon unit carries a value: it would make the idle daemon a likelier
|
||||
# victim than a bash task, and a floor no run under it could go below.
|
||||
# A HIGHER OOMScoreAdjust is a MORE likely victim. The bash daemon puts
|
||||
# each task at +1000 and the subagent daemon each run at +900 itself
|
||||
# (`hive_agent_sock::run_cgroup`), so the harness must sit below 0 for the
|
||||
# agent's own session to go last. Neither daemon unit carries a value: it
|
||||
# would make the idle daemon a likelier victim than a task, and a floor no
|
||||
# run under it could go below.
|
||||
name = "the OOM killer takes the agent's own session last";
|
||||
ok =
|
||||
(agentHarness agentDefault).serviceConfig.OOMScoreAdjust < 0
|
||||
|
|
|
|||
|
|
@ -46,15 +46,20 @@ check() {
|
|||
rm -rf "$out"
|
||||
mkdir -p "$out/normal" "$out/early" "$out/late"
|
||||
env -i "$dir/unit-generator" "$out/normal" "$out/early" "$out/late"
|
||||
local dropin=$out/normal/hive-subagent-daemon.service.d/50-memory-high.conf
|
||||
if [ -z "$want_high" ]; then
|
||||
got=$(find "$out" -type f)
|
||||
if [ -n "$got" ]; then fail "unit generator wrote:"$'\n'"$got"; fi
|
||||
else
|
||||
got=$(cat "$dropin" 2>&1)
|
||||
if [ "$got" != "[Service]"$'\n'"MemoryHigh=$want_high" ]; then
|
||||
fail "unit generator's drop-in:"$'\n'"$got"
|
||||
fi
|
||||
# The subagent daemon and the bash daemon each get one, and nothing else is written.
|
||||
local unit
|
||||
for unit in hive-subagent-daemon hive-bash-daemon; do
|
||||
got=$(cat "$out/normal/$unit.service.d/50-memory-high.conf" 2>&1)
|
||||
if [ "$got" != "[Service]"$'\n'"MemoryHigh=$want_high" ]; then
|
||||
fail "unit generator's drop-in for $unit:"$'\n'"$got"
|
||||
fi
|
||||
done
|
||||
got=$(find "$out" -type f | wc -l)
|
||||
if [ "$got" -ne 2 ]; then fail "unit generator wrote $got files, wanted 2"; fi
|
||||
fi
|
||||
|
||||
# The warning goes to stderr with the journal's `<4>` priority prefix, and
|
||||
|
|
@ -62,7 +67,7 @@ check() {
|
|||
got=$(env -i "$dir/check" 2>&1)
|
||||
local want_warn=
|
||||
if [ -n "$want_why" ]; then
|
||||
want_warn="<4>$cap $want_why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies."
|
||||
want_warn="<4>$cap $want_why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon or hive-bash-daemon; the host's MemoryMax= on the container still applies."
|
||||
fi
|
||||
if [ "$got" != "$want_warn" ]; then
|
||||
fail "boot check printed:"$'\n'"$got"$'\n'"wanted:"$'\n'"$want_warn"
|
||||
|
|
|
|||
Loading…
Reference in a new issue