agents: hand over the kept drop-in's cap, warn at boot without one
When hive-c0re can't read resource-limits.json and a limits drop-in already exists, it keeps the drop-in, but it wrote no memory cap file. A container whose runtime dir had no file yet (hive-c0re upgraded, /run not cleared) then started with no heap ceiling and no subagent MemoryHigh=, silently. `stage_limits` now writes the file on every call: from the new limits, or from the last `MemoryMax=` in the kept drop-in (`infinity` when it has none or can't be read). A missing or malformed cap file at boot is no longer silent. The generators run before the container's journal and an nspawn container has no /dev/kmsg, so a oneshot `hyperhive-memory-cap-check.service` logs a warning naming the path. The agent then runs with no JSC heap ceiling and no subagent MemoryHigh=; the container's MemoryMax= still applies. module-eval-agent-memory gains a case pinning how the ceiling reaches claude: `hive-agent` and `hive-subagent-daemon` neither set nor unset BUN_JSC_forceRAMSize, both generators and the check unit are installed. Part of #5081 (slice S0-2).
This commit is contained in:
atlas
parent
0d619e215e
commit
57955a6c53
6 changed files with 144 additions and 32 deletions
|
|
@ -6,7 +6,8 @@
|
|||
# inside. hive-c0re writes its byte count to `/run/hive/memory-max-bytes`
|
||||
# before every start (`lifecycle::host_config`), and removes the file when the
|
||||
# cap is `infinity` or a RAM percentage. Two values are derived from it, and
|
||||
# neither is set without the file:
|
||||
# neither is set without the file; `hyperhive-memory-cap-check` then logs a
|
||||
# warning at boot:
|
||||
#
|
||||
# - `BUN_JSC_forceRAMSize`, 75% of the cap. Bun/JavaScriptCore, the runtime
|
||||
# under claude-code, sizes its heap ceiling off `/proc/meminfo`, which under
|
||||
|
|
@ -15,6 +16,8 @@
|
|||
# generator hands it to every unit of the container's systemd, `machinectl
|
||||
# shell` sessions (`hivectl choom`) included; `/etc/set-environment` exports
|
||||
# it in login shells that no unit starts (`nixos-container root-login`).
|
||||
# `hive-agent` and `hive-subagent-daemon` spawn `claude` as a plain child,
|
||||
# which inherits it, so neither unit may set or unset it.
|
||||
# - `MemoryHigh=` on `hive-subagent-daemon`, two thirds of the cap, as a
|
||||
# generator drop-in. The daemon spawns nested `claude` sessions as plain
|
||||
# children, so its cgroup already is the "all subagents" cgroup. `High`, not
|
||||
|
|
@ -22,34 +25,50 @@
|
|||
# last third stays free for the agent's own turn, but a subagent that needs
|
||||
# more still gets it when the container has memory to spare.
|
||||
#
|
||||
# Tested by `checks.script-test-agent-memory-cap`.
|
||||
# Tested by `checks.script-test-agent-memory-cap` and
|
||||
# `checks.module-eval-agent-memory`.
|
||||
{ pkgs, ... }:
|
||||
let
|
||||
# Sets `bytes`, or exits when there is no file or its first line is not a
|
||||
# plain byte count. A leading zero would read as octal, and 16 digits or
|
||||
# more could overflow `bytes * 75`. Shell builtins only.
|
||||
readCap = ''
|
||||
# Sets `bytes`, or runs `noCap` when there is no file or its first line is
|
||||
# not a plain byte count. A leading zero would read as octal, and 16 digits
|
||||
# or more could overflow `bytes * 75`. Shell builtins only.
|
||||
readCap = noCap: ''
|
||||
bytes=
|
||||
[ -r /run/hive/memory-max-bytes ] && read -r bytes < /run/hive/memory-max-bytes
|
||||
case "$bytes" in
|
||||
"" | 0* | *[!0-9]* | ????????????????*) exit 0 ;;
|
||||
"" | 0* | *[!0-9]* | ????????????????*) ${noCap} ;;
|
||||
esac
|
||||
'';
|
||||
|
||||
envGenerator = pkgs.writeShellScript "hyperhive-memory-cap-env" ''
|
||||
${readCap}
|
||||
${readCap "exit 0"}
|
||||
printf 'BUN_JSC_forceRAMSize=%s\n' "$((bytes * 75 / 100))"
|
||||
'';
|
||||
|
||||
# `$1` is the generator's normal-priority output dir. A drop-in there merges
|
||||
# with the unit's own file in /etc.
|
||||
unitGenerator = pkgs.writeShellScript "hyperhive-memory-cap" ''
|
||||
${readCap}
|
||||
${readCap "exit 0"}
|
||||
dir="$1/hive-subagent-daemon.service.d"
|
||||
${pkgs.coreutils}/bin/mkdir -p "$dir"
|
||||
printf '[Service]\nMemoryHigh=%s\n' "$((bytes * 2 / 3))" > "$dir/50-memory-high.conf"
|
||||
'';
|
||||
|
||||
# The generators run before the container's journal starts and an nspawn
|
||||
# container has no /dev/kmsg, so the boot warning for an unusable cap comes
|
||||
# from a unit. `<4>` is the journal's warning priority.
|
||||
check = pkgs.writeShellScript "hyperhive-memory-cap-check" ''
|
||||
${readCap ''
|
||||
if [ -e /run/hive/memory-max-bytes ]; then
|
||||
why="does not start with a plain byte count"
|
||||
else
|
||||
why="is missing"
|
||||
fi
|
||||
echo "<4>/run/hive/memory-max-bytes $why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies. hive-c0re writes no file for a cap of infinity or a RAM percentage." >&2
|
||||
exit 0
|
||||
''}
|
||||
'';
|
||||
|
||||
# Sourced by every shell's `/etc/set-environment`, so POSIX sh only.
|
||||
profile = pkgs.writeText "hyperhive-memory-cap.sh" ''
|
||||
__hyperhive_cap=$(${envGenerator})
|
||||
|
|
@ -63,6 +82,14 @@ in
|
|||
environment.etc."systemd/system-environment-generators/50-hyperhive-memory-cap".source =
|
||||
envGenerator;
|
||||
systemd.generators.hyperhive-memory-cap = unitGenerator;
|
||||
systemd.services.hyperhive-memory-cap-check = {
|
||||
description = "Warn when the container's memory cap is unusable";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = check;
|
||||
};
|
||||
};
|
||||
environment.etc."hyperhive/memory-cap.sh".source = profile;
|
||||
environment.extraInit = ". /etc/hyperhive/memory-cap.sh";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,6 +34,26 @@ let
|
|||
name = "the subagent daemon's memory ceiling is not built into its unit";
|
||||
ok = !((agentSubagentDaemon agentDefault).serviceConfig ? MemoryHigh);
|
||||
}
|
||||
{
|
||||
# claude's heap ceiling reaches it only as inherited environment: the
|
||||
# environment generator fills the manager's, systemd hands that to every
|
||||
# unit, and both units spawn claude as a plain child. A unit that sets
|
||||
# or unsets the variable itself cuts that. Without the two generators the
|
||||
# cap reaches nothing, and without the check unit nothing says so.
|
||||
name = "claude's units take the heap ceiling from the memory cap read at boot";
|
||||
ok =
|
||||
let
|
||||
leavesIt =
|
||||
unit:
|
||||
!(unit.environment ? BUN_JSC_forceRAMSize)
|
||||
&& !(lib.elem "BUN_JSC_forceRAMSize" (lib.toList (unit.serviceConfig.UnsetEnvironment or [ ])));
|
||||
in
|
||||
leavesIt (agentHarness agentDefault)
|
||||
&& leavesIt (agentSubagentDaemon agentDefault)
|
||||
&& agentDefault.environment.etc ? "systemd/system-environment-generators/50-hyperhive-memory-cap"
|
||||
&& agentDefault.systemd.generators ? hyperhive-memory-cap
|
||||
&& lib.elem "multi-user.target" agentDefault.systemd.services.hyperhive-memory-cap-check.wantedBy;
|
||||
}
|
||||
{
|
||||
# The sign is the whole property, and it is easy to write backwards:
|
||||
# a HIGHER OOMScoreAdjust is a MORE likely victim. So the subagent
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
# `checks.script-test-agent-memory-cap` — runs the three readers of
|
||||
# `checks.script-test-agent-memory-cap` — runs the four readers of
|
||||
# `/run/hive/memory-max-bytes` in ../agent-modules/memory-cap.nix: the
|
||||
# environment generator, the unit generator, and the login-shell snippet
|
||||
# `/etc/set-environment` sources. The cases are in ./agent-memory-cap.sh.
|
||||
# environment generator, the unit generator, the boot check that warns about
|
||||
# an unusable cap, and the login-shell snippet `/etc/set-environment` sources.
|
||||
# The cases are in ./agent-memory-cap.sh.
|
||||
#
|
||||
# What runs is the rendered scripts. The one edit is the cap file's path, moved
|
||||
# under the build directory, and the snippet's reference to the environment
|
||||
|
|
@ -32,6 +33,7 @@ pkgs.runCommand "hyperhive-script-test-agent-memory-cap"
|
|||
ENV_GENERATOR =
|
||||
machine.environment.etc."systemd/system-environment-generators/50-hyperhive-memory-cap".source;
|
||||
UNIT_GENERATOR = machine.systemd.generators.hyperhive-memory-cap;
|
||||
CHECK = machine.systemd.services.hyperhive-memory-cap-check.serviceConfig.ExecStart;
|
||||
PROFILE = machine.environment.etc."hyperhive/memory-cap.sh".source;
|
||||
EXTRA_INIT = machine.environment.extraInit;
|
||||
SHELLS = "${pkgs.dash}/bin/dash ${pkgs.bash}/bin/sh";
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# Cases for ./agent-memory-cap.nix. Each case writes a cap file (or none), runs
|
||||
# both generators with a clean environment and compares what they produced,
|
||||
# then sources the login-shell snippet under each POSIX shell and compares what
|
||||
# a child process sees.
|
||||
# both generators and the boot check with a clean environment and compares what
|
||||
# they produced, then sources the login-shell snippet under each POSIX shell
|
||||
# and compares what a child process sees.
|
||||
set -uo pipefail
|
||||
|
||||
failures=0
|
||||
|
|
@ -16,12 +16,13 @@ dir=$(mktemp -d)
|
|||
cap=$dir/memory-max-bytes
|
||||
sed "s|/run/hive/memory-max-bytes|$cap|g" "$ENV_GENERATOR" >"$dir/env-generator"
|
||||
sed "s|/run/hive/memory-max-bytes|$cap|g" "$UNIT_GENERATOR" >"$dir/unit-generator"
|
||||
chmod +x "$dir/env-generator" "$dir/unit-generator"
|
||||
sed "s|/run/hive/memory-max-bytes|$cap|g" "$CHECK" >"$dir/check"
|
||||
chmod +x "$dir/env-generator" "$dir/unit-generator" "$dir/check"
|
||||
sed "s|$ENV_GENERATOR|$dir/env-generator|g" "$PROFILE" >"$dir/profile"
|
||||
envbin=$(command -v env)
|
||||
|
||||
case="setup"
|
||||
for g in env-generator unit-generator; do
|
||||
for g in env-generator unit-generator check; do
|
||||
if grep -q '/run/hive/memory-max-bytes' "$dir/$g"; then fail "$g still names /run/hive after the rewrite"; fi
|
||||
if ! grep -qF "$cap" "$dir/$g"; then fail "the rewrite did not reach $g's read"; fi
|
||||
done
|
||||
|
|
@ -31,10 +32,11 @@ if ! printf '%s\n' "$EXTRA_INIT" | grep -qxF '. /etc/hyperhive/memory-cap.sh'; t
|
|||
fi
|
||||
|
||||
# check <description> <expected BUN_JSC_forceRAMSize or ""> <expected MemoryHigh or "">
|
||||
# <why the boot check warns, or "">
|
||||
check() {
|
||||
case=$1
|
||||
count=$((count + 1))
|
||||
local want_env=${2:+BUN_JSC_forceRAMSize=$2} want_high=$3 got
|
||||
local want_env=${2:+BUN_JSC_forceRAMSize=$2} want_high=$3 want_why=${4:-} got
|
||||
got=$(env -i "$dir/env-generator")
|
||||
if [ "$got" != "$want_env" ]; then
|
||||
fail "environment generator printed:"$'\n'"$got"$'\n'"wanted:"$'\n'"$want_env"
|
||||
|
|
@ -55,6 +57,17 @@ check() {
|
|||
fi
|
||||
fi
|
||||
|
||||
# The warning goes to stderr with the journal's `<4>` priority prefix, and
|
||||
# nothing goes anywhere for a usable cap.
|
||||
got=$(env -i "$dir/check" 2>&1)
|
||||
local want_warn=
|
||||
if [ -n "$want_why" ]; then
|
||||
want_warn="<4>$cap $want_why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies. hive-c0re writes no file for a cap of infinity or a RAM percentage."
|
||||
fi
|
||||
if [ "$got" != "$want_warn" ]; then
|
||||
fail "boot check printed:"$'\n'"$got"$'\n'"wanted:"$'\n'"$want_warn"
|
||||
fi
|
||||
|
||||
local sh
|
||||
for sh in $SHELLS; do
|
||||
# Only what the snippet exported reaches the child's environment. No PATH:
|
||||
|
|
@ -76,19 +89,19 @@ printf '4294967296\nLD_PRELOAD=/x\n' >"$cap"
|
|||
check "only the first line is read" 3221225472 2863311530
|
||||
|
||||
printf '8G\n' >"$cap"
|
||||
check "a size with a suffix is ignored" "" ""
|
||||
check "a size with a suffix is ignored" "" "" "does not start with a plain byte count"
|
||||
|
||||
printf '0123\n' >"$cap"
|
||||
check "a leading zero is ignored" "" ""
|
||||
check "a leading zero is ignored" "" "" "does not start with a plain byte count"
|
||||
|
||||
printf '1234567890123456\n' >"$cap"
|
||||
check "sixteen digits are ignored" "" ""
|
||||
check "sixteen digits are ignored" "" "" "does not start with a plain byte count"
|
||||
|
||||
: >"$cap"
|
||||
check "an empty file" "" ""
|
||||
check "an empty file" "" "" "does not start with a plain byte count"
|
||||
|
||||
rm -f "$cap"
|
||||
check "no file" "" ""
|
||||
check "no file" "" "" "is missing"
|
||||
|
||||
echo "$count cases, $failures failed"
|
||||
[ "$failures" -eq 0 ]
|
||||
|
|
|
|||
Loading…
Reference in a new issue