agents: hand over the kept drop-in's cap, warn at boot without one
When hive-c0re can't read resource-limits.json and a limits drop-in already exists, it keeps the drop-in, but it wrote no memory cap file. A container whose runtime dir had no file yet (hive-c0re upgraded, /run not cleared) then started with no heap ceiling and no subagent MemoryHigh=, silently. `stage_limits` now writes the file on every call: from the new limits, or from the last `MemoryMax=` in the kept drop-in (`infinity` when it has none or can't be read). A missing or malformed cap file at boot is no longer silent. The generators run before the container's journal and an nspawn container has no /dev/kmsg, so a oneshot `hyperhive-memory-cap-check.service` logs a warning naming the path. The agent then runs with no JSC heap ceiling and no subagent MemoryHigh=; the container's MemoryMax= still applies. module-eval-agent-memory gains a case pinning how the ceiling reaches claude: `hive-agent` and `hive-subagent-daemon` neither set nor unset BUN_JSC_forceRAMSize, both generators and the check unit are installed. Part of #5081 (slice S0-2).
This commit is contained in:
atlas
parent
0d619e215e
commit
57955a6c53
6 changed files with 144 additions and 32 deletions
|
|
@ -459,6 +459,11 @@ values from it: `BUN_JSC_forceRAMSize` (75%, claude-code's heap ceiling)
|
|||
and `MemoryHigh=` on `hive-subagent-daemon` (two thirds). Neither is in the
|
||||
agent's build, so a changed cap reaches them at the next container start
|
||||
without a rebuild. A cap of `infinity` or a RAM percentage sets neither.
|
||||
When hive-c0re keeps the last drop-in because it can't read the overrides
|
||||
file, the file carries that drop-in's `MemoryMax=`. A missing or malformed
|
||||
file sets neither value, and `hyperhive-memory-cap-check.service` logs a
|
||||
warning naming the path in the container's journal at boot. The
|
||||
container's `MemoryMax=` still applies.
|
||||
|
||||
The same drop-in carries `CPUWeight=` / `IOWeight=` from
|
||||
`agentCpuWeight` / `agentIoWeight`. Those are a different kind of
|
||||
|
|
|
|||
|
|
@ -61,8 +61,8 @@ const MEMORY_CAP_FILE: &str = "memory-max-bytes";
|
|||
/// [`limits_to_apply`], plus handing the memory cap to the agent through
|
||||
/// [`MEMORY_CAP_FILE`] in `runtime_dir`. The container cannot read the cap
|
||||
/// systemd enforces on it, so this file is how it learns the value it sizes
|
||||
/// its heap and its subagents against. When the drop-in is kept as it is, so is
|
||||
/// the file.
|
||||
/// its heap and its subagents against. It is written on every call, from the
|
||||
/// `MemoryMax=` the agent will start under: the new one, or the kept drop-in's.
|
||||
fn stage_limits(
|
||||
agent_name: &str,
|
||||
limits_path: &Path,
|
||||
|
|
@ -71,12 +71,39 @@ fn stage_limits(
|
|||
hive: &HiveEnv,
|
||||
) -> Result<Option<(String, String)>> {
|
||||
let limits = limits_to_apply(agent_name, limits_path, dropin_path, hive);
|
||||
if let Some((_, memory_max)) = &limits {
|
||||
write_memory_cap(runtime_dir, agent_name, memory_max)?;
|
||||
}
|
||||
let memory_max = match &limits {
|
||||
Some((_, memory_max)) => memory_max.clone(),
|
||||
None => dropin_memory_max(agent_name, dropin_path),
|
||||
};
|
||||
write_memory_cap(runtime_dir, agent_name, &memory_max)?;
|
||||
Ok(limits)
|
||||
}
|
||||
|
||||
/// The `MemoryMax=` the drop-in at `path` applies: its last assignment, as
|
||||
/// systemd reads it. `infinity`, systemd's default, when it has none, its last
|
||||
/// one is empty, or the file can't be read.
|
||||
fn dropin_memory_max(agent_name: &str, path: &Path) -> String {
|
||||
let body = match std::fs::read_to_string(path) {
|
||||
Ok(body) => body,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
agent = %agent_name,
|
||||
path = %path.display(),
|
||||
error = ?e,
|
||||
"limits drop-in unreadable; the agent gets no memory cap byte count"
|
||||
);
|
||||
return "infinity".to_owned();
|
||||
}
|
||||
};
|
||||
body.lines()
|
||||
.rev()
|
||||
.find_map(|line| line.trim().strip_prefix("MemoryMax="))
|
||||
.map(str::trim)
|
||||
.filter(|v| !v.is_empty())
|
||||
.unwrap_or("infinity")
|
||||
.to_owned()
|
||||
}
|
||||
|
||||
/// Write `memory_max` into [`MEMORY_CAP_FILE`] as a byte count, or remove the
|
||||
/// file when the value has none: `infinity`, or a RAM percentage, which needs
|
||||
/// the host's `MemTotal` to resolve. With no file the agent sets no heap or
|
||||
|
|
@ -815,15 +842,33 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn a_kept_dropin_keeps_the_agents_copy() {
|
||||
fn a_kept_dropin_hands_the_agent_its_cap() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let limits = dir.path().join("resource-limits.json");
|
||||
let dropin = dir.path().join("hyperhive-limits.conf");
|
||||
let copy = dir.path().join("runtime").join(MEMORY_CAP_FILE);
|
||||
std::fs::write(&limits, r#"{"sock": {"memory_max": "1G"}}"#).expect("seed limits");
|
||||
assert_eq!(stage(&dir).1, Some("1073741824\n".to_owned()));
|
||||
|
||||
// Unreadable limits keep the drop-in; the copy follows the drop-in.
|
||||
std::fs::write(&limits, TRUNCATED_LIMITS).expect("seed limits");
|
||||
std::fs::write(dir.path().join("hyperhive-limits.conf"), "MemoryMax=1G\n")
|
||||
.expect("seed drop-in");
|
||||
assert_eq!(stage(&dir), (None, Some("1073741824\n".to_owned())));
|
||||
std::fs::write(
|
||||
&dropin,
|
||||
"[Unit]\nConditionPathIsDirectory=/rt/sock\n\n[Service]\nRestartSec=5\nMemoryMax=2G\nCPUQuota=200%\n",
|
||||
)
|
||||
.expect("seed drop-in");
|
||||
assert_eq!(stage(&dir), (None, Some("2147483648\n".to_owned())));
|
||||
|
||||
// A kept drop-in with no copy beside it, as after an upgrade.
|
||||
std::fs::remove_file(©).expect("remove copy");
|
||||
assert_eq!(stage(&dir), (None, Some("2147483648\n".to_owned())));
|
||||
|
||||
// The last assignment is the one systemd applies.
|
||||
std::fs::write(&dropin, "[Service]\nMemoryMax=2G\nMemoryMax=3G\n").expect("seed drop-in");
|
||||
assert_eq!(stage(&dir), (None, Some("3221225472\n".to_owned())));
|
||||
|
||||
// No `MemoryMax=` is systemd's default, `infinity`: no byte count.
|
||||
std::fs::write(&dropin, "[Service]\nCPUQuota=200%\n").expect("seed drop-in");
|
||||
assert_eq!(stage(&dir), (None, None));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,8 @@
|
|||
# inside. hive-c0re writes its byte count to `/run/hive/memory-max-bytes`
|
||||
# before every start (`lifecycle::host_config`), and removes the file when the
|
||||
# cap is `infinity` or a RAM percentage. Two values are derived from it, and
|
||||
# neither is set without the file:
|
||||
# neither is set without the file; `hyperhive-memory-cap-check` then logs a
|
||||
# warning at boot:
|
||||
#
|
||||
# - `BUN_JSC_forceRAMSize`, 75% of the cap. Bun/JavaScriptCore, the runtime
|
||||
# under claude-code, sizes its heap ceiling off `/proc/meminfo`, which under
|
||||
|
|
@ -15,6 +16,8 @@
|
|||
# generator hands it to every unit of the container's systemd, `machinectl
|
||||
# shell` sessions (`hivectl choom`) included; `/etc/set-environment` exports
|
||||
# it in login shells that no unit starts (`nixos-container root-login`).
|
||||
# `hive-agent` and `hive-subagent-daemon` spawn `claude` as a plain child,
|
||||
# which inherits it, so neither unit may set or unset it.
|
||||
# - `MemoryHigh=` on `hive-subagent-daemon`, two thirds of the cap, as a
|
||||
# generator drop-in. The daemon spawns nested `claude` sessions as plain
|
||||
# children, so its cgroup already is the "all subagents" cgroup. `High`, not
|
||||
|
|
@ -22,34 +25,50 @@
|
|||
# last third stays free for the agent's own turn, but a subagent that needs
|
||||
# more still gets it when the container has memory to spare.
|
||||
#
|
||||
# Tested by `checks.script-test-agent-memory-cap`.
|
||||
# Tested by `checks.script-test-agent-memory-cap` and
|
||||
# `checks.module-eval-agent-memory`.
|
||||
{ pkgs, ... }:
|
||||
let
|
||||
# Sets `bytes`, or exits when there is no file or its first line is not a
|
||||
# plain byte count. A leading zero would read as octal, and 16 digits or
|
||||
# more could overflow `bytes * 75`. Shell builtins only.
|
||||
readCap = ''
|
||||
# Sets `bytes`, or runs `noCap` when there is no file or its first line is
|
||||
# not a plain byte count. A leading zero would read as octal, and 16 digits
|
||||
# or more could overflow `bytes * 75`. Shell builtins only.
|
||||
readCap = noCap: ''
|
||||
bytes=
|
||||
[ -r /run/hive/memory-max-bytes ] && read -r bytes < /run/hive/memory-max-bytes
|
||||
case "$bytes" in
|
||||
"" | 0* | *[!0-9]* | ????????????????*) exit 0 ;;
|
||||
"" | 0* | *[!0-9]* | ????????????????*) ${noCap} ;;
|
||||
esac
|
||||
'';
|
||||
|
||||
envGenerator = pkgs.writeShellScript "hyperhive-memory-cap-env" ''
|
||||
${readCap}
|
||||
${readCap "exit 0"}
|
||||
printf 'BUN_JSC_forceRAMSize=%s\n' "$((bytes * 75 / 100))"
|
||||
'';
|
||||
|
||||
# `$1` is the generator's normal-priority output dir. A drop-in there merges
|
||||
# with the unit's own file in /etc.
|
||||
unitGenerator = pkgs.writeShellScript "hyperhive-memory-cap" ''
|
||||
${readCap}
|
||||
${readCap "exit 0"}
|
||||
dir="$1/hive-subagent-daemon.service.d"
|
||||
${pkgs.coreutils}/bin/mkdir -p "$dir"
|
||||
printf '[Service]\nMemoryHigh=%s\n' "$((bytes * 2 / 3))" > "$dir/50-memory-high.conf"
|
||||
'';
|
||||
|
||||
# The generators run before the container's journal starts and an nspawn
|
||||
# container has no /dev/kmsg, so the boot warning for an unusable cap comes
|
||||
# from a unit. `<4>` is the journal's warning priority.
|
||||
check = pkgs.writeShellScript "hyperhive-memory-cap-check" ''
|
||||
${readCap ''
|
||||
if [ -e /run/hive/memory-max-bytes ]; then
|
||||
why="does not start with a plain byte count"
|
||||
else
|
||||
why="is missing"
|
||||
fi
|
||||
echo "<4>/run/hive/memory-max-bytes $why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies. hive-c0re writes no file for a cap of infinity or a RAM percentage." >&2
|
||||
exit 0
|
||||
''}
|
||||
'';
|
||||
|
||||
# Sourced by every shell's `/etc/set-environment`, so POSIX sh only.
|
||||
profile = pkgs.writeText "hyperhive-memory-cap.sh" ''
|
||||
__hyperhive_cap=$(${envGenerator})
|
||||
|
|
@ -63,6 +82,14 @@ in
|
|||
environment.etc."systemd/system-environment-generators/50-hyperhive-memory-cap".source =
|
||||
envGenerator;
|
||||
systemd.generators.hyperhive-memory-cap = unitGenerator;
|
||||
systemd.services.hyperhive-memory-cap-check = {
|
||||
description = "Warn when the container's memory cap is unusable";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = check;
|
||||
};
|
||||
};
|
||||
environment.etc."hyperhive/memory-cap.sh".source = profile;
|
||||
environment.extraInit = ". /etc/hyperhive/memory-cap.sh";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,6 +34,26 @@ let
|
|||
name = "the subagent daemon's memory ceiling is not built into its unit";
|
||||
ok = !((agentSubagentDaemon agentDefault).serviceConfig ? MemoryHigh);
|
||||
}
|
||||
{
|
||||
# claude's heap ceiling reaches it only as inherited environment: the
|
||||
# environment generator fills the manager's, systemd hands that to every
|
||||
# unit, and both units spawn claude as a plain child. A unit that sets
|
||||
# or unsets the variable itself cuts that. Without the two generators the
|
||||
# cap reaches nothing, and without the check unit nothing says so.
|
||||
name = "claude's units take the heap ceiling from the memory cap read at boot";
|
||||
ok =
|
||||
let
|
||||
leavesIt =
|
||||
unit:
|
||||
!(unit.environment ? BUN_JSC_forceRAMSize)
|
||||
&& !(lib.elem "BUN_JSC_forceRAMSize" (lib.toList (unit.serviceConfig.UnsetEnvironment or [ ])));
|
||||
in
|
||||
leavesIt (agentHarness agentDefault)
|
||||
&& leavesIt (agentSubagentDaemon agentDefault)
|
||||
&& agentDefault.environment.etc ? "systemd/system-environment-generators/50-hyperhive-memory-cap"
|
||||
&& agentDefault.systemd.generators ? hyperhive-memory-cap
|
||||
&& lib.elem "multi-user.target" agentDefault.systemd.services.hyperhive-memory-cap-check.wantedBy;
|
||||
}
|
||||
{
|
||||
# The sign is the whole property, and it is easy to write backwards:
|
||||
# a HIGHER OOMScoreAdjust is a MORE likely victim. So the subagent
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
# `checks.script-test-agent-memory-cap` — runs the three readers of
|
||||
# `checks.script-test-agent-memory-cap` — runs the four readers of
|
||||
# `/run/hive/memory-max-bytes` in ../agent-modules/memory-cap.nix: the
|
||||
# environment generator, the unit generator, and the login-shell snippet
|
||||
# `/etc/set-environment` sources. The cases are in ./agent-memory-cap.sh.
|
||||
# environment generator, the unit generator, the boot check that warns about
|
||||
# an unusable cap, and the login-shell snippet `/etc/set-environment` sources.
|
||||
# The cases are in ./agent-memory-cap.sh.
|
||||
#
|
||||
# What runs is the rendered scripts. The one edit is the cap file's path, moved
|
||||
# under the build directory, and the snippet's reference to the environment
|
||||
|
|
@ -32,6 +33,7 @@ pkgs.runCommand "hyperhive-script-test-agent-memory-cap"
|
|||
ENV_GENERATOR =
|
||||
machine.environment.etc."systemd/system-environment-generators/50-hyperhive-memory-cap".source;
|
||||
UNIT_GENERATOR = machine.systemd.generators.hyperhive-memory-cap;
|
||||
CHECK = machine.systemd.services.hyperhive-memory-cap-check.serviceConfig.ExecStart;
|
||||
PROFILE = machine.environment.etc."hyperhive/memory-cap.sh".source;
|
||||
EXTRA_INIT = machine.environment.extraInit;
|
||||
SHELLS = "${pkgs.dash}/bin/dash ${pkgs.bash}/bin/sh";
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# Cases for ./agent-memory-cap.nix. Each case writes a cap file (or none), runs
|
||||
# both generators with a clean environment and compares what they produced,
|
||||
# then sources the login-shell snippet under each POSIX shell and compares what
|
||||
# a child process sees.
|
||||
# both generators and the boot check with a clean environment and compares what
|
||||
# they produced, then sources the login-shell snippet under each POSIX shell
|
||||
# and compares what a child process sees.
|
||||
set -uo pipefail
|
||||
|
||||
failures=0
|
||||
|
|
@ -16,12 +16,13 @@ dir=$(mktemp -d)
|
|||
cap=$dir/memory-max-bytes
|
||||
sed "s|/run/hive/memory-max-bytes|$cap|g" "$ENV_GENERATOR" >"$dir/env-generator"
|
||||
sed "s|/run/hive/memory-max-bytes|$cap|g" "$UNIT_GENERATOR" >"$dir/unit-generator"
|
||||
chmod +x "$dir/env-generator" "$dir/unit-generator"
|
||||
sed "s|/run/hive/memory-max-bytes|$cap|g" "$CHECK" >"$dir/check"
|
||||
chmod +x "$dir/env-generator" "$dir/unit-generator" "$dir/check"
|
||||
sed "s|$ENV_GENERATOR|$dir/env-generator|g" "$PROFILE" >"$dir/profile"
|
||||
envbin=$(command -v env)
|
||||
|
||||
case="setup"
|
||||
for g in env-generator unit-generator; do
|
||||
for g in env-generator unit-generator check; do
|
||||
if grep -q '/run/hive/memory-max-bytes' "$dir/$g"; then fail "$g still names /run/hive after the rewrite"; fi
|
||||
if ! grep -qF "$cap" "$dir/$g"; then fail "the rewrite did not reach $g's read"; fi
|
||||
done
|
||||
|
|
@ -31,10 +32,11 @@ if ! printf '%s\n' "$EXTRA_INIT" | grep -qxF '. /etc/hyperhive/memory-cap.sh'; t
|
|||
fi
|
||||
|
||||
# check <description> <expected BUN_JSC_forceRAMSize or ""> <expected MemoryHigh or "">
|
||||
# <why the boot check warns, or "">
|
||||
check() {
|
||||
case=$1
|
||||
count=$((count + 1))
|
||||
local want_env=${2:+BUN_JSC_forceRAMSize=$2} want_high=$3 got
|
||||
local want_env=${2:+BUN_JSC_forceRAMSize=$2} want_high=$3 want_why=${4:-} got
|
||||
got=$(env -i "$dir/env-generator")
|
||||
if [ "$got" != "$want_env" ]; then
|
||||
fail "environment generator printed:"$'\n'"$got"$'\n'"wanted:"$'\n'"$want_env"
|
||||
|
|
@ -55,6 +57,17 @@ check() {
|
|||
fi
|
||||
fi
|
||||
|
||||
# The warning goes to stderr with the journal's `<4>` priority prefix, and
|
||||
# nothing goes anywhere for a usable cap.
|
||||
got=$(env -i "$dir/check" 2>&1)
|
||||
local want_warn=
|
||||
if [ -n "$want_why" ]; then
|
||||
want_warn="<4>$cap $want_why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies. hive-c0re writes no file for a cap of infinity or a RAM percentage."
|
||||
fi
|
||||
if [ "$got" != "$want_warn" ]; then
|
||||
fail "boot check printed:"$'\n'"$got"$'\n'"wanted:"$'\n'"$want_warn"
|
||||
fi
|
||||
|
||||
local sh
|
||||
for sh in $SHELLS; do
|
||||
# Only what the snippet exported reaches the child's environment. No PATH:
|
||||
|
|
@ -76,19 +89,19 @@ printf '4294967296\nLD_PRELOAD=/x\n' >"$cap"
|
|||
check "only the first line is read" 3221225472 2863311530
|
||||
|
||||
printf '8G\n' >"$cap"
|
||||
check "a size with a suffix is ignored" "" ""
|
||||
check "a size with a suffix is ignored" "" "" "does not start with a plain byte count"
|
||||
|
||||
printf '0123\n' >"$cap"
|
||||
check "a leading zero is ignored" "" ""
|
||||
check "a leading zero is ignored" "" "" "does not start with a plain byte count"
|
||||
|
||||
printf '1234567890123456\n' >"$cap"
|
||||
check "sixteen digits are ignored" "" ""
|
||||
check "sixteen digits are ignored" "" "" "does not start with a plain byte count"
|
||||
|
||||
: >"$cap"
|
||||
check "an empty file" "" ""
|
||||
check "an empty file" "" "" "does not start with a plain byte count"
|
||||
|
||||
rm -f "$cap"
|
||||
check "no file" "" ""
|
||||
check "no file" "" "" "is missing"
|
||||
|
||||
echo "$count cases, $failures failed"
|
||||
[ "$failures" -eq 0 ]
|
||||
|
|
|
|||
Loading…
Reference in a new issue