b31db6c7d9
hive-subagent-daemon and hive-bash-daemon are now delegated their cgroup (Delegate=memory pids, DelegateSubgroup=daemon). At start each daemon enables memory+pids below its unit and creates runs/ (subagent) or tasks/ (bash). Each subagent run gets runs/<name>-<n>/ and each bash task tasks/<id>/, both with memory.oom.group=1, and the daemon removes the directory when the run ends. An OOM kill now takes one whole run instead of one process out of the daemon's shared cgroup, and never the daemon itself. Subagent runs get oom_score_adj +1000; bash tasks and both daemons stay at 0; the harness stays at -500. The unit-level OOMScoreAdjust=500 on the subagent daemon goes: it was both the daemon's own score and a floor no run below it could be lowered past. Bash tasks enter their cgroup from the existing pre_exec hook, writing to a cgroup.procs fd opened by the parent. hive-claude builds its Command itself and exposes neither a pre_exec hook nor the child's pid, so a subagent's claude (and an ACP agent's process) is started as the daemon's own binary with the run's cgroup in its environment; main() enters the cgroup, raises oom_score_adj and execs the real program, the trampoline swarm-review-classifier already uses. Both enter before the program executes, so nothing it forks starts outside the run's cgroup. Without a delegated `daemon` leaf (tests, a dev shell) each daemon logs one warning and runs everything in its own cgroup, as before. The shared helper is hive-agent-sock's run_cgroup module, the crate both daemons already use for what in-container plugins share. Closes #5007
103 lines
4.1 KiB
Nix
103 lines
4.1 KiB
Nix
# `checks.module-eval-agent-memory` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
runGroup
|
|
agentHarness
|
|
;
|
|
|
|
agentDefault = shared.agentBare;
|
|
|
|
agentBashDaemon = machine: machine.systemd.services.hive-bash-daemon;
|
|
agentSubagentDaemon = machine: machine.systemd.services.hive-subagent-daemon;
|
|
cases = [
|
|
{
|
|
# The ceiling is sized at boot from the container's cap
|
|
# (../agent-modules/memory-cap.nix). One in the unit would be the cap at
|
|
# the last rebuild, and every agent would carry its own hive's number.
|
|
name = "the subagent daemon's memory ceiling is not built into its unit";
|
|
ok = !((agentSubagentDaemon agentDefault).serviceConfig ? MemoryHigh);
|
|
}
|
|
{
|
|
# claude's heap ceiling reaches it only as inherited environment: the
|
|
# environment generator fills the manager's, systemd hands that to every
|
|
# unit, and both units spawn claude as a plain child. A unit that sets
|
|
# or unsets the variable itself cuts that. Without the two generators the
|
|
# cap reaches nothing, and without the check unit nothing says so.
|
|
name = "claude's units take the heap ceiling from the memory cap read at boot";
|
|
ok =
|
|
let
|
|
leavesIt =
|
|
unit:
|
|
!(unit.environment ? BUN_JSC_forceRAMSize)
|
|
&& !(lib.elem "BUN_JSC_forceRAMSize" (lib.toList (unit.serviceConfig.UnsetEnvironment or [ ])));
|
|
in
|
|
leavesIt (agentHarness agentDefault)
|
|
&& leavesIt (agentSubagentDaemon agentDefault)
|
|
&& agentDefault.environment.etc ? "systemd/system-environment-generators/50-hyperhive-memory-cap"
|
|
&& agentDefault.systemd.generators ? hyperhive-memory-cap
|
|
&& lib.elem "multi-user.target" agentDefault.systemd.services.hyperhive-memory-cap-check.wantedBy;
|
|
}
|
|
{
|
|
# A HIGHER OOMScoreAdjust is a MORE likely victim. The subagent daemon
|
|
# puts each run at +1000 itself and bash tasks stay at 0, so the harness
|
|
# must sit below 0 for the agent's own session to go last. Neither
|
|
# daemon unit carries a value: it would make the idle daemon a likelier
|
|
# victim than a bash task, and a floor no run under it could go below.
|
|
name = "the OOM killer takes the agent's own session last";
|
|
ok =
|
|
(agentHarness agentDefault).serviceConfig.OOMScoreAdjust < 0
|
|
&& !((agentSubagentDaemon agentDefault).serviceConfig ? OOMScoreAdjust)
|
|
&& !((agentBashDaemon agentDefault).serviceConfig ? OOMScoreAdjust);
|
|
}
|
|
{
|
|
# Both daemons create a cgroup per run under their own; without the
|
|
# delegation they cannot, and without the `daemon` leaf (the name
|
|
# hive-agent-sock's `run_cgroup` looks for) they refuse to, so runs
|
|
# would share the daemon's cgroup again.
|
|
name = "the subagent and bash daemons are delegated their cgroup";
|
|
ok =
|
|
lib.all
|
|
(
|
|
unit:
|
|
unit.serviceConfig.Delegate or null == "memory pids"
|
|
&& unit.serviceConfig.DelegateSubgroup or null == "daemon"
|
|
)
|
|
[
|
|
(agentSubagentDaemon agentDefault)
|
|
(agentBashDaemon agentDefault)
|
|
];
|
|
}
|
|
{
|
|
# Preferring a subagent run as the victim is an improvement only if
|
|
# losing one run isn't losing all of them. systemd's default `stop`
|
|
# would take the daemon and every sibling session down with whichever
|
|
# process the kernel chose.
|
|
name = "one subagent losing the OOM draw does not stop the daemon";
|
|
ok = (agentSubagentDaemon agentDefault).serviceConfig.OOMPolicy == "continue";
|
|
}
|
|
{
|
|
# Every bash task runs under the daemon's unit; under `stop` one
|
|
# OOM-killed task would end every running sibling task.
|
|
name = "one bash task losing the OOM draw does not stop the daemon";
|
|
ok = (agentBashDaemon agentDefault).serviceConfig.OOMPolicy or null == "continue";
|
|
}
|
|
];
|
|
in
|
|
runGroup "agent-memory" cases
|