From 4daab7efe4d18a28126c53078557295c25cdd58a Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 13 Sep 2026 13:04:00 +0200 Subject: [PATCH] subagent daemon: throttle at two thirds of the container's memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The daemon spawns every nested claude as a plain child, so its cgroup is already the "all subagents" cgroup — but it ran with MemoryHigh=infinity and MemoryMax=infinity, so nothing slowed a subagent down before the kernel's OOM killer stopped the unit and cut every live session with it. MemoryHigh= and not MemoryMax=: a soft ceiling reclaims and stalls the cgroup past two thirds of the container's cap, which turns a silent kill into a visible throttle, while still letting a single subagent exceed its share when the container has memory free. A hard per-agent cap would make overprovisioning impossible, which is not wanted — most of the time nothing in these sessions is compiling. The fraction is taken from hyperhive.claudeMemoryMaxBytes, the container's own effective MemoryMax= that meta.rs already bakes in per agent. When that is null (an `infinity` or percentage cap) the unit renders no ceiling rather than a fabricated constant, and module-eval pins both arms. Refs #4316 --- nix/agent-modules/claude-settings.nix | 5 ++++- nix/agent-modules/mcp.nix | 22 +++++++++++++++++++ nix/module-eval.nix | 31 +++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/nix/agent-modules/claude-settings.nix b/nix/agent-modules/claude-settings.nix index 45941b5b..371595db 100644 --- a/nix/agent-modules/claude-settings.nix +++ b/nix/agent-modules/claude-settings.nix @@ -143,7 +143,10 @@ in description = '' Effective per-agent memory cap in bytes, when it's a plain byte-size value (null for an unbounded or percentage-based cap). - Used to derive `BUN_JSC_forceRAMSize` in `baseClaudeEnv`. + This is the whole container's cap, not claude's share of it — + the name records its first consumer, `BUN_JSC_forceRAMSize` in + `baseClaudeEnv`. `mcp.nix` reads it too, to size the subagent + daemon's `MemoryHigh=` against the container it runs in. ''; }; diff --git a/nix/agent-modules/mcp.nix b/nix/agent-modules/mcp.nix index 45a91b50..a535e483 100644 --- a/nix/agent-modules/mcp.nix +++ b/nix/agent-modules/mcp.nix @@ -11,6 +11,25 @@ }: let userName = config.hyperhive.user.name; + # This container's own effective `MemoryMax=` in bytes, baked in per + # agent by meta.rs's flake render — see + # `hyperhive.claudeMemoryMaxBytes` in ./claude-settings.nix. `null` + # when the cap is `infinity` or a RAM percentage, i.e. when the module + # has no byte count to size anything against. + containerMemoryMaxBytes = config.hyperhive.claudeMemoryMaxBytes; + # Two thirds of the container's cap, as the soft ceiling on everything + # the subagent daemon runs. The daemon spawns nested `claude` sessions + # as plain children, so its cgroup already *is* the "all subagents" + # cgroup and a unit-level ceiling bounds the set without a slice. + # + # `MemoryHigh=` and not `MemoryMax=`: this throttles rather than walls. + # Past it the kernel reclaims aggressively and the cgroup stalls, so a + # subagent that overshoots gets visibly slow and the remaining third + # stays available for the agent's own turn — but a subagent that + # genuinely needs more than two thirds still gets it when the container + # has the memory to spare, which is what keeps overprovisioning + # (several agents that rarely compile at the same time) working. + subagentMemoryHigh = containerMemoryMaxBytes * 2 / 3; in { options.hyperhive.allowedRecipients = lib.mkOption { @@ -346,6 +365,9 @@ in RestartSec = 3; User = userName; Group = userName; + } + // lib.optionalAttrs (containerMemoryMaxBytes != null) { + MemoryHigh = toString subagentMemoryHigh; }; }; diff --git a/nix/module-eval.nix b/nix/module-eval.nix index a754f806..82968aa9 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -473,7 +473,15 @@ let queue.tokenEndpoint = "https://auth.t.local/api/oidc/token"; }; agentNoQueue = agent { }; + + # The memory-pressure pair. `claudeMemoryMaxBytes` is the container's own + # cap, rendered per agent by meta.rs — the capped arm is the one every + # real deploy gets, the uncapped arm is a hive that set `infinity` or a + # RAM percentage and so hands the module no byte count to size against. + agentCapped = agent { claudeMemoryMaxBytes = 8589934592; }; + agentUncapped = agent { }; agentHarness = machine: machine.systemd.services.hive-agent; + agentSubagentDaemon = machine: machine.systemd.services.hive-subagent-daemon; agentSettings = machine: machine.services.opentelemetry-collector.settings; # This hive's own collector, which is a HOST service — unlike the swarm @@ -1354,6 +1362,29 @@ let && !(u.environment ? HIVE_AGENT_OIDC_CLIENT_SECRET_FILE) && !(u.environment ? HIVE_AGENT_OIDC_CLIENT_ID_FILE); } + { + # Two thirds of the container's cap, and a SOFT ceiling: the daemon's + # cgroup holds every nested claude, so `MemoryHigh=` throttles the + # subagent set as a whole before the kernel picks a victim, while the + # absent `MemoryMax=` is what still lets one subagent use more than + # its share on a container that has the memory free. A hard cap here + # would trade the silent kill for a guaranteed wall, which is the + # shape this deliberately does not have. + name = "the subagent daemon throttles at two thirds of the container's memory"; + ok = + let + c = (agentSubagentDaemon agentCapped).serviceConfig; + in + c.MemoryHigh == "5726623061" && !(c ? MemoryMax); + } + { + # What makes the case above able to fail. With no byte count for the + # container there is no fraction to take, and a hardcoded fallback + # would be a number about some other hive's machine — so the unit + # renders no ceiling at all rather than a fabricated one. + name = "an agent with no byte-valued memory cap renders no subagent ceiling"; + ok = !((agentSubagentDaemon agentUncapped).serviceConfig ? MemoryHigh); + } { # The doctrine three glue files state, as a property a rewrite has to # keep: a client is defined by holding a certificate the store accepts,