From 4daab7efe4d18a28126c53078557295c25cdd58a Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 13 Sep 2026 13:04:00 +0200 Subject: [PATCH 1/3] subagent daemon: throttle at two thirds of the container's memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The daemon spawns every nested claude as a plain child, so its cgroup is already the "all subagents" cgroup — but it ran with MemoryHigh=infinity and MemoryMax=infinity, so nothing slowed a subagent down before the kernel's OOM killer stopped the unit and cut every live session with it. MemoryHigh= and not MemoryMax=: a soft ceiling reclaims and stalls the cgroup past two thirds of the container's cap, which turns a silent kill into a visible throttle, while still letting a single subagent exceed its share when the container has memory free. A hard per-agent cap would make overprovisioning impossible, which is not wanted — most of the time nothing in these sessions is compiling. The fraction is taken from hyperhive.claudeMemoryMaxBytes, the container's own effective MemoryMax= that meta.rs already bakes in per agent. When that is null (an `infinity` or percentage cap) the unit renders no ceiling rather than a fabricated constant, and module-eval pins both arms. Refs #4316 --- nix/agent-modules/claude-settings.nix | 5 ++++- nix/agent-modules/mcp.nix | 22 +++++++++++++++++++ nix/module-eval.nix | 31 +++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/nix/agent-modules/claude-settings.nix b/nix/agent-modules/claude-settings.nix index 45941b5b..371595db 100644 --- a/nix/agent-modules/claude-settings.nix +++ b/nix/agent-modules/claude-settings.nix @@ -143,7 +143,10 @@ in description = '' Effective per-agent memory cap in bytes, when it's a plain byte-size value (null for an unbounded or percentage-based cap). - Used to derive `BUN_JSC_forceRAMSize` in `baseClaudeEnv`. + This is the whole container's cap, not claude's share of it — + the name records its first consumer, `BUN_JSC_forceRAMSize` in + `baseClaudeEnv`. `mcp.nix` reads it too, to size the subagent + daemon's `MemoryHigh=` against the container it runs in. ''; }; diff --git a/nix/agent-modules/mcp.nix b/nix/agent-modules/mcp.nix index 45a91b50..a535e483 100644 --- a/nix/agent-modules/mcp.nix +++ b/nix/agent-modules/mcp.nix @@ -11,6 +11,25 @@ }: let userName = config.hyperhive.user.name; + # This container's own effective `MemoryMax=` in bytes, baked in per + # agent by meta.rs's flake render — see + # `hyperhive.claudeMemoryMaxBytes` in ./claude-settings.nix. `null` + # when the cap is `infinity` or a RAM percentage, i.e. when the module + # has no byte count to size anything against. + containerMemoryMaxBytes = config.hyperhive.claudeMemoryMaxBytes; + # Two thirds of the container's cap, as the soft ceiling on everything + # the subagent daemon runs. The daemon spawns nested `claude` sessions + # as plain children, so its cgroup already *is* the "all subagents" + # cgroup and a unit-level ceiling bounds the set without a slice. + # + # `MemoryHigh=` and not `MemoryMax=`: this throttles rather than walls. + # Past it the kernel reclaims aggressively and the cgroup stalls, so a + # subagent that overshoots gets visibly slow and the remaining third + # stays available for the agent's own turn — but a subagent that + # genuinely needs more than two thirds still gets it when the container + # has the memory to spare, which is what keeps overprovisioning + # (several agents that rarely compile at the same time) working. + subagentMemoryHigh = containerMemoryMaxBytes * 2 / 3; in { options.hyperhive.allowedRecipients = lib.mkOption { @@ -346,6 +365,9 @@ in RestartSec = 3; User = userName; Group = userName; + } + // lib.optionalAttrs (containerMemoryMaxBytes != null) { + MemoryHigh = toString subagentMemoryHigh; }; }; diff --git a/nix/module-eval.nix b/nix/module-eval.nix index a754f806..82968aa9 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -473,7 +473,15 @@ let queue.tokenEndpoint = "https://auth.t.local/api/oidc/token"; }; agentNoQueue = agent { }; + + # The memory-pressure pair. `claudeMemoryMaxBytes` is the container's own + # cap, rendered per agent by meta.rs — the capped arm is the one every + # real deploy gets, the uncapped arm is a hive that set `infinity` or a + # RAM percentage and so hands the module no byte count to size against. + agentCapped = agent { claudeMemoryMaxBytes = 8589934592; }; + agentUncapped = agent { }; agentHarness = machine: machine.systemd.services.hive-agent; + agentSubagentDaemon = machine: machine.systemd.services.hive-subagent-daemon; agentSettings = machine: machine.services.opentelemetry-collector.settings; # This hive's own collector, which is a HOST service — unlike the swarm @@ -1354,6 +1362,29 @@ let && !(u.environment ? HIVE_AGENT_OIDC_CLIENT_SECRET_FILE) && !(u.environment ? HIVE_AGENT_OIDC_CLIENT_ID_FILE); } + { + # Two thirds of the container's cap, and a SOFT ceiling: the daemon's + # cgroup holds every nested claude, so `MemoryHigh=` throttles the + # subagent set as a whole before the kernel picks a victim, while the + # absent `MemoryMax=` is what still lets one subagent use more than + # its share on a container that has the memory free. A hard cap here + # would trade the silent kill for a guaranteed wall, which is the + # shape this deliberately does not have. + name = "the subagent daemon throttles at two thirds of the container's memory"; + ok = + let + c = (agentSubagentDaemon agentCapped).serviceConfig; + in + c.MemoryHigh == "5726623061" && !(c ? MemoryMax); + } + { + # What makes the case above able to fail. With no byte count for the + # container there is no fraction to take, and a hardcoded fallback + # would be a number about some other hive's machine — so the unit + # renders no ceiling at all rather than a fabricated one. + name = "an agent with no byte-valued memory cap renders no subagent ceiling"; + ok = !((agentSubagentDaemon agentUncapped).serviceConfig ? MemoryHigh); + } { # The doctrine three glue files state, as a property a rewrite has to # keep: a client is defined by holding a certificate the store accepts, From d249468db22162476ee0c300ffbf6143c610c242 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 13 Sep 2026 13:04:45 +0200 Subject: [PATCH 2/3] agent: make the OOM killer prefer a subagent over the agent's own turn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both units ran at OOMScoreAdjust=0, so under container memory pressure the kernel picked purely on footprint — and the agent's own claude is often the fattest process in the container, which means the session supervising the work died before the work did. The sign is the load-bearing part and is easy to invert: a HIGHER OOMScoreAdjust means MORE likely to be killed, because the kernel adds it to the badness score it derives from the process's memory footprint and then kills the highest scorer. So hive-subagent-daemon gets +500 (first in line) and hive-agent gets -500 (last in line). Written backwards this makes the reported bug worse rather than better, so module-eval pins the order as an inequality. Both values are inherited by the nested claude each unit spawns as a child, so ordering the units orders the sessions underneath them. -500 rather than -1000 on the harness: fully exempting it would leave the kernel nothing to kill in a container whose only large process is the harness. Refs #4316 --- nix/agent-modules/agent-service.nix | 10 ++++++++++ nix/agent-modules/mcp.nix | 12 ++++++++++++ nix/module-eval.nix | 17 +++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/nix/agent-modules/agent-service.nix b/nix/agent-modules/agent-service.nix index f3721311..db5af618 100644 --- a/nix/agent-modules/agent-service.nix +++ b/nix/agent-modules/agent-service.nix @@ -303,6 +303,16 @@ in SyslogIdentifier = binary; Restart = "on-failure"; RestartSec = 2; + # A LOWER OOMScoreAdjust means LESS likely to be killed, so this + # negative value puts the harness — and the agent's own `claude`, + # which it spawns as a child and which therefore inherits the + # value — last in line under memory pressure, behind + # `hive-subagent-daemon`'s `+500` (see ./mcp.nix). Losing a + # subagent costs one restartable task; losing this costs the + # session that was supervising it. Not `-1000`, which would + # exempt the harness entirely and leave the kernel with nothing + # to kill in a container whose only large process is this one. + OOMScoreAdjust = -500; # Per-service runtime dir owned by `User=` below; the harness # writes its regenerated claude-{mcp-config,settings,system-prompt} # files here (`paths::config_dir`). Separate from /run/hive, diff --git a/nix/agent-modules/mcp.nix b/nix/agent-modules/mcp.nix index a535e483..d5e9f5a6 100644 --- a/nix/agent-modules/mcp.nix +++ b/nix/agent-modules/mcp.nix @@ -363,6 +363,18 @@ in # `start`/`continue`/`status`/`interrupt` with no stdio fallback. Restart = "always"; RestartSec = 3; + # A HIGHER OOMScoreAdjust means MORE likely to be killed: the + # kernel adds it to the badness score it derives from the + # process's memory footprint, then kills the highest scorer. So + # the positive value here and the negative one on `hive-agent` + # put this unit first in line — a subagent turn can be retried, + # the agent's own turn is the thing everything else hangs off. + # `+500` against `hive-agent`'s `-500` is a full half of the + # score range apart, enough that a fat subagent outranks a fatter + # harness rather than merely tying with it. Every nested `claude` + # inherits the value from the daemon, so the ordering covers the + # whole subtree and not just this process. + OOMScoreAdjust = 500; User = userName; Group = userName; } diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 82968aa9..ace26482 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -1385,6 +1385,23 @@ let name = "an agent with no byte-valued memory cap renders no subagent ceiling"; ok = !((agentSubagentDaemon agentUncapped).serviceConfig ? MemoryHigh); } + { + # The sign is the whole property, and it is easy to write backwards: + # a HIGHER OOMScoreAdjust is a MORE likely victim. So the subagent + # daemon must be strictly above zero and the harness strictly below + # it — swap the two and the kernel takes the agent's own turn first, + # which is worse than setting nothing at all. Both nested `claude` + # processes inherit their unit's value, so ordering the units orders + # the sessions. Held as an inequality rather than two constants: what + # must not drift is the order, not the magnitudes. + name = "the OOM killer prefers a subagent over the agent's own session"; + ok = + let + sub = (agentSubagentDaemon agentUncapped).serviceConfig.OOMScoreAdjust; + own = (agentHarness agentUncapped).serviceConfig.OOMScoreAdjust; + in + sub > 0 && own < 0 && sub > own; + } { # The doctrine three glue files state, as a property a rewrite has to # keep: a client is defined by holding a certificate the store accepts, From 33aa2bdbc80b9124ffd6810252be96f2482528d1 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 13 Sep 2026 13:05:17 +0200 Subject: [PATCH 3/3] subagent daemon: scope an OOM kill to the session that lost the draw MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit systemd's default OOMPolicy=stop tears the whole unit down the moment the kernel kills any process in its cgroup, so a single over-large subagent takes the daemon and every sibling session with it — measured on a live agent, both units show OOMPolicy=stop today, which is exactly the "every live subagent session was cut mid-turn" symptom. That blast radius is also what would make the preceding commit a bad trade: deliberately putting this unit first in the OOM queue is only an improvement if losing one subagent isn't losing all of them. OOMPolicy= continue scopes the loss to the process the kernel actually chose, and leaves the daemon alive to report the kill instead of vanishing and being restarted with an empty session map. Refs #4316 --- nix/agent-modules/mcp.nix | 8 ++++++++ nix/module-eval.nix | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/nix/agent-modules/mcp.nix b/nix/agent-modules/mcp.nix index d5e9f5a6..151bbd00 100644 --- a/nix/agent-modules/mcp.nix +++ b/nix/agent-modules/mcp.nix @@ -375,6 +375,14 @@ in # inherits the value from the daemon, so the ordering covers the # whole subtree and not just this process. OOMScoreAdjust = 500; + # systemd's default is `stop`, which tears the whole unit down as + # soon as the kernel kills *any* process in its cgroup — so one + # over-large subagent takes the daemon and every sibling session + # with it, which is the blast radius that makes an OOM here + # expensive. `continue` scopes the loss to the process the kernel + # actually chose and leaves the daemon serving, which is also what + # lets it report the kill rather than vanish and be restarted. + OOMPolicy = "continue"; User = userName; Group = userName; } diff --git a/nix/module-eval.nix b/nix/module-eval.nix index ace26482..25208745 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -1402,6 +1402,16 @@ let in sub > 0 && own < 0 && sub > own; } + { + # The pair the case above only makes sense with: preferring this unit + # as the victim is an improvement only if losing one subagent isn't + # losing all of them. systemd's default `stop` would take the daemon + # and every sibling session down with whichever process the kernel + # chose, which is the blast radius that made the preference a bad + # trade in the first place. + name = "one subagent losing the OOM draw does not stop the daemon"; + ok = (agentSubagentDaemon agentUncapped).serviceConfig.OOMPolicy == "continue"; + } { # The doctrine three glue files state, as a property a rewrite has to # keep: a client is defined by holding a certificate the store accepts,