diff --git a/nix/agent-modules/agent-service.nix b/nix/agent-modules/agent-service.nix index f3721311..db5af618 100644 --- a/nix/agent-modules/agent-service.nix +++ b/nix/agent-modules/agent-service.nix @@ -303,6 +303,16 @@ in SyslogIdentifier = binary; Restart = "on-failure"; RestartSec = 2; + # A LOWER OOMScoreAdjust means LESS likely to be killed, so this + # negative value puts the harness — and the agent's own `claude`, + # which it spawns as a child and which therefore inherits the + # value — last in line under memory pressure, behind + # `hive-subagent-daemon`'s `+500` (see ./mcp.nix). Losing a + # subagent costs one restartable task; losing this costs the + # session that was supervising it. Not `-1000`, which would + # exempt the harness entirely and leave the kernel with nothing + # to kill in a container whose only large process is this one. + OOMScoreAdjust = -500; # Per-service runtime dir owned by `User=` below; the harness # writes its regenerated claude-{mcp-config,settings,system-prompt} # files here (`paths::config_dir`). Separate from /run/hive, diff --git a/nix/agent-modules/claude-settings.nix b/nix/agent-modules/claude-settings.nix index 45941b5b..371595db 100644 --- a/nix/agent-modules/claude-settings.nix +++ b/nix/agent-modules/claude-settings.nix @@ -143,7 +143,10 @@ in description = '' Effective per-agent memory cap in bytes, when it's a plain byte-size value (null for an unbounded or percentage-based cap). - Used to derive `BUN_JSC_forceRAMSize` in `baseClaudeEnv`. + This is the whole container's cap, not claude's share of it — + the name records its first consumer, `BUN_JSC_forceRAMSize` in + `baseClaudeEnv`. `mcp.nix` reads it too, to size the subagent + daemon's `MemoryHigh=` against the container it runs in. ''; }; diff --git a/nix/agent-modules/mcp.nix b/nix/agent-modules/mcp.nix index 45a91b50..151bbd00 100644 --- a/nix/agent-modules/mcp.nix +++ b/nix/agent-modules/mcp.nix @@ -11,6 +11,25 @@ }: let userName = config.hyperhive.user.name; + # This container's own effective `MemoryMax=` in bytes, baked in per + # agent by meta.rs's flake render — see + # `hyperhive.claudeMemoryMaxBytes` in ./claude-settings.nix. `null` + # when the cap is `infinity` or a RAM percentage, i.e. when the module + # has no byte count to size anything against. + containerMemoryMaxBytes = config.hyperhive.claudeMemoryMaxBytes; + # Two thirds of the container's cap, as the soft ceiling on everything + # the subagent daemon runs. The daemon spawns nested `claude` sessions + # as plain children, so its cgroup already *is* the "all subagents" + # cgroup and a unit-level ceiling bounds the set without a slice. + # + # `MemoryHigh=` and not `MemoryMax=`: this throttles rather than walls. + # Past it the kernel reclaims aggressively and the cgroup stalls, so a + # subagent that overshoots gets visibly slow and the remaining third + # stays available for the agent's own turn — but a subagent that + # genuinely needs more than two thirds still gets it when the container + # has the memory to spare, which is what keeps overprovisioning + # (several agents that rarely compile at the same time) working. + subagentMemoryHigh = containerMemoryMaxBytes * 2 / 3; in { options.hyperhive.allowedRecipients = lib.mkOption { @@ -344,8 +363,31 @@ in # `start`/`continue`/`status`/`interrupt` with no stdio fallback. Restart = "always"; RestartSec = 3; + # A HIGHER OOMScoreAdjust means MORE likely to be killed: the + # kernel adds it to the badness score it derives from the + # process's memory footprint, then kills the highest scorer. So + # the positive value here and the negative one on `hive-agent` + # put this unit first in line — a subagent turn can be retried, + # the agent's own turn is the thing everything else hangs off. + # `+500` against `hive-agent`'s `-500` is a full half of the + # score range apart, enough that a fat subagent outranks a fatter + # harness rather than merely tying with it. Every nested `claude` + # inherits the value from the daemon, so the ordering covers the + # whole subtree and not just this process. + OOMScoreAdjust = 500; + # systemd's default is `stop`, which tears the whole unit down as + # soon as the kernel kills *any* process in its cgroup — so one + # over-large subagent takes the daemon and every sibling session + # with it, which is the blast radius that makes an OOM here + # expensive. `continue` scopes the loss to the process the kernel + # actually chose and leaves the daemon serving, which is also what + # lets it report the kill rather than vanish and be restarted. + OOMPolicy = "continue"; User = userName; Group = userName; + } + // lib.optionalAttrs (containerMemoryMaxBytes != null) { + MemoryHigh = toString subagentMemoryHigh; }; }; diff --git a/nix/module-eval.nix b/nix/module-eval.nix index a754f806..25208745 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -473,7 +473,15 @@ let queue.tokenEndpoint = "https://auth.t.local/api/oidc/token"; }; agentNoQueue = agent { }; + + # The memory-pressure pair. `claudeMemoryMaxBytes` is the container's own + # cap, rendered per agent by meta.rs — the capped arm is the one every + # real deploy gets, the uncapped arm is a hive that set `infinity` or a + # RAM percentage and so hands the module no byte count to size against. + agentCapped = agent { claudeMemoryMaxBytes = 8589934592; }; + agentUncapped = agent { }; agentHarness = machine: machine.systemd.services.hive-agent; + agentSubagentDaemon = machine: machine.systemd.services.hive-subagent-daemon; agentSettings = machine: machine.services.opentelemetry-collector.settings; # This hive's own collector, which is a HOST service — unlike the swarm @@ -1354,6 +1362,56 @@ let && !(u.environment ? HIVE_AGENT_OIDC_CLIENT_SECRET_FILE) && !(u.environment ? HIVE_AGENT_OIDC_CLIENT_ID_FILE); } + { + # Two thirds of the container's cap, and a SOFT ceiling: the daemon's + # cgroup holds every nested claude, so `MemoryHigh=` throttles the + # subagent set as a whole before the kernel picks a victim, while the + # absent `MemoryMax=` is what still lets one subagent use more than + # its share on a container that has the memory free. A hard cap here + # would trade the silent kill for a guaranteed wall, which is the + # shape this deliberately does not have. + name = "the subagent daemon throttles at two thirds of the container's memory"; + ok = + let + c = (agentSubagentDaemon agentCapped).serviceConfig; + in + c.MemoryHigh == "5726623061" && !(c ? MemoryMax); + } + { + # What makes the case above able to fail. With no byte count for the + # container there is no fraction to take, and a hardcoded fallback + # would be a number about some other hive's machine — so the unit + # renders no ceiling at all rather than a fabricated one. + name = "an agent with no byte-valued memory cap renders no subagent ceiling"; + ok = !((agentSubagentDaemon agentUncapped).serviceConfig ? MemoryHigh); + } + { + # The sign is the whole property, and it is easy to write backwards: + # a HIGHER OOMScoreAdjust is a MORE likely victim. So the subagent + # daemon must be strictly above zero and the harness strictly below + # it — swap the two and the kernel takes the agent's own turn first, + # which is worse than setting nothing at all. Both nested `claude` + # processes inherit their unit's value, so ordering the units orders + # the sessions. Held as an inequality rather than two constants: what + # must not drift is the order, not the magnitudes. + name = "the OOM killer prefers a subagent over the agent's own session"; + ok = + let + sub = (agentSubagentDaemon agentUncapped).serviceConfig.OOMScoreAdjust; + own = (agentHarness agentUncapped).serviceConfig.OOMScoreAdjust; + in + sub > 0 && own < 0 && sub > own; + } + { + # The pair the case above only makes sense with: preferring this unit + # as the victim is an improvement only if losing one subagent isn't + # losing all of them. systemd's default `stop` would take the daemon + # and every sibling session down with whichever process the kernel + # chose, which is the blast radius that made the preference a bad + # trade in the first place. + name = "one subagent losing the OOM draw does not stop the daemon"; + ok = (agentSubagentDaemon agentUncapped).serviceConfig.OOMPolicy == "continue"; + } { # The doctrine three glue files state, as a property a rewrite has to # keep: a client is defined by holding a certificate the store accepts,