agents: read the memory cap at boot, not from the build
An agent's memory cap was rendered into its flake (`memoryMaxBytes` → `services.hyperhive.agent.claudeMemoryMaxBytes`), so the closure depended on it and a changed cap reached the JSC heap ceiling and the subagent `MemoryHigh=` only after a rebuild. hive-c0re now writes the effective `MemoryMax=` in bytes to the agent's runtime dir (`/run/hive/memory-max-bytes` inside) in `write_dropins`, from the same value the drop-in gets. The file is removed for `infinity` or a RAM percentage, and left alone when the drop-in is kept. Inside the container, `nix/agent-modules/memory-cap.nix` reads it at boot: an environment generator sets `BUN_JSC_forceRAMSize` (75%) for every unit and `machinectl shell`, `/etc/set-environment` exports it in login shells, and a unit generator writes `MemoryHigh=` (2/3) as a drop-in for `hive-subagent-daemon`. The render drops `memoryMaxBytes`, the agent option is removed (both spellings fail eval with a removal message), and `effective_memory_bytes_from` goes with its only caller. Part of #5081 (slice S0-2).
This commit is contained in:
atlas
parent
c8df666bf3
commit
0d619e215e
15 changed files with 416 additions and 287 deletions
|
|
@ -19,11 +19,6 @@ let
|
|||
# meta.rs `otel_config`). Options declared in `otel.nix`, which also
|
||||
# exports the generic OTLP environment this container's producers read.
|
||||
otelCfg = config.services.hyperhive.agent.otel;
|
||||
# Effective per-agent MemoryMax=, in bytes, injected by meta.rs's
|
||||
# per-agent flake render (`services.hyperhive.agent.claudeMemoryMaxBytes`). `null`
|
||||
# when the effective cap is unbounded ("infinity") or a RAM
|
||||
# percentage — see `resource_limits::effective_memory_bytes`.
|
||||
memoryMaxBytes = config.services.hyperhive.agent.claudeMemoryMaxBytes;
|
||||
# Base claude-code environment applied to every agent regardless of OTEL.
|
||||
# Shipped via the managed settings `env` block so claude and `hivectl
|
||||
# choom` both inherit them without a launch wrapper.
|
||||
|
|
@ -47,19 +42,6 @@ let
|
|||
CLAUDE_CODE_RESUME_INTERRUPTED_TURN = "1";
|
||||
# Use the simpler system prompt variant suited to headless operation.
|
||||
CLAUDE_CODE_SIMPLE_SYSTEM_PROMPT = "1";
|
||||
}
|
||||
# Bun/JavaScriptCore (the runtime under claude-code's `.claude-wrapped`
|
||||
# binary) sizes its default heap ceiling off the container's visible
|
||||
# `/proc/meminfo`, which under nspawn is the *host's* physical RAM, not
|
||||
# the systemd `MemoryMax=` cgroup cap actually enforced on this
|
||||
# container — the cap lives on the host's outer cgroup and is invisible
|
||||
# from inside (confirmed: `/sys/fs/cgroup/memory.max` reads `max` at
|
||||
# every level from in here). That mismatch lets JSC's heap grow well
|
||||
# past the real wall before GC kicks in hard, causing choom sessions to
|
||||
# hang. Pin JSC's ceiling to 75% of the *actual* effective cap instead,
|
||||
# once it's known at build time.
|
||||
// lib.optionalAttrs (memoryMaxBytes != null) {
|
||||
BUN_JSC_forceRAMSize = toString (memoryMaxBytes * 75 / 100);
|
||||
};
|
||||
# Claude Code's own telemetry switches — what it emits, and whether it
|
||||
# emits at all. Everything an OTEL SDK reads generically (endpoint,
|
||||
|
|
@ -99,31 +81,6 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
# Build-time implementation surface for the JSC-heap-ceiling fix:
|
||||
# meta.rs's per-agent flake render injects this from the effective
|
||||
# `MemoryMax=` (per-agent `resource-limits.json` override, else the
|
||||
# hive-wide `services.hyperhive.agentMemoryMax`) — see
|
||||
# `resource_limits::effective_memory_bytes_from`. Not meant to be set
|
||||
# directly in an agent.nix, same convention as `services.hyperhive.agent.otel.*`
|
||||
# above; the host option (or `hivectl agent <name> set-limits`) is
|
||||
# the real operator knob, and this only reflects the value baked in at
|
||||
# the agent's *last rebuild* — `set-limits` still applies the
|
||||
# cgroup cap live via a drop-in reload, but this derived heap ceiling
|
||||
# needs a rebuild to pick up a new value.
|
||||
options.services.hyperhive.agent.claudeMemoryMaxBytes = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.ints.positive;
|
||||
default = null;
|
||||
internal = true;
|
||||
description = ''
|
||||
Effective per-agent memory cap in bytes, when it's a plain
|
||||
byte-size value (null for an unbounded or percentage-based cap).
|
||||
This is the whole container's cap, not claude's share of it —
|
||||
the name records its first consumer, `BUN_JSC_forceRAMSize` in
|
||||
`baseClaudeEnv`. `mcp.nix` reads it too, to size the subagent
|
||||
daemon's `MemoryHigh=` against the container it runs in.
|
||||
'';
|
||||
};
|
||||
|
||||
options.services.hyperhive.agent.claudeMarketplaces = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [
|
||||
|
|
@ -257,9 +214,7 @@ in
|
|||
environment.etc."claude-code/managed-settings.json".source =
|
||||
let
|
||||
baseSettings = "${config.services.hyperhive.agent.packages.assets}/share/hyperhive/prompts/claude-settings.json";
|
||||
# Merge base env (always) with OTEL env (when enabled). jq is always
|
||||
# run — `baseClaudeEnv` can carry per-agent values (e.g.
|
||||
# BUN_JSC_forceRAMSize) that can't live in the static store asset.
|
||||
# Merge base env (always) with OTEL env (when enabled).
|
||||
allEnv =
|
||||
baseClaudeEnv
|
||||
// lib.optionalAttrs otelCfg.enable otelSettingsEnv
|
||||
|
|
|
|||
|
|
@ -41,6 +41,11 @@ let
|
|||
which the hive writes into every container, so no agent build sets it.
|
||||
Remove the setting.
|
||||
'';
|
||||
memoryMaxBytesRemoved = ''
|
||||
An agent reads its memory cap at boot from /run/hive/memory-max-bytes,
|
||||
which the hive writes before every start, so no agent build sets it.
|
||||
Remove the setting.
|
||||
'';
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
|
|
@ -60,6 +65,7 @@ in
|
|||
./logs.nix
|
||||
./matrix.nix
|
||||
./mcp.nix
|
||||
./memory-cap.nix
|
||||
./network.nix
|
||||
./otel.nix
|
||||
./packages.nix
|
||||
|
|
@ -94,6 +100,13 @@ in
|
|||
# Both spellings, for the reason given above.
|
||||
(lib.mkRemovedOptionModule [ "hyperhive" "hiveName" ] hiveNameRemoved)
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "agent" "hiveName" ] hiveNameRemoved)
|
||||
(lib.mkRemovedOptionModule [ "hyperhive" "claudeMemoryMaxBytes" ] memoryMaxBytesRemoved)
|
||||
(lib.mkRemovedOptionModule [
|
||||
"services"
|
||||
"hyperhive"
|
||||
"agent"
|
||||
"claudeMemoryMaxBytes"
|
||||
] memoryMaxBytesRemoved)
|
||||
];
|
||||
|
||||
options.services.hyperhive.agent.icon = lib.mkOption {
|
||||
|
|
|
|||
|
|
@ -11,25 +11,6 @@
|
|||
}:
|
||||
let
|
||||
userName = config.services.hyperhive.agent.user.name;
|
||||
# This container's own effective `MemoryMax=` in bytes, baked in per
|
||||
# agent by meta.rs's flake render — see
|
||||
# `services.hyperhive.agent.claudeMemoryMaxBytes` in ./claude-settings.nix. `null`
|
||||
# when the cap is `infinity` or a RAM percentage, i.e. when the module
|
||||
# has no byte count to size anything against.
|
||||
containerMemoryMaxBytes = config.services.hyperhive.agent.claudeMemoryMaxBytes;
|
||||
# Two thirds of the container's cap, as the soft ceiling on everything
|
||||
# the subagent daemon runs. The daemon spawns nested `claude` sessions
|
||||
# as plain children, so its cgroup already *is* the "all subagents"
|
||||
# cgroup and a unit-level ceiling bounds the set without a slice.
|
||||
#
|
||||
# `MemoryHigh=` and not `MemoryMax=`: this throttles rather than walls.
|
||||
# Past it the kernel reclaims aggressively and the cgroup stalls, so a
|
||||
# subagent that overshoots gets visibly slow and the remaining third
|
||||
# stays available for the agent's own turn — but a subagent that
|
||||
# genuinely needs more than two thirds still gets it when the container
|
||||
# has the memory to spare, which is what keeps overprovisioning
|
||||
# (several agents that rarely compile at the same time) working.
|
||||
subagentMemoryHigh = containerMemoryMaxBytes * 2 / 3;
|
||||
# Set on the harness only for an ACP agent on the opencode preset
|
||||
# (./agent-service.nix).
|
||||
acpApiKeyEnv = config.systemd.services.hive-agent.environment.HIVE_ACP_API_KEY_ENV or null;
|
||||
|
|
@ -475,12 +456,11 @@ in
|
|||
# actually chose and leaves the daemon serving, which is also what
|
||||
# lets it report the kill rather than vanish and be restarted.
|
||||
OOMPolicy = "continue";
|
||||
# `MemoryHigh=` is a drop-in from ./memory-cap.nix, sized at boot from
|
||||
# the container's cap.
|
||||
User = userName;
|
||||
Group = userName;
|
||||
}
|
||||
// lib.optionalAttrs (containerMemoryMaxBytes != null) {
|
||||
MemoryHigh = toString subagentMemoryHigh;
|
||||
}
|
||||
// lib.optionalAttrs subagentReadsStore {
|
||||
# Bare id: inherits the credential the container manager passed in,
|
||||
# the same form ./queue-identity.nix uses.
|
||||
|
|
|
|||
68
nix/agent-modules/memory-cap.nix
Normal file
68
nix/agent-modules/memory-cap.nix
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
# The container's memory cap, read at boot rather than built in, so a changed
|
||||
# cap needs no rebuild.
|
||||
#
|
||||
# The cap is `MemoryMax=` on the host's `container@` unit, which nothing in
|
||||
# here can read: `/sys/fs/cgroup/memory.max` reads `max` at every level from
|
||||
# inside. hive-c0re writes its byte count to `/run/hive/memory-max-bytes`
|
||||
# before every start (`lifecycle::host_config`), and removes the file when the
|
||||
# cap is `infinity` or a RAM percentage. Two values are derived from it, and
|
||||
# neither is set without the file:
|
||||
#
|
||||
# - `BUN_JSC_forceRAMSize`, 75% of the cap. Bun/JavaScriptCore, the runtime
|
||||
# under claude-code, sizes its heap ceiling off `/proc/meminfo`, which under
|
||||
# nspawn is the host's RAM, so without it the heap grows well past the real
|
||||
# wall before GC kicks in hard and choom sessions hang. An environment
|
||||
# generator hands it to every unit of the container's systemd, `machinectl
|
||||
# shell` sessions (`hivectl choom`) included; `/etc/set-environment` exports
|
||||
# it in login shells that no unit starts (`nixos-container root-login`).
|
||||
# - `MemoryHigh=` on `hive-subagent-daemon`, two thirds of the cap, as a
|
||||
# generator drop-in. The daemon spawns nested `claude` sessions as plain
|
||||
# children, so its cgroup already is the "all subagents" cgroup. `High`, not
|
||||
# `Max`: past it the kernel reclaims hard and the subagents slow down, so the
|
||||
# last third stays free for the agent's own turn, but a subagent that needs
|
||||
# more still gets it when the container has memory to spare.
|
||||
#
|
||||
# Tested by `checks.script-test-agent-memory-cap`.
|
||||
{ pkgs, ... }:
|
||||
let
|
||||
# Sets `bytes`, or exits when there is no file or its first line is not a
|
||||
# plain byte count. A leading zero would read as octal, and 16 digits or
|
||||
# more could overflow `bytes * 75`. Shell builtins only.
|
||||
readCap = ''
|
||||
bytes=
|
||||
[ -r /run/hive/memory-max-bytes ] && read -r bytes < /run/hive/memory-max-bytes
|
||||
case "$bytes" in
|
||||
"" | 0* | *[!0-9]* | ????????????????*) exit 0 ;;
|
||||
esac
|
||||
'';
|
||||
|
||||
envGenerator = pkgs.writeShellScript "hyperhive-memory-cap-env" ''
|
||||
${readCap}
|
||||
printf 'BUN_JSC_forceRAMSize=%s\n' "$((bytes * 75 / 100))"
|
||||
'';
|
||||
|
||||
# `$1` is the generator's normal-priority output dir. A drop-in there merges
|
||||
# with the unit's own file in /etc.
|
||||
unitGenerator = pkgs.writeShellScript "hyperhive-memory-cap" ''
|
||||
${readCap}
|
||||
dir="$1/hive-subagent-daemon.service.d"
|
||||
${pkgs.coreutils}/bin/mkdir -p "$dir"
|
||||
printf '[Service]\nMemoryHigh=%s\n' "$((bytes * 2 / 3))" > "$dir/50-memory-high.conf"
|
||||
'';
|
||||
|
||||
# Sourced by every shell's `/etc/set-environment`, so POSIX sh only.
|
||||
profile = pkgs.writeText "hyperhive-memory-cap.sh" ''
|
||||
__hyperhive_cap=$(${envGenerator})
|
||||
case "$__hyperhive_cap" in
|
||||
BUN_JSC_forceRAMSize=?*) export "$__hyperhive_cap" ;;
|
||||
esac
|
||||
unset __hyperhive_cap
|
||||
'';
|
||||
in
|
||||
{
|
||||
environment.etc."systemd/system-environment-generators/50-hyperhive-memory-cap".source =
|
||||
envGenerator;
|
||||
systemd.generators.hyperhive-memory-cap = unitGenerator;
|
||||
environment.etc."hyperhive/memory-cap.sh".source = profile;
|
||||
environment.extraInit = ". /etc/hyperhive/memory-cap.sh";
|
||||
}
|
||||
|
|
@ -45,10 +45,6 @@
|
|||
[ "hyperhive" "claudeMarketplaces" ]
|
||||
[ "services" "hyperhive" "agent" "claudeMarketplaces" ]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "hyperhive" "claudeMemoryMaxBytes" ]
|
||||
[ "services" "hyperhive" "agent" "claudeMemoryMaxBytes" ]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "hyperhive" "claudePlugins" ]
|
||||
[ "services" "hyperhive" "agent" "claudePlugins" ]
|
||||
|
|
|
|||
Loading…
Reference in a new issue