agents: hand over the kept drop-in's cap, warn at boot without one

When hive-c0re can't read resource-limits.json and a limits drop-in
already exists, it keeps the drop-in, but it wrote no memory cap file. A
container whose runtime dir had no file yet (hive-c0re upgraded, /run
not cleared) then started with no heap ceiling and no subagent
MemoryHigh=, silently. `stage_limits` now writes the file on every
call: from the new limits, or from the last `MemoryMax=` in the kept
drop-in (`infinity` when it has none or can't be read).

A missing or malformed cap file at boot is no longer silent. The
generators run before the container's journal and an nspawn container
has no /dev/kmsg, so a oneshot `hyperhive-memory-cap-check.service`
logs a warning naming the path. The agent then runs with no JSC heap
ceiling and no subagent MemoryHigh=; the container's MemoryMax= still
applies.

module-eval-agent-memory gains a case pinning how the ceiling reaches
claude: `hive-agent` and `hive-subagent-daemon` neither set nor unset
BUN_JSC_forceRAMSize, both generators and the check unit are installed.

Part of #5081 (slice S0-2).

docs/scheduler/coordinator.md

@@ -459,6 +459,11 @@ values from it: `BUN_JSC_forceRAMSize` (75%, claude-code's heap ceiling)
 and `MemoryHigh=` on `hive-subagent-daemon` (two thirds). Neither is in the
 agent's build, so a changed cap reaches them at the next container start
 without a rebuild. A cap of `infinity` or a RAM percentage sets neither.
+When hive-c0re keeps the last drop-in because it can't read the overrides
+file, the file carries that drop-in's `MemoryMax=`. A missing or malformed
+file sets neither value, and `hyperhive-memory-cap-check.service` logs a
+warning naming the path in the container's journal at boot. The
+container's `MemoryMax=` still applies.
 
 The same drop-in carries `CPUWeight=` / `IOWeight=` from
 `agentCpuWeight` / `agentIoWeight`. Those are a different kind of

hive-c0re/src/lifecycle/host_config.rs

@@ -61,8 +61,8 @@ const MEMORY_CAP_FILE: &str = "memory-max-bytes";
 /// [`limits_to_apply`], plus handing the memory cap to the agent through
 /// [`MEMORY_CAP_FILE`] in `runtime_dir`. The container cannot read the cap
 /// systemd enforces on it, so this file is how it learns the value it sizes
-/// its heap and its subagents against. When the drop-in is kept as it is, so is
+/// its heap and its subagents against. It is written on every call, from the
-/// the file.
+/// `MemoryMax=` the agent will start under: the new one, or the kept drop-in's.
 fn stage_limits(
     agent_name: &str,
     limits_path: &Path,
@@ -71,12 +71,39 @@ fn stage_limits(
     hive: &HiveEnv,
 ) -> Result<Option<(String, String)>> {
     let limits = limits_to_apply(agent_name, limits_path, dropin_path, hive);
-    if let Some((_, memory_max)) = &limits {
+    let memory_max = match &limits {
-        write_memory_cap(runtime_dir, agent_name, memory_max)?;
+        Some((_, memory_max)) => memory_max.clone(),
-    }
+        None => dropin_memory_max(agent_name, dropin_path),
+    };
+    write_memory_cap(runtime_dir, agent_name, &memory_max)?;
     Ok(limits)
 }
 
+/// The `MemoryMax=` the drop-in at `path` applies: its last assignment, as
+/// systemd reads it. `infinity`, systemd's default, when it has none, its last
+/// one is empty, or the file can't be read.
+fn dropin_memory_max(agent_name: &str, path: &Path) -> String {
+    let body = match std::fs::read_to_string(path) {
+        Ok(body) => body,
+        Err(e) => {
+            tracing::warn!(
+                agent = %agent_name,
+                path = %path.display(),
+                error = ?e,
+                "limits drop-in unreadable; the agent gets no memory cap byte count"
+            );
+            return "infinity".to_owned();
+        }
+    };
+    body.lines()
+        .rev()
+        .find_map(|line| line.trim().strip_prefix("MemoryMax="))
+        .map(str::trim)
+        .filter(|v| !v.is_empty())
+        .unwrap_or("infinity")
+        .to_owned()
+}
+
 /// Write `memory_max` into [`MEMORY_CAP_FILE`] as a byte count, or remove the
 /// file when the value has none: `infinity`, or a RAM percentage, which needs
 /// the host's `MemTotal` to resolve. With no file the agent sets no heap or
@@ -815,15 +842,33 @@ mod tests {
     }
 
     #[test]
-    fn a_kept_dropin_keeps_the_agents_copy() {
+    fn a_kept_dropin_hands_the_agent_its_cap() {
         let dir = tempfile::tempdir().expect("tempdir");
         let limits = dir.path().join("resource-limits.json");
+        let dropin = dir.path().join("hyperhive-limits.conf");
+        let copy = dir.path().join("runtime").join(MEMORY_CAP_FILE);
         std::fs::write(&limits, r#"{"sock": {"memory_max": "1G"}}"#).expect("seed limits");
         assert_eq!(stage(&dir).1, Some("1073741824\n".to_owned()));
 
+        // Unreadable limits keep the drop-in; the copy follows the drop-in.
         std::fs::write(&limits, TRUNCATED_LIMITS).expect("seed limits");
-        std::fs::write(dir.path().join("hyperhive-limits.conf"), "MemoryMax=1G\n")
+        std::fs::write(
+            &dropin,
+            "[Unit]\nConditionPathIsDirectory=/rt/sock\n\n[Service]\nRestartSec=5\nMemoryMax=2G\nCPUQuota=200%\n",
+        )
         .expect("seed drop-in");
-        assert_eq!(stage(&dir), (None, Some("1073741824\n".to_owned())));
+        assert_eq!(stage(&dir), (None, Some("2147483648\n".to_owned())));
+
+        // A kept drop-in with no copy beside it, as after an upgrade.
+        std::fs::remove_file(&copy).expect("remove copy");
+        assert_eq!(stage(&dir), (None, Some("2147483648\n".to_owned())));
+
+        // The last assignment is the one systemd applies.
+        std::fs::write(&dropin, "[Service]\nMemoryMax=2G\nMemoryMax=3G\n").expect("seed drop-in");
+        assert_eq!(stage(&dir), (None, Some("3221225472\n".to_owned())));
+
+        // No `MemoryMax=` is systemd's default, `infinity`: no byte count.
+        std::fs::write(&dropin, "[Service]\nCPUQuota=200%\n").expect("seed drop-in");
+        assert_eq!(stage(&dir), (None, None));
     }
 }

nix/agent-modules/memory-cap.nix

@@ -6,7 +6,8 @@
 # inside. hive-c0re writes its byte count to `/run/hive/memory-max-bytes`
 # before every start (`lifecycle::host_config`), and removes the file when the
 # cap is `infinity` or a RAM percentage. Two values are derived from it, and
-# neither is set without the file:
+# neither is set without the file; `hyperhive-memory-cap-check` then logs a
+# warning at boot:
 #
 # - `BUN_JSC_forceRAMSize`, 75% of the cap. Bun/JavaScriptCore, the runtime
 #   under claude-code, sizes its heap ceiling off `/proc/meminfo`, which under
@@ -15,6 +16,8 @@
 #   generator hands it to every unit of the container's systemd, `machinectl
 #   shell` sessions (`hivectl choom`) included; `/etc/set-environment` exports
 #   it in login shells that no unit starts (`nixos-container root-login`).
+#   `hive-agent` and `hive-subagent-daemon` spawn `claude` as a plain child,
+#   which inherits it, so neither unit may set or unset it.
 # - `MemoryHigh=` on `hive-subagent-daemon`, two thirds of the cap, as a
 #   generator drop-in. The daemon spawns nested `claude` sessions as plain
 #   children, so its cgroup already is the "all subagents" cgroup. `High`, not
@@ -22,34 +25,50 @@
 #   last third stays free for the agent's own turn, but a subagent that needs
 #   more still gets it when the container has memory to spare.
 #
-# Tested by `checks.script-test-agent-memory-cap`.
+# Tested by `checks.script-test-agent-memory-cap` and
+# `checks.module-eval-agent-memory`.
 { pkgs, ... }:
 let
-  # Sets `bytes`, or exits when there is no file or its first line is not a
+  # Sets `bytes`, or runs `noCap` when there is no file or its first line is
-  # plain byte count. A leading zero would read as octal, and 16 digits or
+  # not a plain byte count. A leading zero would read as octal, and 16 digits
-  # more could overflow `bytes * 75`. Shell builtins only.
+  # or more could overflow `bytes * 75`. Shell builtins only.
-  readCap = ''
+  readCap = noCap: ''
     bytes=
     [ -r /run/hive/memory-max-bytes ] && read -r bytes < /run/hive/memory-max-bytes
     case "$bytes" in
-      "" | 0* | *[!0-9]* | ????????????????*) exit 0 ;;
+      "" | 0* | *[!0-9]* | ????????????????*) ${noCap} ;;
     esac
   '';
 
   envGenerator = pkgs.writeShellScript "hyperhive-memory-cap-env" ''
-    ${readCap}
+    ${readCap "exit 0"}
     printf 'BUN_JSC_forceRAMSize=%s\n' "$((bytes * 75 / 100))"
   '';
 
   # `$1` is the generator's normal-priority output dir. A drop-in there merges
   # with the unit's own file in /etc.
   unitGenerator = pkgs.writeShellScript "hyperhive-memory-cap" ''
-    ${readCap}
+    ${readCap "exit 0"}
     dir="$1/hive-subagent-daemon.service.d"
     ${pkgs.coreutils}/bin/mkdir -p "$dir"
     printf '[Service]\nMemoryHigh=%s\n' "$((bytes * 2 / 3))" > "$dir/50-memory-high.conf"
   '';
 
+  # The generators run before the container's journal starts and an nspawn
+  # container has no /dev/kmsg, so the boot warning for an unusable cap comes
+  # from a unit. `<4>` is the journal's warning priority.
+  check = pkgs.writeShellScript "hyperhive-memory-cap-check" ''
+    ${readCap ''
+      if [ -e /run/hive/memory-max-bytes ]; then
+        why="does not start with a plain byte count"
+      else
+        why="is missing"
+      fi
+      echo "<4>/run/hive/memory-max-bytes $why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies. hive-c0re writes no file for a cap of infinity or a RAM percentage." >&2
+      exit 0
+    ''}
+  '';
+
   # Sourced by every shell's `/etc/set-environment`, so POSIX sh only.
   profile = pkgs.writeText "hyperhive-memory-cap.sh" ''
     __hyperhive_cap=$(${envGenerator})
@@ -63,6 +82,14 @@ in
   environment.etc."systemd/system-environment-generators/50-hyperhive-memory-cap".source =
     envGenerator;
   systemd.generators.hyperhive-memory-cap = unitGenerator;
+  systemd.services.hyperhive-memory-cap-check = {
+    description = "Warn when the container's memory cap is unusable";
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = check;
+    };
+  };
   environment.etc."hyperhive/memory-cap.sh".source = profile;
   environment.extraInit = ". /etc/hyperhive/memory-cap.sh";
 }

nix/module-eval/agent-memory.nix

@@ -34,6 +34,26 @@ let
       name = "the subagent daemon's memory ceiling is not built into its unit";
       ok = !((agentSubagentDaemon agentDefault).serviceConfig ? MemoryHigh);
     }
+    {
+      # claude's heap ceiling reaches it only as inherited environment: the
+      # environment generator fills the manager's, systemd hands that to every
+      # unit, and both units spawn claude as a plain child. A unit that sets
+      # or unsets the variable itself cuts that. Without the two generators the
+      # cap reaches nothing, and without the check unit nothing says so.
+      name = "claude's units take the heap ceiling from the memory cap read at boot";
+      ok =
+        let
+          leavesIt =
+            unit:
+            !(unit.environment ? BUN_JSC_forceRAMSize)
+            && !(lib.elem "BUN_JSC_forceRAMSize" (lib.toList (unit.serviceConfig.UnsetEnvironment or [ ])));
+        in
+        leavesIt (agentHarness agentDefault)
+        && leavesIt (agentSubagentDaemon agentDefault)
+        && agentDefault.environment.etc ? "systemd/system-environment-generators/50-hyperhive-memory-cap"
+        && agentDefault.systemd.generators ? hyperhive-memory-cap
+        && lib.elem "multi-user.target" agentDefault.systemd.services.hyperhive-memory-cap-check.wantedBy;
+    }
     {
       # The sign is the whole property, and it is easy to write backwards:
       # a HIGHER OOMScoreAdjust is a MORE likely victim. So the subagent

nix/script-tests/agent-memory-cap.nix

@@ -1,7 +1,8 @@
-# `checks.script-test-agent-memory-cap` — runs the three readers of
+# `checks.script-test-agent-memory-cap` — runs the four readers of
 # `/run/hive/memory-max-bytes` in ../agent-modules/memory-cap.nix: the
-# environment generator, the unit generator, and the login-shell snippet
+# environment generator, the unit generator, the boot check that warns about
-# `/etc/set-environment` sources. The cases are in ./agent-memory-cap.sh.
+# an unusable cap, and the login-shell snippet `/etc/set-environment` sources.
+# The cases are in ./agent-memory-cap.sh.
 #
 # What runs is the rendered scripts. The one edit is the cap file's path, moved
 # under the build directory, and the snippet's reference to the environment
@@ -32,6 +33,7 @@ pkgs.runCommand "hyperhive-script-test-agent-memory-cap"
     ENV_GENERATOR =
       machine.environment.etc."systemd/system-environment-generators/50-hyperhive-memory-cap".source;
     UNIT_GENERATOR = machine.systemd.generators.hyperhive-memory-cap;
+    CHECK = machine.systemd.services.hyperhive-memory-cap-check.serviceConfig.ExecStart;
     PROFILE = machine.environment.etc."hyperhive/memory-cap.sh".source;
     EXTRA_INIT = machine.environment.extraInit;
     SHELLS = "${pkgs.dash}/bin/dash ${pkgs.bash}/bin/sh";

nix/script-tests/agent-memory-cap.sh

@@ -1,7 +1,7 @@
 # Cases for ./agent-memory-cap.nix. Each case writes a cap file (or none), runs
-# both generators with a clean environment and compares what they produced,
+# both generators and the boot check with a clean environment and compares what
-# then sources the login-shell snippet under each POSIX shell and compares what
+# they produced, then sources the login-shell snippet under each POSIX shell
-# a child process sees.
+# and compares what a child process sees.
 set -uo pipefail
 
 failures=0
@@ -16,12 +16,13 @@ dir=$(mktemp -d)
 cap=$dir/memory-max-bytes
 sed "s|/run/hive/memory-max-bytes|$cap|g" "$ENV_GENERATOR" >"$dir/env-generator"
 sed "s|/run/hive/memory-max-bytes|$cap|g" "$UNIT_GENERATOR" >"$dir/unit-generator"
-chmod +x "$dir/env-generator" "$dir/unit-generator"
+sed "s|/run/hive/memory-max-bytes|$cap|g" "$CHECK" >"$dir/check"
+chmod +x "$dir/env-generator" "$dir/unit-generator" "$dir/check"
 sed "s|$ENV_GENERATOR|$dir/env-generator|g" "$PROFILE" >"$dir/profile"
 envbin=$(command -v env)
 
 case="setup"
-for g in env-generator unit-generator; do
+for g in env-generator unit-generator check; do
   if grep -q '/run/hive/memory-max-bytes' "$dir/$g"; then fail "$g still names /run/hive after the rewrite"; fi
   if ! grep -qF "$cap" "$dir/$g"; then fail "the rewrite did not reach $g's read"; fi
 done
@@ -31,10 +32,11 @@ if ! printf '%s\n' "$EXTRA_INIT" | grep -qxF '. /etc/hyperhive/memory-cap.sh'; t
 fi
 
 # check <description> <expected BUN_JSC_forceRAMSize or ""> <expected MemoryHigh or "">
+#       <why the boot check warns, or "">
 check() {
   case=$1
   count=$((count + 1))
-  local want_env=${2:+BUN_JSC_forceRAMSize=$2} want_high=$3 got
+  local want_env=${2:+BUN_JSC_forceRAMSize=$2} want_high=$3 want_why=${4:-} got
   got=$(env -i "$dir/env-generator")
   if [ "$got" != "$want_env" ]; then
     fail "environment generator printed:"$'\n'"$got"$'\n'"wanted:"$'\n'"$want_env"
@@ -55,6 +57,17 @@ check() {
     fi
   fi
 
+  # The warning goes to stderr with the journal's `<4>` priority prefix, and
+  # nothing goes anywhere for a usable cap.
+  got=$(env -i "$dir/check" 2>&1)
+  local want_warn=
+  if [ -n "$want_why" ]; then
+    want_warn="<4>$cap $want_why: no BUN_JSC_forceRAMSize heap ceiling and no MemoryHigh= on hive-subagent-daemon; the host's MemoryMax= on the container still applies. hive-c0re writes no file for a cap of infinity or a RAM percentage."
+  fi
+  if [ "$got" != "$want_warn" ]; then
+    fail "boot check printed:"$'\n'"$got"$'\n'"wanted:"$'\n'"$want_warn"
+  fi
+
   local sh
   for sh in $SHELLS; do
     # Only what the snippet exported reaches the child's environment. No PATH:
@@ -76,19 +89,19 @@ printf '4294967296\nLD_PRELOAD=/x\n' >"$cap"
 check "only the first line is read" 3221225472 2863311530
 
 printf '8G\n' >"$cap"
-check "a size with a suffix is ignored" "" ""
+check "a size with a suffix is ignored" "" "" "does not start with a plain byte count"
 
 printf '0123\n' >"$cap"
-check "a leading zero is ignored" "" ""
+check "a leading zero is ignored" "" "" "does not start with a plain byte count"
 
 printf '1234567890123456\n' >"$cap"
-check "sixteen digits are ignored" "" ""
+check "sixteen digits are ignored" "" "" "does not start with a plain byte count"
 
 : >"$cap"
-check "an empty file" "" ""
+check "an empty file" "" "" "does not start with a plain byte count"
 
 rm -f "$cap"
-check "no file" "" ""
+check "no file" "" "" "is missing"
 
 echo "$count cases, $failures failed"
 [ "$failures" -eq 0 ]