From 2689efebcf881bf7873dd8c71f4569a3b1c3d29c Mon Sep 17 00:00:00 2001 From: damocles Date: Wed, 18 Nov 2026 13:32:26 +0100 Subject: [PATCH] hive-agent-sock: kill what a subagent run leaves in its cgroup after it ends --- docs/4-implementation/agent-tools/subagent.md | 5 +- hive-agent-sock/README.md | 4 +- hive-agent-sock/src/run_cgroup.rs | 79 +++++++++++++++++-- hive-subagent-mcp/src/session.rs | 5 +- 4 files changed, 82 insertions(+), 11 deletions(-) diff --git a/docs/4-implementation/agent-tools/subagent.md b/docs/4-implementation/agent-tools/subagent.md index d2f5915d6..8577fe5a1 100644 --- a/docs/4-implementation/agent-tools/subagent.md +++ b/docs/4-implementation/agent-tools/subagent.md @@ -170,7 +170,10 @@ On `claude`, the signal is always SIGINT, and it goes to claude's whole process group. The signal doesn't reach a command that moved itself out of the group (`setsid`), and claude's Bash tool starts every command that way: claude stops those commands itself when it shuts down on SIGINT. A SIGKILL -would leave them running, so nothing escalates to one. `interrupt` returns +would leave them running, so nothing escalates to one while claude is +alive. Once the run has ended, whatever is still in its cgroup is +SIGKILLed after 30 seconds, so a command that left claude's process group +cannot outlive the run. `interrupt` returns at once, and the name stays claimed until claude has exited: `status` says **stopping**, and the daemon refuses `start` and `continue` on that name until then. A claude that ignores the SIGINT stays stopping until someone kills diff --git a/hive-agent-sock/README.md b/hive-agent-sock/README.md index e29809652..c325260a7 100644 --- a/hive-agent-sock/README.md +++ b/hive-agent-sock/README.md @@ -44,6 +44,8 @@ a crate of its own. `hive-bash-mcp` puts each bash task in `/tasks//` and `hive-subagent-mcp` each subagent run in `/runs/-/`, both with `memory.oom.group=1`, so the OOM killer takes one whole run and leaves the -daemon alone. Both daemons are delegated their cgroup with `Delegate=memory +daemon alone. A subagent run's cgroup is also killed (`cgroup.kill`) if +processes are still in it 30 seconds after the run ended; a bash task's is +left alone. Both daemons are delegated their cgroup with `Delegate=memory pids` and `DelegateSubgroup=daemon` (`nix/agent-modules/mcp.nix`); outside such a unit the module logs one warning and runs stay in the daemon's cgroup. diff --git a/hive-agent-sock/src/run_cgroup.rs b/hive-agent-sock/src/run_cgroup.rs index 509818a65..a0b5a8333 100644 --- a/hive-agent-sock/src/run_cgroup.rs +++ b/hive-agent-sock/src/run_cgroup.rs @@ -45,6 +45,8 @@ const ENTRY_OOM_SCORE_ADJ_ENV: &str = "HIVE_RUN_ENTRY_OOM_SCORE_ADJ"; /// removal is left to the next daemon start. const RELEASE_GRACE: Duration = Duration::from_secs(30); const RELEASE_POLL: Duration = Duration::from_millis(250); +/// How long a killed run cgroup gets to empty before it is left in place. +const KILL_GRACE: Duration = Duration::from_secs(5); /// The `//` node run cgroups are created under. #[derive(Debug)] @@ -126,20 +128,36 @@ impl RunCgroups { } other => other?, } - let cgroup = RunCgroup { dir }; + let cgroup = RunCgroup { + dir, + kill_leftovers: false, + }; std::fs::write(cgroup.dir.join("memory.oom.group"), "1")?; Ok(cgroup) } } /// One run's cgroup. Dropping it removes the directory once the run's last -/// process has left, waiting up to 30 seconds on a background thread. +/// process has left, waiting up to 30 seconds on a background thread. With +/// [`RunCgroup::killing_leftovers`], processes still in it after that wait +/// are killed. #[derive(Debug)] pub struct RunCgroup { dir: PathBuf, + kill_leftovers: bool, } impl RunCgroup { + /// Kill whatever is still in the cgroup once the run has ended and the + /// release wait has run out. For a run whose caller has nothing left + /// that could stop a process that left the run's process group + /// (`setsid`, `nohup ... &`): the cgroup is the only handle on it. + #[must_use] + pub fn killing_leftovers(mut self) -> Self { + self.kill_leftovers = true; + self + } + /// The run's `cgroup.procs`, open for [`enter_in_pre_exec`]. Keep it /// open until the spawn has returned. /// @@ -186,7 +204,8 @@ impl Drop for RunCgroup { match std::fs::remove_dir(&self.dir) { Err(e) if e.raw_os_error() == Some(libc::EBUSY) => { let dir = std::mem::take(&mut self.dir); - std::thread::spawn(move || release_when_empty(&dir)); + let kill = self.kill_leftovers; + std::thread::spawn(move || release_when_empty(&dir, kill)); } Err(e) if e.kind() != io::ErrorKind::NotFound => { tracing::warn!(dir = %self.dir.display(), error = %e, "removing run cgroup failed"); @@ -197,17 +216,42 @@ impl Drop for RunCgroup { } /// Retry removing a still-populated run cgroup until it empties or -/// [`RELEASE_GRACE`] runs out; past that the next daemon start removes it. -fn release_when_empty(dir: &Path) { - let deadline = std::time::Instant::now() + RELEASE_GRACE; +/// [`RELEASE_GRACE`] runs out. Past that, `kill` SIGKILLs what is left and +/// waits once more; otherwise (or if that fails) the next daemon start +/// removes it. +fn release_when_empty(dir: &Path, kill: bool) { + if wait_until_removed(dir, RELEASE_GRACE) { + return; + } + if kill { + tracing::warn!(dir = %dir.display(), "run cgroup still populated after the run ended; killing it"); + if let Err(e) = kill_all(dir) { + tracing::warn!(dir = %dir.display(), error = %e, "killing run cgroup failed"); + } else if wait_until_removed(dir, KILL_GRACE) { + return; + } + } + tracing::info!(dir = %dir.display(), "run cgroup still populated after the run ended; left in place"); +} + +/// `true` once `dir` is gone (or cannot be removed for a reason other than +/// processes still in it), `false` when `grace` ran out first. +fn wait_until_removed(dir: &Path, grace: Duration) -> bool { + let deadline = std::time::Instant::now() + grace; while std::time::Instant::now() < deadline { std::thread::sleep(RELEASE_POLL); match std::fs::remove_dir(dir) { Err(e) if e.raw_os_error() == Some(libc::EBUSY) => {} - _ => return, + _ => return true, } } - tracing::info!(dir = %dir.display(), "run cgroup still populated after the run ended; left in place"); + false +} + +/// SIGKILL every process in the cgroup at `dir`, including any forked while +/// the kill runs (`cgroup.kill`, Linux 5.14). +fn kill_all(dir: &Path) -> io::Result<()> { + std::fs::write(dir.join("cgroup.kill"), "1") } /// Move the calling process into the cgroup whose `cgroup.procs` is `procs`. @@ -329,6 +373,25 @@ mod tests { assert!(!dir.exists()); } + #[test] + fn kill_all_writes_one_to_cgroup_kill() { + let dir = tempfile::tempdir().expect("tempdir"); + kill_all(dir.path()).expect("kill"); + assert_eq!( + std::fs::read_to_string(dir.path().join("cgroup.kill")).expect("read"), + "1" + ); + } + + #[test] + fn a_run_cgroup_kills_its_leftovers_only_when_asked() { + let (root, _unit) = unit_tree(); + let cgroups = RunCgroups::init_at(root.path(), OWN, KIND).expect("init"); + let run = cgroups.create("alpha-0").expect("create"); + assert!(!run.kill_leftovers); + assert!(run.killing_leftovers().kill_leftovers); + } + #[test] fn create_replaces_an_empty_leftover_of_the_same_name() { let (root, unit) = unit_tree(); diff --git a/hive-subagent-mcp/src/session.rs b/hive-subagent-mcp/src/session.rs index 9e4a1f813..37dd55be4 100644 --- a/hive-subagent-mcp/src/session.rs +++ b/hive-subagent-mcp/src/session.rs @@ -515,7 +515,9 @@ impl State { /// A new cgroup for one run of `name`; `None` without run cgroups, or /// (logged) when creating it failed, and the run then stays in the - /// daemon's cgroup. + /// daemon's cgroup. Whatever a run left running in it (a tool child that + /// left claude's process group, which `interrupt`'s SIGINT never + /// reaches) is killed once the run has ended. fn run_cgroup(&self, name: &str) -> Option { let cgroups = self.run_cgroups.as_ref()?; let n = self.run_seq.fetch_add(1, Ordering::Relaxed); @@ -523,6 +525,7 @@ impl State { .create(&format!("{name}-{n}")) .inspect_err(|e| tracing::warn!(name, error = %e, "subagent: run cgroup unavailable")) .ok() + .map(RunCgroup::killing_leftovers) } /// `config` with its program started through the entry wrapper, at