Watch
0
0
Fork
You've already forked hyperhive
0

hive-agent-sock: kill what a subagent run leaves in its cgroup after it ends

This commit is contained in:
damocles 2026-11-18 13:32:26 +01:00 • committed by mara
commit 2689efebcf
4 changed files with 82 additions and 11 deletions

View file

@ -170,7 +170,10 @@ On `claude`, the signal is always SIGINT, and it goes to claude's whole
process group. The signal doesn't reach a command that moved itself out of process group. The signal doesn't reach a command that moved itself out of
the group (`setsid`), and claude's Bash tool starts every command that way: the group (`setsid`), and claude's Bash tool starts every command that way:
claude stops those commands itself when it shuts down on SIGINT. A SIGKILL claude stops those commands itself when it shuts down on SIGINT. A SIGKILL
would leave them running, so nothing escalates to one. `interrupt` returns would leave them running, so nothing escalates to one while claude is
alive. Once the run has ended, whatever is still in its cgroup is
SIGKILLed after 30 seconds, so a command that left claude's process group
cannot outlive the run. `interrupt` returns
at once, and the name stays claimed until claude has exited: `status` says at once, and the name stays claimed until claude has exited: `status` says
**stopping**, and the daemon refuses `start` and `continue` on that name **stopping**, and the daemon refuses `start` and `continue` on that name
until then. A claude that ignores the SIGINT stays stopping until someone kills until then. A claude that ignores the SIGINT stays stopping until someone kills

View file

@ -44,6 +44,8 @@ a crate of its own.
`hive-bash-mcp` puts each bash task in `<unit>/tasks/<id>/` and `hive-bash-mcp` puts each bash task in `<unit>/tasks/<id>/` and
`hive-subagent-mcp` each subagent run in `<unit>/runs/<name>-<n>/`, both with `hive-subagent-mcp` each subagent run in `<unit>/runs/<name>-<n>/`, both with
`memory.oom.group=1`, so the OOM killer takes one whole run and leaves the `memory.oom.group=1`, so the OOM killer takes one whole run and leaves the
daemon alone. Both daemons are delegated their cgroup with `Delegate=memory daemon alone. A subagent run's cgroup is also killed (`cgroup.kill`) if
processes are still in it 30 seconds after the run ended; a bash task's is
left alone. Both daemons are delegated their cgroup with `Delegate=memory
pids` and `DelegateSubgroup=daemon` (`nix/agent-modules/mcp.nix`); outside such pids` and `DelegateSubgroup=daemon` (`nix/agent-modules/mcp.nix`); outside such
a unit the module logs one warning and runs stay in the daemon's cgroup. a unit the module logs one warning and runs stay in the daemon's cgroup.

View file

@ -45,6 +45,8 @@ const ENTRY_OOM_SCORE_ADJ_ENV: &str = "HIVE_RUN_ENTRY_OOM_SCORE_ADJ";
/// removal is left to the next daemon start. /// removal is left to the next daemon start.
const RELEASE_GRACE: Duration = Duration::from_secs(30); const RELEASE_GRACE: Duration = Duration::from_secs(30);
const RELEASE_POLL: Duration = Duration::from_millis(250); const RELEASE_POLL: Duration = Duration::from_millis(250);
/// How long a killed run cgroup gets to empty before it is left in place.
const KILL_GRACE: Duration = Duration::from_secs(5);
/// The `<unit>/<kind>/` node run cgroups are created under. /// The `<unit>/<kind>/` node run cgroups are created under.
#[derive(Debug)] #[derive(Debug)]
@ -126,20 +128,36 @@ impl RunCgroups {
} }
other => other?, other => other?,
} }
let cgroup = RunCgroup { dir }; let cgroup = RunCgroup {
dir,
kill_leftovers: false,
};
std::fs::write(cgroup.dir.join("memory.oom.group"), "1")?; std::fs::write(cgroup.dir.join("memory.oom.group"), "1")?;
Ok(cgroup) Ok(cgroup)
} }
} }
/// One run's cgroup. Dropping it removes the directory once the run's last /// One run's cgroup. Dropping it removes the directory once the run's last
/// process has left, waiting up to 30 seconds on a background thread. /// process has left, waiting up to 30 seconds on a background thread. With
/// [`RunCgroup::killing_leftovers`], processes still in it after that wait
/// are killed.
#[derive(Debug)] #[derive(Debug)]
pub struct RunCgroup { pub struct RunCgroup {
dir: PathBuf, dir: PathBuf,
kill_leftovers: bool,
} }
impl RunCgroup { impl RunCgroup {
/// Kill whatever is still in the cgroup once the run has ended and the
/// release wait has run out. For a run whose caller has nothing left
/// that could stop a process that left the run's process group
/// (`setsid`, `nohup ... &`): the cgroup is the only handle on it.
#[must_use]
pub fn killing_leftovers(mut self) -> Self {
self.kill_leftovers = true;
self
}
/// The run's `cgroup.procs`, open for [`enter_in_pre_exec`]. Keep it /// The run's `cgroup.procs`, open for [`enter_in_pre_exec`]. Keep it
/// open until the spawn has returned. /// open until the spawn has returned.
/// ///
@ -186,7 +204,8 @@ impl Drop for RunCgroup {
match std::fs::remove_dir(&self.dir) { match std::fs::remove_dir(&self.dir) {
Err(e) if e.raw_os_error() == Some(libc::EBUSY) => { Err(e) if e.raw_os_error() == Some(libc::EBUSY) => {
let dir = std::mem::take(&mut self.dir); let dir = std::mem::take(&mut self.dir);
std::thread::spawn(move || release_when_empty(&dir)); let kill = self.kill_leftovers;
std::thread::spawn(move || release_when_empty(&dir, kill));
} }
Err(e) if e.kind() != io::ErrorKind::NotFound => { Err(e) if e.kind() != io::ErrorKind::NotFound => {
tracing::warn!(dir = %self.dir.display(), error = %e, "removing run cgroup failed"); tracing::warn!(dir = %self.dir.display(), error = %e, "removing run cgroup failed");
@ -197,17 +216,42 @@ impl Drop for RunCgroup {
} }
/// Retry removing a still-populated run cgroup until it empties or /// Retry removing a still-populated run cgroup until it empties or
/// [`RELEASE_GRACE`] runs out; past that the next daemon start removes it. /// [`RELEASE_GRACE`] runs out. Past that, `kill` SIGKILLs what is left and
fn release_when_empty(dir: &Path) { /// waits once more; otherwise (or if that fails) the next daemon start
let deadline = std::time::Instant::now() + RELEASE_GRACE; /// removes it.
fn release_when_empty(dir: &Path, kill: bool) {
if wait_until_removed(dir, RELEASE_GRACE) {
return;
}
if kill {
tracing::warn!(dir = %dir.display(), "run cgroup still populated after the run ended; killing it");
if let Err(e) = kill_all(dir) {
tracing::warn!(dir = %dir.display(), error = %e, "killing run cgroup failed");
} else if wait_until_removed(dir, KILL_GRACE) {
return;
}
}
tracing::info!(dir = %dir.display(), "run cgroup still populated after the run ended; left in place");
}
/// `true` once `dir` is gone (or cannot be removed for a reason other than
/// processes still in it), `false` when `grace` ran out first.
fn wait_until_removed(dir: &Path, grace: Duration) -> bool {
let deadline = std::time::Instant::now() + grace;
while std::time::Instant::now() < deadline { while std::time::Instant::now() < deadline {
std::thread::sleep(RELEASE_POLL); std::thread::sleep(RELEASE_POLL);
match std::fs::remove_dir(dir) { match std::fs::remove_dir(dir) {
Err(e) if e.raw_os_error() == Some(libc::EBUSY) => {} Err(e) if e.raw_os_error() == Some(libc::EBUSY) => {}
_ => return, _ => return true,
} }
} }
tracing::info!(dir = %dir.display(), "run cgroup still populated after the run ended; left in place"); false
}
/// SIGKILL every process in the cgroup at `dir`, including any forked while
/// the kill runs (`cgroup.kill`, Linux 5.14).
fn kill_all(dir: &Path) -> io::Result<()> {
std::fs::write(dir.join("cgroup.kill"), "1")
} }
/// Move the calling process into the cgroup whose `cgroup.procs` is `procs`. /// Move the calling process into the cgroup whose `cgroup.procs` is `procs`.
@ -329,6 +373,25 @@ mod tests {
assert!(!dir.exists()); assert!(!dir.exists());
} }
#[test]
fn kill_all_writes_one_to_cgroup_kill() {
let dir = tempfile::tempdir().expect("tempdir");
kill_all(dir.path()).expect("kill");
assert_eq!(
std::fs::read_to_string(dir.path().join("cgroup.kill")).expect("read"),
"1"
);
}
#[test]
fn a_run_cgroup_kills_its_leftovers_only_when_asked() {
let (root, _unit) = unit_tree();
let cgroups = RunCgroups::init_at(root.path(), OWN, KIND).expect("init");
let run = cgroups.create("alpha-0").expect("create");
assert!(!run.kill_leftovers);
assert!(run.killing_leftovers().kill_leftovers);
}
#[test] #[test]
fn create_replaces_an_empty_leftover_of_the_same_name() { fn create_replaces_an_empty_leftover_of_the_same_name() {
let (root, unit) = unit_tree(); let (root, unit) = unit_tree();

View file

@ -515,7 +515,9 @@ impl State {
/// A new cgroup for one run of `name`; `None` without run cgroups, or /// A new cgroup for one run of `name`; `None` without run cgroups, or
/// (logged) when creating it failed, and the run then stays in the /// (logged) when creating it failed, and the run then stays in the
/// daemon's cgroup. /// daemon's cgroup. Whatever a run left running in it (a tool child that
/// left claude's process group, which `interrupt`'s SIGINT never
/// reaches) is killed once the run has ended.
fn run_cgroup(&self, name: &str) -> Option<RunCgroup> { fn run_cgroup(&self, name: &str) -> Option<RunCgroup> {
let cgroups = self.run_cgroups.as_ref()?; let cgroups = self.run_cgroups.as_ref()?;
let n = self.run_seq.fetch_add(1, Ordering::Relaxed); let n = self.run_seq.fetch_add(1, Ordering::Relaxed);
@ -523,6 +525,7 @@ impl State {
.create(&format!("{name}-{n}")) .create(&format!("{name}-{n}"))
.inspect_err(|e| tracing::warn!(name, error = %e, "subagent: run cgroup unavailable")) .inspect_err(|e| tracing::warn!(name, error = %e, "subagent: run cgroup unavailable"))
.ok() .ok()
.map(RunCgroup::killing_leftovers)
} }
/// `config` with its program started through the entry wrapper, at /// `config` with its program started through the entry wrapper, at