Watch
0
0
Fork
You've already forked hyperhive
0

hive-agent-sock: kill what a subagent run leaves in its cgroup after it ends

This commit is contained in:
damocles 2026-11-18 13:32:26 +01:00 • committed by mara
commit 2689efebcf
4 changed files with 82 additions and 11 deletions

View file

@ -170,7 +170,10 @@ On `claude`, the signal is always SIGINT, and it goes to claude's whole
process group. The signal doesn't reach a command that moved itself out of
the group (`setsid`), and claude's Bash tool starts every command that way:
claude stops those commands itself when it shuts down on SIGINT. A SIGKILL
would leave them running, so nothing escalates to one. `interrupt` returns
would leave them running, so nothing escalates to one while claude is
alive. Once the run has ended, whatever is still in its cgroup is
SIGKILLed after 30 seconds, so a command that left claude's process group
cannot outlive the run. `interrupt` returns
at once, and the name stays claimed until claude has exited: `status` says
**stopping**, and the daemon refuses `start` and `continue` on that name
until then. A claude that ignores the SIGINT stays stopping until someone kills

View file

@ -44,6 +44,8 @@ a crate of its own.
`hive-bash-mcp` puts each bash task in `<unit>/tasks/<id>/` and
`hive-subagent-mcp` each subagent run in `<unit>/runs/<name>-<n>/`, both with
`memory.oom.group=1`, so the OOM killer takes one whole run and leaves the
daemon alone. Both daemons are delegated their cgroup with `Delegate=memory
daemon alone. A subagent run's cgroup is also killed (`cgroup.kill`) if
processes are still in it 30 seconds after the run ended; a bash task's is
left alone. Both daemons are delegated their cgroup with `Delegate=memory
pids` and `DelegateSubgroup=daemon` (`nix/agent-modules/mcp.nix`); outside such
a unit the module logs one warning and runs stay in the daemon's cgroup.

View file

@ -45,6 +45,8 @@ const ENTRY_OOM_SCORE_ADJ_ENV: &str = "HIVE_RUN_ENTRY_OOM_SCORE_ADJ";
/// removal is left to the next daemon start.
const RELEASE_GRACE: Duration = Duration::from_secs(30);
const RELEASE_POLL: Duration = Duration::from_millis(250);
/// How long a killed run cgroup gets to empty before it is left in place.
const KILL_GRACE: Duration = Duration::from_secs(5);
/// The `<unit>/<kind>/` node run cgroups are created under.
#[derive(Debug)]
@ -126,20 +128,36 @@ impl RunCgroups {
}
other => other?,
}
let cgroup = RunCgroup { dir };
let cgroup = RunCgroup {
dir,
kill_leftovers: false,
};
std::fs::write(cgroup.dir.join("memory.oom.group"), "1")?;
Ok(cgroup)
}
}
/// One run's cgroup. Dropping it removes the directory once the run's last
/// process has left, waiting up to 30 seconds on a background thread.
/// process has left, waiting up to 30 seconds on a background thread. With
/// [`RunCgroup::killing_leftovers`], processes still in it after that wait
/// are killed.
#[derive(Debug)]
pub struct RunCgroup {
dir: PathBuf,
kill_leftovers: bool,
}
impl RunCgroup {
/// Kill whatever is still in the cgroup once the run has ended and the
/// release wait has run out. For a run whose caller has nothing left
/// that could stop a process that left the run's process group
/// (`setsid`, `nohup ... &`): the cgroup is the only handle on it.
#[must_use]
pub fn killing_leftovers(mut self) -> Self {
self.kill_leftovers = true;
self
}
/// The run's `cgroup.procs`, open for [`enter_in_pre_exec`]. Keep it
/// open until the spawn has returned.
///
@ -186,7 +204,8 @@ impl Drop for RunCgroup {
match std::fs::remove_dir(&self.dir) {
Err(e) if e.raw_os_error() == Some(libc::EBUSY) => {
let dir = std::mem::take(&mut self.dir);
std::thread::spawn(move || release_when_empty(&dir));
let kill = self.kill_leftovers;
std::thread::spawn(move || release_when_empty(&dir, kill));
}
Err(e) if e.kind() != io::ErrorKind::NotFound => {
tracing::warn!(dir = %self.dir.display(), error = %e, "removing run cgroup failed");
@ -197,17 +216,42 @@ impl Drop for RunCgroup {
}
/// Retry removing a still-populated run cgroup until it empties or
/// [`RELEASE_GRACE`] runs out; past that the next daemon start removes it.
fn release_when_empty(dir: &Path) {
let deadline = std::time::Instant::now() + RELEASE_GRACE;
/// [`RELEASE_GRACE`] runs out. Past that, `kill` SIGKILLs what is left and
/// waits once more; otherwise (or if that fails) the next daemon start
/// removes it.
fn release_when_empty(dir: &Path, kill: bool) {
if wait_until_removed(dir, RELEASE_GRACE) {
return;
}
if kill {
tracing::warn!(dir = %dir.display(), "run cgroup still populated after the run ended; killing it");
if let Err(e) = kill_all(dir) {
tracing::warn!(dir = %dir.display(), error = %e, "killing run cgroup failed");
} else if wait_until_removed(dir, KILL_GRACE) {
return;
}
}
tracing::info!(dir = %dir.display(), "run cgroup still populated after the run ended; left in place");
}
/// `true` once `dir` is gone (or cannot be removed for a reason other than
/// processes still in it), `false` when `grace` ran out first.
fn wait_until_removed(dir: &Path, grace: Duration) -> bool {
let deadline = std::time::Instant::now() + grace;
while std::time::Instant::now() < deadline {
std::thread::sleep(RELEASE_POLL);
match std::fs::remove_dir(dir) {
Err(e) if e.raw_os_error() == Some(libc::EBUSY) => {}
_ => return,
_ => return true,
}
}
tracing::info!(dir = %dir.display(), "run cgroup still populated after the run ended; left in place");
false
}
/// SIGKILL every process in the cgroup at `dir`, including any forked while
/// the kill runs (`cgroup.kill`, Linux 5.14).
fn kill_all(dir: &Path) -> io::Result<()> {
std::fs::write(dir.join("cgroup.kill"), "1")
}
/// Move the calling process into the cgroup whose `cgroup.procs` is `procs`.
@ -329,6 +373,25 @@ mod tests {
assert!(!dir.exists());
}
#[test]
fn kill_all_writes_one_to_cgroup_kill() {
let dir = tempfile::tempdir().expect("tempdir");
kill_all(dir.path()).expect("kill");
assert_eq!(
std::fs::read_to_string(dir.path().join("cgroup.kill")).expect("read"),
"1"
);
}
#[test]
fn a_run_cgroup_kills_its_leftovers_only_when_asked() {
let (root, _unit) = unit_tree();
let cgroups = RunCgroups::init_at(root.path(), OWN, KIND).expect("init");
let run = cgroups.create("alpha-0").expect("create");
assert!(!run.kill_leftovers);
assert!(run.killing_leftovers().kill_leftovers);
}
#[test]
fn create_replaces_an_empty_leftover_of_the_same_name() {
let (root, unit) = unit_tree();

View file

@ -515,7 +515,9 @@ impl State {
/// A new cgroup for one run of `name`; `None` without run cgroups, or
/// (logged) when creating it failed, and the run then stays in the
/// daemon's cgroup.
/// daemon's cgroup. Whatever a run left running in it (a tool child that
/// left claude's process group, which `interrupt`'s SIGINT never
/// reaches) is killed once the run has ended.
fn run_cgroup(&self, name: &str) -> Option<RunCgroup> {
let cgroups = self.run_cgroups.as_ref()?;
let n = self.run_seq.fetch_add(1, Ordering::Relaxed);
@ -523,6 +525,7 @@ impl State {
.create(&format!("{name}-{n}"))
.inspect_err(|e| tracing::warn!(name, error = %e, "subagent: run cgroup unavailable"))
.ok()
.map(RunCgroup::killing_leftovers)
}
/// `config` with its program started through the entry wrapper, at