Watch
0
0
Fork
You've already forked hyperhive
0

hive-agent-sock: kill what a subagent run leaves in its cgroup after it ends

This commit is contained in:
damocles 2026-11-18 13:32:26 +01:00 • committed by mara
commit 2689efebcf
2 changed files with 82 additions and 11 deletions

View file

@ -44,6 +44,8 @@ a crate of its own.
`hive-bash-mcp` puts each bash task in `<unit>/tasks/<id>/` and
`hive-subagent-mcp` each subagent run in `<unit>/runs/<name>-<n>/`, both with
`memory.oom.group=1`, so the OOM killer takes one whole run and leaves the
daemon alone. Both daemons are delegated their cgroup with `Delegate=memory
daemon alone. A subagent run's cgroup is also killed (`cgroup.kill`) if
processes are still in it 30 seconds after the run ended; a bash task's is
left alone. Both daemons are delegated their cgroup with `Delegate=memory
pids` and `DelegateSubgroup=daemon` (`nix/agent-modules/mcp.nix`); outside such
a unit the module logs one warning and runs stay in the daemon's cgroup.

View file

@ -45,6 +45,8 @@ const ENTRY_OOM_SCORE_ADJ_ENV: &str = "HIVE_RUN_ENTRY_OOM_SCORE_ADJ";
/// removal is left to the next daemon start.
const RELEASE_GRACE: Duration = Duration::from_secs(30);
const RELEASE_POLL: Duration = Duration::from_millis(250);
/// How long a killed run cgroup gets to empty before it is left in place.
const KILL_GRACE: Duration = Duration::from_secs(5);
/// The `<unit>/<kind>/` node run cgroups are created under.
#[derive(Debug)]
@ -126,20 +128,36 @@ impl RunCgroups {
}
other => other?,
}
let cgroup = RunCgroup { dir };
let cgroup = RunCgroup {
dir,
kill_leftovers: false,
};
std::fs::write(cgroup.dir.join("memory.oom.group"), "1")?;
Ok(cgroup)
}
}
/// One run's cgroup. Dropping it removes the directory once the run's last
/// process has left, waiting up to 30 seconds on a background thread.
/// process has left, waiting up to 30 seconds on a background thread. With
/// [`RunCgroup::killing_leftovers`], processes still in it after that wait
/// are killed.
#[derive(Debug)]
pub struct RunCgroup {
dir: PathBuf,
kill_leftovers: bool,
}
impl RunCgroup {
/// Kill whatever is still in the cgroup once the run has ended and the
/// release wait has run out. For a run whose caller has nothing left
/// that could stop a process that left the run's process group
/// (`setsid`, `nohup ... &`): the cgroup is the only handle on it.
#[must_use]
pub fn killing_leftovers(mut self) -> Self {
self.kill_leftovers = true;
self
}
/// The run's `cgroup.procs`, open for [`enter_in_pre_exec`]. Keep it
/// open until the spawn has returned.
///
@ -186,7 +204,8 @@ impl Drop for RunCgroup {
match std::fs::remove_dir(&self.dir) {
Err(e) if e.raw_os_error() == Some(libc::EBUSY) => {
let dir = std::mem::take(&mut self.dir);
std::thread::spawn(move || release_when_empty(&dir));
let kill = self.kill_leftovers;
std::thread::spawn(move || release_when_empty(&dir, kill));
}
Err(e) if e.kind() != io::ErrorKind::NotFound => {
tracing::warn!(dir = %self.dir.display(), error = %e, "removing run cgroup failed");
@ -197,17 +216,42 @@ impl Drop for RunCgroup {
}
/// Retry removing a still-populated run cgroup until it empties or
/// [`RELEASE_GRACE`] runs out; past that the next daemon start removes it.
fn release_when_empty(dir: &Path) {
let deadline = std::time::Instant::now() + RELEASE_GRACE;
/// [`RELEASE_GRACE`] runs out. Past that, `kill` SIGKILLs what is left and
/// waits once more; otherwise (or if that fails) the next daemon start
/// removes it.
fn release_when_empty(dir: &Path, kill: bool) {
if wait_until_removed(dir, RELEASE_GRACE) {
return;
}
if kill {
tracing::warn!(dir = %dir.display(), "run cgroup still populated after the run ended; killing it");
if let Err(e) = kill_all(dir) {
tracing::warn!(dir = %dir.display(), error = %e, "killing run cgroup failed");
} else if wait_until_removed(dir, KILL_GRACE) {
return;
}
}
tracing::info!(dir = %dir.display(), "run cgroup still populated after the run ended; left in place");
}
/// `true` once `dir` is gone (or cannot be removed for a reason other than
/// processes still in it), `false` when `grace` ran out first.
fn wait_until_removed(dir: &Path, grace: Duration) -> bool {
let deadline = std::time::Instant::now() + grace;
while std::time::Instant::now() < deadline {
std::thread::sleep(RELEASE_POLL);
match std::fs::remove_dir(dir) {
Err(e) if e.raw_os_error() == Some(libc::EBUSY) => {}
_ => return,
_ => return true,
}
}
tracing::info!(dir = %dir.display(), "run cgroup still populated after the run ended; left in place");
false
}
/// SIGKILL every process in the cgroup at `dir`, including any forked while
/// the kill runs (`cgroup.kill`, Linux 5.14).
fn kill_all(dir: &Path) -> io::Result<()> {
std::fs::write(dir.join("cgroup.kill"), "1")
}
/// Move the calling process into the cgroup whose `cgroup.procs` is `procs`.
@ -329,6 +373,25 @@ mod tests {
assert!(!dir.exists());
}
#[test]
fn kill_all_writes_one_to_cgroup_kill() {
let dir = tempfile::tempdir().expect("tempdir");
kill_all(dir.path()).expect("kill");
assert_eq!(
std::fs::read_to_string(dir.path().join("cgroup.kill")).expect("read"),
"1"
);
}
#[test]
fn a_run_cgroup_kills_its_leftovers_only_when_asked() {
let (root, _unit) = unit_tree();
let cgroups = RunCgroups::init_at(root.path(), OWN, KIND).expect("init");
let run = cgroups.create("alpha-0").expect("create");
assert!(!run.kill_leftovers);
assert!(run.killing_leftovers().kill_leftovers);
}
#[test]
fn create_replaces_an_empty_leftover_of_the_same_name() {
let (root, unit) = unit_tree();