hive-agent-sock: kill what a subagent run leaves in its cgroup after it ends
This commit is contained in:
parent
566758483d
commit
2689efebcf
2 changed files with 82 additions and 11 deletions
|
|
@ -44,6 +44,8 @@ a crate of its own.
|
|||
`hive-bash-mcp` puts each bash task in `<unit>/tasks/<id>/` and
|
||||
`hive-subagent-mcp` each subagent run in `<unit>/runs/<name>-<n>/`, both with
|
||||
`memory.oom.group=1`, so the OOM killer takes one whole run and leaves the
|
||||
daemon alone. Both daemons are delegated their cgroup with `Delegate=memory
|
||||
daemon alone. A subagent run's cgroup is also killed (`cgroup.kill`) if
|
||||
processes are still in it 30 seconds after the run ended; a bash task's is
|
||||
left alone. Both daemons are delegated their cgroup with `Delegate=memory
|
||||
pids` and `DelegateSubgroup=daemon` (`nix/agent-modules/mcp.nix`); outside such
|
||||
a unit the module logs one warning and runs stay in the daemon's cgroup.
|
||||
|
|
|
|||
|
|
@ -45,6 +45,8 @@ const ENTRY_OOM_SCORE_ADJ_ENV: &str = "HIVE_RUN_ENTRY_OOM_SCORE_ADJ";
|
|||
/// removal is left to the next daemon start.
|
||||
const RELEASE_GRACE: Duration = Duration::from_secs(30);
|
||||
const RELEASE_POLL: Duration = Duration::from_millis(250);
|
||||
/// How long a killed run cgroup gets to empty before it is left in place.
|
||||
const KILL_GRACE: Duration = Duration::from_secs(5);
|
||||
|
||||
/// The `<unit>/<kind>/` node run cgroups are created under.
|
||||
#[derive(Debug)]
|
||||
|
|
@ -126,20 +128,36 @@ impl RunCgroups {
|
|||
}
|
||||
other => other?,
|
||||
}
|
||||
let cgroup = RunCgroup { dir };
|
||||
let cgroup = RunCgroup {
|
||||
dir,
|
||||
kill_leftovers: false,
|
||||
};
|
||||
std::fs::write(cgroup.dir.join("memory.oom.group"), "1")?;
|
||||
Ok(cgroup)
|
||||
}
|
||||
}
|
||||
|
||||
/// One run's cgroup. Dropping it removes the directory once the run's last
|
||||
/// process has left, waiting up to 30 seconds on a background thread.
|
||||
/// process has left, waiting up to 30 seconds on a background thread. With
|
||||
/// [`RunCgroup::killing_leftovers`], processes still in it after that wait
|
||||
/// are killed.
|
||||
#[derive(Debug)]
|
||||
pub struct RunCgroup {
|
||||
dir: PathBuf,
|
||||
kill_leftovers: bool,
|
||||
}
|
||||
|
||||
impl RunCgroup {
|
||||
/// Kill whatever is still in the cgroup once the run has ended and the
|
||||
/// release wait has run out. For a run whose caller has nothing left
|
||||
/// that could stop a process that left the run's process group
|
||||
/// (`setsid`, `nohup ... &`): the cgroup is the only handle on it.
|
||||
#[must_use]
|
||||
pub fn killing_leftovers(mut self) -> Self {
|
||||
self.kill_leftovers = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// The run's `cgroup.procs`, open for [`enter_in_pre_exec`]. Keep it
|
||||
/// open until the spawn has returned.
|
||||
///
|
||||
|
|
@ -186,7 +204,8 @@ impl Drop for RunCgroup {
|
|||
match std::fs::remove_dir(&self.dir) {
|
||||
Err(e) if e.raw_os_error() == Some(libc::EBUSY) => {
|
||||
let dir = std::mem::take(&mut self.dir);
|
||||
std::thread::spawn(move || release_when_empty(&dir));
|
||||
let kill = self.kill_leftovers;
|
||||
std::thread::spawn(move || release_when_empty(&dir, kill));
|
||||
}
|
||||
Err(e) if e.kind() != io::ErrorKind::NotFound => {
|
||||
tracing::warn!(dir = %self.dir.display(), error = %e, "removing run cgroup failed");
|
||||
|
|
@ -197,17 +216,42 @@ impl Drop for RunCgroup {
|
|||
}
|
||||
|
||||
/// Retry removing a still-populated run cgroup until it empties or
|
||||
/// [`RELEASE_GRACE`] runs out; past that the next daemon start removes it.
|
||||
fn release_when_empty(dir: &Path) {
|
||||
let deadline = std::time::Instant::now() + RELEASE_GRACE;
|
||||
/// [`RELEASE_GRACE`] runs out. Past that, `kill` SIGKILLs what is left and
|
||||
/// waits once more; otherwise (or if that fails) the next daemon start
|
||||
/// removes it.
|
||||
fn release_when_empty(dir: &Path, kill: bool) {
|
||||
if wait_until_removed(dir, RELEASE_GRACE) {
|
||||
return;
|
||||
}
|
||||
if kill {
|
||||
tracing::warn!(dir = %dir.display(), "run cgroup still populated after the run ended; killing it");
|
||||
if let Err(e) = kill_all(dir) {
|
||||
tracing::warn!(dir = %dir.display(), error = %e, "killing run cgroup failed");
|
||||
} else if wait_until_removed(dir, KILL_GRACE) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
tracing::info!(dir = %dir.display(), "run cgroup still populated after the run ended; left in place");
|
||||
}
|
||||
|
||||
/// `true` once `dir` is gone (or cannot be removed for a reason other than
|
||||
/// processes still in it), `false` when `grace` ran out first.
|
||||
fn wait_until_removed(dir: &Path, grace: Duration) -> bool {
|
||||
let deadline = std::time::Instant::now() + grace;
|
||||
while std::time::Instant::now() < deadline {
|
||||
std::thread::sleep(RELEASE_POLL);
|
||||
match std::fs::remove_dir(dir) {
|
||||
Err(e) if e.raw_os_error() == Some(libc::EBUSY) => {}
|
||||
_ => return,
|
||||
_ => return true,
|
||||
}
|
||||
}
|
||||
tracing::info!(dir = %dir.display(), "run cgroup still populated after the run ended; left in place");
|
||||
false
|
||||
}
|
||||
|
||||
/// SIGKILL every process in the cgroup at `dir`, including any forked while
|
||||
/// the kill runs (`cgroup.kill`, Linux 5.14).
|
||||
fn kill_all(dir: &Path) -> io::Result<()> {
|
||||
std::fs::write(dir.join("cgroup.kill"), "1")
|
||||
}
|
||||
|
||||
/// Move the calling process into the cgroup whose `cgroup.procs` is `procs`.
|
||||
|
|
@ -329,6 +373,25 @@ mod tests {
|
|||
assert!(!dir.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kill_all_writes_one_to_cgroup_kill() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
kill_all(dir.path()).expect("kill");
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(dir.path().join("cgroup.kill")).expect("read"),
|
||||
"1"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_run_cgroup_kills_its_leftovers_only_when_asked() {
|
||||
let (root, _unit) = unit_tree();
|
||||
let cgroups = RunCgroups::init_at(root.path(), OWN, KIND).expect("init");
|
||||
let run = cgroups.create("alpha-0").expect("create");
|
||||
assert!(!run.kill_leftovers);
|
||||
assert!(run.killing_leftovers().kill_leftovers);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_replaces_an_empty_leftover_of_the_same_name() {
|
||||
let (root, unit) = unit_tree();
|
||||
|
|
|
|||
Loading…
Reference in a new issue