hive-agent-sock: kill what a subagent run leaves in its cgroup after it ends
This commit is contained in:
parent
566758483d
commit
2689efebcf
4 changed files with 82 additions and 11 deletions
|
|
@ -170,7 +170,10 @@ On `claude`, the signal is always SIGINT, and it goes to claude's whole
|
||||||
process group. The signal doesn't reach a command that moved itself out of
|
process group. The signal doesn't reach a command that moved itself out of
|
||||||
the group (`setsid`), and claude's Bash tool starts every command that way:
|
the group (`setsid`), and claude's Bash tool starts every command that way:
|
||||||
claude stops those commands itself when it shuts down on SIGINT. A SIGKILL
|
claude stops those commands itself when it shuts down on SIGINT. A SIGKILL
|
||||||
would leave them running, so nothing escalates to one. `interrupt` returns
|
would leave them running, so nothing escalates to one while claude is
|
||||||
|
alive. Once the run has ended, whatever is still in its cgroup is
|
||||||
|
SIGKILLed after 30 seconds, so a command that left claude's process group
|
||||||
|
cannot outlive the run. `interrupt` returns
|
||||||
at once, and the name stays claimed until claude has exited: `status` says
|
at once, and the name stays claimed until claude has exited: `status` says
|
||||||
**stopping**, and the daemon refuses `start` and `continue` on that name
|
**stopping**, and the daemon refuses `start` and `continue` on that name
|
||||||
until then. A claude that ignores the SIGINT stays stopping until someone kills
|
until then. A claude that ignores the SIGINT stays stopping until someone kills
|
||||||
|
|
|
||||||
|
|
@ -44,6 +44,8 @@ a crate of its own.
|
||||||
`hive-bash-mcp` puts each bash task in `<unit>/tasks/<id>/` and
|
`hive-bash-mcp` puts each bash task in `<unit>/tasks/<id>/` and
|
||||||
`hive-subagent-mcp` each subagent run in `<unit>/runs/<name>-<n>/`, both with
|
`hive-subagent-mcp` each subagent run in `<unit>/runs/<name>-<n>/`, both with
|
||||||
`memory.oom.group=1`, so the OOM killer takes one whole run and leaves the
|
`memory.oom.group=1`, so the OOM killer takes one whole run and leaves the
|
||||||
daemon alone. Both daemons are delegated their cgroup with `Delegate=memory
|
daemon alone. A subagent run's cgroup is also killed (`cgroup.kill`) if
|
||||||
|
processes are still in it 30 seconds after the run ended; a bash task's is
|
||||||
|
left alone. Both daemons are delegated their cgroup with `Delegate=memory
|
||||||
pids` and `DelegateSubgroup=daemon` (`nix/agent-modules/mcp.nix`); outside such
|
pids` and `DelegateSubgroup=daemon` (`nix/agent-modules/mcp.nix`); outside such
|
||||||
a unit the module logs one warning and runs stay in the daemon's cgroup.
|
a unit the module logs one warning and runs stay in the daemon's cgroup.
|
||||||
|
|
|
||||||
|
|
@ -45,6 +45,8 @@ const ENTRY_OOM_SCORE_ADJ_ENV: &str = "HIVE_RUN_ENTRY_OOM_SCORE_ADJ";
|
||||||
/// removal is left to the next daemon start.
|
/// removal is left to the next daemon start.
|
||||||
const RELEASE_GRACE: Duration = Duration::from_secs(30);
|
const RELEASE_GRACE: Duration = Duration::from_secs(30);
|
||||||
const RELEASE_POLL: Duration = Duration::from_millis(250);
|
const RELEASE_POLL: Duration = Duration::from_millis(250);
|
||||||
|
/// How long a killed run cgroup gets to empty before it is left in place.
|
||||||
|
const KILL_GRACE: Duration = Duration::from_secs(5);
|
||||||
|
|
||||||
/// The `<unit>/<kind>/` node run cgroups are created under.
|
/// The `<unit>/<kind>/` node run cgroups are created under.
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
|
|
@ -126,20 +128,36 @@ impl RunCgroups {
|
||||||
}
|
}
|
||||||
other => other?,
|
other => other?,
|
||||||
}
|
}
|
||||||
let cgroup = RunCgroup { dir };
|
let cgroup = RunCgroup {
|
||||||
|
dir,
|
||||||
|
kill_leftovers: false,
|
||||||
|
};
|
||||||
std::fs::write(cgroup.dir.join("memory.oom.group"), "1")?;
|
std::fs::write(cgroup.dir.join("memory.oom.group"), "1")?;
|
||||||
Ok(cgroup)
|
Ok(cgroup)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// One run's cgroup. Dropping it removes the directory once the run's last
|
/// One run's cgroup. Dropping it removes the directory once the run's last
|
||||||
/// process has left, waiting up to 30 seconds on a background thread.
|
/// process has left, waiting up to 30 seconds on a background thread. With
|
||||||
|
/// [`RunCgroup::killing_leftovers`], processes still in it after that wait
|
||||||
|
/// are killed.
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct RunCgroup {
|
pub struct RunCgroup {
|
||||||
dir: PathBuf,
|
dir: PathBuf,
|
||||||
|
kill_leftovers: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RunCgroup {
|
impl RunCgroup {
|
||||||
|
/// Kill whatever is still in the cgroup once the run has ended and the
|
||||||
|
/// release wait has run out. For a run whose caller has nothing left
|
||||||
|
/// that could stop a process that left the run's process group
|
||||||
|
/// (`setsid`, `nohup ... &`): the cgroup is the only handle on it.
|
||||||
|
#[must_use]
|
||||||
|
pub fn killing_leftovers(mut self) -> Self {
|
||||||
|
self.kill_leftovers = true;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
/// The run's `cgroup.procs`, open for [`enter_in_pre_exec`]. Keep it
|
/// The run's `cgroup.procs`, open for [`enter_in_pre_exec`]. Keep it
|
||||||
/// open until the spawn has returned.
|
/// open until the spawn has returned.
|
||||||
///
|
///
|
||||||
|
|
@ -186,7 +204,8 @@ impl Drop for RunCgroup {
|
||||||
match std::fs::remove_dir(&self.dir) {
|
match std::fs::remove_dir(&self.dir) {
|
||||||
Err(e) if e.raw_os_error() == Some(libc::EBUSY) => {
|
Err(e) if e.raw_os_error() == Some(libc::EBUSY) => {
|
||||||
let dir = std::mem::take(&mut self.dir);
|
let dir = std::mem::take(&mut self.dir);
|
||||||
std::thread::spawn(move || release_when_empty(&dir));
|
let kill = self.kill_leftovers;
|
||||||
|
std::thread::spawn(move || release_when_empty(&dir, kill));
|
||||||
}
|
}
|
||||||
Err(e) if e.kind() != io::ErrorKind::NotFound => {
|
Err(e) if e.kind() != io::ErrorKind::NotFound => {
|
||||||
tracing::warn!(dir = %self.dir.display(), error = %e, "removing run cgroup failed");
|
tracing::warn!(dir = %self.dir.display(), error = %e, "removing run cgroup failed");
|
||||||
|
|
@ -197,17 +216,42 @@ impl Drop for RunCgroup {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Retry removing a still-populated run cgroup until it empties or
|
/// Retry removing a still-populated run cgroup until it empties or
|
||||||
/// [`RELEASE_GRACE`] runs out; past that the next daemon start removes it.
|
/// [`RELEASE_GRACE`] runs out. Past that, `kill` SIGKILLs what is left and
|
||||||
fn release_when_empty(dir: &Path) {
|
/// waits once more; otherwise (or if that fails) the next daemon start
|
||||||
let deadline = std::time::Instant::now() + RELEASE_GRACE;
|
/// removes it.
|
||||||
|
fn release_when_empty(dir: &Path, kill: bool) {
|
||||||
|
if wait_until_removed(dir, RELEASE_GRACE) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if kill {
|
||||||
|
tracing::warn!(dir = %dir.display(), "run cgroup still populated after the run ended; killing it");
|
||||||
|
if let Err(e) = kill_all(dir) {
|
||||||
|
tracing::warn!(dir = %dir.display(), error = %e, "killing run cgroup failed");
|
||||||
|
} else if wait_until_removed(dir, KILL_GRACE) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tracing::info!(dir = %dir.display(), "run cgroup still populated after the run ended; left in place");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `true` once `dir` is gone (or cannot be removed for a reason other than
|
||||||
|
/// processes still in it), `false` when `grace` ran out first.
|
||||||
|
fn wait_until_removed(dir: &Path, grace: Duration) -> bool {
|
||||||
|
let deadline = std::time::Instant::now() + grace;
|
||||||
while std::time::Instant::now() < deadline {
|
while std::time::Instant::now() < deadline {
|
||||||
std::thread::sleep(RELEASE_POLL);
|
std::thread::sleep(RELEASE_POLL);
|
||||||
match std::fs::remove_dir(dir) {
|
match std::fs::remove_dir(dir) {
|
||||||
Err(e) if e.raw_os_error() == Some(libc::EBUSY) => {}
|
Err(e) if e.raw_os_error() == Some(libc::EBUSY) => {}
|
||||||
_ => return,
|
_ => return true,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
tracing::info!(dir = %dir.display(), "run cgroup still populated after the run ended; left in place");
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
/// SIGKILL every process in the cgroup at `dir`, including any forked while
|
||||||
|
/// the kill runs (`cgroup.kill`, Linux 5.14).
|
||||||
|
fn kill_all(dir: &Path) -> io::Result<()> {
|
||||||
|
std::fs::write(dir.join("cgroup.kill"), "1")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Move the calling process into the cgroup whose `cgroup.procs` is `procs`.
|
/// Move the calling process into the cgroup whose `cgroup.procs` is `procs`.
|
||||||
|
|
@ -329,6 +373,25 @@ mod tests {
|
||||||
assert!(!dir.exists());
|
assert!(!dir.exists());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn kill_all_writes_one_to_cgroup_kill() {
|
||||||
|
let dir = tempfile::tempdir().expect("tempdir");
|
||||||
|
kill_all(dir.path()).expect("kill");
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(dir.path().join("cgroup.kill")).expect("read"),
|
||||||
|
"1"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_run_cgroup_kills_its_leftovers_only_when_asked() {
|
||||||
|
let (root, _unit) = unit_tree();
|
||||||
|
let cgroups = RunCgroups::init_at(root.path(), OWN, KIND).expect("init");
|
||||||
|
let run = cgroups.create("alpha-0").expect("create");
|
||||||
|
assert!(!run.kill_leftovers);
|
||||||
|
assert!(run.killing_leftovers().kill_leftovers);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn create_replaces_an_empty_leftover_of_the_same_name() {
|
fn create_replaces_an_empty_leftover_of_the_same_name() {
|
||||||
let (root, unit) = unit_tree();
|
let (root, unit) = unit_tree();
|
||||||
|
|
|
||||||
|
|
@ -515,7 +515,9 @@ impl State {
|
||||||
|
|
||||||
/// A new cgroup for one run of `name`; `None` without run cgroups, or
|
/// A new cgroup for one run of `name`; `None` without run cgroups, or
|
||||||
/// (logged) when creating it failed, and the run then stays in the
|
/// (logged) when creating it failed, and the run then stays in the
|
||||||
/// daemon's cgroup.
|
/// daemon's cgroup. Whatever a run left running in it (a tool child that
|
||||||
|
/// left claude's process group, which `interrupt`'s SIGINT never
|
||||||
|
/// reaches) is killed once the run has ended.
|
||||||
fn run_cgroup(&self, name: &str) -> Option<RunCgroup> {
|
fn run_cgroup(&self, name: &str) -> Option<RunCgroup> {
|
||||||
let cgroups = self.run_cgroups.as_ref()?;
|
let cgroups = self.run_cgroups.as_ref()?;
|
||||||
let n = self.run_seq.fetch_add(1, Ordering::Relaxed);
|
let n = self.run_seq.fetch_add(1, Ordering::Relaxed);
|
||||||
|
|
@ -523,6 +525,7 @@ impl State {
|
||||||
.create(&format!("{name}-{n}"))
|
.create(&format!("{name}-{n}"))
|
||||||
.inspect_err(|e| tracing::warn!(name, error = %e, "subagent: run cgroup unavailable"))
|
.inspect_err(|e| tracing::warn!(name, error = %e, "subagent: run cgroup unavailable"))
|
||||||
.ok()
|
.ok()
|
||||||
|
.map(RunCgroup::killing_leftovers)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `config` with its program started through the entry wrapper, at
|
/// `config` with its program started through the entry wrapper, at
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue