subagent, bash: run each subagent run and bash task in its own cgroup
hive-subagent-daemon and hive-bash-daemon are now delegated their cgroup (Delegate=memory pids, DelegateSubgroup=daemon). At start each daemon enables memory+pids below its unit and creates runs/ (subagent) or tasks/ (bash). Each subagent run gets runs/<name>-<n>/ and each bash task tasks/<id>/, both with memory.oom.group=1, and the daemon removes the directory when the run ends. An OOM kill now takes one whole run instead of one process out of the daemon's shared cgroup, and never the daemon itself. Subagent runs get oom_score_adj +1000; bash tasks and both daemons stay at 0; the harness stays at -500. The unit-level OOMScoreAdjust=500 on the subagent daemon goes: it was both the daemon's own score and a floor no run below it could be lowered past. Bash tasks enter their cgroup from the existing pre_exec hook, writing to a cgroup.procs fd opened by the parent. hive-claude builds its Command itself and exposes neither a pre_exec hook nor the child's pid, so a subagent's claude (and an ACP agent's process) is started as the daemon's own binary with the run's cgroup in its environment; main() enters the cgroup, raises oom_score_adj and execs the real program, the trampoline swarm-review-classifier already uses. Both enter before the program executes, so nothing it forks starts outside the run's cgroup. Without a delegated `daemon` leaf (tests, a dev shell) each daemon logs one warning and runs everything in its own cgroup, as before. The shared helper is hive-agent-sock's run_cgroup module, the crate both daemons already use for what in-container plugins share. Closes #5007
This commit is contained in:
parent
baf282131f
commit
b31db6c7d9
14 changed files with 600 additions and 56 deletions
|
|
@ -39,9 +39,11 @@ async fn main() -> Result<()> {
|
|||
"hive-bash-daemon starting"
|
||||
);
|
||||
|
||||
let cgroups = hive_agent_sock::run_cgroup::RunCgroups::init("tasks");
|
||||
|
||||
// Start the background runner loop — scans for pending tasks and
|
||||
// spawns them, pushing todos to the harness on task transitions.
|
||||
hive_bash_mcp::runner::spawn_loop(todo_socket);
|
||||
hive_bash_mcp::runner::spawn_loop(todo_socket, cgroups);
|
||||
|
||||
// Serve the MCP tools over streamable-http forever.
|
||||
hive_bash_mcp::mcp::serve_http(cli.http).await
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
|||
|
||||
use anyhow::{Result, bail};
|
||||
use hive_agent_sock::Request as TodoReq;
|
||||
use hive_agent_sock::run_cgroup::{self, RunCgroup, RunCgroups};
|
||||
use tokio::io::AsyncWriteExt as _;
|
||||
use tokio::sync::Notify;
|
||||
|
||||
|
|
@ -431,13 +432,14 @@ pub fn kill_task(id: &str, force: bool) -> (bool, bool) {
|
|||
/// Spawn the background runner loop as a detached tokio task. `socket` is
|
||||
/// the harness's in-agent todo socket (`HIVE_AGENT_SOCKET`), where task
|
||||
/// transitions are pushed as todos. Call once at daemon startup.
|
||||
pub fn spawn_loop(socket: PathBuf) {
|
||||
pub fn spawn_loop(socket: PathBuf, cgroups: Option<RunCgroups>) {
|
||||
let cgroups = Arc::new(cgroups);
|
||||
tokio::spawn(async move {
|
||||
run_loop(socket).await;
|
||||
run_loop(socket, cgroups).await;
|
||||
});
|
||||
}
|
||||
|
||||
async fn run_loop(socket: PathBuf) {
|
||||
async fn run_loop(socket: PathBuf, cgroups: Arc<Option<RunCgroups>>) {
|
||||
if let Err(e) = std::fs::create_dir_all(paths::tasks_dir()) {
|
||||
tracing::warn!(error = ?e, "bash_runner: create tasks dir failed");
|
||||
}
|
||||
|
|
@ -450,7 +452,7 @@ async fn run_loop(socket: PathBuf) {
|
|||
let claimed: Arc<Mutex<HashSet<String>>> = Arc::new(Mutex::new(HashSet::new()));
|
||||
|
||||
loop {
|
||||
poll_once(&socket, &claimed);
|
||||
poll_once(&socket, &claimed, &cgroups);
|
||||
tokio::time::sleep(POLL_INTERVAL).await;
|
||||
}
|
||||
}
|
||||
|
|
@ -541,7 +543,11 @@ async fn mark_interrupted(socket: &Path) {
|
|||
}
|
||||
}
|
||||
|
||||
fn poll_once(socket: &Path, claimed: &Arc<Mutex<HashSet<String>>>) {
|
||||
fn poll_once(
|
||||
socket: &Path,
|
||||
claimed: &Arc<Mutex<HashSet<String>>>,
|
||||
cgroups: &Arc<Option<RunCgroups>>,
|
||||
) {
|
||||
let Ok(rd) = std::fs::read_dir(paths::tasks_dir()) else {
|
||||
return;
|
||||
};
|
||||
|
|
@ -567,8 +573,9 @@ fn poll_once(socket: &Path, claimed: &Arc<Mutex<HashSet<String>>>) {
|
|||
claimed.lock().unwrap().insert(id.clone());
|
||||
let socket = socket.to_path_buf();
|
||||
let claimed = claimed.clone();
|
||||
let cgroups = Arc::clone(cgroups);
|
||||
tokio::spawn(async move {
|
||||
run_task(task, &socket).await;
|
||||
run_task(task, &socket, cgroups.as_ref().as_ref()).await;
|
||||
claimed.lock().unwrap().remove(&id);
|
||||
});
|
||||
}
|
||||
|
|
@ -578,7 +585,7 @@ fn poll_once(socket: &Path, claimed: &Arc<Mutex<HashSet<String>>>) {
|
|||
// Task execution
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async fn run_task(mut task: TaskFile, socket: &Path) {
|
||||
async fn run_task(mut task: TaskFile, socket: &Path, cgroups: Option<&RunCgroups>) {
|
||||
let id = task.id.clone();
|
||||
tracing::info!(id = %id, cmd = %task.cmd, "bash_runner: starting task");
|
||||
|
||||
|
|
@ -616,6 +623,7 @@ async fn run_task(mut task: TaskFile, socket: &Path) {
|
|||
},
|
||||
);
|
||||
|
||||
let cgroup = cgroups.and_then(|cgroups| task_cgroup(cgroups, &id));
|
||||
let outcome = exec_cmd(
|
||||
&task.cmd,
|
||||
&out_path,
|
||||
|
|
@ -623,8 +631,10 @@ async fn run_task(mut task: TaskFile, socket: &Path) {
|
|||
task.timeout_secs,
|
||||
&cancel,
|
||||
&force,
|
||||
cgroup.as_ref().map(|(_, procs)| procs),
|
||||
)
|
||||
.await;
|
||||
drop(cgroup);
|
||||
|
||||
// Deregister before writing terminal state — the task is no longer killable.
|
||||
running().lock().unwrap().remove(&id);
|
||||
|
|
@ -693,6 +703,20 @@ async fn run_task(mut task: TaskFile, socket: &Path) {
|
|||
.await;
|
||||
}
|
||||
|
||||
/// `id`'s cgroup and its open `cgroup.procs`, or `None` (logged) when either
|
||||
/// fails, in which case the task runs in the daemon's cgroup.
|
||||
fn task_cgroup(cgroups: &RunCgroups, id: &str) -> Option<(RunCgroup, std::fs::File)> {
|
||||
let opened = cgroups.create(id).and_then(|cgroup| {
|
||||
let procs = cgroup.procs()?;
|
||||
Ok((cgroup, procs))
|
||||
});
|
||||
opened
|
||||
.inspect_err(|e| {
|
||||
tracing::warn!(id = %id, error = %e, "bash_runner: task cgroup unavailable");
|
||||
})
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// Run `bash -c cmd` in its own process group, streaming output to files.
|
||||
/// `timeout_secs = None` means no timeout — run until natural exit.
|
||||
///
|
||||
|
|
@ -721,6 +745,7 @@ async fn exec_cmd(
|
|||
timeout_secs: Option<u64>,
|
||||
cancel: &Notify,
|
||||
force: &AtomicBool,
|
||||
cgroup_procs: Option<&std::fs::File>,
|
||||
) -> Result<ExecOutcome> {
|
||||
use tokio::process::Command;
|
||||
// `/usr/bin/env bash` rather than a bare `bash`: `/usr/bin/env` is at a
|
||||
|
|
@ -734,14 +759,20 @@ async fn exec_cmd(
|
|||
.arg(format!("set -o pipefail; {cmd}"))
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped());
|
||||
// SAFETY: `setpgid`/`nice` are async-signal-safe and only touch the
|
||||
// calling (child) process before exec — no allocations, no locks.
|
||||
let cgroup_procs = cgroup_procs.map(std::os::fd::AsRawFd::as_raw_fd);
|
||||
// SAFETY: `setpgid`/`nice`/`write` are async-signal-safe and only touch
|
||||
// the calling (child) process before exec — no allocations, no locks.
|
||||
// `setpgid(0, 0)` makes the child its own process-group leader so the
|
||||
// whole subtree can be signalled as a group (`kill(-pgid, …)`).
|
||||
// Entering the task's cgroup before exec means nothing the task forks
|
||||
// starts outside it.
|
||||
unsafe {
|
||||
cmd_builder.pre_exec(|| {
|
||||
cmd_builder.pre_exec(move || {
|
||||
libc::setpgid(0, 0);
|
||||
libc::nice(10);
|
||||
if let Some(procs) = cgroup_procs {
|
||||
run_cgroup::enter_in_pre_exec(procs)?;
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue