subagent, bash: run each subagent run and bash task in its own cgroup
hive-subagent-daemon and hive-bash-daemon are now delegated their cgroup (Delegate=memory pids, DelegateSubgroup=daemon). At start each daemon enables memory+pids below its unit and creates runs/ (subagent) or tasks/ (bash). Each subagent run gets runs/<name>-<n>/ and each bash task tasks/<id>/, both with memory.oom.group=1, and the daemon removes the directory when the run ends. An OOM kill now takes one whole run instead of one process out of the daemon's shared cgroup, and never the daemon itself. Subagent runs get oom_score_adj +1000; bash tasks and both daemons stay at 0; the harness stays at -500. The unit-level OOMScoreAdjust=500 on the subagent daemon goes: it was both the daemon's own score and a floor no run below it could be lowered past. Bash tasks enter their cgroup from the existing pre_exec hook, writing to a cgroup.procs fd opened by the parent. hive-claude builds its Command itself and exposes neither a pre_exec hook nor the child's pid, so a subagent's claude (and an ACP agent's process) is started as the daemon's own binary with the run's cgroup in its environment; main() enters the cgroup, raises oom_score_adj and execs the real program, the trampoline swarm-review-classifier already uses. Both enter before the program executes, so nothing it forks starts outside the run's cgroup. Without a delegated `daemon` leaf (tests, a dev shell) each daemon logs one warning and runs everything in its own cgroup, as before. The shared helper is hive-agent-sock's run_cgroup module, the crate both daemons already use for what in-container plugins share. Closes #5007
This commit is contained in:
parent
baf282131f
commit
b31db6c7d9
14 changed files with 600 additions and 56 deletions
|
|
@ -22,8 +22,15 @@ struct Cli {
|
|||
http: std::net::SocketAddr,
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
// Each subagent run starts as this binary, which enters the run's cgroup
|
||||
// and execs the run's program (`RunCgroup::entry`).
|
||||
hive_agent_sock::run_cgroup::exec_entry();
|
||||
serve()
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
async fn serve() -> Result<()> {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_env("RUST_LOG")
|
||||
|
|
@ -57,7 +64,8 @@ async fn main() -> Result<()> {
|
|||
hive_subagent_mcp::session::State::new(todo_socket, signal_base)
|
||||
.on_runtime(runtime)
|
||||
.publishing_to(hive_subagent_mcp::swarm_term::Publisher::spawn())
|
||||
.reading_models_from(hive_subagent_mcp::swarm_models::Reader::spawn()),
|
||||
.reading_models_from(hive_subagent_mcp::swarm_models::Reader::spawn())
|
||||
.in_run_cgroups(hive_agent_sock::run_cgroup::RunCgroups::init("runs")),
|
||||
);
|
||||
|
||||
// Serve the MCP tools over streamable-http forever. No background poll
|
||||
|
|
|
|||
|
|
@ -89,9 +89,11 @@
|
|||
use std::collections::HashMap;
|
||||
use std::os::unix::process::ExitStatusExt as _;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use hive_agent_sock::run_cgroup::{RunCgroup, RunCgroups};
|
||||
use hive_claude::{Attach, Claude, Config, SessionStore};
|
||||
use hive_runtime::{
|
||||
AcpCommand, AcpRuntime, PermissionAsk, PermissionPolicy, Runtime as _, RuntimeSpec, TurnStop,
|
||||
|
|
@ -405,6 +407,11 @@ pub struct State {
|
|||
/// The agent's own model list, checked when the swarm sets none:
|
||||
/// [`available_models`] outside tests.
|
||||
local_models: fn() -> Option<String>,
|
||||
/// Where each run gets its own cgroup, when this daemon was delegated
|
||||
/// one ([`State::in_run_cgroups`]).
|
||||
run_cgroups: Option<RunCgroups>,
|
||||
/// The `<n>` of the next run cgroup `<name>-<n>`.
|
||||
run_seq: AtomicU64,
|
||||
}
|
||||
|
||||
/// Which opaque URL segment belongs to which session — the whole of a
|
||||
|
|
@ -460,6 +467,8 @@ impl State {
|
|||
term: None,
|
||||
models: None,
|
||||
local_models: available_models,
|
||||
run_cgroups: None,
|
||||
run_seq: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -478,6 +487,26 @@ impl State {
|
|||
self
|
||||
}
|
||||
|
||||
/// Run each subagent run in its own cgroup under `cgroups`, at
|
||||
/// `oom_score_adj` +1000.
|
||||
#[must_use]
|
||||
pub fn in_run_cgroups(mut self, cgroups: Option<RunCgroups>) -> Self {
|
||||
self.run_cgroups = cgroups;
|
||||
self
|
||||
}
|
||||
|
||||
/// A new cgroup for one run of `name`; `None` without run cgroups, or
|
||||
/// (logged) when creating it failed, and the run then stays in the
|
||||
/// daemon's cgroup.
|
||||
fn run_cgroup(&self, name: &str) -> Option<RunCgroup> {
|
||||
let cgroups = self.run_cgroups.as_ref()?;
|
||||
let n = self.run_seq.fetch_add(1, Ordering::Relaxed);
|
||||
cgroups
|
||||
.create(&format!("{name}-{n}"))
|
||||
.inspect_err(|e| tracing::warn!(name, error = %e, "subagent: run cgroup unavailable"))
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// Refuse a `model` outside the list in effect for this agent (see
|
||||
/// `check_model`). The swarm key is read only for a named model.
|
||||
async fn allow_model(&self, model: Option<&str>) -> anyhow::Result<()> {
|
||||
|
|
@ -1394,7 +1423,8 @@ fn start_on_runtime(
|
|||
trigger: String,
|
||||
) -> anyhow::Result<String> {
|
||||
if let SubagentRuntime::Acp { command, sessions } = &state.runtime {
|
||||
let runtime = acp_runtime(command.clone(), sessions, name);
|
||||
let cgroup = state.run_cgroup(name);
|
||||
let runtime = acp_runtime(command.clone(), sessions, name, cgroup.as_ref());
|
||||
if let Some(archived) = runtime
|
||||
.archive()
|
||||
.map_err(|e| anyhow::anyhow!("archiving the prior `{name}` session failed: {e}"))?
|
||||
|
|
@ -1409,7 +1439,7 @@ fn start_on_runtime(
|
|||
system_prompt_file: system_prompt,
|
||||
..config
|
||||
};
|
||||
return spawn_and_track_acp(state, name, runtime, config, trigger, None);
|
||||
return spawn_and_track_acp(state, name, runtime, cgroup, config, trigger, None);
|
||||
}
|
||||
let store = build_store(&config)?;
|
||||
if store.find_by_title(name).is_some() {
|
||||
|
|
@ -1608,11 +1638,13 @@ fn continue_on_runtime(
|
|||
);
|
||||
}
|
||||
state.begin_continue(name, dir);
|
||||
let runtime = acp_runtime(command.clone(), sessions, name);
|
||||
let cgroup = state.run_cgroup(name);
|
||||
let runtime = acp_runtime(command.clone(), sessions, name, cgroup.as_ref());
|
||||
return spawn_and_track_acp(
|
||||
state,
|
||||
name,
|
||||
runtime,
|
||||
cgroup,
|
||||
config,
|
||||
prompt,
|
||||
Some(Arc::clone(verdict)),
|
||||
|
|
@ -1736,6 +1768,10 @@ fn spawn_and_track(
|
|||
prompt: String,
|
||||
verdict: Option<VerdictTx>,
|
||||
) -> anyhow::Result<String> {
|
||||
// Every turn of the run, and all they start, share one cgroup, released
|
||||
// when the background task below ends.
|
||||
let cgroup = state.run_cgroup(name);
|
||||
let config = &entered_config(config, cgroup.as_ref());
|
||||
let running = Claude::spawn(config, attach)
|
||||
.map_err(|e| anyhow::anyhow!("starting the subagent process failed: {e}"))?;
|
||||
state.track(name, running.cancel_handle().into());
|
||||
|
|
@ -1761,6 +1797,7 @@ fn spawn_and_track(
|
|||
let state = Arc::clone(state);
|
||||
let task_name = name.to_owned();
|
||||
tokio::spawn(async move {
|
||||
let _cgroup = cgroup;
|
||||
let mut running = running;
|
||||
let mut prompt = prompt;
|
||||
loop {
|
||||
|
|
@ -1853,11 +1890,12 @@ async fn after_turn(
|
|||
/// `runtime`, whose agent process is spawned by the first turn and killed
|
||||
/// when the run ends and the runtime is dropped. Returns before the agent
|
||||
/// exists, so a spawn failure is the first turn's `Failed` end rather than
|
||||
/// this call's error.
|
||||
/// this call's error. The run's `cgroup` is released after the runtime.
|
||||
fn spawn_and_track_acp(
|
||||
state: &Arc<State>,
|
||||
name: &str,
|
||||
runtime: AcpRuntime<hive_claude::NeverCompact>,
|
||||
cgroup: Option<RunCgroup>,
|
||||
config: Config,
|
||||
prompt: String,
|
||||
verdict: Option<VerdictTx>,
|
||||
|
|
@ -1884,12 +1922,14 @@ fn spawn_and_track_acp(
|
|||
Err(e) => TurnEnd::Failed(format!("ACP error: {e}")),
|
||||
};
|
||||
let Some(next) = after_turn(&state, &task_name, end, verdict.as_ref()).await else {
|
||||
return;
|
||||
break;
|
||||
};
|
||||
state.track(&task_name, canceller.clone().into());
|
||||
state.note_event(&task_name);
|
||||
prompt = next;
|
||||
}
|
||||
drop(runtime);
|
||||
drop(cgroup);
|
||||
});
|
||||
|
||||
Ok(format!("subagent `{name}` started"))
|
||||
|
|
@ -1901,19 +1941,51 @@ fn acp_session_file(sessions: &Path, name: &str) -> PathBuf {
|
|||
sessions.join(format!("subagent-acp-session-{name}"))
|
||||
}
|
||||
|
||||
/// A subagent run's `oom_score_adj`: the maximum, so under memory pressure
|
||||
/// the kernel takes a subagent run before a bash task (0) or the agent's
|
||||
/// own session (-500, `nix/agent-modules/agent-service.nix`).
|
||||
const RUN_OOM_SCORE_ADJ: i32 = 1000;
|
||||
|
||||
/// `config` with its program started inside `cgroup`, if there is one.
|
||||
/// `hive-claude` builds the `Command` itself and offers no `pre_exec`, so
|
||||
/// the program becomes this daemon's binary as a [`RunCgroup::entry`]
|
||||
/// wrapper.
|
||||
fn entered_config(config: &Config, cgroup: Option<&RunCgroup>) -> Config {
|
||||
let mut config = config.clone();
|
||||
if let Some(cgroup) = cgroup {
|
||||
// `hive-claude`'s own default program.
|
||||
let program = config.program.take().unwrap_or_else(|| "claude".into());
|
||||
let (entry, env) = cgroup.entry(&program, RUN_OOM_SCORE_ADJ);
|
||||
config.program = Some(entry);
|
||||
config.env.extend(env);
|
||||
}
|
||||
config
|
||||
}
|
||||
|
||||
/// [`entered_config`] for an ACP agent's command.
|
||||
fn entered_command(mut command: AcpCommand, cgroup: Option<&RunCgroup>) -> AcpCommand {
|
||||
if let Some(cgroup) = cgroup {
|
||||
let (entry, env) = cgroup.entry(Path::new(&command.command), RUN_OOM_SCORE_ADJ);
|
||||
command.command = entry.to_string_lossy().into_owned();
|
||||
command.env.extend(env);
|
||||
}
|
||||
command
|
||||
}
|
||||
|
||||
/// The ACP runtime `name`'s run drives, keeping its session id in
|
||||
/// [`acp_session_file`] and answering permission requests per
|
||||
/// [`acp_permits`].
|
||||
/// [`acp_permits`]. Its agent process starts inside `cgroup`.
|
||||
fn acp_runtime(
|
||||
command: AcpCommand,
|
||||
sessions: &Path,
|
||||
name: &str,
|
||||
cgroup: Option<&RunCgroup>,
|
||||
) -> AcpRuntime<hive_claude::NeverCompact> {
|
||||
let groups = hive_sh4re::permissions::effective_tool_groups();
|
||||
let permit: PermissionPolicy =
|
||||
Arc::new(move |ask: &PermissionAsk<'_>| acp_permits(ask, &groups));
|
||||
AcpRuntime::new(
|
||||
command,
|
||||
entered_command(command, cgroup),
|
||||
acp_session_file(sessions, name),
|
||||
permit,
|
||||
hive_claude::NeverCompact,
|
||||
|
|
|
|||
28
hive-subagent-mcp/tests/run_entry.rs
Normal file
28
hive-subagent-mcp/tests/run_entry.rs
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
//! The daemon binary as a run's entry wrapper (`run_cgroup::RunCgroup::entry`
|
||||
//! sets these variables; its unit test pins the names).
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
#[test]
|
||||
fn the_entry_enters_the_cgroup_raises_the_score_and_execs_the_program() {
|
||||
let cgroup = std::env::temp_dir().join(format!("hive-run-entry-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&cgroup).expect("fake cgroup");
|
||||
let out = Command::new(env!("CARGO_BIN_EXE_hive-subagent-daemon"))
|
||||
// Stands in for `claude`: prints what it was started with.
|
||||
.env("HIVE_RUN_ENTRY_PROGRAM", "/bin/sh")
|
||||
.env("HIVE_RUN_ENTRY_CGROUP", &cgroup)
|
||||
.env("HIVE_RUN_ENTRY_OOM_SCORE_ADJ", "1000")
|
||||
.args([
|
||||
"-c",
|
||||
r#"printf '%s|%s|%s' "$(cat /proc/self/oom_score_adj)" "${HIVE_RUN_ENTRY_CGROUP-unset}" "$1""#,
|
||||
"sh",
|
||||
"an-arg",
|
||||
])
|
||||
.output()
|
||||
.expect("the binary runs");
|
||||
let procs = std::fs::read_to_string(cgroup.join("cgroup.procs"));
|
||||
std::fs::remove_dir_all(&cgroup).ok();
|
||||
assert!(out.status.success(), "{out:?}");
|
||||
assert_eq!(String::from_utf8_lossy(&out.stdout), "1000|unset|an-arg");
|
||||
assert_eq!(procs.expect("cgroup.procs written"), "0");
|
||||
}
|
||||
Loading…
Reference in a new issue