//! Bash subprocess runner: spawns `bash -c ` tasks, writes status //! files under `harness/bash-tasks/`, and fires hyperhive wake signals //! on completion. //! //! Files under `tasks_dir()`: //! - `.json` — task metadata + status (pending → running → done) //! - `.out` — captured stdout (streamed while running) //! - `.err` — captured stderr (streamed while running) //! //! Tasks with status `running` on daemon boot are marked `interrupted` //! (the process died with the previous daemon). A best-effort wake is //! still sent so the agent is not silently blocked. //! //! The runner kills the child process on timeout — `tokio::process::Child::drop()` //! does not kill children, so we explicitly call `child.kill().await`. use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use anyhow::{Result, bail}; use tokio::io::AsyncWriteExt as _; use tokio::sync::Notify; use crate::paths; use crate::protocol::{TaskFile, TaskStatus}; /// Poll interval for the runner background loop. const POLL_INTERVAL: Duration = Duration::from_millis(200); /// Soft cap on stdout/stderr captured in the done JSON summary. /// Full output always lives in the `.out`/`.err` files. pub const SUMMARY_BYTES: usize = 4096; /// Maximum inline wait (cap on `wait_seconds`). pub const MAX_WAIT_SECS: u64 = 30; /// Poll interval used by the inline-wait loops. const POLL_MS: u64 = 100; static TASK_SEQ: AtomicU64 = AtomicU64::new(0); // --------------------------------------------------------------------------- // Running-task registry (for on-demand kill) // --------------------------------------------------------------------------- /// Handle to a currently-executing task, used by [`kill_task`] to signal it. /// `exec_cmd` selects on `cancel`; when notified it signals the task's /// process group with `SIGKILL` (if `force`) or `SIGINT`. struct RunningHandle { cancel: Arc, force: Arc, } /// Global registry of running tasks (id → handle). Populated by `run_task` /// for the duration of execution and read by [`kill_task`] from the socket /// dispatch path — a separate async context from the runner loop, so a /// shared global (rather than the loop-local `claimed` set) is needed. fn running() -> &'static Mutex> { static RUNNING: OnceLock>> = OnceLock::new(); RUNNING.get_or_init(|| Mutex::new(HashMap::new())) } /// Outcome of one `exec_cmd` run. enum ExecOutcome { /// Process exited on its own with this status code. Exited(i32), /// Killed by the per-task timeout (`timeout_secs`). TimedOut, /// Killed on request via [`kill_task`]. `forced` = SIGKILL vs SIGINT. Killed { forced: bool }, } /// Send `sig` to the task's process group. A negative pgid targets the whole /// group, so `bash -c` plus any children it spawned all receive the signal. /// No-op if the child had no pid (already reaped). fn signal_group(pgid: Option, sig: i32) { if let Some(pgid) = pgid { // SAFETY: plain libc `kill(2)`; `-pgid` targets the child's own // process group (it called `setpgid(0, 0)` so it leads its group). unsafe { libc::kill(-pgid, sig); } } } // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- use hive_sh4re::wire_time::now_unix; /// Generate a task ID: ``. #[must_use] pub fn new_task_id() -> String { let t = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_default() .as_millis(); let seq = TASK_SEQ.fetch_add(1, Ordering::Relaxed); format!("{t:013x}{seq:04x}") } /// Write a task file atomically (tmp + rename). fn write_task(task: &TaskFile) -> std::io::Result<()> { let json = serde_json::to_string_pretty(task) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; let dest = paths::task_json(&task.id); let tmp = dest.with_extension("json.tmp"); std::fs::write(&tmp, &json)?; std::fs::rename(&tmp, &dest) } /// Read a task file. Returns `None` if the file doesn't exist or is /// unparseable. #[must_use] pub fn read_task(id: &str) -> Option { let s = std::fs::read_to_string(paths::task_json(id)).ok()?; serde_json::from_str(&s).ok() } // --------------------------------------------------------------------------- // Loose-ends file (generic MCP loose-ends protocol) // --------------------------------------------------------------------------- /// Rewrite `mcp-loose-ends/bash.json` with a summary of all currently /// active (Pending or Running) tasks. The harness scans this directory /// generically in `get_loose_ends` — no bash-specific code needed there. /// /// File format: a JSON array of plain-text summary strings, one per /// loose-end item. The harness includes them verbatim in the output. /// Atomic write (tmp + rename) so the harness never reads a partial file. fn refresh_loose_ends() { let active = active_tasks(); let dir = paths::mcp_loose_ends_dir(); if let Err(e) = std::fs::create_dir_all(&dir) { tracing::warn!(error = ?e, "bash_runner: create mcp-loose-ends dir failed"); return; } let items: Vec = active .iter() .map(|t| { let age = now_unix() - t.created_at; format!( "bash task `{}` status={:?}, cmd: `{}`, age {}s", t.id, t.status, t.cmd, age ) }) .collect(); let dest = dir.join("bash.json"); let tmp = dest.with_extension("json.tmp"); let json = serde_json::to_string(&items).unwrap_or_else(|_| "[]".to_owned()); if let Err(e) = std::fs::write(&tmp, &json).and_then(|()| std::fs::rename(&tmp, &dest)) { tracing::warn!(error = ?e, "bash_runner: write mcp-loose-ends/bash.json failed"); } } // --------------------------------------------------------------------------- // Public API used by daemon dispatch // --------------------------------------------------------------------------- /// Longest accepted caller-chosen task name. const MAX_TASK_NAME_LEN: usize = 64; /// Validate a caller-chosen task name. The name doubles as the task id /// and therefore as the `.json` filename, so it must be a single /// safe path segment. Allows ASCII alphanumerics plus `.`, `_`, `-`; /// rejects empties, over-long names, `.`/`..`, and anything that could /// escape the tasks dir or collide with the `.json.tmp` scratch suffix. fn validate_task_name(name: &str) -> Result<()> { if name.is_empty() { bail!("task name must not be empty"); } if name.len() > MAX_TASK_NAME_LEN { bail!("task name too long (max {MAX_TASK_NAME_LEN} chars)"); } if name == "." || name == ".." { bail!("task name {name:?} is reserved"); } if !name .chars() .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')) { bail!("task name {name:?} may only contain ASCII letters, digits, '.', '_', '-'"); } Ok(()) } /// Submit a new pending task. Returns the task ID. /// /// When `name` is `Some`, it is validated and used as the task id (so it /// surfaces in the wake `from`, status lookups, and the loose-ends list). /// A name may be reused once any prior task of that name has finished; /// submitting a name whose task is still `Pending`/`Running` is rejected. /// `None` falls back to the auto-generated timestamp id. /// /// # Errors /// /// Returns an error if the name is invalid, a task of that name is still /// running, the tasks directory cannot be created, or the task file /// cannot be written. pub fn submit_task(cmd: String, timeout_secs: Option, name: Option) -> Result { std::fs::create_dir_all(paths::tasks_dir())?; let id = match name { Some(name) => { validate_task_name(&name)?; if let Some(existing) = read_task(&name) && matches!(existing.status, TaskStatus::Pending | TaskStatus::Running) { bail!( "a bash task named `{name}` is already running — wait for it to finish or pick another name" ); } name } None => new_task_id(), }; let task = TaskFile { id: id.clone(), cmd, timeout_secs, status: TaskStatus::Pending, created_at: now_unix(), started_at: None, completed_at: None, exit_code: None, stdout_tail: None, stderr_tail: None, }; write_task(&task)?; refresh_loose_ends(); Ok(id) } /// Return all tasks currently in `Pending` or `Running` state. #[must_use] pub fn active_tasks() -> Vec { let Ok(rd) = std::fs::read_dir(paths::tasks_dir()) else { return Vec::new(); }; let mut out = Vec::new(); for entry in rd.flatten() { let path = entry.path(); if path.extension().and_then(|e| e.to_str()) != Some("json") { continue; } let Some(id) = path.file_stem().and_then(|s| s.to_str()).map(str::to_owned) else { continue; }; let Some(task) = read_task(&id) else { continue }; if matches!(task.status, TaskStatus::Pending | TaskStatus::Running) { out.push(task); } } out } /// Inline wait: poll `read_task(id)` until terminal state or deadline. /// Returns the final task on success, or `None` if it never completed. pub async fn wait_for_task(id: &str, wait_secs: u64) -> Option { let cap = wait_secs.min(MAX_WAIT_SECS); if cap == 0 { return read_task(id); } let deadline = tokio::time::Instant::now() + Duration::from_secs(cap); loop { match read_task(id) { None => break, Some(task) => { if matches!( task.status, TaskStatus::Done | TaskStatus::TimedOut | TaskStatus::Interrupted | TaskStatus::Killed ) { return Some(task); } } } if tokio::time::Instant::now() >= deadline { break; } tokio::time::sleep(Duration::from_millis(POLL_MS)).await; } read_task(id) } /// Kill a running or still-pending task. /// /// Fire-and-forget: this **sends a signal and returns** — it does not wait /// for the process to actually exit. /// /// - **Running** (in the registry): signals its process group via the /// `exec_cmd` cancel path — `SIGKILL` if `force`, else `SIGINT`. If the /// process ignores `SIGINT`, the caller re-invokes with `force` to send /// `SIGKILL` (no daemon-side auto-escalation). The task transitions to /// [`TaskStatus::Killed`] once the process exits and a completion wake fires. /// - **Pending** (queued, not yet started): marked `Killed` directly so the /// runner loop never starts it. No process exists yet, so `force` is moot. /// - **Terminal or unknown id**: no-op. /// /// Returns `(killed, was_running)`: `killed` = a kill was issued (signal sent /// or pending task cancelled); `was_running` = the task was actively running. /// /// # Panics /// /// Panics if the running-task registry mutex is poisoned (a prior holder /// panicked) — unrecoverable, consistent with the rest of the daemon's /// `Mutex` usage. #[must_use] pub fn kill_task(id: &str, force: bool) -> (bool, bool) { // Running task: signal via the registry. { let guard = running().lock().unwrap(); if let Some(handle) = guard.get(id) { handle.force.store(force, Ordering::SeqCst); handle.cancel.notify_one(); return (true, true); } } // Pending task: mark Killed so the poll loop never starts it. if let Some(mut task) = read_task(id) && task.status == TaskStatus::Pending { task.status = TaskStatus::Killed; task.completed_at = Some(now_unix()); let _ = write_task(&task); refresh_loose_ends(); return (true, false); } (false, false) } // --------------------------------------------------------------------------- // Runner background loop // --------------------------------------------------------------------------- /// Spawn the background runner loop as a detached tokio task. `socket` /// is the path to the per-agent broker socket used to deliver completion /// wake signals. Call once at daemon startup. pub fn spawn_loop(socket: PathBuf) { tokio::spawn(async move { run_loop(socket).await; }); } async fn run_loop(socket: PathBuf) { if let Err(e) = std::fs::create_dir_all(paths::tasks_dir()) { tracing::warn!(error = ?e, "bash_runner: create tasks dir failed"); } mark_interrupted(&socket).await; let claimed: Arc>> = Arc::new(Mutex::new(HashSet::new())); loop { poll_once(&socket, &claimed); tokio::time::sleep(POLL_INTERVAL).await; } } /// On boot, flip any `running` tasks to `interrupted` and fire a wake. async fn mark_interrupted(socket: &Path) { let Ok(rd) = std::fs::read_dir(paths::tasks_dir()) else { return; }; for entry in rd.flatten() { let path = entry.path(); if path.extension().and_then(|e| e.to_str()) != Some("json") { continue; } let Some(id) = path.file_stem().and_then(|s| s.to_str()).map(str::to_owned) else { continue; }; let Some(mut task) = read_task(&id) else { continue; }; if task.status != TaskStatus::Running { continue; } tracing::warn!(id = %id, "bash_runner: marking interrupted task"); task.status = TaskStatus::Interrupted; task.completed_at = Some(now_unix()); if let Err(e) = write_task(&task) { tracing::warn!(id = %id, error = ?e, "bash_runner: write interrupted state failed"); } refresh_loose_ends(); send_wake(socket, &id, "interrupted (daemon restarted)", None).await; } } fn poll_once(socket: &Path, claimed: &Arc>>) { let Ok(rd) = std::fs::read_dir(paths::tasks_dir()) else { return; }; for entry in rd.flatten() { let path = entry.path(); if path.extension().and_then(|e| e.to_str()) != Some("json") { continue; } let Some(id) = path.file_stem().and_then(|s| s.to_str()).map(str::to_owned) else { continue; }; { let guard = claimed.lock().unwrap(); if guard.contains(&id) { continue; } } let Some(task) = read_task(&id) else { continue }; if task.status != TaskStatus::Pending { continue; } // Claim before spawning to avoid double-spawn across poll iterations. claimed.lock().unwrap().insert(id.clone()); let socket = socket.to_path_buf(); let claimed = claimed.clone(); tokio::spawn(async move { run_task(task, &socket).await; claimed.lock().unwrap().remove(&id); }); } } // --------------------------------------------------------------------------- // Task execution // --------------------------------------------------------------------------- async fn run_task(mut task: TaskFile, socket: &Path) { let id = task.id.clone(); tracing::info!(id = %id, cmd = %task.cmd, "bash_runner: starting task"); task.status = TaskStatus::Running; task.started_at = Some(now_unix()); if let Err(e) = write_task(&task) { tracing::warn!(id = %id, error = ?e, "bash_runner: write running state failed"); } refresh_loose_ends(); // Best-effort: tally the normalised command head for the /stats // "favorite tools" view. Counted once per execution, regardless of // exit status. Never fails the task. crate::stats::record_command(&task.cmd); let out_path = paths::task_out(&id); let err_path = paths::task_err(&id); // Register a kill handle for the duration of execution so `kill_task` // (called from the socket dispatch path) can signal this task. let cancel = Arc::new(Notify::new()); let force = Arc::new(AtomicBool::new(false)); running().lock().unwrap().insert( id.clone(), RunningHandle { cancel: cancel.clone(), force: force.clone(), }, ); let outcome = exec_cmd( &task.cmd, &out_path, &err_path, task.timeout_secs, &cancel, &force, ) .await; // Deregister before writing terminal state — the task is no longer killable. running().lock().unwrap().remove(&id); let stdout_tail = tail_file(&out_path, SUMMARY_BYTES); let stderr_tail = tail_file(&err_path, SUMMARY_BYTES); let (status, exit_code, summary) = match outcome { Ok(ExecOutcome::Exited(code)) => (TaskStatus::Done, Some(code), format!("exit={code}")), Ok(ExecOutcome::TimedOut) => { tracing::warn!(id = %id, "bash_runner: task timed out"); let secs = task.timeout_secs.unwrap_or(0); ( TaskStatus::TimedOut, None, format!("timed out after {secs}s"), ) } Ok(ExecOutcome::Killed { forced }) => { let sig = if forced { "SIGKILL" } else { "SIGINT" }; tracing::warn!(id = %id, sig, "bash_runner: task killed"); (TaskStatus::Killed, None, format!("killed ({sig})")) } Err(e) => { tracing::warn!(id = %id, error = ?e, "bash_runner: exec error"); (TaskStatus::Done, None, "exec error".to_owned()) } }; task.status = status; task.completed_at = Some(now_unix()); task.exit_code = exit_code; task.stdout_tail = stdout_tail.clone().filter(|s| !s.is_empty()); task.stderr_tail = stderr_tail.clone().filter(|s| !s.is_empty()); if let Err(e) = write_task(&task) { tracing::warn!(id = %id, error = ?e, "bash_runner: write done state failed"); } refresh_loose_ends(); let out_snippet = stdout_tail.as_deref().unwrap_or("").trim(); let err_snippet = stderr_tail.as_deref().unwrap_or("").trim(); send_wake(socket, &id, &summary, Some((out_snippet, err_snippet))).await; } /// Run `bash -c cmd` in its own process group, streaming output to files. /// `timeout_secs = None` means no timeout — run until natural exit. /// /// Tasks run under `bash`, not `sh` (on NixOS `/bin/sh` is bash in POSIX /// mode, which disables bashisms — arrays, `[[ … ]]`, `local`, process /// substitution), invoked via `/usr/bin/env bash`. /// /// The task is cancellable via `cancel` (set `force` first): on cancel the /// whole process group is signalled — `SIGKILL` if `force`, else `SIGINT` — /// so children of the shell die too, not just the shell. The timeout path /// likewise `SIGKILL`s the group. async fn exec_cmd( cmd: &str, out_path: &Path, err_path: &Path, timeout_secs: Option, cancel: &Notify, force: &AtomicBool, ) -> Result { use tokio::process::Command; // `/usr/bin/env bash` rather than a bare `bash`: `/usr/bin/env` is at a // fixed absolute path (coreutils, present on NixOS), and it resolves // `bash` via PATH — the same controlled PATH the daemon's systemd unit // sets. Avoids hardcoding a nix store / generation path in the binary. let mut cmd_builder = Command::new("/usr/bin/env"); cmd_builder .arg("bash") .arg("-c") .arg(cmd) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()); // SAFETY: `setpgid`/`nice` are async-signal-safe and only touch the // calling (child) process before exec — no allocations, no locks. // `setpgid(0, 0)` makes the child its own process-group leader so the // whole subtree can be signalled as a group (`kill(-pgid, …)`). unsafe { cmd_builder.pre_exec(|| { libc::setpgid(0, 0); libc::nice(10); Ok(()) }); } let mut child = cmd_builder.spawn()?; // With `setpgid(0, 0)` the child's pgid equals its pid. let pgid = child.id().map(u32_to_i32); let stdout = child.stdout.take().expect("stdout piped"); let stderr = child.stderr.take().expect("stderr piped"); let out_path = out_path.to_path_buf(); let err_path = err_path.to_path_buf(); let copy_out = tokio::spawn(copy_stream_to_file( tokio::io::BufReader::new(stdout), out_path, )); let copy_err = tokio::spawn(copy_stream_to_file( tokio::io::BufReader::new(stderr), err_path, )); // A future that fires when the timeout elapses, or never if `None`. let timeout_fut = async { match timeout_secs { Some(secs) => tokio::time::sleep(Duration::from_secs(secs)).await, None => std::future::pending::<()>().await, } }; let mut timed_out = false; let mut killed: Option = None; let wait_res = { let wait = child.wait(); tokio::pin!(wait); tokio::pin!(timeout_fut); loop { tokio::select! { res = &mut wait => break res, () = &mut timeout_fut => { // Per-task timeout: hard-kill the whole group and reap. signal_group(pgid, libc::SIGKILL); timed_out = true; break (&mut wait).await; } () = cancel.notified() => { // A kill was requested: send the signal and keep waiting. // `kill` is fire-and-forget — we do NOT block-for-exit or // auto-escalate. If the process ignores SIGINT the caller // re-invokes kill with `force`, which re-fires this arm // with SIGKILL. `killed` tracks the last signal sent. let forced = force.load(Ordering::SeqCst); signal_group(pgid, if forced { libc::SIGKILL } else { libc::SIGINT }); killed = Some(forced); } } } }; let _ = copy_out.await; let _ = copy_err.await; let status = wait_res?; let outcome = if let Some(forced) = killed { ExecOutcome::Killed { forced } } else if timed_out { ExecOutcome::TimedOut } else { ExecOutcome::Exited(status.code().unwrap_or(-1)) }; Ok(outcome) } /// `u32` pid → `i32` for `kill(2)`. Pids fit in `i32`; saturates defensively. fn u32_to_i32(v: u32) -> i32 { i32::try_from(v).unwrap_or(i32::MAX) } async fn copy_stream_to_file(mut reader: R, path: PathBuf) where R: tokio::io::AsyncRead + Unpin, { match tokio::fs::File::create(&path).await { Ok(mut f) => { let _ = tokio::io::copy(&mut reader, &mut f).await; let _ = f.flush().await; } Err(e) => { tracing::warn!(path = %path.display(), error = ?e, "bash_runner: open output file failed"); } } } fn tail_file(path: &Path, max_bytes: usize) -> Option { let data = std::fs::read(path).ok()?; let slice = if data.len() > max_bytes { &data[data.len() - max_bytes..] } else { &data }; Some(String::from_utf8_lossy(slice).into_owned()) } // --------------------------------------------------------------------------- // Wake delivery // --------------------------------------------------------------------------- pub(crate) async fn send_wake( socket: &Path, id: &str, summary: &str, output: Option<(&str, &str)>, ) { use tokio::io::{AsyncBufReadExt as _, BufReader}; use tokio::net::UnixStream; let mut body = format!("bash task `{id}` finished: {summary}"); if let Some((stdout, stderr)) = output { if !stdout.is_empty() { body.push_str("\n\nstdout:\n```\n"); body.push_str(stdout); body.push_str("\n```"); } if !stderr.is_empty() { body.push_str("\n\nstderr:\n```\n"); body.push_str(stderr); body.push_str("\n```"); } } let req = hive_sh4re::AgentRequest::Wake { from: format!("bash-task-{id}"), body, }; match UnixStream::connect(socket).await { Ok(stream) => { let (read, mut write) = stream.into_split(); let line = match serde_json::to_string(&req) { Ok(mut s) => { s.push('\n'); s } Err(e) => { tracing::warn!(id = %id, error = ?e, "bash_runner: serialise wake failed"); return; } }; if write.write_all(line.as_bytes()).await.is_err() { tracing::warn!(id = %id, "bash_runner: write wake failed"); return; } let _ = write.shutdown().await; // Drain the response so the server doesn't get ECONNRESET. let mut resp = String::new(); let _ = BufReader::new(read).read_line(&mut resp).await; tracing::info!(id = %id, "bash_runner: wake delivered"); } Err(e) => { tracing::warn!(id = %id, error = ?e, "bash_runner: connect wake socket failed"); } } } #[cfg(test)] mod tests { use super::{MAX_TASK_NAME_LEN, validate_task_name}; #[test] fn accepts_reasonable_names() { for ok in ["build", "ci-check", "nix_flake.check", "t1", "A.B-C_9"] { assert!(validate_task_name(ok).is_ok(), "{ok} should be valid"); } } #[test] fn rejects_unsafe_names() { // Empty, traversal, separators, control/space, and over-long. for bad in [ "", ".", "..", "a/b", "../escape", "has space", "tab\tname", "slash\\back", ] { assert!( validate_task_name(bad).is_err(), "{bad:?} should be rejected" ); } assert!(validate_task_name(&"x".repeat(MAX_TASK_NAME_LEN + 1)).is_err()); // Exactly at the cap is allowed. assert!(validate_task_name(&"x".repeat(MAX_TASK_NAME_LEN)).is_ok()); } }