- subvol upgrade waits for the queued stop DAG before migrating (was snapshotting + swapping state under a live bind mount) and for the restart job after - history trim gets a 5-min grace for fresh terminals so broad stop/start waits can't miss a failed DAG evicted by the per-template cap (cap still applies past the grace) - restart-all returns its DAG ids so hivectl actually waits - hard stops await their agent DAGs (bounded) before infra goes down, restoring the agents-before-infra invariant - hivectl wait uses node-level terminality so the after-any recovery reconcile is watched to completion; infra render errors no longer skip watching already-queued agent DAGs - fold hive-bash-mcp's last local now_unix into wire_time
749 lines
26 KiB
Rust
749 lines
26 KiB
Rust
//! Bash subprocess runner: spawns `bash -c <cmd>` tasks, writes status
|
|
//! files under `harness/bash-tasks/`, and fires hyperhive wake signals
|
|
//! on completion.
|
|
//!
|
|
//! Files under `tasks_dir()`:
|
|
//! - `<id>.json` — task metadata + status (pending → running → done)
|
|
//! - `<id>.out` — captured stdout (streamed while running)
|
|
//! - `<id>.err` — captured stderr (streamed while running)
|
|
//!
|
|
//! Tasks with status `running` on daemon boot are marked `interrupted`
|
|
//! (the process died with the previous daemon). A best-effort wake is
|
|
//! still sent so the agent is not silently blocked.
|
|
//!
|
|
//! The runner kills the child process on timeout — `tokio::process::Child::drop()`
|
|
//! does not kill children, so we explicitly call `child.kill().await`.
|
|
|
|
use std::collections::{HashMap, HashSet};
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
|
use std::sync::{Arc, Mutex, OnceLock};
|
|
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
|
|
|
use anyhow::{Result, bail};
|
|
use tokio::io::AsyncWriteExt as _;
|
|
use tokio::sync::Notify;
|
|
|
|
use crate::paths;
|
|
use crate::protocol::{TaskFile, TaskStatus};
|
|
|
|
/// Poll interval for the runner background loop.
|
|
const POLL_INTERVAL: Duration = Duration::from_millis(200);
|
|
|
|
/// Soft cap on stdout/stderr captured in the done JSON summary.
|
|
/// Full output always lives in the `.out`/`.err` files.
|
|
pub const SUMMARY_BYTES: usize = 4096;
|
|
|
|
/// Maximum inline wait (cap on `wait_seconds`).
|
|
pub const MAX_WAIT_SECS: u64 = 30;
|
|
|
|
/// Poll interval used by the inline-wait loops.
|
|
const POLL_MS: u64 = 100;
|
|
|
|
static TASK_SEQ: AtomicU64 = AtomicU64::new(0);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Running-task registry (for on-demand kill)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Handle to a currently-executing task, used by [`kill_task`] to signal it.
|
|
/// `exec_cmd` selects on `cancel`; when notified it signals the task's
|
|
/// process group with `SIGKILL` (if `force`) or `SIGINT`.
|
|
struct RunningHandle {
|
|
cancel: Arc<Notify>,
|
|
force: Arc<AtomicBool>,
|
|
}
|
|
|
|
/// Global registry of running tasks (id → handle). Populated by `run_task`
|
|
/// for the duration of execution and read by [`kill_task`] from the socket
|
|
/// dispatch path — a separate async context from the runner loop, so a
|
|
/// shared global (rather than the loop-local `claimed` set) is needed.
|
|
fn running() -> &'static Mutex<HashMap<String, RunningHandle>> {
|
|
static RUNNING: OnceLock<Mutex<HashMap<String, RunningHandle>>> = OnceLock::new();
|
|
RUNNING.get_or_init(|| Mutex::new(HashMap::new()))
|
|
}
|
|
|
|
/// Outcome of one `exec_cmd` run.
|
|
enum ExecOutcome {
|
|
/// Process exited on its own with this status code.
|
|
Exited(i32),
|
|
/// Killed by the per-task timeout (`timeout_secs`).
|
|
TimedOut,
|
|
/// Killed on request via [`kill_task`]. `forced` = SIGKILL vs SIGINT.
|
|
Killed { forced: bool },
|
|
}
|
|
|
|
/// Send `sig` to the task's process group. A negative pgid targets the whole
|
|
/// group, so `bash -c` plus any children it spawned all receive the signal.
|
|
/// No-op if the child had no pid (already reaped).
|
|
fn signal_group(pgid: Option<i32>, sig: i32) {
|
|
if let Some(pgid) = pgid {
|
|
// SAFETY: plain libc `kill(2)`; `-pgid` targets the child's own
|
|
// process group (it called `setpgid(0, 0)` so it leads its group).
|
|
unsafe {
|
|
libc::kill(-pgid, sig);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Helpers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
use hive_sh4re::wire_time::now_unix;
|
|
|
|
/// Generate a task ID: `<timestamp_hex><seq_hex>`.
|
|
#[must_use]
|
|
pub fn new_task_id() -> String {
|
|
let t = SystemTime::now()
|
|
.duration_since(UNIX_EPOCH)
|
|
.unwrap_or_default()
|
|
.as_millis();
|
|
let seq = TASK_SEQ.fetch_add(1, Ordering::Relaxed);
|
|
format!("{t:013x}{seq:04x}")
|
|
}
|
|
|
|
/// Write a task file atomically (tmp + rename).
|
|
fn write_task(task: &TaskFile) -> std::io::Result<()> {
|
|
let json = serde_json::to_string_pretty(task)
|
|
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
|
let dest = paths::task_json(&task.id);
|
|
let tmp = dest.with_extension("json.tmp");
|
|
std::fs::write(&tmp, &json)?;
|
|
std::fs::rename(&tmp, &dest)
|
|
}
|
|
|
|
/// Read a task file. Returns `None` if the file doesn't exist or is
|
|
/// unparseable.
|
|
#[must_use]
|
|
pub fn read_task(id: &str) -> Option<TaskFile> {
|
|
let s = std::fs::read_to_string(paths::task_json(id)).ok()?;
|
|
serde_json::from_str(&s).ok()
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Loose-ends file (generic MCP loose-ends protocol)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Rewrite `mcp-loose-ends/bash.json` with a summary of all currently
|
|
/// active (Pending or Running) tasks. The harness scans this directory
|
|
/// generically in `get_loose_ends` — no bash-specific code needed there.
|
|
///
|
|
/// File format: a JSON array of plain-text summary strings, one per
|
|
/// loose-end item. The harness includes them verbatim in the output.
|
|
/// Atomic write (tmp + rename) so the harness never reads a partial file.
|
|
fn refresh_loose_ends() {
|
|
let active = active_tasks();
|
|
let dir = paths::mcp_loose_ends_dir();
|
|
if let Err(e) = std::fs::create_dir_all(&dir) {
|
|
tracing::warn!(error = ?e, "bash_runner: create mcp-loose-ends dir failed");
|
|
return;
|
|
}
|
|
let items: Vec<String> = active
|
|
.iter()
|
|
.map(|t| {
|
|
let age = now_unix() - t.created_at;
|
|
format!(
|
|
"bash task `{}` status={:?}, cmd: `{}`, age {}s",
|
|
t.id, t.status, t.cmd, age
|
|
)
|
|
})
|
|
.collect();
|
|
let dest = dir.join("bash.json");
|
|
let tmp = dest.with_extension("json.tmp");
|
|
let json = serde_json::to_string(&items).unwrap_or_else(|_| "[]".to_owned());
|
|
if let Err(e) = std::fs::write(&tmp, &json).and_then(|()| std::fs::rename(&tmp, &dest)) {
|
|
tracing::warn!(error = ?e, "bash_runner: write mcp-loose-ends/bash.json failed");
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Public API used by daemon dispatch
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Longest accepted caller-chosen task name.
|
|
const MAX_TASK_NAME_LEN: usize = 64;
|
|
|
|
/// Validate a caller-chosen task name. The name doubles as the task id
|
|
/// and therefore as the `<name>.json` filename, so it must be a single
|
|
/// safe path segment. Allows ASCII alphanumerics plus `.`, `_`, `-`;
|
|
/// rejects empties, over-long names, `.`/`..`, and anything that could
|
|
/// escape the tasks dir or collide with the `.json.tmp` scratch suffix.
|
|
fn validate_task_name(name: &str) -> Result<()> {
|
|
if name.is_empty() {
|
|
bail!("task name must not be empty");
|
|
}
|
|
if name.len() > MAX_TASK_NAME_LEN {
|
|
bail!("task name too long (max {MAX_TASK_NAME_LEN} chars)");
|
|
}
|
|
if name == "." || name == ".." {
|
|
bail!("task name {name:?} is reserved");
|
|
}
|
|
if !name
|
|
.chars()
|
|
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-'))
|
|
{
|
|
bail!("task name {name:?} may only contain ASCII letters, digits, '.', '_', '-'");
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Submit a new pending task. Returns the task ID.
|
|
///
|
|
/// When `name` is `Some`, it is validated and used as the task id (so it
|
|
/// surfaces in the wake `from`, status lookups, and the loose-ends list).
|
|
/// A name may be reused once any prior task of that name has finished;
|
|
/// submitting a name whose task is still `Pending`/`Running` is rejected.
|
|
/// `None` falls back to the auto-generated timestamp id.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns an error if the name is invalid, a task of that name is still
|
|
/// running, the tasks directory cannot be created, or the task file
|
|
/// cannot be written.
|
|
pub fn submit_task(cmd: String, timeout_secs: Option<u64>, name: Option<String>) -> Result<String> {
|
|
std::fs::create_dir_all(paths::tasks_dir())?;
|
|
let id = match name {
|
|
Some(name) => {
|
|
validate_task_name(&name)?;
|
|
if let Some(existing) = read_task(&name)
|
|
&& matches!(existing.status, TaskStatus::Pending | TaskStatus::Running)
|
|
{
|
|
bail!(
|
|
"a bash task named `{name}` is already running — wait for it to finish or pick another name"
|
|
);
|
|
}
|
|
name
|
|
}
|
|
None => new_task_id(),
|
|
};
|
|
let task = TaskFile {
|
|
id: id.clone(),
|
|
cmd,
|
|
timeout_secs,
|
|
status: TaskStatus::Pending,
|
|
created_at: now_unix(),
|
|
started_at: None,
|
|
completed_at: None,
|
|
exit_code: None,
|
|
stdout_tail: None,
|
|
stderr_tail: None,
|
|
};
|
|
write_task(&task)?;
|
|
refresh_loose_ends();
|
|
Ok(id)
|
|
}
|
|
|
|
/// Return all tasks currently in `Pending` or `Running` state.
|
|
#[must_use]
|
|
pub fn active_tasks() -> Vec<TaskFile> {
|
|
let Ok(rd) = std::fs::read_dir(paths::tasks_dir()) else {
|
|
return Vec::new();
|
|
};
|
|
let mut out = Vec::new();
|
|
for entry in rd.flatten() {
|
|
let path = entry.path();
|
|
if path.extension().and_then(|e| e.to_str()) != Some("json") {
|
|
continue;
|
|
}
|
|
let Some(id) = path.file_stem().and_then(|s| s.to_str()).map(str::to_owned) else {
|
|
continue;
|
|
};
|
|
let Some(task) = read_task(&id) else { continue };
|
|
if matches!(task.status, TaskStatus::Pending | TaskStatus::Running) {
|
|
out.push(task);
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// Inline wait: poll `read_task(id)` until terminal state or deadline.
|
|
/// Returns the final task on success, or `None` if it never completed.
|
|
pub async fn wait_for_task(id: &str, wait_secs: u64) -> Option<TaskFile> {
|
|
let cap = wait_secs.min(MAX_WAIT_SECS);
|
|
if cap == 0 {
|
|
return read_task(id);
|
|
}
|
|
let deadline = tokio::time::Instant::now() + Duration::from_secs(cap);
|
|
loop {
|
|
match read_task(id) {
|
|
None => break,
|
|
Some(task) => {
|
|
if matches!(
|
|
task.status,
|
|
TaskStatus::Done
|
|
| TaskStatus::TimedOut
|
|
| TaskStatus::Interrupted
|
|
| TaskStatus::Killed
|
|
) {
|
|
return Some(task);
|
|
}
|
|
}
|
|
}
|
|
if tokio::time::Instant::now() >= deadline {
|
|
break;
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(POLL_MS)).await;
|
|
}
|
|
read_task(id)
|
|
}
|
|
|
|
/// Kill a running or still-pending task.
|
|
///
|
|
/// Fire-and-forget: this **sends a signal and returns** — it does not wait
|
|
/// for the process to actually exit.
|
|
///
|
|
/// - **Running** (in the registry): signals its process group via the
|
|
/// `exec_cmd` cancel path — `SIGKILL` if `force`, else `SIGINT`. If the
|
|
/// process ignores `SIGINT`, the caller re-invokes with `force` to send
|
|
/// `SIGKILL` (no daemon-side auto-escalation). The task transitions to
|
|
/// [`TaskStatus::Killed`] once the process exits and a completion wake fires.
|
|
/// - **Pending** (queued, not yet started): marked `Killed` directly so the
|
|
/// runner loop never starts it. No process exists yet, so `force` is moot.
|
|
/// - **Terminal or unknown id**: no-op.
|
|
///
|
|
/// Returns `(killed, was_running)`: `killed` = a kill was issued (signal sent
|
|
/// or pending task cancelled); `was_running` = the task was actively running.
|
|
///
|
|
/// # Panics
|
|
///
|
|
/// Panics if the running-task registry mutex is poisoned (a prior holder
|
|
/// panicked) — unrecoverable, consistent with the rest of the daemon's
|
|
/// `Mutex` usage.
|
|
#[must_use]
|
|
pub fn kill_task(id: &str, force: bool) -> (bool, bool) {
|
|
// Running task: signal via the registry.
|
|
{
|
|
let guard = running().lock().unwrap();
|
|
if let Some(handle) = guard.get(id) {
|
|
handle.force.store(force, Ordering::SeqCst);
|
|
handle.cancel.notify_one();
|
|
return (true, true);
|
|
}
|
|
}
|
|
// Pending task: mark Killed so the poll loop never starts it.
|
|
if let Some(mut task) = read_task(id)
|
|
&& task.status == TaskStatus::Pending
|
|
{
|
|
task.status = TaskStatus::Killed;
|
|
task.completed_at = Some(now_unix());
|
|
let _ = write_task(&task);
|
|
refresh_loose_ends();
|
|
return (true, false);
|
|
}
|
|
(false, false)
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Runner background loop
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Spawn the background runner loop as a detached tokio task. `socket`
|
|
/// is the path to the per-agent broker socket used to deliver completion
|
|
/// wake signals. Call once at daemon startup.
|
|
pub fn spawn_loop(socket: PathBuf) {
|
|
tokio::spawn(async move {
|
|
run_loop(socket).await;
|
|
});
|
|
}
|
|
|
|
async fn run_loop(socket: PathBuf) {
|
|
if let Err(e) = std::fs::create_dir_all(paths::tasks_dir()) {
|
|
tracing::warn!(error = ?e, "bash_runner: create tasks dir failed");
|
|
}
|
|
mark_interrupted(&socket).await;
|
|
|
|
let claimed: Arc<Mutex<HashSet<String>>> = Arc::new(Mutex::new(HashSet::new()));
|
|
|
|
loop {
|
|
poll_once(&socket, &claimed);
|
|
tokio::time::sleep(POLL_INTERVAL).await;
|
|
}
|
|
}
|
|
|
|
/// On boot, flip any `running` tasks to `interrupted` and fire a wake.
|
|
async fn mark_interrupted(socket: &Path) {
|
|
let Ok(rd) = std::fs::read_dir(paths::tasks_dir()) else {
|
|
return;
|
|
};
|
|
for entry in rd.flatten() {
|
|
let path = entry.path();
|
|
if path.extension().and_then(|e| e.to_str()) != Some("json") {
|
|
continue;
|
|
}
|
|
let Some(id) = path.file_stem().and_then(|s| s.to_str()).map(str::to_owned) else {
|
|
continue;
|
|
};
|
|
let Some(mut task) = read_task(&id) else {
|
|
continue;
|
|
};
|
|
if task.status != TaskStatus::Running {
|
|
continue;
|
|
}
|
|
tracing::warn!(id = %id, "bash_runner: marking interrupted task");
|
|
task.status = TaskStatus::Interrupted;
|
|
task.completed_at = Some(now_unix());
|
|
if let Err(e) = write_task(&task) {
|
|
tracing::warn!(id = %id, error = ?e, "bash_runner: write interrupted state failed");
|
|
}
|
|
refresh_loose_ends();
|
|
send_wake(socket, &id, "interrupted (daemon restarted)", None).await;
|
|
}
|
|
}
|
|
|
|
fn poll_once(socket: &Path, claimed: &Arc<Mutex<HashSet<String>>>) {
|
|
let Ok(rd) = std::fs::read_dir(paths::tasks_dir()) else {
|
|
return;
|
|
};
|
|
for entry in rd.flatten() {
|
|
let path = entry.path();
|
|
if path.extension().and_then(|e| e.to_str()) != Some("json") {
|
|
continue;
|
|
}
|
|
let Some(id) = path.file_stem().and_then(|s| s.to_str()).map(str::to_owned) else {
|
|
continue;
|
|
};
|
|
{
|
|
let guard = claimed.lock().unwrap();
|
|
if guard.contains(&id) {
|
|
continue;
|
|
}
|
|
}
|
|
let Some(task) = read_task(&id) else { continue };
|
|
if task.status != TaskStatus::Pending {
|
|
continue;
|
|
}
|
|
// Claim before spawning to avoid double-spawn across poll iterations.
|
|
claimed.lock().unwrap().insert(id.clone());
|
|
let socket = socket.to_path_buf();
|
|
let claimed = claimed.clone();
|
|
tokio::spawn(async move {
|
|
run_task(task, &socket).await;
|
|
claimed.lock().unwrap().remove(&id);
|
|
});
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Task execution
|
|
// ---------------------------------------------------------------------------
|
|
|
|
async fn run_task(mut task: TaskFile, socket: &Path) {
|
|
let id = task.id.clone();
|
|
tracing::info!(id = %id, cmd = %task.cmd, "bash_runner: starting task");
|
|
|
|
task.status = TaskStatus::Running;
|
|
task.started_at = Some(now_unix());
|
|
if let Err(e) = write_task(&task) {
|
|
tracing::warn!(id = %id, error = ?e, "bash_runner: write running state failed");
|
|
}
|
|
refresh_loose_ends();
|
|
// Best-effort: tally the normalised command head for the /stats
|
|
// "favorite tools" view. Counted once per execution, regardless of
|
|
// exit status. Never fails the task.
|
|
crate::stats::record_command(&task.cmd);
|
|
|
|
let out_path = paths::task_out(&id);
|
|
let err_path = paths::task_err(&id);
|
|
|
|
// Register a kill handle for the duration of execution so `kill_task`
|
|
// (called from the socket dispatch path) can signal this task.
|
|
let cancel = Arc::new(Notify::new());
|
|
let force = Arc::new(AtomicBool::new(false));
|
|
running().lock().unwrap().insert(
|
|
id.clone(),
|
|
RunningHandle {
|
|
cancel: cancel.clone(),
|
|
force: force.clone(),
|
|
},
|
|
);
|
|
|
|
let outcome = exec_cmd(
|
|
&task.cmd,
|
|
&out_path,
|
|
&err_path,
|
|
task.timeout_secs,
|
|
&cancel,
|
|
&force,
|
|
)
|
|
.await;
|
|
|
|
// Deregister before writing terminal state — the task is no longer killable.
|
|
running().lock().unwrap().remove(&id);
|
|
|
|
let stdout_tail = tail_file(&out_path, SUMMARY_BYTES);
|
|
let stderr_tail = tail_file(&err_path, SUMMARY_BYTES);
|
|
|
|
let (status, exit_code, summary) = match outcome {
|
|
Ok(ExecOutcome::Exited(code)) => (TaskStatus::Done, Some(code), format!("exit={code}")),
|
|
Ok(ExecOutcome::TimedOut) => {
|
|
tracing::warn!(id = %id, "bash_runner: task timed out");
|
|
let secs = task.timeout_secs.unwrap_or(0);
|
|
(
|
|
TaskStatus::TimedOut,
|
|
None,
|
|
format!("timed out after {secs}s"),
|
|
)
|
|
}
|
|
Ok(ExecOutcome::Killed { forced }) => {
|
|
let sig = if forced { "SIGKILL" } else { "SIGINT" };
|
|
tracing::warn!(id = %id, sig, "bash_runner: task killed");
|
|
(TaskStatus::Killed, None, format!("killed ({sig})"))
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(id = %id, error = ?e, "bash_runner: exec error");
|
|
(TaskStatus::Done, None, "exec error".to_owned())
|
|
}
|
|
};
|
|
|
|
task.status = status;
|
|
task.completed_at = Some(now_unix());
|
|
task.exit_code = exit_code;
|
|
task.stdout_tail = stdout_tail.clone().filter(|s| !s.is_empty());
|
|
task.stderr_tail = stderr_tail.clone().filter(|s| !s.is_empty());
|
|
|
|
if let Err(e) = write_task(&task) {
|
|
tracing::warn!(id = %id, error = ?e, "bash_runner: write done state failed");
|
|
}
|
|
refresh_loose_ends();
|
|
|
|
let out_snippet = stdout_tail.as_deref().unwrap_or("").trim();
|
|
let err_snippet = stderr_tail.as_deref().unwrap_or("").trim();
|
|
send_wake(socket, &id, &summary, Some((out_snippet, err_snippet))).await;
|
|
}
|
|
|
|
/// Run `bash -c cmd` in its own process group, streaming output to files.
|
|
/// `timeout_secs = None` means no timeout — run until natural exit.
|
|
///
|
|
/// Tasks run under `bash`, not `sh` (on NixOS `/bin/sh` is bash in POSIX
|
|
/// mode, which disables bashisms — arrays, `[[ … ]]`, `local`, process
|
|
/// substitution), invoked via `/usr/bin/env bash`.
|
|
///
|
|
/// The task is cancellable via `cancel` (set `force` first): on cancel the
|
|
/// whole process group is signalled — `SIGKILL` if `force`, else `SIGINT` —
|
|
/// so children of the shell die too, not just the shell. The timeout path
|
|
/// likewise `SIGKILL`s the group.
|
|
async fn exec_cmd(
|
|
cmd: &str,
|
|
out_path: &Path,
|
|
err_path: &Path,
|
|
timeout_secs: Option<u64>,
|
|
cancel: &Notify,
|
|
force: &AtomicBool,
|
|
) -> Result<ExecOutcome> {
|
|
use tokio::process::Command;
|
|
// `/usr/bin/env bash` rather than a bare `bash`: `/usr/bin/env` is at a
|
|
// fixed absolute path (coreutils, present on NixOS), and it resolves
|
|
// `bash` via PATH — the same controlled PATH the daemon's systemd unit
|
|
// sets. Avoids hardcoding a nix store / generation path in the binary.
|
|
let mut cmd_builder = Command::new("/usr/bin/env");
|
|
cmd_builder
|
|
.arg("bash")
|
|
.arg("-c")
|
|
.arg(cmd)
|
|
.stdout(std::process::Stdio::piped())
|
|
.stderr(std::process::Stdio::piped());
|
|
// SAFETY: `setpgid`/`nice` are async-signal-safe and only touch the
|
|
// calling (child) process before exec — no allocations, no locks.
|
|
// `setpgid(0, 0)` makes the child its own process-group leader so the
|
|
// whole subtree can be signalled as a group (`kill(-pgid, …)`).
|
|
unsafe {
|
|
cmd_builder.pre_exec(|| {
|
|
libc::setpgid(0, 0);
|
|
libc::nice(10);
|
|
Ok(())
|
|
});
|
|
}
|
|
let mut child = cmd_builder.spawn()?;
|
|
// With `setpgid(0, 0)` the child's pgid equals its pid.
|
|
let pgid = child.id().map(u32_to_i32);
|
|
|
|
let stdout = child.stdout.take().expect("stdout piped");
|
|
let stderr = child.stderr.take().expect("stderr piped");
|
|
|
|
let out_path = out_path.to_path_buf();
|
|
let err_path = err_path.to_path_buf();
|
|
|
|
let copy_out = tokio::spawn(copy_stream_to_file(
|
|
tokio::io::BufReader::new(stdout),
|
|
out_path,
|
|
));
|
|
let copy_err = tokio::spawn(copy_stream_to_file(
|
|
tokio::io::BufReader::new(stderr),
|
|
err_path,
|
|
));
|
|
|
|
// A future that fires when the timeout elapses, or never if `None`.
|
|
let timeout_fut = async {
|
|
match timeout_secs {
|
|
Some(secs) => tokio::time::sleep(Duration::from_secs(secs)).await,
|
|
None => std::future::pending::<()>().await,
|
|
}
|
|
};
|
|
|
|
let mut timed_out = false;
|
|
let mut killed: Option<bool> = None;
|
|
let wait_res = {
|
|
let wait = child.wait();
|
|
tokio::pin!(wait);
|
|
tokio::pin!(timeout_fut);
|
|
loop {
|
|
tokio::select! {
|
|
res = &mut wait => break res,
|
|
() = &mut timeout_fut => {
|
|
// Per-task timeout: hard-kill the whole group and reap.
|
|
signal_group(pgid, libc::SIGKILL);
|
|
timed_out = true;
|
|
break (&mut wait).await;
|
|
}
|
|
() = cancel.notified() => {
|
|
// A kill was requested: send the signal and keep waiting.
|
|
// `kill` is fire-and-forget — we do NOT block-for-exit or
|
|
// auto-escalate. If the process ignores SIGINT the caller
|
|
// re-invokes kill with `force`, which re-fires this arm
|
|
// with SIGKILL. `killed` tracks the last signal sent.
|
|
let forced = force.load(Ordering::SeqCst);
|
|
signal_group(pgid, if forced { libc::SIGKILL } else { libc::SIGINT });
|
|
killed = Some(forced);
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
let _ = copy_out.await;
|
|
let _ = copy_err.await;
|
|
|
|
let status = wait_res?;
|
|
let outcome = if let Some(forced) = killed {
|
|
ExecOutcome::Killed { forced }
|
|
} else if timed_out {
|
|
ExecOutcome::TimedOut
|
|
} else {
|
|
ExecOutcome::Exited(status.code().unwrap_or(-1))
|
|
};
|
|
Ok(outcome)
|
|
}
|
|
|
|
/// `u32` pid → `i32` for `kill(2)`. Pids fit in `i32`; saturates defensively.
|
|
fn u32_to_i32(v: u32) -> i32 {
|
|
i32::try_from(v).unwrap_or(i32::MAX)
|
|
}
|
|
|
|
async fn copy_stream_to_file<R>(mut reader: R, path: PathBuf)
|
|
where
|
|
R: tokio::io::AsyncRead + Unpin,
|
|
{
|
|
match tokio::fs::File::create(&path).await {
|
|
Ok(mut f) => {
|
|
let _ = tokio::io::copy(&mut reader, &mut f).await;
|
|
let _ = f.flush().await;
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(path = %path.display(), error = ?e, "bash_runner: open output file failed");
|
|
}
|
|
}
|
|
}
|
|
|
|
fn tail_file(path: &Path, max_bytes: usize) -> Option<String> {
|
|
let data = std::fs::read(path).ok()?;
|
|
let slice = if data.len() > max_bytes {
|
|
&data[data.len() - max_bytes..]
|
|
} else {
|
|
&data
|
|
};
|
|
Some(String::from_utf8_lossy(slice).into_owned())
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Wake delivery
|
|
// ---------------------------------------------------------------------------
|
|
|
|
pub(crate) async fn send_wake(
|
|
socket: &Path,
|
|
id: &str,
|
|
summary: &str,
|
|
output: Option<(&str, &str)>,
|
|
) {
|
|
use tokio::io::{AsyncBufReadExt as _, BufReader};
|
|
use tokio::net::UnixStream;
|
|
let mut body = format!("bash task `{id}` finished: {summary}");
|
|
if let Some((stdout, stderr)) = output {
|
|
if !stdout.is_empty() {
|
|
body.push_str("\n\nstdout:\n```\n");
|
|
body.push_str(stdout);
|
|
body.push_str("\n```");
|
|
}
|
|
if !stderr.is_empty() {
|
|
body.push_str("\n\nstderr:\n```\n");
|
|
body.push_str(stderr);
|
|
body.push_str("\n```");
|
|
}
|
|
}
|
|
let req = hive_sh4re::AgentRequest::Wake {
|
|
from: format!("bash-task-{id}"),
|
|
body,
|
|
};
|
|
|
|
match UnixStream::connect(socket).await {
|
|
Ok(stream) => {
|
|
let (read, mut write) = stream.into_split();
|
|
let line = match serde_json::to_string(&req) {
|
|
Ok(mut s) => {
|
|
s.push('\n');
|
|
s
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(id = %id, error = ?e, "bash_runner: serialise wake failed");
|
|
return;
|
|
}
|
|
};
|
|
if write.write_all(line.as_bytes()).await.is_err() {
|
|
tracing::warn!(id = %id, "bash_runner: write wake failed");
|
|
return;
|
|
}
|
|
let _ = write.shutdown().await;
|
|
// Drain the response so the server doesn't get ECONNRESET.
|
|
let mut resp = String::new();
|
|
let _ = BufReader::new(read).read_line(&mut resp).await;
|
|
tracing::info!(id = %id, "bash_runner: wake delivered");
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(id = %id, error = ?e, "bash_runner: connect wake socket failed");
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::{MAX_TASK_NAME_LEN, validate_task_name};
|
|
|
|
#[test]
|
|
fn accepts_reasonable_names() {
|
|
for ok in ["build", "ci-check", "nix_flake.check", "t1", "A.B-C_9"] {
|
|
assert!(validate_task_name(ok).is_ok(), "{ok} should be valid");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn rejects_unsafe_names() {
|
|
// Empty, traversal, separators, control/space, and over-long.
|
|
for bad in [
|
|
"",
|
|
".",
|
|
"..",
|
|
"a/b",
|
|
"../escape",
|
|
"has space",
|
|
"tab\tname",
|
|
"slash\\back",
|
|
] {
|
|
assert!(
|
|
validate_task_name(bad).is_err(),
|
|
"{bad:?} should be rejected"
|
|
);
|
|
}
|
|
assert!(validate_task_name(&"x".repeat(MAX_TASK_NAME_LEN + 1)).is_err());
|
|
// Exactly at the cap is allowed.
|
|
assert!(validate_task_name(&"x".repeat(MAX_TASK_NAME_LEN)).is_ok());
|
|
}
|
|
}
|