feat(#2109): harness-side idle watchdog to bail on anthropic api stall storms

hive-claude/src/config.rs

@@ -1,6 +1,7 @@
 //! Invocation config: how to build one `claude --print` command line.
 
 use std::path::PathBuf;
+use std::time::Duration;
 
 /// How a run attaches to a claude session — the low-level session flag. Kept
 /// separate from [`Config`] so one config can drive resume + create +
@@ -56,4 +57,11 @@ pub struct Config {
     pub extra_args: Vec<String>,
     /// Program to spawn. `None` defaults to `claude` (resolved on `PATH`).
     pub program: Option<String>,
+    /// Idle watchdog: kill the child and return [`crate::Error::IdleTimeout`]
+    /// if no stdout line arrives for this long. The timer resets on every
+    /// stdout line, so a large/slow but still-streaming turn is never cut;
+    /// only complete output silence trips it. `None` waits indefinitely.
+    /// The driver stays policy-free — the caller decides the window (and
+    /// whether to read it from the environment).
+    pub idle_timeout: Option<Duration>,
 }

hive-claude/src/driver.rs

@@ -2,10 +2,12 @@
 //! assemble the result.
 
 use std::collections::VecDeque;
-use std::process::Stdio;
+use std::process::{ExitStatus, Stdio};
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::time::{Duration, Instant};
 
 use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
-use tokio::process::{ChildStderr, ChildStdout, Command};
+use tokio::process::{Child, ChildStderr, ChildStdout, Command};
 
 use crate::classify::Sentinels;
 use crate::{Attach, Config, Error, Result, Sink};
@@ -16,6 +18,11 @@ const DEFAULT_PROGRAM: &str = "claude";
 /// How many trailing stderr lines to keep for [`Error::Exit`].
 const STDERR_TAIL_LINES: usize = 20;
 
+/// Idle-watchdog probe cadence: re-check output silence this often while the
+/// child runs. Fine-grained enough to fire within ~one probe of the deadline,
+/// cheap enough to ignore.
+const IDLE_PROBE: Duration = Duration::from_secs(5);
+
 /// The low-level driver entry point. A namespace for the run function — there
 /// is nothing to construct; call `Claude::run(…)` directly. For a durable,
 /// self-compacting session, use [`crate::InfiniteSession`] instead.
@@ -37,6 +44,8 @@ impl Claude {
     ///   [`Error::AuthFailed`], [`Error::SessionNotFound`].
     /// - [`Error::Spawn`] if the binary can't be launched.
     /// - [`Error::Stdin`] / [`Error::Wait`] on stdin-write / child-wait failure.
+    /// - [`Error::IdleTimeout`] if `config.idle_timeout` is set and no stdout
+    ///   line arrives within that window (the child is killed).
     /// - [`Error::Exit`] on a non-zero exit that raised no sentinel.
     pub async fn run(
         config: &Config,
@@ -69,20 +78,29 @@ impl Claude {
         let stderr = child.stderr.take().expect("stderr piped");
 
         let sentinels = Sentinels::default();
+        // Idle watchdog clock: `last_activity` holds seconds-since-`base` of the
+        // last stdout line, bumped by the pump; the waiter reads it to detect a
+        // fully silent stall. Monotonic (`Instant`) so a wall-clock jump can't
+        // spuriously fire it.
+        let base = Instant::now();
+        let last_activity = AtomicU64::new(0);
         // Pump both streams and wait for exit concurrently on this task — no
         // `spawn`, so the sink needn't be `'static` and borrows stay simple.
-        let ((), stderr_tail, status) = tokio::join!(
-            pump_stdout(stdout, sink, &sentinels),
+        let ((), stderr_tail, (status, timed_out)) = tokio::join!(
+            pump_stdout(stdout, sink, &sentinels, base, &last_activity),
             pump_stderr(stderr, sink, &sentinels),
-            child.wait(),
+            wait_with_idle(&mut child, base, &last_activity, config.idle_timeout),
         );
         let status = status.map_err(Error::Wait)?;
 
-        // A recognized sentinel takes precedence over the exit code; otherwise
-        // a non-zero exit with no sentinel is a hard failure.
+        // A recognized sentinel takes precedence over everything (most specific
+        // reason). Then an idle-watchdog kill; then a plain non-zero exit.
         if let Some(sentinel) = sentinels.soft_error() {
             return Err(sentinel);
         }
+        if timed_out {
+            return Err(Error::IdleTimeout);
+        }
         if !status.success() {
             return Err(Error::Exit {
                 status,
@@ -93,6 +111,36 @@ impl Claude {
     }
 }
 
+/// Wait for the child to exit, enforcing the optional idle watchdog. With no
+/// `idle_timeout` this is a plain `child.wait()`. Otherwise it re-checks on a
+/// fixed probe cadence: if no stdout line arrived within `idle_timeout` (per
+/// `last_activity`, bumped by [`pump_stdout`]), it kills the child and reaps
+/// it. Returns the exit status and whether the watchdog fired.
+async fn wait_with_idle(
+    child: &mut Child,
+    base: Instant,
+    last_activity: &AtomicU64,
+    idle_timeout: Option<Duration>,
+) -> (std::io::Result<ExitStatus>, bool) {
+    let Some(window) = idle_timeout else {
+        return (child.wait().await, false);
+    };
+    // `child.wait()` is cancel-safe, so dropping it on a probe timeout doesn't
+    // lose the exit.
+    loop {
+        match tokio::time::timeout(IDLE_PROBE, child.wait()).await {
+            Ok(status) => return (status, false),
+            Err(_probe_expired) => {
+                let last = Duration::from_secs(last_activity.load(Ordering::Relaxed));
+                if base.elapsed().saturating_sub(last) >= window {
+                    let _ = child.kill().await;
+                    return (child.wait().await, true);
+                }
+            }
+        }
+    }
+}
+
 /// Assemble the argv. `--print --verbose --output-format stream-json` are
 /// mandatory (the driver parses that shape); everything else is gated on the
 /// [`Config`] / [`Attach`].
@@ -149,9 +197,18 @@ fn build_command(program: &str, config: &Config, attach: &Attach) -> Command {
 
 /// Read stdout line by line: classify each line, parse JSON, hand events (or
 /// raw non-JSON lines) to the sink.
-async fn pump_stdout(stdout: ChildStdout, sink: &impl Sink, sentinels: &Sentinels) {
+async fn pump_stdout(
+    stdout: ChildStdout,
+    sink: &impl Sink,
+    sentinels: &Sentinels,
+    base: Instant,
+    last_activity: &AtomicU64,
+) {
     let mut lines = BufReader::new(stdout).lines();
     while let Ok(Some(line)) = lines.next_line().await {
+        // Poke the idle watchdog: any stdout line resets the silence timer.
+        // Seconds granularity is plenty — the probe cadence is coarser still.
+        last_activity.store(base.elapsed().as_secs(), Ordering::Relaxed);
         if let Ok(event) = serde_json::from_str::<serde_json::Value>(&line) {
             // JSON stdout: classify with the model-content gate so an
             // `assistant`/`user` message quoting a marker can't trip it.

hive-claude/src/error.rs

@@ -57,6 +57,14 @@ pub enum Error {
     #[error("waiting on claude failed: {0}")]
     Wait(#[source] std::io::Error),
 
+    /// The child produced no stdout for longer than the configured idle
+    /// window (`Config::idle_timeout`) and was killed. Indicative of an
+    /// Anthropic API stall (e.g. a multi-retry connection storm that goes
+    /// silent for minutes). Callers typically park briefly and retry, like
+    /// the rate-limit path.
+    #[error("claude idle timeout: no output for the configured window")]
+    IdleTimeout,
+
     /// claude exited non-zero and raised none of the recognized sentinels.
     /// `stderr_tail` is the last handful of stderr lines (empty if there were
     /// none), included so the caller can surface a real diagnostic.