feat(#2109): harness-side idle watchdog to bail on anthropic api stall storms

hive-ag3nt/src/bin/hive-agent.rs

@@ -553,6 +553,22 @@ async fn handle_turn<S: Surface>(
         S::requeue_inflight(socket).await;
         bus.emit_status("online");
     }
+    if matches!(outcome, Err(turn::TurnError::ApiStall)) {
+        // Idle watchdog killed claude on a suspected API stall. Park briefly to
+        // let the API recover, then requeue — same shape as the rate-limit path.
+        let secs = turn::stall_sleep_secs();
+        bus.emit_status("api_stall");
+        bus.emit(LiveEvent::Note {
+            text: format!(
+                "API stall timeout — sleeping {secs}s before retry \
+                 (tune HIVE_STALL_SLEEP_SECS; disable the watchdog with HIVE_TURN_IDLE_SECS=0)"
+            ),
+        });
+        tracing::warn!(sleep_secs = secs, "API stall; parking before retry");
+        tokio::time::sleep(Duration::from_secs(secs)).await;
+        S::requeue_inflight(socket).await;
+        bus.emit_status("online");
+    }
     if matches!(outcome, Err(turn::TurnError::AuthFailed)) {
         bus.emit_status("needs_login_idle");
         bus.emit(LiveEvent::Note {

hive-ag3nt/src/serve_common.rs

@@ -98,6 +98,7 @@ pub fn build_row(args: TurnRowArgs<'_>) -> TurnStatRow {
         Err(TurnError::RateLimited) => ("rate_limited", None),
         Err(TurnError::AuthFailed) => ("auth_failed", None),
         Err(TurnError::SessionNotFound) => ("session_not_found", None),
+        Err(TurnError::ApiStall) => ("api_stall", None),
         Err(TurnError::Failed(e)) => ("failed", Some(format!("{e:#}"))),
     };
     let wake_from = if wake_from.starts_with("bash-task-") {

hive-ag3nt/src/turn.rs

@@ -47,6 +47,19 @@ const DEFAULT_SESSION_TITLE: &str = "hive-session";
 /// capacity limits.
 const DEFAULT_RATE_LIMIT_SLEEP_SECS: u64 = 300;
 
+/// Idle watchdog window: kill claude and surface `ApiStall` if it produces no
+/// stdout for this long. The timer resets on every stdout line, so a large but
+/// still-streaming turn is never cut — only a complete silence trips it.
+/// Default is 10 minutes: long enough for legitimate big-context turns, short
+/// enough to cut a multi-retry Anthropic connection storm (atlas telemetry:
+/// attempt:11 took ~6.7min on a 371k-token context). Override via
+/// `HIVE_TURN_IDLE_SECS`; `0` disables the watchdog (wait indefinitely).
+const DEFAULT_TURN_IDLE_SECS: u64 = 600;
+
+/// How long to park after an `ApiStall` before requeueing the message, giving
+/// the API a chance to recover. Overridable via `HIVE_STALL_SLEEP_SECS`.
+const DEFAULT_STALL_SLEEP_SECS: u64 = 60;
+
 /// Assumed prompt-cache TTL. Claude caches prompt prefixes — ~5 minutes on
 /// the API (pay-per-token), ~1 hour on Claude Max (subscription). When the
 /// idle gap exceeds this, the cache prefix has likely expired and the next
@@ -178,6 +191,12 @@ pub enum TurnError {
     /// the wake message, the serve loop requeues it so the next turn retries;
     /// no status park.
     SessionNotFound,
+    /// The harness-side idle watchdog killed claude after `turn_idle_secs()` of
+    /// output silence — indicative of an Anthropic API stall (e.g. a multi-retry
+    /// connection storm burning minutes of wall-clock with no stream progress).
+    /// The serve loop parks for `stall_sleep_secs()` and requeues, like the
+    /// rate-limit path — NOT a crash.
+    ApiStall,
     /// A hard failure with no recovery — the serve loop escalates it to the
     /// parent (`send_to_parent`).
     Failed(anyhow::Error),
@@ -204,6 +223,20 @@ pub fn rate_limit_sleep_secs() -> u64 {
     env_u64_positive("HIVE_RATE_LIMIT_SLEEP_SECS", DEFAULT_RATE_LIMIT_SLEEP_SECS)
 }
 
+/// Idle-watchdog window in seconds. Reads `HIVE_TURN_IDLE_SECS`; `0` disables
+/// the watchdog. Absent / unparseable falls back to [`DEFAULT_TURN_IDLE_SECS`].
+#[must_use]
+pub fn turn_idle_secs() -> u64 {
+    env_u64("HIVE_TURN_IDLE_SECS").unwrap_or(DEFAULT_TURN_IDLE_SECS)
+}
+
+/// How long to park after an `ApiStall` before requeueing. Reads
+/// `HIVE_STALL_SLEEP_SECS` if set to a valid positive integer.
+#[must_use]
+pub fn stall_sleep_secs() -> u64 {
+    env_u64_positive("HIVE_STALL_SLEEP_SECS", DEFAULT_STALL_SLEEP_SECS)
+}
+
 /// Resolve the effective context-window size for watermark calculations.
 /// Priority order (first wins):
 /// 1. API-reported window from the last `result` event's `modelUsage.*.contextWindow`.
@@ -445,6 +478,16 @@ pub fn emit_turn_end(bus: &Bus, outcome: &TurnOutcome) {
             });
             tracing::warn!("turn session-not-found; requeueing message");
         }
+        Err(TurnError::ApiStall) => {
+            bus.emit(LiveEvent::TurnEnd {
+                ok: false,
+                note: Some(format!(
+                    "claude killed after {}s of output silence — API stall suspected, parking + requeueing",
+                    turn_idle_secs()
+                )),
+            });
+            tracing::warn!("turn killed: API stall (idle watchdog)");
+        }
         Err(TurnError::Failed(e)) => {
             let note = format!("{e:#}");
             bus.emit(LiveEvent::TurnEnd {
@@ -542,6 +585,12 @@ fn claude_config(bus: &Bus, files: &TurnFiles) -> Config {
         tools: Some(mcp_config::builtin_tools_arg()),
         allowed_tools: Some(mcp_config::allowed_tools_arg()),
         add_dirs,
+        // Idle watchdog: `0` disables (wait indefinitely), any positive value
+        // caps output silence. Policy lives here; the driver just enforces it.
+        idle_timeout: match turn_idle_secs() {
+            0 => None,
+            secs => Some(std::time::Duration::from_secs(secs)),
+        },
         ..Config::default()
     }
 }
@@ -557,6 +606,7 @@ fn error_to_turn(err: hive_claude::Error) -> TurnOutcome {
         Error::RateLimited => Err(TurnError::RateLimited),
         Error::AuthFailed => Err(TurnError::AuthFailed),
         Error::SessionNotFound => Err(TurnError::SessionNotFound),
+        Error::IdleTimeout => Err(TurnError::ApiStall),
         other => Err(TurnError::Failed(other.into())),
     }
 }