feat(#2109): harness-side idle watchdog to bail on anthropic api stall storms

This commit is contained in:
damocles 2026-07-07 17:17:45 +02:00 committed by mara
commit 5027068e31
7 changed files with 157 additions and 8 deletions

View file

@ -553,6 +553,22 @@ async fn handle_turn<S: Surface>(
S::requeue_inflight(socket).await;
bus.emit_status("online");
}
if matches!(outcome, Err(turn::TurnError::ApiStall)) {
// Idle watchdog killed claude on a suspected API stall. Park briefly to
// let the API recover, then requeue — same shape as the rate-limit path.
let secs = turn::stall_sleep_secs();
bus.emit_status("api_stall");
bus.emit(LiveEvent::Note {
text: format!(
"API stall timeout — sleeping {secs}s before retry \
(tune HIVE_STALL_SLEEP_SECS; disable the watchdog with HIVE_TURN_IDLE_SECS=0)"
),
});
tracing::warn!(sleep_secs = secs, "API stall; parking before retry");
tokio::time::sleep(Duration::from_secs(secs)).await;
S::requeue_inflight(socket).await;
bus.emit_status("online");
}
if matches!(outcome, Err(turn::TurnError::AuthFailed)) {
bus.emit_status("needs_login_idle");
bus.emit(LiveEvent::Note {