diff --git a/Cargo.lock b/Cargo.lock index e8c42a52..3ff35e3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1835,9 +1835,9 @@ dependencies = [ [[package]] name = "hive-claude" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c87ac627823f7bfc38478d7494e614e3f680dda6007dbcd306af51d3eed3bb3" +checksum = "4d7c5638f8f59e438d6416b3048709c9b3d6a10c23872ecf3825e3fe4c1bd7e8" dependencies = [ "libc", "serde", diff --git a/Cargo.toml b/Cargo.toml index ab3840c7..7f73e0ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,7 +97,7 @@ hive-jobq-metrics = { path = "hive-jobq-metrics" } hive-jobq-wire = { path = "hive-jobq-wire" } hive-log = { path = "hive-log" } hive-core-agent-sock = { path = "hive-core-agent-sock" } -hive-claude = "0.1.3" +hive-claude = "0.1.4" hive-host-sock = { path = "hive-host-sock" } hive-priv-sock = { path = "hive-priv-sock" } hive-runtime = { path = "hive-runtime" } diff --git a/docs/turn-loop/README.md b/docs/turn-loop/README.md index 961f3a6d..ee2327b1 100644 --- a/docs/turn-loop/README.md +++ b/docs/turn-loop/README.md @@ -142,8 +142,8 @@ else a `TurnError`) drives the post-claude branch: | `Err(RateLimited)` | sleep `HIVE_RATE_LIMIT_SLEEP_SECS` (default 300), requeue inflight, status back to `online` | | `Err(AuthFailed)` | emit `needs_login_idle` sentinel, requeue inflight, park in `wait_for_login` | | `Err(SessionNotFound)` | resume + create self-heal both missed ("shouldn't happen"); requeue inflight so the next turn creates fresh — no status park, message not dropped | -| `Err(ApiStall)` | idle watchdog killed claude after `HIVE_TURN_IDLE_SECS` (default 600) of output silence; sleep `HIVE_STALL_SLEEP_SECS` (default 60), requeue inflight, status back to `online` | -| `Err(AgentStall(note))` | the same, for an ACP agent: after `HIVE_TURN_IDLE_SECS` with no `session/update` the harness sent the agent `session/cancel`, and killed it if it ignored that; `note` says which | +| `Err(ApiStall)` | idle watchdog sent claude SIGINT after `HIVE_TURN_IDLE_SECS` (default 600) of output silence and claude exited; sleep `HIVE_STALL_SLEEP_SECS` (default 60), requeue inflight, status back to `online` | +| `Err(AgentStall(note))` | the same, for an ACP agent: after `HIVE_TURN_IDLE_SECS` with no `session/update` the harness sent the agent `session/cancel`, and SIGINT if it ignored that; `note` says which | | `Err(Failed(err))` | route `[system] \`\` claude turn failed:\n` to `operator` via `send_to_operator` | `ApiStall` catches an Anthropic API stall — a multi-retry connection storm where @@ -152,7 +152,11 @@ the stream goes silent for minutes. On `claude` the idle watchdog lives in resets on every stdout line, so a large but still-streaming turn is never cut — only complete output silence for the window trips it. The harness sets the window from `HIVE_TURN_IDLE_SECS` (`0` disables) and maps the driver's -`Error::IdleTimeout` onto `TurnError::ApiStall`. +`Error::IdleTimeout` onto `TurnError::ApiStall`. The watchdog sends claude's +process group SIGINT, so claude stops the commands it started, and then +waits for claude to exit. Nothing escalates to SIGKILL: a claude that +ignores the SIGINT keeps the turn open until someone kills it or the unit +stops. `hive-runtime` applies the same window to an ACP agent's turns. That's also how a provider HTTP 429 ends an ACP turn when the agent retries it without diff --git a/docs/web-ui/agent.md b/docs/web-ui/agent.md index ea836349..50ef0adf 100644 --- a/docs/web-ui/agent.md +++ b/docs/web-ui/agent.md @@ -229,7 +229,7 @@ Slash commands today: - `/clear` — wipe the local terminal view (server history kept). - `/cancel` — `POST /api/cancel` → SIGINT to the turn's claude and every command it started (its process group), emits a Note. On an ACP agent - it sends the agent `session/cancel` instead, and kills it if the turn + it sends the agent `session/cancel` instead, and SIGINT if the turn hasn't ended 10s later. Also surfaces as a `■ cancel turn` button in the state row while state=thinking. - `/compact` — `POST /api/compact` → sets the deferred diff --git a/hive-agent/src/turn.rs b/hive-agent/src/turn.rs index ee970c70..3bcc338f 100644 --- a/hive-agent/src/turn.rs +++ b/hive-agent/src/turn.rs @@ -50,8 +50,8 @@ const DEFAULT_SESSION_TITLE: &str = "hive-session"; /// capacity limits. const DEFAULT_RATE_LIMIT_SLEEP_SECS: u64 = 300; -/// Idle watchdog window: kill claude and surface `ApiStall` if it produces no -/// stdout for this long. The timer resets on every stdout line, so a large but +/// Idle watchdog window: send claude SIGINT and surface `ApiStall` once it +/// exits, if it produces no stdout for this long. The timer resets on every stdout line, so a large but /// still-streaming turn is never cut — only a complete silence trips it. /// Default is 10 minutes: long enough for legitimate big-context turns, short /// enough to cut a multi-retry Anthropic connection storm (atlas telemetry: @@ -189,14 +189,14 @@ pub enum TurnError { /// the wake message, the serve loop requeues it so the next turn retries; /// no status park. SessionNotFound, - /// The harness-side idle watchdog killed claude after `turn_idle_secs()` of + /// The harness-side idle watchdog stopped claude after `turn_idle_secs()` of /// output silence — indicative of an Anthropic API stall (e.g. a multi-retry /// connection storm burning minutes of wall-clock with no stream progress). /// The serve loop parks for `stall_sleep_secs()` and requeues, like the /// rate-limit path — NOT a crash. ApiStall, /// The ACP backend's idle watchdog stopped a turn that sent nothing for - /// `turn_idle_secs()`: the agent was sent `session/cancel`, and killed if + /// `turn_idle_secs()`: the agent was sent `session/cancel`, and SIGINT if /// it ignored that. Carries what happened. An ACP agent retrying a failing /// provider (e.g. HTTP 429) without reporting it ends up here. Handled /// like [`Self::ApiStall`]. @@ -558,11 +558,11 @@ pub fn emit_turn_end(bus: &Bus, outcome: &TurnOutcome) { bus.emit(LiveEvent::TurnEnd { ok: false, note: Some(format!( - "claude killed after {}s of output silence — API stall suspected, parking + requeueing", + "claude stopped with SIGINT after {}s of output silence — API stall suspected, parking + requeueing", turn_idle_secs() )), }); - tracing::warn!("turn killed: API stall (idle watchdog)"); + tracing::warn!("turn stopped: API stall (idle watchdog)"); } Err(TurnError::AgentStall(note)) => { bus.emit(LiveEvent::TurnEnd {