harness: flip into needs_login on 401 mid-turn (closes #419)

hive-ag3nt/src/turn.rs

@@ -47,6 +47,21 @@ const RATE_LIMIT_MARKERS: &[&str] = &[
     "Request rate limit exceeded",
 ];
 
+/// Substrings that indicate the Anthropic API rejected the request as
+/// unauthenticated — the OAuth session in `/root/.claude/` has expired
+/// or been revoked. Surfaced as `TurnOutcome::AuthFailed`, which the
+/// harness uses to flip the container into `needs_login_idle` so the
+/// dashboard's re-auth flow takes over (closes #419). Matched against
+/// both stdout JSON `error` events and stderr; the markers come from
+/// claude-code's `api_retry` events (`{"error":"authentication_failed",
+/// "error_status":401,...}`) and the human-readable
+/// "Failed to authenticate. API Error: 401" line claude prints on giveup.
+const AUTH_FAIL_MARKERS: &[&str] = &[
+    "\"error\":\"authentication_failed\"",
+    "\"error_status\":401",
+    "Failed to authenticate. API Error: 401",
+];
+
 /// How long to sleep after detecting a rate-limit before re-entering the
 /// serve loop. Overridable via `HIVE_RATE_LIMIT_SLEEP_SECS`. Default is
 /// 5 minutes — enough for most short-lived throttles; the operator can
@@ -196,6 +211,11 @@ pub enum TurnOutcome {
     /// usage cap, or exhausted credit balance. The serve loop should park for
     /// `rate_limit_sleep_secs()` and retry — NOT bubble up as a crash.
     RateLimited,
+    /// The Anthropic API rejected the request with 401 (OAuth session
+    /// expired or revoked). The serve loop should flip the container
+    /// into `needs_login_idle` and stop driving turns until the
+    /// operator re-auths via the per-agent web UI (closes #419).
+    AuthFailed,
     Failed(anyhow::Error),
 }
 
@@ -343,6 +363,9 @@ async fn maybe_checkpoint_and_compact(files: &TurnFiles, bus: &Bus) -> bool {
         TurnOutcome::RateLimited => bus.emit(LiveEvent::Note {
             text: "checkpoint turn was rate-limited — compacting anyway".into(),
         }),
+        TurnOutcome::AuthFailed => bus.emit(LiveEvent::Note {
+            text: "checkpoint turn hit 401 — skipping compaction, parking for re-login".into(),
+        }),
         TurnOutcome::Failed(e) => bus.emit(LiveEvent::Note {
             text: format!("checkpoint turn failed ({e:#}) — compacting anyway"),
         }),
@@ -413,6 +436,13 @@ pub fn emit_turn_end(bus: &Bus, outcome: &TurnOutcome) {
             });
             tracing::warn!("turn rate-limited");
         }
+        TurnOutcome::AuthFailed => {
+            bus.emit(LiveEvent::TurnEnd {
+                ok: false,
+                note: Some("authentication failed (401) — waiting for re-login".into()),
+            });
+            tracing::warn!("turn auth-failed (401)");
+        }
         TurnOutcome::Failed(e) => {
             let note = format!("{e:#}");
             bus.emit(LiveEvent::TurnEnd {
@@ -461,8 +491,9 @@ pub async fn wait_for_login(
 /// doesn't stall mid-turn — hyperhive owns compaction.
 pub async fn run_turn(prompt: &str, files: &TurnFiles, bus: &Bus) -> TurnOutcome {
     match run_claude(prompt, files, bus).await {
-        Ok((too_long, _)) if too_long => TurnOutcome::PromptTooLong,
-        Ok((_, rate_limited)) if rate_limited => TurnOutcome::RateLimited,
+        Ok((too_long, _, _)) if too_long => TurnOutcome::PromptTooLong,
+        Ok((_, rate_limited, _)) if rate_limited => TurnOutcome::RateLimited,
+        Ok((_, _, auth_failed)) if auth_failed => TurnOutcome::AuthFailed,
         Ok(_) => TurnOutcome::Ok,
         Err(e) => TurnOutcome::Failed(e),
     }
@@ -483,7 +514,7 @@ pub async fn compact_session(files: &TurnFiles, bus: &Bus) -> Result<()> {
     bus.emit(LiveEvent::Note {
         text: "context overflow — running /compact on the persistent session".into(),
     });
-    let (_, _) = run_claude("/compact", files, bus).await?;
+    let (_, _, _) = run_claude("/compact", files, bus).await?;
     bus.emit(LiveEvent::Note {
         text: "/compact done".into(),
     });
@@ -491,7 +522,7 @@ pub async fn compact_session(files: &TurnFiles, bus: &Bus) -> Result<()> {
 }
 
 #[allow(clippy::too_many_lines)]
-async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool, bool)> {
+async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool, bool, bool)> {
     // Keep the last STDERR_TAIL_LINES of stderr so a non-zero exit can
     // include real context in the bail message (and downstream in the
     // failure notification to the manager) instead of just "exit 1".
@@ -547,10 +578,13 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool,
 
     let prompt_too_long = Arc::new(AtomicBool::new(false));
     let rate_limited = Arc::new(AtomicBool::new(false));
+    let auth_failed = Arc::new(AtomicBool::new(false));
     let flag_out = prompt_too_long.clone();
     let flag_err = prompt_too_long.clone();
     let rate_out = rate_limited.clone();
     let rate_err = rate_limited.clone();
+    let auth_out = auth_failed.clone();
+    let auth_err = auth_failed.clone();
     let bus_out = bus.clone();
     let bus_err = bus.clone();
     let pump_stdout = tokio::spawn(async move {
@@ -566,6 +600,13 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool,
             if line.contains(PROMPT_TOO_LONG_MARKER) {
                 flag_out.store(true, Ordering::Relaxed);
             }
+            // Auth-fail check happens on the raw line first so we
+            // catch both the `api_retry` JSON events (which can land
+            // before they're fully parseable) and any stderr-shaped
+            // text that snuck onto stdout.
+            if AUTH_FAIL_MARKERS.iter().any(|m| line.contains(m)) {
+                auth_out.store(true, Ordering::Relaxed);
+            }
             match serde_json::from_str::<serde_json::Value>(&line) {
                 Ok(v) => {
                     // Rate-limit detection: only fire on JSON `error` events,
@@ -628,6 +669,9 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool,
             if RATE_LIMIT_MARKERS.iter().any(|m| line.contains(m)) {
                 rate_err.store(true, Ordering::Relaxed);
             }
+            if AUTH_FAIL_MARKERS.iter().any(|m| line.contains(m)) {
+                auth_err.store(true, Ordering::Relaxed);
+            }
             // Mirror to journald so post-mortems work without the web UI
             // or the events sqlite. The bus event is what the dashboard
             // renders; the tracing line is what `journalctl -M <c> -b`
@@ -649,7 +693,8 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool,
     let _ = pump_stderr.await;
     let too_long = prompt_too_long.load(Ordering::Relaxed);
     let is_rate_limited = rate_limited.load(Ordering::Relaxed);
-    if !status.success() && !too_long && !is_rate_limited {
+    let is_auth_failed = auth_failed.load(Ordering::Relaxed);
+    if !status.success() && !too_long && !is_rate_limited && !is_auth_failed {
         let tail = stderr_tail.lock().unwrap();
         if tail.is_empty() {
             bail!("claude exited {status} (no stderr)");
@@ -657,5 +702,5 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool,
         let tail_str = tail.iter().cloned().collect::<Vec<_>>().join("\n");
         bail!("claude exited {status}\nstderr tail:\n{tail_str}");
     }
-    Ok((too_long, is_rate_limited))
+    Ok((too_long, is_rate_limited, is_auth_failed))
 }