329 lines
15 KiB
Rust
329 lines
15 KiB
Rust
//! Sentinel detection: mapping claude-code CLI output onto [`crate::Error`]
|
|
//! variants.
|
|
//!
|
|
//! These marker strings are claude-code CLI knowledge, not app knowledge. They
|
|
//! are empirically stable across CLI versions; if one drifts the run degrades
|
|
//! gracefully (a clean turn, or a hard [`crate::Error::Exit`] on a non-zero
|
|
//! exit) rather than misbehaving.
|
|
//!
|
|
//! Stdout events are parsed into a typed [`StreamEvent`] (a serde-tagged enum)
|
|
//! rather than walked as a `serde_json::Value` — the event-type dispatch and
|
|
//! the terminal result event's fields (`is_error`, `api_error_status`) are
|
|
//! typed, so a shape drift is a clean fallback instead of a silent
|
|
//! mis-detection. The marker *strings* still gate detection (they're the
|
|
//! API's error text); typing removes the fragile field-probing around them.
|
|
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
|
|
use serde::Deserialize;
|
|
|
|
/// Emitted when the prompt/context exceeds the model's window.
|
|
const PROMPT_TOO_LONG_MARKER: &str = "Prompt is too long";
|
|
|
|
/// Substrings indicating the API refused for rate-limit / usage-cap / credit
|
|
/// reasons. On stdout these are only trusted inside a JSON `error` event (see
|
|
/// [`Sentinels::scan_stdout_json`] / [`Sentinels::scan_rate_limit_text`]) so a
|
|
/// model *discussing* a rate limit in prose can't trigger a false positive.
|
|
const RATE_LIMIT_MARKERS: [&str; 5] = [
|
|
"rate_limit_error",
|
|
"overloaded_error",
|
|
"Credit balance is too low",
|
|
"Usage limit reached",
|
|
"Request rate limit exceeded",
|
|
];
|
|
|
|
/// Substrings indicating the API rejected the request as unauthenticated (401)
|
|
/// — an expired/revoked OAuth session. Sourced from claude-code's `api_retry`
|
|
/// JSON events and its human-readable give-up line.
|
|
const AUTH_FAIL_MARKERS: [&str; 3] = [
|
|
"\"error\":\"authentication_failed\"",
|
|
"\"error_status\":401",
|
|
"Failed to authenticate. API Error: 401",
|
|
];
|
|
|
|
/// Substrings indicating `--resume` could not resolve its target: no session
|
|
/// with the given title, or no conversation with the given id.
|
|
const SESSION_NOT_FOUND_MARKERS: [&str; 2] = [
|
|
"does not match any session title",
|
|
"No conversation found with session ID",
|
|
];
|
|
|
|
/// The claude-code stdout stream-json events we classify, internally tagged
|
|
/// on `type`. Typing the dispatch (and the `result` event's fields below)
|
|
/// replaces walking a `serde_json::Value` with `get("field").and_then(...)`,
|
|
/// so a field-shape drift is a compile error / a clean fallback rather than a
|
|
/// silent mis-detection. Unknown event types fall into [`StreamEvent::Other`]
|
|
/// (scanned raw like any other control event).
|
|
#[derive(Deserialize)]
|
|
#[serde(tag = "type", rename_all = "snake_case")]
|
|
enum StreamEvent {
|
|
/// Model-authored turn output — never scanned (it can quote any marker).
|
|
Assistant,
|
|
/// Model-authored tool results — never scanned.
|
|
User,
|
|
/// The terminal result event (see [`ResultEvent`]).
|
|
Result(ResultEvent),
|
|
/// A control `error` event — the only event a rate-limit marker is
|
|
/// trusted on (a model *discussing* a rate limit can't forge this type).
|
|
Error,
|
|
/// `system` and any other/unknown control event — claude-authored, so
|
|
/// scanned raw for the failure markers.
|
|
#[serde(other)]
|
|
Other,
|
|
}
|
|
|
|
/// The terminal `result` event's fields we act on. `subtype` is deliberately
|
|
/// absent: it reads `"success"` even on a hard failure, so it's not a usable
|
|
/// error signal — `is_error` is the discriminator (the CLI sets it from the
|
|
/// real outcome; the model can't forge it). On `is_error` the `result` /
|
|
/// `error` fields carry claude-code's own failure text.
|
|
#[derive(Deserialize)]
|
|
struct ResultEvent {
|
|
#[serde(default)]
|
|
is_error: bool,
|
|
/// claude-code's failure text on `is_error` (a genuine "Prompt is too
|
|
/// long" / auth message); the model's final answer on success.
|
|
#[serde(default)]
|
|
result: Option<String>,
|
|
/// Structured error payload (a string, or an object like
|
|
/// `{"type":"rate_limit_error"}`) present on some failures.
|
|
#[serde(default)]
|
|
error: Option<serde_json::Value>,
|
|
/// API status on an auth failure (e.g. `401`) — a typed signal that
|
|
/// doesn't depend on the human-readable string landing in `result`.
|
|
#[serde(default)]
|
|
api_error_status: Option<u32>,
|
|
}
|
|
|
|
/// Shared, lock-free sentinel flags accumulated while both output streams are
|
|
/// pumped concurrently. Read once after the child exits.
|
|
#[derive(Default)]
|
|
pub(crate) struct Sentinels {
|
|
prompt_too_long: AtomicBool,
|
|
rate_limited: AtomicBool,
|
|
auth_failed: AtomicBool,
|
|
session_not_found: AtomicBool,
|
|
}
|
|
|
|
impl Sentinels {
|
|
/// Scan a CLI-authored line — stderr, or a non-JSON stdout line — for every
|
|
/// marker. These bytes are always claude-code's own output, never model
|
|
/// conversation, so all markers are trusted.
|
|
pub(crate) fn scan_cli_line(&self, line: &str) {
|
|
self.scan_failure_markers(line);
|
|
if RATE_LIMIT_MARKERS.iter().any(|m| line.contains(m)) {
|
|
self.rate_limited.store(true, Ordering::Relaxed);
|
|
}
|
|
}
|
|
|
|
/// Scan a parsed stdout JSON event. **Skips model-authored `assistant` /
|
|
/// `user` message events**, whose serialized content can quote any marker
|
|
/// verbatim (an agent discussing this very code, say) — a false positive
|
|
/// that would otherwise trip a needless compact/retry or a spurious
|
|
/// auth/session error. Every real signal here is emitted *instead of* a
|
|
/// model turn (the API rejected the prompt, the auth failed, or `--resume`
|
|
/// missed before any inference), so it can only appear on a control event
|
|
/// (`error` / `result` / `system`) or as raw non-JSON text — never inside
|
|
/// an assistant/user message. This holds whatever exact shape claude-code
|
|
/// uses for the message, so the gate can't suppress a genuine signal.
|
|
pub(crate) fn scan_stdout_json(&self, event: &serde_json::Value, raw: &str) {
|
|
// Type the event instead of walking the `Value`. A parse failure
|
|
// (missing / non-string `type`) is rare and falls through to a raw
|
|
// scan — the safe default, matching the old "unknown type" branch.
|
|
match StreamEvent::deserialize(event) {
|
|
// Model-authored output — its serialized content can quote any
|
|
// marker verbatim (an agent discussing this very code), so it is
|
|
// never scanned. A genuine signal is emitted *instead of* a model
|
|
// turn, so it never rides an assistant/user event.
|
|
Ok(StreamEvent::Assistant | StreamEvent::User) => {}
|
|
|
|
// The terminal result event. On success its `result` field is the
|
|
// model's final answer (same trust level as an assistant message
|
|
// — must NOT scan). On a FAILED result (`is_error`) the `result` /
|
|
// `error` fields carry claude-code's OWN failure text — the real
|
|
// event (verified against captured stream-json) is
|
|
// {"type":"result","is_error":true,"subtype":"success",
|
|
// "result":"Prompt is too long","terminal_reason":"blocking_limit"}
|
|
// — note `subtype` is "success" even on a hard failure, so
|
|
// `is_error` is the discriminator (the model can't forge it).
|
|
Ok(StreamEvent::Result(r)) => {
|
|
if r.is_error {
|
|
if let Some(text) = &r.result {
|
|
self.scan_failure_markers(text);
|
|
}
|
|
if let Some(err) = &r.error {
|
|
// `to_string()` re-serializes the JSON value (a string
|
|
// value round-trips to `"...quoted..."`), but the marker
|
|
// substring still appears inside the quoted form, so the
|
|
// scan finds it either way.
|
|
self.scan_failure_markers(&err.to_string());
|
|
}
|
|
// Typed 401: an auth failure that set the status code but
|
|
// may not have put the human-readable string in `result`.
|
|
if r.api_error_status == Some(401) {
|
|
self.auth_failed.store(true, Ordering::Relaxed);
|
|
}
|
|
}
|
|
}
|
|
|
|
// A control `error` event — claude-authored end to end, and the
|
|
// only place a rate-limit marker is trusted.
|
|
Ok(StreamEvent::Error) => {
|
|
self.scan_failure_markers(raw);
|
|
if RATE_LIMIT_MARKERS.iter().any(|m| raw.contains(m)) {
|
|
self.rate_limited.store(true, Ordering::Relaxed);
|
|
}
|
|
}
|
|
|
|
// `system` / unknown control events (claude-authored) → scan raw.
|
|
Ok(StreamEvent::Other) | Err(_) => self.scan_failure_markers(raw),
|
|
}
|
|
}
|
|
|
|
/// The prompt-too-long / auth-failed / session-not-found markers. Callers
|
|
/// gate *where* this runs (see `scan_cli_line` / `scan_stdout_json`).
|
|
fn scan_failure_markers(&self, line: &str) {
|
|
if line.contains(PROMPT_TOO_LONG_MARKER) {
|
|
self.prompt_too_long.store(true, Ordering::Relaxed);
|
|
}
|
|
if AUTH_FAIL_MARKERS.iter().any(|m| line.contains(m)) {
|
|
self.auth_failed.store(true, Ordering::Relaxed);
|
|
}
|
|
if SESSION_NOT_FOUND_MARKERS.iter().any(|m| line.contains(m)) {
|
|
self.session_not_found.store(true, Ordering::Relaxed);
|
|
}
|
|
}
|
|
|
|
/// The recognized-sentinel error, if any fired — `None` means no sentinel
|
|
/// (so the run either completed or failed hard on its exit code). The
|
|
/// sentinels keep a fixed priority (too-long > rate > auth); a
|
|
/// session-not-found can only arise on a resume that made no model call,
|
|
/// so it never coincides with the others.
|
|
pub(crate) fn soft_error(&self) -> Option<crate::Error> {
|
|
use crate::Error;
|
|
if self.prompt_too_long.load(Ordering::Relaxed) {
|
|
Some(Error::PromptTooLong)
|
|
} else if self.rate_limited.load(Ordering::Relaxed) {
|
|
Some(Error::RateLimited)
|
|
} else if self.auth_failed.load(Ordering::Relaxed) {
|
|
Some(Error::AuthFailed)
|
|
} else if self.session_not_found.load(Ordering::Relaxed) {
|
|
Some(Error::SessionNotFound)
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::Error;
|
|
|
|
fn json(raw: &str) -> serde_json::Value {
|
|
serde_json::from_str(raw).unwrap()
|
|
}
|
|
|
|
#[test]
|
|
fn assistant_content_quoting_marker_is_ignored() {
|
|
// An agent discussing this code emits the marker verbatim in an
|
|
// assistant message — must NOT trip a sentinel.
|
|
let s = Sentinels::default();
|
|
let raw = r#"{"type":"assistant","message":{"content":[{"type":"text","text":"the CLI prints Prompt is too long on overflow"}]}}"#;
|
|
s.scan_stdout_json(&json(raw), raw);
|
|
assert!(s.soft_error().is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn genuine_failure_result_is_detected() {
|
|
// Both fixtures are the real claude-code failure shape, verified against
|
|
// captured stream-json: the marker lives in the `result` field with
|
|
// `is_error: true` and — counterintuitively — `subtype: "success"`. The
|
|
// earlier unconditional `result` scrub blinded this; `is_error` is the
|
|
// discriminator that restores it (and the model can't forge it).
|
|
let s = Sentinels::default();
|
|
let raw = r#"{"type":"result","is_error":true,"subtype":"success","result":"Prompt is too long","terminal_reason":"blocking_limit","stop_reason":"stop_sequence"}"#;
|
|
s.scan_stdout_json(&json(raw), raw);
|
|
assert!(matches!(s.soft_error(), Some(Error::PromptTooLong)));
|
|
|
|
let s = Sentinels::default();
|
|
let raw = r#"{"type":"result","is_error":true,"subtype":"success","result":"Failed to authenticate. API Error: 401 Invalid authentication credentials","api_error_status":401}"#;
|
|
s.scan_stdout_json(&json(raw), raw);
|
|
assert!(matches!(s.soft_error(), Some(Error::AuthFailed)));
|
|
}
|
|
|
|
#[test]
|
|
fn typed_api_error_status_401_trips_auth_without_string_marker() {
|
|
// The typed `api_error_status` field catches an auth failure even when
|
|
// the human-readable 401 string isn't in `result` — a signal the old
|
|
// substring scan would have missed.
|
|
let s = Sentinels::default();
|
|
let raw = r#"{"type":"result","is_error":true,"subtype":"success","result":"request failed","api_error_status":401}"#;
|
|
s.scan_stdout_json(&json(raw), raw);
|
|
assert!(matches!(s.soft_error(), Some(Error::AuthFailed)));
|
|
}
|
|
|
|
#[test]
|
|
fn error_payload_object_on_result_is_scanned() {
|
|
// A failure result can carry the marker in a structured `error`
|
|
// payload rather than `result`; the typed `error: Value` is stringified
|
|
// and scanned.
|
|
let s = Sentinels::default();
|
|
let raw = r#"{"type":"result","is_error":true,"error":"Prompt is too long","result":"ok"}"#;
|
|
s.scan_stdout_json(&json(raw), raw);
|
|
assert!(matches!(s.soft_error(), Some(Error::PromptTooLong)));
|
|
}
|
|
|
|
#[test]
|
|
fn result_field_model_text_quoting_marker_is_ignored() {
|
|
// The terminal `result` event's `result` field is the model's final
|
|
// answer. A marker quoted there (the model discussing this very code,
|
|
// or echoing an API error string) must NOT trip a sentinel — that was
|
|
// a turn-kill DoS. Covers prompt-too-long, auth, and session markers.
|
|
for marker in [
|
|
"Prompt is too long",
|
|
"Failed to authenticate. API Error: 401",
|
|
"does not match any session title",
|
|
] {
|
|
let s = Sentinels::default();
|
|
let raw = format!(
|
|
r#"{{"type":"result","subtype":"success","is_error":false,"result":"the harness scans stdout for {marker} — see classify.rs"}}"#
|
|
);
|
|
s.scan_stdout_json(&json(&raw), &raw);
|
|
assert!(
|
|
s.soft_error().is_none(),
|
|
"marker {marker:?} in the model-authored result field must be ignored"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn result_control_field_still_trips_even_with_clean_result_text() {
|
|
// Scrubbing `result` must not blind us to a genuine signal in a
|
|
// control field of the same event.
|
|
let s = Sentinels::default();
|
|
let raw = r#"{"type":"result","subtype":"error","is_error":true,"error":"Failed to authenticate. API Error: 401","result":"ok"}"#;
|
|
s.scan_stdout_json(&json(raw), raw);
|
|
assert!(matches!(s.soft_error(), Some(Error::AuthFailed)));
|
|
}
|
|
|
|
#[test]
|
|
fn raw_non_json_marker_is_detected() {
|
|
let s = Sentinels::default();
|
|
s.scan_cli_line("API Error: Prompt is too long");
|
|
assert!(matches!(s.soft_error(), Some(Error::PromptTooLong)));
|
|
}
|
|
|
|
#[test]
|
|
fn rate_limit_still_only_on_error_event() {
|
|
// A non-error control event mentioning the marker must not trip it.
|
|
let s = Sentinels::default();
|
|
let raw = r#"{"type":"result","summary":"we hit a rate_limit_error earlier"}"#;
|
|
s.scan_stdout_json(&json(raw), raw);
|
|
assert!(s.soft_error().is_none());
|
|
// A genuine error event does.
|
|
let raw2 = r#"{"type":"error","error":{"type":"rate_limit_error"}}"#;
|
|
s.scan_stdout_json(&json(raw2), raw2);
|
|
assert!(matches!(s.soft_error(), Some(Error::RateLimited)));
|
|
}
|
|
}
|