vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(#2019): pin harness claude session via --resume so choom can't clobber it

Browse source
This commit is contained in:
damocles 2026-06-27 14:57:01 +02:00 • committed by mara
commit c9115bdbf5
1 changed files with 79 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

90
hive-ag3nt/src/turn.rs
View file

@ -63,6 +63,19 @@ const AUTH_FAIL_MARKERS: &[&str] = &[
"Failed to authenticate. API Error: 401",
];
/// Substring claude-code emits when `--resume <id>` is handed a session id
/// that doesn't exist in this cwd's project (stale persisted id, or a crash
/// before the first turn ever completed a `.jsonl`). On a hit we clear the
/// persisted id so the NEXT turn starts a fresh session and re-captures —
/// the agent self-heals instead of failing `--resume` forever.
const SESSION_NOT_FOUND_MARKER: &str = "No conversation found with session ID";
/// Name of the harness-owned file under `paths::harness_dir()` that holds
/// the claude session id to resume. Written after every turn with the id
/// claude reported on its stream (the id can change across resume turns in
/// some claude-code versions, so we always rewrite with the last-seen value).
const CLAUDE_SESSION_ID_FILE: &str = "claude-session-id";
/// How long to sleep after detecting a rate-limit before re-entering the
/// serve loop. Overridable via `HIVE_RATE_LIMIT_SLEEP_SECS`. Default is
/// 5 minutes — enough for most short-lived throttles; the operator can
@ -552,8 +565,11 @@ fn session_refreshed(prev: DirSnapshot, now: DirSnapshot) -> bool {
/// Spawn `claude` for one turn and pump `stream-json` stdout into the
/// live event bus. Prompt goes over stdin (variadic
/// `--allowedTools`/`--tools` would otherwise eat a trailing positional
/// prompt). The session is persistent across turns via `--continue` and
/// claude's in-session auto-compact is disabled via the managed
/// prompt). The session is persistent across turns via `--resume <id>`
/// against the harness's own captured session id (NOT bare `--continue`,
/// which resumes the *latest* session in this cwd and so lets a `choom`
/// session hijack the live harness context). claude's in-session
/// auto-compact is disabled via the managed
/// settings at `/etc/claude-code/managed-settings.json` so it doesn't
/// stall mid-turn — hyperhive owns compaction.
pub async fn run_turn(prompt: &str, files: &TurnFiles, bus: &Bus) -> TurnOutcome {
@ -624,16 +640,32 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool,
const STDERR_TAIL_LINES: usize = 20;
let model = bus.model();
let effort = bus.effort();
let resume = !bus.take_skip_continue();
if !resume {
// Flag the fresh session so the bin loop mints a new `sessions`
// row + stamps its id onto this turn's stats (and subsequent
// turns until the next fresh start).
// Resolve which claude session to resume. We NEVER pass bare
// `--continue`: that resumes the latest session in this cwd, which a
// `choom` invocation (same cwd) can hijack, wiping the harness context.
// Instead we `--resume <id>` against the id claude reported on a prior
// turn, persisted under `harness_dir()/claude-session-id`.
let persist_path = crate::paths::harness_dir().join(CLAUDE_SESSION_ID_FILE);
let resume_id: Option<String> = if bus.take_skip_continue() {
// Fresh session requested: mint a new one (no --resume). Flag it so
// the bin loop mints a new `sessions` row + stamps its id onto this
// turn's stats. Drop any stale persisted id — the new id claude
// reports this turn is captured + written below.
bus.mark_fresh_session();
let _ = std::fs::remove_file(&persist_path);
bus.emit(LiveEvent::Note {
text: "fresh session (--continue suppressed for this turn)".into(),
text: "fresh session (continue suppressed for this turn)".into(),
});
}
None
} else {
// Continue: resume OUR captured id. Absent (first turn / just
// self-healed from a stale id) → fall through to a fresh session
// and capture the new id below.
match std::fs::read_to_string(&persist_path) {
Ok(s) if !s.trim().is_empty() => Some(s.trim().to_string()),
_ => None,
}
};
let mut cmd = Command::new("claude");
// Spawn inside the agent's state dir so relative paths in tool calls
// (Read foo.md, Bash ls, Write notes.md) land in the durable dir
@ -651,8 +683,8 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool,
.arg(&model)
.arg("--effort")
.arg(&effort);
if resume {
cmd.arg("--continue");
if let Some(id) = &resume_id {
cmd.arg("--resume").arg(id);
}
cmd.arg("--system-prompt-file").arg(&files.system_prompt);
cmd.arg("--mcp-config")
@ -679,12 +711,21 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool,
let prompt_too_long = Arc::new(AtomicBool::new(false));
let rate_limited = Arc::new(AtomicBool::new(false));
let auth_failed = Arc::new(AtomicBool::new(false));
// `--resume` against a stale/missing id: clear the persist file so the
// next turn self-heals into a fresh session.
let session_not_found = Arc::new(AtomicBool::new(false));
// Last `session_id` claude reported on its stream this turn; persisted
// after the child exits so the next turn `--resume`s it.
let session_id_seen = Arc::new(Mutex::new(None::<String>));
let flag_out = prompt_too_long.clone();
let flag_err = prompt_too_long.clone();
let rate_out = rate_limited.clone();
let rate_err = rate_limited.clone();
let auth_out = auth_failed.clone();
let auth_err = auth_failed.clone();
let notfound_out = session_not_found.clone();
let notfound_err = session_not_found.clone();
let session_id_out = session_id_seen.clone();
let bus_out = bus.clone();
let bus_err = bus.clone();
let pump_stdout = tokio::spawn(async move {
@ -712,7 +753,19 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool,
if AUTH_FAIL_MARKERS.iter().any(|m| line.contains(m)) {
auth_out.store(true, Ordering::Relaxed);
}
if line.contains(SESSION_NOT_FOUND_MARKER) {
notfound_out.store(true, Ordering::Relaxed);
}
if let Ok(v) = serde_json::from_str::<serde_json::Value>(&line) {
// Track the session id claude reports (init + result events
// both carry it). Persisted after exit so the next turn
// `--resume`s it; re-captured each turn since the id can
// change across resumes in some claude-code versions.
if let Some(sid) = v.get("session_id").and_then(|s| s.as_str())
&& !sid.is_empty()
{
*session_id_out.lock().unwrap() = Some(sid.to_string());
}
// Rate-limit detection: only fire on JSON `error` events,
// not on arbitrary text content. An agent discussing a past
// rate limit in its response would otherwise trigger a false
@ -779,6 +832,9 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool,
if AUTH_FAIL_MARKERS.iter().any(|m| line.contains(m)) {
auth_err.store(true, Ordering::Relaxed);
}
if line.contains(SESSION_NOT_FOUND_MARKER) {
notfound_err.store(true, Ordering::Relaxed);
}
// Mirror to journald so post-mortems work without the web UI
// or the events sqlite. The bus event is what the dashboard
// renders; the tracing line is what `journalctl -M <c> -b`
@ -801,6 +857,18 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool,
let too_long = prompt_too_long.load(Ordering::Relaxed);
let is_rate_limited = rate_limited.load(Ordering::Relaxed);
let is_auth_failed = auth_failed.load(Ordering::Relaxed);
// Session-id bookkeeping. On a stale/missing `--resume` id, drop the
// persist file so the next turn starts fresh and self-heals. Otherwise
// rewrite it with the id claude reported this turn (handles the id
// changing across resumes in some claude-code versions).
if session_not_found.load(Ordering::Relaxed) {
let _ = std::fs::remove_file(&persist_path);
} else if let Some(sid) = session_id_seen.lock().unwrap().clone() {
if let Some(parent) = persist_path.parent() {
let _ = std::fs::create_dir_all(parent);
}
let _ = std::fs::write(&persist_path, sid);
}
if !status.success() && !too_long && !is_rate_limited && !is_auth_failed {
let tail = stderr_tail.lock().unwrap();
if tail.is_empty() {