//! Unified hyperhive harness binary. Dispatches one of three subcommands //! (`serve` / `mcp` / `wake`). There is one role: agent. The `Surface` //! trait + `AgentSurface` zero-sized type tag keeps the turn loop //! generic and testable. Architecture lives in //! [`docs/turn-loop.md::Harness binary shape`](../../../docs/turn-loop.md). use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use std::time::Duration; use hive_ag3nt::web_ui::TurnLock; use anyhow::Result; use clap::{Parser, Subcommand}; use hive_ag3nt::events::{Bus, LiveEvent, TurnState}; use hive_ag3nt::login::{self, LoginState}; use hive_ag3nt::turn_stats::TurnStats; use hive_ag3nt::{ DEFAULT_SOCKET, DEFAULT_WEB_PORT, client, mcp, plugins, serve_common, turn, web_ui, }; use hive_sh4re::{AgentRequest, AgentResponse, HelperEvent, SYSTEM_SENDER}; #[derive(Parser)] #[command(name = "hive", about = "hyperhive harness")] struct Cli { /// Path to the per-agent MCP socket (bind-mounted from the host). #[arg(long, global = true, default_value = DEFAULT_SOCKET)] socket: PathBuf, #[command(subcommand)] cmd: Cmd, } #[derive(Subcommand)] enum Cmd { /// Run the long-lived harness loop. Polls inbox; replies via /// `claude --print` when available. Serve { /// Inbox poll interval in milliseconds. #[arg(long, default_value_t = 1000)] poll_ms: u64, }, /// Run the MCP server. Default is stdio — spawned by `claude` via /// `--mcp-config`; tools dispatch through `/run/hive/mcp.sock` back /// into the hyperhive broker. Pass `--http ` to instead run a /// long-lived streamable-http listener (persistent daemon) that /// claude reconnects to each turn, avoiding the per-turn stdio /// re-registration race. Mcp { /// Serve over streamable-http on this address (e.g. /// `127.0.0.1:8790`) instead of stdio. Bind loopback only. #[arg(long)] http: Option, }, /// Inject a wake-up event into this harness's inbox so the next /// turn fires with the given body. Intended for extra MCP servers /// / helpers (matrix bridge, scraper, webhook listener, etc.) that /// need to nudge claude on external events. Wake { #[arg(long)] from: String, /// Body of the wake message. Pass `-` to read from stdin. #[arg(long)] body: String, }, } #[tokio::main] async fn main() -> Result<()> { tracing_subscriber::fmt() .with_env_filter( tracing_subscriber::EnvFilter::try_from_default_env() .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), ) .init(); let cli = Cli::parse(); match cli.cmd { Cmd::Serve { poll_ms } => serve_main::(&cli.socket, poll_ms).await, Cmd::Mcp { http } => match http { Some(addr) => mcp::serve_http(cli.socket, addr).await, None => mcp::serve_agent_stdio(cli.socket).await, }, Cmd::Wake { from, body } => wake::(&cli.socket, from, body).await, } } // ---------- shared turn helpers ---------- /// Surface a `SYSTEM_SENDER` message in the live event bus + tracing /// log. Both agents and the manager receive `QuestionAnswered`, /// `ContainerCrash`, reparent notifications, and friends; the parse /// and log path is identical. Quiet no-op when `from` isn't /// `SYSTEM_SENDER`. fn log_system_event(bus: &Bus, from: &str, body: &str) { if from != SYSTEM_SENDER { return; } let parsed = serde_json::from_str::(body).ok(); if let Some(event) = parsed { tracing::info!(?event, "helper event"); } else { tracing::info!(%from, %body, "system message"); } bus.emit(LiveEvent::Note { text: format!("[system] {body}"), }); } /// Body string for the turn-failure notification we route to /// `` on `TurnOutcome::Failed`. Reads the hive-qualified /// identity so the receiver sees `agent@hive` rather than relying on /// the caller threading a `label` through every turn-handling layer. /// Falls back to `` when `HIVE_LABEL` is missing so a /// misconfigured harness still produces a parseable line. fn format_turn_failure(err: &anyhow::Error) -> String { let who = hive_ag3nt::identity::qualified_label(); let who = if who.is_empty() { "".to_owned() } else { who }; format!("[system] `{who}` claude turn failed:\n{err:#}") } /// Check for the `hyperhive-continue` sentinel under the state dir /// (dropped by the `request_next_turn` MCP tool). Returns true and /// consumes the file when present; false otherwise. Caller fires /// the role-specific `Wake` request — the sentinel itself is wire- /// agnostic so this helper lives outside both surfaces. fn consume_continue_sentinel() -> bool { let sentinel = hive_ag3nt::paths::state_dir().join("hyperhive-continue"); if !sentinel.exists() { return false; } if let Err(e) = std::fs::remove_file(&sentinel) { tracing::warn!(error = %e, "consume_continue_sentinel: remove sentinel failed"); return false; } true } /// What a finished turn tells the serve loop to do next. Replaces the /// bare `auth_failed` bool so the loop can also act on a pending /// `request_next_turn` without round-tripping a synthetic message /// through the broker. struct TurnControl { /// The turn ended in `AuthFailed` — caller parks on login. auth_failed: bool, /// `request_next_turn` was called during the turn (the /// `hyperhive-continue` sentinel was dropped + consumed). continue_requested: bool, /// Inbox unread count observed right after the turn. Used to /// decide whether a self-continue is actually needed. pending: u64, } /// Decide whether the serve loop should drive a self-continue turn /// in-process. A continue is only "needed" when nothing else will /// wake the agent: if real messages are already pending they drive /// the next turn(s) and the continue is dropped (matches the /// `request_next_turn` contract — "no effect if a new inbox message /// arrives before this turn ends"). Auth-failed parks the loop on /// login, so it suppresses the continue too. fn should_self_continue(ctrl: &TurnControl) -> bool { ctrl.continue_requested && !ctrl.auth_failed && ctrl.pending == 0 } /// Synthesize the `from: "self"` / `body: "continue"` message that a /// `request_next_turn` self-continue drives. Built in-process rather /// than fetched from the broker — it never touches the send/recv /// path, so it doesn't persist to sqlite or pollute the inbox. /// `id = 0` is a non-broker sentinel: the synthetic message /// has no DB row, and `AckTurn` keys off the recipient's in-flight /// list (which is empty here) rather than this id. fn synthetic_continue() -> hive_sh4re::DeliveredMessage { hive_sh4re::DeliveredMessage { from: "self".into(), body: "continue".into(), id: 0, redelivered: false, in_reply_to: None, } } /// Synthetic message that drives the single stop-checkpoint turn when c0re /// signals a graceful stop. The agent gets one final turn to flush durable /// `/state` before the container is stopped; new inbound is already fenced. fn graceful_stop_message() -> hive_sh4re::DeliveredMessage { hive_sh4re::DeliveredMessage { from: "graceful-stop".into(), body: "You are being gracefully stopped — the container will shut down after this turn, \ and new inbound messages are already fenced. This is your one checkpoint turn: \ flush anything worth keeping into your durable /state files now — update your \ notes / CLAUDE.md / TODO.md with in-flight task state, decisions made, important \ file paths, and whatever you'd need to resume cleanly later with only a summary \ of this conversation to go on. Do not start new work or reply to anyone; just \ write your notes and end your turn. The session may be compacted after this turn \ so a later cold start resumes cheaply." .into(), id: 0, redelivered: false, in_reply_to: None, } } // ---------- surface trait ---------- /// What a `Recv` long-poll returned. Decoupled from the per-role /// Response enum so `serve_loop` can pattern-match without seeing /// either `AgentResponse` or `ManagerResponse` directly. enum RecvOutcome { /// Long-poll returned at least one message; first one is detached. Message(hive_sh4re::DeliveredMessage), /// Long-poll timed out cleanly (empty `Messages` response). Caller /// sleeps then retries. Empty, /// Wire returned an error / unexpected variant. Caller logs + /// retries; the surface impl is responsible for tracing the /// detail before returning this. TransportError, /// c0re signalled a graceful stop for this agent. The serve loop runs /// one stop-checkpoint turn (flush durable `/state`), reports /// `GracefulStopComplete`, and exits so the container can be stopped. GracefulStop, } /// Wire surface abstraction. `AgentSurface` is the only impl — the trait /// exists to keep the turn loop generic and testable. Every function that /// talks to the broker goes through this so there are zero hard-coded /// `AgentRequest` / `AgentResponse` references in the turn loop itself. trait Surface { /// Ack the in-flight turn. Logs warnings on transport/broker /// errors but never propagates — turn loop continues either way. fn ack_turn(socket: &Path) -> impl Future; /// Requeue any messages that were "in-flight" (delivered but not /// ack'd) — fires on harness boot to recover from a crash mid-turn. fn requeue_inflight(socket: &Path) -> impl Future; /// Current inbox unread count via `Status`. Returns 0 on any /// transport/wire error so the caller falls through cleanly. fn inbox_unread(socket: &Path) -> impl Future; /// `(open_threads, open_reminders)` for the post-turn stats row. /// Either field is `None` when the underlying request errors. fn post_turn_counts(socket: &Path) -> impl Future, Option)>; /// Tell c0re the graceful-stop checkpoint is done and the harness is /// exiting its serve loop (fire-and-forget; logs on error). Lets the /// `GracefulStop` orchestration stop the container without waiting out /// its timeout fallback. fn graceful_stop_complete(socket: &Path) -> impl Future; /// Send a message addressed to `` (broker resolves the /// sentinel via `topology::parent_of` at delivery time; root /// agents/manager fall through to operator). fn send_to_parent(socket: &Path, body: String) -> impl Future; /// Long-poll the broker for the next message. Wraps the /// `Messages`/empty/error trichotomy in `RecvOutcome` so the /// generic `serve_loop` doesn't need the per-role Response enum /// at all. fn recv_next(socket: &Path) -> impl Future; /// External `wake` subcommand (the `hive wake` CLI command, used /// by co-process daemons like matrix to push events into the /// harness inbox). Errors out via `anyhow::bail!` so the calling /// binary surfaces them on stderr. fn wake_external(socket: &Path, from: String, body: String) -> impl Future>; } // ---------- AgentSurface ---------- /// Zero-sized type tag for the agent wire surface. /// Talks `AgentRequest` / `AgentResponse`. struct AgentSurface; impl Surface for AgentSurface { async fn ack_turn(socket: &Path) { match client::request::<_, AgentResponse>(socket, &AgentRequest::AckTurn).await { Ok(AgentResponse::Ok) => {} Ok(AgentResponse::Err { message }) => { tracing::warn!(%message, "ack_turn rejected by broker"); } Ok(other) => tracing::warn!(?other, "ack_turn unexpected response"), Err(e) => tracing::warn!(error = ?e, "ack_turn transport error"), } } async fn requeue_inflight(socket: &Path) { match client::request::<_, AgentResponse>(socket, &AgentRequest::RequeueInflight).await { Ok(AgentResponse::Ok) => {} Ok(AgentResponse::Err { message }) => { tracing::warn!(%message, "requeue_inflight rejected by broker"); } Ok(other) => tracing::warn!(?other, "requeue_inflight unexpected response"), Err(e) => tracing::warn!(error = ?e, "requeue_inflight transport error"), } } async fn graceful_stop_complete(socket: &Path) { match client::request::<_, AgentResponse>(socket, &AgentRequest::GracefulStopComplete).await { Ok(AgentResponse::Ok) => {} Ok(AgentResponse::Err { message }) => { tracing::warn!(%message, "graceful_stop_complete rejected by broker"); } Ok(other) => tracing::warn!(?other, "graceful_stop_complete unexpected response"), Err(e) => tracing::warn!(error = ?e, "graceful_stop_complete transport error"), } } async fn inbox_unread(socket: &Path) -> u64 { match client::request::<_, AgentResponse>(socket, &AgentRequest::Status).await { Ok(AgentResponse::Status { unread }) => unread, _ => 0, } } async fn post_turn_counts(socket: &Path) -> (Option, Option) { let threads = match client::request::<_, AgentResponse>( socket, &AgentRequest::GetLooseEnds { agent: None }, ) .await { Ok(AgentResponse::LooseEnds { loose_ends }) => u64::try_from(loose_ends.len()).ok(), _ => None, }; let reminders = match client::request::<_, AgentResponse>( socket, &AgentRequest::CountPendingReminders { agent: None }, ) .await { Ok(AgentResponse::PendingRemindersCount { count }) => Some(count), _ => None, }; (threads, reminders) } async fn send_to_parent(socket: &Path, body: String) { let res = client::request::<_, AgentResponse>( socket, &AgentRequest::Send { to: hive_sh4re::PARENT_RECIPIENT.into(), body, in_reply_to: None, }, ) .await; if let Err(e) = res { tracing::warn!(error = ?e, "failed to notify parent of turn failure"); } } async fn recv_next(socket: &Path) -> RecvOutcome { let recv: Result = client::request( socket, &AgentRequest::Recv { wait_seconds: Some(180), max: None, }, ) .await; match recv { Ok(AgentResponse::Messages { messages }) if !messages.is_empty() => { let first = messages.into_iter().next().expect("checked non-empty"); RecvOutcome::Message(first) } Ok(AgentResponse::Messages { .. }) => RecvOutcome::Empty, Ok(AgentResponse::GracefulStop) => RecvOutcome::GracefulStop, Ok(AgentResponse::Err { message }) => { tracing::warn!(%message, "recv error"); RecvOutcome::TransportError } Ok(other) => { tracing::warn!(?other, "recv produced unexpected response kind"); RecvOutcome::TransportError } Err(e) => { tracing::warn!(error = ?e, "recv failed; retrying"); RecvOutcome::TransportError } } } async fn wake_external(socket: &Path, from: String, body: String) -> Result<()> { let resp: AgentResponse = client::request(socket, &AgentRequest::Wake { from, body }).await?; match resp { AgentResponse::Ok => Ok(()), AgentResponse::Err { message } => anyhow::bail!("wake: {message}"), other => anyhow::bail!("wake: unexpected response {other:?}"), } } } // ---------- generic turn loop ---------- /// Boot — wires up the web UI, login state, stats, plugins, forge /// notifier, and either drops into `serve_loop` directly (`Online`) or /// parks on the login flow first (`NeedsLogin`). See /// `docs/turn-loop.md::Boot wiring`. async fn serve_main(socket: &Path, poll_ms: u64) -> Result<()> { let port = std::env::var("HIVE_PORT") .ok() .and_then(|s| s.parse::().ok()) .unwrap_or(DEFAULT_WEB_PORT); // `HIVE_LABEL` is set unconditionally by the meta-flake envelope // for any container-deployed agent; the `"hive"` fallback here // covers standalone `nix run .#hive` invocations and pre-meta // dev shells. Role-independent: no semantic reason for the // fallback to differ when the env var is missing. let label = std::env::var("HIVE_LABEL").unwrap_or_else(|_| "hive".into()); let claude_dir = login::default_dir(); let initial = LoginState::from_dir(&claude_dir); tracing::info!(state = ?initial, claude_dir = %claude_dir.display(), "harness boot"); let login_state = Arc::new(Mutex::new(initial)); let bus = Bus::new(); let stats = TurnStats::open_default(); if let Some(s) = &stats { let (ctx, cost) = s.last_usage(); if ctx.is_some() || cost.is_some() { bus.seed_usage(ctx, cost); } } let files = turn::TurnFiles::prepare(socket, &label).await?; let turn_lock: TurnLock = Arc::new(tokio::sync::Mutex::new(())); // Plugin install failures come back as a Vec — route each // through `` via the `send_to_parent` failure-notify path. // The broker resolves `` per `topology::parent_of`; // root agents fall through to operator. for failure in plugins::install_configured(socket).await { S::send_to_parent(socket, failure).await; } tokio::spawn(hive_ag3nt::forge_notify::run(socket.to_path_buf())); // Agent-side cleanup of this agent's own harness artifacts (completed // bash-task files + verbose event rows). Runs here, not host-side in // hive-c0re, because the files are agent-owned — see `vacuum` module docs. tokio::spawn(hive_ag3nt::vacuum::run()); // Log web_ui::serve's error instead of dropping it. A bare // `tokio::spawn(web_ui::serve(...))` discards the JoinHandle, so // any Err (e.g. EACCES from `bind_unix` when HIVE_WEB_SOCKET points // at a dir the agent user can't write) vanishes — leaving an // operator with no log line and no socket, debuggable only by // staring at lifecycle.rs. let web_ui_args = ( label.clone(), port, login_state.clone(), bus.clone(), socket.to_path_buf(), ); tokio::spawn(async move { let (label, port, login_state, bus, socket) = web_ui_args; if let Err(e) = web_ui::serve(label, port, login_state, bus, socket).await { tracing::error!(error = %e, "web_ui::serve exited with error"); } }); if matches!(initial, LoginState::NeedsLogin) { login::wait_for_login(&claude_dir, login_state.clone(), &bus, poll_ms).await; } else { // Clear any stale `hyperhive-needs-login` sentinel left over // from a prior boot — `online` status writes the sentinel // cleanup in `Bus::emit_status`. bus.emit_status("online"); } serve_loop::( socket, Duration::from_millis(poll_ms), login_state, claude_dir, bus, stats, &files, turn_lock, ) .await } /// The long-running message loop. Long-polls the broker via /// `S::recv_next`, drives a turn per message, parks on auth-failed, /// otherwise retries. #[allow( clippy::too_many_arguments, reason = "the harness's long-lived deps threaded into one serve loop, \ wired once from main; bundling into a struct would just move the \ same fields one level out (cf. Coordinator::open)" )] async fn serve_loop( socket: &Path, interval: Duration, login_state: Arc>, claude_dir: std::path::PathBuf, bus: Bus, stats: Option, files: &turn::TurnFiles, turn_lock: TurnLock, ) -> Result<()> { tracing::info!(socket = %socket.display(), "harness serve"); S::requeue_inflight(socket).await; // Set when a turn calls `request_next_turn` and no real work is // pending — the next iteration drives this synthetic message // in-process instead of long-polling the broker. Never // persisted: it lives entirely in this loop's stack. let mut self_continue: Option = None; loop { let next = match self_continue.take() { Some(msg) => msg, None => match S::recv_next(socket).await { RecvOutcome::Message(first) => first, RecvOutcome::Empty => { // Idle: no message this poll. Service a queued operator // `/compact` here so it runs even when no turn is driving // (the in-flight case is handled at the end of drive_turn). let compacted = { let _guard = turn_lock.lock().await; turn::run_pending_compact(files, &bus).await }; if !compacted { tokio::time::sleep(interval).await; } continue; } RecvOutcome::TransportError => { // `recv_next` already logged the detail; just retry. // No backoff: the long-poll wait is itself the throttle. continue; } RecvOutcome::GracefulStop => { // c0re fenced our inbox and wants a clean stop. Run one // checkpoint turn so the agent flushes durable /state, // report completion, then exit the loop → the harness // process ends and the container can be stopped. tracing::info!( "graceful stop signalled — running stop-checkpoint turn, then exiting" ); let _ = handle_turn::( socket, &bus, stats.as_ref(), files, &turn_lock, graceful_stop_message(), ) .await; S::graceful_stop_complete(socket).await; return Ok(()); } }, }; let ctrl = handle_turn::(socket, &bus, stats.as_ref(), files, &turn_lock, next).await; if ctrl.auth_failed { *login_state.lock().unwrap() = LoginState::NeedsLogin; login::wait_for_login( &claude_dir, login_state.clone(), &bus, u64::try_from(interval.as_millis()).unwrap_or(2000), ) .await; } else if should_self_continue(&ctrl) { tracing::info!("request_next_turn: driving self-continue turn in-process"); self_continue = Some(synthetic_continue()); } } } /// Drive a single turn: emit boot-of-turn events, run claude, ack on /// success / requeue on rate-limit-or-401 / notify parent on failure, /// record stats, then pick up the `request_next_turn` sentinel if it's /// been dropped during the turn. Returns a `TurnControl` carrying the /// auth-failed flag, whether a self-continue was requested, and the /// post-turn inbox count — the serve loop decides what to do next. async fn handle_turn( socket: &Path, bus: &Bus, stats: Option<&TurnStats>, files: &turn::TurnFiles, turn_lock: &TurnLock, first: hive_sh4re::DeliveredMessage, ) -> TurnControl { let from = first.from; let body = first.body; let redelivered = first.redelivered; let msg_id = first.id; log_system_event(bus, &from, &body); tracing::info!(%from, %body, %redelivered, "inbox"); let unread = S::inbox_unread(socket).await; bus.emit(LiveEvent::TurnStart { from: from.clone(), body: body.clone(), unread, }); bus.set_state(TurnState::Thinking); let started_at = serve_common::now_unix(); let started_instant = std::time::Instant::now(); let model_at_start = bus.model(); let prompt = serve_common::format_wake_prompt(msg_id, &from, &body, unread, redelivered); let outcome = { let _guard = turn_lock.lock().await; turn::drive_turn(&prompt, files, bus).await }; turn::emit_turn_end(bus, &outcome); bus.set_state(TurnState::Idle); if matches!( outcome, turn::TurnOutcome::Ok | turn::TurnOutcome::Compacted ) { S::ack_turn(socket).await; } if matches!(outcome, turn::TurnOutcome::RateLimited) { let secs = turn::rate_limit_sleep_secs(); bus.emit_status("rate_limited"); bus.emit(LiveEvent::Note { text: format!("API rate-limited — sleeping {secs}s before retry"), }); tracing::warn!(sleep_secs = secs, "rate-limited; parking"); tokio::time::sleep(Duration::from_secs(secs)).await; S::requeue_inflight(socket).await; bus.emit_status("online"); } if matches!(outcome, turn::TurnOutcome::AuthFailed) { bus.emit_status("needs_login_idle"); bus.emit(LiveEvent::Note { text: "API 401 — waiting for re-login via web UI".into(), }); tracing::warn!("auth-failed; parking until re-login"); S::requeue_inflight(socket).await; } if let turn::TurnOutcome::Failed(e) = &outcome { S::send_to_parent(socket, format_turn_failure(e)).await; } if let Some(stats) = stats { // Fresh session this turn → mint a `sessions` row and set its id on // the bus so this turn (and subsequent ones until the next fresh // start) stamp `turn_stats.session_id`. Takes the one-shot flag // `run_claude` set when it suppressed `--continue`. if bus.take_fresh_session() { let sid = stats.start_session(started_at, &model_at_start); bus.set_session_id(sid); } let ended_at = serve_common::now_unix(); let duration_ms = i64::try_from(started_instant.elapsed().as_millis()).unwrap_or(i64::MAX); let (open_threads, open_reminders) = S::post_turn_counts(socket).await; let row = serve_common::build_row(serve_common::TurnRowArgs { started_at, ended_at, duration_ms, model: model_at_start, wake_from: from.clone(), outcome: &outcome, bus, open_threads_count: open_threads, open_reminders_count: open_reminders, }); stats.record(&row); } let pending = S::inbox_unread(socket).await; if pending > 0 { tracing::info!(%pending, "pending messages after turn; fetching next"); } TurnControl { auth_failed: matches!(outcome, turn::TurnOutcome::AuthFailed), continue_requested: consume_continue_sentinel(), pending, } } /// External `hive wake` subcommand — push a message into our own /// inbox so the next turn fires with the given body. Reads the body /// from stdin when `body == "-"`. async fn wake(socket: &Path, from: String, body: String) -> Result<()> { let body = if body == "-" { let mut buf = String::new(); std::io::Read::read_to_string(&mut std::io::stdin(), &mut buf)?; buf } else { body }; S::wake_external(socket, from, body).await } #[cfg(test)] mod continue_tests { use super::{TurnControl, should_self_continue, synthetic_continue}; fn ctrl(auth_failed: bool, continue_requested: bool, pending: u64) -> TurnControl { TurnControl { auth_failed, continue_requested, pending, } } #[test] fn self_continue_when_requested_and_inbox_empty() { assert!(should_self_continue(&ctrl(false, true, 0))); } #[test] fn no_self_continue_when_not_requested() { assert!(!should_self_continue(&ctrl(false, false, 0))); } #[test] fn no_self_continue_when_real_messages_pending() { // A real message will drive the next turn via recv — the // continue is superseded, not needed (request_next_turn contract). assert!(!should_self_continue(&ctrl(false, true, 3))); } #[test] fn no_self_continue_when_auth_failed() { // Auth-failed parks the loop on login; a queued continue must // not jump the gate. assert!(!should_self_continue(&ctrl(true, true, 0))); } #[test] fn synthetic_continue_shape() { let m = synthetic_continue(); assert_eq!(m.from, "self"); assert_eq!(m.body, "continue"); assert_eq!(m.id, 0); assert!(!m.redelivered); assert!(m.in_reply_to.is_none()); } }