hyperhive/hive-agent/src/client.rs

use std::path::Path;
use std::time::Duration;

use anyhow::{Result, anyhow};
use serde::Serialize;
use serde::de::DeserializeOwned;
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use tokio::net::UnixStream;

/// Backoff schedule between attempts. Five entries → up to 5 retries on
/// top of the initial attempt; total wall-clock cap = 2+4+8+16+30 = 60s.
/// Sized to ride out a hive-c0re restart (systemd usually has the unix
/// socket back inside ~5s) without the agent-side claude session having
/// to handle the transient itself — burning tokens on a tool-error retry
/// loop is more expensive than 60s of in-harness sleep.
const RETRY_BACKOFFS_MS: &[u64] = &[2_000, 4_000, 8_000, 16_000, 30_000];

/// Transparent retry wrapper around [`request_retried`] that throws away
/// the retry count. Use this from non-tool callers (the harness serve
/// loop, web UI, CLI subcommands) where we just want the socket-restart
/// resilience without surfacing the bookkeeping.
///
/// # Errors
///
/// Returns an error if the socket is unreachable after all retries, or if
/// serialization / deserialization of the request or response fails.
pub async fn request<Req, Resp>(socket: &Path, req: &Req) -> Result<Resp>
where
    Req: Serialize + ?Sized,
    Resp: DeserializeOwned,
{
    request_retried(socket, req).await.map(|(resp, _)| resp)
}

/// Same wire shape as [`request`], but reports how many retries it took
/// past the initial attempt (0 = succeeded first try). MCP tool handlers
/// use this so they can append a one-line hint to the tool result when
/// retries happened — that way claude knows the prior socket flake
/// wasn't a content error and shouldn't trigger an LLM-level retry of
/// its own.
///
/// # Errors
///
/// Returns an error if all retries are exhausted, or on a fatal protocol
/// error (serialization / deserialization failure).
///
/// # Panics
///
/// Panics if `RETRY_BACKOFFS_MS.len()` does not fit in a `u32`, which
/// cannot happen with the current compile-time constant.
pub async fn request_retried<Req, Resp>(socket: &Path, req: &Req) -> Result<(Resp, u32)>
where
    Req: Serialize + ?Sized,
    Resp: DeserializeOwned,
{
    let mut last_err: Option<anyhow::Error> = None;
    let max_retries = u32::try_from(RETRY_BACKOFFS_MS.len()).unwrap();
    for attempt in 0..=max_retries {
        match try_once::<Req, Resp>(socket, req).await {
            Ok(resp) => return Ok((resp, attempt)),
            Err(RequestError::Fatal(e)) => return Err(e),
            Err(RequestError::Transient(e)) => {
                if attempt < max_retries {
                    let sleep_ms = RETRY_BACKOFFS_MS[attempt as usize];
                    tracing::warn!(
                        attempt = attempt + 1,
                        sleep_ms,
                        error = %e,
                        "hive socket attempt failed; retrying"
                    );
                    last_err = Some(e);
                    tokio::time::sleep(Duration::from_millis(sleep_ms)).await;
                } else {
                    last_err = Some(e);
                }
            }
        }
    }
    // Reaching here means the final attempt returned `Transient`, which always
    // sets `last_err` — so this is infallible.
    Err(last_err.expect("a transient failure on the final attempt set last_err"))
}

/// Transient = connect / IO error worth a retry (server restart, broken
/// pipe). Fatal = serialization / deserialization / protocol error
/// where retrying would just repeat the same failure.
enum RequestError {
    Transient(anyhow::Error),
    Fatal(anyhow::Error),
}

async fn try_once<Req, Resp>(socket: &Path, req: &Req) -> Result<Resp, RequestError>
where
    Req: Serialize + ?Sized,
    Resp: DeserializeOwned,
{
    let stream = match UnixStream::connect(socket).await {
        Ok(stream) => stream,
        Err(e) => {
            // A refused or missing socket usually means hive-c0re is
            // mid-restart (operator redeploy / rebuild) — the socket is
            // recreated on its boot and `request_retried` rides it out. When
            // the error *does* surface (retries exhausted, or a non-retried
            // caller) add that context so claude reads it as a likely
            // transient rather than a hard failure worth escalating.
            let restarting = matches!(
                e.kind(),
                std::io::ErrorKind::ConnectionRefused | std::io::ErrorKind::NotFound
            );
            let mut err = anyhow::Error::new(e).context(format!("connect to {}", socket.display()));
            if restarting {
                err = err.context(
                    "hive-c0re may be restarting (e.g. an operator redeploy); \
                     the harness already retried ~60s before surfacing this",
                );
            }
            return Err(RequestError::Transient(err));
        }
    };
    let (read, mut write) = stream.into_split();

    let mut payload = serde_json::to_string(req).map_err(|e| RequestError::Fatal(e.into()))?;
    payload.push('\n');
    write
        .write_all(payload.as_bytes())
        .await
        .map_err(|e| RequestError::Transient(e.into()))?;
    write
        .flush()
        .await
        .map_err(|e| RequestError::Transient(e.into()))?;

    let mut reader = BufReader::new(read);
    let mut line = String::new();
    let read_bytes = reader
        .read_line(&mut line)
        .await
        .map_err(|e| RequestError::Transient(e.into()))?;
    if read_bytes == 0 || line.is_empty() {
        return Err(RequestError::Transient(anyhow!(
            "server closed connection without responding"
        )));
    }
    serde_json::from_str(line.trim()).map_err(|e| RequestError::Fatal(e.into()))
}

#[cfg(test)]
mod tests {
    use super::{RequestError, try_once};

    /// A connect to a non-existent socket path (ENOENT → `NotFound`) is
    /// classified transient AND annotated with the "hive-c0re is restarting"
    /// hint, so a surfaced tool error reads as the expected transient.
    #[tokio::test]
    async fn missing_socket_connect_is_transient_with_restart_hint() {
        let bogus = std::path::Path::new("/nonexistent/hive/mcp.sock");
        match try_once::<(), serde_json::Value>(bogus, &()).await {
            Err(RequestError::Transient(e)) => {
                let msg = format!("{e:#}");
                assert!(msg.contains("restarting"), "missing restart hint: {msg}");
                assert!(msg.contains("connect to"), "missing connect context: {msg}");
            }
            Err(RequestError::Fatal(e)) => panic!("expected transient, got fatal: {e:#}"),
            Ok(_) => panic!("expected connect failure to a non-existent socket"),
        }
    }
}