From 3d5957c87bd4f3079fcea82f69352d258b316126 Mon Sep 17 00:00:00 2001 From: damocles Date: Thu, 2 Jul 2026 19:54:37 +0200 Subject: [PATCH 1/2] web-ui: deadline-bound all broker-backed fetches so a busy hive-c0re can't hang api/state (closes #2148) --- hive-ag3nt/src/web_ui.rs | 74 ++++++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/hive-ag3nt/src/web_ui.rs b/hive-ag3nt/src/web_ui.rs index 5444a394..982d05b0 100644 --- a/hive-ag3nt/src/web_ui.rs +++ b/hive-ag3nt/src/web_ui.rs @@ -32,6 +32,14 @@ use crate::login::LoginState; use crate::login_session::{LoginSession, drop_if_finished}; use crate::turn::TurnFiles; +/// Deadline for broker-backed fetches on web-UI request paths. The +/// page's critical fields (status, turn state, usage) are all +/// in-memory; a busy or stalled hive-c0re must degrade the +/// socket-backed extras (inbox rows, loose ends, reminder stats) +/// instead of hanging the whole response — an unbounded await here is +/// what let `/api/state` stall long enough to bork the terminal. +const SOCKET_FETCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(3); + /// Live login state for the web UI. The harness updates this in place as it /// transitions between `NeedsLogin` and `Online`; the UI reads on each /// render. @@ -512,18 +520,22 @@ struct SessionView { /// the `mcp__hyperhive__get_loose_ends` tool sees from inside the /// container. async fn api_loose_ends(State(state): State) -> Response { - let loose_ends: Vec = match client::request::<_, hive_sh4re::Response>( - &state.socket, - &hive_sh4re::Request::GetLooseEnds { agent: None }, + let loose_ends: Vec = match tokio::time::timeout( + SOCKET_FETCH_TIMEOUT, + client::request::<_, hive_sh4re::Response>( + &state.socket, + &hive_sh4re::Request::GetLooseEnds { agent: None }, + ), ) .await { - Ok(hive_sh4re::Response::LooseEnds { loose_ends }) => loose_ends, - Ok(hive_sh4re::Response::Err { message }) => { + Ok(Ok(hive_sh4re::Response::LooseEnds { loose_ends })) => loose_ends, + Ok(Ok(hive_sh4re::Response::Err { message })) => { return error_response(&format!("get_loose_ends: {message}")); } - Ok(other) => return error_response(&format!("unexpected response: {other:?}")), - Err(e) => return error_response(&format!("transport: {e:#}")), + Ok(Ok(other)) => return error_response(&format!("unexpected response: {other:?}")), + Ok(Err(e)) => return error_response(&format!("transport: {e:#}")), + Err(_) => return error_response("get_loose_ends: timed out — hive-c0re busy, retry"), }; axum::Json(serde_json::json!({ "loose_ends": loose_ends })).into_response() } @@ -813,13 +825,18 @@ struct ExtraLink { /// failure — the inbox section is decorative, not authoritative. async fn recent_inbox(socket: &std::path::Path) -> Vec { const LIMIT: u64 = 30; - match client::request::<_, hive_sh4re::Response>( - socket, - &hive_sh4re::Request::Recent { limit: LIMIT }, + // Deadline-bounded: `/api/state` must render even when hive-c0re is + // busy — an empty inbox section beats a hung snapshot. + match tokio::time::timeout( + SOCKET_FETCH_TIMEOUT, + client::request::<_, hive_sh4re::Response>( + socket, + &hive_sh4re::Request::Recent { limit: LIMIT }, + ), ) .await { - Ok(hive_sh4re::Response::Recent { rows }) => rows, + Ok(Ok(hive_sh4re::Response::Recent { rows })) => rows, _ => Vec::new(), } } @@ -831,16 +848,19 @@ async fn fetch_reminder_stats( socket: &std::path::Path, window_secs: u64, ) -> Option { - match client::request::<_, hive_sh4re::Response>( - socket, - &hive_sh4re::Request::ReminderRollup { - since_secs: window_secs, - agent: None, - }, + match tokio::time::timeout( + SOCKET_FETCH_TIMEOUT, + client::request::<_, hive_sh4re::Response>( + socket, + &hive_sh4re::Request::ReminderRollup { + since_secs: window_secs, + agent: None, + }, + ), ) .await { - Ok(hive_sh4re::Response::ReminderRollup(stats)) => Some(stats), + Ok(Ok(hive_sh4re::Response::ReminderRollup(stats))) => Some(stats), _ => None, } } @@ -859,16 +879,20 @@ async fn post_send(State(state): State, Form(form): Form) -> if body.is_empty() { return error_response("send: `body` required"); } - let result = match client::request::<_, hive_sh4re::Response>( - &state.socket, - &hive_sh4re::Request::OperatorMsg { body }, + let result = match tokio::time::timeout( + SOCKET_FETCH_TIMEOUT, + client::request::<_, hive_sh4re::Response>( + &state.socket, + &hive_sh4re::Request::OperatorMsg { body }, + ), ) .await { - Ok(hive_sh4re::Response::Ok) => Ok(()), - Ok(hive_sh4re::Response::Err { message }) => Err(message), - Ok(other) => Err(format!("unexpected response: {other:?}")), - Err(e) => Err(format!("transport: {e:#}")), + Ok(Ok(hive_sh4re::Response::Ok)) => Ok(()), + Ok(Ok(hive_sh4re::Response::Err { message })) => Err(message), + Ok(Ok(other)) => Err(format!("unexpected response: {other:?}")), + Ok(Err(e)) => Err(format!("transport: {e:#}")), + Err(_) => Err("timed out — hive-c0re busy, retry".to_owned()), }; match result { // 200 instead of 303 → the client doesn't refetch /api/state. From cf1f7288bffd7848cd742928ae3c7e587565b07e Mon Sep 17 00:00:00 2001 From: damocles Date: Thu, 2 Jul 2026 20:17:00 +0200 Subject: [PATCH 2/2] =?UTF-8?q?make=20agent=5Fconfig=5Fpending=20async=20?= =?UTF-8?q?=E2=80=94=20the=20sync=20git=20fork=20on=20every=20sweep=20star?= =?UTF-8?q?ved=20the=20runtime=20under=20IO=20load?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hive-c0re/src/auto_update.rs | 13 ++++++++++--- hive-c0re/src/container_view.rs | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/hive-c0re/src/auto_update.rs b/hive-c0re/src/auto_update.rs index b2d9ef0a..40d81b6e 100644 --- a/hive-c0re/src/auto_update.rs +++ b/hive-c0re/src/auto_update.rs @@ -38,9 +38,15 @@ pub fn current_flake_rev(hyperhive_flake: &str) -> Option { /// deployed (i.e. the applied HEAD differs from the sha currently locked in /// meta's flake.lock). This is the semantic the dashboard `needs_update` chip /// conveys: "there is a config change ready to apply via rebuild." -#[must_use] -pub fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> bool { - let applied_head = std::process::Command::new("git") +/// +/// Async on purpose: this runs per agent inside `container_view::build_all`, +/// which fires on the ~10s dashboard sweep, every `AgentStatus` request, and +/// every `rescan_containers_and_emit` after a lifecycle step. A synchronous +/// `git` fork here blocks a tokio worker for the whole exec — under +/// nix-build disk saturation that's long enough that concurrent sweeps +/// starved the runtime and stalled the per-agent sockets. +pub async fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> bool { + let applied_head = tokio::process::Command::new("git") .args([ "-C", &format!("/var/lib/hyperhive/applied/{name}"), @@ -48,6 +54,7 @@ pub fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> bool { "HEAD", ]) .output() + .await .ok() .filter(|o| o.status.success()) .and_then(|o| String::from_utf8(o.stdout).ok()) diff --git a/hive-c0re/src/container_view.rs b/hive-c0re/src/container_view.rs index 74c41dd1..9cdda653 100644 --- a/hive-c0re/src/container_view.rs +++ b/hive-c0re/src/container_view.rs @@ -70,7 +70,7 @@ pub async fn build_all(coord: &Coordinator) -> Vec { let deployed_full = locked .get(&format!("agent-{logical}")) .map(std::string::String::as_str); - let needs_update = crate::auto_update::agent_config_pending(&logical, deployed_full); + let needs_update = crate::auto_update::agent_config_pending(&logical, deployed_full).await; let deployed_sha = deployed_full.map(|s| s[..s.len().min(12)].to_owned()); let pending_reminders = coord .broker