Compare commits

..
3 changed files with 29 additions and 60 deletions

View file

@ -32,14 +32,6 @@ use crate::login::LoginState;
use crate::login_session::{LoginSession, drop_if_finished}; use crate::login_session::{LoginSession, drop_if_finished};
use crate::turn::TurnFiles; use crate::turn::TurnFiles;
/// Deadline for broker-backed fetches on web-UI request paths. The
/// page's critical fields (status, turn state, usage) are all
/// in-memory; a busy or stalled hive-c0re must degrade the
/// socket-backed extras (inbox rows, loose ends, reminder stats)
/// instead of hanging the whole response — an unbounded await here is
/// what let `/api/state` stall long enough to bork the terminal.
const SOCKET_FETCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(3);
/// Live login state for the web UI. The harness updates this in place as it /// Live login state for the web UI. The harness updates this in place as it
/// transitions between `NeedsLogin` and `Online`; the UI reads on each /// transitions between `NeedsLogin` and `Online`; the UI reads on each
/// render. /// render.
@ -520,22 +512,18 @@ struct SessionView {
/// the `mcp__hyperhive__get_loose_ends` tool sees from inside the /// the `mcp__hyperhive__get_loose_ends` tool sees from inside the
/// container. /// container.
async fn api_loose_ends(State(state): State<AppState>) -> Response { async fn api_loose_ends(State(state): State<AppState>) -> Response {
let loose_ends: Vec<hive_sh4re::LooseEnd> = match tokio::time::timeout( let loose_ends: Vec<hive_sh4re::LooseEnd> = match client::request::<_, hive_sh4re::Response>(
SOCKET_FETCH_TIMEOUT, &state.socket,
client::request::<_, hive_sh4re::Response>( &hive_sh4re::Request::GetLooseEnds { agent: None },
&state.socket,
&hive_sh4re::Request::GetLooseEnds { agent: None },
),
) )
.await .await
{ {
Ok(Ok(hive_sh4re::Response::LooseEnds { loose_ends })) => loose_ends, Ok(hive_sh4re::Response::LooseEnds { loose_ends }) => loose_ends,
Ok(Ok(hive_sh4re::Response::Err { message })) => { Ok(hive_sh4re::Response::Err { message }) => {
return error_response(&format!("get_loose_ends: {message}")); return error_response(&format!("get_loose_ends: {message}"));
} }
Ok(Ok(other)) => return error_response(&format!("unexpected response: {other:?}")), Ok(other) => return error_response(&format!("unexpected response: {other:?}")),
Ok(Err(e)) => return error_response(&format!("transport: {e:#}")), Err(e) => return error_response(&format!("transport: {e:#}")),
Err(_) => return error_response("get_loose_ends: timed out — hive-c0re busy, retry"),
}; };
axum::Json(serde_json::json!({ "loose_ends": loose_ends })).into_response() axum::Json(serde_json::json!({ "loose_ends": loose_ends })).into_response()
} }
@ -825,18 +813,13 @@ struct ExtraLink {
/// failure — the inbox section is decorative, not authoritative. /// failure — the inbox section is decorative, not authoritative.
async fn recent_inbox(socket: &std::path::Path) -> Vec<hive_sh4re::InboxRow> { async fn recent_inbox(socket: &std::path::Path) -> Vec<hive_sh4re::InboxRow> {
const LIMIT: u64 = 30; const LIMIT: u64 = 30;
// Deadline-bounded: `/api/state` must render even when hive-c0re is match client::request::<_, hive_sh4re::Response>(
// busy — an empty inbox section beats a hung snapshot. socket,
match tokio::time::timeout( &hive_sh4re::Request::Recent { limit: LIMIT },
SOCKET_FETCH_TIMEOUT,
client::request::<_, hive_sh4re::Response>(
socket,
&hive_sh4re::Request::Recent { limit: LIMIT },
),
) )
.await .await
{ {
Ok(Ok(hive_sh4re::Response::Recent { rows })) => rows, Ok(hive_sh4re::Response::Recent { rows }) => rows,
_ => Vec::new(), _ => Vec::new(),
} }
} }
@ -848,19 +831,16 @@ async fn fetch_reminder_stats(
socket: &std::path::Path, socket: &std::path::Path,
window_secs: u64, window_secs: u64,
) -> Option<hive_sh4re::ReminderStats> { ) -> Option<hive_sh4re::ReminderStats> {
match tokio::time::timeout( match client::request::<_, hive_sh4re::Response>(
SOCKET_FETCH_TIMEOUT, socket,
client::request::<_, hive_sh4re::Response>( &hive_sh4re::Request::ReminderRollup {
socket, since_secs: window_secs,
&hive_sh4re::Request::ReminderRollup { agent: None,
since_secs: window_secs, },
agent: None,
},
),
) )
.await .await
{ {
Ok(Ok(hive_sh4re::Response::ReminderRollup(stats))) => Some(stats), Ok(hive_sh4re::Response::ReminderRollup(stats)) => Some(stats),
_ => None, _ => None,
} }
} }
@ -879,20 +859,16 @@ async fn post_send(State(state): State<AppState>, Form(form): Form<SendForm>) ->
if body.is_empty() { if body.is_empty() {
return error_response("send: `body` required"); return error_response("send: `body` required");
} }
let result = match tokio::time::timeout( let result = match client::request::<_, hive_sh4re::Response>(
SOCKET_FETCH_TIMEOUT, &state.socket,
client::request::<_, hive_sh4re::Response>( &hive_sh4re::Request::OperatorMsg { body },
&state.socket,
&hive_sh4re::Request::OperatorMsg { body },
),
) )
.await .await
{ {
Ok(Ok(hive_sh4re::Response::Ok)) => Ok(()), Ok(hive_sh4re::Response::Ok) => Ok(()),
Ok(Ok(hive_sh4re::Response::Err { message })) => Err(message), Ok(hive_sh4re::Response::Err { message }) => Err(message),
Ok(Ok(other)) => Err(format!("unexpected response: {other:?}")), Ok(other) => Err(format!("unexpected response: {other:?}")),
Ok(Err(e)) => Err(format!("transport: {e:#}")), Err(e) => Err(format!("transport: {e:#}")),
Err(_) => Err("timed out — hive-c0re busy, retry".to_owned()),
}; };
match result { match result {
// 200 instead of 303 → the client doesn't refetch /api/state. // 200 instead of 303 → the client doesn't refetch /api/state.

View file

@ -38,15 +38,9 @@ pub fn current_flake_rev(hyperhive_flake: &str) -> Option<String> {
/// deployed (i.e. the applied HEAD differs from the sha currently locked in /// deployed (i.e. the applied HEAD differs from the sha currently locked in
/// meta's flake.lock). This is the semantic the dashboard `needs_update` chip /// meta's flake.lock). This is the semantic the dashboard `needs_update` chip
/// conveys: "there is a config change ready to apply via rebuild." /// conveys: "there is a config change ready to apply via rebuild."
/// #[must_use]
/// Async on purpose: this runs per agent inside `container_view::build_all`, pub fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> bool {
/// which fires on the ~10s dashboard sweep, every `AgentStatus` request, and let applied_head = std::process::Command::new("git")
/// every `rescan_containers_and_emit` after a lifecycle step. A synchronous
/// `git` fork here blocks a tokio worker for the whole exec — under
/// nix-build disk saturation that's long enough that concurrent sweeps
/// starved the runtime and stalled the per-agent sockets.
pub async fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> bool {
let applied_head = tokio::process::Command::new("git")
.args([ .args([
"-C", "-C",
&format!("/var/lib/hyperhive/applied/{name}"), &format!("/var/lib/hyperhive/applied/{name}"),
@ -54,7 +48,6 @@ pub async fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> boo
"HEAD", "HEAD",
]) ])
.output() .output()
.await
.ok() .ok()
.filter(|o| o.status.success()) .filter(|o| o.status.success())
.and_then(|o| String::from_utf8(o.stdout).ok()) .and_then(|o| String::from_utf8(o.stdout).ok())

View file

@ -70,7 +70,7 @@ pub async fn build_all(coord: &Coordinator) -> Vec<ContainerView> {
let deployed_full = locked let deployed_full = locked
.get(&format!("agent-{logical}")) .get(&format!("agent-{logical}"))
.map(std::string::String::as_str); .map(std::string::String::as_str);
let needs_update = crate::auto_update::agent_config_pending(&logical, deployed_full).await; let needs_update = crate::auto_update::agent_config_pending(&logical, deployed_full);
let deployed_sha = deployed_full.map(|s| s[..s.len().min(12)].to_owned()); let deployed_sha = deployed_full.map(|s| s[..s.len().min(12)].to_owned());
let pending_reminders = coord let pending_reminders = coord
.broker .broker