diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index d3dd110e..800c2934 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -414,11 +414,46 @@ async fn cmd_serve( let mut matrix_shutdown = coord.shutdown_rx(); tokio::spawn(async move { let interval = std::time::Duration::from_mins(30); - matrix::ensure_all().await; + // Debounced banner: a lone bad sweep (homeserver mid-restart, a + // transient HTTP blip) shouldn't flap the dashboard, but a sweep + // that's been failing for hours (missing agent invites, a broken + // admin token) should surface. Cleared the moment a sweep is clean. + let mut health = sweep_health::SweepHealth::new("matrix_ensure_all", "warn", 2); + if matrix::ensure_all().await { + health.record_ok(); + } else { + health.record_err(|ctx| { + let age = ctx.since_last_ok.map_or_else( + || "no success this session".to_owned(), + |d| format!("last ok {} ago", sweep_health::fmt_age(d)), + ); + format!( + "matrix user/space sweep failing ({} consecutive, {age}) \ + — some agents may be missing matrix accounts, space membership, \ + or chat-room invites", + ctx.consecutive + ) + }); + } loop { tokio::select! { () = tokio::time::sleep(interval) => { - matrix::ensure_all().await; + if matrix::ensure_all().await { + health.record_ok(); + } else { + health.record_err(|ctx| { + let age = ctx.since_last_ok.map_or_else( + || "no success this session".to_owned(), + |d| format!("last ok {} ago", sweep_health::fmt_age(d)), + ); + format!( + "matrix user/space sweep failing ({} consecutive, {age}) \ + — some agents may be missing matrix accounts, space membership, \ + or chat-room invites", + ctx.consecutive + ) + }); + } } _ = matrix_shutdown.changed() => { tracing::info!("matrix ensure_all: shutdown signal received"); diff --git a/hive-c0re/src/matrix.rs b/hive-c0re/src/matrix.rs index cbbc7123..2de2078e 100644 --- a/hive-c0re/src/matrix.rs +++ b/hive-c0re/src/matrix.rs @@ -1503,20 +1503,28 @@ async fn resolve_room_alias( } /// Sweep every existing container (manager + sub-agents) and ensure -/// each has a matrix user + token on the local homeserver. Called once -/// at hive-c0re startup, alongside `forge::ensure_all`. No-op when the +/// each has a matrix user + token on the local homeserver. Called at +/// hive-c0re startup, alongside `forge::ensure_all`, and then +/// periodically (see the caller in `main.rs`). No-op when the /// hive-matrix container isn't running. Per-step failures are logged /// but don't abort the sweep. -pub async fn ensure_all() { +/// +/// Returns `true` when every step of the sweep succeeded, `false` when at +/// least one step failed — the caller feeds this into a +/// [`crate::stats::sweep_health::SweepHealth`] to raise a debounced +/// dashboard banner on persistent failure (this sweep re-runs every 30 +/// minutes, so a one-off blip self-heals without ever bannering). +pub async fn ensure_all() -> bool { if !is_present().await { tracing::debug!("matrix: hive-matrix container absent, skipping user sweep"); - return; + return true; } + let mut ok = true; let register_token = match ensure_register_token() { Ok(t) => t, Err(e) => { tracing::warn!(error = ?e, "matrix: ensure_register_token failed"); - return; + return false; } }; // One HTTP client for the whole sweep — connection pool is @@ -1528,7 +1536,7 @@ pub async fn ensure_all() { Ok(c) => c, Err(e) => { tracing::warn!(error = ?e, "matrix: build HTTP client failed; skipping sweep"); - return; + return false; } }; // Provision hive admin user FIRST so it's the first registered @@ -1536,10 +1544,11 @@ pub async fn ensure_all() { // registered user admin automatically). if let Err(e) = ensure_admin_user(&client, ®ister_token).await { tracing::warn!(error = ?e, "matrix: ensure_admin_user failed"); + ok = false; } let Ok(containers) = crate::lifecycle::list().await else { tracing::warn!("matrix: nixos-container list failed; skipping user sweep"); - return; + return false; }; let mut agent_names: Vec = Vec::new(); for c in &containers { @@ -1556,7 +1565,7 @@ pub async fn ensure_all() { Ok(t) => t, Err(e) => { tracing::warn!(error = ?e, "matrix: skipping hive space provisioning (no admin token)"); - return; + return false; } }; // server_name first — the agent invites need it (fully-qualified user ids). @@ -1564,14 +1573,14 @@ pub async fn ensure_all() { Ok(s) => s, Err(e) => { tracing::warn!(error = ?e, "matrix: discover_server_name failed; skipping space provisioning"); - return; + return false; } }; let room_id = match ensure_hive_space(&client, &admin_token).await { Ok(id) => id, Err(e) => { tracing::warn!(error = ?e, "matrix: ensure_hive_space failed"); - return; + return false; } }; // Invite @hive admin first, then all agents. @@ -1585,10 +1594,12 @@ pub async fn ensure_all() { .await { tracing::warn!(error = ?e, "matrix: invite @hive to space failed"); + ok = false; } for name in &agent_names { if let Err(e) = invite_to_room(&client, &admin_token, &room_id, name, &server_name).await { tracing::warn!(%name, error = ?e, "matrix: invite agent to space failed"); + ok = false; } } @@ -1609,19 +1620,23 @@ pub async fn ensure_all() { .await { tracing::warn!(error = ?e, "matrix: invite @hive to chat room failed"); + ok = false; } for name in &agent_names { if let Err(e) = invite_to_room(&client, &admin_token, &chat_room_id, name, &server_name).await { tracing::warn!(%name, error = ?e, "matrix: invite agent to chat room failed"); + ok = false; } } } Err(e) => { tracing::warn!(error = ?e, "matrix: ensure_hive_chat_room failed"); + ok = false; } } + ok } #[cfg(test)]