From b889f403d541eb23e7be1a1ff3cce9669d930f7b Mon Sep 17 00:00:00 2001 From: iris Date: Thu, 16 Jul 2026 20:04:35 +0200 Subject: [PATCH 1/5] sweep: wire matrix::ensure_all() into the warning-banner registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fast-follow for #2289 — matrix::ensure_all() (admin user, per-agent sync, hive Space + chat-room provisioning/invites) ran periodically every 30 minutes but only ever warn!'d to the journal on failure, so a persistent problem (missing invites, broken admin token, etc.) was invisible to the operator. ensure_all() now returns bool (aggregate ok/fail across every sub-step) instead of (), and both call sites in main.rs feed that into a debounced SweepHealth("matrix_ensure_all", warn, threshold=2) — matches the existing knowledge_pull pattern. A lone bad sweep self-heals silently; two consecutive failures raise a banner that clears on the next clean sweep. forge::ensure_all()'s remaining independent steps are still open — that sweep only runs once at startup (no periodic loop), so the debounced pattern doesn't map as directly; left for a follow-up. --- hive-c0re/src/main.rs | 39 +++++++++++++++++++++++++++++++++++++-- hive-c0re/src/matrix.rs | 35 +++++++++++++++++++++++++---------- 2 files changed, 62 insertions(+), 12 deletions(-) diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index d3dd110e..800c2934 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -414,11 +414,46 @@ async fn cmd_serve( let mut matrix_shutdown = coord.shutdown_rx(); tokio::spawn(async move { let interval = std::time::Duration::from_mins(30); - matrix::ensure_all().await; + // Debounced banner: a lone bad sweep (homeserver mid-restart, a + // transient HTTP blip) shouldn't flap the dashboard, but a sweep + // that's been failing for hours (missing agent invites, a broken + // admin token) should surface. Cleared the moment a sweep is clean. + let mut health = sweep_health::SweepHealth::new("matrix_ensure_all", "warn", 2); + if matrix::ensure_all().await { + health.record_ok(); + } else { + health.record_err(|ctx| { + let age = ctx.since_last_ok.map_or_else( + || "no success this session".to_owned(), + |d| format!("last ok {} ago", sweep_health::fmt_age(d)), + ); + format!( + "matrix user/space sweep failing ({} consecutive, {age}) \ + — some agents may be missing matrix accounts, space membership, \ + or chat-room invites", + ctx.consecutive + ) + }); + } loop { tokio::select! { () = tokio::time::sleep(interval) => { - matrix::ensure_all().await; + if matrix::ensure_all().await { + health.record_ok(); + } else { + health.record_err(|ctx| { + let age = ctx.since_last_ok.map_or_else( + || "no success this session".to_owned(), + |d| format!("last ok {} ago", sweep_health::fmt_age(d)), + ); + format!( + "matrix user/space sweep failing ({} consecutive, {age}) \ + — some agents may be missing matrix accounts, space membership, \ + or chat-room invites", + ctx.consecutive + ) + }); + } } _ = matrix_shutdown.changed() => { tracing::info!("matrix ensure_all: shutdown signal received"); diff --git a/hive-c0re/src/matrix.rs b/hive-c0re/src/matrix.rs index cbbc7123..2de2078e 100644 --- a/hive-c0re/src/matrix.rs +++ b/hive-c0re/src/matrix.rs @@ -1503,20 +1503,28 @@ async fn resolve_room_alias( } /// Sweep every existing container (manager + sub-agents) and ensure -/// each has a matrix user + token on the local homeserver. Called once -/// at hive-c0re startup, alongside `forge::ensure_all`. No-op when the +/// each has a matrix user + token on the local homeserver. Called at +/// hive-c0re startup, alongside `forge::ensure_all`, and then +/// periodically (see the caller in `main.rs`). No-op when the /// hive-matrix container isn't running. Per-step failures are logged /// but don't abort the sweep. -pub async fn ensure_all() { +/// +/// Returns `true` when every step of the sweep succeeded, `false` when at +/// least one step failed — the caller feeds this into a +/// [`crate::stats::sweep_health::SweepHealth`] to raise a debounced +/// dashboard banner on persistent failure (this sweep re-runs every 30 +/// minutes, so a one-off blip self-heals without ever bannering). +pub async fn ensure_all() -> bool { if !is_present().await { tracing::debug!("matrix: hive-matrix container absent, skipping user sweep"); - return; + return true; } + let mut ok = true; let register_token = match ensure_register_token() { Ok(t) => t, Err(e) => { tracing::warn!(error = ?e, "matrix: ensure_register_token failed"); - return; + return false; } }; // One HTTP client for the whole sweep — connection pool is @@ -1528,7 +1536,7 @@ pub async fn ensure_all() { Ok(c) => c, Err(e) => { tracing::warn!(error = ?e, "matrix: build HTTP client failed; skipping sweep"); - return; + return false; } }; // Provision hive admin user FIRST so it's the first registered @@ -1536,10 +1544,11 @@ pub async fn ensure_all() { // registered user admin automatically). if let Err(e) = ensure_admin_user(&client, ®ister_token).await { tracing::warn!(error = ?e, "matrix: ensure_admin_user failed"); + ok = false; } let Ok(containers) = crate::lifecycle::list().await else { tracing::warn!("matrix: nixos-container list failed; skipping user sweep"); - return; + return false; }; let mut agent_names: Vec = Vec::new(); for c in &containers { @@ -1556,7 +1565,7 @@ pub async fn ensure_all() { Ok(t) => t, Err(e) => { tracing::warn!(error = ?e, "matrix: skipping hive space provisioning (no admin token)"); - return; + return false; } }; // server_name first — the agent invites need it (fully-qualified user ids). @@ -1564,14 +1573,14 @@ pub async fn ensure_all() { Ok(s) => s, Err(e) => { tracing::warn!(error = ?e, "matrix: discover_server_name failed; skipping space provisioning"); - return; + return false; } }; let room_id = match ensure_hive_space(&client, &admin_token).await { Ok(id) => id, Err(e) => { tracing::warn!(error = ?e, "matrix: ensure_hive_space failed"); - return; + return false; } }; // Invite @hive admin first, then all agents. @@ -1585,10 +1594,12 @@ pub async fn ensure_all() { .await { tracing::warn!(error = ?e, "matrix: invite @hive to space failed"); + ok = false; } for name in &agent_names { if let Err(e) = invite_to_room(&client, &admin_token, &room_id, name, &server_name).await { tracing::warn!(%name, error = ?e, "matrix: invite agent to space failed"); + ok = false; } } @@ -1609,19 +1620,23 @@ pub async fn ensure_all() { .await { tracing::warn!(error = ?e, "matrix: invite @hive to chat room failed"); + ok = false; } for name in &agent_names { if let Err(e) = invite_to_room(&client, &admin_token, &chat_room_id, name, &server_name).await { tracing::warn!(%name, error = ?e, "matrix: invite agent to chat room failed"); + ok = false; } } } Err(e) => { tracing::warn!(error = ?e, "matrix: ensure_hive_chat_room failed"); + ok = false; } } + ok } #[cfg(test)] From 84ea7b8e7d1426e0db9d4b49f0973d252ba499f1 Mon Sep 17 00:00:00 2001 From: iris Date: Thu, 16 Jul 2026 20:14:00 +0200 Subject: [PATCH 2/5] sweep: dedupe matrix ensure_all banner closure --- hive-c0re/src/main.rs | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index 800c2934..26a98ec4 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -401,6 +401,21 @@ async fn cmd_serve( } } }); + /// Banner message for a failing matrix `ensure_all` sweep, shared by + /// both the initial and periodic `record_err` call sites below so the + /// wording can't drift between them. + fn matrix_sweep_banner(ctx: sweep_health::SweepFailure) -> String { + let age = ctx.since_last_ok.map_or_else( + || "no success this session".to_owned(), + |d| format!("last ok {} ago", sweep_health::fmt_age(d)), + ); + format!( + "matrix user/space sweep failing ({} consecutive, {age}) \ + — some agents may be missing matrix accounts, space membership, \ + or chat-room invites", + ctx.consecutive + ) + } // Matrix user sweep: same shape — ensure every container has // an account on the local matrix-tuwunel homeserver with an // access_token persisted to `/matrix-token`. No-op when @@ -422,18 +437,7 @@ async fn cmd_serve( if matrix::ensure_all().await { health.record_ok(); } else { - health.record_err(|ctx| { - let age = ctx.since_last_ok.map_or_else( - || "no success this session".to_owned(), - |d| format!("last ok {} ago", sweep_health::fmt_age(d)), - ); - format!( - "matrix user/space sweep failing ({} consecutive, {age}) \ - — some agents may be missing matrix accounts, space membership, \ - or chat-room invites", - ctx.consecutive - ) - }); + health.record_err(matrix_sweep_banner); } loop { tokio::select! { @@ -441,18 +445,7 @@ async fn cmd_serve( if matrix::ensure_all().await { health.record_ok(); } else { - health.record_err(|ctx| { - let age = ctx.since_last_ok.map_or_else( - || "no success this session".to_owned(), - |d| format!("last ok {} ago", sweep_health::fmt_age(d)), - ); - format!( - "matrix user/space sweep failing ({} consecutive, {age}) \ - — some agents may be missing matrix accounts, space membership, \ - or chat-room invites", - ctx.consecutive - ) - }); + health.record_err(matrix_sweep_banner); } } _ = matrix_shutdown.changed() => { From cfed36582ebd381aa030a5c5752df108a3c904bb Mon Sep 17 00:00:00 2001 From: iris Date: Fri, 17 Jul 2026 01:10:48 +0200 Subject: [PATCH 3/5] fix: move matrix_sweep_banner to module scope (items_after_statements) --- hive-c0re/src/main.rs | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index 26a98ec4..a97aff7b 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -197,6 +197,22 @@ async fn main() -> Result<()> { } } +/// Banner message for a failing matrix `ensure_all` sweep, shared by both +/// the initial and periodic `record_err` call sites in `cmd_serve` so the +/// wording can't drift between them. +fn matrix_sweep_banner(ctx: sweep_health::SweepFailure) -> String { + let age = ctx.since_last_ok.map_or_else( + || "no success this session".to_owned(), + |d| format!("last ok {} ago", sweep_health::fmt_age(d)), + ); + format!( + "matrix user/space sweep failing ({} consecutive, {age}) \ + — some agents may be missing matrix accounts, space membership, \ + or chat-room invites", + ctx.consecutive + ) +} + /// Start the coordinator daemon: open the broker, run migrations, spawn /// background tasks (auto-update, vacuums, crash-watcher, reminder-scheduler, /// dashboard), then serve the admin socket until a signal arrives. @@ -401,21 +417,6 @@ async fn cmd_serve( } } }); - /// Banner message for a failing matrix `ensure_all` sweep, shared by - /// both the initial and periodic `record_err` call sites below so the - /// wording can't drift between them. - fn matrix_sweep_banner(ctx: sweep_health::SweepFailure) -> String { - let age = ctx.since_last_ok.map_or_else( - || "no success this session".to_owned(), - |d| format!("last ok {} ago", sweep_health::fmt_age(d)), - ); - format!( - "matrix user/space sweep failing ({} consecutive, {age}) \ - — some agents may be missing matrix accounts, space membership, \ - or chat-room invites", - ctx.consecutive - ) - } // Matrix user sweep: same shape — ensure every container has // an account on the local matrix-tuwunel homeserver with an // access_token persisted to `/matrix-token`. No-op when From 8cb130b8d703cbb34528524081a07799b639ac29 Mon Sep 17 00:00:00 2001 From: iris Date: Fri, 17 Jul 2026 01:27:13 +0200 Subject: [PATCH 4/5] fix: allow too_many_lines on matrix::ensure_all (108/100, aggregate-bool sweep) --- hive-c0re/src/matrix.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hive-c0re/src/matrix.rs b/hive-c0re/src/matrix.rs index 2de2078e..b2af5b27 100644 --- a/hive-c0re/src/matrix.rs +++ b/hive-c0re/src/matrix.rs @@ -1514,6 +1514,13 @@ async fn resolve_room_alias( /// [`crate::stats::sweep_health::SweepHealth`] to raise a debounced /// dashboard banner on persistent failure (this sweep re-runs every 30 /// minutes, so a one-off blip self-heals without ever bannering). +#[allow( + clippy::too_many_lines, + reason = "sequential provisioning sweep — admin user, per-agent user/token \ + sync, space + chat-room membership — each step's failure handling \ + (log + continue + flip `ok`) is inherent to the aggregate bool \ + contract, not something to factor away just to hit a line count" +)] pub async fn ensure_all() -> bool { if !is_present().await { tracing::debug!("matrix: hive-matrix container absent, skipping user sweep"); From bf913df67aad9e7fbcb7207f35b418ec26852f2e Mon Sep 17 00:00:00 2001 From: iris Date: Fri, 17 Jul 2026 01:56:15 +0200 Subject: [PATCH 5/5] refactor: extract provision_space from matrix::ensure_all instead of allow Per mara's standing calibration (#2463): extraction > silencing for too_many_lines. Splits the space + chat-room provisioning tail into its own fn, bringing ensure_all back under the 100-line threshold without an #[allow]. --- hive-c0re/src/matrix.rs | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/hive-c0re/src/matrix.rs b/hive-c0re/src/matrix.rs index b2af5b27..156d50b6 100644 --- a/hive-c0re/src/matrix.rs +++ b/hive-c0re/src/matrix.rs @@ -1514,13 +1514,6 @@ async fn resolve_room_alias( /// [`crate::stats::sweep_health::SweepHealth`] to raise a debounced /// dashboard banner on persistent failure (this sweep re-runs every 30 /// minutes, so a one-off blip self-heals without ever bannering). -#[allow( - clippy::too_many_lines, - reason = "sequential provisioning sweep — admin user, per-agent user/token \ - sync, space + chat-room membership — each step's failure handling \ - (log + continue + flip `ok`) is inherent to the aggregate bool \ - contract, not something to factor away just to hit a line count" -)] pub async fn ensure_all() -> bool { if !is_present().await { tracing::debug!("matrix: hive-matrix container absent, skipping user sweep"); @@ -1566,7 +1559,19 @@ pub async fn ensure_all() -> bool { agent_names.push(name.to_owned()); } - // Provision the hive Space and invite all agents (+ the admin account). + if !provision_space(&client, &agent_names).await { + ok = false; + } + ok +} + +/// Provision the hive Space + default chat room and invite every agent +/// (+ the admin account) to both. Split out of [`ensure_all`] purely to +/// keep that function under the `too_many_lines` threshold — this is the +/// tail half of the same sequential sweep and shares its aggregate-bool, +/// log-and-continue failure handling. +async fn provision_space(client: &reqwest::Client, agent_names: &[String]) -> bool { + let mut ok = true; // server_name is needed to form full Matrix user IDs for invites. let admin_token = match read_admin_token() { Ok(t) => t, @@ -1576,14 +1581,14 @@ pub async fn ensure_all() -> bool { } }; // server_name first — the agent invites need it (fully-qualified user ids). - let server_name = match discover_server_name(&client).await { + let server_name = match discover_server_name(client).await { Ok(s) => s, Err(e) => { tracing::warn!(error = ?e, "matrix: discover_server_name failed; skipping space provisioning"); return false; } }; - let room_id = match ensure_hive_space(&client, &admin_token).await { + let room_id = match ensure_hive_space(client, &admin_token).await { Ok(id) => id, Err(e) => { tracing::warn!(error = ?e, "matrix: ensure_hive_space failed"); @@ -1592,7 +1597,7 @@ pub async fn ensure_all() -> bool { }; // Invite @hive admin first, then all agents. if let Err(e) = invite_to_room( - &client, + client, &admin_token, &room_id, HIVE_ADMIN_LOCALPART, @@ -1603,8 +1608,8 @@ pub async fn ensure_all() -> bool { tracing::warn!(error = ?e, "matrix: invite @hive to space failed"); ok = false; } - for name in &agent_names { - if let Err(e) = invite_to_room(&client, &admin_token, &room_id, name, &server_name).await { + for name in agent_names { + if let Err(e) = invite_to_room(client, &admin_token, &room_id, name, &server_name).await { tracing::warn!(%name, error = ?e, "matrix: invite agent to space failed"); ok = false; } @@ -1615,10 +1620,10 @@ pub async fn ensure_all() -> bool { // rooms to chat in (Matrix semantics — children aren't auto-joined), so // without this the Space is empty. The restricted join rule additionally // lets the operator (a Space member) join from the Space hierarchy. - match ensure_hive_chat_room(&client, &admin_token, &room_id, &server_name).await { + match ensure_hive_chat_room(client, &admin_token, &room_id, &server_name).await { Ok(chat_room_id) => { if let Err(e) = invite_to_room( - &client, + client, &admin_token, &chat_room_id, HIVE_ADMIN_LOCALPART, @@ -1629,9 +1634,9 @@ pub async fn ensure_all() -> bool { tracing::warn!(error = ?e, "matrix: invite @hive to chat room failed"); ok = false; } - for name in &agent_names { + for name in agent_names { if let Err(e) = - invite_to_room(&client, &admin_token, &chat_room_id, name, &server_name).await + invite_to_room(client, &admin_token, &chat_room_id, name, &server_name).await { tracing::warn!(%name, error = ?e, "matrix: invite agent to chat room failed"); ok = false;