From a1cd50610af7ccfc9bdeb0b7425c8b7b7359aeaa Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 4 Jul 2026 12:16:44 +0200 Subject: [PATCH 1/6] fix(#2095): retry secondary matrix account bring-up on transient failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a transient error (network blip, DNS not ready, homeserver 5xx) a secondary account's bring-up was immediately skipped for the entire daemon lifetime. This bit janet's catgirl account repeatedly when the host DNS resolver wasn't ready at daemon start — the account would silently disappear until the next restart. Add a PermanentBringUpError sentinel in client.rs so callers can distinguish M_UNKNOWN_TOKEN (stale/expired token — permanent, don't retry) from transient network/homeserver errors. In main.rs, replace the immediate skip with a bounded retry loop for secondary accounts: up to 4 attempts with 2s/5s/15s/30s backoffs (~52s total wait). On a transient error the daemon now stays alive serving the primary and any other healthy accounts while the failing secondary gets another chance. Permanent failures (PermanentBringUpError) still skip immediately with no retry. The primary account keep its existing behaviour: fatal on non-permanent error so systemd restarts the whole daemon (systemd is the right retry mechanism for primary bring-up failure). --- hive-matrix-mcp/src/client.rs | 29 ++++++++++--- hive-matrix-mcp/src/main.rs | 76 +++++++++++++++++++++++++++++++++-- 2 files changed, 97 insertions(+), 8 deletions(-) diff --git a/hive-matrix-mcp/src/client.rs b/hive-matrix-mcp/src/client.rs index 5303341e..76447820 100644 --- a/hive-matrix-mcp/src/client.rs +++ b/hive-matrix-mcp/src/client.rs @@ -29,6 +29,25 @@ use matrix_sdk::{ use serde::Deserialize; use tokio::fs; +/// Sentinel returned when `build_and_restore` detects that the token is +/// permanently invalid (M_UNKNOWN_TOKEN). The token has already been +/// removed from disk. Callers should NOT retry — the account needs +/// re-provisioning by hive-c0re. +/// +/// Distinct from the general `anyhow::Error` path so callers can use +/// `err.downcast_ref::()` to distinguish "retry +/// won't help" from a transient network/DNS/5xx failure. +#[derive(Debug)] +pub struct PermanentBringUpError(pub String); + +impl std::fmt::Display for PermanentBringUpError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for PermanentBringUpError {} + /// Subset of the `/_matrix/client/v3/account/whoami` response we care /// about. matrix-spec field names; `device_id` is optional per spec /// (servers MAY omit it for legacy bearer scopes) but tuwunel always @@ -112,11 +131,11 @@ pub async fn build_and_restore( // Secondary: token removed (so it's cleanly skipped next boot // rather than re-erroring); leave the sdk state in place in case // the operator re-provisions a fresh token for the same device. - // Return Err so the caller logs + skips this one account and the - // daemon keeps serving the primary and any other healthy account. - return Err(anyhow!( - "matrix token rejected (M_UNKNOWN_TOKEN); removed stale token, skipping account" - )); + // Return a PermanentBringUpError so the caller can distinguish + // "don't retry" from a transient network/DNS failure. + return Err(anyhow::Error::new(PermanentBringUpError( + "matrix token rejected (M_UNKNOWN_TOKEN); removed stale token, skipping account".into(), + ))); } return Err(e); } diff --git a/hive-matrix-mcp/src/main.rs b/hive-matrix-mcp/src/main.rs index 57043435..db60ae99 100644 --- a/hive-matrix-mcp/src/main.rs +++ b/hive-matrix-mcp/src/main.rs @@ -39,6 +39,7 @@ mod timeline; mod wake; use accounts::{AccountCfg, Registry}; +use client::PermanentBringUpError; /// A per-account sync loop, boxed so loops for N accounts can be driven /// concurrently on the main task. Deliberately NOT `Send`: matrix-sdk's @@ -52,6 +53,14 @@ type SyncLoop = std::pin::Pin>>> /// a tiny file) is negligible. const ACCOUNTS_HEARTBEAT_SECS: u64 = 30; +/// Retry delays (seconds) for transient secondary-account failures. +/// After the initial attempt we back off through these before giving up +/// and skipping the account for the rest of the daemon lifetime. +/// Transient = anything that is NOT a PermanentBringUpError (bad/expired +/// token). A down homeserver at DNS-not-ready boot time is the typical +/// case; the total wait is ~52s before we give up. +const SECONDARY_RETRY_DELAYS_SECS: &[u64] = &[2, 5, 15, 30]; + #[tokio::main] async fn main() -> Result<()> { tracing_subscriber::fmt() @@ -75,7 +84,7 @@ async fn main() -> Result<()> { // Account-tag the wakes only in multi-account mode so single- // account wake bodies stay byte-identical to the legacy format. let tag = multi.then(|| cfg.name.clone()); - match bring_up_account(&cfg, &hyperhive_socket, tag, is_primary).await { + match bring_up_account(&cfg, &hyperhive_socket, tag.clone(), is_primary).await { Ok(Some((client, sync_loop))) => { registry.insert(cfg.name, client); sync_loops.push(sync_loop); @@ -96,10 +105,71 @@ async fn main() -> Result<()> { } // The primary failing to restore is fatal (propagate so // systemd retries on a transient blip — matches legacy - // behaviour); a secondary failing is logged and skipped. + // behaviour); a secondary failing is retried with backoff + // before being skipped for this daemon lifetime. Err(e) if is_primary => return Err(e.context("bring up primary matrix account")), Err(e) => { - tracing::error!(account = %cfg.name, error = %format!("{e:#}"), "secondary matrix account failed to restore; skipping"); + // Permanent failure (bad/expired token already removed): + // no point retrying. + if e.downcast_ref::().is_some() { + tracing::error!( + account = %cfg.name, + error = %format!("{e:#}"), + "secondary matrix account: permanent failure; skipping" + ); + continue; + } + // Transient failure: retry with backoff before giving up. + tracing::warn!( + account = %cfg.name, + error = %format!("{e:#}"), + "secondary matrix account bring-up failed (transient); will retry" + ); + let mut recovered = false; + for &delay in SECONDARY_RETRY_DELAYS_SECS { + tracing::info!( + account = %cfg.name, + delay_s = delay, + "retrying secondary account bring-up after backoff" + ); + tokio::time::sleep(std::time::Duration::from_secs(delay)).await; + match bring_up_account(&cfg, &hyperhive_socket, tag.clone(), false).await { + Ok(Some((client, sync_loop))) => { + tracing::info!(account = %cfg.name, "secondary matrix account recovered"); + registry.insert(cfg.name.clone(), client); + sync_loops.push(sync_loop); + recovered = true; + break; + } + Ok(None) => { + tracing::warn!(account = %cfg.name, "secondary matrix account has no token; skipping"); + break; + } + Err(re) if re.downcast_ref::().is_some() => { + tracing::error!( + account = %cfg.name, + error = %format!("{re:#}"), + "secondary matrix account: permanent failure on retry; skipping" + ); + break; + } + Err(re) => { + tracing::warn!( + account = %cfg.name, + error = %format!("{re:#}"), + "secondary matrix account bring-up still failing (transient)" + ); + } + } + } + if !recovered { + tracing::error!( + account = %cfg.name, + retries = SECONDARY_RETRY_DELAYS_SECS.len(), + "secondary matrix account failed after all retries; skipping for this daemon lifetime" + ); + } + continue; } } } From 317e545d7cf7b9d6e7137d5d44e80031f6815c4b Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 4 Jul 2026 12:30:47 +0200 Subject: [PATCH 2/6] fix: wrap M_UNKNOWN_TOKEN in backticks in doc comment (clippy::doc_markdown) --- hive-matrix-mcp/src/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive-matrix-mcp/src/client.rs b/hive-matrix-mcp/src/client.rs index 76447820..ca24e1c3 100644 --- a/hive-matrix-mcp/src/client.rs +++ b/hive-matrix-mcp/src/client.rs @@ -30,7 +30,7 @@ use serde::Deserialize; use tokio::fs; /// Sentinel returned when `build_and_restore` detects that the token is -/// permanently invalid (M_UNKNOWN_TOKEN). The token has already been +/// permanently invalid (`M_UNKNOWN_TOKEN`). The token has already been /// removed from disk. Callers should NOT retry — the account needs /// re-provisioning by hive-c0re. /// From 3de576141bba2247b0a0815918d730bcb0e7e9db Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 4 Jul 2026 12:40:00 +0200 Subject: [PATCH 3/6] refactor: extract bring_up_secondary_with_retry (fix clippy too_many_lines + needless_continue) --- hive-matrix-mcp/src/main.rs | 133 ++++++++++++++++++++---------------- 1 file changed, 73 insertions(+), 60 deletions(-) diff --git a/hive-matrix-mcp/src/main.rs b/hive-matrix-mcp/src/main.rs index db60ae99..e15d0983 100644 --- a/hive-matrix-mcp/src/main.rs +++ b/hive-matrix-mcp/src/main.rs @@ -109,67 +109,12 @@ async fn main() -> Result<()> { // before being skipped for this daemon lifetime. Err(e) if is_primary => return Err(e.context("bring up primary matrix account")), Err(e) => { - // Permanent failure (bad/expired token already removed): - // no point retrying. - if e.downcast_ref::().is_some() { - tracing::error!( - account = %cfg.name, - error = %format!("{e:#}"), - "secondary matrix account: permanent failure; skipping" - ); - continue; + if let Some((client, sync_loop)) = + bring_up_secondary_with_retry(&cfg, &hyperhive_socket, tag, e).await + { + registry.insert(cfg.name, client); + sync_loops.push(sync_loop); } - // Transient failure: retry with backoff before giving up. - tracing::warn!( - account = %cfg.name, - error = %format!("{e:#}"), - "secondary matrix account bring-up failed (transient); will retry" - ); - let mut recovered = false; - for &delay in SECONDARY_RETRY_DELAYS_SECS { - tracing::info!( - account = %cfg.name, - delay_s = delay, - "retrying secondary account bring-up after backoff" - ); - tokio::time::sleep(std::time::Duration::from_secs(delay)).await; - match bring_up_account(&cfg, &hyperhive_socket, tag.clone(), false).await { - Ok(Some((client, sync_loop))) => { - tracing::info!(account = %cfg.name, "secondary matrix account recovered"); - registry.insert(cfg.name.clone(), client); - sync_loops.push(sync_loop); - recovered = true; - break; - } - Ok(None) => { - tracing::warn!(account = %cfg.name, "secondary matrix account has no token; skipping"); - break; - } - Err(re) if re.downcast_ref::().is_some() => { - tracing::error!( - account = %cfg.name, - error = %format!("{re:#}"), - "secondary matrix account: permanent failure on retry; skipping" - ); - break; - } - Err(re) => { - tracing::warn!( - account = %cfg.name, - error = %format!("{re:#}"), - "secondary matrix account bring-up still failing (transient)" - ); - } - } - } - if !recovered { - tracing::error!( - account = %cfg.name, - retries = SECONDARY_RETRY_DELAYS_SECS.len(), - "secondary matrix account failed after all retries; skipping for this daemon lifetime" - ); - } - continue; } } } @@ -237,6 +182,74 @@ async fn main() -> Result<()> { Ok(()) } +/// Try to bring up a secondary account, retrying with exponential backoff +/// on transient failures (anything that is NOT a `PermanentBringUpError`). +/// Returns `Some((client, sync_loop))` on success, or `None` to signal +/// that the account should be skipped for this daemon lifetime (permanent +/// failure, no token, or all retries exhausted). +async fn bring_up_secondary_with_retry( + cfg: &AccountCfg, + hyperhive_socket: &std::path::Path, + tag: Option, + first_error: anyhow::Error, +) -> Option<(Client, SyncLoop)> { + // Permanent failure: the token was invalid/expired and has already been + // removed from disk. Retrying won't help — skip immediately. + if first_error.downcast_ref::().is_some() { + tracing::error!( + account = %cfg.name, + error = %format!("{first_error:#}"), + "secondary matrix account: permanent failure; skipping" + ); + return None; + } + // Transient failure (network/DNS/homeserver 5xx): retry with backoff. + tracing::warn!( + account = %cfg.name, + error = %format!("{first_error:#}"), + "secondary matrix account bring-up failed (transient); will retry" + ); + for &delay in SECONDARY_RETRY_DELAYS_SECS { + tracing::info!( + account = %cfg.name, + delay_s = delay, + "retrying secondary account bring-up after backoff" + ); + tokio::time::sleep(std::time::Duration::from_secs(delay)).await; + match bring_up_account(cfg, hyperhive_socket, tag.clone(), false).await { + Ok(Some((client, sync_loop))) => { + tracing::info!(account = %cfg.name, "secondary matrix account recovered"); + return Some((client, sync_loop)); + } + Ok(None) => { + tracing::warn!(account = %cfg.name, "secondary matrix account has no token; skipping"); + return None; + } + Err(re) if re.downcast_ref::().is_some() => { + tracing::error!( + account = %cfg.name, + error = %format!("{re:#}"), + "secondary matrix account: permanent failure on retry; skipping" + ); + return None; + } + Err(re) => { + tracing::warn!( + account = %cfg.name, + error = %format!("{re:#}"), + "secondary matrix account bring-up still failing (transient)" + ); + } + } + } + tracing::error!( + account = %cfg.name, + retries = SECONDARY_RETRY_DELAYS_SECS.len(), + "secondary matrix account failed after all retries; skipping for this daemon lifetime" + ); + None +} + /// Restore one account's client (when its token exists), install its /// message handler, and build its sync loop. Returns /// `Ok(Some((client, sync_loop)))` when the account came up, `Ok(None)` From 793f6ce185407989ed24555127d281806058f2e5 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 4 Jul 2026 18:40:34 +0200 Subject: [PATCH 4/6] fix(fmt): split method chain in bring_up_secondary_with_retry for treefmt --- hive-matrix-mcp/src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hive-matrix-mcp/src/main.rs b/hive-matrix-mcp/src/main.rs index e15d0983..12d23bee 100644 --- a/hive-matrix-mcp/src/main.rs +++ b/hive-matrix-mcp/src/main.rs @@ -195,7 +195,10 @@ async fn bring_up_secondary_with_retry( ) -> Option<(Client, SyncLoop)> { // Permanent failure: the token was invalid/expired and has already been // removed from disk. Retrying won't help — skip immediately. - if first_error.downcast_ref::().is_some() { + if first_error + .downcast_ref::() + .is_some() + { tracing::error!( account = %cfg.name, error = %format!("{first_error:#}"), From 7bdc3a827eac4d515f8278ca0b04bf9e613c36c8 Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 9 Jul 2026 00:28:14 +0200 Subject: [PATCH 5/6] ci: retrigger nix flake check (wasip2 path missing from lix db) From 8d2ebcf51f9dca600cd5af5d7cf7a265774f7bb8 Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 9 Jul 2026 00:40:54 +0200 Subject: [PATCH 6/6] fix(clippy): wrap PermanentBringUpError in backticks in doc comment (main.rs:59) --- hive-matrix-mcp/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive-matrix-mcp/src/main.rs b/hive-matrix-mcp/src/main.rs index 12d23bee..a6af4a57 100644 --- a/hive-matrix-mcp/src/main.rs +++ b/hive-matrix-mcp/src/main.rs @@ -56,7 +56,7 @@ const ACCOUNTS_HEARTBEAT_SECS: u64 = 30; /// Retry delays (seconds) for transient secondary-account failures. /// After the initial attempt we back off through these before giving up /// and skipping the account for the rest of the daemon lifetime. -/// Transient = anything that is NOT a PermanentBringUpError (bad/expired +/// Transient = anything that is NOT a `PermanentBringUpError` (bad/expired /// token). A down homeserver at DNS-not-ready boot time is the typical /// case; the total wait is ~52s before we give up. const SECONDARY_RETRY_DELAYS_SECS: &[u64] = &[2, 5, 15, 30];