diff --git a/hive-matrix-mcp/src/client.rs b/hive-matrix-mcp/src/client.rs index 5303341e..76447820 100644 --- a/hive-matrix-mcp/src/client.rs +++ b/hive-matrix-mcp/src/client.rs @@ -29,6 +29,25 @@ use matrix_sdk::{ use serde::Deserialize; use tokio::fs; +/// Sentinel returned when `build_and_restore` detects that the token is +/// permanently invalid (M_UNKNOWN_TOKEN). The token has already been +/// removed from disk. Callers should NOT retry — the account needs +/// re-provisioning by hive-c0re. +/// +/// Distinct from the general `anyhow::Error` path so callers can use +/// `err.downcast_ref::()` to distinguish "retry +/// won't help" from a transient network/DNS/5xx failure. +#[derive(Debug)] +pub struct PermanentBringUpError(pub String); + +impl std::fmt::Display for PermanentBringUpError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for PermanentBringUpError {} + /// Subset of the `/_matrix/client/v3/account/whoami` response we care /// about. matrix-spec field names; `device_id` is optional per spec /// (servers MAY omit it for legacy bearer scopes) but tuwunel always @@ -112,11 +131,11 @@ pub async fn build_and_restore( // Secondary: token removed (so it's cleanly skipped next boot // rather than re-erroring); leave the sdk state in place in case // the operator re-provisions a fresh token for the same device. - // Return Err so the caller logs + skips this one account and the - // daemon keeps serving the primary and any other healthy account. - return Err(anyhow!( - "matrix token rejected (M_UNKNOWN_TOKEN); removed stale token, skipping account" - )); + // Return a PermanentBringUpError so the caller can distinguish + // "don't retry" from a transient network/DNS failure. + return Err(anyhow::Error::new(PermanentBringUpError( + "matrix token rejected (M_UNKNOWN_TOKEN); removed stale token, skipping account".into(), + ))); } return Err(e); } diff --git a/hive-matrix-mcp/src/main.rs b/hive-matrix-mcp/src/main.rs index 57043435..db60ae99 100644 --- a/hive-matrix-mcp/src/main.rs +++ b/hive-matrix-mcp/src/main.rs @@ -39,6 +39,7 @@ mod timeline; mod wake; use accounts::{AccountCfg, Registry}; +use client::PermanentBringUpError; /// A per-account sync loop, boxed so loops for N accounts can be driven /// concurrently on the main task. Deliberately NOT `Send`: matrix-sdk's @@ -52,6 +53,14 @@ type SyncLoop = std::pin::Pin>>> /// a tiny file) is negligible. const ACCOUNTS_HEARTBEAT_SECS: u64 = 30; +/// Retry delays (seconds) for transient secondary-account failures. +/// After the initial attempt we back off through these before giving up +/// and skipping the account for the rest of the daemon lifetime. +/// Transient = anything that is NOT a PermanentBringUpError (bad/expired +/// token). A down homeserver at DNS-not-ready boot time is the typical +/// case; the total wait is ~52s before we give up. +const SECONDARY_RETRY_DELAYS_SECS: &[u64] = &[2, 5, 15, 30]; + #[tokio::main] async fn main() -> Result<()> { tracing_subscriber::fmt() @@ -75,7 +84,7 @@ async fn main() -> Result<()> { // Account-tag the wakes only in multi-account mode so single- // account wake bodies stay byte-identical to the legacy format. let tag = multi.then(|| cfg.name.clone()); - match bring_up_account(&cfg, &hyperhive_socket, tag, is_primary).await { + match bring_up_account(&cfg, &hyperhive_socket, tag.clone(), is_primary).await { Ok(Some((client, sync_loop))) => { registry.insert(cfg.name, client); sync_loops.push(sync_loop); @@ -96,10 +105,71 @@ async fn main() -> Result<()> { } // The primary failing to restore is fatal (propagate so // systemd retries on a transient blip — matches legacy - // behaviour); a secondary failing is logged and skipped. + // behaviour); a secondary failing is retried with backoff + // before being skipped for this daemon lifetime. Err(e) if is_primary => return Err(e.context("bring up primary matrix account")), Err(e) => { - tracing::error!(account = %cfg.name, error = %format!("{e:#}"), "secondary matrix account failed to restore; skipping"); + // Permanent failure (bad/expired token already removed): + // no point retrying. + if e.downcast_ref::().is_some() { + tracing::error!( + account = %cfg.name, + error = %format!("{e:#}"), + "secondary matrix account: permanent failure; skipping" + ); + continue; + } + // Transient failure: retry with backoff before giving up. + tracing::warn!( + account = %cfg.name, + error = %format!("{e:#}"), + "secondary matrix account bring-up failed (transient); will retry" + ); + let mut recovered = false; + for &delay in SECONDARY_RETRY_DELAYS_SECS { + tracing::info!( + account = %cfg.name, + delay_s = delay, + "retrying secondary account bring-up after backoff" + ); + tokio::time::sleep(std::time::Duration::from_secs(delay)).await; + match bring_up_account(&cfg, &hyperhive_socket, tag.clone(), false).await { + Ok(Some((client, sync_loop))) => { + tracing::info!(account = %cfg.name, "secondary matrix account recovered"); + registry.insert(cfg.name.clone(), client); + sync_loops.push(sync_loop); + recovered = true; + break; + } + Ok(None) => { + tracing::warn!(account = %cfg.name, "secondary matrix account has no token; skipping"); + break; + } + Err(re) if re.downcast_ref::().is_some() => { + tracing::error!( + account = %cfg.name, + error = %format!("{re:#}"), + "secondary matrix account: permanent failure on retry; skipping" + ); + break; + } + Err(re) => { + tracing::warn!( + account = %cfg.name, + error = %format!("{re:#}"), + "secondary matrix account bring-up still failing (transient)" + ); + } + } + } + if !recovered { + tracing::error!( + account = %cfg.name, + retries = SECONDARY_RETRY_DELAYS_SECS.len(), + "secondary matrix account failed after all retries; skipping for this daemon lifetime" + ); + } + continue; } } }