Compare commits

..
2 changed files with 8 additions and 113 deletions

View file

@ -29,25 +29,6 @@ use matrix_sdk::{
use serde::Deserialize;
use tokio::fs;
/// Sentinel returned when `build_and_restore` detects that the token is
/// permanently invalid (`M_UNKNOWN_TOKEN`). The token has already been
/// removed from disk. Callers should NOT retry — the account needs
/// re-provisioning by hive-c0re.
///
/// Distinct from the general `anyhow::Error` path so callers can use
/// `err.downcast_ref::<PermanentBringUpError>()` to distinguish "retry
/// won't help" from a transient network/DNS/5xx failure.
#[derive(Debug)]
pub struct PermanentBringUpError(pub String);
impl std::fmt::Display for PermanentBringUpError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
impl std::error::Error for PermanentBringUpError {}
/// Subset of the `/_matrix/client/v3/account/whoami` response we care
/// about. matrix-spec field names; `device_id` is optional per spec
/// (servers MAY omit it for legacy bearer scopes) but tuwunel always
@ -131,11 +112,11 @@ pub async fn build_and_restore(
// Secondary: token removed (so it's cleanly skipped next boot
// rather than re-erroring); leave the sdk state in place in case
// the operator re-provisions a fresh token for the same device.
// Return a PermanentBringUpError so the caller can distinguish
// "don't retry" from a transient network/DNS failure.
return Err(anyhow::Error::new(PermanentBringUpError(
"matrix token rejected (M_UNKNOWN_TOKEN); removed stale token, skipping account".into(),
)));
// Return Err so the caller logs + skips this one account and the
// daemon keeps serving the primary and any other healthy account.
return Err(anyhow!(
"matrix token rejected (M_UNKNOWN_TOKEN); removed stale token, skipping account"
));
}
return Err(e);
}

View file

@ -39,7 +39,6 @@ mod timeline;
mod wake;
use accounts::{AccountCfg, Registry};
use client::PermanentBringUpError;
/// A per-account sync loop, boxed so loops for N accounts can be driven
/// concurrently on the main task. Deliberately NOT `Send`: matrix-sdk's
@ -53,14 +52,6 @@ type SyncLoop = std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>>>>
/// a tiny file) is negligible.
const ACCOUNTS_HEARTBEAT_SECS: u64 = 30;
/// Retry delays (seconds) for transient secondary-account failures.
/// After the initial attempt we back off through these before giving up
/// and skipping the account for the rest of the daemon lifetime.
/// Transient = anything that is NOT a `PermanentBringUpError` (bad/expired
/// token). A down homeserver at DNS-not-ready boot time is the typical
/// case; the total wait is ~52s before we give up.
const SECONDARY_RETRY_DELAYS_SECS: &[u64] = &[2, 5, 15, 30];
#[tokio::main]
async fn main() -> Result<()> {
tracing_subscriber::fmt()
@ -84,7 +75,7 @@ async fn main() -> Result<()> {
// Account-tag the wakes only in multi-account mode so single-
// account wake bodies stay byte-identical to the legacy format.
let tag = multi.then(|| cfg.name.clone());
match bring_up_account(&cfg, &hyperhive_socket, tag.clone(), is_primary).await {
match bring_up_account(&cfg, &hyperhive_socket, tag, is_primary).await {
Ok(Some((client, sync_loop))) => {
registry.insert(cfg.name, client);
sync_loops.push(sync_loop);
@ -105,16 +96,10 @@ async fn main() -> Result<()> {
}
// The primary failing to restore is fatal (propagate so
// systemd retries on a transient blip — matches legacy
// behaviour); a secondary failing is retried with backoff
// before being skipped for this daemon lifetime.
// behaviour); a secondary failing is logged and skipped.
Err(e) if is_primary => return Err(e.context("bring up primary matrix account")),
Err(e) => {
if let Some((client, sync_loop)) =
bring_up_secondary_with_retry(&cfg, &hyperhive_socket, tag, e).await
{
registry.insert(cfg.name, client);
sync_loops.push(sync_loop);
}
tracing::error!(account = %cfg.name, error = %format!("{e:#}"), "secondary matrix account failed to restore; skipping");
}
}
}
@ -182,77 +167,6 @@ async fn main() -> Result<()> {
Ok(())
}
/// Try to bring up a secondary account, retrying with exponential backoff
/// on transient failures (anything that is NOT a `PermanentBringUpError`).
/// Returns `Some((client, sync_loop))` on success, or `None` to signal
/// that the account should be skipped for this daemon lifetime (permanent
/// failure, no token, or all retries exhausted).
async fn bring_up_secondary_with_retry(
cfg: &AccountCfg,
hyperhive_socket: &std::path::Path,
tag: Option<String>,
first_error: anyhow::Error,
) -> Option<(Client, SyncLoop)> {
// Permanent failure: the token was invalid/expired and has already been
// removed from disk. Retrying won't help — skip immediately.
if first_error
.downcast_ref::<PermanentBringUpError>()
.is_some()
{
tracing::error!(
account = %cfg.name,
error = %format!("{first_error:#}"),
"secondary matrix account: permanent failure; skipping"
);
return None;
}
// Transient failure (network/DNS/homeserver 5xx): retry with backoff.
tracing::warn!(
account = %cfg.name,
error = %format!("{first_error:#}"),
"secondary matrix account bring-up failed (transient); will retry"
);
for &delay in SECONDARY_RETRY_DELAYS_SECS {
tracing::info!(
account = %cfg.name,
delay_s = delay,
"retrying secondary account bring-up after backoff"
);
tokio::time::sleep(std::time::Duration::from_secs(delay)).await;
match bring_up_account(cfg, hyperhive_socket, tag.clone(), false).await {
Ok(Some((client, sync_loop))) => {
tracing::info!(account = %cfg.name, "secondary matrix account recovered");
return Some((client, sync_loop));
}
Ok(None) => {
tracing::warn!(account = %cfg.name, "secondary matrix account has no token; skipping");
return None;
}
Err(re) if re.downcast_ref::<PermanentBringUpError>().is_some() => {
tracing::error!(
account = %cfg.name,
error = %format!("{re:#}"),
"secondary matrix account: permanent failure on retry; skipping"
);
return None;
}
Err(re) => {
tracing::warn!(
account = %cfg.name,
error = %format!("{re:#}"),
"secondary matrix account bring-up still failing (transient)"
);
}
}
}
tracing::error!(
account = %cfg.name,
retries = SECONDARY_RETRY_DELAYS_SECS.len(),
"secondary matrix account failed after all retries; skipping for this daemon lifetime"
);
None
}
/// Restore one account's client (when its token exists), install its
/// message handler, and build its sync loop. Returns
/// `Ok(Some((client, sync_loop)))` when the account came up, `Ok(None)`