Compare commits

..
2 changed files with 8 additions and 113 deletions

View file

@ -29,25 +29,6 @@ use matrix_sdk::{
use serde::Deserialize; use serde::Deserialize;
use tokio::fs; use tokio::fs;
/// Sentinel returned when `build_and_restore` detects that the token is
/// permanently invalid (`M_UNKNOWN_TOKEN`). The token has already been
/// removed from disk. Callers should NOT retry — the account needs
/// re-provisioning by hive-c0re.
///
/// Distinct from the general `anyhow::Error` path so callers can use
/// `err.downcast_ref::<PermanentBringUpError>()` to distinguish "retry
/// won't help" from a transient network/DNS/5xx failure.
#[derive(Debug)]
pub struct PermanentBringUpError(pub String);
impl std::fmt::Display for PermanentBringUpError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
impl std::error::Error for PermanentBringUpError {}
/// Subset of the `/_matrix/client/v3/account/whoami` response we care /// Subset of the `/_matrix/client/v3/account/whoami` response we care
/// about. matrix-spec field names; `device_id` is optional per spec /// about. matrix-spec field names; `device_id` is optional per spec
/// (servers MAY omit it for legacy bearer scopes) but tuwunel always /// (servers MAY omit it for legacy bearer scopes) but tuwunel always
@ -131,11 +112,11 @@ pub async fn build_and_restore(
// Secondary: token removed (so it's cleanly skipped next boot // Secondary: token removed (so it's cleanly skipped next boot
// rather than re-erroring); leave the sdk state in place in case // rather than re-erroring); leave the sdk state in place in case
// the operator re-provisions a fresh token for the same device. // the operator re-provisions a fresh token for the same device.
// Return a PermanentBringUpError so the caller can distinguish // Return Err so the caller logs + skips this one account and the
// "don't retry" from a transient network/DNS failure. // daemon keeps serving the primary and any other healthy account.
return Err(anyhow::Error::new(PermanentBringUpError( return Err(anyhow!(
"matrix token rejected (M_UNKNOWN_TOKEN); removed stale token, skipping account".into(), "matrix token rejected (M_UNKNOWN_TOKEN); removed stale token, skipping account"
))); ));
} }
return Err(e); return Err(e);
} }

View file

@ -39,7 +39,6 @@ mod timeline;
mod wake; mod wake;
use accounts::{AccountCfg, Registry}; use accounts::{AccountCfg, Registry};
use client::PermanentBringUpError;
/// A per-account sync loop, boxed so loops for N accounts can be driven /// A per-account sync loop, boxed so loops for N accounts can be driven
/// concurrently on the main task. Deliberately NOT `Send`: matrix-sdk's /// concurrently on the main task. Deliberately NOT `Send`: matrix-sdk's
@ -53,14 +52,6 @@ type SyncLoop = std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>>>>
/// a tiny file) is negligible. /// a tiny file) is negligible.
const ACCOUNTS_HEARTBEAT_SECS: u64 = 30; const ACCOUNTS_HEARTBEAT_SECS: u64 = 30;
/// Retry delays (seconds) for transient secondary-account failures.
/// After the initial attempt we back off through these before giving up
/// and skipping the account for the rest of the daemon lifetime.
/// Transient = anything that is NOT a `PermanentBringUpError` (bad/expired
/// token). A down homeserver at DNS-not-ready boot time is the typical
/// case; the total wait is ~52s before we give up.
const SECONDARY_RETRY_DELAYS_SECS: &[u64] = &[2, 5, 15, 30];
#[tokio::main] #[tokio::main]
async fn main() -> Result<()> { async fn main() -> Result<()> {
tracing_subscriber::fmt() tracing_subscriber::fmt()
@ -84,7 +75,7 @@ async fn main() -> Result<()> {
// Account-tag the wakes only in multi-account mode so single- // Account-tag the wakes only in multi-account mode so single-
// account wake bodies stay byte-identical to the legacy format. // account wake bodies stay byte-identical to the legacy format.
let tag = multi.then(|| cfg.name.clone()); let tag = multi.then(|| cfg.name.clone());
match bring_up_account(&cfg, &hyperhive_socket, tag.clone(), is_primary).await { match bring_up_account(&cfg, &hyperhive_socket, tag, is_primary).await {
Ok(Some((client, sync_loop))) => { Ok(Some((client, sync_loop))) => {
registry.insert(cfg.name, client); registry.insert(cfg.name, client);
sync_loops.push(sync_loop); sync_loops.push(sync_loop);
@ -105,16 +96,10 @@ async fn main() -> Result<()> {
} }
// The primary failing to restore is fatal (propagate so // The primary failing to restore is fatal (propagate so
// systemd retries on a transient blip — matches legacy // systemd retries on a transient blip — matches legacy
// behaviour); a secondary failing is retried with backoff // behaviour); a secondary failing is logged and skipped.
// before being skipped for this daemon lifetime.
Err(e) if is_primary => return Err(e.context("bring up primary matrix account")), Err(e) if is_primary => return Err(e.context("bring up primary matrix account")),
Err(e) => { Err(e) => {
if let Some((client, sync_loop)) = tracing::error!(account = %cfg.name, error = %format!("{e:#}"), "secondary matrix account failed to restore; skipping");
bring_up_secondary_with_retry(&cfg, &hyperhive_socket, tag, e).await
{
registry.insert(cfg.name, client);
sync_loops.push(sync_loop);
}
} }
} }
} }
@ -182,77 +167,6 @@ async fn main() -> Result<()> {
Ok(()) Ok(())
} }
/// Try to bring up a secondary account, retrying with exponential backoff
/// on transient failures (anything that is NOT a `PermanentBringUpError`).
/// Returns `Some((client, sync_loop))` on success, or `None` to signal
/// that the account should be skipped for this daemon lifetime (permanent
/// failure, no token, or all retries exhausted).
async fn bring_up_secondary_with_retry(
cfg: &AccountCfg,
hyperhive_socket: &std::path::Path,
tag: Option<String>,
first_error: anyhow::Error,
) -> Option<(Client, SyncLoop)> {
// Permanent failure: the token was invalid/expired and has already been
// removed from disk. Retrying won't help — skip immediately.
if first_error
.downcast_ref::<PermanentBringUpError>()
.is_some()
{
tracing::error!(
account = %cfg.name,
error = %format!("{first_error:#}"),
"secondary matrix account: permanent failure; skipping"
);
return None;
}
// Transient failure (network/DNS/homeserver 5xx): retry with backoff.
tracing::warn!(
account = %cfg.name,
error = %format!("{first_error:#}"),
"secondary matrix account bring-up failed (transient); will retry"
);
for &delay in SECONDARY_RETRY_DELAYS_SECS {
tracing::info!(
account = %cfg.name,
delay_s = delay,
"retrying secondary account bring-up after backoff"
);
tokio::time::sleep(std::time::Duration::from_secs(delay)).await;
match bring_up_account(cfg, hyperhive_socket, tag.clone(), false).await {
Ok(Some((client, sync_loop))) => {
tracing::info!(account = %cfg.name, "secondary matrix account recovered");
return Some((client, sync_loop));
}
Ok(None) => {
tracing::warn!(account = %cfg.name, "secondary matrix account has no token; skipping");
return None;
}
Err(re) if re.downcast_ref::<PermanentBringUpError>().is_some() => {
tracing::error!(
account = %cfg.name,
error = %format!("{re:#}"),
"secondary matrix account: permanent failure on retry; skipping"
);
return None;
}
Err(re) => {
tracing::warn!(
account = %cfg.name,
error = %format!("{re:#}"),
"secondary matrix account bring-up still failing (transient)"
);
}
}
}
tracing::error!(
account = %cfg.name,
retries = SECONDARY_RETRY_DELAYS_SECS.len(),
"secondary matrix account failed after all retries; skipping for this daemon lifetime"
);
None
}
/// Restore one account's client (when its token exists), install its /// Restore one account's client (when its token exists), install its
/// message handler, and build its sync loop. Returns /// message handler, and build its sync loop. Returns
/// `Ok(Some((client, sync_loop)))` when the account came up, `Ok(None)` /// `Ok(Some((client, sync_loop)))` when the account came up, `Ok(None)`