Compare commits
2 changed files with 8 additions and 113 deletions
|
|
@ -29,25 +29,6 @@ use matrix_sdk::{
|
|||
use serde::Deserialize;
|
||||
use tokio::fs;
|
||||
|
||||
/// Sentinel returned when `build_and_restore` detects that the token is
|
||||
/// permanently invalid (`M_UNKNOWN_TOKEN`). The token has already been
|
||||
/// removed from disk. Callers should NOT retry — the account needs
|
||||
/// re-provisioning by hive-c0re.
|
||||
///
|
||||
/// Distinct from the general `anyhow::Error` path so callers can use
|
||||
/// `err.downcast_ref::<PermanentBringUpError>()` to distinguish "retry
|
||||
/// won't help" from a transient network/DNS/5xx failure.
|
||||
#[derive(Debug)]
|
||||
pub struct PermanentBringUpError(pub String);
|
||||
|
||||
impl std::fmt::Display for PermanentBringUpError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for PermanentBringUpError {}
|
||||
|
||||
/// Subset of the `/_matrix/client/v3/account/whoami` response we care
|
||||
/// about. matrix-spec field names; `device_id` is optional per spec
|
||||
/// (servers MAY omit it for legacy bearer scopes) but tuwunel always
|
||||
|
|
@ -131,11 +112,11 @@ pub async fn build_and_restore(
|
|||
// Secondary: token removed (so it's cleanly skipped next boot
|
||||
// rather than re-erroring); leave the sdk state in place in case
|
||||
// the operator re-provisions a fresh token for the same device.
|
||||
// Return a PermanentBringUpError so the caller can distinguish
|
||||
// "don't retry" from a transient network/DNS failure.
|
||||
return Err(anyhow::Error::new(PermanentBringUpError(
|
||||
"matrix token rejected (M_UNKNOWN_TOKEN); removed stale token, skipping account".into(),
|
||||
)));
|
||||
// Return Err so the caller logs + skips this one account and the
|
||||
// daemon keeps serving the primary and any other healthy account.
|
||||
return Err(anyhow!(
|
||||
"matrix token rejected (M_UNKNOWN_TOKEN); removed stale token, skipping account"
|
||||
));
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,7 +39,6 @@ mod timeline;
|
|||
mod wake;
|
||||
|
||||
use accounts::{AccountCfg, Registry};
|
||||
use client::PermanentBringUpError;
|
||||
|
||||
/// A per-account sync loop, boxed so loops for N accounts can be driven
|
||||
/// concurrently on the main task. Deliberately NOT `Send`: matrix-sdk's
|
||||
|
|
@ -53,14 +52,6 @@ type SyncLoop = std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>>>>
|
|||
/// a tiny file) is negligible.
|
||||
const ACCOUNTS_HEARTBEAT_SECS: u64 = 30;
|
||||
|
||||
/// Retry delays (seconds) for transient secondary-account failures.
|
||||
/// After the initial attempt we back off through these before giving up
|
||||
/// and skipping the account for the rest of the daemon lifetime.
|
||||
/// Transient = anything that is NOT a `PermanentBringUpError` (bad/expired
|
||||
/// token). A down homeserver at DNS-not-ready boot time is the typical
|
||||
/// case; the total wait is ~52s before we give up.
|
||||
const SECONDARY_RETRY_DELAYS_SECS: &[u64] = &[2, 5, 15, 30];
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
tracing_subscriber::fmt()
|
||||
|
|
@ -84,7 +75,7 @@ async fn main() -> Result<()> {
|
|||
// Account-tag the wakes only in multi-account mode so single-
|
||||
// account wake bodies stay byte-identical to the legacy format.
|
||||
let tag = multi.then(|| cfg.name.clone());
|
||||
match bring_up_account(&cfg, &hyperhive_socket, tag.clone(), is_primary).await {
|
||||
match bring_up_account(&cfg, &hyperhive_socket, tag, is_primary).await {
|
||||
Ok(Some((client, sync_loop))) => {
|
||||
registry.insert(cfg.name, client);
|
||||
sync_loops.push(sync_loop);
|
||||
|
|
@ -105,16 +96,10 @@ async fn main() -> Result<()> {
|
|||
}
|
||||
// The primary failing to restore is fatal (propagate so
|
||||
// systemd retries on a transient blip — matches legacy
|
||||
// behaviour); a secondary failing is retried with backoff
|
||||
// before being skipped for this daemon lifetime.
|
||||
// behaviour); a secondary failing is logged and skipped.
|
||||
Err(e) if is_primary => return Err(e.context("bring up primary matrix account")),
|
||||
Err(e) => {
|
||||
if let Some((client, sync_loop)) =
|
||||
bring_up_secondary_with_retry(&cfg, &hyperhive_socket, tag, e).await
|
||||
{
|
||||
registry.insert(cfg.name, client);
|
||||
sync_loops.push(sync_loop);
|
||||
}
|
||||
tracing::error!(account = %cfg.name, error = %format!("{e:#}"), "secondary matrix account failed to restore; skipping");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -182,77 +167,6 @@ async fn main() -> Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Try to bring up a secondary account, retrying with exponential backoff
|
||||
/// on transient failures (anything that is NOT a `PermanentBringUpError`).
|
||||
/// Returns `Some((client, sync_loop))` on success, or `None` to signal
|
||||
/// that the account should be skipped for this daemon lifetime (permanent
|
||||
/// failure, no token, or all retries exhausted).
|
||||
async fn bring_up_secondary_with_retry(
|
||||
cfg: &AccountCfg,
|
||||
hyperhive_socket: &std::path::Path,
|
||||
tag: Option<String>,
|
||||
first_error: anyhow::Error,
|
||||
) -> Option<(Client, SyncLoop)> {
|
||||
// Permanent failure: the token was invalid/expired and has already been
|
||||
// removed from disk. Retrying won't help — skip immediately.
|
||||
if first_error
|
||||
.downcast_ref::<PermanentBringUpError>()
|
||||
.is_some()
|
||||
{
|
||||
tracing::error!(
|
||||
account = %cfg.name,
|
||||
error = %format!("{first_error:#}"),
|
||||
"secondary matrix account: permanent failure; skipping"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
// Transient failure (network/DNS/homeserver 5xx): retry with backoff.
|
||||
tracing::warn!(
|
||||
account = %cfg.name,
|
||||
error = %format!("{first_error:#}"),
|
||||
"secondary matrix account bring-up failed (transient); will retry"
|
||||
);
|
||||
for &delay in SECONDARY_RETRY_DELAYS_SECS {
|
||||
tracing::info!(
|
||||
account = %cfg.name,
|
||||
delay_s = delay,
|
||||
"retrying secondary account bring-up after backoff"
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_secs(delay)).await;
|
||||
match bring_up_account(cfg, hyperhive_socket, tag.clone(), false).await {
|
||||
Ok(Some((client, sync_loop))) => {
|
||||
tracing::info!(account = %cfg.name, "secondary matrix account recovered");
|
||||
return Some((client, sync_loop));
|
||||
}
|
||||
Ok(None) => {
|
||||
tracing::warn!(account = %cfg.name, "secondary matrix account has no token; skipping");
|
||||
return None;
|
||||
}
|
||||
Err(re) if re.downcast_ref::<PermanentBringUpError>().is_some() => {
|
||||
tracing::error!(
|
||||
account = %cfg.name,
|
||||
error = %format!("{re:#}"),
|
||||
"secondary matrix account: permanent failure on retry; skipping"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
Err(re) => {
|
||||
tracing::warn!(
|
||||
account = %cfg.name,
|
||||
error = %format!("{re:#}"),
|
||||
"secondary matrix account bring-up still failing (transient)"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
tracing::error!(
|
||||
account = %cfg.name,
|
||||
retries = SECONDARY_RETRY_DELAYS_SECS.len(),
|
||||
"secondary matrix account failed after all retries; skipping for this daemon lifetime"
|
||||
);
|
||||
None
|
||||
}
|
||||
|
||||
/// Restore one account's client (when its token exists), install its
|
||||
/// message handler, and build its sync loop. Returns
|
||||
/// `Ok(Some((client, sync_loop)))` when the account came up, `Ok(None)`
|
||||
|
|
|
|||
Loading…
Reference in a new issue