refactor: extract bring_up_secondary_with_retry (fix clippy too_many_lines + needless_continue)
This commit is contained in:
parent
317e545d7c
commit
3de576141b
1 changed files with 73 additions and 60 deletions
|
|
@ -109,67 +109,12 @@ async fn main() -> Result<()> {
|
||||||
// before being skipped for this daemon lifetime.
|
// before being skipped for this daemon lifetime.
|
||||||
Err(e) if is_primary => return Err(e.context("bring up primary matrix account")),
|
Err(e) if is_primary => return Err(e.context("bring up primary matrix account")),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
// Permanent failure (bad/expired token already removed):
|
if let Some((client, sync_loop)) =
|
||||||
// no point retrying.
|
bring_up_secondary_with_retry(&cfg, &hyperhive_socket, tag, e).await
|
||||||
if e.downcast_ref::<PermanentBringUpError>().is_some() {
|
{
|
||||||
tracing::error!(
|
registry.insert(cfg.name, client);
|
||||||
account = %cfg.name,
|
sync_loops.push(sync_loop);
|
||||||
error = %format!("{e:#}"),
|
|
||||||
"secondary matrix account: permanent failure; skipping"
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
// Transient failure: retry with backoff before giving up.
|
|
||||||
tracing::warn!(
|
|
||||||
account = %cfg.name,
|
|
||||||
error = %format!("{e:#}"),
|
|
||||||
"secondary matrix account bring-up failed (transient); will retry"
|
|
||||||
);
|
|
||||||
let mut recovered = false;
|
|
||||||
for &delay in SECONDARY_RETRY_DELAYS_SECS {
|
|
||||||
tracing::info!(
|
|
||||||
account = %cfg.name,
|
|
||||||
delay_s = delay,
|
|
||||||
"retrying secondary account bring-up after backoff"
|
|
||||||
);
|
|
||||||
tokio::time::sleep(std::time::Duration::from_secs(delay)).await;
|
|
||||||
match bring_up_account(&cfg, &hyperhive_socket, tag.clone(), false).await {
|
|
||||||
Ok(Some((client, sync_loop))) => {
|
|
||||||
tracing::info!(account = %cfg.name, "secondary matrix account recovered");
|
|
||||||
registry.insert(cfg.name.clone(), client);
|
|
||||||
sync_loops.push(sync_loop);
|
|
||||||
recovered = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Ok(None) => {
|
|
||||||
tracing::warn!(account = %cfg.name, "secondary matrix account has no token; skipping");
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Err(re) if re.downcast_ref::<PermanentBringUpError>().is_some() => {
|
|
||||||
tracing::error!(
|
|
||||||
account = %cfg.name,
|
|
||||||
error = %format!("{re:#}"),
|
|
||||||
"secondary matrix account: permanent failure on retry; skipping"
|
|
||||||
);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Err(re) => {
|
|
||||||
tracing::warn!(
|
|
||||||
account = %cfg.name,
|
|
||||||
error = %format!("{re:#}"),
|
|
||||||
"secondary matrix account bring-up still failing (transient)"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !recovered {
|
|
||||||
tracing::error!(
|
|
||||||
account = %cfg.name,
|
|
||||||
retries = SECONDARY_RETRY_DELAYS_SECS.len(),
|
|
||||||
"secondary matrix account failed after all retries; skipping for this daemon lifetime"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -237,6 +182,74 @@ async fn main() -> Result<()> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Try to bring up a secondary account, retrying with exponential backoff
|
||||||
|
/// on transient failures (anything that is NOT a `PermanentBringUpError`).
|
||||||
|
/// Returns `Some((client, sync_loop))` on success, or `None` to signal
|
||||||
|
/// that the account should be skipped for this daemon lifetime (permanent
|
||||||
|
/// failure, no token, or all retries exhausted).
|
||||||
|
async fn bring_up_secondary_with_retry(
|
||||||
|
cfg: &AccountCfg,
|
||||||
|
hyperhive_socket: &std::path::Path,
|
||||||
|
tag: Option<String>,
|
||||||
|
first_error: anyhow::Error,
|
||||||
|
) -> Option<(Client, SyncLoop)> {
|
||||||
|
// Permanent failure: the token was invalid/expired and has already been
|
||||||
|
// removed from disk. Retrying won't help — skip immediately.
|
||||||
|
if first_error.downcast_ref::<PermanentBringUpError>().is_some() {
|
||||||
|
tracing::error!(
|
||||||
|
account = %cfg.name,
|
||||||
|
error = %format!("{first_error:#}"),
|
||||||
|
"secondary matrix account: permanent failure; skipping"
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
// Transient failure (network/DNS/homeserver 5xx): retry with backoff.
|
||||||
|
tracing::warn!(
|
||||||
|
account = %cfg.name,
|
||||||
|
error = %format!("{first_error:#}"),
|
||||||
|
"secondary matrix account bring-up failed (transient); will retry"
|
||||||
|
);
|
||||||
|
for &delay in SECONDARY_RETRY_DELAYS_SECS {
|
||||||
|
tracing::info!(
|
||||||
|
account = %cfg.name,
|
||||||
|
delay_s = delay,
|
||||||
|
"retrying secondary account bring-up after backoff"
|
||||||
|
);
|
||||||
|
tokio::time::sleep(std::time::Duration::from_secs(delay)).await;
|
||||||
|
match bring_up_account(cfg, hyperhive_socket, tag.clone(), false).await {
|
||||||
|
Ok(Some((client, sync_loop))) => {
|
||||||
|
tracing::info!(account = %cfg.name, "secondary matrix account recovered");
|
||||||
|
return Some((client, sync_loop));
|
||||||
|
}
|
||||||
|
Ok(None) => {
|
||||||
|
tracing::warn!(account = %cfg.name, "secondary matrix account has no token; skipping");
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Err(re) if re.downcast_ref::<PermanentBringUpError>().is_some() => {
|
||||||
|
tracing::error!(
|
||||||
|
account = %cfg.name,
|
||||||
|
error = %format!("{re:#}"),
|
||||||
|
"secondary matrix account: permanent failure on retry; skipping"
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Err(re) => {
|
||||||
|
tracing::warn!(
|
||||||
|
account = %cfg.name,
|
||||||
|
error = %format!("{re:#}"),
|
||||||
|
"secondary matrix account bring-up still failing (transient)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tracing::error!(
|
||||||
|
account = %cfg.name,
|
||||||
|
retries = SECONDARY_RETRY_DELAYS_SECS.len(),
|
||||||
|
"secondary matrix account failed after all retries; skipping for this daemon lifetime"
|
||||||
|
);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
/// Restore one account's client (when its token exists), install its
|
/// Restore one account's client (when its token exists), install its
|
||||||
/// message handler, and build its sync loop. Returns
|
/// message handler, and build its sync loop. Returns
|
||||||
/// `Ok(Some((client, sync_loop)))` when the account came up, `Ok(None)`
|
/// `Ok(Some((client, sync_loop)))` when the account came up, `Ok(None)`
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue