vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

retry: share one backon helper for the client secret reads and convert the remaining doubling tables

Browse source
This commit is contained in:
damocles 2026-10-17 10:39:26 +02:00 • committed by mara
commit 9a3153cb08
14 changed files with 354 additions and 320 deletions
Download patch file Download diff file Expand all files Collapse all files

40
swarm-controller/src/main.rs
View file

@ -35,6 +35,7 @@ use axum::{
extract::{Path, State},
routing::get,
};
use backon::{ExponentialBuilder, Retryable as _};
use hive_jobq_wire::GraphWire as _;
use serde::{Deserialize, Serialize};
use swarm_authelia_bridge_sock::BridgeResponse;
@ -3321,31 +3322,34 @@ fn register_swarm_webhooks(forge: Arc<forge::Client>, secret: Option<Arc<str>>)
return;
};
tokio::spawn(async move {
// Seconds to wait before each retry — a bounded schedule, not a poll
// loop: it exists to outlast a forge that is slow to start, not to
// A bounded schedule, not a poll loop: 15s doubling to 240s, five
// retries. It exists to outlast a forge that is slow to start, not to
// re-register periodically. Registration is idempotent, so a repeat
// costs one API call and changes nothing.
const RETRY_DELAYS_S: [u64; 5] = [15, 30, 60, 120, 240];
const RETRIES: usize = 5;
for (attempt, delay) in RETRY_DELAYS_S.iter().enumerate() {
match forge.ensure_swarm_webhooks(&public_url, &secret).await {
Ok(()) => return,
Err(e) => tracing::info!(
let mut failed = 0;
let registered = (|| forge.ensure_swarm_webhooks(&public_url, &secret))
.retry(
ExponentialBuilder::default()
.with_min_delay(std::time::Duration::from_secs(15))
.with_max_delay(std::time::Duration::from_mins(4))
.with_max_times(RETRIES),
)
.notify(|e, wait| {
failed += 1;
tracing::info!(
error = %format!("{e:#}"),
attempt = attempt + 1,
retry_in_s = *delay,
attempt = failed,
retry_in = ?wait,
"registering swarm-wide forge webhooks failed; retrying"
),
}
tokio::time::sleep(std::time::Duration::from_secs(*delay)).await;
}
// Last attempt after the final wait, so the schedule above reads as
// "delay before the next try" rather than one entry meaning two things.
if let Err(e) = forge.ensure_swarm_webhooks(&public_url, &secret).await {
);
})
.await;
if let Err(e) = registered {
tracing::warn!(
error = %format!("{e:#}"),
attempts = RETRY_DELAYS_S.len() + 1,
attempts = RETRIES + 1,
"registering swarm-wide forge webhooks failed; giving up until the next start"
);
}