vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

retry: share one backon helper for the client secret reads and convert the remaining doubling tables

Browse source
This commit is contained in:
damocles 2026-10-17 10:39:26 +02:00 • committed by mara
commit 9a3153cb08
14 changed files with 354 additions and 320 deletions
Download patch file Download diff file Expand all files Collapse all files

1
swarm-controller/Cargo.toml
View file

@ -10,6 +10,7 @@ path = "src/main.rs"
[dependencies]
anyhow.workspace = true
backon.workspace = true
# `kv` (which pulls `jetstream`) on top of the workspace's feature set: the
# queue is this daemon's *store*, not just its transport - a hive's last
# status snapshot is read out of a JetStream KV bucket. Declared here rather

113
swarm-controller/src/forge.rs
View file

@ -21,6 +21,7 @@
//! either.
use anyhow::{Context, Result};
use backon::{ExponentialBuilder, Retryable as _};
use forgejo_api::structs::{
AddCollaboratorOption, AddCollaboratorOptionPermission, ChangeFileOperation,
ChangeFileOperationOperation, ChangeFilesOptions, CommitStatusState,
@ -272,52 +273,48 @@ pub fn url_from_env() -> Result<Option<url::Url>> {
.map(Some)
}
/// How long the background re-read waits before its first attempt. The caller
/// has just made one, so an immediate retry would only repeat it.
const BACKGROUND_FIRST: std::time::Duration = std::time::Duration::from_secs(30);
/// The longest the background re-read waits between attempts.
const BACKGROUND_CAP: std::time::Duration = std::time::Duration::from_mins(5);
/// How long the background re-read waits before its `n`-th attempt: 30s,
/// doubling, capped at [`BACKGROUND_CAP`].
fn background_delay(n: u32) -> std::time::Duration {
const FIRST_S: u64 = 30;
std::time::Duration::from_secs(
FIRST_S
.saturating_mul(1u64.checked_shl(n).unwrap_or(u64::MAX))
.min(BACKGROUND_CAP.as_secs()),
)
}
/// Whether the `n`-th failed attempt is logged at `info` rather than `debug`:
/// once the backoff is capped, so a controller that never gets its token says
/// so once per [`BACKGROUND_CAP`] without a line per early retry.
fn failure_logs_at_info(n: u32) -> bool {
background_delay(n) >= BACKGROUND_CAP
}
/// Fill `slot` with the first value `attempt` yields, sleeping
/// [`background_delay`] before each attempt. Never gives up. A failure is logged
/// as [`failure_logs_at_info`] says; the caller logs the first one and the
/// recovery.
async fn fill_when_ready<T, F, Fut>(slot: &OnceLock<T>, mut attempt: F) -> &T
/// Fill `slot` with the first value `attempt` yields: after
/// [`BACKGROUND_FIRST`], then at 60s, doubling, capped at [`BACKGROUND_CAP`].
/// Never gives up. A failure is logged at `info` once the next wait is the cap,
/// so a controller that never gets its token says so once per
/// [`BACKGROUND_CAP`] without a line per early retry, and at `debug` before
/// that. The caller logs the first failure and the recovery.
async fn fill_when_ready<T, F, Fut>(slot: &OnceLock<T>, attempt: F) -> &T
where
F: FnMut() -> Fut,
Fut: std::future::Future<Output = Result<T>>,
{
let mut n = 0;
loop {
tokio::time::sleep(background_delay(n)).await;
match attempt().await {
Ok(value) => return slot.get_or_init(|| value),
Err(e) if failure_logs_at_info(n) => tracing::info!(
error = %format!("{e:#}"),
"still no forge access: the controller's forge token is unavailable"
),
Err(e) => tracing::debug!(
error = %format!("{e:#}"),
"the controller's forge token is still unavailable"
),
}
n = n.saturating_add(1);
}
tokio::time::sleep(BACKGROUND_FIRST).await;
let value = attempt
.retry(
ExponentialBuilder::default()
.with_min_delay(BACKGROUND_FIRST * 2)
.with_max_delay(BACKGROUND_CAP)
.without_max_times(),
)
.notify(|e, wait| {
if wait >= BACKGROUND_CAP {
tracing::info!(
error = %format!("{e:#}"),
"still no forge access: the controller's forge token is unavailable"
);
} else {
tracing::debug!(
error = %format!("{e:#}"),
"the controller's forge token is still unavailable"
);
}
})
.await
.expect("a retry with no cap on its tries returns only once it succeeds");
slot.get_or_init(|| value)
}
/// The longest one unit of forge work, a periodic pass or a job node, may run
@ -2020,21 +2017,6 @@ mod tests {
}
}
#[test]
fn a_failed_re_read_logs_at_info_only_once_the_backoff_is_capped() {
assert!((0..4).all(|n| !failure_logs_at_info(n)));
assert!(failure_logs_at_info(4));
assert!(failure_logs_at_info(40));
}
#[test]
fn the_background_re_read_backs_off_from_30s_to_a_5_minute_cap() {
let s = |n| background_delay(n).as_secs();
assert_eq!([s(0), s(1), s(2), s(3), s(4)], [30, 60, 120, 240, 300]);
assert_eq!(s(40), 300);
assert_eq!(s(u32::MAX), 300);
}
// Paused time: each sleep completes at once and advances the clock, so the
// elapsed time is the backoff schedule itself.
#[tokio::test(start_paused = true)]
@ -2059,6 +2041,29 @@ mod tests {
assert_eq!(start.elapsed().as_secs(), 30 + 60 + 120);
}
#[tokio::test(start_paused = true)]
async fn a_token_that_stays_absent_is_retried_every_five_minutes() {
let slot = OnceLock::new();
let start = tokio::time::Instant::now();
let mut calls = 0;
fill_when_ready(&slot, || {
calls += 1;
let n = calls;
async move {
if n < 7 {
anyhow::bail!("nothing at the controller's forge token path")
}
Ok("client")
}
})
.await;
assert_eq!(calls, 7);
assert_eq!(
start.elapsed().as_secs(),
30 + 60 + 120 + 240 + 300 + 300 + 300
);
}
#[tokio::test(start_paused = true)]
async fn a_pass_that_never_answers_is_abandoned_at_the_deadline() {
let start = tokio::time::Instant::now();

40
swarm-controller/src/main.rs
View file

@ -35,6 +35,7 @@ use axum::{
extract::{Path, State},
routing::get,
};
use backon::{ExponentialBuilder, Retryable as _};
use hive_jobq_wire::GraphWire as _;
use serde::{Deserialize, Serialize};
use swarm_authelia_bridge_sock::BridgeResponse;
@ -3321,31 +3322,34 @@ fn register_swarm_webhooks(forge: Arc<forge::Client>, secret: Option<Arc<str>>)
return;
};
tokio::spawn(async move {
// Seconds to wait before each retry — a bounded schedule, not a poll
// loop: it exists to outlast a forge that is slow to start, not to
// A bounded schedule, not a poll loop: 15s doubling to 240s, five
// retries. It exists to outlast a forge that is slow to start, not to
// re-register periodically. Registration is idempotent, so a repeat
// costs one API call and changes nothing.
const RETRY_DELAYS_S: [u64; 5] = [15, 30, 60, 120, 240];
const RETRIES: usize = 5;
for (attempt, delay) in RETRY_DELAYS_S.iter().enumerate() {
match forge.ensure_swarm_webhooks(&public_url, &secret).await {
Ok(()) => return,
Err(e) => tracing::info!(
let mut failed = 0;
let registered = (|| forge.ensure_swarm_webhooks(&public_url, &secret))
.retry(
ExponentialBuilder::default()
.with_min_delay(std::time::Duration::from_secs(15))
.with_max_delay(std::time::Duration::from_mins(4))
.with_max_times(RETRIES),
)
.notify(|e, wait| {
failed += 1;
tracing::info!(
error = %format!("{e:#}"),
attempt = attempt + 1,
retry_in_s = *delay,
attempt = failed,
retry_in = ?wait,
"registering swarm-wide forge webhooks failed; retrying"
),
}
tokio::time::sleep(std::time::Duration::from_secs(*delay)).await;
}
// Last attempt after the final wait, so the schedule above reads as
// "delay before the next try" rather than one entry meaning two things.
if let Err(e) = forge.ensure_swarm_webhooks(&public_url, &secret).await {
);
})
.await;
if let Err(e) = registered {
tracing::warn!(
error = %format!("{e:#}"),
attempts = RETRY_DELAYS_S.len() + 1,
attempts = RETRIES + 1,
"registering swarm-wide forge webhooks failed; giving up until the next start"
);
}

80
swarm-controller/src/queue_identity.rs
View file

@ -11,10 +11,8 @@
//!
//! [`init`] runs once in `main`, before anything calls [`get`].
use std::future::Future;
use std::path::PathBuf;
use std::sync::OnceLock;
use std::time::Duration;
use anyhow::{Context, Result, bail};
use swarm_queue_client::stored_secret::StoredSecret;
@ -24,11 +22,6 @@ use swarm_secret_client::queue::{self, ControllerCredential};
/// The environment prefix the nix module's `queueEnv` sets.
const PREFIX: &str = "SWARM_CONTROLLER";
/// Seconds to wait after each failed fetch before the next one. About a minute
/// in all, which rides out a store restarting alongside this daemon. A longer
/// outage fails the start, and the unit's `Restart=` tries again from the top.
const RETRY_DELAYS_S: [u64; 6] = [1, 2, 4, 8, 16, 30];
static IDENTITY: OnceLock<Option<QueueConfig>> = OnceLock::new();
/// Everything in a [`QueueConfig`] except the secret.
@ -59,14 +52,19 @@ impl Coordinates {
///
/// # Errors
/// A half-set environment, which is a deployment bug and not an absent queue;
/// a secret the store still would not give up after [`RETRY_DELAYS_S`]; or a
/// a secret the store still would not give up after the startup retries; or a
/// second call.
pub async fn init() -> Result<()> {
let identity = match coordinates_from(|name| std::env::var(name).ok())? {
None => None,
Some(coordinates) => {
let delays = RETRY_DELAYS_S.map(Duration::from_secs);
Some(coordinates.with_secret(fetch_with_retry(&delays, fetch_secret).await?))
let secret = swarm_queue_client::read_with_startup_retry(
"fetching the queue client secret from the swarm secret store",
fetch_secret,
)
.await
.context("fetching the queue client secret from the swarm secret store")?;
Some(coordinates.with_secret(secret))
}
};
IDENTITY
@ -122,34 +120,6 @@ async fn fetch_secret() -> Result<String> {
}
}
/// Run `attempt`, waiting each of `delays` after a failure, then once more.
/// Answers with the first success, or the last attempt's error.
async fn fetch_with_retry<F, Fut>(delays: &[Duration], mut attempt: F) -> Result<String>
where
F: FnMut() -> Fut,
Fut: Future<Output = Result<String>>,
{
for delay in delays {
match attempt().await {
Ok(secret) => return Ok(secret),
Err(e) => {
tracing::warn!(
error = %format!("{e:#}"),
retry_in = ?delay,
"fetching the queue client secret from the swarm secret store failed"
);
tokio::time::sleep(*delay).await;
}
}
}
attempt().await.with_context(|| {
format!(
"fetching the queue client secret from the swarm secret store, after {} attempts",
delays.len() + 1
)
})
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
@ -218,38 +188,4 @@ mod tests {
cfg.client_secret
);
}
#[tokio::test]
async fn a_store_that_comes_back_within_the_window_is_waited_for() {
let mut calls = 0;
let got = fetch_with_retry(&[Duration::ZERO; 3], || {
calls += 1;
let n = calls;
async move {
if n < 3 {
bail!("store sealed")
}
Ok("s3cret".to_owned())
}
})
.await
.expect("the third attempt succeeds");
assert_eq!(got, "s3cret");
assert_eq!(calls, 3);
}
#[tokio::test]
async fn a_store_that_stays_down_fails_after_one_attempt_per_delay_plus_one() {
let mut calls = 0;
let err = fetch_with_retry(&[Duration::ZERO; 3], || {
calls += 1;
async { bail!("store sealed") }
})
.await
.expect_err("every attempt fails");
assert_eq!(calls, 4);
let msg = format!("{err:#}");
assert!(msg.contains("after 4 attempts"), "{msg}");
assert!(msg.contains("store sealed"), "{msg}");
}
}