vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

retry: share one backon helper for the client secret reads and convert the remaining doubling tables

Browse source
This commit is contained in:
damocles 2026-10-17 10:39:26 +02:00 • committed by mara
commit 9a3153cb08
14 changed files with 354 additions and 320 deletions
Download patch file Download diff file Expand all files Collapse all files

80
swarm-controller/src/queue_identity.rs
View file

@ -11,10 +11,8 @@
//!
//! [`init`] runs once in `main`, before anything calls [`get`].
use std::future::Future;
use std::path::PathBuf;
use std::sync::OnceLock;
use std::time::Duration;
use anyhow::{Context, Result, bail};
use swarm_queue_client::stored_secret::StoredSecret;
@ -24,11 +22,6 @@ use swarm_secret_client::queue::{self, ControllerCredential};
/// The environment prefix the nix module's `queueEnv` sets.
const PREFIX: &str = "SWARM_CONTROLLER";
/// Seconds to wait after each failed fetch before the next one. About a minute
/// in all, which rides out a store restarting alongside this daemon. A longer
/// outage fails the start, and the unit's `Restart=` tries again from the top.
const RETRY_DELAYS_S: [u64; 6] = [1, 2, 4, 8, 16, 30];
static IDENTITY: OnceLock<Option<QueueConfig>> = OnceLock::new();
/// Everything in a [`QueueConfig`] except the secret.
@ -59,14 +52,19 @@ impl Coordinates {
///
/// # Errors
/// A half-set environment, which is a deployment bug and not an absent queue;
/// a secret the store still would not give up after [`RETRY_DELAYS_S`]; or a
/// a secret the store still would not give up after the startup retries; or a
/// second call.
pub async fn init() -> Result<()> {
let identity = match coordinates_from(|name| std::env::var(name).ok())? {
None => None,
Some(coordinates) => {
let delays = RETRY_DELAYS_S.map(Duration::from_secs);
Some(coordinates.with_secret(fetch_with_retry(&delays, fetch_secret).await?))
let secret = swarm_queue_client::read_with_startup_retry(
"fetching the queue client secret from the swarm secret store",
fetch_secret,
)
.await
.context("fetching the queue client secret from the swarm secret store")?;
Some(coordinates.with_secret(secret))
}
};
IDENTITY
@ -122,34 +120,6 @@ async fn fetch_secret() -> Result<String> {
}
}
/// Run `attempt`, waiting each of `delays` after a failure, then once more.
/// Answers with the first success, or the last attempt's error.
async fn fetch_with_retry<F, Fut>(delays: &[Duration], mut attempt: F) -> Result<String>
where
F: FnMut() -> Fut,
Fut: Future<Output = Result<String>>,
{
for delay in delays {
match attempt().await {
Ok(secret) => return Ok(secret),
Err(e) => {
tracing::warn!(
error = %format!("{e:#}"),
retry_in = ?delay,
"fetching the queue client secret from the swarm secret store failed"
);
tokio::time::sleep(*delay).await;
}
}
}
attempt().await.with_context(|| {
format!(
"fetching the queue client secret from the swarm secret store, after {} attempts",
delays.len() + 1
)
})
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
@ -218,38 +188,4 @@ mod tests {
cfg.client_secret
);
}
#[tokio::test]
async fn a_store_that_comes_back_within_the_window_is_waited_for() {
let mut calls = 0;
let got = fetch_with_retry(&[Duration::ZERO; 3], || {
calls += 1;
let n = calls;
async move {
if n < 3 {
bail!("store sealed")
}
Ok("s3cret".to_owned())
}
})
.await
.expect("the third attempt succeeds");
assert_eq!(got, "s3cret");
assert_eq!(calls, 3);
}
#[tokio::test]
async fn a_store_that_stays_down_fails_after_one_attempt_per_delay_plus_one() {
let mut calls = 0;
let err = fetch_with_retry(&[Duration::ZERO; 3], || {
calls += 1;
async { bail!("store sealed") }
})
.await
.expect_err("every attempt fails");
assert_eq!(calls, 4);
let msg = format!("{err:#}");
assert!(msg.contains("after 4 attempts"), "{msg}");
assert!(msg.contains("store sealed"), "{msg}");
}
}