vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

retry: share one backon helper for the client secret reads and convert the remaining doubling tables

Browse source
This commit is contained in:
damocles 2026-10-17 10:39:26 +02:00 • committed by mara
commit 9a3153cb08
14 changed files with 354 additions and 320 deletions
Download patch file Download diff file Expand all files Collapse all files

143
hive-agent/src/swarm_queue.rs
View file

@ -7,13 +7,14 @@
//!
//! **The connection is required.** [`init`] fails when the queue's address or
//! the store's coordinates are missing, and [`client`] retries the first
//! connect on [`CONNECT_BACKOFF`] and then exits the process, so the harness
//! connect on a doubling backoff and then exits the process, so the harness
//! unit fails and systemd restarts it.
use std::sync::{Arc, Mutex, OnceLock};
use std::time::Duration;
use anyhow::{Context as _, anyhow};
use backon::{ExponentialBuilder, Retryable as _};
use tokio::sync::OnceCell;
use swarm_secret_client::{
@ -34,16 +35,16 @@ const ENV_AGENT_NAME: &str = "HIVE_AGENT_NAME";
/// every reconnect behind it.
const STORE_READ_TIMEOUT: Duration = Duration::from_secs(10);
/// The waits between first-connect attempts: 5s doubling, about five minutes
/// in all. After the last attempt fails the process exits.
const CONNECT_BACKOFF: [Duration; 6] = [
Duration::from_secs(5),
Duration::from_secs(10),
Duration::from_secs(20),
Duration::from_secs(40),
Duration::from_secs(80),
Duration::from_secs(160),
];
/// The first wait between first-connect attempts, doubled per failure up to
/// [`CONNECT_MAX_WAIT`].
const CONNECT_FIRST_WAIT: Duration = Duration::from_secs(5);
/// The longest wait between first-connect attempts.
const CONNECT_MAX_WAIT: Duration = Duration::from_secs(160);
/// Waits of 5s doubling to 160s, about five minutes in all. After the last
/// attempt fails the process exits.
const CONNECT_RETRIES: usize = 6;
/// Logged at `error` once every first-connect attempt has failed, just before
/// the process exits. The Grafana rule `hh-swarm-queue-connect-failed`
@ -294,7 +295,7 @@ pub fn init() -> anyhow::Result<()> {
/// one client, so a second `connect` would be a second live connection for the
/// same identity rather than a second credential.
///
/// The first call retries the connect on [`CONNECT_BACKOFF`], and every other
/// The first call retries the connect (see [`CONNECT_RETRIES`]), and every other
/// caller waits on it. When no attempt succeeds the process exits with status
/// 1, so this never returns without a client.
pub async fn client() -> async_nats::Client {
@ -303,7 +304,7 @@ pub async fn client() -> async_nats::Client {
async fn connect_or_exit() -> async_nats::Client {
let connected = match AGENT.get() {
Some(agent) => retry_first_connect(|| connect_once(agent), tokio::time::sleep).await,
Some(agent) => retry_first_connect(|| connect_once(agent)).await,
None => Err(anyhow!("swarm_queue::init has not run")),
};
connected.unwrap_or_else(|e| {
@ -328,34 +329,38 @@ async fn connect_once(agent: &AgentPath) -> anyhow::Result<async_nats::Client> {
Ok(client)
}
/// Run `connect` until it succeeds, sleeping each wait in [`CONNECT_BACKOFF`]
/// between attempts, and return the last error once every attempt has failed.
async fn retry_first_connect<T, C, CFut, S, SFut>(mut connect: C, mut sleep: S) -> anyhow::Result<T>
/// Run `connect` until it succeeds, waiting between attempts as
/// [`CONNECT_RETRIES`] says, and return the last error once every attempt has
/// failed.
async fn retry_first_connect<T, C, CFut>(connect: C) -> anyhow::Result<T>
where
C: FnMut() -> CFut,
CFut: Future<Output = anyhow::Result<T>>,
S: FnMut(Duration) -> SFut,
SFut: Future<Output = ()>,
{
let mut waits = CONNECT_BACKOFF.iter();
let mut attempt = 1;
loop {
let e = match connect().await {
Ok(client) => return Ok(client),
Err(e) => e,
};
let Some(&wait) = waits.next() else {
return Err(e.context(format!("{attempt} connection attempts failed")));
};
tracing::warn!(
error = format!("{e:#}"),
attempt,
retry_in = ?wait,
"connecting to the swarm queue failed"
);
sleep(wait).await;
attempt += 1;
}
connect
.retry(
ExponentialBuilder::default()
.with_min_delay(CONNECT_FIRST_WAIT)
.with_max_delay(CONNECT_MAX_WAIT)
.with_max_times(CONNECT_RETRIES),
)
.notify(|e, wait| {
tracing::warn!(
error = format!("{e:#}"),
attempt,
retry_in = ?wait,
"connecting to the swarm queue failed"
);
attempt += 1;
})
.await
.map_err(|e| {
e.context(format!(
"{} connection attempts failed",
CONNECT_RETRIES + 1
))
})
}
/// Connect presenting this agent's own token as `name`.
@ -401,7 +406,7 @@ mod tests {
use tracing_subscriber::layer::SubscriberExt as _;
use super::{
AgentPath, AgentSecret, CONNECT_BACKOFF, SecretSource, StoreSource, agent_path,
AgentPath, AgentSecret, CONNECT_RETRIES, SecretSource, StoreSource, agent_path,
connect_as_agent, log_connect_exhausted, retry_first_connect, store_source,
};
@ -489,65 +494,55 @@ mod tests {
}
/// Runs `retry_first_connect` against a connect that fails `failures`
/// times and then succeeds, recording each wait instead of sleeping it.
async fn retry_with(failures: usize) -> (anyhow::Result<usize>, usize, Vec<Duration>) {
/// times and then succeeds, on paused time so each wait costs nothing and
/// the elapsed time is the backoff itself.
async fn retry_with(failures: usize) -> (anyhow::Result<usize>, usize, Duration) {
let mut calls = 0;
let mut waits = Vec::new();
let result = retry_first_connect(
|| {
calls += 1;
let n = calls;
async move {
if n > failures {
Ok(n)
} else {
Err(anyhow!("attempt {n} refused"))
}
let started = tokio::time::Instant::now();
let result = retry_first_connect(|| {
calls += 1;
let n = calls;
async move {
if n > failures {
Ok(n)
} else {
Err(anyhow!("attempt {n} refused"))
}
},
|wait| {
waits.push(wait);
async {}
},
)
}
})
.await;
(result, calls, waits)
(result, calls, started.elapsed())
}
#[tokio::test]
#[tokio::test(start_paused = true)]
async fn a_first_connect_that_succeeds_waits_for_nothing() {
let (result, calls, waits) = retry_with(0).await;
let (result, calls, waited) = retry_with(0).await;
assert_eq!(result.expect("connected"), 1);
assert_eq!(calls, 1);
assert!(waits.is_empty(), "{waits:?}");
assert_eq!(waited, Duration::ZERO);
}
/// 5s doubling, one wait per failed attempt, and the attempt after the
/// last wait is the one that connects.
#[tokio::test]
#[tokio::test(start_paused = true)]
async fn a_failed_connect_is_retried_on_a_doubling_backoff() {
let (result, calls, waits) = retry_with(3).await;
let (result, calls, waited) = retry_with(3).await;
assert_eq!(result.expect("connected on the fourth attempt"), 4);
assert_eq!(calls, 4);
assert_eq!(
waits,
[5, 10, 20].map(Duration::from_secs),
"each wait doubles the one before"
);
assert_eq!(waited, Duration::from_secs(5 + 10 + 20));
}
/// About five minutes of waits, then the last attempt's error rather than
/// another wait.
#[tokio::test]
#[tokio::test(start_paused = true)]
async fn the_retry_gives_up_with_the_last_error_after_about_five_minutes() {
let (result, calls, waits) = retry_with(usize::MAX).await;
let (result, calls, waited) = retry_with(usize::MAX).await;
let e = result.expect_err("every attempt failed");
assert_eq!(calls, CONNECT_BACKOFF.len() + 1);
assert_eq!(waits, CONNECT_BACKOFF);
let total: Duration = waits.iter().sum();
assert_eq!(calls, CONNECT_RETRIES + 1);
assert_eq!(waited, Duration::from_secs(5 + 10 + 20 + 40 + 80 + 160));
assert!(
(Duration::from_mins(4)..=Duration::from_mins(6)).contains(&total),
"{total:?}"
(Duration::from_mins(4)..=Duration::from_mins(6)).contains(&waited),
"{waited:?}"
);
assert!(
format!("{e:#}").contains(&format!("attempt {calls} refused")),