vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

retry: replace three hand-rolled doubling loops with backon

Browse source
This commit is contained in:
damocles 2026-10-17 09:23:46 +02:00 • committed by mara
commit 0f67ec34b1
7 changed files with 141 additions and 83 deletions
Download patch file Download diff file Expand all files Collapse all files

45
hive-agent/src/swarm_agent_icon.rs
View file

@ -22,6 +22,7 @@ use std::path::Path;
use std::time::Duration;
use async_nats::jetstream::context::PublishErrorKind;
use backon::{ExponentialBuilder, Retryable};
/// Where the agent's NixOS config puts the icon
/// (`services.hyperhive.agent.icon`), and the file `GET /icon` serves. Absent
@ -29,7 +30,7 @@ use async_nats::jetstream::context::PublishErrorKind;
const ICON_PATH: &str = "/etc/hyperhive/icon.svg";
/// First retry delay after a failed write, doubled per failure up to
/// [`MAX_RETRY`].
/// [`MAX_RETRY`], with no cap on the number of tries.
const FIRST_RETRY: Duration = Duration::from_secs(5);
/// Longest wait between retries.
@ -80,27 +81,33 @@ async fn run(agent: String) {
}
};
let js = async_nats::jetstream::new(client.clone());
let mut delay = FIRST_RETRY;
loop {
match write(&client, &js, &subject, &icon).await {
Ok(()) => {
tracing::info!(
subject,
set = matches!(icon, Icon::Set(_)),
"published this agent's icon"
);
return;
}
Err(Failure::TooLarge(e)) => {
tracing::warn!(error = %e, "the agent icon is over the queue's payload limit; not publishing it");
return;
}
Err(Failure::Retry(e)) => {
let written = (|| write(&client, &js, &subject, &icon))
.retry(
ExponentialBuilder::default()
.with_min_delay(FIRST_RETRY)
.with_max_delay(MAX_RETRY)
.without_max_times(),
)
.when(|failure| matches!(failure, Failure::Retry(_)))
.notify(|failure, delay| {
if let Failure::Retry(e) = failure {
tracing::warn!(error = e, retry_in = ?delay, "publishing the agent icon failed");
}
})
.await;
match written {
Ok(()) => tracing::info!(
subject,
set = matches!(icon, Icon::Set(_)),
"published this agent's icon"
),
Err(Failure::TooLarge(e)) => {
tracing::warn!(error = %e, "the agent icon is over the queue's payload limit; not publishing it");
}
// Only a `TooLarge` ends the retries.
Err(Failure::Retry(e)) => {
tracing::error!(error = e, "gave up publishing the agent icon");
}
tokio::time::sleep(delay).await;
delay = (delay * 2).min(MAX_RETRY);
}
}