vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

retry the remaining plain schedules with backon

Browse source
This commit is contained in:
damocles 2026-10-17 15:25:22 +02:00 • committed by mara
commit ed76419be7
7 changed files with 132 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

52
swarm-controller/src/agent_permissions.rs
View file

@ -10,10 +10,12 @@
use std::collections::BTreeSet;
use std::sync::Arc;
use std::time::Duration;
use axum::Json;
use axum::extract::{Path, State};
use axum::http::StatusCode;
use backon::{ConstantBuilder, Retryable as _};
use futures_util::StreamExt as _;
use hive_sh4re::permissions::Permission;
use problem_details::ProblemDetails;
@ -335,32 +337,44 @@ pub(crate) async fn restore(client: &async_nats::Client, agent: &str, replaced:
/// the queue is not serving it, rather than that nothing is set.
pub(crate) fn spawn(client: async_nats::Client, wanted: Arc<WantedWriter>, hives: Vec<String>) {
tokio::spawn(async move {
const RETRY: std::time::Duration = std::time::Duration::from_secs(5);
let store = loop {
if swarm_queue_client::ensure_connected(&client).is_ok() {
match agent_permissions::open_or_create(&client).await {
Ok(store) => break store,
Err(e) => tracing::warn!(
error = swarm_queue_client::chain(&e),
"creating the agent-permissions bucket failed; retrying"
),
}
const RETRY: Duration = Duration::from_secs(5);
// The first two retry until they work: the queue is the only thing either waits on.
let every_five_seconds = || {
ConstantBuilder::default()
.with_delay(RETRY)
.without_max_times()
};
let Ok(store) = (|| async {
swarm_queue_client::ensure_connected(&client)?;
agent_permissions::open_or_create(&client).await
})
.retry(every_five_seconds())
.notify(|e: &swarm_queue_client::Error, _| {
// Not connected is the queue being late, and it says so itself.
if !matches!(e, swarm_queue_client::Error::NotConnected(_)) {
tracing::warn!(
error = swarm_queue_client::chain(e),
"creating the agent-permissions bucket failed; retrying"
);
}
tokio::time::sleep(RETRY).await;
})
.await
else {
return;
};
tokio::spawn({
let (store, wanted, hives) = (store.clone(), Arc::clone(&wanted), hives.clone());
async move {
loop {
match seed_ruth_grant(&store, &wanted, &hives).await {
Ok(()) => break,
Err(e) => tracing::warn!(
// Retries without end, so it answers only once it has worked.
let _ = (|| seed_ruth_grant(&store, &wanted, &hives))
.retry(every_five_seconds())
.notify(|e: &anyhow::Error, _| {
tracing::warn!(
error = %format!("{e:#}"),
"agent permissions: seeding ruth's grant failed; retrying"
),
}
tokio::time::sleep(RETRY).await;
}
);
})
.await;
}
});
let subject = agent_permissions::import_subject_wildcard();

15
swarm-controller/src/read_policy.rs
View file

@ -29,6 +29,7 @@
use std::{future::Future, time::Duration};
use anyhow::{Context, Result};
use backon::{BackoffBuilder as _, ConstantBuilder};
use swarm_secret_client::{Error, SecretStore, client::DEFAULT_CERT_MOUNT, policy};
/// File holding the authority hives are issued from, as
@ -111,12 +112,14 @@ where
F: FnMut() -> Fut,
Fut: Future<Output = Result<(), Unreachable>>,
{
let mut ticker = tokio::time::interval(interval);
// `interval`'s first tick is immediate, and that pass is the one the caller
// already ran before deciding to retry at all.
ticker.tick().await;
for n in 1..=attempts {
ticker.tick().await;
// One delay per attempt, waited before it: the pass the caller already ran
// before deciding to retry at all is the attempt that needs no wait.
let delays = ConstantBuilder::default()
.with_delay(interval)
.with_max_times(attempts as usize)
.build();
for (n, delay) in (1..).zip(delays) {
tokio::time::sleep(delay).await;
if attempt().await.is_ok() {
return Some(n);
}