vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

retry the remaining plain schedules with backon

Browse source
This commit is contained in:
damocles 2026-10-17 15:25:22 +02:00 • committed by mara
commit ed76419be7
7 changed files with 132 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

52
swarm-controller/src/agent_permissions.rs
View file

@ -10,10 +10,12 @@
use std::collections::BTreeSet;
use std::sync::Arc;
use std::time::Duration;
use axum::Json;
use axum::extract::{Path, State};
use axum::http::StatusCode;
use backon::{ConstantBuilder, Retryable as _};
use futures_util::StreamExt as _;
use hive_sh4re::permissions::Permission;
use problem_details::ProblemDetails;
@ -335,32 +337,44 @@ pub(crate) async fn restore(client: &async_nats::Client, agent: &str, replaced:
/// the queue is not serving it, rather than that nothing is set.
pub(crate) fn spawn(client: async_nats::Client, wanted: Arc<WantedWriter>, hives: Vec<String>) {
tokio::spawn(async move {
const RETRY: std::time::Duration = std::time::Duration::from_secs(5);
let store = loop {
if swarm_queue_client::ensure_connected(&client).is_ok() {
match agent_permissions::open_or_create(&client).await {
Ok(store) => break store,
Err(e) => tracing::warn!(
error = swarm_queue_client::chain(&e),
"creating the agent-permissions bucket failed; retrying"
),
}
const RETRY: Duration = Duration::from_secs(5);
// The first two retry until they work: the queue is the only thing either waits on.
let every_five_seconds = || {
ConstantBuilder::default()
.with_delay(RETRY)
.without_max_times()
};
let Ok(store) = (|| async {
swarm_queue_client::ensure_connected(&client)?;
agent_permissions::open_or_create(&client).await
})
.retry(every_five_seconds())
.notify(|e: &swarm_queue_client::Error, _| {
// Not connected is the queue being late, and it says so itself.
if !matches!(e, swarm_queue_client::Error::NotConnected(_)) {
tracing::warn!(
error = swarm_queue_client::chain(e),
"creating the agent-permissions bucket failed; retrying"
);
}
tokio::time::sleep(RETRY).await;
})
.await
else {
return;
};
tokio::spawn({
let (store, wanted, hives) = (store.clone(), Arc::clone(&wanted), hives.clone());
async move {
loop {
match seed_ruth_grant(&store, &wanted, &hives).await {
Ok(()) => break,
Err(e) => tracing::warn!(
// Retries without end, so it answers only once it has worked.
let _ = (|| seed_ruth_grant(&store, &wanted, &hives))
.retry(every_five_seconds())
.notify(|e: &anyhow::Error, _| {
tracing::warn!(
error = %format!("{e:#}"),
"agent permissions: seeding ruth's grant failed; retrying"
),
}
tokio::time::sleep(RETRY).await;
}
);
})
.await;
}
});
let subject = agent_permissions::import_subject_wildcard();