retry: share one backon helper for the client secret reads and convert the remaining doubling tables
This commit is contained in:
parent
cf2f69d19c
commit
9a3153cb08
14 changed files with 354 additions and 320 deletions
|
|
@ -13,7 +13,6 @@
|
|||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use futures_util::StreamExt as _;
|
||||
|
|
@ -31,10 +30,6 @@ const CERT_ROLE: &str = "swarm-claude-login";
|
|||
/// The environment prefix `swarm-claude-login.nix` sets.
|
||||
const PREFIX: &str = "SWARM_CLAUDE_LOGIN";
|
||||
|
||||
/// Seconds to wait after each failed read of the queue client secret. A
|
||||
/// longer outage fails the start, and the unit's `Restart=` tries again.
|
||||
const RETRY_DELAYS_S: [u64; 6] = [1, 2, 4, 8, 16, 30];
|
||||
|
||||
/// The agent's store leaf, written under this service's certificate.
|
||||
struct AgentLeaf;
|
||||
|
||||
|
|
@ -67,7 +62,11 @@ async fn main() -> Result<()> {
|
|||
std::env::temp_dir(),
|
||||
STEP_DEADLINE,
|
||||
));
|
||||
let secret = client_secret_with_retry(&client_id).await?;
|
||||
let secret =
|
||||
swarm_queue_client::read_with_startup_retry("reading the queue client secret", || {
|
||||
client_secret(&client_id)
|
||||
})
|
||||
.await?;
|
||||
let reread_id = client_id.clone();
|
||||
let secret = StoredSecret::new(client_id.clone(), secret, move || {
|
||||
let client_id = reread_id.clone();
|
||||
|
|
@ -166,21 +165,6 @@ async fn handle(payload: &[u8], logins: &Arc<login::Logins>) -> Reply {
|
|||
reply
|
||||
}
|
||||
|
||||
/// The queue client secret, read from the store, retried through a store that
|
||||
/// is restarting alongside this service.
|
||||
async fn client_secret_with_retry(client_id: &str) -> Result<String> {
|
||||
for delay in RETRY_DELAYS_S.map(Duration::from_secs) {
|
||||
match client_secret(client_id).await {
|
||||
Ok(secret) => return Ok(secret),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %format!("{e:#}"), retry_in = ?delay, "reading the queue client secret failed");
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
client_secret(client_id).await
|
||||
}
|
||||
|
||||
async fn client_secret(client_id: &str) -> Result<String> {
|
||||
use swarm_secret_client::builder::ClientSecret;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue