diff --git a/swarm-builder/src/main.rs b/swarm-builder/src/main.rs index d7a21f935..dc46d9594 100644 --- a/swarm-builder/src/main.rs +++ b/swarm-builder/src/main.rs @@ -113,11 +113,15 @@ async fn main() -> Result<()> { system: var("SYSTEM")?, cache: push_target()?, }); - let secret = - swarm_secret_client::read_with_startup_retry("reading the queue client secret", || { - client_secret(&client_id) - }) - .await?; + let Some(secret) = swarm_secret_client::read_with_startup_retry_or_stop( + "reading the queue client secret", + stop.clone(), + || client_secret(&client_id), + ) + .await? + else { + return Ok(()); + }; let evaluator = archetypes::Evaluator { nix: builder.nix.clone(), hyperhive: builder.hyperhive.clone(), diff --git a/swarm-claude-login/src/main.rs b/swarm-claude-login/src/main.rs index 930f9ffa5..b4674158e 100644 --- a/swarm-claude-login/src/main.rs +++ b/swarm-claude-login/src/main.rs @@ -119,11 +119,15 @@ async fn main() -> Result<()> { std::env::temp_dir(), STEP_DEADLINE, )); - let secret = - swarm_secret_client::read_with_startup_retry("reading the queue client secret", || { - client_secret(&client_id) - }) - .await?; + let Some(secret) = swarm_secret_client::read_with_startup_retry_or_stop( + "reading the queue client secret", + stop.clone(), + || client_secret(&client_id), + ) + .await? + else { + return Ok(()); + }; let reread_id = client_id.clone(); let secret = StoredSecret::new(client_id.clone(), secret, move || { let client_id = reread_id.clone(); diff --git a/swarm-notify-router/src/main.rs b/swarm-notify-router/src/main.rs index 2e6180de1..2b49270cf 100644 --- a/swarm-notify-router/src/main.rs +++ b/swarm-notify-router/src/main.rs @@ -100,11 +100,15 @@ async fn main() -> Result<()> { .await .context("starting the health listener")?; let client_id = var("OIDC_CLIENT_ID")?; - let secret = - swarm_secret_client::read_with_startup_retry("reading the queue client secret", || { - client_secret(&client_id) - }) - .await?; + let Some(secret) = swarm_secret_client::read_with_startup_retry_or_stop( + "reading the queue client secret", + stop.clone(), + || client_secret(&client_id), + ) + .await? + else { + return Ok(()); + }; let reread_id = client_id.clone(); let secret = StoredSecret::new(client_id.clone(), secret, move || { let client_id = reread_id.clone(); diff --git a/swarm-review-classifier/src/main.rs b/swarm-review-classifier/src/main.rs index 93ec2241c..044d8021a 100644 --- a/swarm-review-classifier/src/main.rs +++ b/swarm-review-classifier/src/main.rs @@ -75,11 +75,15 @@ async fn serve() -> Result<()> { cwd: std::env::temp_dir().join("review-classifier-acp-cwd"), }, }; - let secret = - swarm_secret_client::read_with_startup_retry("reading the queue client secret", || { - client_secret(&client_id) - }) - .await?; + let Some(secret) = swarm_secret_client::read_with_startup_retry_or_stop( + "reading the queue client secret", + stop.clone(), + || client_secret(&client_id), + ) + .await? + else { + return Ok(()); + }; let reread_id = client_id.clone(); let secret = StoredSecret::new(client_id.clone(), secret, move || { let client_id = reread_id.clone(); diff --git a/swarm-secret-client/src/lib.rs b/swarm-secret-client/src/lib.rs index 5100af2f9..ab4003041 100644 --- a/swarm-secret-client/src/lib.rs +++ b/swarm-secret-client/src/lib.rs @@ -62,7 +62,7 @@ pub mod stored_secret; pub use client::SecretStore; pub use shutdown::{DRAIN, InFlight, Shutdown, serve_bounded, shutdown_signal}; -pub use startup_read::read_with_startup_retry; +pub use startup_read::{read_with_startup_retry, read_with_startup_retry_or_stop}; /// What can go wrong between "we have a client certificate" and "we have the /// credential". diff --git a/swarm-secret-client/src/startup_read.rs b/swarm-secret-client/src/startup_read.rs index 73839f1b5..ca6f1aaf3 100644 --- a/swarm-secret-client/src/startup_read.rs +++ b/swarm-secret-client/src/startup_read.rs @@ -25,6 +25,9 @@ const RETRIES: usize = 6; /// with the first success or the last attempt's error. Each failure is logged /// as a warning naming `what`. /// +/// This read doesn't hear a stop signal: a caller that already has one in hand +/// uses [`read_with_startup_retry_or_stop`]. +/// /// # Errors /// The error of the final attempt, once every read has failed. pub async fn read_with_startup_retry(what: &str, read: F) -> Result @@ -45,6 +48,36 @@ where .await } +/// [`read_with_startup_retry`], ended by `stop`. +/// +/// When `stop` resolves, in a sleep or during an attempt, the read is dropped +/// without another attempt and the answer is `Ok(None)`. The caller exits 0, as +/// it does for the same signal later on, so `systemctl stop` or a pod delete +/// during a store outage returns at once instead of after the schedule (about a +/// minute) or the unit's stop timeout. +/// +/// # Errors +/// The error of the final attempt, once every read has failed. +pub async fn read_with_startup_retry_or_stop( + what: &str, + stop: impl Future, + read: F, +) -> Result, E> +where + F: FnMut() -> Fut, + Fut: Future>, + E: Display, +{ + tokio::select! { + biased; + () = stop => { + tracing::info!("stop arrived while {what}; exiting before startup finished"); + Ok(None) + } + read = read_with_startup_retry(what, read) => read.map(Some), + } +} + #[cfg(test)] mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; @@ -85,6 +118,48 @@ mod tests { assert_eq!(started.elapsed(), Duration::from_secs(1 + 2 + 4)); } + /// Resolves `after` from now, like a SIGTERM that arrives mid-outage. + async fn stop_after(after: Duration) { + tokio::time::sleep(after).await; + } + + #[tokio::test(start_paused = true)] + async fn a_stop_during_a_retry_wait_ends_the_read_without_another_attempt() { + // Attempts at 0 s and 1 s fail; the wait after the second runs 1 s to 3 s. + let calls = AtomicUsize::new(0); + let started = Instant::now(); + let got = + read_with_startup_retry_or_stop("reading", stop_after(Duration::from_secs(2)), || { + read_after(usize::MAX, &calls) + }) + .await; + assert_eq!(got, Ok(None)); + assert_eq!(calls.load(Ordering::SeqCst), 2); + assert_eq!(started.elapsed(), Duration::from_secs(2)); + } + + #[tokio::test(start_paused = true)] + async fn a_stop_that_has_already_fired_makes_no_attempt() { + let calls = AtomicUsize::new(0); + let got = read_with_startup_retry_or_stop("reading", std::future::ready(()), || { + read_after(0, &calls) + }) + .await; + assert_eq!(got, Ok(None)); + assert_eq!(calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test(start_paused = true)] + async fn a_stop_after_the_read_succeeded_changes_nothing() { + let calls = AtomicUsize::new(0); + let got = + read_with_startup_retry_or_stop("reading", stop_after(Duration::from_mins(1)), || { + read_after(1, &calls) + }) + .await; + assert_eq!(got, Ok(Some("secret"))); + } + #[tokio::test(start_paused = true)] async fn a_store_that_stays_down_fails_after_the_whole_schedule() { let calls = AtomicUsize::new(0); diff --git a/swarm-stt/src/main.rs b/swarm-stt/src/main.rs index b9cdbc893..2bc79d902 100644 --- a/swarm-stt/src/main.rs +++ b/swarm-stt/src/main.rs @@ -87,20 +87,28 @@ fn settings(var: Var<'_>, number: Number<'_>) -> Result { }) } -/// The queue connection, as the service's own OIDC client. -async fn connect_queue(var: Var<'_>) -> Result { +/// The queue connection, as the service's own OIDC client. `None` when `stop` +/// fired while the secret was still being waited for. +async fn connect_queue( + var: Var<'_>, + stop: swarm_secret_client::Shutdown, +) -> Result> { let client_id = var("OIDC_CLIENT_ID")?; - let secret = - swarm_secret_client::read_with_startup_retry("reading the queue client secret", || { - client_secret(&client_id) - }) - .await?; + let Some(secret) = swarm_secret_client::read_with_startup_retry_or_stop( + "reading the queue client secret", + stop.clone(), + || client_secret(&client_id), + ) + .await? + else { + return Ok(None); + }; let reread_id = client_id.clone(); let secret = StoredSecret::new(client_id.clone(), secret, move || { let client_id = reread_id.clone(); async move { client_secret(&client_id).await.map_err(Into::into) } }); - swarm_queue_client::connect(QueueConfig { + let queue = swarm_queue_client::connect(QueueConfig { url: var("NATS_URL")?, token_endpoint: var("OIDC_TOKEN_ENDPOINT")?, client_id, @@ -109,8 +117,8 @@ async fn connect_queue(var: Var<'_>) -> Result { .ok() .map(PathBuf::from), }) - .await - .map_err(Into::into) + .await?; + Ok(Some(queue)) } /// Removes expired transcripts from `cache` every [`PURGE_EVERY`], for as long as @@ -155,7 +163,9 @@ async fn main() -> Result<()> { waiting, } = settings(&var, &number)?; let per_agent: Arc>> = Arc::default(); - let queue = connect_queue(&var).await?; + let Some(queue) = connect_queue(&var, stop.clone()).await? else { + return Ok(()); + }; swarm_queue_client::ready_when_connected(&health, &queue); let metrics = Arc::new(Metrics::from_env());