move startup_read from swarm-queue-client to swarm-secret-client

read_with_startup_retry retries the first read of a client secret through a store that is restarting. Nothing in swarm-queue-client calls it; its only callers are the four binaries that read the secret (swarm-builder, swarm-claude-login, swarm-controller, swarm-review-classifier), and they already depend on swarm-secret-client. It sits beside stored_secret, which moved there in #5538. swarm-queue-client drops its backon dependency and the tokio test-util dev-dependency that only these tests used.

Cargo.lock

@@ -5027,7 +5027,6 @@ version = "0.1.0"
 dependencies = [
  "async-nats",
  "axum",
- "backon",
  "data-encoding",
  "hive-sh4re",
  "reqwest",
@@ -5067,6 +5066,7 @@ dependencies = [
 name = "swarm-secret-client"
 version = "0.1.0"
 dependencies = [
+ "backon",
  "futures-util",
  "reqwest",
  "rustify",

swarm-builder/src/main.rs

@@ -74,7 +74,7 @@ async fn main() -> Result<()> {
         deadline: BUILD_DEADLINE,
     });
     let secret =
-        swarm_queue_client::read_with_startup_retry("reading the queue client secret", || {
+        swarm_secret_client::read_with_startup_retry("reading the queue client secret", || {
             client_secret(&client_id)
         })
         .await?;

swarm-claude-login/src/main.rs

@@ -63,7 +63,7 @@ async fn main() -> Result<()> {
         STEP_DEADLINE,
     ));
     let secret =
-        swarm_queue_client::read_with_startup_retry("reading the queue client secret", || {
+        swarm_secret_client::read_with_startup_retry("reading the queue client secret", || {
             client_secret(&client_id)
         })
         .await?;

swarm-controller/src/queue_identity.rs

@@ -58,7 +58,7 @@ pub async fn init() -> Result<()> {
     let identity = match coordinates_from(|name| std::env::var(name).ok())? {
         None => None,
         Some(coordinates) => {
-            let secret = swarm_queue_client::read_with_startup_retry(
+            let secret = swarm_secret_client::read_with_startup_retry(
                 "fetching the queue client secret from the swarm secret store",
                 fetch_secret,
             )

swarm-queue-client/Cargo.toml

@@ -71,7 +71,6 @@ agents-md-initial = ["async-nats/jetstream"]
 # Bare (no `kv`/`jetstream`) unless a consumer opts into the `kv` feature
 # above - the connect itself needs none of them.
 async-nats.workspace = true
-backon.workspace = true
 hive-sh4re = { workspace = true, optional = true }
 # `blocking` on top of the workspace default (`form`/`json`/`rustls`) —
 # `mint_token_for_blocking` needs `reqwest::blocking::Client` for a caller
@@ -100,9 +99,6 @@ utoipa = { workspace = true, optional = true }
 # A loopback stand-in token endpoint for the client secret re-read tests.
 axum.workspace = true
 data-encoding.workspace = true
-# `test-util` for `#[tokio::test(start_paused = true)]`: the startup-read
-# retry tests run on paused time.
-tokio = { workspace = true, features = ["test-util"] }
 
 [lints]
 workspace = true

swarm-queue-client/src/lib.rs

@@ -496,9 +496,6 @@ pub mod subagent_term;
 /// same shapes.
 pub mod review_classifier;
 
-mod startup_read;
-pub use startup_read::read_with_startup_retry;
-
 /// Only the fields this needs; authelia returns several.
 #[derive(serde::Deserialize)]
 struct TokenResponse {

swarm-review-classifier/src/main.rs

@@ -68,7 +68,7 @@ async fn serve() -> Result<()> {
         },
     };
     let secret =
-        swarm_queue_client::read_with_startup_retry("reading the queue client secret", || {
+        swarm_secret_client::read_with_startup_retry("reading the queue client secret", || {
             client_secret(&client_id)
         })
         .await?;

swarm-secret-client/Cargo.toml

@@ -4,6 +4,8 @@ version.workspace = true
 edition.workspace = true
 
 [dependencies]
+# `read_with_startup_retry`'s backoff schedule.
+backon.workspace = true
 # `Shared`, so every caller refused while a secret re-read runs waits on that
 # one read (`stored_secret`).
 futures-util.workspace = true

swarm-secret-client/src/lib.rs

@@ -44,9 +44,11 @@ pub mod path;
 pub mod policy;
 pub mod queue;
 pub mod review_classifier;
+pub mod startup_read;
 pub mod stored_secret;
 
 pub use client::SecretStore;
+pub use startup_read::read_with_startup_retry;
 
 /// What can go wrong between "we have a client certificate" and "we have the
 /// credential".

swarm-secret-client/src/startup_read.rs

@@ -0,0 +1,100 @@
+//! The first read of a client secret, retried through a store that is
+//! restarting alongside the service reading it.
+//!
+//! Every service that logs in to the queue reads its secret once at start.
+//! The store may still be coming up, so a failed read waits and tries again;
+//! a store down for longer than the schedule fails the start, and the unit's
+//! `Restart=` tries again.
+
+use std::fmt::Display;
+use std::future::Future;
+use std::time::Duration;
+
+use backon::{ExponentialBuilder, Retryable as _};
+
+/// The wait after the first failed read.
+const FIRST_RETRY: Duration = Duration::from_secs(1);
+
+/// The longest wait between two reads.
+const MAX_RETRY: Duration = Duration::from_secs(30);
+
+/// Reads after the first one: waits of 1, 2, 4, 8, 16 and 30 seconds.
+const RETRIES: usize = 6;
+
+/// Run `read`, waiting after each failure up to six times, and answer
+/// with the first success or the last attempt's error. Each failure is logged
+/// as a warning naming `what`.
+///
+/// # Errors
+/// The error of the final attempt, once every read has failed.
+pub async fn read_with_startup_retry<T, E, F, Fut>(what: &str, read: F) -> Result<T, E>
+where
+    F: FnMut() -> Fut,
+    Fut: Future<Output = Result<T, E>>,
+    E: Display,
+{
+    read.retry(
+        ExponentialBuilder::default()
+            .with_min_delay(FIRST_RETRY)
+            .with_max_delay(MAX_RETRY)
+            .with_max_times(RETRIES),
+    )
+    .notify(|e: &E, wait| {
+        tracing::warn!(error = %format!("{e:#}"), retry_in = ?wait, "{what} failed");
+    })
+    .await
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    use tokio::time::Instant;
+
+    use super::*;
+
+    /// A store that fails `failures` times and then answers.
+    fn read_after(
+        failures: usize,
+        calls: &AtomicUsize,
+    ) -> std::future::Ready<Result<&'static str, String>> {
+        std::future::ready(if calls.fetch_add(1, Ordering::SeqCst) < failures {
+            Err("store is down".to_owned())
+        } else {
+            Ok("secret")
+        })
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn a_store_that_answers_at_once_costs_no_wait() {
+        let calls = AtomicUsize::new(0);
+        let started = Instant::now();
+        let got = read_with_startup_retry("reading", || read_after(0, &calls)).await;
+        assert_eq!(got, Ok("secret"));
+        assert_eq!(calls.load(Ordering::SeqCst), 1);
+        assert_eq!(started.elapsed(), Duration::ZERO);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn a_store_that_comes_up_late_is_waited_for() {
+        let calls = AtomicUsize::new(0);
+        let started = Instant::now();
+        let got = read_with_startup_retry("reading", || read_after(3, &calls)).await;
+        assert_eq!(got, Ok("secret"));
+        assert_eq!(calls.load(Ordering::SeqCst), 4);
+        assert_eq!(started.elapsed(), Duration::from_secs(1 + 2 + 4));
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn a_store_that_stays_down_fails_after_the_whole_schedule() {
+        let calls = AtomicUsize::new(0);
+        let started = Instant::now();
+        let got = read_with_startup_retry("reading", || read_after(usize::MAX, &calls)).await;
+        assert_eq!(got, Err("store is down".to_owned()));
+        assert_eq!(calls.load(Ordering::SeqCst), RETRIES + 1);
+        assert_eq!(
+            started.elapsed(),
+            Duration::from_secs(1 + 2 + 4 + 8 + 16 + 30)
+        );
+    }
+}