swarm-controller: keep retrying the forge webhook secret and registration after a startup outage

The webhook secret slot is filled in the background, on the forge token's
capped unbounded backoff, when the store can't answer at start. The hook
registration waits for the secret and retries on the same backoff instead
of giving up after five tries.

Closes #6101

swarm-controller/src/agent_limits.rs

@@ -472,7 +472,7 @@ pub(crate) mod tests {
                 hive_jobq::Graph::new(),
                 hive_jobq::resources::ResourceTable::new(),
             ))),
-            webhook_secret: None,
+            webhook_secret: std::sync::Arc::default(),
             config_prs: Arc::default(),
             swarm_name: None,
             auth: None,

swarm-controller/src/backend_env.rs

@@ -164,7 +164,7 @@ mod tests {
                 hive_jobq::Graph::new(),
                 hive_jobq::resources::ResourceTable::new(),
             ))),
-            webhook_secret: None,
+            webhook_secret: std::sync::Arc::default(),
             config_prs: std::sync::Arc::default(),
             swarm_name: None,
             auth: None,

swarm-controller/src/forge.rs

@@ -317,40 +317,53 @@ const BACKGROUND_FIRST: std::time::Duration = std::time::Duration::from_secs(30)
 /// The longest the background re-read waits between attempts.
 const BACKGROUND_CAP: std::time::Duration = std::time::Duration::from_mins(5);
 
-/// Fill `slot` with the first value `attempt` yields: after
-/// [`BACKGROUND_FIRST`], then at 60s, doubling, capped at [`BACKGROUND_CAP`].
-/// Never gives up. A failure is logged at `info` once the next wait is the cap,
-/// so a controller that never gets its token says so once per
+/// Run `attempt` until it succeeds: after `min_delay`, doubling, capped at
+/// [`BACKGROUND_CAP`]. Never gives up. A failure is logged at `info` once the
+/// next wait is the cap, so work that never succeeds says so once per
 /// [`BACKGROUND_CAP`] without a line per early retry, and at `debug` before
-/// that. The caller logs the first failure and the recovery.
-async fn fill_when_ready<T, F, Fut>(slot: &OnceLock<T>, attempt: F) -> &T
+/// that. The caller logs the first failure and the recovery. `what` names the
+/// thing waited for in those lines.
+pub(crate) async fn retry_unbounded<T, F, Fut>(
+    what: &str,
+    min_delay: std::time::Duration,
+    attempt: F,
+) -> T
 where
     F: FnMut() -> Fut,
     Fut: std::future::Future<Output = Result<T>>,
 {
-    tokio::time::sleep(BACKGROUND_FIRST).await;
-    let value = attempt
+    attempt
         .retry(
             ExponentialBuilder::default()
-                .with_min_delay(BACKGROUND_FIRST * 2)
+                .with_min_delay(min_delay)
                 .with_max_delay(BACKGROUND_CAP)
                 .without_max_times(),
         )
         .notify(|e, wait| {
             if wait >= BACKGROUND_CAP {
-                tracing::info!(
-                    error = %format!("{e:#}"),
-                    "still no forge access: the controller's forge token is unavailable"
-                );
+                tracing::info!(what, error = %format!("{e:#}"), "still unavailable; retrying");
             } else {
-                tracing::debug!(
-                    error = %format!("{e:#}"),
-                    "the controller's forge token is still unavailable"
-                );
+                tracing::debug!(what, error = %format!("{e:#}"), "unavailable; retrying");
             }
         })
         .await
-        .expect("a retry with no cap on its tries returns only once it succeeds");
+        .expect("a retry with no cap on its tries returns only once it succeeds")
+}
+
+/// Fill `slot` with the first value `attempt` yields: after
+/// [`BACKGROUND_FIRST`], then at 60s, doubling, capped at [`BACKGROUND_CAP`],
+/// as [`retry_unbounded`] does. `what` names the value in its log lines.
+pub(crate) async fn fill_when_ready<'a, T, F, Fut>(
+    slot: &'a OnceLock<T>,
+    what: &str,
+    attempt: F,
+) -> &'a T
+where
+    F: FnMut() -> Fut,
+    Fut: std::future::Future<Output = Result<T>>,
+{
+    tokio::time::sleep(BACKGROUND_FIRST).await;
+    let value = retry_unbounded(what, BACKGROUND_FIRST * 2, attempt).await;
     slot.get_or_init(|| value)
 }
 
@@ -400,7 +413,7 @@ pub async fn enable(
             );
             tokio::spawn(async move {
                 let client = Arc::clone(
-                    fill_when_ready(&slot, || async {
+                    fill_when_ready(&slot, "the controller's forge token", || async {
                         Client::connect(&url).await.map(Arc::new)
                     })
                     .await,
@@ -2330,7 +2343,7 @@ mod tests {
         let slot = OnceLock::new();
         let start = tokio::time::Instant::now();
         let mut calls = 0;
-        let got = fill_when_ready(&slot, || {
+        let got = fill_when_ready(&slot, "a test value", || {
             calls += 1;
             let n = calls;
             async move {
@@ -2352,7 +2365,7 @@ mod tests {
         let slot = OnceLock::new();
         let start = tokio::time::Instant::now();
         let mut calls = 0;
-        fill_when_ready(&slot, || {
+        fill_when_ready(&slot, "a test value", || {
             calls += 1;
             let n = calls;
             async move {

swarm-controller/src/forge_account.rs

@@ -170,7 +170,7 @@ mod tests {
                 hive_jobq::Graph::new(),
                 hive_jobq::resources::ResourceTable::new(),
             ))),
-            webhook_secret: None,
+            webhook_secret: std::sync::Arc::default(),
             config_prs: std::sync::Arc::default(),
             swarm_name: None,
             auth: None,

swarm-controller/src/github_account.rs

@@ -132,7 +132,7 @@ mod tests {
                 hive_jobq::Graph::new(),
                 hive_jobq::resources::ResourceTable::new(),
             ))),
-            webhook_secret: None,
+            webhook_secret: std::sync::Arc::default(),
             config_prs: std::sync::Arc::default(),
             swarm_name: None,
             auth: None,

swarm-controller/src/gitlab_account.rs

@@ -223,7 +223,7 @@ mod tests {
                 hive_jobq::Graph::new(),
                 hive_jobq::resources::ResourceTable::new(),
             ))),
-            webhook_secret: None,
+            webhook_secret: std::sync::Arc::default(),
             config_prs: std::sync::Arc::default(),
             swarm_name: None,
             auth: None,

swarm-controller/src/linked_accounts.rs

@@ -1131,7 +1131,7 @@ pub(crate) mod tests {
                 hive_jobq::Graph::new(),
                 hive_jobq::resources::ResourceTable::new(),
             ))),
-            webhook_secret: None,
+            webhook_secret: std::sync::Arc::default(),
             config_prs: std::sync::Arc::default(),
             swarm_name: None,
             auth: None,

swarm-controller/src/main.rs

@@ -94,8 +94,8 @@ use routes::AppState;
 use startup::{
     agent_icon_reader, agent_status_reader, agents_md_reader, build_app,
     configured_matrix_homeserver, connect_status_reader, load_hives, load_links, load_swarm_name,
-    load_webhook_secret, reported_inference, socket_path, spawn_agent_renewal,
-    spawn_matrix_account_backfill, spawn_per_agent_upkeep, start_forge, wanted_writer,
+    reported_inference, socket_path, spawn_agent_renewal, spawn_matrix_account_backfill,
+    spawn_per_agent_upkeep, start_forge, wanted_writer,
 };
 
 #[tokio::main]
@@ -192,7 +192,7 @@ async fn main() -> Result<()> {
     register_cert_gauges(jobq_metrics_provider.as_ref());
     held_deploys::register(jobq_metrics_provider.as_ref(), &jobq);
 
-    let webhook_secret = load_webhook_secret().await;
+    let webhook_secret = webhook::load().await;
 
     // Diagnostic only — `put_matrix_account` does not call this yet (see its
     // own doc comment), so this cannot change any request's outcome. Logged
@@ -215,7 +215,7 @@ async fn main() -> Result<()> {
         Arc::clone(&config_prs),
         Arc::clone(&jobq),
         hives.clone(),
-        webhook_secret.clone(),
+        Arc::clone(&webhook_secret),
         auth.clone(),
         status.as_ref().map(|s| s.queue_client()),
     )

swarm-controller/src/matrix_account.rs

@@ -474,7 +474,7 @@ mod tests {
                 hive_jobq::Graph::new(),
                 hive_jobq::resources::ResourceTable::new(),
             ))),
-            webhook_secret: None,
+            webhook_secret: std::sync::Arc::default(),
             config_prs: std::sync::Arc::default(),
             swarm_name: None,
             auth: None,

swarm-controller/src/routes/mod.rs

@@ -93,17 +93,17 @@ pub(crate) struct AppState {
     /// either has or does not. `POST /api/forge/users/{name}/admin` reads
     /// it for every name and refuses when it cannot (`admin_unless_agent`).
     pub(crate) auth: Option<Arc<auth::AuthBridge>>,
-    /// HMAC secret for swarm-wide forge webhooks, loaded once at startup.
-    /// `None` when it could not be read or created — the webhook endpoint
-    /// then refuses every delivery with 503 rather than admitting one it
-    /// cannot verify. Deliberately not fatal to startup: nothing is
-    /// registered against that endpoint yet, and the rest of this daemon's
-    /// surface is unaffected. See [`load_or_generate_secret`](crate::webhook::load_or_generate_secret).
+    /// HMAC secret for swarm-wide forge webhooks. Empty while it could not be
+    /// read or created — the webhook endpoint then refuses every delivery with
+    /// 503 rather than admitting one it cannot verify, and the hook
+    /// registration waits — and filled by [`load`](crate::webhook::load)
+    /// once the store answers. Deliberately not fatal to startup: the rest of
+    /// this daemon's surface is unaffected.
     ///
-    /// `Arc<str>`, not `Arc<String>`: the value is never mutated after
-    /// startup, and this way `as_deref()` yields the `&str` the verifier
-    /// takes without a second hop through `String`.
-    pub(crate) webhook_secret: Option<Arc<str>>,
+    /// `Arc<str>`, not `Arc<String>`: the value is never mutated once set, and
+    /// this way `AsRef::as_ref` yields the `&str` the verifier takes without a
+    /// second hop through `String`.
+    pub(crate) webhook_secret: Late<Arc<str>>,
     /// Last successful `agent-configs/*` scan, kept current by
     /// [`config_pr::spawn`]. Empty while this daemon has no forge access —
     /// same "absent means don't ask" shape as `status` and `forge`, not a
@@ -296,7 +296,7 @@ pub(crate) mod tests {
             reported_inference: None,
             agent_login: None,
             jobq: std::sync::Arc::clone(&sched),
-            webhook_secret: None,
+            webhook_secret: std::sync::Arc::default(),
             config_prs: std::sync::Arc::default(),
             swarm_name: None,
             // No bridge: these tests drive agent *creation*, which queues a

swarm-controller/src/startup.rs

@@ -15,7 +15,6 @@ use std::sync::{Arc, Mutex};
 
 use anyhow::Result;
 use axum::{Json, routing::get};
-use backon::{ExponentialBuilder, Retryable as _};
 use utoipa::OpenApi;
 use utoipa_axum::{router::OpenApiRouter, routes};
 
@@ -214,24 +213,6 @@ pub(crate) fn agent_login(
     Some(agent_login::Reported::spawn(status?.queue_client()))
 }
 
-/// Same "log and carry on" shape as the queue/bridge/forge wiring in `main`.
-/// A controller that cannot hold a webhook secret still serves every other
-/// route; the webhook endpoint answers 503, which is the honest answer
-/// rather than a silent accept. Lifted out of `main` for
-/// `clippy::too_many_lines`.
-pub(crate) async fn load_webhook_secret() -> Option<Arc<str>> {
-    match webhook::load_or_generate_secret().await {
-        Ok(secret) => Some(Arc::from(secret)),
-        Err(e) => {
-            tracing::warn!(
-                error = %format!("{e:#}"),
-                "webhook secret unavailable; swarm-wide forge webhooks are off"
-            );
-            None
-        }
-    }
-}
-
 /// The homeserver agents' own matrix accounts are minted on. Unset is this
 /// controller creating agents with no matrix account, which the mint node
 /// reports by name rather than `main` refusing to start.
@@ -444,21 +425,21 @@ pub(crate) const PUBLIC_URL_ENV: &str = "SWARM_CONTROLLER_PUBLIC_URL";
 /// briefly down at boot, and none of the daemon's other routes depend on a
 /// hook existing.
 ///
-/// Retried on a backoff rather than left to the next process start. Losing
-/// that race is the *common* case, not an edge one — the controller and the
-/// forge come up together on a rebuild, and a deploy was measured where both
-/// consecutive starts got `502 Bad Gateway` from the gateway because forgejo
-/// was not serving yet. Nothing schedules another start, so "the next restart
-/// retries" can leave a swarm with no hooks registered for as long as the
-/// daemon keeps running — and the failure is silent at both ends, because the
-/// forge has nothing to report about a call that never arrived.
+/// Retried on a capped backoff for as long as it takes, rather than left to
+/// the next process start. Losing that race is the *common* case, not an edge
+/// one — the controller and the forge come up together on a rebuild, and a
+/// deploy was measured where both consecutive starts got `502 Bad Gateway`
+/// from the gateway because forgejo was not serving yet. Nothing schedules
+/// another start, so "the next restart retries" can leave a swarm with no
+/// hooks registered for as long as the daemon keeps running — and the failure
+/// is silent at both ends, because the forge has nothing to report about a
+/// call that never arrived.
 ///
-/// Runs once there is a forge client, and silently does nothing when either
-/// other precondition is missing — each is a legitimate deployment shape (no
-/// state directory to hold a secret, no public vhost), and each is already
-/// logged where it is discovered.
-fn register_swarm_webhooks(forge: Arc<forge::Client>, secret: Option<Arc<str>>) {
-    let Some(secret) = secret else { return };
+/// Runs once there is a forge client, and silently does nothing when the
+/// public URL is missing: no public vhost is a legitimate deployment shape,
+/// and it is logged here. A webhook secret that is not loaded yet is waited
+/// for, as a forge that is not answering is.
+fn register_swarm_webhooks(forge: Arc<forge::Client>, secret: Late<Arc<str>>) {
     let Ok(public_url) = std::env::var(PUBLIC_URL_ENV) else {
         tracing::info!(
             "{PUBLIC_URL_ENV} unset; not registering swarm-wide forge webhooks (the \
@@ -467,40 +448,40 @@ fn register_swarm_webhooks(forge: Arc<forge::Client>, secret: Option<Arc<str>>)
         return;
     };
     tokio::spawn(async move {
-        // A bounded schedule, not a poll loop: 15s doubling to 240s, five
-        // retries. It exists to outlast a forge that is slow to start, not to
-        // re-register periodically. Registration is idempotent, so a repeat
-        // costs one API call and changes nothing.
-        const RETRIES: usize = 5;
-
-        let mut failed = 0;
-        let registered = (|| forge.ensure_swarm_webhooks(&public_url, &secret))
-            .retry(
-                ExponentialBuilder::default()
-                    .with_min_delay(std::time::Duration::from_secs(15))
-                    .with_max_delay(std::time::Duration::from_mins(4))
-                    .with_max_times(RETRIES),
-            )
-            .notify(|e, wait| {
-                failed += 1;
-                tracing::info!(
-                    error = %format!("{e:#}"),
-                    attempt = failed,
-                    retry_in = ?wait,
-                    "registering swarm-wide forge webhooks failed; retrying"
-                );
-            })
-            .await;
-        if let Err(e) = registered {
-            tracing::warn!(
-                error = %format!("{e:#}"),
-                attempts = RETRIES + 1,
-                "registering swarm-wide forge webhooks failed; giving up until the next start"
-            );
-        }
+        register_when_ready(&secret, |secret| {
+            let forge = Arc::clone(&forge);
+            let public_url = public_url.clone();
+            async move { forge.ensure_swarm_webhooks(&public_url, &secret).await }
+        })
+        .await;
     });
 }
 
+/// How long the first retry of the registration waits.
+const REGISTRATION_FIRST: std::time::Duration = std::time::Duration::from_secs(15);
+
+/// Run `ensure` with the webhook secret until it succeeds, waiting for
+/// `secret` to be filled first. Registration is idempotent, so a repeat costs
+/// one API call and changes nothing.
+async fn register_when_ready<F, Fut>(secret: &Late<Arc<str>>, mut ensure: F)
+where
+    F: FnMut(Arc<str>) -> Fut,
+    Fut: std::future::Future<Output = Result<()>>,
+{
+    forge::retry_unbounded("the swarm-wide forge webhooks", REGISTRATION_FIRST, || {
+        let secret = secret.get().cloned();
+        let registering = secret.map(&mut ensure);
+        async move {
+            match registering {
+                Some(registering) => registering.await,
+                None => anyhow::bail!("the webhook secret is not loaded yet"),
+            }
+        }
+    })
+    .await;
+    tracing::info!("registered the swarm-wide forge webhooks");
+}
+
 /// Connect to the swarm queue when this deployment wired one up, extracted
 /// out of `main` purely to keep that function under clippy's line-count
 /// lint — no behavior split, every comment below is unchanged from where
@@ -556,7 +537,7 @@ pub(crate) async fn start_forge(
     config_prs: Late<Arc<config_pr::ConfigPrCache>>,
     jobq: Arc<Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>>,
     hives: Vec<HiveEntry>,
-    webhook_secret: Option<Arc<str>>,
+    webhook_secret: Late<Arc<str>>,
     auth: Option<Arc<auth::AuthBridge>>,
     queue: Option<async_nats::Client>,
 ) {
@@ -978,4 +959,55 @@ mod tests {
             );
         }
     }
+
+    // Paused time: each sleep completes at once and advances the clock.
+    #[tokio::test(start_paused = true)]
+    async fn registration_waits_for_the_secret_and_then_registers_with_it() {
+        let secret = Late::<Arc<str>>::default();
+        let registered = Arc::new(Mutex::new(Vec::<String>::new()));
+        let filling = Arc::clone(&secret);
+        tokio::spawn(async move {
+            tokio::time::sleep(std::time::Duration::from_secs(100)).await;
+            filling.get_or_init(|| Arc::from("the-secret"));
+        });
+
+        let seen = Arc::clone(&registered);
+        register_when_ready(&secret, move |secret| {
+            let seen = Arc::clone(&seen);
+            async move {
+                seen.lock().unwrap().push(secret.to_string());
+                Ok(())
+            }
+        })
+        .await;
+
+        assert_eq!(*registered.lock().unwrap(), ["the-secret"]);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn a_forge_that_fails_past_the_old_bound_is_still_registered_with() {
+        let secret = Late::<Arc<str>>::default();
+        secret.get_or_init(|| Arc::from("the-secret"));
+        let start = tokio::time::Instant::now();
+        let mut calls = 0;
+
+        register_when_ready(&secret, |_| {
+            calls += 1;
+            let n = calls;
+            async move {
+                if n < 9 {
+                    anyhow::bail!("502 Bad Gateway")
+                }
+                Ok(())
+            }
+        })
+        .await;
+
+        assert_eq!(calls, 9);
+        // 15, 30, 60, 120, 240, then the 300s cap three times.
+        assert_eq!(
+            start.elapsed().as_secs(),
+            15 + 30 + 60 + 120 + 240 + 3 * 300
+        );
+    }
 }

swarm-controller/src/webhook.rs

@@ -24,7 +24,9 @@ use axum::{
 };
 use hmac::{Hmac, KeyInit as _, Mac as _};
 use sha2::Sha256;
+use std::sync::Arc;
 
+use crate::jobq::Late;
 use crate::routes::AppState;
 
 /// The legacy home of the HMAC secret, relative to the daemon's state directory.
@@ -116,6 +118,45 @@ pub(super) async fn load_or_generate_secret() -> Result<String> {
     resolve(&StoreSlot, &secret_path()).await
 }
 
+/// The slot the webhook secret lands in: filled before this returns if the
+/// store answers, otherwise from a background task that reads it again until it
+/// does. The webhook endpoint answers 503 until the slot is filled, and the
+/// registration of the forge hooks waits for it.
+///
+/// Every attempt is [`load_or_generate_secret`], so a retry only reads again: a
+/// secret is minted only when the store answers that it holds none, never on a
+/// failed read.
+pub(crate) async fn load() -> Late<Arc<str>> {
+    load_with(|| async { load_or_generate_secret().await.map(Arc::from) }).await
+}
+
+/// [`load`], with `load` standing in for the store.
+async fn load_with<F, Fut>(mut load: F) -> Late<Arc<str>>
+where
+    F: FnMut() -> Fut + Send + 'static,
+    Fut: Future<Output = Result<Arc<str>>> + Send,
+{
+    let slot = Late::<Arc<str>>::default();
+    match load().await {
+        Ok(secret) => {
+            slot.get_or_init(|| secret);
+        }
+        Err(e) => {
+            tracing::warn!(
+                error = %format!("{e:#}"),
+                "webhook secret unavailable; swarm-wide forge webhooks are off until it is read \
+                 from the swarm secret store in the background"
+            );
+            let filling = Arc::clone(&slot);
+            tokio::spawn(async move {
+                crate::forge::fill_when_ready(&filling, "the webhook secret", load).await;
+                tracing::info!("the webhook secret is loaded; forge webhooks are on");
+            });
+        }
+    }
+    slot
+}
+
 /// What the store holds at the webhook secret's path.
 pub(super) enum Stored {
     /// The secret.
@@ -551,7 +592,11 @@ pub(super) async fn post_webhook_forge(
     // Order matters: verify BEFORE looking at the path. An unauthenticated
     // caller learning which `{kind}` values exist, from the difference
     // between 404 and 401, is a small leak — but it is free to close.
-    if let Err(refusal) = verify(state.webhook_secret.as_deref(), &headers, &body) {
+    if let Err(refusal) = verify(
+        state.webhook_secret.get().map(AsRef::as_ref),
+        &headers,
+        &body,
+    ) {
         let message = refusal.message();
         tracing::warn!(%kind, %message, "webhook: refused delivery");
         return (refusal.status(), message).into_response();
@@ -671,13 +716,14 @@ async fn announce_knowledge_change(state: &AppState) {
 #[cfg(test)]
 mod tests {
     use super::{
-        DeliveryKind, Refusal, SecretSlot, Stored, load_or_generate_at, resolve, secret_path_from,
-        verify,
+        DeliveryKind, Refusal, SecretSlot, Stored, load_or_generate_at, load_with, resolve,
+        secret_path_from, verify,
     };
     use axum::{body::Bytes, http::HeaderMap, http::StatusCode};
     use hmac::{Hmac, KeyInit as _, Mac as _};
     use sha2::Sha256;
     use std::fmt::Write as _;
+    use std::sync::Arc;
 
     /// Mint the header Forgejo would send. Written out rather than reusing
     /// the verifier's internals so a test cannot pass by sharing a bug with
@@ -950,6 +996,8 @@ mod tests {
         /// What `read` answers while it holds nothing.
         empty: fn() -> anyhow::Result<Stored>,
         creates: std::sync::Mutex<u32>,
+        /// How many more reads fail as an unreachable store does, before it answers.
+        unreachable_reads: std::sync::Mutex<u32>,
     }
 
     impl FakeSlot {
@@ -958,6 +1006,7 @@ mod tests {
                 held: std::sync::Mutex::new(secret.map(str::to_owned)),
                 empty: || Ok(Stored::Absent),
                 creates: std::sync::Mutex::new(0),
+                unreachable_reads: std::sync::Mutex::new(0),
             }
         }
 
@@ -968,6 +1017,12 @@ mod tests {
             }
         }
 
+        /// Reads fail as an unreachable store does until `reads` have been made.
+        fn unreachable_for(self, reads: u32) -> Self {
+            *self.unreachable_reads.lock().expect("unpoisoned") = reads;
+            self
+        }
+
         fn held(&self) -> Option<String> {
             self.held.lock().expect("unpoisoned").clone()
         }
@@ -979,6 +1034,13 @@ mod tests {
 
     impl SecretSlot for FakeSlot {
         async fn read(&self) -> anyhow::Result<Stored> {
+            {
+                let mut left = self.unreachable_reads.lock().expect("unpoisoned");
+                if *left > 0 {
+                    *left -= 1;
+                    anyhow::bail!("connection refused");
+                }
+            }
             match self.held() {
                 Some(secret) => Ok(Stored::Value(secret)),
                 None => (self.empty)(),
@@ -1107,4 +1169,60 @@ mod tests {
         assert!(!path.exists());
         assert_eq!(slot.creates(), 0);
     }
+
+    /// `load_with`'s source: `resolve` against `slot` and the file at
+    /// `path`, as `load_or_generate_secret` does against the store.
+    fn resolving(
+        slot: &Arc<FakeSlot>,
+        path: &std::path::Path,
+    ) -> impl FnMut() -> std::pin::Pin<Box<dyn Future<Output = anyhow::Result<Arc<str>>> + Send>>
+    + Send
+    + 'static {
+        let (slot, path) = (Arc::clone(slot), path.to_owned());
+        move || {
+            let (slot, path) = (Arc::clone(&slot), path.clone());
+            Box::pin(async move { resolve(&*slot, &path).await.map(Arc::from) })
+        }
+    }
+
+    // Paused time: each sleep completes at once and advances the clock.
+    #[tokio::test(start_paused = true)]
+    async fn a_store_down_at_start_fills_the_slot_once_it_answers() {
+        let (_dir, path) = legacy_file(None);
+        let store = Arc::new(FakeSlot::holding(Some(IN_STORE)).unreachable_for(3));
+        let slot = load_with(resolving(&store, &path)).await;
+        assert!(slot.get().is_none(), "the first read failed");
+
+        tokio::time::sleep(std::time::Duration::from_mins(10)).await;
+
+        assert_eq!(slot.get().map(|s| &**s), Some(IN_STORE));
+        assert_eq!(store.creates(), 0);
+        assert!(!path.exists(), "nothing was minted on disk");
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn a_store_that_stays_down_mints_nothing_however_often_it_is_read() {
+        let (_dir, path) = legacy_file(None);
+        let store = Arc::new(FakeSlot::holding(None).unreachable_for(u32::MAX));
+        let slot = load_with(resolving(&store, &path)).await;
+        tokio::time::sleep(std::time::Duration::from_hours(3)).await;
+
+        assert!(slot.get().is_none());
+        assert_eq!(store.creates(), 0);
+        assert_eq!(store.held(), None);
+        assert!(!path.exists());
+        assert!(
+            *store.unreachable_reads.lock().expect("unpoisoned") < u32::MAX - 10,
+            "the background task kept reading"
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn a_store_answering_at_start_fills_the_slot_without_a_background_task() {
+        let (_dir, path) = legacy_file(None);
+        let store = Arc::new(FakeSlot::holding(Some(IN_STORE)));
+        let slot = load_with(resolving(&store, &path)).await;
+
+        assert_eq!(slot.get().map(|s| &**s), Some(IN_STORE));
+    }
 }