retry the remaining plain schedules with backon

hive-forge-notify/src/notify.rs

@@ -27,9 +27,11 @@
 
 use std::collections::{HashMap, HashSet};
 use std::fmt::Write as _;
+use std::future::Future;
 use std::path::Path;
 use std::time::Duration;
 
+use backon::{ConstantBuilder, Retryable as _};
 use forgejo_api::structs::NotifySubjectType;
 use serde::{Deserialize, Deserializer};
 use time::OffsetDateTime;
@@ -54,10 +56,30 @@ const BODY_TRUNCATE: usize = 500;
 /// How long to wait between token-read retries when the token file is
 /// missing or unreadable at startup (e.g. hive-priv hasn't provisioned
 /// it yet, or a chown race left it temporarily root-owned).
-pub const TOKEN_RETRY_SECS: u64 = 30;
-/// Give up waiting for the token after this many retries (~10 minutes).
+const TOKEN_RETRY_SECS: u64 = 30;
+/// Give up waiting for the token after this many reads (~10 minutes).
 /// Avoids an infinite wait on agents that genuinely have no forge account.
-pub const TOKEN_RETRY_MAX: u32 = 20;
+const TOKEN_READS_MAX: usize = 20;
+
+/// Read a token that another process writes out of band, waiting for it to
+/// appear: `read` is called up to 20 times, 30 seconds apart (about ten
+/// minutes). `None` when it never did, which is the normal answer for an agent
+/// with no account for this source. `what` names the token in the log.
+pub async fn wait_for_token<F, Fut>(what: &str, read: F) -> Option<String>
+where
+    F: Fn() -> Fut,
+    Fut: Future<Output = Option<String>>,
+{
+    (|| async { read().await.ok_or(()) })
+        .retry(
+            ConstantBuilder::default()
+                .with_delay(Duration::from_secs(TOKEN_RETRY_SECS))
+                .with_max_times(TOKEN_READS_MAX - 1),
+        )
+        .notify(|(), _| debug!("forge_notify: no {what} yet"))
+        .await
+        .ok()
+}
 /// How close (seconds) the notification's event time must be to a
 /// subject's `created_at` for us to call it a genuine creation and emit
 /// a `new <kind>` label. Later activity that lands on the state-change
@@ -1186,6 +1208,47 @@ mod tests {
         assert!(should_deliver(&delivered, "99", "2026-06-22T16:00:00Z"));
     }
 
+    /// A token that shows up on the `n`th read.
+    fn token_on_read(n: usize, reads: &std::sync::atomic::AtomicUsize) -> Option<String> {
+        let read = reads.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1;
+        (read >= n).then(|| "secret".to_owned())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn a_token_that_is_already_there_costs_no_wait() {
+        let reads = std::sync::atomic::AtomicUsize::new(0);
+        let started = tokio::time::Instant::now();
+        let got = wait_for_token("token", || async { token_on_read(1, &reads) }).await;
+        assert_eq!(got.as_deref(), Some("secret"));
+        assert_eq!(started.elapsed(), Duration::ZERO);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn a_token_that_arrives_late_is_waited_for() {
+        let reads = std::sync::atomic::AtomicUsize::new(0);
+        let started = tokio::time::Instant::now();
+        let got = wait_for_token("token", || async { token_on_read(4, &reads) }).await;
+        assert_eq!(got.as_deref(), Some("secret"));
+        assert_eq!(started.elapsed(), Duration::from_secs(3 * TOKEN_RETRY_SECS));
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn a_token_that_never_comes_is_given_up_on_after_the_bound() {
+        let reads = std::sync::atomic::AtomicUsize::new(0);
+        let started = tokio::time::Instant::now();
+        let got = wait_for_token("token", || async { token_on_read(usize::MAX, &reads) }).await;
+        assert_eq!(got, None);
+        assert_eq!(
+            reads.load(std::sync::atomic::Ordering::SeqCst),
+            TOKEN_READS_MAX
+        );
+        assert_eq!(
+            started.elapsed(),
+            Duration::from_secs((TOKEN_READS_MAX as u64 - 1) * TOKEN_RETRY_SECS),
+            "no wait after the last read"
+        );
+    }
+
     #[test]
     fn escape_md_headings_escapes_top_level_atx() {
         // Argus reviews start with `## argus review`, which would