retry the remaining plain schedules with backon

hive-forge-notify/src/bin/hive-forge-notify/main.rs

@@ -12,9 +12,7 @@ use std::collections::HashMap;
 use std::path::Path;
 use std::time::Duration;
 
-use hive_forge_notify::notify::{
-    POLL_INTERVAL_SECS, TOKEN_RETRY_MAX, TOKEN_RETRY_SECS, poll_once, resolve_own_login,
-};
+use hive_forge_notify::notify::{POLL_INTERVAL_SECS, poll_once, resolve_own_login, wait_for_token};
 use hive_forge_notify::{HTTP_TIMEOUT_SECS, TODO_SOCKET_RETRY, agent_socket, init_tracing};
 use source::ForgejoSource;
 use tracing::{debug, info, warn};
@@ -41,22 +39,13 @@ async fn forgejo_loop(state_dir: String, socket: std::path::PathBuf) {
     // Retry reading the token to handle races where the agent's fetch of
     // it from the swarm secret store lands after this unit starts, or where
     // a parent-container chown briefly makes the file unreadable.
-    let mut token = {
-        let mut attempts = 0u32;
-        loop {
-            if let Some(t) = hive_forge_notify::read_first_token(&token_paths) {
-                break t;
-            }
-            debug!(?token_paths, "forge_notify: no forge token yet");
-            attempts += 1;
-            if attempts >= TOKEN_RETRY_MAX {
-                debug!(
-                    "forge_notify: token not available after {TOKEN_RETRY_MAX} retries — disabled"
-                );
-                return;
-            }
-            tokio::time::sleep(Duration::from_secs(TOKEN_RETRY_SECS)).await;
-        }
+    let Some(mut token) = wait_for_token("forge token", || async {
+        hive_forge_notify::read_first_token(&token_paths)
+    })
+    .await
+    else {
+        debug!(?token_paths, "forge_notify: token not available — disabled");
+        return;
     };
 
     let Some(mut source) = ForgejoSource::new(&forge_url, &token) else {

hive-forge-notify/src/bin/hive-github-notify/main.rs

@@ -24,9 +24,7 @@ mod source;
 use std::collections::HashMap;
 use std::time::Duration;
 
-use hive_forge_notify::notify::{
-    POLL_INTERVAL_SECS, TOKEN_RETRY_MAX, TOKEN_RETRY_SECS, poll_once, resolve_own_login,
-};
+use hive_forge_notify::notify::{POLL_INTERVAL_SECS, poll_once, resolve_own_login, wait_for_token};
 use hive_forge_notify::{HTTP_TIMEOUT_SECS, agent_socket, init_tracing};
 use source::GithubSource;
 use tracing::{debug, info, warn};
@@ -50,20 +48,14 @@ async fn main() {
 /// common case for an agent that has the unit but no account.
 async fn github_loop(state_dir: String, socket: std::path::PathBuf) {
     let token_path = format!("{state_dir}/github-token");
-    let mut attempts = 0u32;
-    let token = loop {
-        match tokio::fs::read_to_string(&token_path).await {
-            Ok(t) if !t.trim().is_empty() => break t.trim().to_owned(),
-            _ => debug!("forge_notify: no github token at {token_path} yet"),
-        }
-        attempts += 1;
-        if attempts >= TOKEN_RETRY_MAX {
-            debug!(
-                "forge_notify: no github token after {TOKEN_RETRY_MAX} retries — github disabled"
-            );
-            return;
-        }
-        tokio::time::sleep(Duration::from_secs(TOKEN_RETRY_SECS)).await;
+    let Some(token) = wait_for_token("github token", || async {
+        let text = tokio::fs::read_to_string(&token_path).await.ok()?;
+        Some(text.trim().to_owned()).filter(|t| !t.is_empty())
+    })
+    .await
+    else {
+        debug!("forge_notify: no github token at {token_path} — github disabled");
+        return;
     };
 
     let client = match reqwest::Client::builder()

hive-forge-notify/src/notify.rs

@@ -27,9 +27,11 @@
 
 use std::collections::{HashMap, HashSet};
 use std::fmt::Write as _;
+use std::future::Future;
 use std::path::Path;
 use std::time::Duration;
 
+use backon::{ConstantBuilder, Retryable as _};
 use forgejo_api::structs::NotifySubjectType;
 use serde::{Deserialize, Deserializer};
 use time::OffsetDateTime;
@@ -54,10 +56,30 @@ const BODY_TRUNCATE: usize = 500;
 /// How long to wait between token-read retries when the token file is
 /// missing or unreadable at startup (e.g. hive-priv hasn't provisioned
 /// it yet, or a chown race left it temporarily root-owned).
-pub const TOKEN_RETRY_SECS: u64 = 30;
-/// Give up waiting for the token after this many retries (~10 minutes).
+const TOKEN_RETRY_SECS: u64 = 30;
+/// Give up waiting for the token after this many reads (~10 minutes).
 /// Avoids an infinite wait on agents that genuinely have no forge account.
-pub const TOKEN_RETRY_MAX: u32 = 20;
+const TOKEN_READS_MAX: usize = 20;
+
+/// Read a token that another process writes out of band, waiting for it to
+/// appear: `read` is called up to 20 times, 30 seconds apart (about ten
+/// minutes). `None` when it never did, which is the normal answer for an agent
+/// with no account for this source. `what` names the token in the log.
+pub async fn wait_for_token<F, Fut>(what: &str, read: F) -> Option<String>
+where
+    F: Fn() -> Fut,
+    Fut: Future<Output = Option<String>>,
+{
+    (|| async { read().await.ok_or(()) })
+        .retry(
+            ConstantBuilder::default()
+                .with_delay(Duration::from_secs(TOKEN_RETRY_SECS))
+                .with_max_times(TOKEN_READS_MAX - 1),
+        )
+        .notify(|(), _| debug!("forge_notify: no {what} yet"))
+        .await
+        .ok()
+}
 /// How close (seconds) the notification's event time must be to a
 /// subject's `created_at` for us to call it a genuine creation and emit
 /// a `new <kind>` label. Later activity that lands on the state-change
@@ -1186,6 +1208,47 @@ mod tests {
         assert!(should_deliver(&delivered, "99", "2026-06-22T16:00:00Z"));
     }
 
+    /// A token that shows up on the `n`th read.
+    fn token_on_read(n: usize, reads: &std::sync::atomic::AtomicUsize) -> Option<String> {
+        let read = reads.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1;
+        (read >= n).then(|| "secret".to_owned())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn a_token_that_is_already_there_costs_no_wait() {
+        let reads = std::sync::atomic::AtomicUsize::new(0);
+        let started = tokio::time::Instant::now();
+        let got = wait_for_token("token", || async { token_on_read(1, &reads) }).await;
+        assert_eq!(got.as_deref(), Some("secret"));
+        assert_eq!(started.elapsed(), Duration::ZERO);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn a_token_that_arrives_late_is_waited_for() {
+        let reads = std::sync::atomic::AtomicUsize::new(0);
+        let started = tokio::time::Instant::now();
+        let got = wait_for_token("token", || async { token_on_read(4, &reads) }).await;
+        assert_eq!(got.as_deref(), Some("secret"));
+        assert_eq!(started.elapsed(), Duration::from_secs(3 * TOKEN_RETRY_SECS));
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn a_token_that_never_comes_is_given_up_on_after_the_bound() {
+        let reads = std::sync::atomic::AtomicUsize::new(0);
+        let started = tokio::time::Instant::now();
+        let got = wait_for_token("token", || async { token_on_read(usize::MAX, &reads) }).await;
+        assert_eq!(got, None);
+        assert_eq!(
+            reads.load(std::sync::atomic::Ordering::SeqCst),
+            TOKEN_READS_MAX
+        );
+        assert_eq!(
+            started.elapsed(),
+            Duration::from_secs((TOKEN_READS_MAX as u64 - 1) * TOKEN_RETRY_SECS),
+            "no wait after the last read"
+        );
+    }
+
     #[test]
     fn escape_md_headings_escapes_top_level_atx() {
         // Argus reviews start with `## argus review`, which would