retry: replace three hand-rolled doubling loops with backon

hive-agent/src/web_ui/mod.rs

@@ -21,6 +21,7 @@ use std::net::SocketAddr;
 use std::path::{Path, PathBuf};
 use std::sync::atomic::AtomicBool;
 use std::sync::{Arc, Mutex};
+use std::time::Duration;
 
 use anyhow::{Context, Result};
 use axum::{
@@ -29,6 +30,7 @@ use axum::{
     response::{IntoResponse, Response},
     routing::{get, post},
 };
+use backon::{ExponentialBuilder, Retryable};
 use tower_http::services::ServeDir;
 
 use crate::events::Bus;
@@ -221,38 +223,34 @@ const MAX_BIND_ATTEMPTS: u32 = 12;
 /// Retry rationale + dashboard-banner-on-real-collision:
 /// see `docs/web-ui/shape.md::Listener bind`.
 async fn bind_with_retry(addr: SocketAddr, label: &str) -> Result<tokio::net::TcpListener> {
-    let mut delay_ms = 250u64;
-    let mut attempts = 0u32;
-    loop {
-        match try_bind(addr) {
-            Ok(l) => {
-                if attempts > 0 {
-                    tracing::info!(
-                        %addr, attempts,
-                        "{label}: bind succeeded after retry"
-                    );
-                }
-                return Ok(l);
-            }
-            Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => {
-                let attempt = attempts + 1;
-                if attempt >= MAX_BIND_ATTEMPTS {
-                    return Err(e).with_context(|| {
-                        format!("bind {label} on {addr}: still AddrInUse after {attempt} attempts")
-                    });
-                }
-                tracing::warn!(
-                    %addr, attempt,
-                    "{label}: AddrInUse, retrying in {delay_ms}ms"
-                );
-                tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
-                attempts += 1;
-                delay_ms = (delay_ms * 2).min(2000);
-            }
-            Err(e) => {
-                return Err(e).with_context(|| format!("bind {label} on {addr}"));
+    let mut retries = 0u32;
+    let bound = (|| async { try_bind(addr) })
+        .retry(
+            ExponentialBuilder::default()
+                .with_min_delay(Duration::from_millis(250))
+                .with_max_delay(Duration::from_secs(2))
+                .with_max_times((MAX_BIND_ATTEMPTS - 1) as usize),
+        )
+        .when(|e| e.kind() == std::io::ErrorKind::AddrInUse)
+        .notify(|_, delay| {
+            retries += 1;
+            tracing::warn!(
+                %addr, attempt = retries,
+                "{label}: AddrInUse, retrying in {}ms", delay.as_millis()
+            );
+        })
+        .await;
+    match bound {
+        Ok(l) => {
+            if retries > 0 {
+                tracing::info!(%addr, attempts = retries, "{label}: bind succeeded after retry");
             }
+            Ok(l)
         }
+        Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => Err(e).with_context(|| {
+            format!("bind {label} on {addr}: still AddrInUse after {MAX_BIND_ATTEMPTS} attempts")
+        }),
+        Err(e) => Err(e).with_context(|| format!("bind {label} on {addr}")),
     }
 }
 
@@ -484,10 +482,49 @@ mod tests {
     use axum::http::{Request, StatusCode, header};
     use tower::ServiceExt;
 
-    use super::{AppState, SigintOutcome, routes, sigint_turn};
+    use super::{AppState, SigintOutcome, bind_with_retry, routes, sigint_turn, try_bind};
     use crate::events::Bus;
     use crate::login::LoginState;
 
+    /// A port another listener holds for a moment is waited out.
+    #[tokio::test(start_paused = true)]
+    async fn bind_waits_out_a_port_that_is_briefly_held() {
+        let holder = try_bind("127.0.0.1:0".parse().expect("addr")).expect("holder");
+        let addr = holder.local_addr().expect("local addr");
+        let release = tokio::spawn(async move {
+            tokio::time::sleep(Duration::from_secs(1)).await;
+            drop(holder);
+        });
+        let bound = bind_with_retry(addr, "test")
+            .await
+            .expect("bind after release");
+        assert_eq!(bound.local_addr().expect("local addr"), addr);
+        release.await.expect("release task");
+    }
+
+    /// A port that stays held ends in an `AddrInUse` error after 12 attempts:
+    /// 11 waits of 250ms doubling to the 2s cap.
+    #[tokio::test(start_paused = true)]
+    async fn bind_gives_up_when_the_port_stays_held() {
+        let holder = try_bind("127.0.0.1:0".parse().expect("addr")).expect("holder");
+        let addr = holder.local_addr().expect("local addr");
+        let start = tokio::time::Instant::now();
+        let err = bind_with_retry(addr, "test")
+            .await
+            .expect_err("port is held");
+        assert_eq!(
+            start.elapsed(),
+            Duration::from_millis(250 + 500 + 1000 + 8 * 2000)
+        );
+        assert!(err.to_string().contains("after 12 attempts"), "{err:#}");
+        assert_eq!(
+            err.downcast_ref::<std::io::Error>()
+                .map(std::io::Error::kind),
+            Some(std::io::ErrorKind::AddrInUse)
+        );
+        drop(holder);
+    }
+
     fn form(path: &str, body: &'static str) -> Request<Body> {
         Request::post(path)
             .header(header::CONTENT_TYPE, "application/x-www-form-urlencoded")