retry: replace three hand-rolled doubling loops with backon

hive-c0re/Cargo.toml

@@ -16,6 +16,7 @@ anyhow.workspace = true
 # around; the connect itself lives in `swarm-queue-client` below.
 async-nats.workspace = true
 axum.workspace = true
+backon.workspace = true
 chrono.workspace = true
 base64.workspace = true
 reqwest.workspace = true

hive-c0re/src/dashboard/mod.rs

@@ -3,6 +3,7 @@
 
 use std::net::SocketAddr;
 use std::sync::Arc;
+use std::time::Duration;
 
 use anyhow::{Context, Result};
 use axum::{
@@ -11,6 +12,7 @@ use axum::{
     response::{IntoResponse, Response},
     routing::get,
 };
+use backon::{ExponentialBuilder, Retryable};
 use utoipa::OpenApi;
 use utoipa_axum::{router::OpenApiRouter, routes};
 
@@ -166,41 +168,33 @@ pub async fn serve(port: u16, coord: Arc<Coordinator>) -> Result<()> {
 /// `SO_REUSEADDR` bind with retry. Retry mechanics, attempt-cap
 /// rationale, and log-level cadence: `docs/web-ui/shape.md::Listener bind`.
 async fn bind_with_retry(addr: SocketAddr) -> Result<tokio::net::TcpListener> {
-    let mut delay_ms = 250u64;
-    let mut attempts = 0u32;
-    loop {
-        match try_bind(addr) {
-            Ok(l) => {
-                if attempts > 0 {
-                    tracing::info!(
-                        %addr, attempts,
-                        "dashboard: bind succeeded after retry"
-                    );
-                }
-                return Ok(l);
+    let mut retries = 0u32;
+    let listener = (|| async { try_bind(addr) })
+        .retry(
+            ExponentialBuilder::default()
+                .with_min_delay(Duration::from_millis(250))
+                .with_max_delay(Duration::from_secs(2))
+                .without_max_times(),
+        )
+        .when(|e| e.kind() == std::io::ErrorKind::AddrInUse)
+        .notify(|_, delay| {
+            retries += 1;
+            let ms = delay.as_millis();
+            if retries <= 12 {
+                tracing::warn!(%addr, attempt = retries, "dashboard: AddrInUse, retrying in {ms}ms");
+            } else {
+                tracing::info!(
+                    %addr, attempt = retries,
+                    "dashboard: AddrInUse still holding, retrying in {ms}ms"
+                );
             }
-            Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => {
-                let attempt = attempts + 1;
-                if attempt <= 12 {
-                    tracing::warn!(
-                        %addr, attempt,
-                        "dashboard: AddrInUse, retrying in {delay_ms}ms"
-                    );
-                } else {
-                    tracing::info!(
-                        %addr, attempt,
-                        "dashboard: AddrInUse still holding, retrying in {delay_ms}ms"
-                    );
-                }
-                tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
-                attempts += 1;
-                delay_ms = (delay_ms * 2).min(2000);
-            }
-            Err(e) => {
-                return Err(e).with_context(|| format!("bind dashboard on {addr}"));
-            }
-        }
+        })
+        .await
+        .with_context(|| format!("bind dashboard on {addr}"))?;
+    if retries > 0 {
+        tracing::info!(%addr, attempts = retries, "dashboard: bind succeeded after retry");
     }
+    Ok(listener)
 }
 
 fn try_bind(addr: SocketAddr) -> std::io::Result<tokio::net::TcpListener> {
@@ -312,6 +306,21 @@ mod tests {
             .unwrap_or_else(|| panic!("no status line in {reply:?}"))
     }
 
+    /// The dashboard never gives up on a held port: a holder that lets go well
+    /// after the attempt count where the log level drops still gets waited out.
+    #[tokio::test(start_paused = true)]
+    async fn bind_keeps_retrying_until_the_port_is_free() {
+        let holder = try_bind("127.0.0.1:0".parse().expect("addr")).expect("holder");
+        let addr = holder.local_addr().expect("local addr");
+        let release = tokio::spawn(async move {
+            tokio::time::sleep(Duration::from_mins(5)).await;
+            drop(holder);
+        });
+        let bound = bind_with_retry(addr).await.expect("bind after release");
+        assert_eq!(bound.local_addr().expect("local addr"), addr);
+        release.await.expect("release task");
+    }
+
     #[tokio::test]
     async fn operator_inbox_route_is_gone() {
         assert_eq!(status_of("/health/live").await, 200);