From 196805bfc02ff18a8302482d7f08d2cc71d4419a Mon Sep 17 00:00:00 2001
From: atlas <atlas@pr1ma.darkest.space>
Date: Mon, 31 Aug 2026 13:18:50 +0200
Subject: [PATCH] swarm-controller: retry webhook registration instead of
 waiting for a restart
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Registration ran once at startup and, on failure, deferred to the next
process start. Nothing schedules one, so a controller that lost the boot
race kept running with no hooks registered — and the failure is silent at
both ends, since the forge has nothing to report about a call that never
arrived.

That race is the common case rather than an edge: the controller and the
forge come up together on a rebuild. Measured on a deploy where both
consecutive starts got 502 from the gateway because forgejo was not yet
serving; the forge was healthy two minutes later.

Bounded backoff, not a poll loop — it exists to outlast a slow forge, not
to re-register periodically. Unit ordering would not fix this: the forge
is a remote host in a spread deployment, where no After= can reach it.

Closes #3828
---
 swarm-controller/src/main.rs | 35 +++++++++++++++++++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)

diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs
index 7a82c0ef..3a617fbf 100644
--- a/swarm-controller/src/main.rs
+++ b/swarm-controller/src/main.rs
@@ -1087,7 +1087,16 @@ const PUBLIC_URL_ENV: &str = "SWARM_CONTROLLER_PUBLIC_URL";
 ///
 /// Detached rather than awaited, and never fatal: the forge may be slow or
 /// briefly down at boot, and none of the daemon's other routes depend on a
-/// hook existing. Registration is idempotent, so the next restart retries.
+/// hook existing.
+///
+/// Retried on a backoff rather than left to the next process start. Losing
+/// that race is the *common* case, not an edge one — the controller and the
+/// forge come up together on a rebuild, and a deploy was measured where both
+/// consecutive starts got `502 Bad Gateway` from the gateway because forgejo
+/// was not serving yet. Nothing schedules another start, so "the next restart
+/// retries" can leave a swarm with no hooks registered for as long as the
+/// daemon keeps running — and the failure is silent at both ends, because the
+/// forge has nothing to report about a call that never arrived.
 ///
 /// Silently does nothing when any of the three preconditions is missing —
 /// each is a legitimate deployment shape (no forge here, no state directory
@@ -1104,10 +1113,32 @@ fn register_swarm_webhooks(forge: Option<Arc<forge::Client>>, secret: Option<Arc
         return;
     };
     tokio::spawn(async move {
+        // Seconds to wait before each retry — a bounded schedule, not a poll
+        // loop: it exists to outlast a forge that is slow to start, not to
+        // re-register periodically. Registration is idempotent, so a repeat
+        // costs one API call and changes nothing.
+        const RETRY_DELAYS_S: [u64; 5] = [15, 30, 60, 120, 240];
+
+        for (attempt, delay) in RETRY_DELAYS_S.iter().enumerate() {
+            match forge.ensure_swarm_webhooks(&public_url, &secret).await {
+                Ok(()) => return,
+                Err(e) => tracing::info!(
+                    error = %format!("{e:#}"),
+                    attempt = attempt + 1,
+                    retry_in_s = *delay,
+                    "registering swarm-wide forge webhooks failed; retrying"
+                ),
+            }
+            tokio::time::sleep(std::time::Duration::from_secs(*delay)).await;
+        }
+
+        // Last attempt after the final wait, so the schedule above reads as
+        // "delay before the next try" rather than one entry meaning two things.
         if let Err(e) = forge.ensure_swarm_webhooks(&public_url, &secret).await {
             tracing::warn!(
                 error = %format!("{e:#}"),
-                "registering swarm-wide forge webhooks failed; retrying on next start"
+                attempts = RETRY_DELAYS_S.len() + 1,
+                "registering swarm-wide forge webhooks failed; giving up until the next start"
             );
         }
     });