bridge waits out a slow authelia; controller retries a bridge 503

swarm-controller/src/auth.rs

@@ -127,14 +127,56 @@ impl AuthBridge {
             .await
             .context("minting a bearer token for swarm-authelia-bridge")?;
 
-        let response = self
-            .http
-            .post(format!("{}/requests", self.base_url))
+        post_with_retry(&self.http, &self.base_url, &token, req, RETRY_BACKOFF).await
+    }
+}
+
+/// Waits between attempts at one bridge call: three tries in all.
+const RETRY_BACKOFF: &[std::time::Duration] = &[
+    std::time::Duration::from_secs(1),
+    std::time::Duration::from_secs(4),
+];
+
+/// POST `req` to the bridge, retrying a `503` or a failure to connect after
+/// each wait in `backoff`.
+///
+/// `503` is the bridge saying it could not get an answer out of authelia, so
+/// the token was never evaluated; the same call usually succeeds once
+/// authelia recovers. `401` and `403` are verdicts on the token and are
+/// final, as is any other status. A timeout after the connection was made
+/// is not retried: the request may have been acted on, and a whole attempt
+/// already took [`HTTP_TIMEOUT`]. Every bridge request is safe to repeat
+/// regardless (idempotent ensure, or a read).
+async fn post_with_retry<T: serde::de::DeserializeOwned>(
+    http: &reqwest::Client,
+    base_url: &str,
+    token: &str,
+    req: &BridgeRequest,
+    backoff: &[std::time::Duration],
+) -> Result<T> {
+    let mut waits = backoff.iter();
+    loop {
+        let sent = http
+            .post(format!("{base_url}/requests"))
             .bearer_auth(token)
             .json(req)
             .send()
-            .await
-            .map_err(|e| {
+            .await;
+        let (failure, retryable) = match sent {
+            Ok(response) => {
+                let status = response.status();
+                let body = response.text().await.unwrap_or_default();
+                if status.is_success() {
+                    return serde_json::from_str(&body)
+                        .context("parsing swarm-authelia-bridge's response");
+                }
+                (
+                    anyhow::anyhow!("swarm-authelia-bridge refused the request ({status}): {body}"),
+                    status == reqwest::StatusCode::SERVICE_UNAVAILABLE,
+                )
+            }
+            Err(e) => {
+                let retryable = e.is_connect();
                 let what = if e.is_connect() && e.is_timeout() {
                     format!(
                         "calling swarm-authelia-bridge: connect timed out after {HTTP_CONNECT_TIMEOUT:?}"
@@ -144,16 +186,16 @@ impl AuthBridge {
                 } else {
                     "calling swarm-authelia-bridge".to_owned()
                 };
-                anyhow::Error::new(e).context(what)
-            })?;
-
-        let status = response.status();
-        let body = response.text().await.unwrap_or_default();
-        if !status.is_success() {
-            anyhow::bail!("swarm-authelia-bridge refused the request ({status}): {body}");
+                (anyhow::Error::new(e).context(what), retryable)
+            }
+        };
+        match waits.next() {
+            Some(wait) if retryable => {
+                tracing::warn!(error = %failure, ?wait, "swarm-authelia-bridge call failed; retrying");
+                tokio::time::sleep(*wait).await;
+            }
+            _ => return Err(failure),
         }
-
-        serde_json::from_str(&body).context("parsing swarm-authelia-bridge's response")
     }
 }
 
@@ -161,6 +203,106 @@ impl AuthBridge {
 mod tests {
     use super::*;
 
+    use std::sync::Arc;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::time::Duration;
+
+    const NO_WAIT: &[Duration] = &[Duration::ZERO, Duration::ZERO];
+
+    /// A bridge on loopback answering each request with the next entry of
+    /// `statuses` (the last one repeats), and counting the requests.
+    async fn bridge(statuses: &'static [u16]) -> (String, Arc<AtomicUsize>) {
+        use axum::http::StatusCode;
+        let hits = Arc::new(AtomicUsize::new(0));
+        let counter = hits.clone();
+        let app = axum::Router::new().route(
+            "/requests",
+            axum::routing::post(move || {
+                let n = counter.fetch_add(1, Ordering::SeqCst);
+                let code = statuses[n.min(statuses.len() - 1)];
+                async move {
+                    let body = if code == 200 {
+                        r#"{"status":"operators","operators":["mara"]}"#
+                    } else {
+                        r#"{"status":"error","message":"refused"}"#
+                    };
+                    (StatusCode::from_u16(code).unwrap(), body)
+                }
+            }),
+        );
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let url = format!("http://{}", listener.local_addr().unwrap());
+        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+        (url, hits)
+    }
+
+    async fn call(url: &str, backoff: &[Duration]) -> Result<BridgeResponse> {
+        post_with_retry(
+            &reqwest::Client::new(),
+            url,
+            "tok",
+            &BridgeRequest::ListOperators,
+            backoff,
+        )
+        .await
+    }
+
+    #[tokio::test]
+    async fn a_503_is_retried_until_the_bridge_answers() {
+        let (url, hits) = bridge(&[503, 503, 200]).await;
+        let got = call(&url, NO_WAIT).await.expect("third try succeeds");
+        assert!(matches!(got, BridgeResponse::Operators { .. }), "{got:?}");
+        assert_eq!(hits.load(Ordering::SeqCst), 3);
+    }
+
+    #[tokio::test]
+    async fn a_persistent_503_fails_after_three_tries() {
+        let (url, hits) = bridge(&[503]).await;
+        let err = call(&url, NO_WAIT).await.expect_err("never recovers");
+        assert!(format!("{err:#}").contains("503"), "{err:#}");
+        assert_eq!(hits.load(Ordering::SeqCst), 3);
+    }
+
+    /// 401 and 403 are verdicts on the token: one try each.
+    #[tokio::test]
+    async fn a_401_or_403_is_final() {
+        for code in [401, 403] {
+            let (url, hits) = bridge(match code {
+                401 => &[401],
+                _ => &[403],
+            })
+            .await;
+            let err = call(&url, NO_WAIT).await.expect_err("refused");
+            assert!(format!("{err:#}").contains(&code.to_string()), "{err:#}");
+            assert_eq!(hits.load(Ordering::SeqCst), 1, "status {code}");
+        }
+    }
+
+    #[tokio::test]
+    async fn a_connection_refusal_is_retried_then_fails() {
+        // Bind then drop, so the port refuses connections.
+        let port = {
+            let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+            l.local_addr().unwrap().port()
+        };
+        let started = tokio::time::Instant::now();
+        let err = call(
+            &format!("http://127.0.0.1:{port}"),
+            &[Duration::from_millis(150); 2],
+        )
+        .await
+        .expect_err("nothing listens");
+        assert!(
+            format!("{err:#}").contains("calling swarm-authelia-bridge"),
+            "{err:#}"
+        );
+        assert!(
+            started.elapsed() >= Duration::from_millis(300),
+            "both waits were taken: {:?}",
+            started.elapsed()
+        );
+    }
+
     /// The common case: no bridge wired up for this deployment.
     #[test]
     fn absent_url_is_not_an_error() {