hive-sock-client, web proxy, HTTP clients: bound connect and response waits

hive-sock-client: each attempt now bounds connect (5s), write (10s) and
the wait for the response (60s by default). The response bound is per
call through the new `request_within`, which hive-agent's serve-loop
`Recv` uses with its 180s long-poll plus 30s headroom. A response
timeout is terminal rather than retried: the server holds the request,
so a retry re-sends something it may still act on and multiplies the
wait by the backoff schedule.

Outbound HTTP: the matrix login/whoami clients in swarm-controller and
hive-c0re's dashboard (5s connect, 30s request), the authelia-bridge
client (5s/30s; ensuring an identity runs an argon2 hash first) and the
ci-runner forge calls (5s/15s, config_pr_poll's forge budget) get a
connect_timeout and a request timeout. Timeout errors name the bound
that fired.

hive-agent's unix-socket extra web proxy bounds the connect (5s) and
the wait for the response head (30s, the http sibling's budget); the
body read stays unbounded.

Refs #4723

hive-c0re/src/dashboard/matrix_accounts.rs

@@ -369,6 +369,33 @@ pub(super) async fn get_github_account(Query(q): Query<GithubAccountQuery>) -> R
     axum::Json(GithubAccountStatus { present }).into_response()
 }
 
+/// Bound on reaching the homeserver.
+const HTTP_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
+/// Bound on one whole homeserver round trip, body included. A password
+/// login makes the homeserver hash the password before it answers, so this
+/// is looser than a plain API call needs.
+const HTTP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
+
+/// A client with both homeserver bounds applied.
+fn http_client() -> Result<reqwest::Client, String> {
+    reqwest::Client::builder()
+        .connect_timeout(HTTP_CONNECT_TIMEOUT)
+        .timeout(HTTP_TIMEOUT)
+        .build()
+        .map_err(|e| format!("build HTTP client: {e}"))
+}
+
+/// `what` failed with `e`; a timeout names the bound that fired.
+fn http_error(what: &str, e: &reqwest::Error) -> String {
+    if e.is_connect() && e.is_timeout() {
+        format!("{what}: connect timed out after {HTTP_CONNECT_TIMEOUT:?}")
+    } else if e.is_timeout() {
+        format!("{what}: timed out after {HTTP_TIMEOUT:?}")
+    } else {
+        format!("{what}: {e}")
+    }
+}
+
 /// POST `m.login.password` to `<homeserver>/_matrix/client/v3/login`.
 /// Returns `(access_token, user_id)`.
 async fn matrix_password_login(
@@ -383,17 +410,17 @@ async fn matrix_password_login(
         "password": password,
         "initial_device_display_name": "hyperhive",
     });
-    let resp = reqwest::Client::new()
+    let resp = http_client()?
         .post(&url)
         .json(&body)
         .send()
         .await
-        .map_err(|e| format!("POST /login: {e}"))?;
+        .map_err(|e| http_error("POST /login", &e))?;
     let status = resp.status();
     let json: serde_json::Value = resp
         .json()
         .await
-        .map_err(|e| format!("parse /login response: {e}"))?;
+        .map_err(|e| http_error("parse /login response", &e))?;
     if !status.is_success() {
         let err = json
             .get("error")
@@ -416,17 +443,17 @@ async fn matrix_password_login(
 /// to validate it and recover the `user_id`.
 async fn matrix_whoami(homeserver: &str, token: &str) -> Result<String, String> {
     let url = format!("{homeserver}/_matrix/client/v3/account/whoami");
-    let resp = reqwest::Client::new()
+    let resp = http_client()?
         .get(&url)
         .bearer_auth(token)
         .send()
         .await
-        .map_err(|e| format!("GET /whoami: {e}"))?;
+        .map_err(|e| http_error("GET /whoami", &e))?;
     let status = resp.status();
     let json: serde_json::Value = resp
         .json()
         .await
-        .map_err(|e| format!("parse /whoami response: {e}"))?;
+        .map_err(|e| http_error("parse /whoami response", &e))?;
     if !status.is_success() {
         let err = json
             .get("error")

hive-c0re/src/forge/ci_runner.rs

@@ -126,13 +126,46 @@ fn runner_address_matches(raw: &str, configured: &str) -> bool {
     }
 }
 
+/// Bound on reaching the forge.
+const HTTP_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
+/// Bound on one whole forge admin call, body included; the same budget
+/// `config_pr_poll` gives this forge.
+const HTTP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15);
+
+/// A client with both forge bounds applied.
+fn http_client() -> Result<reqwest::Client> {
+    reqwest::Client::builder()
+        .connect_timeout(HTTP_CONNECT_TIMEOUT)
+        .timeout(HTTP_TIMEOUT)
+        .build()
+        .context("build HTTP client")
+}
+
+/// Context for `what` failing with `e`; a timeout names the bound that fired.
+fn http_context(what: &str, e: &reqwest::Error) -> String {
+    if e.is_connect() && e.is_timeout() {
+        format!("{what}: connect timed out after {HTTP_CONNECT_TIMEOUT:?}")
+    } else if e.is_timeout() {
+        format!("{what}: timed out after {HTTP_TIMEOUT:?}")
+    } else {
+        what.to_owned()
+    }
+}
+
 /// `GET /admin/runners/{id}` — `true` iff the runner still exists on the forge
 /// (HTTP 200). A 404 (deleted from the admin panel) or any other status means
 /// re-registration is needed. A transport error (forge unreachable) is treated
 /// as "keep the existing creds" so a network blip never wipes a valid runner.
 async fn runner_valid(core_token: &str, id: u64) -> bool {
     let url = format!("{}/api/v1/admin/runners/{id}", forge_http_base());
-    match reqwest::Client::new()
+    let client = match http_client() {
+        Ok(c) => c,
+        Err(e) => {
+            tracing::warn!(error = ?e, "ci runner: build HTTP client failed; keeping existing creds");
+            return true;
+        }
+    };
+    match client
         .get(&url)
         .header("Authorization", format!("token {core_token}"))
         .send()
@@ -140,7 +173,8 @@ async fn runner_valid(core_token: &str, id: u64) -> bool {
     {
         Ok(resp) => resp.status().is_success(),
         Err(e) => {
-            tracing::warn!(error = ?e, "ci runner: validation request failed; keeping existing creds");
+            let what = http_context("validation request failed", &e);
+            tracing::warn!(error = ?e, "ci runner: {what}; keeping existing creds");
             true
         }
     }
@@ -155,17 +189,20 @@ async fn fetch_registration_token(core_token: &str) -> Result<String> {
         "{}/api/v1/admin/runners/registration-token",
         forge_http_base()
     );
-    let resp = reqwest::Client::new()
+    let resp = http_client()?
         .get(&url)
         .header("Authorization", format!("token {core_token}"))
         .send()
         .await
-        .context("GET admin/runners/registration-token")?;
+        .map_err(|e| {
+            let what = http_context("GET admin/runners/registration-token", &e);
+            anyhow::Error::new(e).context(what)
+        })?;
     let status = resp.status();
-    let json: Value = resp
-        .json()
-        .await
-        .context("parse registration-token response")?;
+    let json: Value = resp.json().await.map_err(|e| {
+        let what = http_context("parse registration-token response", &e);
+        anyhow::Error::new(e).context(what)
+    })?;
     if !status.is_success() {
         anyhow::bail!("registration-token HTTP {status}: {json}");
     }