hive-sock-client, web proxy, HTTP clients: bound connect and response waits
hive-sock-client: each attempt now bounds connect (5s), write (10s) and the wait for the response (60s by default). The response bound is per call through the new `request_within`, which hive-agent's serve-loop `Recv` uses with its 180s long-poll plus 30s headroom. A response timeout is terminal rather than retried: the server holds the request, so a retry re-sends something it may still act on and multiplies the wait by the backoff schedule. Outbound HTTP: the matrix login/whoami clients in swarm-controller and hive-c0re's dashboard (5s connect, 30s request), the authelia-bridge client (5s/30s; ensuring an identity runs an argon2 hash first) and the ci-runner forge calls (5s/15s, config_pr_poll's forge budget) get a connect_timeout and a request timeout. Timeout errors name the bound that fired. hive-agent's unix-socket extra web proxy bounds the connect (5s) and the wait for the response head (30s, the http sibling's budget); the body read stays unbounded. Refs #4723
This commit is contained in:
parent
6fac00dcc5
commit
7b1fe5f9d3
7 changed files with 412 additions and 60 deletions
|
|
@ -48,6 +48,15 @@ const DEFAULT_SOCKET: &str = "/run/hive/mcp.sock";
|
|||
/// so a hive-c0re restart is worth waiting out rather than surfacing.
|
||||
const CONTROL_SOCKET_RETRY: Retry = Retry::RideOutRestart;
|
||||
|
||||
/// How long the serve loop's `Recv` asks the broker to park when the inbox
|
||||
/// is empty.
|
||||
const RECV_WAIT: Duration = Duration::from_mins(3);
|
||||
|
||||
/// Response deadline for that `Recv`: the park itself plus headroom for the
|
||||
/// broker to answer once it ends, so an idle poll never reads as a stuck
|
||||
/// hive-c0re.
|
||||
const RECV_RESPONSE_TIMEOUT: Duration = RECV_WAIT.saturating_add(Duration::from_secs(30));
|
||||
|
||||
/// Default web UI port — used when `HIVE_PORT` env is unset.
|
||||
const DEFAULT_WEB_PORT: u16 = 8042;
|
||||
|
||||
|
|
@ -389,13 +398,14 @@ impl Surface for AgentSurface {
|
|||
}
|
||||
|
||||
async fn recv_next(socket: &Path) -> RecvOutcome {
|
||||
let recv: Result<Response> = hive_sock_client::request(
|
||||
let recv: Result<Response> = hive_sock_client::request_within(
|
||||
socket,
|
||||
&Request::Recv {
|
||||
wait_seconds: Some(180),
|
||||
wait_seconds: Some(RECV_WAIT.as_secs()),
|
||||
max: None,
|
||||
},
|
||||
CONTROL_SOCKET_RETRY,
|
||||
RECV_RESPONSE_TIMEOUT,
|
||||
)
|
||||
.await;
|
||||
match recv {
|
||||
|
|
|
|||
|
|
@ -40,6 +40,16 @@ const HOP_BY_HOP: [&str; 6] = [
|
|||
"upgrade",
|
||||
];
|
||||
|
||||
/// Bound on dialing a `unix:` upstream. The socket is local, so a connect
|
||||
/// still pending after this long is not going to complete.
|
||||
const UNIX_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
|
||||
/// Bound on a `unix:` upstream answering with its response head — the same
|
||||
/// budget the `http(s)://` path gives a whole request. Only the head is
|
||||
/// bounded: once the upstream has answered, a long body is the upstream
|
||||
/// working, not stuck.
|
||||
const UNIX_RESPONSE_HEAD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
|
||||
|
||||
/// Nest every proxy declared in `HIVE_EXTRA_WEB_PROXIES` (a JSON object
|
||||
/// `{"<name>": "<upstream_url>"}`) under `/extra/<name>/` on `app`. Absent /
|
||||
/// blank / invalid JSON is a no-op (logged). Called from [`super::serve`]
|
||||
|
|
@ -213,7 +223,12 @@ async fn forward_over_unix_socket(
|
|||
mut headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> anyhow::Result<Response> {
|
||||
let stream = tokio::net::UnixStream::connect(sock_path).await?;
|
||||
let stream = tokio::time::timeout(
|
||||
UNIX_CONNECT_TIMEOUT,
|
||||
tokio::net::UnixStream::connect(sock_path),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| anyhow::anyhow!("connect timed out after {UNIX_CONNECT_TIMEOUT:?}"))??;
|
||||
let io = TokioIo::new(stream);
|
||||
|
||||
let (mut sender, conn) = hyper::client::conn::http1::handshake(io).await?;
|
||||
|
|
@ -241,7 +256,13 @@ async fn forward_over_unix_socket(
|
|||
}
|
||||
let req = req_builder.body(Full::new(body))?;
|
||||
|
||||
let upstream_resp = sender.send_request(req).await?;
|
||||
let upstream_resp = tokio::time::timeout(UNIX_RESPONSE_HEAD_TIMEOUT, sender.send_request(req))
|
||||
.await
|
||||
.map_err(|_| {
|
||||
anyhow::anyhow!(
|
||||
"waiting for the response head timed out after {UNIX_RESPONSE_HEAD_TIMEOUT:?}"
|
||||
)
|
||||
})??;
|
||||
let status = StatusCode::from_u16(upstream_resp.status().as_u16())?;
|
||||
let mut resp_headers = HeaderMap::new();
|
||||
for (name, value) in upstream_resp.headers() {
|
||||
|
|
|
|||
Loading…
Reference in a new issue