bridge waits out a slow authelia; controller retries a bridge 503
This commit is contained in:
parent
bcd698eb43
commit
2de07c4575
2 changed files with 279 additions and 22 deletions
|
|
@ -27,11 +27,15 @@ use std::time::Duration;
|
|||
|
||||
use serde::Deserialize;
|
||||
|
||||
/// Generous relative to `swarm-nats-auth`'s 1.5s (that one is bounded by
|
||||
/// the NATS server's own 2s `auth_callout` timeout; an HTTP request here
|
||||
/// has no such external deadline to stay under), but still bounded — a
|
||||
/// hung introspection call must not wedge a `CreateIdentity` job forever.
|
||||
pub const INTROSPECTION_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
/// One attempt, no retry. authelia's introspection endpoint takes up to
|
||||
/// ~19s when its host is overloaded, and this covers that. It must stay
|
||||
/// below the controller's 30s bound on a whole bridge call
|
||||
/// (`swarm-controller`'s `auth::HTTP_TIMEOUT`), or the controller gives up
|
||||
/// on the bridge before the bridge has answered. A timeout is still
|
||||
/// [`Verdict::Unavailable`], never an admission.
|
||||
pub const INTROSPECTION_TIMEOUT: Duration = Duration::from_secs(20);
|
||||
|
||||
const _: () = assert!(INTROSPECTION_TIMEOUT.as_secs() < 30);
|
||||
|
||||
/// `active` decides whether to admit; `client_id` says as whom, which is
|
||||
/// what lets one verb be scoped to a single client. Same two fields, for the
|
||||
|
|
@ -91,12 +95,33 @@ pub async fn introspect(
|
|||
client_id: &str,
|
||||
client_secret: &str,
|
||||
token: &str,
|
||||
) -> Verdict {
|
||||
introspect_within(
|
||||
http,
|
||||
url,
|
||||
client_id,
|
||||
client_secret,
|
||||
token,
|
||||
INTROSPECTION_TIMEOUT,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// [`introspect`] with the deadline as a parameter, so a test can exercise
|
||||
/// the timeout without waiting out [`INTROSPECTION_TIMEOUT`].
|
||||
async fn introspect_within(
|
||||
http: &reqwest::Client,
|
||||
url: &str,
|
||||
client_id: &str,
|
||||
client_secret: &str,
|
||||
token: &str,
|
||||
timeout: Duration,
|
||||
) -> Verdict {
|
||||
let resp = match http
|
||||
.post(url)
|
||||
.basic_auth(client_id, Some(client_secret))
|
||||
.form(&[("token", token)])
|
||||
.timeout(INTROSPECTION_TIMEOUT)
|
||||
.timeout(timeout)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
|
|
@ -129,6 +154,94 @@ pub async fn introspect(
|
|||
mod tests {
|
||||
use super::*;
|
||||
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
|
||||
/// An HTTP server on loopback. Each connection is answered with
|
||||
/// `status` and `body` after `delay`; a `None` delay holds the
|
||||
/// connection open and never answers.
|
||||
async fn serve(delay: Option<Duration>, status: u16, body: &'static str) -> String {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}/introspect", listener.local_addr().unwrap());
|
||||
tokio::spawn(async move {
|
||||
while let Ok((mut conn, _)) = listener.accept().await {
|
||||
tokio::spawn(async move {
|
||||
let mut buf = vec![0u8; 8192];
|
||||
let _ = conn.read(&mut buf).await;
|
||||
let Some(delay) = delay else {
|
||||
std::future::pending::<()>().await;
|
||||
return;
|
||||
};
|
||||
tokio::time::sleep(delay).await;
|
||||
let resp = format!(
|
||||
"HTTP/1.1 {status} X\r\ncontent-type: application/json\r\n\
|
||||
content-length: {}\r\nconnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
let _ = conn.write_all(resp.as_bytes()).await;
|
||||
});
|
||||
}
|
||||
});
|
||||
url
|
||||
}
|
||||
|
||||
const ACTIVE: &str = r#"{"active":true,"client_id":"c"}"#;
|
||||
|
||||
fn active() -> Verdict {
|
||||
Verdict::Active {
|
||||
client_id: Some("c".to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn ask(url: &str, timeout: Duration) -> Verdict {
|
||||
introspect_within(
|
||||
&reqwest::Client::new(),
|
||||
url,
|
||||
"bridge",
|
||||
"secret",
|
||||
"tok",
|
||||
timeout,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Presence control for the timeout tests: the same server shape admits
|
||||
/// when it answers in time.
|
||||
#[tokio::test]
|
||||
async fn an_active_answer_within_the_deadline_admits() {
|
||||
let url = serve(Some(Duration::ZERO), 200, ACTIVE).await;
|
||||
assert_eq!(ask(&url, Duration::from_secs(5)).await, active());
|
||||
}
|
||||
|
||||
/// A slow authelia is waited for: the answer's latency does not matter
|
||||
/// as long as it beats the deadline.
|
||||
#[tokio::test]
|
||||
async fn a_slow_answer_inside_the_deadline_still_admits() {
|
||||
let url = serve(Some(Duration::from_millis(600)), 200, ACTIVE).await;
|
||||
assert_eq!(ask(&url, Duration::from_secs(5)).await, active());
|
||||
}
|
||||
|
||||
/// Fail closed: no answer by the deadline is `Unavailable`, not an
|
||||
/// admission.
|
||||
#[tokio::test]
|
||||
async fn no_answer_by_the_deadline_is_unavailable() {
|
||||
let url = serve(None, 200, "").await;
|
||||
assert_eq!(
|
||||
ask(&url, Duration::from_millis(300)).await,
|
||||
Verdict::Unavailable
|
||||
);
|
||||
}
|
||||
|
||||
/// An answer that arrives after the deadline is not used, even though
|
||||
/// it would have admitted.
|
||||
#[tokio::test]
|
||||
async fn an_active_answer_after_the_deadline_is_unavailable() {
|
||||
let url = serve(Some(Duration::from_millis(900)), 200, ACTIVE).await;
|
||||
assert_eq!(
|
||||
ask(&url, Duration::from_millis(300)).await,
|
||||
Verdict::Unavailable
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_active_true_deserializes_to_a_grant() {
|
||||
let yes: IntrospectionResponse = serde_json::from_str(r#"{"active":true}"#).unwrap();
|
||||
|
|
|
|||
|
|
@ -284,8 +284,10 @@ impl Refusal {
|
|||
/// 503 rather than 500 because the overwhelmingly common cause is
|
||||
/// transient by construction: this bridge's own secret is minted on
|
||||
/// authelia's first boot and delivered by a separate unit, so the
|
||||
/// window where it is absent closes on its own. A client retrying is
|
||||
/// doing the right thing.
|
||||
/// window where it is absent closes on its own; an authelia that
|
||||
/// answers slower than [`introspect::INTROSPECTION_TIMEOUT`] is
|
||||
/// transient too. A client retrying is doing the right thing, and
|
||||
/// `swarm-controller` does.
|
||||
fn status(self) -> StatusCode {
|
||||
match self {
|
||||
Refusal::Caller => StatusCode::UNAUTHORIZED,
|
||||
|
|
|
|||
Loading…
Reference in a new issue