readiness follows a cached store check instead of a constant or a login per probe
This commit is contained in:
parent
2eacb83b21
commit
b0239914b8
2 changed files with 369 additions and 40 deletions
|
|
@ -294,6 +294,7 @@ async fn serve(
|
|||
state: AppState,
|
||||
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||
) -> Result<()> {
|
||||
routes::watch_store();
|
||||
review_classifier::review::spawn_queue(&state, stop.clone());
|
||||
request_reviewer::spawn(&state);
|
||||
let serving = axum::serve(listener, build_app(state)).with_graceful_shutdown(stop.clone());
|
||||
|
|
|
|||
|
|
@ -7,8 +7,7 @@ pub(crate) mod forge_admin;
|
|||
pub(crate) mod hives;
|
||||
pub(crate) mod jobq_graph;
|
||||
|
||||
use std::future::Future;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
|
||||
use axum::extract::State;
|
||||
use axum::http::StatusCode;
|
||||
|
|
@ -32,11 +31,10 @@ pub(crate) async fn health() -> &'static str {
|
|||
concat!("swarm-controller ", env!("CARGO_PKG_VERSION"), "\n")
|
||||
}
|
||||
|
||||
/// How long the readiness probe waits for the secret store to answer a login.
|
||||
const STORE_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
|
||||
/// Readiness probe: 200 when the swarm queue is connected and the secret store
|
||||
/// accepts the controller's login, 503 otherwise. No body.
|
||||
/// has accepted the controller's login within the last 90 s, 503 otherwise. No
|
||||
/// body. The login is made by a background task every 30 s
|
||||
/// ([`watch_store`]); a probe makes no store call.
|
||||
///
|
||||
/// Not in the `OpenAPI` document: it is for the supervisor, not for API clients.
|
||||
pub(crate) async fn readyz(State(state): State<AppState>) -> StatusCode {
|
||||
|
|
@ -44,18 +42,28 @@ pub(crate) async fn readyz(State(state): State<AppState>) -> StatusCode {
|
|||
.status
|
||||
.as_ref()
|
||||
.is_some_and(|status| swarm_queue_client::ensure_connected(&status.queue_client()).is_ok());
|
||||
ready(queue_connected, async {
|
||||
tokio::time::timeout(STORE_PROBE_TIMEOUT, crate::store::connect())
|
||||
.await
|
||||
.is_ok_and(|login| login.is_ok())
|
||||
})
|
||||
.await
|
||||
let store_fresh = STORE_READY
|
||||
.get()
|
||||
.is_some_and(swarm_secret_client::health::CachedCheck::is_fresh);
|
||||
ready(queue_connected, store_fresh)
|
||||
}
|
||||
|
||||
/// 200 when the queue is connected and `store_reachable` resolves true. The
|
||||
/// store is not asked while the queue is down.
|
||||
async fn ready(queue_connected: bool, store_reachable: impl Future<Output = bool>) -> StatusCode {
|
||||
if queue_connected && store_reachable.await {
|
||||
/// The controller's own store login, tried every 30 s once [`watch_store`] has
|
||||
/// started it. Process-wide, as the controller has one store login to ask about.
|
||||
static STORE_READY: OnceLock<swarm_secret_client::health::CachedCheck> = OnceLock::new();
|
||||
|
||||
/// Starts the store login `/readyz` reads. Called once, where serving starts.
|
||||
pub(crate) fn watch_store() {
|
||||
STORE_READY.get_or_init(|| {
|
||||
swarm_secret_client::health::CachedCheck::every_30s(|| async {
|
||||
crate::store::connect().await.map(drop)
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
/// 200 when the queue is connected and the store's last login is fresh.
|
||||
fn ready(queue_connected: bool, store_fresh: bool) -> StatusCode {
|
||||
if queue_connected && store_fresh {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
|
|
@ -238,25 +246,50 @@ pub(crate) fn wanted_error_status(e: &anyhow::Error) -> axum::http::StatusCode {
|
|||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
|
||||
/// `/readyz` needs both the queue and the store, and does not ask the store
|
||||
/// while the queue is down.
|
||||
#[tokio::test]
|
||||
async fn ready_needs_the_queue_and_the_store() {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
/// `/readyz` needs both the queue and a fresh store login.
|
||||
#[test]
|
||||
fn ready_needs_the_queue_and_the_store() {
|
||||
assert_eq!(ready(true, true), StatusCode::OK);
|
||||
assert_eq!(ready(true, false), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(ready(false, true), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(ready(false, false), StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
|
||||
assert_eq!(ready(true, async { true }).await, StatusCode::OK);
|
||||
assert_eq!(
|
||||
ready(true, async { false }).await,
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"store unreachable"
|
||||
);
|
||||
let asked = AtomicBool::new(false);
|
||||
let store = async {
|
||||
asked.store(true, Ordering::SeqCst);
|
||||
true
|
||||
};
|
||||
assert_eq!(ready(false, store).await, StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert!(!asked.load(Ordering::SeqCst), "queue down: store not asked");
|
||||
/// A probe makes no store call: the login runs once at start and then every
|
||||
/// 30 s, however many probes arrive.
|
||||
#[tokio::test]
|
||||
async fn probing_readyz_makes_no_store_call() {
|
||||
use axum::body::Body;
|
||||
use axum::http::Request;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
let logins = Arc::new(AtomicUsize::new(0));
|
||||
let counted = Arc::clone(&logins);
|
||||
let (state, _sched) = state_with_roster();
|
||||
// The one test that starts the check; the others leave it unset.
|
||||
STORE_READY.get_or_init(|| {
|
||||
swarm_secret_client::health::CachedCheck::every_30s(move || {
|
||||
counted.fetch_add(1, Ordering::SeqCst);
|
||||
std::future::ready(Ok::<(), String>(()))
|
||||
})
|
||||
});
|
||||
// Let the first login run.
|
||||
while logins.load(Ordering::SeqCst) == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
let app = crate::startup::build_app(state);
|
||||
for _ in 0..50 {
|
||||
app.clone()
|
||||
.oneshot(
|
||||
Request::get("/readyz")
|
||||
.body(Body::empty())
|
||||
.expect("a request"),
|
||||
)
|
||||
.await
|
||||
.expect("answers");
|
||||
}
|
||||
assert_eq!(logins.load(Ordering::SeqCst), 1, "50 probes, one login");
|
||||
}
|
||||
|
||||
/// Served by the real router: with no queue wired up `/readyz` is 503,
|
||||
|
|
|
|||
Loading…
Reference in a new issue