Watch
0
0
Fork
You've already forked hyperhive
0

readiness follows a cached store check instead of a constant or a login per probe

This commit is contained in:
maia 2026-11-19 06:07:18 +01:00 • committed by mara
commit b0239914b8
2 changed files with 369 additions and 40 deletions

View file

@ -294,6 +294,7 @@ async fn serve(
state: AppState,
stop: impl Future<Output = ()> + Clone + Send + 'static,
) -> Result<()> {
routes::watch_store();
review_classifier::review::spawn_queue(&state, stop.clone());
request_reviewer::spawn(&state);
let serving = axum::serve(listener, build_app(state)).with_graceful_shutdown(stop.clone());

View file

@ -7,8 +7,7 @@ pub(crate) mod forge_admin;
pub(crate) mod hives;
pub(crate) mod jobq_graph;
use std::future::Future;
use std::sync::{Arc, Mutex};
use std::sync::{Arc, Mutex, OnceLock};
use axum::extract::State;
use axum::http::StatusCode;
@ -32,11 +31,10 @@ pub(crate) async fn health() -> &'static str {
concat!("swarm-controller ", env!("CARGO_PKG_VERSION"), "\n")
}
/// How long the readiness probe waits for the secret store to answer a login.
const STORE_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
/// Readiness probe: 200 when the swarm queue is connected and the secret store
/// accepts the controller's login, 503 otherwise. No body.
/// has accepted the controller's login within the last 90 s, 503 otherwise. No
/// body. The login is made by a background task every 30 s
/// ([`watch_store`]); a probe makes no store call.
///
/// Not in the `OpenAPI` document: it is for the supervisor, not for API clients.
pub(crate) async fn readyz(State(state): State<AppState>) -> StatusCode {
@ -44,18 +42,28 @@ pub(crate) async fn readyz(State(state): State<AppState>) -> StatusCode {
.status
.as_ref()
.is_some_and(|status| swarm_queue_client::ensure_connected(&status.queue_client()).is_ok());
ready(queue_connected, async {
tokio::time::timeout(STORE_PROBE_TIMEOUT, crate::store::connect())
.await
.is_ok_and(|login| login.is_ok())
})
.await
let store_fresh = STORE_READY
.get()
.is_some_and(swarm_secret_client::health::CachedCheck::is_fresh);
ready(queue_connected, store_fresh)
}
/// 200 when the queue is connected and `store_reachable` resolves true. The
/// store is not asked while the queue is down.
async fn ready(queue_connected: bool, store_reachable: impl Future<Output = bool>) -> StatusCode {
if queue_connected && store_reachable.await {
/// The controller's own store login, tried every 30 s once [`watch_store`] has
/// started it. Process-wide, as the controller has one store login to ask about.
static STORE_READY: OnceLock<swarm_secret_client::health::CachedCheck> = OnceLock::new();
/// Starts the store login `/readyz` reads. Called once, where serving starts.
pub(crate) fn watch_store() {
STORE_READY.get_or_init(|| {
swarm_secret_client::health::CachedCheck::every_30s(|| async {
crate::store::connect().await.map(drop)
})
});
}
/// 200 when the queue is connected and the store's last login is fresh.
fn ready(queue_connected: bool, store_fresh: bool) -> StatusCode {
if queue_connected && store_fresh {
StatusCode::OK
} else {
StatusCode::SERVICE_UNAVAILABLE
@ -238,25 +246,50 @@ pub(crate) fn wanted_error_status(e: &anyhow::Error) -> axum::http::StatusCode {
pub(crate) mod tests {
use super::*;
/// `/readyz` needs both the queue and the store, and does not ask the store
/// while the queue is down.
#[tokio::test]
async fn ready_needs_the_queue_and_the_store() {
use std::sync::atomic::{AtomicBool, Ordering};
/// `/readyz` needs both the queue and a fresh store login.
#[test]
fn ready_needs_the_queue_and_the_store() {
assert_eq!(ready(true, true), StatusCode::OK);
assert_eq!(ready(true, false), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(ready(false, true), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(ready(false, false), StatusCode::SERVICE_UNAVAILABLE);
}
assert_eq!(ready(true, async { true }).await, StatusCode::OK);
assert_eq!(
ready(true, async { false }).await,
StatusCode::SERVICE_UNAVAILABLE,
"store unreachable"
);
let asked = AtomicBool::new(false);
let store = async {
asked.store(true, Ordering::SeqCst);
true
};
assert_eq!(ready(false, store).await, StatusCode::SERVICE_UNAVAILABLE);
assert!(!asked.load(Ordering::SeqCst), "queue down: store not asked");
/// A probe makes no store call: the login runs once at start and then every
/// 30 s, however many probes arrive.
#[tokio::test]
async fn probing_readyz_makes_no_store_call() {
use axum::body::Body;
use axum::http::Request;
use std::sync::atomic::{AtomicUsize, Ordering};
use tower::ServiceExt as _;
let logins = Arc::new(AtomicUsize::new(0));
let counted = Arc::clone(&logins);
let (state, _sched) = state_with_roster();
// The one test that starts the check; the others leave it unset.
STORE_READY.get_or_init(|| {
swarm_secret_client::health::CachedCheck::every_30s(move || {
counted.fetch_add(1, Ordering::SeqCst);
std::future::ready(Ok::<(), String>(()))
})
});
// Let the first login run.
while logins.load(Ordering::SeqCst) == 0 {
tokio::task::yield_now().await;
}
let app = crate::startup::build_app(state);
for _ in 0..50 {
app.clone()
.oneshot(
Request::get("/readyz")
.body(Body::empty())
.expect("a request"),
)
.await
.expect("answers");
}
assert_eq!(logins.load(Ordering::SeqCst), 1, "50 probes, one login");
}
/// Served by the real router: with no queue wired up `/readyz` is 503,