swarm daemons: opt-in /healthz and /readyz listener, and /readyz on the controller
This commit is contained in:
parent
377bc1d5df
commit
c21b158884
2 changed files with 787 additions and 26 deletions
|
|
@ -7,8 +7,12 @@ pub(crate) mod forge_admin;
|
|||
pub(crate) mod hives;
|
||||
pub(crate) mod jobq_graph;
|
||||
|
||||
use std::future::Future;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use axum::extract::State;
|
||||
use axum::http::StatusCode;
|
||||
|
||||
use crate::jobq::{Late, SwarmNodeKind, SwarmResourceKind};
|
||||
use crate::routes::hives::{HiveEntry, ServiceLink};
|
||||
use crate::{
|
||||
|
|
@ -28,6 +32,36 @@ pub(crate) async fn health() -> &'static str {
|
|||
concat!("swarm-controller ", env!("CARGO_PKG_VERSION"), "\n")
|
||||
}
|
||||
|
||||
/// How long the readiness probe waits for the secret store to answer a login.
|
||||
const STORE_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
|
||||
/// Readiness probe: 200 when the swarm queue is connected and the secret store
|
||||
/// accepts the controller's login, 503 otherwise. No body.
|
||||
///
|
||||
/// Not in the `OpenAPI` document: it is for the supervisor, not for API clients.
|
||||
pub(crate) async fn readyz(State(state): State<AppState>) -> StatusCode {
|
||||
let queue_connected = state
|
||||
.status
|
||||
.as_ref()
|
||||
.is_some_and(|status| swarm_queue_client::ensure_connected(&status.queue_client()).is_ok());
|
||||
ready(queue_connected, async {
|
||||
tokio::time::timeout(STORE_PROBE_TIMEOUT, crate::store::connect())
|
||||
.await
|
||||
.is_ok_and(|login| login.is_ok())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// 200 when the queue is connected and `store_reachable` resolves true. The
|
||||
/// store is not asked while the queue is down.
|
||||
async fn ready(queue_connected: bool, store_reachable: impl Future<Output = bool>) -> StatusCode {
|
||||
if queue_connected && store_reachable.await {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct AppState {
|
||||
/// Loaded once at startup (`load_hives`); never mutated, so an
|
||||
|
|
@ -204,6 +238,51 @@ pub(crate) fn wanted_error_status(e: &anyhow::Error) -> axum::http::StatusCode {
|
|||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
|
||||
/// `/readyz` needs both the queue and the store, and does not ask the store
|
||||
/// while the queue is down.
|
||||
#[tokio::test]
|
||||
async fn ready_needs_the_queue_and_the_store() {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
assert_eq!(ready(true, async { true }).await, StatusCode::OK);
|
||||
assert_eq!(
|
||||
ready(true, async { false }).await,
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"store unreachable"
|
||||
);
|
||||
let asked = AtomicBool::new(false);
|
||||
let store = async {
|
||||
asked.store(true, Ordering::SeqCst);
|
||||
true
|
||||
};
|
||||
assert_eq!(ready(false, store).await, StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert!(!asked.load(Ordering::SeqCst), "queue down: store not asked");
|
||||
}
|
||||
|
||||
/// Served by the real router: with no queue wired up `/readyz` is 503,
|
||||
/// while `/health` on the same router stays 200.
|
||||
#[tokio::test]
|
||||
async fn the_router_answers_readyz_503_without_a_queue_and_health_200() {
|
||||
use axum::body::Body;
|
||||
use axum::http::Request;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
let (state, _sched) = state_with_roster();
|
||||
let app = crate::startup::build_app(state);
|
||||
for (path, expected) in [
|
||||
("/readyz", StatusCode::SERVICE_UNAVAILABLE),
|
||||
("/health", StatusCode::OK),
|
||||
] {
|
||||
let status = app
|
||||
.clone()
|
||||
.oneshot(Request::get(path).body(Body::empty()).expect("a request"))
|
||||
.await
|
||||
.expect("answers")
|
||||
.status();
|
||||
assert_eq!(status, expected, "{path}");
|
||||
}
|
||||
}
|
||||
|
||||
/// The 503 this route returns is what the operator UIs' shared error
|
||||
/// component renders, so assert the RENDERED response rather than the
|
||||
/// `problem_details` crate: the contract a UI depends on is the content
|
||||
|
|
|
|||
|
|
@ -771,6 +771,7 @@ pub(crate) fn build_app(state: AppState) -> axum::Router {
|
|||
// `Clone`; each request gets its own owned copy for `Json` to
|
||||
// serialize, same as `hive-c0re::dashboard::serve`.
|
||||
router
|
||||
.route("/readyz", get(routes::readyz))
|
||||
.route(
|
||||
"/api/openapi.json",
|
||||
get(move || async move { Json(api.clone()) }),
|
||||
|
|
|
|||
Loading…
Reference in a new issue