Watch
0
0
Fork
You've already forked hyperhive
0

swarm daemons: opt-in /healthz and /readyz listener, and /readyz on the controller

This commit is contained in:
atlas 2026-11-18 13:52:36 +01:00 • committed by mara
commit c21b158884
2 changed files with 787 additions and 26 deletions

View file

@ -7,8 +7,12 @@ pub(crate) mod forge_admin;
pub(crate) mod hives;
pub(crate) mod jobq_graph;
use std::future::Future;
use std::sync::{Arc, Mutex};
use axum::extract::State;
use axum::http::StatusCode;
use crate::jobq::{Late, SwarmNodeKind, SwarmResourceKind};
use crate::routes::hives::{HiveEntry, ServiceLink};
use crate::{
@ -28,6 +32,36 @@ pub(crate) async fn health() -> &'static str {
concat!("swarm-controller ", env!("CARGO_PKG_VERSION"), "\n")
}
/// How long the readiness probe waits for the secret store to answer a login.
const STORE_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
/// Readiness probe: 200 when the swarm queue is connected and the secret store
/// accepts the controller's login, 503 otherwise. No body.
///
/// Not in the `OpenAPI` document: it is for the supervisor, not for API clients.
pub(crate) async fn readyz(State(state): State<AppState>) -> StatusCode {
let queue_connected = state
.status
.as_ref()
.is_some_and(|status| swarm_queue_client::ensure_connected(&status.queue_client()).is_ok());
ready(queue_connected, async {
tokio::time::timeout(STORE_PROBE_TIMEOUT, crate::store::connect())
.await
.is_ok_and(|login| login.is_ok())
})
.await
}
/// 200 when the queue is connected and `store_reachable` resolves true. The
/// store is not asked while the queue is down.
async fn ready(queue_connected: bool, store_reachable: impl Future<Output = bool>) -> StatusCode {
if queue_connected && store_reachable.await {
StatusCode::OK
} else {
StatusCode::SERVICE_UNAVAILABLE
}
}
#[derive(Clone)]
pub(crate) struct AppState {
/// Loaded once at startup (`load_hives`); never mutated, so an
@ -204,6 +238,51 @@ pub(crate) fn wanted_error_status(e: &anyhow::Error) -> axum::http::StatusCode {
pub(crate) mod tests {
use super::*;
/// `/readyz` needs both the queue and the store, and does not ask the store
/// while the queue is down.
#[tokio::test]
async fn ready_needs_the_queue_and_the_store() {
use std::sync::atomic::{AtomicBool, Ordering};
assert_eq!(ready(true, async { true }).await, StatusCode::OK);
assert_eq!(
ready(true, async { false }).await,
StatusCode::SERVICE_UNAVAILABLE,
"store unreachable"
);
let asked = AtomicBool::new(false);
let store = async {
asked.store(true, Ordering::SeqCst);
true
};
assert_eq!(ready(false, store).await, StatusCode::SERVICE_UNAVAILABLE);
assert!(!asked.load(Ordering::SeqCst), "queue down: store not asked");
}
/// Served by the real router: with no queue wired up `/readyz` is 503,
/// while `/health` on the same router stays 200.
#[tokio::test]
async fn the_router_answers_readyz_503_without_a_queue_and_health_200() {
use axum::body::Body;
use axum::http::Request;
use tower::ServiceExt as _;
let (state, _sched) = state_with_roster();
let app = crate::startup::build_app(state);
for (path, expected) in [
("/readyz", StatusCode::SERVICE_UNAVAILABLE),
("/health", StatusCode::OK),
] {
let status = app
.clone()
.oneshot(Request::get(path).body(Body::empty()).expect("a request"))
.await
.expect("answers")
.status();
assert_eq!(status, expected, "{path}");
}
}
/// The 503 this route returns is what the operator UIs' shared error
/// component renders, so assert the RENDERED response rather than the
/// `problem_details` crate: the contract a UI depends on is the content

View file

@ -771,6 +771,7 @@ pub(crate) fn build_app(state: AppState) -> axum::Router {
// `Clone`; each request gets its own owned copy for `Json` to
// serialize, same as `hive-c0re::dashboard::serve`.
router
.route("/readyz", get(routes::readyz))
.route(
"/api/openapi.json",
get(move || async move { Json(api.clone()) }),