Watch
0
0
Fork
You've already forked hyperhive
0

swarm-controller: an opt-in plain /healthz and /readyz listener a kubelet can probe

This commit is contained in:
maia 2026-11-19 09:43:02 +01:00 • committed by mara
commit ffbd2f4dea
2 changed files with 147 additions and 50 deletions

View file

@ -110,8 +110,7 @@ use startup::{
#[tokio::main]
async fn main() -> Result<()> {
hive_log::init();
let stop = install_process_hooks();
let (stop, health) = start_process().await?;
let path = socket_path();
@ -269,15 +268,7 @@ async fn main() -> Result<()> {
reviews: Arc::default(),
alerts: alerts::from_env(),
};
serve(listener, state, stop).await
}
/// The TLS provider, and the stop signal. The signal is installed first thing,
/// so a stop that arrives while the startup reads wait on a store that is down
/// ends them instead of being ignored until `serve`.
fn install_process_hooks() -> swarm_secret_client::Shutdown {
swarm_secret_client::install_crypto_provider();
swarm_secret_client::shutdown_signal()
serve(listener, state, stop, health).await
}
/// The value a startup step produced, or exit 0 when the stop signal ended the
@ -286,6 +277,32 @@ fn exit_if_stopped<T>(started: Option<T>) -> T {
started.unwrap_or_else(|| std::process::exit(0))
}
/// The environment variable that names the plain probe listener's address.
const HEALTH_ADDR_VAR: &str = "SWARM_CONTROLLER_HEALTH_ADDR";
/// What `main` sets up before anything else: logging, the TLS provider, the
/// stop signal (installed first thing, so a stop that arrives while the startup
/// reads wait on a store that is down ends them instead of being ignored until
/// `serve`), and the opt-in `/healthz` and `/readyz` listener on the address
/// [`HEALTH_ADDR_VAR`] names, which a kubelet can reach where the socket's
/// `/readyz` is out of its way. Nothing binds when the variable is unset. The
/// controller is a server with no loop of its own to beat, so the beat comes
/// from a task on its runtime. `/readyz` stays not ready until [`serve`] sets
/// its condition.
async fn start_process() -> Result<(
swarm_secret_client::Shutdown,
swarm_secret_client::health::Health,
)> {
hive_log::init();
swarm_secret_client::install_crypto_provider();
let stop = swarm_secret_client::shutdown_signal();
let health = swarm_secret_client::health::Health::from_env(HEALTH_ADDR_VAR, stop.clone())
.await
.context("starting the health listener")?;
health.beat_on_runtime();
Ok((stop, health))
}
/// Registers the `bao.client_cert.*` gauges for the controller's own store
/// certificate (`controller.pem`) on the jobq exporter, the one provider built
/// at startup rather than on first use. Nothing when no exporter is configured.
@ -302,13 +319,15 @@ fn register_cert_gauges(provider: Option<&opentelemetry_sdk::metrics::SdkMeterPr
/// Start taking the queued review requests and answering the notification
/// router's, then serve `state` on `listener` until `stop` resolves. The
/// requests in hand finish for at most the drain; the review queue takes no
/// request after `stop`.
/// request after `stop`. `health` answers `/readyz` as the socket's does.
async fn serve(
listener: tokio::net::UnixListener,
state: AppState,
stop: impl Future<Output = ()> + Clone + Send + 'static,
health: swarm_secret_client::health::Health,
) -> Result<()> {
routes::watch_store();
let store = routes::watch_store();
health.ready_when(routes::probe_condition(state.status.clone(), store));
review_classifier::review::spawn_queue(&state, stop.clone());
request_reviewer::spawn(&state);
let serving = axum::serve(listener, build_app(state)).with_graceful_shutdown(stop.clone());

View file

@ -11,6 +11,7 @@ use std::sync::{Arc, Mutex, OnceLock};
use axum::extract::State;
use axum::http::StatusCode;
use swarm_secret_client::health::CachedCheck;
use crate::jobq::{Late, SwarmNodeKind, SwarmResourceKind};
use crate::routes::hives::{HiveEntry, ServiceLink};
@ -38,38 +39,49 @@ pub(crate) async fn health() -> &'static str {
///
/// Not in the `OpenAPI` document: it is for the supervisor, not for API clients.
pub(crate) async fn readyz(State(state): State<AppState>) -> StatusCode {
let queue_connected = state
.status
.as_ref()
.is_some_and(|status| swarm_queue_client::ensure_connected(&status.queue_client()).is_ok());
let store_fresh = STORE_READY
.get()
.is_some_and(swarm_secret_client::health::CachedCheck::is_fresh);
ready(queue_connected, store_fresh)
}
/// The controller's own store login, tried every 30 s once [`watch_store`] has
/// started it. Process-wide, as the controller has one store login to ask about.
static STORE_READY: OnceLock<swarm_secret_client::health::CachedCheck> = OnceLock::new();
/// Starts the store login `/readyz` reads. Called once, where serving starts.
pub(crate) fn watch_store() {
STORE_READY.get_or_init(|| {
swarm_secret_client::health::CachedCheck::every_30s(|| async {
crate::store::connect().await.map(drop)
})
});
}
/// 200 when the queue is connected and the store's last login is fresh.
fn ready(queue_connected: bool, store_fresh: bool) -> StatusCode {
if queue_connected && store_fresh {
if is_ready(state.status.as_deref(), STORE_READY.get()) {
StatusCode::OK
} else {
StatusCode::SERVICE_UNAVAILABLE
}
}
/// Whether the controller has what it needs to work: its queue connection and a
/// fresh store login. The one answer behind `/readyz` on the socket and on the
/// opt-in plain listener (`SWARM_CONTROLLER_HEALTH_ADDR`). Reads memory only.
pub(crate) fn is_ready(status: Option<&status::StatusReader>, store: Option<&CachedCheck>) -> bool {
let queue_connected = status
.is_some_and(|status| swarm_queue_client::ensure_connected(&status.queue_client()).is_ok());
ready(queue_connected, store.is_some_and(CachedCheck::is_fresh))
}
/// What the plain listener's `/readyz` asks: [`is_ready`], as the socket's does.
pub(crate) fn probe_condition(
status: Option<Arc<status::StatusReader>>,
store: CachedCheck,
) -> impl Fn() -> bool + Send + Sync + 'static {
move || is_ready(status.as_deref(), Some(&store))
}
/// The controller's own store login, tried every 30 s once [`watch_store`] has
/// started it. Process-wide, as the controller has one store login to ask about.
static STORE_READY: OnceLock<CachedCheck> = OnceLock::new();
/// Starts the store login `/readyz` reads, and returns it for the plain
/// listener to read too. Called once, where serving starts.
pub(crate) fn watch_store() -> CachedCheck {
STORE_READY
.get_or_init(|| {
CachedCheck::every_30s(|| async { crate::store::connect().await.map(drop) })
})
.clone()
}
/// Ready when the queue is connected and the store's last login is fresh.
fn ready(queue_connected: bool, store_fresh: bool) -> bool {
queue_connected && store_fresh
}
#[derive(Clone)]
pub(crate) struct AppState {
/// Loaded once at startup (`load_hives`); never mutated, so an
@ -249,10 +261,69 @@ pub(crate) mod tests {
/// `/readyz` needs both the queue and a fresh store login.
#[test]
fn ready_needs_the_queue_and_the_store() {
assert_eq!(ready(true, true), StatusCode::OK);
assert_eq!(ready(true, false), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(ready(false, true), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(ready(false, false), StatusCode::SERVICE_UNAVAILABLE);
assert!(ready(true, true));
assert!(!ready(true, false));
assert!(!ready(false, true));
assert!(!ready(false, false));
}
/// A store check that has succeeded, so it reads fresh.
async fn fresh_store() -> CachedCheck {
let check = CachedCheck::every_30s(|| std::future::ready(Ok::<(), String>(())));
while !check.is_fresh() {
tokio::task::yield_now().await;
}
check
}
/// A store check that never succeeds, as a store that is down.
fn failing_store() -> CachedCheck {
CachedCheck::every_30s(|| std::future::ready(Err::<(), String>("store is down".into())))
}
/// Readiness is a connected queue *and* a fresh store login, whichever of
/// the two is missing.
#[tokio::test]
async fn is_ready_needs_a_connected_queue_and_a_fresh_store() {
let (_nats, state, _sched) = state_with_roster_queued().await;
let queue = state.status.as_deref();
let (fresh, failing) = (fresh_store().await, failing_store());
assert!(is_ready(queue, Some(&fresh)));
assert!(!is_ready(queue, Some(&failing)), "store login failing");
assert!(!is_ready(queue, None), "no store check started");
assert!(!is_ready(None, Some(&fresh)), "no queue wired up");
}
/// The plain listener's `/readyz` follows the same answer as the socket's,
/// and its `/healthz` is 200 whatever readiness says.
#[tokio::test]
async fn the_plain_listener_answers_readyz_like_the_socket() {
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
let (_nats, state, _sched) = state_with_roster_queued().await;
let cases = [
(state.status.clone(), fresh_store().await, 200, "ready"),
(state.status.clone(), failing_store(), 503, "store failing"),
(None, fresh_store().await, 503, "no queue"),
];
for (status, store, expected, why) in cases {
let health =
swarm_secret_client::health::Health::new(std::time::Duration::from_mins(1));
health.ready_when(probe_condition(status, store));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(health.serve(listener, std::future::pending()));
for (path, want) in [("/readyz", expected), ("/healthz", 200)] {
let mut conn = tokio::net::TcpStream::connect(addr).await.unwrap();
let request =
format!("GET {path} HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n");
conn.write_all(request.as_bytes()).await.unwrap();
let mut reply = String::new();
conn.read_to_string(&mut reply).await.unwrap();
let code = reply.split_whitespace().nth(1).unwrap_or_default();
assert_eq!(code, want.to_string(), "{path} ({why}): {reply:?}");
}
}
}
/// A probe makes no store call: the login runs once at start and then every