From ffbd2f4dea2c1aaa99cc5688b41f77d4d7fa3b4d Mon Sep 17 00:00:00 2001 From: maia Date: Thu, 19 Nov 2026 09:43:02 +0100 Subject: [PATCH] swarm-controller: an opt-in plain /healthz and /readyz listener a kubelet can probe --- docs/2-handbook/swarm/services.md | 25 +++-- docs/4-implementation/swarm/readiness.md | 6 +- swarm-controller/src/main.rs | 45 +++++--- swarm-controller/src/routes/mod.rs | 131 +++++++++++++++++------ 4 files changed, 152 insertions(+), 55 deletions(-) diff --git a/docs/2-handbook/swarm/services.md b/docs/2-handbook/swarm/services.md index 63f5e6b85..455fa736e 100644 --- a/docs/2-handbook/swarm/services.md +++ b/docs/2-handbook/swarm/services.md @@ -270,37 +270,40 @@ spells out. The settings, limits and reasons: The queue daemons (`swarm-stt`, `swarm-review-classifier`, `swarm-notify-router`, `swarm-builder`, `swarm-claude-login`), -`swarm-nats-auth` and the three auth responders (`swarm-logs-auth`, -`swarm-metrics-auth`, `swarm-authelia-bridge`) can serve two probe endpoints. -They answer 200 or 503 and send no body. +`swarm-nats-auth`, `swarm-controller` and the three auth responders +(`swarm-logs-auth`, `swarm-metrics-auth`, `swarm-authelia-bridge`) can serve two +probe endpoints. They answer 200 or 503 and send no body. - `GET /healthz` is 200 while the daemon's main loop is running. The loop beats as it takes messages, so a loop that stops polling turns the answer to 503 after 30 seconds (5 minutes for `swarm-notify-router`, whose loop handles a batch of events between beats). Before the loop's first beat the answer is 200: waiting on the secret store at startup doesn't count as stuck. A - daemon that only serves requests (the three auth responders) beats from a task - on its runtime, so `/healthz` shows the runtime still runs tasks, not that - its accept loop is healthy. + daemon that only serves requests (`swarm-controller` and the three auth + responders) beats from a task on its runtime, so `/healthz` shows the runtime + still runs tasks, not that its accept loop is healthy. - `GET /readyz` is 200 while the daemon has what it needs to work: its queue connection works (`swarm-stt`, `swarm-review-classifier`, `swarm-notify-router`, `swarm-builder`, `swarm-claude-login`, `swarm-nats-auth`), or what it answers from is there: `swarm-logs-auth` and `swarm-metrics-auth` are ready while the secret store answers, - `swarm-authelia-bridge` while its users file loads. The mechanics are in + `swarm-authelia-bridge` while its users file loads, and `swarm-controller` + while both its queue connection works and the store answers. The mechanics are in [Readiness checks](../../4-implementation/swarm/readiness.md). Nothing listens by default. Set `SWARM_STT_HEALTH_ADDR`, `SWARM_REVIEW_CLASSIFIER_HEALTH_ADDR`, `SWARM_NOTIFY_ROUTER_HEALTH_ADDR`, `SWARM_BUILDER_HEALTH_ADDR`, `SWARM_CLAUDE_LOGIN_HEALTH_ADDR` or -`SWARM_AUTHELIA_BRIDGE_HEALTH_ADDR` to a `host:port` to bind the listener. For +`SWARM_AUTHELIA_BRIDGE_HEALTH_ADDR` or `SWARM_CONTROLLER_HEALTH_ADDR` to a +`host:port` to bind the listener. For `swarm-nats-auth`, `swarm-logs-auth` and `swarm-metrics-auth`, pass `--health-addr`, or set `SWARM_NATS_AUTH_HEALTH_ADDR`, `SWARM_LOGS_AUTH_HEALTH_ADDR` or `SWARM_METRICS_AUTH_HEALTH_ADDR`; the flag wins. No NixOS module sets one. The listener stops on the same SIGTERM as the daemon. -`swarm-controller` serves `GET /readyz` on its existing socket, next to -`/health`. It answers 200 while the queue connection works and the secret store -answers the controller's login, and 503 otherwise (see +`swarm-controller` also serves `GET /readyz` on its existing socket, next to +`/health`, with or without the listener. The socket and the listener give the +same answer: 200 while the queue connection works and the secret store answers +the controller's login, and 503 otherwise (see [Readiness checks](../../4-implementation/swarm/readiness.md)). diff --git a/docs/4-implementation/swarm/readiness.md b/docs/4-implementation/swarm/readiness.md index f6a9b42fb..6b4f9ebd9 100644 --- a/docs/4-implementation/swarm/readiness.md +++ b/docs/4-implementation/swarm/readiness.md @@ -29,4 +29,8 @@ ends when the last clone of the check drops. The controller starts its check once, when it starts serving (`routes::watch_store`), and `/readyz` reads the last result. Before that, and -in tests that don't start it, the store reads as not ready. +in tests that don't start it, the store reads as not ready. The socket's +`/readyz` and the plain listener's (`SWARM_CONTROLLER_HEALTH_ADDR`) both ask +`routes::is_ready`, so they can't disagree. The listener binds at the top of +`main`, before the store is reachable, so a probe gets a 503 and not a refused +connection while the controller starts. diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 55fccad7d..dce44a752 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -110,8 +110,7 @@ use startup::{ #[tokio::main] async fn main() -> Result<()> { - hive_log::init(); - let stop = install_process_hooks(); + let (stop, health) = start_process().await?; let path = socket_path(); @@ -269,15 +268,7 @@ async fn main() -> Result<()> { reviews: Arc::default(), alerts: alerts::from_env(), }; - serve(listener, state, stop).await -} - -/// The TLS provider, and the stop signal. The signal is installed first thing, -/// so a stop that arrives while the startup reads wait on a store that is down -/// ends them instead of being ignored until `serve`. -fn install_process_hooks() -> swarm_secret_client::Shutdown { - swarm_secret_client::install_crypto_provider(); - swarm_secret_client::shutdown_signal() + serve(listener, state, stop, health).await } /// The value a startup step produced, or exit 0 when the stop signal ended the @@ -286,6 +277,32 @@ fn exit_if_stopped(started: Option) -> T { started.unwrap_or_else(|| std::process::exit(0)) } +/// The environment variable that names the plain probe listener's address. +const HEALTH_ADDR_VAR: &str = "SWARM_CONTROLLER_HEALTH_ADDR"; + +/// What `main` sets up before anything else: logging, the TLS provider, the +/// stop signal (installed first thing, so a stop that arrives while the startup +/// reads wait on a store that is down ends them instead of being ignored until +/// `serve`), and the opt-in `/healthz` and `/readyz` listener on the address +/// [`HEALTH_ADDR_VAR`] names, which a kubelet can reach where the socket's +/// `/readyz` is out of its way. Nothing binds when the variable is unset. The +/// controller is a server with no loop of its own to beat, so the beat comes +/// from a task on its runtime. `/readyz` stays not ready until [`serve`] sets +/// its condition. +async fn start_process() -> Result<( + swarm_secret_client::Shutdown, + swarm_secret_client::health::Health, +)> { + hive_log::init(); + swarm_secret_client::install_crypto_provider(); + let stop = swarm_secret_client::shutdown_signal(); + let health = swarm_secret_client::health::Health::from_env(HEALTH_ADDR_VAR, stop.clone()) + .await + .context("starting the health listener")?; + health.beat_on_runtime(); + Ok((stop, health)) +} + /// Registers the `bao.client_cert.*` gauges for the controller's own store /// certificate (`controller.pem`) on the jobq exporter, the one provider built /// at startup rather than on first use. Nothing when no exporter is configured. @@ -302,13 +319,15 @@ fn register_cert_gauges(provider: Option<&opentelemetry_sdk::metrics::SdkMeterPr /// Start taking the queued review requests and answering the notification /// router's, then serve `state` on `listener` until `stop` resolves. The /// requests in hand finish for at most the drain; the review queue takes no -/// request after `stop`. +/// request after `stop`. `health` answers `/readyz` as the socket's does. async fn serve( listener: tokio::net::UnixListener, state: AppState, stop: impl Future + Clone + Send + 'static, + health: swarm_secret_client::health::Health, ) -> Result<()> { - routes::watch_store(); + let store = routes::watch_store(); + health.ready_when(routes::probe_condition(state.status.clone(), store)); review_classifier::review::spawn_queue(&state, stop.clone()); request_reviewer::spawn(&state); let serving = axum::serve(listener, build_app(state)).with_graceful_shutdown(stop.clone()); diff --git a/swarm-controller/src/routes/mod.rs b/swarm-controller/src/routes/mod.rs index 9dca6e369..78ee2f3d5 100644 --- a/swarm-controller/src/routes/mod.rs +++ b/swarm-controller/src/routes/mod.rs @@ -11,6 +11,7 @@ use std::sync::{Arc, Mutex, OnceLock}; use axum::extract::State; use axum::http::StatusCode; +use swarm_secret_client::health::CachedCheck; use crate::jobq::{Late, SwarmNodeKind, SwarmResourceKind}; use crate::routes::hives::{HiveEntry, ServiceLink}; @@ -38,38 +39,49 @@ pub(crate) async fn health() -> &'static str { /// /// Not in the `OpenAPI` document: it is for the supervisor, not for API clients. pub(crate) async fn readyz(State(state): State) -> StatusCode { - let queue_connected = state - .status - .as_ref() - .is_some_and(|status| swarm_queue_client::ensure_connected(&status.queue_client()).is_ok()); - let store_fresh = STORE_READY - .get() - .is_some_and(swarm_secret_client::health::CachedCheck::is_fresh); - ready(queue_connected, store_fresh) -} - -/// The controller's own store login, tried every 30 s once [`watch_store`] has -/// started it. Process-wide, as the controller has one store login to ask about. -static STORE_READY: OnceLock = OnceLock::new(); - -/// Starts the store login `/readyz` reads. Called once, where serving starts. -pub(crate) fn watch_store() { - STORE_READY.get_or_init(|| { - swarm_secret_client::health::CachedCheck::every_30s(|| async { - crate::store::connect().await.map(drop) - }) - }); -} - -/// 200 when the queue is connected and the store's last login is fresh. -fn ready(queue_connected: bool, store_fresh: bool) -> StatusCode { - if queue_connected && store_fresh { + if is_ready(state.status.as_deref(), STORE_READY.get()) { StatusCode::OK } else { StatusCode::SERVICE_UNAVAILABLE } } +/// Whether the controller has what it needs to work: its queue connection and a +/// fresh store login. The one answer behind `/readyz` on the socket and on the +/// opt-in plain listener (`SWARM_CONTROLLER_HEALTH_ADDR`). Reads memory only. +pub(crate) fn is_ready(status: Option<&status::StatusReader>, store: Option<&CachedCheck>) -> bool { + let queue_connected = status + .is_some_and(|status| swarm_queue_client::ensure_connected(&status.queue_client()).is_ok()); + ready(queue_connected, store.is_some_and(CachedCheck::is_fresh)) +} + +/// What the plain listener's `/readyz` asks: [`is_ready`], as the socket's does. +pub(crate) fn probe_condition( + status: Option>, + store: CachedCheck, +) -> impl Fn() -> bool + Send + Sync + 'static { + move || is_ready(status.as_deref(), Some(&store)) +} + +/// The controller's own store login, tried every 30 s once [`watch_store`] has +/// started it. Process-wide, as the controller has one store login to ask about. +static STORE_READY: OnceLock = OnceLock::new(); + +/// Starts the store login `/readyz` reads, and returns it for the plain +/// listener to read too. Called once, where serving starts. +pub(crate) fn watch_store() -> CachedCheck { + STORE_READY + .get_or_init(|| { + CachedCheck::every_30s(|| async { crate::store::connect().await.map(drop) }) + }) + .clone() +} + +/// Ready when the queue is connected and the store's last login is fresh. +fn ready(queue_connected: bool, store_fresh: bool) -> bool { + queue_connected && store_fresh +} + #[derive(Clone)] pub(crate) struct AppState { /// Loaded once at startup (`load_hives`); never mutated, so an @@ -249,10 +261,69 @@ pub(crate) mod tests { /// `/readyz` needs both the queue and a fresh store login. #[test] fn ready_needs_the_queue_and_the_store() { - assert_eq!(ready(true, true), StatusCode::OK); - assert_eq!(ready(true, false), StatusCode::SERVICE_UNAVAILABLE); - assert_eq!(ready(false, true), StatusCode::SERVICE_UNAVAILABLE); - assert_eq!(ready(false, false), StatusCode::SERVICE_UNAVAILABLE); + assert!(ready(true, true)); + assert!(!ready(true, false)); + assert!(!ready(false, true)); + assert!(!ready(false, false)); + } + + /// A store check that has succeeded, so it reads fresh. + async fn fresh_store() -> CachedCheck { + let check = CachedCheck::every_30s(|| std::future::ready(Ok::<(), String>(()))); + while !check.is_fresh() { + tokio::task::yield_now().await; + } + check + } + + /// A store check that never succeeds, as a store that is down. + fn failing_store() -> CachedCheck { + CachedCheck::every_30s(|| std::future::ready(Err::<(), String>("store is down".into()))) + } + + /// Readiness is a connected queue *and* a fresh store login, whichever of + /// the two is missing. + #[tokio::test] + async fn is_ready_needs_a_connected_queue_and_a_fresh_store() { + let (_nats, state, _sched) = state_with_roster_queued().await; + let queue = state.status.as_deref(); + let (fresh, failing) = (fresh_store().await, failing_store()); + assert!(is_ready(queue, Some(&fresh))); + assert!(!is_ready(queue, Some(&failing)), "store login failing"); + assert!(!is_ready(queue, None), "no store check started"); + assert!(!is_ready(None, Some(&fresh)), "no queue wired up"); + } + + /// The plain listener's `/readyz` follows the same answer as the socket's, + /// and its `/healthz` is 200 whatever readiness says. + #[tokio::test] + async fn the_plain_listener_answers_readyz_like_the_socket() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + let (_nats, state, _sched) = state_with_roster_queued().await; + let cases = [ + (state.status.clone(), fresh_store().await, 200, "ready"), + (state.status.clone(), failing_store(), 503, "store failing"), + (None, fresh_store().await, 503, "no queue"), + ]; + for (status, store, expected, why) in cases { + let health = + swarm_secret_client::health::Health::new(std::time::Duration::from_mins(1)); + health.ready_when(probe_condition(status, store)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(health.serve(listener, std::future::pending())); + for (path, want) in [("/readyz", expected), ("/healthz", 200)] { + let mut conn = tokio::net::TcpStream::connect(addr).await.unwrap(); + let request = + format!("GET {path} HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n"); + conn.write_all(request.as_bytes()).await.unwrap(); + let mut reply = String::new(); + conn.read_to_string(&mut reply).await.unwrap(); + let code = reply.split_whitespace().nth(1).unwrap_or_default(); + assert_eq!(code, want.to_string(), "{path} ({why}): {reply:?}"); + } + } } /// A probe makes no store call: the login runs once at start and then every