diff --git a/docs/2-handbook/swarm/services.md b/docs/2-handbook/swarm/services.md index d93c0a36e..63f5e6b85 100644 --- a/docs/2-handbook/swarm/services.md +++ b/docs/2-handbook/swarm/services.md @@ -285,7 +285,10 @@ They answer 200 or 503 and send no body. - `GET /readyz` is 200 while the daemon has what it needs to work: its queue connection works (`swarm-stt`, `swarm-review-classifier`, `swarm-notify-router`, `swarm-builder`, `swarm-claude-login`, - `swarm-nats-auth`), or it has bound its socket (the three auth responders). + `swarm-nats-auth`), or what it answers from is there: `swarm-logs-auth` and + `swarm-metrics-auth` are ready while the secret store answers, + `swarm-authelia-bridge` while its users file loads. The mechanics are in + [Readiness checks](../../4-implementation/swarm/readiness.md). Nothing listens by default. Set `SWARM_STT_HEALTH_ADDR`, `SWARM_REVIEW_CLASSIFIER_HEALTH_ADDR`, `SWARM_NOTIFY_ROUTER_HEALTH_ADDR`, @@ -299,4 +302,5 @@ daemon. `swarm-controller` serves `GET /readyz` on its existing socket, next to `/health`. It answers 200 while the queue connection works and the secret store -accepts the controller's login, and 503 otherwise. +answers the controller's login, and 503 otherwise (see +[Readiness checks](../../4-implementation/swarm/readiness.md)). diff --git a/docs/4-implementation/swarm/readiness.md b/docs/4-implementation/swarm/readiness.md new file mode 100644 index 000000000..f6a9b42fb --- /dev/null +++ b/docs/4-implementation/swarm/readiness.md @@ -0,0 +1,32 @@ +# Readiness checks + +The swarm daemons that serve `/readyz` (see the [services +page](../../2-handbook/swarm/services.md)) answer it one of two ways. A daemon +whose readiness is its queue connection reads that connection's state. A daemon +whose readiness is "its store answers" reads a cached check, so a probe never +waits on the store and never logs in to it. + +## The cached check + +`swarm_secret_client::health::CachedCheck` runs a check at start and then every +30 s, each run bounded by the 30 s period. A success records its time. The +check counts as fresh for 90 s after that, three periods, so one failed run +doesn't flip readiness. Before the first success there is nothing fresh, so the +daemon isn't ready. A failed or hung run logs a warning and changes nothing else. The task +ends when the last clone of the check drops. + +`CachedCheck::is_fresh` reads one atomic: the probe path does no I/O. +`Health::ready_while_checked` wires a check to a daemon's `/readyz`. + +## What each daemon checks + +| Daemon | The check | +| ----------------------- | ------------------------------------------------------------------------------------- | +| `swarm-logs-auth` | a login to the store under its own certificate | +| `swarm-metrics-auth` | the read of the stored bearer, the lookup it makes anyway | +| `swarm-authelia-bridge` | the users file loads (the bridge has no secret store) | +| `swarm-controller` | a login to the store; the probe reads the queue connection live, since it's in memory | + +The controller starts its check once, when it starts serving +(`routes::watch_store`), and `/readyz` reads the last result. Before that, and +in tests that don't start it, the store reads as not ready. diff --git a/swarm-agent-verify/src/lib.rs b/swarm-agent-verify/src/lib.rs index e06eb94f8..12617a433 100644 --- a/swarm-agent-verify/src/lib.rs +++ b/swarm-agent-verify/src/lib.rs @@ -53,6 +53,23 @@ impl Store { } impl Store { + /// Logs in and reads nothing: whether the store is reachable and accepts + /// this verifier's certificate. + /// + /// # Errors + /// + /// When the login fails. + pub async fn check_login(&self) -> anyhow::Result<()> { + swarm_secret_client::SecretStore::connect( + &self.settings, + &self.cert_role, + DEFAULT_CERT_MOUNT, + ) + .await + .context("logging in to the secret store")?; + Ok(()) + } + /// The object at `path`, `None` when nothing is stored there. Logs in per /// read, so no token is left to expire inside a long-lived process. /// diff --git a/swarm-authelia-bridge/src/main.rs b/swarm-authelia-bridge/src/main.rs index dbdec29d4..c15715320 100644 --- a/swarm-authelia-bridge/src/main.rs +++ b/swarm-authelia-bridge/src/main.rs @@ -125,11 +125,16 @@ async fn main() -> Result<()> { let app = Router::new() .route("/requests", post(handle_request)) - .with_state(state); + .with_state(Arc::clone(&state)); let listener = bind_socket(&socket)?; tracing::info!(socket = %socket.display(), "swarm-authelia-bridge listening"); - health.ready_when(|| true); + // Ready means the users file loads: without it every request fails. + let users_file = state.config.users_file.clone(); + health.ready_while_checked(move || { + let users_file = users_file.clone(); + async move { store::load_store(&users_file).map(drop) } + }); serve(listener, app, stop).await } diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index f35a6c171..741b657a3 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -294,6 +294,7 @@ async fn serve( state: AppState, stop: impl Future + Clone + Send + 'static, ) -> Result<()> { + routes::watch_store(); review_classifier::review::spawn_queue(&state, stop.clone()); request_reviewer::spawn(&state); let serving = axum::serve(listener, build_app(state)).with_graceful_shutdown(stop.clone()); diff --git a/swarm-controller/src/routes/mod.rs b/swarm-controller/src/routes/mod.rs index 3821d2ca6..9dca6e369 100644 --- a/swarm-controller/src/routes/mod.rs +++ b/swarm-controller/src/routes/mod.rs @@ -7,8 +7,7 @@ pub(crate) mod forge_admin; pub(crate) mod hives; pub(crate) mod jobq_graph; -use std::future::Future; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, OnceLock}; use axum::extract::State; use axum::http::StatusCode; @@ -32,11 +31,10 @@ pub(crate) async fn health() -> &'static str { concat!("swarm-controller ", env!("CARGO_PKG_VERSION"), "\n") } -/// How long the readiness probe waits for the secret store to answer a login. -const STORE_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); - /// Readiness probe: 200 when the swarm queue is connected and the secret store -/// accepts the controller's login, 503 otherwise. No body. +/// has accepted the controller's login within the last 90 s, 503 otherwise. No +/// body. The login is made by a background task every 30 s +/// ([`watch_store`]); a probe makes no store call. /// /// Not in the `OpenAPI` document: it is for the supervisor, not for API clients. pub(crate) async fn readyz(State(state): State) -> StatusCode { @@ -44,18 +42,28 @@ pub(crate) async fn readyz(State(state): State) -> StatusCode { .status .as_ref() .is_some_and(|status| swarm_queue_client::ensure_connected(&status.queue_client()).is_ok()); - ready(queue_connected, async { - tokio::time::timeout(STORE_PROBE_TIMEOUT, crate::store::connect()) - .await - .is_ok_and(|login| login.is_ok()) - }) - .await + let store_fresh = STORE_READY + .get() + .is_some_and(swarm_secret_client::health::CachedCheck::is_fresh); + ready(queue_connected, store_fresh) } -/// 200 when the queue is connected and `store_reachable` resolves true. The -/// store is not asked while the queue is down. -async fn ready(queue_connected: bool, store_reachable: impl Future) -> StatusCode { - if queue_connected && store_reachable.await { +/// The controller's own store login, tried every 30 s once [`watch_store`] has +/// started it. Process-wide, as the controller has one store login to ask about. +static STORE_READY: OnceLock = OnceLock::new(); + +/// Starts the store login `/readyz` reads. Called once, where serving starts. +pub(crate) fn watch_store() { + STORE_READY.get_or_init(|| { + swarm_secret_client::health::CachedCheck::every_30s(|| async { + crate::store::connect().await.map(drop) + }) + }); +} + +/// 200 when the queue is connected and the store's last login is fresh. +fn ready(queue_connected: bool, store_fresh: bool) -> StatusCode { + if queue_connected && store_fresh { StatusCode::OK } else { StatusCode::SERVICE_UNAVAILABLE @@ -238,25 +246,50 @@ pub(crate) fn wanted_error_status(e: &anyhow::Error) -> axum::http::StatusCode { pub(crate) mod tests { use super::*; - /// `/readyz` needs both the queue and the store, and does not ask the store - /// while the queue is down. - #[tokio::test] - async fn ready_needs_the_queue_and_the_store() { - use std::sync::atomic::{AtomicBool, Ordering}; + /// `/readyz` needs both the queue and a fresh store login. + #[test] + fn ready_needs_the_queue_and_the_store() { + assert_eq!(ready(true, true), StatusCode::OK); + assert_eq!(ready(true, false), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(ready(false, true), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(ready(false, false), StatusCode::SERVICE_UNAVAILABLE); + } - assert_eq!(ready(true, async { true }).await, StatusCode::OK); - assert_eq!( - ready(true, async { false }).await, - StatusCode::SERVICE_UNAVAILABLE, - "store unreachable" - ); - let asked = AtomicBool::new(false); - let store = async { - asked.store(true, Ordering::SeqCst); - true - }; - assert_eq!(ready(false, store).await, StatusCode::SERVICE_UNAVAILABLE); - assert!(!asked.load(Ordering::SeqCst), "queue down: store not asked"); + /// A probe makes no store call: the login runs once at start and then every + /// 30 s, however many probes arrive. + #[tokio::test] + async fn probing_readyz_makes_no_store_call() { + use axum::body::Body; + use axum::http::Request; + use std::sync::atomic::{AtomicUsize, Ordering}; + use tower::ServiceExt as _; + + let logins = Arc::new(AtomicUsize::new(0)); + let counted = Arc::clone(&logins); + let (state, _sched) = state_with_roster(); + // The one test that starts the check; the others leave it unset. + STORE_READY.get_or_init(|| { + swarm_secret_client::health::CachedCheck::every_30s(move || { + counted.fetch_add(1, Ordering::SeqCst); + std::future::ready(Ok::<(), String>(())) + }) + }); + // Let the first login run. + while logins.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + let app = crate::startup::build_app(state); + for _ in 0..50 { + app.clone() + .oneshot( + Request::get("/readyz") + .body(Body::empty()) + .expect("a request"), + ) + .await + .expect("answers"); + } + assert_eq!(logins.load(Ordering::SeqCst), 1, "50 probes, one login"); } /// Served by the real router: with no queue wired up `/readyz` is 503, diff --git a/swarm-logs-auth/src/main.rs b/swarm-logs-auth/src/main.rs index dc59d5a16..50ab16895 100644 --- a/swarm-logs-auth/src/main.rs +++ b/swarm-logs-auth/src/main.rs @@ -240,7 +240,12 @@ async fn main() -> anyhow::Result<()> { std::fs::set_permissions(&args.socket, std::fs::Permissions::from_mode(0o666)) .with_context(|| format!("chmod {}", args.socket.display()))?; tracing::info!(socket = %args.socket.display(), "swarm-logs-auth listening"); - health.ready_when(|| true); + // Ready means the store answers: with it down every request is a 401. + let probe = Arc::clone(&responder); + health.ready_while_checked(move || { + let probe = Arc::clone(&probe); + async move { probe.source.check_login().await } + }); let app = axum::Router::new().fallback(handle).with_state(responder); serve(listener, app, stop).await diff --git a/swarm-metrics-auth/src/main.rs b/swarm-metrics-auth/src/main.rs index 0e63d24e5..daecb77a3 100644 --- a/swarm-metrics-auth/src/main.rs +++ b/swarm-metrics-auth/src/main.rs @@ -327,7 +327,12 @@ async fn main() -> anyhow::Result<()> { std::fs::set_permissions(&args.socket, std::fs::Permissions::from_mode(0o666)) .with_context(|| format!("chmod {}", args.socket.display()))?; tracing::info!(socket = %args.socket.display(), "swarm-metrics-auth listening"); - health.ready_when(|| true); + // Ready means the store answers: with it down the bearer can't be checked. + let probe = Arc::clone(&responder); + health.ready_while_checked(move || { + let probe = Arc::clone(&probe); + async move { probe.source.lookup().await.map(drop) } + }); let app = axum::Router::new().fallback(handle).with_state(responder); serve(listener, app, stop).await diff --git a/swarm-secret-client/src/health.rs b/swarm-secret-client/src/health.rs index c9e7d5ce9..e5fe77d95 100644 --- a/swarm-secret-client/src/health.rs +++ b/swarm-secret-client/src/health.rs @@ -13,7 +13,10 @@ //! own) calls [`Health::beat_on_runtime`] instead, which shows the daemon's //! runtime still schedules tasks, not that the accept loop is healthy. //! - `GET /readyz` is 200 while the condition set with [`Health::ready_when`] -//! holds, 503 while it does not or none is set yet. +//! holds, 503 while it does not or none is set yet. A daemon whose readiness +//! is "its store answers" uses [`Health::ready_while_checked`]: a task checks +//! every 30 s off the probe path and the probe reads the last result, ready +//! for 90 s after a success. //! //! The answers carry a status code and no body. Nothing listens unless the //! operator names an address (see [`Health::start`]), so a daemon that is not @@ -46,6 +49,99 @@ pub const STALE_AFTER: Duration = Duration::from_secs(30); /// the daemon is alive and not ready. const NO_BEAT: u64 = u64::MAX; +/// How often a [`CachedCheck`] made with [`Health::ready_while_checked`] runs. +pub const CHECK_EVERY: Duration = Duration::from_secs(30); + +/// How long after its last success a check still counts: three periods, so a +/// single failed check does not flap readiness. +pub const FRESH_FOR: Duration = Duration::from_secs(90); + +/// `last_ok` before the check has first succeeded. +const NEVER: u64 = u64::MAX; + +struct CheckState { + born: Instant, + /// Milliseconds after `born` of the last success, or [`NEVER`]. + last_ok: AtomicU64, + fresh_for: Duration, +} + +/// The result of a check that runs off the probe path: a background task runs +/// it every period and records when it last succeeded, and [`is_fresh`] reads +/// that record without any I/O. The task ends when the last clone is dropped. +/// +/// [`is_fresh`]: CachedCheck::is_fresh +#[derive(Clone)] +pub struct CachedCheck(Arc); + +impl CachedCheck { + /// Runs `check` now and then every `every`, each run bounded by `every`. A + /// success keeps [`is_fresh`](Self::is_fresh) true for `fresh_for`; a failure + /// is logged as a warning and changes nothing. Before the first success + /// there is no freshness. + /// + /// # Panics + /// + /// Outside a tokio runtime. + pub fn spawn(every: Duration, fresh_for: Duration, check: F) -> Self + where + F: Fn() -> Fut + Send + 'static, + Fut: Future> + Send, + E: std::fmt::Display + Send, + { + let state = Arc::new(CheckState { + born: Instant::now(), + last_ok: AtomicU64::new(NEVER), + fresh_for, + }); + let task = Arc::downgrade(&state); + tokio::spawn(async move { + loop { + let outcome = tokio::time::timeout(every, check()).await; + let Some(state) = task.upgrade() else { return }; + match outcome { + Ok(Ok(())) => state.last_ok.store( + u64::try_from(state.born.elapsed().as_millis()).unwrap_or(u64::MAX), + Ordering::Relaxed, + ), + Ok(Err(e)) => { + tracing::warn!(error = %format!("{e:#}"), "readiness check failed"); + } + Err(_) => tracing::warn!("readiness check timed out"), + } + drop(state); + tokio::time::sleep(every).await; + } + }); + Self(state) + } + + /// [`CachedCheck::spawn`] at [`CHECK_EVERY`] and [`FRESH_FOR`]. + /// + /// # Panics + /// + /// Outside a tokio runtime. + pub fn every_30s(check: F) -> Self + where + F: Fn() -> Fut + Send + 'static, + Fut: Future> + Send, + E: std::fmt::Display + Send, + { + Self::spawn(CHECK_EVERY, FRESH_FOR, check) + } + + /// Whether the check has succeeded within `fresh_for`. Reads memory only. + #[must_use] + pub fn is_fresh(&self) -> bool { + let last = self.0.last_ok.load(Ordering::Relaxed); + if last == NEVER { + return false; + } + let now = u64::try_from(self.0.born.elapsed().as_millis()).unwrap_or(u64::MAX); + Duration::from_millis(now.saturating_sub(last)) <= self.0.fresh_for + } +} + type Condition = Box bool + Send + Sync>; struct Inner { @@ -92,6 +188,24 @@ impl Health { .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Box::new(condition)); } + /// Makes `/readyz` follow `check`, run every [`CHECK_EVERY`] off the probe + /// path (see [`CachedCheck`]): ready while it has succeeded within + /// [`FRESH_FOR`], not ready before its first success. Replaces an earlier + /// condition. + /// + /// # Panics + /// + /// Outside a tokio runtime. + pub fn ready_while_checked(&self, check: F) + where + F: Fn() -> Fut + Send + 'static, + Fut: Future> + Send, + E: std::fmt::Display + Send, + { + let cached = CachedCheck::every_30s(check); + self.ready_when(move || cached.is_fresh()); + } + /// Whether the main loop has beaten within `stale_after`. #[must_use] pub fn is_alive(&self) -> bool { @@ -445,6 +559,119 @@ mod tests { futures_util::stream::poll_fn(move |cx| rx.poll_recv(cx)).boxed() } + /// A check that answers from `ok` and counts its runs. + fn counted_check( + ok: &Arc, + runs: &Arc, + ) -> impl Fn() -> std::future::Ready> + Send + 'static { + let (ok, runs) = (Arc::clone(ok), Arc::clone(runs)); + move || { + runs.fetch_add(1, Ordering::SeqCst); + std::future::ready(if ok.load(Ordering::SeqCst) { + Ok(()) + } else { + Err("the store did not answer".to_owned()) + }) + } + } + + const EVERY: Duration = Duration::from_secs(30); + const FRESH: Duration = Duration::from_secs(90); + + #[tokio::test(start_paused = true)] + async fn a_check_that_never_succeeded_is_not_fresh() { + let (ok, runs) = ( + Arc::new(AtomicBool::new(false)), + Arc::new(AtomicU64::new(0)), + ); + let cached = CachedCheck::spawn(EVERY, FRESH, counted_check(&ok, &runs)); + tokio::time::sleep(EVERY * 10).await; + assert!(runs.load(Ordering::SeqCst) >= 10, "it kept trying"); + assert!(!cached.is_fresh()); + } + + #[tokio::test(start_paused = true)] + async fn a_fresh_success_is_ready_and_a_stale_one_is_not() { + let (ok, runs) = (Arc::new(AtomicBool::new(true)), Arc::new(AtomicU64::new(0))); + let cached = CachedCheck::spawn(EVERY, FRESH, counted_check(&ok, &runs)); + tokio::time::sleep(Duration::from_secs(1)).await; + assert!(cached.is_fresh(), "the first run succeeded"); + + ok.store(false, Ordering::SeqCst); + // The last success was at 0 s. Two failed runs (30 s, 60 s) are a blip. + tokio::time::sleep(Duration::from_secs(70)).await; + assert!(cached.is_fresh(), "within three periods of the success"); + tokio::time::sleep(Duration::from_secs(30)).await; + assert!(!cached.is_fresh(), "100 s since the last success"); + + ok.store(true, Ordering::SeqCst); + tokio::time::sleep(EVERY).await; + assert!(cached.is_fresh(), "a success makes it ready again"); + } + + /// The probe reads memory: a thousand probes cost no run of the check. + #[tokio::test(start_paused = true)] + async fn probes_do_not_run_the_check() { + let (ok, runs) = (Arc::new(AtomicBool::new(true)), Arc::new(AtomicU64::new(0))); + let cached = CachedCheck::spawn(EVERY, FRESH, counted_check(&ok, &runs)); + tokio::time::sleep(Duration::from_secs(1)).await; + let before = runs.load(Ordering::SeqCst); + for _ in 0..1000 { + assert!(cached.is_fresh()); + } + assert_eq!(runs.load(Ordering::SeqCst), before); + tokio::time::sleep(EVERY * 4).await; + assert!( + runs.load(Ordering::SeqCst) <= before + 5, + "one run a period" + ); + } + + #[tokio::test(start_paused = true)] + async fn a_check_that_hangs_is_cut_off_and_counts_as_failed() { + let cached = CachedCheck::spawn(EVERY, FRESH, || async { + std::future::pending::<()>().await; + Ok::<(), String>(()) + }); + tokio::time::sleep(EVERY * 5).await; + assert!(!cached.is_fresh()); + } + + #[tokio::test(start_paused = true)] + async fn dropping_the_last_clone_ends_the_checking() { + let (ok, runs) = (Arc::new(AtomicBool::new(true)), Arc::new(AtomicU64::new(0))); + let cached = CachedCheck::spawn(EVERY, FRESH, counted_check(&ok, &runs)); + tokio::time::sleep(Duration::from_secs(1)).await; + drop(cached); + tokio::time::sleep(EVERY * 2).await; + let after_drop = runs.load(Ordering::SeqCst); + tokio::time::sleep(EVERY * 10).await; + assert_eq!(runs.load(Ordering::SeqCst), after_drop); + } + + /// `/readyz` end to end: 503 until the first check succeeds, 200 after. + #[tokio::test] + async fn readyz_follows_the_cached_check() { + let runs = Arc::new(AtomicU64::new(0)); + let mut served = Vec::new(); + for (answers, expected) in [(false, 503), (true, 200)] { + let health = Health::new(STALE_AFTER); + let ok = Arc::new(AtomicBool::new(answers)); + health.ready_while_checked(counted_check(&ok, &runs)); + let (addr, stop, task) = listening(&health).await; + let mut got = 0; + for _ in 0..100 { + got = get_status(addr, "/readyz").await; + if got == expected { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert_eq!(got, expected, "a check that answers {answers}"); + served.push((stop, task)); + } + } + #[test] fn addr_takes_the_flag_else_the_variable_and_names_a_bad_one() { let flag: SocketAddr = "127.0.0.1:9".parse().unwrap();