//! Opt-in liveness and readiness endpoints for a swarm daemon. //! //! A daemon that lost its queue connection, or whose main loop is stuck, still //! looks "running" to systemd and to a container runtime. [`Health`] is the one //! state a daemon shares with a tiny HTTP listener that says so: //! //! - `GET /healthz` is 200 while the daemon's main loop has beaten within //! `stale_after`, 503 after, and 200 before its first beat, so a long startup is //! not judged. The loop beats by being polled: wrap its message //! stream in [`Health::beating`], or call [`Health::beat`] each iteration. A //! loop stuck in its body stops polling and the beat goes stale, which a //! constant 200 would not show. A daemon that is a server (no loop of its //! own) calls [`Health::beat_on_runtime`] instead, which shows the daemon's //! runtime still schedules tasks, not that the accept loop is healthy. //! - `GET /readyz` is 200 while the condition set with [`Health::ready_when`] //! holds, 503 while it does not or none is set yet. A daemon whose readiness //! is "its store answers" uses [`Health::ready_while_checked`]: a task checks //! every 30 s off the probe path and the probe reads the last result, ready //! for 90 s after a success. //! //! The answers carry a status code and no body. Nothing listens unless the //! operator names an address (see [`Health::start`]), so a daemon that is not //! given one binds nothing. use std::future::Future; use std::net::SocketAddr; use std::pin::Pin; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, RwLock}; use std::task::{Context, Poll}; use std::time::Duration; use axum::extract::State; use axum::http::StatusCode; use axum::routing::get; use futures_util::Stream; use tokio::time::{Instant, Interval, MissedTickBehavior}; /// How often an idle [`Beating`] stream and the runtime beat wake themselves. pub const BEAT_EVERY: Duration = Duration::from_secs(5); /// How long without a beat before `/healthz` answers 503, for a loop that /// beats at least every [`BEAT_EVERY`]. pub const STALE_AFTER: Duration = Duration::from_secs(30); /// `last_beat` before the main loop has first beaten. Startup, which may wait /// on the secret store for a long time, is not judged: the saturating /// subtraction in [`Health::is_alive`] makes the time since this beat zero, so /// the daemon is alive and not ready. const NO_BEAT: u64 = u64::MAX; /// How often a [`CachedCheck`] made with [`Health::ready_while_checked`] runs. pub const CHECK_EVERY: Duration = Duration::from_secs(30); /// How long after its last success a check still counts: three periods, so a /// single failed check does not flap readiness. pub const FRESH_FOR: Duration = Duration::from_secs(90); /// `last_ok` before the check has first succeeded. const NEVER: u64 = u64::MAX; struct CheckState { born: Instant, /// Milliseconds after `born` of the last success, or [`NEVER`]. last_ok: AtomicU64, fresh_for: Duration, } /// The result of a check that runs off the probe path: a background task runs /// it every period and records when it last succeeded, and [`is_fresh`] reads /// that record without any I/O. The task ends when the last clone is dropped. /// /// [`is_fresh`]: CachedCheck::is_fresh #[derive(Clone)] pub struct CachedCheck(Arc); impl CachedCheck { /// Runs `check` now and then every `every`, each run bounded by `every`. A /// success keeps [`is_fresh`](Self::is_fresh) true for `fresh_for`; a failure /// is logged as a warning and changes nothing. Before the first success /// there is no freshness. /// /// # Panics /// /// Outside a tokio runtime. pub fn spawn(every: Duration, fresh_for: Duration, check: F) -> Self where F: Fn() -> Fut + Send + 'static, Fut: Future> + Send, E: std::fmt::Display + Send, { let state = Arc::new(CheckState { born: Instant::now(), last_ok: AtomicU64::new(NEVER), fresh_for, }); let task = Arc::downgrade(&state); tokio::spawn(async move { loop { let outcome = tokio::time::timeout(every, check()).await; let Some(state) = task.upgrade() else { return }; match outcome { Ok(Ok(())) => state.last_ok.store( u64::try_from(state.born.elapsed().as_millis()).unwrap_or(u64::MAX), Ordering::Relaxed, ), Ok(Err(e)) => { tracing::warn!(error = %format!("{e:#}"), "readiness check failed"); } Err(_) => tracing::warn!("readiness check timed out"), } drop(state); tokio::time::sleep(every).await; } }); Self(state) } /// [`CachedCheck::spawn`] at [`CHECK_EVERY`] and [`FRESH_FOR`]. /// /// # Panics /// /// Outside a tokio runtime. pub fn every_30s(check: F) -> Self where F: Fn() -> Fut + Send + 'static, Fut: Future> + Send, E: std::fmt::Display + Send, { Self::spawn(CHECK_EVERY, FRESH_FOR, check) } /// Whether the check has succeeded within `fresh_for`. Reads memory only. #[must_use] pub fn is_fresh(&self) -> bool { let last = self.0.last_ok.load(Ordering::Relaxed); if last == NEVER { return false; } let now = u64::try_from(self.0.born.elapsed().as_millis()).unwrap_or(u64::MAX); Duration::from_millis(now.saturating_sub(last)) <= self.0.fresh_for } } type Condition = Box bool + Send + Sync>; struct Inner { born: Instant, /// Milliseconds after `born` of the last beat, or [`NO_BEAT`]. last_beat: AtomicU64, stale_after: Duration, ready: RwLock>, } /// What a daemon reports and the listener serves. Clones share one state. #[derive(Clone)] pub struct Health(Arc); impl Health { /// Alive until the first beat, then judged by the beats; not ready. `stale_after` is how long the main loop may go /// without a beat before the daemon counts as wedged. #[must_use] pub fn new(stale_after: Duration) -> Self { Self(Arc::new(Inner { born: Instant::now(), last_beat: AtomicU64::new(NO_BEAT), stale_after, ready: RwLock::new(None), })) } fn elapsed_ms(&self) -> u64 { u64::try_from(self.0.born.elapsed().as_millis()).unwrap_or(u64::MAX) } /// Records that the main loop is running. pub fn beat(&self) { self.0.last_beat.store(self.elapsed_ms(), Ordering::Relaxed); } /// Sets what `/readyz` checks, replacing an earlier condition. It runs on /// every probe, so it must not block or do I/O. pub fn ready_when(&self, condition: impl Fn() -> bool + Send + Sync + 'static) { *self .0 .ready .write() .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Box::new(condition)); } /// Makes `/readyz` follow `check`, run every [`CHECK_EVERY`] off the probe /// path (see [`CachedCheck`]): ready while it has succeeded within /// [`FRESH_FOR`], not ready before its first success. Replaces an earlier /// condition. /// /// # Panics /// /// Outside a tokio runtime. pub fn ready_while_checked(&self, check: F) where F: Fn() -> Fut + Send + 'static, Fut: Future> + Send, E: std::fmt::Display + Send, { let cached = CachedCheck::every_30s(check); self.ready_when(move || cached.is_fresh()); } /// Whether the main loop has beaten within `stale_after`. #[must_use] pub fn is_alive(&self) -> bool { let since = self .elapsed_ms() .saturating_sub(self.0.last_beat.load(Ordering::Relaxed)); Duration::from_millis(since) <= self.0.stale_after } /// Whether the readiness condition is set and holds. #[must_use] pub fn is_ready(&self) -> bool { self.0 .ready .read() .unwrap_or_else(std::sync::PoisonError::into_inner) .as_ref() .is_some_and(|holds| holds()) } /// Wraps the stream a main loop reads from so that each poll beats, and an /// idle loop wakes itself every [`BEAT_EVERY`] to beat. A loop stuck in /// the body of its `while let` stops polling, and the beat goes stale. #[must_use] pub fn beating(&self, stream: S) -> Beating { let mut tick = tokio::time::interval(BEAT_EVERY); tick.set_missed_tick_behavior(MissedTickBehavior::Delay); Beating { health: self.clone(), tick, inner: stream, } } /// Beats from a task on the daemon's runtime, for a daemon that is a /// server and has no loop of its own to beat. Stops beating only when /// the runtime stops scheduling. pub fn beat_on_runtime(&self) { let health = self.clone(); tokio::spawn(async move { loop { health.beat(); tokio::time::sleep(BEAT_EVERY).await; } }); } /// A [`Health`] that serves on the address in `flag`, else in the /// environment variable `var`. With neither set nothing binds and the /// daemon behaves as one without this module. The listener stops when /// `stop` resolves. /// /// # Errors /// /// [`StartError`] when the address does not parse or cannot be bound: an /// address the operator asked for and did not get is a startup failure, /// not a daemon without probes. pub async fn start( stale_after: Duration, flag: Option, var: &str, stop: impl Future + Clone + Send + 'static, ) -> Result { let (health, _) = Self::start_on(stale_after, addr(flag, var)?, stop).await?; Ok(health) } /// [`Health::start`] for a daemon with no flag, and the default /// [`STALE_AFTER`]. /// /// # Errors /// /// As [`Health::start`]. pub async fn from_env( var: &str, stop: impl Future + Clone + Send + 'static, ) -> Result { Self::start(STALE_AFTER, None, var, stop).await } /// [`Health::start`] for an address already resolved; also returns the /// address bound, `None` when `addr` is. async fn start_on( stale_after: Duration, addr: Option, stop: impl Future + Clone + Send + 'static, ) -> Result<(Self, Option), StartError> { let health = Self::new(stale_after); let bound = match addr { Some(addr) => Some( health .listen(addr, stop) .await .map_err(|source| StartError::Bind { addr, source })?, ), None => None, }; Ok((health, bound)) } /// Binds `addr` and serves the two endpoints from a task until `stop` /// resolves, then for at most the drain. Returns the address bound. async fn listen( &self, addr: SocketAddr, stop: impl Future + Clone + Send + 'static, ) -> std::io::Result { let listener = tokio::net::TcpListener::bind(addr).await?; let bound = listener.local_addr()?; tracing::info!(addr = %bound, "serving /healthz and /readyz"); tokio::spawn(self.clone().serve(listener, stop)); Ok(bound) } /// Serves on `listener` until `stop` resolves, then for at most the drain. pub async fn serve( self, listener: tokio::net::TcpListener, stop: impl Future + Clone + Send + 'static, ) { let app = axum::Router::new() .route("/healthz", get(healthz)) .route("/readyz", get(readyz)) .with_state(self); let serving = axum::serve(listener, app).with_graceful_shutdown(stop.clone()); let served = crate::shutdown::serve_bounded(stop, std::future::IntoFuture::into_future(serving)) .await; if let Some(Err(e)) = served { tracing::warn!(error = %e, "the health listener stopped"); } } } fn status(up: bool) -> StatusCode { if up { StatusCode::OK } else { StatusCode::SERVICE_UNAVAILABLE } } async fn healthz(State(health): State) -> StatusCode { status(health.is_alive()) } async fn readyz(State(health): State) -> StatusCode { status(health.is_ready()) } /// A stream that beats [`Health`] on every poll. See [`Health::beating`]. pub struct Beating { health: Health, tick: Interval, inner: S, } impl Stream for Beating { type Item = S::Item; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = &mut *self; this.health.beat(); // Registers the wake-up for the next beat; `Delay` makes the second // poll after a tick Pending, so the loop ends. while this.tick.poll_tick(cx).is_ready() {} Pin::new(&mut this.inner).poll_next(cx) } } /// Why [`Health::start`] could not start the listener. #[derive(Debug, thiserror::Error)] pub enum StartError { #[error("{origin} is not a socket address: {value:?}")] Addr { origin: String, value: String }, #[error("binding the health listener on {addr}")] Bind { addr: SocketAddr, #[source] source: std::io::Error, }, } /// The address the health listener binds: `flag` when the daemon has a flag /// and it was given, else the environment variable `var`, else `None`. /// /// # Errors /// /// [`StartError::Addr`] when `var` is set to something that is not `host:port`. fn addr(flag: Option, var: &str) -> Result, StartError> { if flag.is_some() { return Ok(flag); } match std::env::var(var) { Ok(value) if !value.is_empty() => parse(var, value).map(Some), _ => Ok(None), } } fn parse(origin: &str, value: String) -> Result { value.parse().map_err(|_| StartError::Addr { origin: origin.to_owned(), value, }) } #[cfg(test)] mod tests { use std::sync::atomic::AtomicBool; use futures_util::StreamExt as _; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; use tokio::sync::oneshot; use super::*; /// The status code of `GET path` on `addr`, over a bare TCP connection. pub(crate) async fn get_status(addr: SocketAddr, path: &str) -> u16 { let mut conn = tokio::net::TcpStream::connect(addr) .await .expect("the health listener accepts"); conn.write_all( format!("GET {path} HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n").as_bytes(), ) .await .expect("sending the request"); let mut reply = String::new(); conn.read_to_string(&mut reply).await.expect("the reply"); reply .split_whitespace() .nth(1) .and_then(|code| code.parse().ok()) .unwrap_or_else(|| panic!("no status line in {reply:?}")) } async fn listening( health: &Health, ) -> (SocketAddr, oneshot::Sender<()>, tokio::task::JoinHandle<()>) { let (stop_tx, stop_rx) = oneshot::channel::<()>(); let stop = futures_util::FutureExt::shared(async move { let _ = stop_rx.await; }); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let task = tokio::spawn(health.clone().serve(listener, stop)); (addr, stop_tx, task) } /// `/readyz` follows the condition, and 503 is what it says before one is /// set; `/healthz` is 200 while beats are fresh. #[tokio::test] async fn readyz_follows_the_condition_and_healthz_the_beat() { let health = Health::new(Duration::from_mins(1)); let (addr, _stop, _task) = listening(&health).await; assert_eq!(get_status(addr, "/healthz").await, 200); assert_eq!(get_status(addr, "/readyz").await, 503, "no condition set"); let up = Arc::new(AtomicBool::new(false)); let seen = Arc::clone(&up); health.ready_when(move || seen.load(Ordering::SeqCst)); assert_eq!(get_status(addr, "/readyz").await, 503, "condition false"); up.store(true, Ordering::SeqCst); assert_eq!(get_status(addr, "/readyz").await, 200, "condition true"); up.store(false, Ordering::SeqCst); assert_eq!(get_status(addr, "/readyz").await, 503, "back to false"); assert_eq!(get_status(addr, "/nope").await, 404); } /// `/healthz` goes 503 once the loop stops beating, and 200 again after a /// beat. #[tokio::test] async fn healthz_goes_503_when_the_beat_goes_stale() { let health = Health::new(Duration::from_millis(300)); let (addr, _stop, _task) = listening(&health).await; health.beat(); assert_eq!(get_status(addr, "/healthz").await, 200); tokio::time::sleep(Duration::from_millis(700)).await; assert_eq!(get_status(addr, "/healthz").await, 503, "no beat for 700ms"); health.beat(); assert_eq!(get_status(addr, "/healthz").await, 200, "beat again"); } /// The listener stops on the signal: the serving task ends and the port /// refuses a new connection. #[tokio::test] async fn the_listener_stops_on_the_signal() { let health = Health::new(STALE_AFTER); let (addr, stop, task) = listening(&health).await; assert_eq!( get_status(addr, "/healthz").await, 200, "up before the signal" ); stop.send(()).unwrap(); tokio::time::timeout(Duration::from_secs(5), task) .await .expect("the listener ends on the signal") .unwrap(); assert!(tokio::net::TcpStream::connect(addr).await.is_err()); } /// With no address, nothing binds; with one, it does, on the address given. #[tokio::test] async fn listen_binds_only_when_given_an_address() { let (_, none) = Health::start_on(STALE_AFTER, None, std::future::pending()) .await .unwrap(); assert_eq!(none, None); let (_, bound) = Health::start_on( STALE_AFTER, Some("127.0.0.1:0".parse().unwrap()), std::future::pending(), ) .await .unwrap(); let bound = bound.expect("a requested address is bound"); assert_eq!( get_status(bound, "/healthz").await, 200, "control: it serves" ); } /// A loop that stops polling its beating stream goes stale; one that is /// idle in `next()` keeps beating on its own. #[tokio::test(start_paused = true)] async fn a_beating_stream_beats_while_idle_and_not_while_the_loop_is_stuck() { let health = Health::new(STALE_AFTER); tokio::time::sleep(STALE_AFTER * 2).await; assert!( health.is_alive(), "startup is not judged before the first beat" ); let (_tx, rx) = tokio::sync::mpsc::channel::(1); let mut stream = health.beating(tokio_stream_of(rx)); // Idle in `next()` for far longer than `STALE_AFTER`. let idle = tokio::time::timeout(STALE_AFTER * 4, stream.next()).await; assert!(idle.is_err(), "nothing was sent"); assert!(health.is_alive(), "an idle loop still beats"); // Stuck: the loop stops polling the stream. tokio::time::sleep(STALE_AFTER * 2).await; assert!(!health.is_alive(), "no poll, no beat"); // Polled again. let _ = tokio::time::timeout(Duration::from_millis(1), stream.next()).await; assert!(health.is_alive()); } fn tokio_stream_of( mut rx: tokio::sync::mpsc::Receiver, ) -> futures_util::stream::BoxStream<'static, u8> { futures_util::stream::poll_fn(move |cx| rx.poll_recv(cx)).boxed() } /// A check that answers from `ok` and counts its runs. fn counted_check( ok: &Arc, runs: &Arc, ) -> impl Fn() -> std::future::Ready> + Send + 'static { let (ok, runs) = (Arc::clone(ok), Arc::clone(runs)); move || { runs.fetch_add(1, Ordering::SeqCst); std::future::ready(if ok.load(Ordering::SeqCst) { Ok(()) } else { Err("the store did not answer".to_owned()) }) } } const EVERY: Duration = Duration::from_secs(30); const FRESH: Duration = Duration::from_secs(90); #[tokio::test(start_paused = true)] async fn a_check_that_never_succeeded_is_not_fresh() { let (ok, runs) = ( Arc::new(AtomicBool::new(false)), Arc::new(AtomicU64::new(0)), ); let cached = CachedCheck::spawn(EVERY, FRESH, counted_check(&ok, &runs)); tokio::time::sleep(EVERY * 10).await; assert!(runs.load(Ordering::SeqCst) >= 10, "it kept trying"); assert!(!cached.is_fresh()); } #[tokio::test(start_paused = true)] async fn a_fresh_success_is_ready_and_a_stale_one_is_not() { let (ok, runs) = (Arc::new(AtomicBool::new(true)), Arc::new(AtomicU64::new(0))); let cached = CachedCheck::spawn(EVERY, FRESH, counted_check(&ok, &runs)); tokio::time::sleep(Duration::from_secs(1)).await; assert!(cached.is_fresh(), "the first run succeeded"); ok.store(false, Ordering::SeqCst); // The last success was at 0 s. Two failed runs (30 s, 60 s) are a blip. tokio::time::sleep(Duration::from_secs(70)).await; assert!(cached.is_fresh(), "within three periods of the success"); tokio::time::sleep(Duration::from_secs(30)).await; assert!(!cached.is_fresh(), "100 s since the last success"); ok.store(true, Ordering::SeqCst); tokio::time::sleep(EVERY).await; assert!(cached.is_fresh(), "a success makes it ready again"); } /// The probe reads memory: a thousand probes cost no run of the check. #[tokio::test(start_paused = true)] async fn probes_do_not_run_the_check() { let (ok, runs) = (Arc::new(AtomicBool::new(true)), Arc::new(AtomicU64::new(0))); let cached = CachedCheck::spawn(EVERY, FRESH, counted_check(&ok, &runs)); tokio::time::sleep(Duration::from_secs(1)).await; let before = runs.load(Ordering::SeqCst); for _ in 0..1000 { assert!(cached.is_fresh()); } assert_eq!(runs.load(Ordering::SeqCst), before); tokio::time::sleep(EVERY * 4).await; assert!( runs.load(Ordering::SeqCst) <= before + 5, "one run a period" ); } #[tokio::test(start_paused = true)] async fn a_check_that_hangs_is_cut_off_and_counts_as_failed() { let cached = CachedCheck::spawn(EVERY, FRESH, || async { std::future::pending::<()>().await; Ok::<(), String>(()) }); tokio::time::sleep(EVERY * 5).await; assert!(!cached.is_fresh()); } #[tokio::test(start_paused = true)] async fn dropping_the_last_clone_ends_the_checking() { let (ok, runs) = (Arc::new(AtomicBool::new(true)), Arc::new(AtomicU64::new(0))); let cached = CachedCheck::spawn(EVERY, FRESH, counted_check(&ok, &runs)); tokio::time::sleep(Duration::from_secs(1)).await; drop(cached); tokio::time::sleep(EVERY * 2).await; let after_drop = runs.load(Ordering::SeqCst); tokio::time::sleep(EVERY * 10).await; assert_eq!(runs.load(Ordering::SeqCst), after_drop); } /// `/readyz` end to end: 503 until the first check succeeds, 200 after. #[tokio::test] async fn readyz_follows_the_cached_check() { let runs = Arc::new(AtomicU64::new(0)); let mut served = Vec::new(); for (answers, expected) in [(false, 503), (true, 200)] { let health = Health::new(STALE_AFTER); let ok = Arc::new(AtomicBool::new(answers)); health.ready_while_checked(counted_check(&ok, &runs)); let (addr, stop, task) = listening(&health).await; let mut got = 0; for _ in 0..100 { got = get_status(addr, "/readyz").await; if got == expected { break; } tokio::time::sleep(Duration::from_millis(20)).await; } assert_eq!(got, expected, "a check that answers {answers}"); served.push((stop, task)); } } #[test] fn addr_takes_the_flag_else_the_variable_and_names_a_bad_one() { let flag: SocketAddr = "127.0.0.1:9".parse().unwrap(); assert_eq!(addr(Some(flag), "HEALTH_TEST_UNSET").unwrap(), Some(flag)); assert_eq!(addr(None, "HEALTH_TEST_UNSET").unwrap(), None); assert_eq!(parse("V", "127.0.0.1:9".into()).unwrap(), flag); let bad = parse("SWARM_X_HEALTH_ADDR", "nonsense".into()).unwrap_err(); assert!(bad.to_string().contains("SWARM_X_HEALTH_ADDR"), "{bad}"); } }