684 lines
25 KiB
Rust
684 lines
25 KiB
Rust
//! Opt-in liveness and readiness endpoints for a swarm daemon.
|
|
//!
|
|
//! A daemon that lost its queue connection, or whose main loop is stuck, still
|
|
//! looks "running" to systemd and to a container runtime. [`Health`] is the one
|
|
//! state a daemon shares with a tiny HTTP listener that says so:
|
|
//!
|
|
//! - `GET /healthz` is 200 while the daemon's main loop has beaten within
|
|
//! `stale_after`, 503 after, and 200 before its first beat, so a long startup is
|
|
//! not judged. The loop beats by being polled: wrap its message
|
|
//! stream in [`Health::beating`], or call [`Health::beat`] each iteration. A
|
|
//! loop stuck in its body stops polling and the beat goes stale, which a
|
|
//! constant 200 would not show. A daemon that is a server (no loop of its
|
|
//! own) calls [`Health::beat_on_runtime`] instead, which shows the daemon's
|
|
//! runtime still schedules tasks, not that the accept loop is healthy.
|
|
//! - `GET /readyz` is 200 while the condition set with [`Health::ready_when`]
|
|
//! holds, 503 while it does not or none is set yet. A daemon whose readiness
|
|
//! is "its store answers" uses [`Health::ready_while_checked`]: a task checks
|
|
//! every 30 s off the probe path and the probe reads the last result, ready
|
|
//! for 90 s after a success.
|
|
//!
|
|
//! The answers carry a status code and no body. Nothing listens unless the
|
|
//! operator names an address (see [`Health::start`]), so a daemon that is not
|
|
//! given one binds nothing.
|
|
|
|
use std::future::Future;
|
|
use std::net::SocketAddr;
|
|
use std::pin::Pin;
|
|
use std::sync::atomic::{AtomicU64, Ordering};
|
|
use std::sync::{Arc, RwLock};
|
|
use std::task::{Context, Poll};
|
|
use std::time::Duration;
|
|
|
|
use axum::extract::State;
|
|
use axum::http::StatusCode;
|
|
use axum::routing::get;
|
|
use futures_util::Stream;
|
|
use tokio::time::{Instant, Interval, MissedTickBehavior};
|
|
|
|
/// How often an idle [`Beating`] stream and the runtime beat wake themselves.
|
|
pub const BEAT_EVERY: Duration = Duration::from_secs(5);
|
|
|
|
/// How long without a beat before `/healthz` answers 503, for a loop that
|
|
/// beats at least every [`BEAT_EVERY`].
|
|
pub const STALE_AFTER: Duration = Duration::from_secs(30);
|
|
|
|
/// `last_beat` before the main loop has first beaten. Startup, which may wait
|
|
/// on the secret store for a long time, is not judged: the saturating
|
|
/// subtraction in [`Health::is_alive`] makes the time since this beat zero, so
|
|
/// the daemon is alive and not ready.
|
|
const NO_BEAT: u64 = u64::MAX;
|
|
|
|
/// How often a [`CachedCheck`] made with [`Health::ready_while_checked`] runs.
|
|
pub const CHECK_EVERY: Duration = Duration::from_secs(30);
|
|
|
|
/// How long after its last success a check still counts: three periods, so a
|
|
/// single failed check does not flap readiness.
|
|
pub const FRESH_FOR: Duration = Duration::from_secs(90);
|
|
|
|
/// `last_ok` before the check has first succeeded.
|
|
const NEVER: u64 = u64::MAX;
|
|
|
|
struct CheckState {
|
|
born: Instant,
|
|
/// Milliseconds after `born` of the last success, or [`NEVER`].
|
|
last_ok: AtomicU64,
|
|
fresh_for: Duration,
|
|
}
|
|
|
|
/// The result of a check that runs off the probe path: a background task runs
|
|
/// it every period and records when it last succeeded, and [`is_fresh`] reads
|
|
/// that record without any I/O. The task ends when the last clone is dropped.
|
|
///
|
|
/// [`is_fresh`]: CachedCheck::is_fresh
|
|
#[derive(Clone)]
|
|
pub struct CachedCheck(Arc<CheckState>);
|
|
|
|
impl CachedCheck {
|
|
/// Runs `check` now and then every `every`, each run bounded by `every`. A
|
|
/// success keeps [`is_fresh`](Self::is_fresh) true for `fresh_for`; a failure
|
|
/// is logged as a warning and changes nothing. Before the first success
|
|
/// there is no freshness.
|
|
///
|
|
/// # Panics
|
|
///
|
|
/// Outside a tokio runtime.
|
|
pub fn spawn<F, Fut, E>(every: Duration, fresh_for: Duration, check: F) -> Self
|
|
where
|
|
F: Fn() -> Fut + Send + 'static,
|
|
Fut: Future<Output = Result<(), E>> + Send,
|
|
E: std::fmt::Display + Send,
|
|
{
|
|
let state = Arc::new(CheckState {
|
|
born: Instant::now(),
|
|
last_ok: AtomicU64::new(NEVER),
|
|
fresh_for,
|
|
});
|
|
let task = Arc::downgrade(&state);
|
|
tokio::spawn(async move {
|
|
loop {
|
|
let outcome = tokio::time::timeout(every, check()).await;
|
|
let Some(state) = task.upgrade() else { return };
|
|
match outcome {
|
|
Ok(Ok(())) => state.last_ok.store(
|
|
u64::try_from(state.born.elapsed().as_millis()).unwrap_or(u64::MAX),
|
|
Ordering::Relaxed,
|
|
),
|
|
Ok(Err(e)) => {
|
|
tracing::warn!(error = %format!("{e:#}"), "readiness check failed");
|
|
}
|
|
Err(_) => tracing::warn!("readiness check timed out"),
|
|
}
|
|
drop(state);
|
|
tokio::time::sleep(every).await;
|
|
}
|
|
});
|
|
Self(state)
|
|
}
|
|
|
|
/// [`CachedCheck::spawn`] at [`CHECK_EVERY`] and [`FRESH_FOR`].
|
|
///
|
|
/// # Panics
|
|
///
|
|
/// Outside a tokio runtime.
|
|
pub fn every_30s<F, Fut, E>(check: F) -> Self
|
|
where
|
|
F: Fn() -> Fut + Send + 'static,
|
|
Fut: Future<Output = Result<(), E>> + Send,
|
|
E: std::fmt::Display + Send,
|
|
{
|
|
Self::spawn(CHECK_EVERY, FRESH_FOR, check)
|
|
}
|
|
|
|
/// Whether the check has succeeded within `fresh_for`. Reads memory only.
|
|
#[must_use]
|
|
pub fn is_fresh(&self) -> bool {
|
|
let last = self.0.last_ok.load(Ordering::Relaxed);
|
|
if last == NEVER {
|
|
return false;
|
|
}
|
|
let now = u64::try_from(self.0.born.elapsed().as_millis()).unwrap_or(u64::MAX);
|
|
Duration::from_millis(now.saturating_sub(last)) <= self.0.fresh_for
|
|
}
|
|
}
|
|
|
|
type Condition = Box<dyn Fn() -> bool + Send + Sync>;
|
|
|
|
struct Inner {
|
|
born: Instant,
|
|
/// Milliseconds after `born` of the last beat, or [`NO_BEAT`].
|
|
last_beat: AtomicU64,
|
|
stale_after: Duration,
|
|
ready: RwLock<Option<Condition>>,
|
|
}
|
|
|
|
/// What a daemon reports and the listener serves. Clones share one state.
|
|
#[derive(Clone)]
|
|
pub struct Health(Arc<Inner>);
|
|
|
|
impl Health {
|
|
/// Alive until the first beat, then judged by the beats; not ready. `stale_after` is how long the main loop may go
|
|
/// without a beat before the daemon counts as wedged.
|
|
#[must_use]
|
|
pub fn new(stale_after: Duration) -> Self {
|
|
Self(Arc::new(Inner {
|
|
born: Instant::now(),
|
|
last_beat: AtomicU64::new(NO_BEAT),
|
|
stale_after,
|
|
ready: RwLock::new(None),
|
|
}))
|
|
}
|
|
|
|
fn elapsed_ms(&self) -> u64 {
|
|
u64::try_from(self.0.born.elapsed().as_millis()).unwrap_or(u64::MAX)
|
|
}
|
|
|
|
/// Records that the main loop is running.
|
|
pub fn beat(&self) {
|
|
self.0.last_beat.store(self.elapsed_ms(), Ordering::Relaxed);
|
|
}
|
|
|
|
/// Sets what `/readyz` checks, replacing an earlier condition. It runs on
|
|
/// every probe, so it must not block or do I/O.
|
|
pub fn ready_when(&self, condition: impl Fn() -> bool + Send + Sync + 'static) {
|
|
*self
|
|
.0
|
|
.ready
|
|
.write()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Box::new(condition));
|
|
}
|
|
|
|
/// Makes `/readyz` follow `check`, run every [`CHECK_EVERY`] off the probe
|
|
/// path (see [`CachedCheck`]): ready while it has succeeded within
|
|
/// [`FRESH_FOR`], not ready before its first success. Replaces an earlier
|
|
/// condition.
|
|
///
|
|
/// # Panics
|
|
///
|
|
/// Outside a tokio runtime.
|
|
pub fn ready_while_checked<F, Fut, E>(&self, check: F)
|
|
where
|
|
F: Fn() -> Fut + Send + 'static,
|
|
Fut: Future<Output = Result<(), E>> + Send,
|
|
E: std::fmt::Display + Send,
|
|
{
|
|
let cached = CachedCheck::every_30s(check);
|
|
self.ready_when(move || cached.is_fresh());
|
|
}
|
|
|
|
/// Whether the main loop has beaten within `stale_after`.
|
|
#[must_use]
|
|
pub fn is_alive(&self) -> bool {
|
|
let since = self
|
|
.elapsed_ms()
|
|
.saturating_sub(self.0.last_beat.load(Ordering::Relaxed));
|
|
Duration::from_millis(since) <= self.0.stale_after
|
|
}
|
|
|
|
/// Whether the readiness condition is set and holds.
|
|
#[must_use]
|
|
pub fn is_ready(&self) -> bool {
|
|
self.0
|
|
.ready
|
|
.read()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
.as_ref()
|
|
.is_some_and(|holds| holds())
|
|
}
|
|
|
|
/// Wraps the stream a main loop reads from so that each poll beats, and an
|
|
/// idle loop wakes itself every [`BEAT_EVERY`] to beat. A loop stuck in
|
|
/// the body of its `while let` stops polling, and the beat goes stale.
|
|
#[must_use]
|
|
pub fn beating<S: Stream + Unpin>(&self, stream: S) -> Beating<S> {
|
|
let mut tick = tokio::time::interval(BEAT_EVERY);
|
|
tick.set_missed_tick_behavior(MissedTickBehavior::Delay);
|
|
Beating {
|
|
health: self.clone(),
|
|
tick,
|
|
inner: stream,
|
|
}
|
|
}
|
|
|
|
/// Beats from a task on the daemon's runtime, for a daemon that is a
|
|
/// server and has no loop of its own to beat. Stops beating only when
|
|
/// the runtime stops scheduling.
|
|
pub fn beat_on_runtime(&self) {
|
|
let health = self.clone();
|
|
tokio::spawn(async move {
|
|
loop {
|
|
health.beat();
|
|
tokio::time::sleep(BEAT_EVERY).await;
|
|
}
|
|
});
|
|
}
|
|
|
|
/// A [`Health`] that serves on the address in `flag`, else in the
|
|
/// environment variable `var`. With neither set nothing binds and the
|
|
/// daemon behaves as one without this module. The listener stops when
|
|
/// `stop` resolves.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// [`StartError`] when the address does not parse or cannot be bound: an
|
|
/// address the operator asked for and did not get is a startup failure,
|
|
/// not a daemon without probes.
|
|
pub async fn start(
|
|
stale_after: Duration,
|
|
flag: Option<SocketAddr>,
|
|
var: &str,
|
|
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
|
) -> Result<Self, StartError> {
|
|
let (health, _) = Self::start_on(stale_after, addr(flag, var)?, stop).await?;
|
|
Ok(health)
|
|
}
|
|
|
|
/// [`Health::start`] for a daemon with no flag, and the default
|
|
/// [`STALE_AFTER`].
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// As [`Health::start`].
|
|
pub async fn from_env(
|
|
var: &str,
|
|
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
|
) -> Result<Self, StartError> {
|
|
Self::start(STALE_AFTER, None, var, stop).await
|
|
}
|
|
|
|
/// [`Health::start`] for an address already resolved; also returns the
|
|
/// address bound, `None` when `addr` is.
|
|
async fn start_on(
|
|
stale_after: Duration,
|
|
addr: Option<SocketAddr>,
|
|
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
|
) -> Result<(Self, Option<SocketAddr>), StartError> {
|
|
let health = Self::new(stale_after);
|
|
let bound = match addr {
|
|
Some(addr) => Some(
|
|
health
|
|
.listen(addr, stop)
|
|
.await
|
|
.map_err(|source| StartError::Bind { addr, source })?,
|
|
),
|
|
None => None,
|
|
};
|
|
Ok((health, bound))
|
|
}
|
|
|
|
/// Binds `addr` and serves the two endpoints from a task until `stop`
|
|
/// resolves, then for at most the drain. Returns the address bound.
|
|
async fn listen(
|
|
&self,
|
|
addr: SocketAddr,
|
|
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
|
) -> std::io::Result<SocketAddr> {
|
|
let listener = tokio::net::TcpListener::bind(addr).await?;
|
|
let bound = listener.local_addr()?;
|
|
tracing::info!(addr = %bound, "serving /healthz and /readyz");
|
|
tokio::spawn(self.clone().serve(listener, stop));
|
|
Ok(bound)
|
|
}
|
|
|
|
/// Serves on `listener` until `stop` resolves, then for at most the drain.
|
|
pub async fn serve(
|
|
self,
|
|
listener: tokio::net::TcpListener,
|
|
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
|
) {
|
|
let app = axum::Router::new()
|
|
.route("/healthz", get(healthz))
|
|
.route("/readyz", get(readyz))
|
|
.with_state(self);
|
|
let serving = axum::serve(listener, app).with_graceful_shutdown(stop.clone());
|
|
let served =
|
|
crate::shutdown::serve_bounded(stop, std::future::IntoFuture::into_future(serving))
|
|
.await;
|
|
if let Some(Err(e)) = served {
|
|
tracing::warn!(error = %e, "the health listener stopped");
|
|
}
|
|
}
|
|
}
|
|
|
|
fn status(up: bool) -> StatusCode {
|
|
if up {
|
|
StatusCode::OK
|
|
} else {
|
|
StatusCode::SERVICE_UNAVAILABLE
|
|
}
|
|
}
|
|
|
|
async fn healthz(State(health): State<Health>) -> StatusCode {
|
|
status(health.is_alive())
|
|
}
|
|
|
|
async fn readyz(State(health): State<Health>) -> StatusCode {
|
|
status(health.is_ready())
|
|
}
|
|
|
|
/// A stream that beats [`Health`] on every poll. See [`Health::beating`].
|
|
pub struct Beating<S> {
|
|
health: Health,
|
|
tick: Interval,
|
|
inner: S,
|
|
}
|
|
|
|
impl<S: Stream + Unpin> Stream for Beating<S> {
|
|
type Item = S::Item;
|
|
|
|
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<S::Item>> {
|
|
let this = &mut *self;
|
|
this.health.beat();
|
|
// Registers the wake-up for the next beat; `Delay` makes the second
|
|
// poll after a tick Pending, so the loop ends.
|
|
while this.tick.poll_tick(cx).is_ready() {}
|
|
Pin::new(&mut this.inner).poll_next(cx)
|
|
}
|
|
}
|
|
|
|
/// Why [`Health::start`] could not start the listener.
|
|
#[derive(Debug, thiserror::Error)]
|
|
pub enum StartError {
|
|
#[error("{origin} is not a socket address: {value:?}")]
|
|
Addr { origin: String, value: String },
|
|
#[error("binding the health listener on {addr}")]
|
|
Bind {
|
|
addr: SocketAddr,
|
|
#[source]
|
|
source: std::io::Error,
|
|
},
|
|
}
|
|
|
|
/// The address the health listener binds: `flag` when the daemon has a flag
|
|
/// and it was given, else the environment variable `var`, else `None`.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// [`StartError::Addr`] when `var` is set to something that is not `host:port`.
|
|
fn addr(flag: Option<SocketAddr>, var: &str) -> Result<Option<SocketAddr>, StartError> {
|
|
if flag.is_some() {
|
|
return Ok(flag);
|
|
}
|
|
match std::env::var(var) {
|
|
Ok(value) if !value.is_empty() => parse(var, value).map(Some),
|
|
_ => Ok(None),
|
|
}
|
|
}
|
|
|
|
fn parse(origin: &str, value: String) -> Result<SocketAddr, StartError> {
|
|
value.parse().map_err(|_| StartError::Addr {
|
|
origin: origin.to_owned(),
|
|
value,
|
|
})
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use std::sync::atomic::AtomicBool;
|
|
|
|
use futures_util::StreamExt as _;
|
|
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
|
use tokio::sync::oneshot;
|
|
|
|
use super::*;
|
|
|
|
/// The status code of `GET path` on `addr`, over a bare TCP connection.
|
|
pub(crate) async fn get_status(addr: SocketAddr, path: &str) -> u16 {
|
|
let mut conn = tokio::net::TcpStream::connect(addr)
|
|
.await
|
|
.expect("the health listener accepts");
|
|
conn.write_all(
|
|
format!("GET {path} HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n").as_bytes(),
|
|
)
|
|
.await
|
|
.expect("sending the request");
|
|
let mut reply = String::new();
|
|
conn.read_to_string(&mut reply).await.expect("the reply");
|
|
reply
|
|
.split_whitespace()
|
|
.nth(1)
|
|
.and_then(|code| code.parse().ok())
|
|
.unwrap_or_else(|| panic!("no status line in {reply:?}"))
|
|
}
|
|
|
|
async fn listening(
|
|
health: &Health,
|
|
) -> (SocketAddr, oneshot::Sender<()>, tokio::task::JoinHandle<()>) {
|
|
let (stop_tx, stop_rx) = oneshot::channel::<()>();
|
|
let stop = futures_util::FutureExt::shared(async move {
|
|
let _ = stop_rx.await;
|
|
});
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let addr = listener.local_addr().unwrap();
|
|
let task = tokio::spawn(health.clone().serve(listener, stop));
|
|
(addr, stop_tx, task)
|
|
}
|
|
|
|
/// `/readyz` follows the condition, and 503 is what it says before one is
|
|
/// set; `/healthz` is 200 while beats are fresh.
|
|
#[tokio::test]
|
|
async fn readyz_follows_the_condition_and_healthz_the_beat() {
|
|
let health = Health::new(Duration::from_mins(1));
|
|
let (addr, _stop, _task) = listening(&health).await;
|
|
assert_eq!(get_status(addr, "/healthz").await, 200);
|
|
assert_eq!(get_status(addr, "/readyz").await, 503, "no condition set");
|
|
|
|
let up = Arc::new(AtomicBool::new(false));
|
|
let seen = Arc::clone(&up);
|
|
health.ready_when(move || seen.load(Ordering::SeqCst));
|
|
assert_eq!(get_status(addr, "/readyz").await, 503, "condition false");
|
|
up.store(true, Ordering::SeqCst);
|
|
assert_eq!(get_status(addr, "/readyz").await, 200, "condition true");
|
|
up.store(false, Ordering::SeqCst);
|
|
assert_eq!(get_status(addr, "/readyz").await, 503, "back to false");
|
|
assert_eq!(get_status(addr, "/nope").await, 404);
|
|
}
|
|
|
|
/// `/healthz` goes 503 once the loop stops beating, and 200 again after a
|
|
/// beat.
|
|
#[tokio::test]
|
|
async fn healthz_goes_503_when_the_beat_goes_stale() {
|
|
let health = Health::new(Duration::from_millis(300));
|
|
let (addr, _stop, _task) = listening(&health).await;
|
|
health.beat();
|
|
assert_eq!(get_status(addr, "/healthz").await, 200);
|
|
tokio::time::sleep(Duration::from_millis(700)).await;
|
|
assert_eq!(get_status(addr, "/healthz").await, 503, "no beat for 700ms");
|
|
health.beat();
|
|
assert_eq!(get_status(addr, "/healthz").await, 200, "beat again");
|
|
}
|
|
|
|
/// The listener stops on the signal: the serving task ends and the port
|
|
/// refuses a new connection.
|
|
#[tokio::test]
|
|
async fn the_listener_stops_on_the_signal() {
|
|
let health = Health::new(STALE_AFTER);
|
|
let (addr, stop, task) = listening(&health).await;
|
|
assert_eq!(
|
|
get_status(addr, "/healthz").await,
|
|
200,
|
|
"up before the signal"
|
|
);
|
|
stop.send(()).unwrap();
|
|
tokio::time::timeout(Duration::from_secs(5), task)
|
|
.await
|
|
.expect("the listener ends on the signal")
|
|
.unwrap();
|
|
assert!(tokio::net::TcpStream::connect(addr).await.is_err());
|
|
}
|
|
|
|
/// With no address, nothing binds; with one, it does, on the address given.
|
|
#[tokio::test]
|
|
async fn listen_binds_only_when_given_an_address() {
|
|
let (_, none) = Health::start_on(STALE_AFTER, None, std::future::pending())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(none, None);
|
|
let (_, bound) = Health::start_on(
|
|
STALE_AFTER,
|
|
Some("127.0.0.1:0".parse().unwrap()),
|
|
std::future::pending(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let bound = bound.expect("a requested address is bound");
|
|
assert_eq!(
|
|
get_status(bound, "/healthz").await,
|
|
200,
|
|
"control: it serves"
|
|
);
|
|
}
|
|
|
|
/// A loop that stops polling its beating stream goes stale; one that is
|
|
/// idle in `next()` keeps beating on its own.
|
|
#[tokio::test(start_paused = true)]
|
|
async fn a_beating_stream_beats_while_idle_and_not_while_the_loop_is_stuck() {
|
|
let health = Health::new(STALE_AFTER);
|
|
tokio::time::sleep(STALE_AFTER * 2).await;
|
|
assert!(
|
|
health.is_alive(),
|
|
"startup is not judged before the first beat"
|
|
);
|
|
let (_tx, rx) = tokio::sync::mpsc::channel::<u8>(1);
|
|
let mut stream = health.beating(tokio_stream_of(rx));
|
|
// Idle in `next()` for far longer than `STALE_AFTER`.
|
|
let idle = tokio::time::timeout(STALE_AFTER * 4, stream.next()).await;
|
|
assert!(idle.is_err(), "nothing was sent");
|
|
assert!(health.is_alive(), "an idle loop still beats");
|
|
// Stuck: the loop stops polling the stream.
|
|
tokio::time::sleep(STALE_AFTER * 2).await;
|
|
assert!(!health.is_alive(), "no poll, no beat");
|
|
// Polled again.
|
|
let _ = tokio::time::timeout(Duration::from_millis(1), stream.next()).await;
|
|
assert!(health.is_alive());
|
|
}
|
|
|
|
fn tokio_stream_of(
|
|
mut rx: tokio::sync::mpsc::Receiver<u8>,
|
|
) -> futures_util::stream::BoxStream<'static, u8> {
|
|
futures_util::stream::poll_fn(move |cx| rx.poll_recv(cx)).boxed()
|
|
}
|
|
|
|
/// A check that answers from `ok` and counts its runs.
|
|
fn counted_check(
|
|
ok: &Arc<AtomicBool>,
|
|
runs: &Arc<AtomicU64>,
|
|
) -> impl Fn() -> std::future::Ready<Result<(), String>> + Send + 'static {
|
|
let (ok, runs) = (Arc::clone(ok), Arc::clone(runs));
|
|
move || {
|
|
runs.fetch_add(1, Ordering::SeqCst);
|
|
std::future::ready(if ok.load(Ordering::SeqCst) {
|
|
Ok(())
|
|
} else {
|
|
Err("the store did not answer".to_owned())
|
|
})
|
|
}
|
|
}
|
|
|
|
const EVERY: Duration = Duration::from_secs(30);
|
|
const FRESH: Duration = Duration::from_secs(90);
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn a_check_that_never_succeeded_is_not_fresh() {
|
|
let (ok, runs) = (
|
|
Arc::new(AtomicBool::new(false)),
|
|
Arc::new(AtomicU64::new(0)),
|
|
);
|
|
let cached = CachedCheck::spawn(EVERY, FRESH, counted_check(&ok, &runs));
|
|
tokio::time::sleep(EVERY * 10).await;
|
|
assert!(runs.load(Ordering::SeqCst) >= 10, "it kept trying");
|
|
assert!(!cached.is_fresh());
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn a_fresh_success_is_ready_and_a_stale_one_is_not() {
|
|
let (ok, runs) = (Arc::new(AtomicBool::new(true)), Arc::new(AtomicU64::new(0)));
|
|
let cached = CachedCheck::spawn(EVERY, FRESH, counted_check(&ok, &runs));
|
|
tokio::time::sleep(Duration::from_secs(1)).await;
|
|
assert!(cached.is_fresh(), "the first run succeeded");
|
|
|
|
ok.store(false, Ordering::SeqCst);
|
|
// The last success was at 0 s. Two failed runs (30 s, 60 s) are a blip.
|
|
tokio::time::sleep(Duration::from_secs(70)).await;
|
|
assert!(cached.is_fresh(), "within three periods of the success");
|
|
tokio::time::sleep(Duration::from_secs(30)).await;
|
|
assert!(!cached.is_fresh(), "100 s since the last success");
|
|
|
|
ok.store(true, Ordering::SeqCst);
|
|
tokio::time::sleep(EVERY).await;
|
|
assert!(cached.is_fresh(), "a success makes it ready again");
|
|
}
|
|
|
|
/// The probe reads memory: a thousand probes cost no run of the check.
|
|
#[tokio::test(start_paused = true)]
|
|
async fn probes_do_not_run_the_check() {
|
|
let (ok, runs) = (Arc::new(AtomicBool::new(true)), Arc::new(AtomicU64::new(0)));
|
|
let cached = CachedCheck::spawn(EVERY, FRESH, counted_check(&ok, &runs));
|
|
tokio::time::sleep(Duration::from_secs(1)).await;
|
|
let before = runs.load(Ordering::SeqCst);
|
|
for _ in 0..1000 {
|
|
assert!(cached.is_fresh());
|
|
}
|
|
assert_eq!(runs.load(Ordering::SeqCst), before);
|
|
tokio::time::sleep(EVERY * 4).await;
|
|
assert!(
|
|
runs.load(Ordering::SeqCst) <= before + 5,
|
|
"one run a period"
|
|
);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn a_check_that_hangs_is_cut_off_and_counts_as_failed() {
|
|
let cached = CachedCheck::spawn(EVERY, FRESH, || async {
|
|
std::future::pending::<()>().await;
|
|
Ok::<(), String>(())
|
|
});
|
|
tokio::time::sleep(EVERY * 5).await;
|
|
assert!(!cached.is_fresh());
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn dropping_the_last_clone_ends_the_checking() {
|
|
let (ok, runs) = (Arc::new(AtomicBool::new(true)), Arc::new(AtomicU64::new(0)));
|
|
let cached = CachedCheck::spawn(EVERY, FRESH, counted_check(&ok, &runs));
|
|
tokio::time::sleep(Duration::from_secs(1)).await;
|
|
drop(cached);
|
|
tokio::time::sleep(EVERY * 2).await;
|
|
let after_drop = runs.load(Ordering::SeqCst);
|
|
tokio::time::sleep(EVERY * 10).await;
|
|
assert_eq!(runs.load(Ordering::SeqCst), after_drop);
|
|
}
|
|
|
|
/// `/readyz` end to end: 503 until the first check succeeds, 200 after.
|
|
#[tokio::test]
|
|
async fn readyz_follows_the_cached_check() {
|
|
let runs = Arc::new(AtomicU64::new(0));
|
|
let mut served = Vec::new();
|
|
for (answers, expected) in [(false, 503), (true, 200)] {
|
|
let health = Health::new(STALE_AFTER);
|
|
let ok = Arc::new(AtomicBool::new(answers));
|
|
health.ready_while_checked(counted_check(&ok, &runs));
|
|
let (addr, stop, task) = listening(&health).await;
|
|
let mut got = 0;
|
|
for _ in 0..100 {
|
|
got = get_status(addr, "/readyz").await;
|
|
if got == expected {
|
|
break;
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(20)).await;
|
|
}
|
|
assert_eq!(got, expected, "a check that answers {answers}");
|
|
served.push((stop, task));
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn addr_takes_the_flag_else_the_variable_and_names_a_bad_one() {
|
|
let flag: SocketAddr = "127.0.0.1:9".parse().unwrap();
|
|
assert_eq!(addr(Some(flag), "HEALTH_TEST_UNSET").unwrap(), Some(flag));
|
|
assert_eq!(addr(None, "HEALTH_TEST_UNSET").unwrap(), None);
|
|
assert_eq!(parse("V", "127.0.0.1:9".into()).unwrap(), flag);
|
|
let bad = parse("SWARM_X_HEALTH_ADDR", "nonsense".into()).unwrap_err();
|
|
assert!(bad.to_string().contains("SWARM_X_HEALTH_ADDR"), "{bad}");
|
|
}
|
|
}
|