readiness follows a cached store check instead of a constant or a login per probe
This commit is contained in:
parent
2eacb83b21
commit
b0239914b8
1 changed files with 369 additions and 40 deletions
|
|
@ -13,7 +13,10 @@
|
|||
//! own) calls [`Health::beat_on_runtime`] instead, which shows the daemon's
|
||||
//! runtime still schedules tasks, not that the accept loop is healthy.
|
||||
//! - `GET /readyz` is 200 while the condition set with [`Health::ready_when`]
|
||||
//! holds, 503 while it does not or none is set yet.
|
||||
//! holds, 503 while it does not or none is set yet. A daemon whose readiness
|
||||
//! is "its store answers" uses [`Health::ready_while_checked`]: a task checks
|
||||
//! every 30 s off the probe path and the probe reads the last result, ready
|
||||
//! for 90 s after a success.
|
||||
//!
|
||||
//! The answers carry a status code and no body. Nothing listens unless the
|
||||
//! operator names an address (see [`Health::start`]), so a daemon that is not
|
||||
|
|
@ -46,6 +49,99 @@ pub const STALE_AFTER: Duration = Duration::from_secs(30);
|
|||
/// the daemon is alive and not ready.
|
||||
const NO_BEAT: u64 = u64::MAX;
|
||||
|
||||
/// How often a [`CachedCheck`] made with [`Health::ready_while_checked`] runs.
|
||||
pub const CHECK_EVERY: Duration = Duration::from_secs(30);
|
||||
|
||||
/// How long after its last success a check still counts: three periods, so a
|
||||
/// single failed check does not flap readiness.
|
||||
pub const FRESH_FOR: Duration = Duration::from_secs(90);
|
||||
|
||||
/// `last_ok` before the check has first succeeded.
|
||||
const NEVER: u64 = u64::MAX;
|
||||
|
||||
struct CheckState {
|
||||
born: Instant,
|
||||
/// Milliseconds after `born` of the last success, or [`NEVER`].
|
||||
last_ok: AtomicU64,
|
||||
fresh_for: Duration,
|
||||
}
|
||||
|
||||
/// The result of a check that runs off the probe path: a background task runs
|
||||
/// it every period and records when it last succeeded, and [`is_fresh`] reads
|
||||
/// that record without any I/O. The task ends when the last clone is dropped.
|
||||
///
|
||||
/// [`is_fresh`]: CachedCheck::is_fresh
|
||||
#[derive(Clone)]
|
||||
pub struct CachedCheck(Arc<CheckState>);
|
||||
|
||||
impl CachedCheck {
|
||||
/// Runs `check` now and then every `every`, each run bounded by `every`. A
|
||||
/// success keeps [`is_fresh`](Self::is_fresh) true for `fresh_for`; a failure
|
||||
/// is logged as a warning and changes nothing. Before the first success
|
||||
/// there is no freshness.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Outside a tokio runtime.
|
||||
pub fn spawn<F, Fut, E>(every: Duration, fresh_for: Duration, check: F) -> Self
|
||||
where
|
||||
F: Fn() -> Fut + Send + 'static,
|
||||
Fut: Future<Output = Result<(), E>> + Send,
|
||||
E: std::fmt::Display + Send,
|
||||
{
|
||||
let state = Arc::new(CheckState {
|
||||
born: Instant::now(),
|
||||
last_ok: AtomicU64::new(NEVER),
|
||||
fresh_for,
|
||||
});
|
||||
let task = Arc::downgrade(&state);
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let outcome = tokio::time::timeout(every, check()).await;
|
||||
let Some(state) = task.upgrade() else { return };
|
||||
match outcome {
|
||||
Ok(Ok(())) => state.last_ok.store(
|
||||
u64::try_from(state.born.elapsed().as_millis()).unwrap_or(u64::MAX),
|
||||
Ordering::Relaxed,
|
||||
),
|
||||
Ok(Err(e)) => {
|
||||
tracing::warn!(error = %format!("{e:#}"), "readiness check failed");
|
||||
}
|
||||
Err(_) => tracing::warn!("readiness check timed out"),
|
||||
}
|
||||
drop(state);
|
||||
tokio::time::sleep(every).await;
|
||||
}
|
||||
});
|
||||
Self(state)
|
||||
}
|
||||
|
||||
/// [`CachedCheck::spawn`] at [`CHECK_EVERY`] and [`FRESH_FOR`].
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Outside a tokio runtime.
|
||||
pub fn every_30s<F, Fut, E>(check: F) -> Self
|
||||
where
|
||||
F: Fn() -> Fut + Send + 'static,
|
||||
Fut: Future<Output = Result<(), E>> + Send,
|
||||
E: std::fmt::Display + Send,
|
||||
{
|
||||
Self::spawn(CHECK_EVERY, FRESH_FOR, check)
|
||||
}
|
||||
|
||||
/// Whether the check has succeeded within `fresh_for`. Reads memory only.
|
||||
#[must_use]
|
||||
pub fn is_fresh(&self) -> bool {
|
||||
let last = self.0.last_ok.load(Ordering::Relaxed);
|
||||
if last == NEVER {
|
||||
return false;
|
||||
}
|
||||
let now = u64::try_from(self.0.born.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
Duration::from_millis(now.saturating_sub(last)) <= self.0.fresh_for
|
||||
}
|
||||
}
|
||||
|
||||
type Condition = Box<dyn Fn() -> bool + Send + Sync>;
|
||||
|
||||
struct Inner {
|
||||
|
|
@ -92,6 +188,24 @@ impl Health {
|
|||
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Box::new(condition));
|
||||
}
|
||||
|
||||
/// Makes `/readyz` follow `check`, run every [`CHECK_EVERY`] off the probe
|
||||
/// path (see [`CachedCheck`]): ready while it has succeeded within
|
||||
/// [`FRESH_FOR`], not ready before its first success. Replaces an earlier
|
||||
/// condition.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Outside a tokio runtime.
|
||||
pub fn ready_while_checked<F, Fut, E>(&self, check: F)
|
||||
where
|
||||
F: Fn() -> Fut + Send + 'static,
|
||||
Fut: Future<Output = Result<(), E>> + Send,
|
||||
E: std::fmt::Display + Send,
|
||||
{
|
||||
let cached = CachedCheck::every_30s(check);
|
||||
self.ready_when(move || cached.is_fresh());
|
||||
}
|
||||
|
||||
/// Whether the main loop has beaten within `stale_after`.
|
||||
#[must_use]
|
||||
pub fn is_alive(&self) -> bool {
|
||||
|
|
@ -445,6 +559,119 @@ mod tests {
|
|||
futures_util::stream::poll_fn(move |cx| rx.poll_recv(cx)).boxed()
|
||||
}
|
||||
|
||||
/// A check that answers from `ok` and counts its runs.
|
||||
fn counted_check(
|
||||
ok: &Arc<AtomicBool>,
|
||||
runs: &Arc<AtomicU64>,
|
||||
) -> impl Fn() -> std::future::Ready<Result<(), String>> + Send + 'static {
|
||||
let (ok, runs) = (Arc::clone(ok), Arc::clone(runs));
|
||||
move || {
|
||||
runs.fetch_add(1, Ordering::SeqCst);
|
||||
std::future::ready(if ok.load(Ordering::SeqCst) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err("the store did not answer".to_owned())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const EVERY: Duration = Duration::from_secs(30);
|
||||
const FRESH: Duration = Duration::from_secs(90);
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_check_that_never_succeeded_is_not_fresh() {
|
||||
let (ok, runs) = (
|
||||
Arc::new(AtomicBool::new(false)),
|
||||
Arc::new(AtomicU64::new(0)),
|
||||
);
|
||||
let cached = CachedCheck::spawn(EVERY, FRESH, counted_check(&ok, &runs));
|
||||
tokio::time::sleep(EVERY * 10).await;
|
||||
assert!(runs.load(Ordering::SeqCst) >= 10, "it kept trying");
|
||||
assert!(!cached.is_fresh());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_fresh_success_is_ready_and_a_stale_one_is_not() {
|
||||
let (ok, runs) = (Arc::new(AtomicBool::new(true)), Arc::new(AtomicU64::new(0)));
|
||||
let cached = CachedCheck::spawn(EVERY, FRESH, counted_check(&ok, &runs));
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
assert!(cached.is_fresh(), "the first run succeeded");
|
||||
|
||||
ok.store(false, Ordering::SeqCst);
|
||||
// The last success was at 0 s. Two failed runs (30 s, 60 s) are a blip.
|
||||
tokio::time::sleep(Duration::from_secs(70)).await;
|
||||
assert!(cached.is_fresh(), "within three periods of the success");
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
assert!(!cached.is_fresh(), "100 s since the last success");
|
||||
|
||||
ok.store(true, Ordering::SeqCst);
|
||||
tokio::time::sleep(EVERY).await;
|
||||
assert!(cached.is_fresh(), "a success makes it ready again");
|
||||
}
|
||||
|
||||
/// The probe reads memory: a thousand probes cost no run of the check.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn probes_do_not_run_the_check() {
|
||||
let (ok, runs) = (Arc::new(AtomicBool::new(true)), Arc::new(AtomicU64::new(0)));
|
||||
let cached = CachedCheck::spawn(EVERY, FRESH, counted_check(&ok, &runs));
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
let before = runs.load(Ordering::SeqCst);
|
||||
for _ in 0..1000 {
|
||||
assert!(cached.is_fresh());
|
||||
}
|
||||
assert_eq!(runs.load(Ordering::SeqCst), before);
|
||||
tokio::time::sleep(EVERY * 4).await;
|
||||
assert!(
|
||||
runs.load(Ordering::SeqCst) <= before + 5,
|
||||
"one run a period"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_check_that_hangs_is_cut_off_and_counts_as_failed() {
|
||||
let cached = CachedCheck::spawn(EVERY, FRESH, || async {
|
||||
std::future::pending::<()>().await;
|
||||
Ok::<(), String>(())
|
||||
});
|
||||
tokio::time::sleep(EVERY * 5).await;
|
||||
assert!(!cached.is_fresh());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn dropping_the_last_clone_ends_the_checking() {
|
||||
let (ok, runs) = (Arc::new(AtomicBool::new(true)), Arc::new(AtomicU64::new(0)));
|
||||
let cached = CachedCheck::spawn(EVERY, FRESH, counted_check(&ok, &runs));
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
drop(cached);
|
||||
tokio::time::sleep(EVERY * 2).await;
|
||||
let after_drop = runs.load(Ordering::SeqCst);
|
||||
tokio::time::sleep(EVERY * 10).await;
|
||||
assert_eq!(runs.load(Ordering::SeqCst), after_drop);
|
||||
}
|
||||
|
||||
/// `/readyz` end to end: 503 until the first check succeeds, 200 after.
|
||||
#[tokio::test]
|
||||
async fn readyz_follows_the_cached_check() {
|
||||
let runs = Arc::new(AtomicU64::new(0));
|
||||
let mut served = Vec::new();
|
||||
for (answers, expected) in [(false, 503), (true, 200)] {
|
||||
let health = Health::new(STALE_AFTER);
|
||||
let ok = Arc::new(AtomicBool::new(answers));
|
||||
health.ready_while_checked(counted_check(&ok, &runs));
|
||||
let (addr, stop, task) = listening(&health).await;
|
||||
let mut got = 0;
|
||||
for _ in 0..100 {
|
||||
got = get_status(addr, "/readyz").await;
|
||||
if got == expected {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
assert_eq!(got, expected, "a check that answers {answers}");
|
||||
served.push((stop, task));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn addr_takes_the_flag_else_the_variable_and_names_a_bad_one() {
|
||||
let flag: SocketAddr = "127.0.0.1:9".parse().unwrap();
|
||||
|
|
|
|||
Loading…
Reference in a new issue