swarm daemons exit 0 when told to stop while still waiting for the store at startup
This commit is contained in:
parent
5dd39ba88c
commit
ae37c228b1
2 changed files with 133 additions and 32 deletions
|
|
@ -62,7 +62,7 @@ pub mod stored_secret;
|
|||
|
||||
pub use client::SecretStore;
|
||||
pub use shutdown::{DRAIN, InFlight, Shutdown, serve_bounded, shutdown_signal};
|
||||
pub use startup_read::read_with_startup_retry;
|
||||
pub use startup_read::{read_with_startup_retry, read_with_startup_retry_or_stop};
|
||||
|
||||
/// What can go wrong between "we have a client certificate" and "we have the
|
||||
/// credential".
|
||||
|
|
|
|||
|
|
@ -25,6 +25,9 @@ const RETRIES: usize = 6;
|
|||
/// with the first success or the last attempt's error. Each failure is logged
|
||||
/// as a warning naming `what`.
|
||||
///
|
||||
/// This read doesn't hear a stop signal: a caller that already has one in hand
|
||||
/// uses [`read_with_startup_retry_or_stop`].
|
||||
///
|
||||
/// # Errors
|
||||
/// The error of the final attempt, once every read has failed.
|
||||
pub async fn read_with_startup_retry<T, E, F, Fut>(what: &str, read: F) -> Result<T, E>
|
||||
|
|
@ -45,6 +48,36 @@ where
|
|||
.await
|
||||
}
|
||||
|
||||
/// [`read_with_startup_retry`], ended by `stop`.
|
||||
///
|
||||
/// When `stop` resolves, in a sleep or during an attempt, the read is dropped
|
||||
/// without another attempt and the answer is `Ok(None)`. The caller exits 0, as
|
||||
/// it does for the same signal later on, so `systemctl stop` or a pod delete
|
||||
/// during a store outage returns at once instead of after the schedule (about a
|
||||
/// minute) or the unit's stop timeout.
|
||||
///
|
||||
/// # Errors
|
||||
/// The error of the final attempt, once every read has failed.
|
||||
pub async fn read_with_startup_retry_or_stop<T, E, F, Fut>(
|
||||
what: &str,
|
||||
stop: impl Future<Output = ()>,
|
||||
read: F,
|
||||
) -> Result<Option<T>, E>
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = Result<T, E>>,
|
||||
E: Display,
|
||||
{
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = stop => {
|
||||
tracing::info!("stop arrived while {what}; exiting before startup finished");
|
||||
Ok(None)
|
||||
}
|
||||
read = read_with_startup_retry(what, read) => read.map(Some),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
|
@ -85,6 +118,48 @@ mod tests {
|
|||
assert_eq!(started.elapsed(), Duration::from_secs(1 + 2 + 4));
|
||||
}
|
||||
|
||||
/// Resolves `after` from now, like a SIGTERM that arrives mid-outage.
|
||||
async fn stop_after(after: Duration) {
|
||||
tokio::time::sleep(after).await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_stop_during_a_retry_wait_ends_the_read_without_another_attempt() {
|
||||
// Attempts at 0 s and 1 s fail; the wait after the second runs 1 s to 3 s.
|
||||
let calls = AtomicUsize::new(0);
|
||||
let started = Instant::now();
|
||||
let got =
|
||||
read_with_startup_retry_or_stop("reading", stop_after(Duration::from_secs(2)), || {
|
||||
read_after(usize::MAX, &calls)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(got, Ok(None));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(started.elapsed(), Duration::from_secs(2));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_stop_that_has_already_fired_makes_no_attempt() {
|
||||
let calls = AtomicUsize::new(0);
|
||||
let got = read_with_startup_retry_or_stop("reading", std::future::ready(()), || {
|
||||
read_after(0, &calls)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(got, Ok(None));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_stop_after_the_read_succeeded_changes_nothing() {
|
||||
let calls = AtomicUsize::new(0);
|
||||
let got =
|
||||
read_with_startup_retry_or_stop("reading", stop_after(Duration::from_mins(1)), || {
|
||||
read_after(1, &calls)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(got, Ok(Some("secret")));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_store_that_stays_down_fails_after_the_whole_schedule() {
|
||||
let calls = AtomicUsize::new(0);
|
||||
|
|
|
|||
Loading…
Reference in a new issue