swarm-secret-client: add shutdown_signal, serve_bounded and InFlight
This commit is contained in:
parent
e0002095ba
commit
39c29b64e3
3 changed files with 213 additions and 0 deletions
|
|
@ -47,6 +47,8 @@ workspace = true
|
|||
# provider compiled in rustls picks it without an install, so the provider
|
||||
# test would pass with `install_crypto_provider` doing nothing.
|
||||
[dev-dependencies]
|
||||
# `raise`, to send the test process a real SIGTERM.
|
||||
libc.workspace = true
|
||||
# `InMemoryMetricExporter`, to read back what `cert_metrics` exports.
|
||||
opentelemetry_sdk = { workspace = true, features = ["testing"] }
|
||||
rustls = { workspace = true, features = ["ring"] }
|
||||
|
|
|
|||
|
|
@ -55,10 +55,12 @@ pub mod policy;
|
|||
pub mod proxy;
|
||||
pub mod queue;
|
||||
pub mod review_classifier;
|
||||
pub mod shutdown;
|
||||
pub mod startup_read;
|
||||
pub mod stored_secret;
|
||||
|
||||
pub use client::SecretStore;
|
||||
pub use shutdown::{DRAIN, InFlight, Shutdown, serve_bounded, shutdown_signal};
|
||||
pub use startup_read::read_with_startup_retry;
|
||||
|
||||
/// What can go wrong between "we have a client certificate" and "we have the
|
||||
|
|
|
|||
206
swarm-secret-client/src/shutdown.rs
Normal file
206
swarm-secret-client/src/shutdown.rs
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
//! Stopping a daemon when its unit is stopped.
|
||||
//!
|
||||
//! `systemctl stop` and a container runtime both send SIGTERM. A process with
|
||||
//! no handler dies on the default action, mid-request; one that waits on
|
||||
//! [`shutdown_signal`] finishes what it has in hand and exits 0. Every swarm
|
||||
//! daemon's `main` shares this one definition of "told to stop" and of how
|
||||
//! long it then gets ([`DRAIN`]).
|
||||
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures_util::FutureExt as _;
|
||||
use futures_util::future::{BoxFuture, Shared};
|
||||
use tokio::signal::unix::{SignalKind, signal};
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
/// How long work in hand may take once the signal has arrived. Well under
|
||||
/// systemd's default `TimeoutStopSec` (90 s) and a pod's 30 s grace period, so
|
||||
/// the process exits 0 before either escalates to SIGKILL.
|
||||
pub const DRAIN: Duration = Duration::from_secs(10);
|
||||
|
||||
/// What [`shutdown_signal`] returns: a future that is `Clone`, and whose clones
|
||||
/// all resolve on the one signal.
|
||||
pub type Shutdown = Shared<BoxFuture<'static, ()>>;
|
||||
|
||||
/// Resolves when the process receives SIGTERM or SIGINT.
|
||||
///
|
||||
/// The handlers are installed when this is called, not when the future is
|
||||
/// first polled, so call it early in `main`: a signal arriving between the
|
||||
/// call and the first poll is still seen, and one before the call kills the
|
||||
/// process as before. Clones of the [`Shutdown`] it returns all resolve on the
|
||||
/// one signal, so a server, a message loop and a bound can all wait on it.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// When the runtime is not running, or the kernel refuses a handler.
|
||||
pub fn shutdown_signal() -> Shutdown {
|
||||
let mut term = signal(SignalKind::terminate()).expect("installing the SIGTERM handler");
|
||||
let mut int = signal(SignalKind::interrupt()).expect("installing the SIGINT handler");
|
||||
async move {
|
||||
tokio::select! {
|
||||
_ = term.recv() => tracing::info!("SIGTERM received; shutting down"),
|
||||
_ = int.recv() => tracing::info!("SIGINT received; shutting down"),
|
||||
}
|
||||
}
|
||||
.boxed()
|
||||
.shared()
|
||||
}
|
||||
|
||||
/// Runs `serving`, a server built with `with_graceful_shutdown` on a clone of
|
||||
/// `stop`, and stops waiting for it [`DRAIN`] after `stop` resolves. `None`
|
||||
/// means connections were still open then.
|
||||
///
|
||||
/// A graceful shutdown waits for every connection to close, and a streaming
|
||||
/// response never does on its own; without this bound such a daemon sits out
|
||||
/// the unit's stop timeout and is killed.
|
||||
pub async fn serve_bounded<T>(
|
||||
stop: impl Future<Output = ()>,
|
||||
serving: impl Future<Output = T>,
|
||||
) -> Option<T> {
|
||||
tokio::pin!(serving);
|
||||
tokio::select! {
|
||||
out = &mut serving => Some(out),
|
||||
() = async {
|
||||
stop.await;
|
||||
tokio::time::sleep(DRAIN).await;
|
||||
} => {
|
||||
tracing::warn!(drain = ?DRAIN, "still running after the drain; exiting anyway");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The handlers a daemon has spawned for messages it already took.
|
||||
///
|
||||
/// After the signal the daemon takes no more, calls [`InFlight::drain`], and
|
||||
/// whatever is still running then is abandoned. Nothing is acknowledged on
|
||||
/// its behalf, so an unfinished message is redelivered or times out at its
|
||||
/// sender exactly as it would after a kill.
|
||||
#[derive(Default)]
|
||||
pub struct InFlight(JoinSet<()>);
|
||||
|
||||
impl InFlight {
|
||||
/// An empty set.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Runs `handler` on its own task. Handlers that already finished are
|
||||
/// dropped first, so the set stays as small as the work in hand.
|
||||
pub fn spawn(&mut self, handler: impl Future<Output = ()> + Send + 'static) {
|
||||
while self.0.try_join_next().is_some() {}
|
||||
self.0.spawn(handler);
|
||||
}
|
||||
|
||||
/// Waits up to [`DRAIN`] for the handlers still running.
|
||||
pub async fn drain(mut self) {
|
||||
let finished =
|
||||
tokio::time::timeout(DRAIN, async { while self.0.join_next().await.is_some() {} })
|
||||
.await;
|
||||
if finished.is_err() {
|
||||
tracing::warn!(
|
||||
abandoned = self.0.len(),
|
||||
drain = ?DRAIN,
|
||||
"handlers still running after the drain; abandoning them"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use tokio::sync::Notify;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// A real SIGTERM, then a real SIGINT, raised at this process: the handlers
|
||||
/// are installed by the call, so the signals are consumed rather than
|
||||
/// killing the test binary. One test, so the two raises never overlap.
|
||||
#[tokio::test]
|
||||
async fn resolves_on_a_sigterm_and_on_a_sigint_raised_at_the_process() {
|
||||
let stop = shutdown_signal();
|
||||
let mut stop = std::pin::pin!(stop);
|
||||
assert!(
|
||||
stop.as_mut().now_or_never().is_none(),
|
||||
"resolved with no signal"
|
||||
);
|
||||
// SAFETY: `raise` takes a signal number and touches no memory.
|
||||
assert_eq!(unsafe { libc::raise(libc::SIGTERM) }, 0);
|
||||
tokio::time::timeout(Duration::from_secs(5), stop)
|
||||
.await
|
||||
.expect("SIGTERM resolves the future");
|
||||
|
||||
let stop = shutdown_signal();
|
||||
// SAFETY: as above.
|
||||
assert_eq!(unsafe { libc::raise(libc::SIGINT) }, 0);
|
||||
tokio::time::timeout(Duration::from_secs(5), stop)
|
||||
.await
|
||||
.expect("SIGINT resolves the future");
|
||||
}
|
||||
|
||||
/// A server that finishes by itself is returned from at once, and its
|
||||
/// output is the caller's.
|
||||
#[tokio::test]
|
||||
async fn serve_bounded_returns_what_the_server_returns() {
|
||||
let out = serve_bounded(std::future::pending(), async { 7 }).await;
|
||||
assert_eq!(out, Some(7));
|
||||
}
|
||||
|
||||
/// After `stop`, a server that never ends is given up on once [`DRAIN`]
|
||||
/// has passed, and not before.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn serve_bounded_gives_up_only_after_the_drain() {
|
||||
let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let bounded = tokio::spawn(serve_bounded(
|
||||
async { stop_rx.await.unwrap() },
|
||||
std::future::pending::<()>(),
|
||||
));
|
||||
tokio::time::sleep(DRAIN * 3).await;
|
||||
assert!(!bounded.is_finished(), "gave up with no stop");
|
||||
stop_tx.send(()).unwrap();
|
||||
tokio::time::sleep(DRAIN / 2).await;
|
||||
assert!(!bounded.is_finished(), "gave up inside the drain");
|
||||
tokio::time::sleep(DRAIN / 2 + Duration::from_secs(1)).await;
|
||||
assert!(bounded.is_finished(), "still waiting past the drain");
|
||||
assert_eq!(bounded.await.unwrap(), None);
|
||||
}
|
||||
|
||||
/// A handler that finishes inside the drain is waited for.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn drain_waits_for_a_handler_that_finishes_in_time() {
|
||||
let done = Arc::new(AtomicBool::new(false));
|
||||
let mut in_flight = InFlight::new();
|
||||
let flag = Arc::clone(&done);
|
||||
in_flight.spawn(async move {
|
||||
tokio::time::sleep(DRAIN / 2).await;
|
||||
flag.store(true, Ordering::SeqCst);
|
||||
});
|
||||
in_flight.drain().await;
|
||||
assert!(done.load(Ordering::SeqCst));
|
||||
}
|
||||
|
||||
/// A handler that outlives the drain is abandoned: the drain returns at
|
||||
/// the bound with the handler unfinished.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn drain_abandons_a_handler_that_outlives_it() {
|
||||
let done = Arc::new(AtomicBool::new(false));
|
||||
let started = Arc::new(Notify::new());
|
||||
let mut in_flight = InFlight::new();
|
||||
let (flag, up) = (Arc::clone(&done), Arc::clone(&started));
|
||||
in_flight.spawn(async move {
|
||||
up.notify_one();
|
||||
tokio::time::sleep(DRAIN * 5).await;
|
||||
flag.store(true, Ordering::SeqCst);
|
||||
});
|
||||
started.notified().await;
|
||||
let before = tokio::time::Instant::now();
|
||||
in_flight.drain().await;
|
||||
assert_eq!(before.elapsed(), DRAIN);
|
||||
assert!(!done.load(Ordering::SeqCst));
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue