Watch
0
0
Fork
You've already forked hyperhive
0
hyperhive/swarm-secret-client/src/shutdown.rs

206 lines
7.8 KiB
Rust

//! Stopping a daemon when its unit is stopped.
//!
//! `systemctl stop` and a container runtime both send SIGTERM. A process with
//! no handler dies on the default action, mid-request; one that waits on
//! [`shutdown_signal`] finishes what it has in hand and exits 0. Every swarm
//! daemon's `main` shares this one definition of "told to stop" and of how
//! long it then gets ([`DRAIN`]).
use std::future::Future;
use std::time::Duration;
use futures_util::FutureExt as _;
use futures_util::future::{BoxFuture, Shared};
use tokio::signal::unix::{SignalKind, signal};
use tokio::task::JoinSet;
/// How long work in hand may take once the signal has arrived. Well under
/// systemd's default `TimeoutStopSec` (90 s) and a pod's 30 s grace period, so
/// the process exits 0 before either escalates to SIGKILL.
pub const DRAIN: Duration = Duration::from_secs(10);
/// What [`shutdown_signal`] returns: a future that is `Clone`, and whose clones
/// all resolve on the one signal.
pub type Shutdown = Shared<BoxFuture<'static, ()>>;
/// Resolves when the process receives SIGTERM or SIGINT.
///
/// The handlers are installed when this is called, not when the future is
/// first polled, so call it early in `main`: a signal arriving between the
/// call and the first poll is still seen, and one before the call kills the
/// process as before. Clones of the [`Shutdown`] it returns all resolve on the
/// one signal, so a server, a message loop and a bound can all wait on it.
///
/// # Panics
///
/// When the runtime is not running, or the kernel refuses a handler.
pub fn shutdown_signal() -> Shutdown {
let mut term = signal(SignalKind::terminate()).expect("installing the SIGTERM handler");
let mut int = signal(SignalKind::interrupt()).expect("installing the SIGINT handler");
async move {
tokio::select! {
_ = term.recv() => tracing::info!("SIGTERM received; shutting down"),
_ = int.recv() => tracing::info!("SIGINT received; shutting down"),
}
}
.boxed()
.shared()
}
/// Runs `serving`, a server built with `with_graceful_shutdown` on a clone of
/// `stop`, and stops waiting for it [`DRAIN`] after `stop` resolves. `None`
/// means connections were still open then.
///
/// A graceful shutdown waits for every connection to close, and a streaming
/// response never does on its own; without this bound such a daemon sits out
/// the unit's stop timeout and is killed.
pub async fn serve_bounded<T>(
stop: impl Future<Output = ()>,
serving: impl Future<Output = T>,
) -> Option<T> {
tokio::pin!(serving);
tokio::select! {
out = &mut serving => Some(out),
() = async {
stop.await;
tokio::time::sleep(DRAIN).await;
} => {
tracing::warn!(drain = ?DRAIN, "still running after the drain; exiting anyway");
None
}
}
}
/// The handlers a daemon has spawned for messages it already took.
///
/// After the signal the daemon takes no more, calls [`InFlight::drain`], and
/// whatever is still running then is abandoned. Nothing is acknowledged on
/// its behalf, so an unfinished message is redelivered or times out at its
/// sender exactly as it would after a kill.
#[derive(Default)]
pub struct InFlight(JoinSet<()>);
impl InFlight {
/// An empty set.
#[must_use]
pub fn new() -> Self {
Self::default()
}
/// Runs `handler` on its own task. Handlers that already finished are
/// dropped first, so the set stays as small as the work in hand.
pub fn spawn(&mut self, handler: impl Future<Output = ()> + Send + 'static) {
while self.0.try_join_next().is_some() {}
self.0.spawn(handler);
}
/// Waits up to [`DRAIN`] for the handlers still running.
pub async fn drain(mut self) {
let finished =
tokio::time::timeout(DRAIN, async { while self.0.join_next().await.is_some() {} })
.await;
if finished.is_err() {
tracing::warn!(
abandoned = self.0.len(),
drain = ?DRAIN,
"handlers still running after the drain; abandoning them"
);
}
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use tokio::sync::Notify;
use super::*;
/// A real SIGTERM, then a real SIGINT, raised at this process: the handlers
/// are installed by the call, so the signals are consumed rather than
/// killing the test binary. One test, so the two raises never overlap.
#[tokio::test]
async fn resolves_on_a_sigterm_and_on_a_sigint_raised_at_the_process() {
let stop = shutdown_signal();
let mut stop = std::pin::pin!(stop);
assert!(
stop.as_mut().now_or_never().is_none(),
"resolved with no signal"
);
// SAFETY: `raise` takes a signal number and touches no memory.
assert_eq!(unsafe { libc::raise(libc::SIGTERM) }, 0);
tokio::time::timeout(Duration::from_secs(5), stop)
.await
.expect("SIGTERM resolves the future");
let stop = shutdown_signal();
// SAFETY: as above.
assert_eq!(unsafe { libc::raise(libc::SIGINT) }, 0);
tokio::time::timeout(Duration::from_secs(5), stop)
.await
.expect("SIGINT resolves the future");
}
/// A server that finishes by itself is returned from at once, and its
/// output is the caller's.
#[tokio::test]
async fn serve_bounded_returns_what_the_server_returns() {
let out = serve_bounded(std::future::pending(), async { 7 }).await;
assert_eq!(out, Some(7));
}
/// After `stop`, a server that never ends is given up on once [`DRAIN`]
/// has passed, and not before.
#[tokio::test(start_paused = true)]
async fn serve_bounded_gives_up_only_after_the_drain() {
let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>();
let bounded = tokio::spawn(serve_bounded(
async { stop_rx.await.unwrap() },
std::future::pending::<()>(),
));
tokio::time::sleep(DRAIN * 3).await;
assert!(!bounded.is_finished(), "gave up with no stop");
stop_tx.send(()).unwrap();
tokio::time::sleep(DRAIN / 2).await;
assert!(!bounded.is_finished(), "gave up inside the drain");
tokio::time::sleep(DRAIN / 2 + Duration::from_secs(1)).await;
assert!(bounded.is_finished(), "still waiting past the drain");
assert_eq!(bounded.await.unwrap(), None);
}
/// A handler that finishes inside the drain is waited for.
#[tokio::test(start_paused = true)]
async fn drain_waits_for_a_handler_that_finishes_in_time() {
let done = Arc::new(AtomicBool::new(false));
let mut in_flight = InFlight::new();
let flag = Arc::clone(&done);
in_flight.spawn(async move {
tokio::time::sleep(DRAIN / 2).await;
flag.store(true, Ordering::SeqCst);
});
in_flight.drain().await;
assert!(done.load(Ordering::SeqCst));
}
/// A handler that outlives the drain is abandoned: the drain returns at
/// the bound with the handler unfinished.
#[tokio::test(start_paused = true)]
async fn drain_abandons_a_handler_that_outlives_it() {
let done = Arc::new(AtomicBool::new(false));
let started = Arc::new(Notify::new());
let mut in_flight = InFlight::new();
let (flag, up) = (Arc::clone(&done), Arc::clone(&started));
in_flight.spawn(async move {
up.notify_one();
tokio::time::sleep(DRAIN * 5).await;
flag.store(true, Ordering::SeqCst);
});
started.notified().await;
let before = tokio::time::Instant::now();
in_flight.drain().await;
assert_eq!(before.elapsed(), DRAIN);
assert!(!done.load(Ordering::SeqCst));
}
}