206 lines
7.8 KiB
Rust
206 lines
7.8 KiB
Rust
//! Stopping a daemon when its unit is stopped.
|
|
//!
|
|
//! `systemctl stop` and a container runtime both send SIGTERM. A process with
|
|
//! no handler dies on the default action, mid-request; one that waits on
|
|
//! [`shutdown_signal`] finishes what it has in hand and exits 0. Every swarm
|
|
//! daemon's `main` shares this one definition of "told to stop" and of how
|
|
//! long it then gets ([`DRAIN`]).
|
|
|
|
use std::future::Future;
|
|
use std::time::Duration;
|
|
|
|
use futures_util::FutureExt as _;
|
|
use futures_util::future::{BoxFuture, Shared};
|
|
use tokio::signal::unix::{SignalKind, signal};
|
|
use tokio::task::JoinSet;
|
|
|
|
/// How long work in hand may take once the signal has arrived. Well under
|
|
/// systemd's default `TimeoutStopSec` (90 s) and a pod's 30 s grace period, so
|
|
/// the process exits 0 before either escalates to SIGKILL.
|
|
pub const DRAIN: Duration = Duration::from_secs(10);
|
|
|
|
/// What [`shutdown_signal`] returns: a future that is `Clone`, and whose clones
|
|
/// all resolve on the one signal.
|
|
pub type Shutdown = Shared<BoxFuture<'static, ()>>;
|
|
|
|
/// Resolves when the process receives SIGTERM or SIGINT.
|
|
///
|
|
/// The handlers are installed when this is called, not when the future is
|
|
/// first polled, so call it early in `main`: a signal arriving between the
|
|
/// call and the first poll is still seen, and one before the call kills the
|
|
/// process as before. Clones of the [`Shutdown`] it returns all resolve on the
|
|
/// one signal, so a server, a message loop and a bound can all wait on it.
|
|
///
|
|
/// # Panics
|
|
///
|
|
/// When the runtime is not running, or the kernel refuses a handler.
|
|
pub fn shutdown_signal() -> Shutdown {
|
|
let mut term = signal(SignalKind::terminate()).expect("installing the SIGTERM handler");
|
|
let mut int = signal(SignalKind::interrupt()).expect("installing the SIGINT handler");
|
|
async move {
|
|
tokio::select! {
|
|
_ = term.recv() => tracing::info!("SIGTERM received; shutting down"),
|
|
_ = int.recv() => tracing::info!("SIGINT received; shutting down"),
|
|
}
|
|
}
|
|
.boxed()
|
|
.shared()
|
|
}
|
|
|
|
/// Runs `serving`, a server built with `with_graceful_shutdown` on a clone of
|
|
/// `stop`, and stops waiting for it [`DRAIN`] after `stop` resolves. `None`
|
|
/// means connections were still open then.
|
|
///
|
|
/// A graceful shutdown waits for every connection to close, and a streaming
|
|
/// response never does on its own; without this bound such a daemon sits out
|
|
/// the unit's stop timeout and is killed.
|
|
pub async fn serve_bounded<T>(
|
|
stop: impl Future<Output = ()>,
|
|
serving: impl Future<Output = T>,
|
|
) -> Option<T> {
|
|
tokio::pin!(serving);
|
|
tokio::select! {
|
|
out = &mut serving => Some(out),
|
|
() = async {
|
|
stop.await;
|
|
tokio::time::sleep(DRAIN).await;
|
|
} => {
|
|
tracing::warn!(drain = ?DRAIN, "still running after the drain; exiting anyway");
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The handlers a daemon has spawned for messages it already took.
|
|
///
|
|
/// After the signal the daemon takes no more, calls [`InFlight::drain`], and
|
|
/// whatever is still running then is abandoned. Nothing is acknowledged on
|
|
/// its behalf, so an unfinished message is redelivered or times out at its
|
|
/// sender exactly as it would after a kill.
|
|
#[derive(Default)]
|
|
pub struct InFlight(JoinSet<()>);
|
|
|
|
impl InFlight {
|
|
/// An empty set.
|
|
#[must_use]
|
|
pub fn new() -> Self {
|
|
Self::default()
|
|
}
|
|
|
|
/// Runs `handler` on its own task. Handlers that already finished are
|
|
/// dropped first, so the set stays as small as the work in hand.
|
|
pub fn spawn(&mut self, handler: impl Future<Output = ()> + Send + 'static) {
|
|
while self.0.try_join_next().is_some() {}
|
|
self.0.spawn(handler);
|
|
}
|
|
|
|
/// Waits up to [`DRAIN`] for the handlers still running.
|
|
pub async fn drain(mut self) {
|
|
let finished =
|
|
tokio::time::timeout(DRAIN, async { while self.0.join_next().await.is_some() {} })
|
|
.await;
|
|
if finished.is_err() {
|
|
tracing::warn!(
|
|
abandoned = self.0.len(),
|
|
drain = ?DRAIN,
|
|
"handlers still running after the drain; abandoning them"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
|
|
use tokio::sync::Notify;
|
|
|
|
use super::*;
|
|
|
|
/// A real SIGTERM, then a real SIGINT, raised at this process: the handlers
|
|
/// are installed by the call, so the signals are consumed rather than
|
|
/// killing the test binary. One test, so the two raises never overlap.
|
|
#[tokio::test]
|
|
async fn resolves_on_a_sigterm_and_on_a_sigint_raised_at_the_process() {
|
|
let stop = shutdown_signal();
|
|
let mut stop = std::pin::pin!(stop);
|
|
assert!(
|
|
stop.as_mut().now_or_never().is_none(),
|
|
"resolved with no signal"
|
|
);
|
|
// SAFETY: `raise` takes a signal number and touches no memory.
|
|
assert_eq!(unsafe { libc::raise(libc::SIGTERM) }, 0);
|
|
tokio::time::timeout(Duration::from_secs(5), stop)
|
|
.await
|
|
.expect("SIGTERM resolves the future");
|
|
|
|
let stop = shutdown_signal();
|
|
// SAFETY: as above.
|
|
assert_eq!(unsafe { libc::raise(libc::SIGINT) }, 0);
|
|
tokio::time::timeout(Duration::from_secs(5), stop)
|
|
.await
|
|
.expect("SIGINT resolves the future");
|
|
}
|
|
|
|
/// A server that finishes by itself is returned from at once, and its
|
|
/// output is the caller's.
|
|
#[tokio::test]
|
|
async fn serve_bounded_returns_what_the_server_returns() {
|
|
let out = serve_bounded(std::future::pending(), async { 7 }).await;
|
|
assert_eq!(out, Some(7));
|
|
}
|
|
|
|
/// After `stop`, a server that never ends is given up on once [`DRAIN`]
|
|
/// has passed, and not before.
|
|
#[tokio::test(start_paused = true)]
|
|
async fn serve_bounded_gives_up_only_after_the_drain() {
|
|
let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>();
|
|
let bounded = tokio::spawn(serve_bounded(
|
|
async { stop_rx.await.unwrap() },
|
|
std::future::pending::<()>(),
|
|
));
|
|
tokio::time::sleep(DRAIN * 3).await;
|
|
assert!(!bounded.is_finished(), "gave up with no stop");
|
|
stop_tx.send(()).unwrap();
|
|
tokio::time::sleep(DRAIN / 2).await;
|
|
assert!(!bounded.is_finished(), "gave up inside the drain");
|
|
tokio::time::sleep(DRAIN / 2 + Duration::from_secs(1)).await;
|
|
assert!(bounded.is_finished(), "still waiting past the drain");
|
|
assert_eq!(bounded.await.unwrap(), None);
|
|
}
|
|
|
|
/// A handler that finishes inside the drain is waited for.
|
|
#[tokio::test(start_paused = true)]
|
|
async fn drain_waits_for_a_handler_that_finishes_in_time() {
|
|
let done = Arc::new(AtomicBool::new(false));
|
|
let mut in_flight = InFlight::new();
|
|
let flag = Arc::clone(&done);
|
|
in_flight.spawn(async move {
|
|
tokio::time::sleep(DRAIN / 2).await;
|
|
flag.store(true, Ordering::SeqCst);
|
|
});
|
|
in_flight.drain().await;
|
|
assert!(done.load(Ordering::SeqCst));
|
|
}
|
|
|
|
/// A handler that outlives the drain is abandoned: the drain returns at
|
|
/// the bound with the handler unfinished.
|
|
#[tokio::test(start_paused = true)]
|
|
async fn drain_abandons_a_handler_that_outlives_it() {
|
|
let done = Arc::new(AtomicBool::new(false));
|
|
let started = Arc::new(Notify::new());
|
|
let mut in_flight = InFlight::new();
|
|
let (flag, up) = (Arc::clone(&done), Arc::clone(&started));
|
|
in_flight.spawn(async move {
|
|
up.notify_one();
|
|
tokio::time::sleep(DRAIN * 5).await;
|
|
flag.store(true, Ordering::SeqCst);
|
|
});
|
|
started.notified().await;
|
|
let before = tokio::time::Instant::now();
|
|
in_flight.drain().await;
|
|
assert_eq!(before.elapsed(), DRAIN);
|
|
assert!(!done.load(Ordering::SeqCst));
|
|
}
|
|
}
|