//! Stopping a daemon when its unit is stopped. //! //! `systemctl stop` and a container runtime both send SIGTERM. A process with //! no handler dies on the default action, mid-request; one that waits on //! [`shutdown_signal`] finishes what it has in hand and exits 0. Every swarm //! daemon's `main` shares this one definition of "told to stop" and of how //! long it then gets ([`DRAIN`]). use std::future::Future; use std::time::Duration; use futures_util::FutureExt as _; use futures_util::future::{BoxFuture, Shared}; use tokio::signal::unix::{SignalKind, signal}; use tokio::task::JoinSet; /// How long work in hand may take once the signal has arrived. Well under /// systemd's default `TimeoutStopSec` (90 s) and a pod's 30 s grace period, so /// the process exits 0 before either escalates to SIGKILL. pub const DRAIN: Duration = Duration::from_secs(10); /// What [`shutdown_signal`] returns: a future that is `Clone`, and whose clones /// all resolve on the one signal. pub type Shutdown = Shared>; /// Resolves when the process receives SIGTERM or SIGINT. /// /// The handlers are installed when this is called, not when the future is /// first polled, so call it early in `main`: a signal arriving between the /// call and the first poll is still seen, and one before the call kills the /// process as before. Clones of the [`Shutdown`] it returns all resolve on the /// one signal, so a server, a message loop and a bound can all wait on it. /// /// # Panics /// /// When the runtime is not running, or the kernel refuses a handler. pub fn shutdown_signal() -> Shutdown { let mut term = signal(SignalKind::terminate()).expect("installing the SIGTERM handler"); let mut int = signal(SignalKind::interrupt()).expect("installing the SIGINT handler"); async move { tokio::select! { _ = term.recv() => tracing::info!("SIGTERM received; shutting down"), _ = int.recv() => tracing::info!("SIGINT received; shutting down"), } } .boxed() .shared() } /// Runs `serving`, a server built with `with_graceful_shutdown` on a clone of /// `stop`, and stops waiting for it [`DRAIN`] after `stop` resolves. `None` /// means connections were still open then. /// /// A graceful shutdown waits for every connection to close, and a streaming /// response never does on its own; without this bound such a daemon sits out /// the unit's stop timeout and is killed. pub async fn serve_bounded( stop: impl Future, serving: impl Future, ) -> Option { tokio::pin!(serving); tokio::select! { out = &mut serving => Some(out), () = async { stop.await; tokio::time::sleep(DRAIN).await; } => { tracing::warn!(drain = ?DRAIN, "still running after the drain; exiting anyway"); None } } } /// The handlers a daemon has spawned for messages it already took. /// /// After the signal the daemon takes no more, calls [`InFlight::drain`], and /// whatever is still running then is abandoned. Nothing is acknowledged on /// its behalf, so an unfinished message is redelivered or times out at its /// sender exactly as it would after a kill. #[derive(Default)] pub struct InFlight(JoinSet<()>); impl InFlight { /// An empty set. #[must_use] pub fn new() -> Self { Self::default() } /// Runs `handler` on its own task. Handlers that already finished are /// dropped first, so the set stays as small as the work in hand. pub fn spawn(&mut self, handler: impl Future + Send + 'static) { while self.0.try_join_next().is_some() {} self.0.spawn(handler); } /// Waits up to [`DRAIN`] for the handlers still running. pub async fn drain(mut self) { let finished = tokio::time::timeout(DRAIN, async { while self.0.join_next().await.is_some() {} }) .await; if finished.is_err() { tracing::warn!( abandoned = self.0.len(), drain = ?DRAIN, "handlers still running after the drain; abandoning them" ); } } } #[cfg(test)] mod tests { use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use tokio::sync::Notify; use super::*; /// A real SIGTERM, then a real SIGINT, raised at this process: the handlers /// are installed by the call, so the signals are consumed rather than /// killing the test binary. One test, so the two raises never overlap. #[tokio::test] async fn resolves_on_a_sigterm_and_on_a_sigint_raised_at_the_process() { let stop = shutdown_signal(); let mut stop = std::pin::pin!(stop); assert!( stop.as_mut().now_or_never().is_none(), "resolved with no signal" ); // SAFETY: `raise` takes a signal number and touches no memory. assert_eq!(unsafe { libc::raise(libc::SIGTERM) }, 0); tokio::time::timeout(Duration::from_secs(5), stop) .await .expect("SIGTERM resolves the future"); let stop = shutdown_signal(); // SAFETY: as above. assert_eq!(unsafe { libc::raise(libc::SIGINT) }, 0); tokio::time::timeout(Duration::from_secs(5), stop) .await .expect("SIGINT resolves the future"); } /// A server that finishes by itself is returned from at once, and its /// output is the caller's. #[tokio::test] async fn serve_bounded_returns_what_the_server_returns() { let out = serve_bounded(std::future::pending(), async { 7 }).await; assert_eq!(out, Some(7)); } /// After `stop`, a server that never ends is given up on once [`DRAIN`] /// has passed, and not before. #[tokio::test(start_paused = true)] async fn serve_bounded_gives_up_only_after_the_drain() { let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>(); let bounded = tokio::spawn(serve_bounded( async { stop_rx.await.unwrap() }, std::future::pending::<()>(), )); tokio::time::sleep(DRAIN * 3).await; assert!(!bounded.is_finished(), "gave up with no stop"); stop_tx.send(()).unwrap(); tokio::time::sleep(DRAIN / 2).await; assert!(!bounded.is_finished(), "gave up inside the drain"); tokio::time::sleep(DRAIN / 2 + Duration::from_secs(1)).await; assert!(bounded.is_finished(), "still waiting past the drain"); assert_eq!(bounded.await.unwrap(), None); } /// A handler that finishes inside the drain is waited for. #[tokio::test(start_paused = true)] async fn drain_waits_for_a_handler_that_finishes_in_time() { let done = Arc::new(AtomicBool::new(false)); let mut in_flight = InFlight::new(); let flag = Arc::clone(&done); in_flight.spawn(async move { tokio::time::sleep(DRAIN / 2).await; flag.store(true, Ordering::SeqCst); }); in_flight.drain().await; assert!(done.load(Ordering::SeqCst)); } /// A handler that outlives the drain is abandoned: the drain returns at /// the bound with the handler unfinished. #[tokio::test(start_paused = true)] async fn drain_abandons_a_handler_that_outlives_it() { let done = Arc::new(AtomicBool::new(false)); let started = Arc::new(Notify::new()); let mut in_flight = InFlight::new(); let (flag, up) = (Arc::clone(&done), Arc::clone(&started)); in_flight.spawn(async move { up.notify_one(); tokio::time::sleep(DRAIN * 5).await; flag.store(true, Ordering::SeqCst); }); started.notified().await; let before = tokio::time::Instant::now(); in_flight.drain().await; assert_eq!(before.elapsed(), DRAIN); assert!(!done.load(Ordering::SeqCst)); } }