From 39c29b64e3d79e5c0a34d11f05688dad736cce62 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 18 Nov 2026 03:28:12 +0100 Subject: [PATCH] swarm-secret-client: add shutdown_signal, serve_bounded and InFlight --- Cargo.lock | 3 + swarm-secret-client/Cargo.toml | 2 + swarm-secret-client/src/lib.rs | 2 + swarm-secret-client/src/shutdown.rs | 206 ++++++++++++++++++++++++++++ 4 files changed, 213 insertions(+) create mode 100644 swarm-secret-client/src/shutdown.rs diff --git a/Cargo.lock b/Cargo.lock index e0660ac22..0ae9dd629 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5278,6 +5278,7 @@ dependencies = [ "serde_json", "serde_norway", "swarm-authelia-bridge-sock", + "swarm-secret-client", "tokio", "tracing", "tracing-subscriber", @@ -5438,6 +5439,7 @@ dependencies = [ "anyhow", "axum", "clap", + "futures-util", "sha2 0.11.1", "swarm-agent-verify", "swarm-secret-client", @@ -5599,6 +5601,7 @@ version = "0.1.0" dependencies = [ "backon", "futures-util", + "libc", "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", diff --git a/swarm-secret-client/Cargo.toml b/swarm-secret-client/Cargo.toml index 5904e4400..fd3484b67 100644 --- a/swarm-secret-client/Cargo.toml +++ b/swarm-secret-client/Cargo.toml @@ -47,6 +47,8 @@ workspace = true # provider compiled in rustls picks it without an install, so the provider # test would pass with `install_crypto_provider` doing nothing. [dev-dependencies] +# `raise`, to send the test process a real SIGTERM. +libc.workspace = true # `InMemoryMetricExporter`, to read back what `cert_metrics` exports. opentelemetry_sdk = { workspace = true, features = ["testing"] } rustls = { workspace = true, features = ["ring"] } diff --git a/swarm-secret-client/src/lib.rs b/swarm-secret-client/src/lib.rs index 7e059ee81..d0ea6c07d 100644 --- a/swarm-secret-client/src/lib.rs +++ b/swarm-secret-client/src/lib.rs @@ -55,10 +55,12 @@ pub mod policy; pub mod proxy; pub mod queue; pub mod review_classifier; +pub mod shutdown; pub mod startup_read; pub mod stored_secret; pub use client::SecretStore; +pub use shutdown::{DRAIN, InFlight, Shutdown, serve_bounded, shutdown_signal}; pub use startup_read::read_with_startup_retry; /// What can go wrong between "we have a client certificate" and "we have the diff --git a/swarm-secret-client/src/shutdown.rs b/swarm-secret-client/src/shutdown.rs new file mode 100644 index 000000000..4fb9b5e00 --- /dev/null +++ b/swarm-secret-client/src/shutdown.rs @@ -0,0 +1,206 @@ +//! Stopping a daemon when its unit is stopped. +//! +//! `systemctl stop` and a container runtime both send SIGTERM. A process with +//! no handler dies on the default action, mid-request; one that waits on +//! [`shutdown_signal`] finishes what it has in hand and exits 0. Every swarm +//! daemon's `main` shares this one definition of "told to stop" and of how +//! long it then gets ([`DRAIN`]). + +use std::future::Future; +use std::time::Duration; + +use futures_util::FutureExt as _; +use futures_util::future::{BoxFuture, Shared}; +use tokio::signal::unix::{SignalKind, signal}; +use tokio::task::JoinSet; + +/// How long work in hand may take once the signal has arrived. Well under +/// systemd's default `TimeoutStopSec` (90 s) and a pod's 30 s grace period, so +/// the process exits 0 before either escalates to SIGKILL. +pub const DRAIN: Duration = Duration::from_secs(10); + +/// What [`shutdown_signal`] returns: a future that is `Clone`, and whose clones +/// all resolve on the one signal. +pub type Shutdown = Shared>; + +/// Resolves when the process receives SIGTERM or SIGINT. +/// +/// The handlers are installed when this is called, not when the future is +/// first polled, so call it early in `main`: a signal arriving between the +/// call and the first poll is still seen, and one before the call kills the +/// process as before. Clones of the [`Shutdown`] it returns all resolve on the +/// one signal, so a server, a message loop and a bound can all wait on it. +/// +/// # Panics +/// +/// When the runtime is not running, or the kernel refuses a handler. +pub fn shutdown_signal() -> Shutdown { + let mut term = signal(SignalKind::terminate()).expect("installing the SIGTERM handler"); + let mut int = signal(SignalKind::interrupt()).expect("installing the SIGINT handler"); + async move { + tokio::select! { + _ = term.recv() => tracing::info!("SIGTERM received; shutting down"), + _ = int.recv() => tracing::info!("SIGINT received; shutting down"), + } + } + .boxed() + .shared() +} + +/// Runs `serving`, a server built with `with_graceful_shutdown` on a clone of +/// `stop`, and stops waiting for it [`DRAIN`] after `stop` resolves. `None` +/// means connections were still open then. +/// +/// A graceful shutdown waits for every connection to close, and a streaming +/// response never does on its own; without this bound such a daemon sits out +/// the unit's stop timeout and is killed. +pub async fn serve_bounded( + stop: impl Future, + serving: impl Future, +) -> Option { + tokio::pin!(serving); + tokio::select! { + out = &mut serving => Some(out), + () = async { + stop.await; + tokio::time::sleep(DRAIN).await; + } => { + tracing::warn!(drain = ?DRAIN, "still running after the drain; exiting anyway"); + None + } + } +} + +/// The handlers a daemon has spawned for messages it already took. +/// +/// After the signal the daemon takes no more, calls [`InFlight::drain`], and +/// whatever is still running then is abandoned. Nothing is acknowledged on +/// its behalf, so an unfinished message is redelivered or times out at its +/// sender exactly as it would after a kill. +#[derive(Default)] +pub struct InFlight(JoinSet<()>); + +impl InFlight { + /// An empty set. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Runs `handler` on its own task. Handlers that already finished are + /// dropped first, so the set stays as small as the work in hand. + pub fn spawn(&mut self, handler: impl Future + Send + 'static) { + while self.0.try_join_next().is_some() {} + self.0.spawn(handler); + } + + /// Waits up to [`DRAIN`] for the handlers still running. + pub async fn drain(mut self) { + let finished = + tokio::time::timeout(DRAIN, async { while self.0.join_next().await.is_some() {} }) + .await; + if finished.is_err() { + tracing::warn!( + abandoned = self.0.len(), + drain = ?DRAIN, + "handlers still running after the drain; abandoning them" + ); + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + use tokio::sync::Notify; + + use super::*; + + /// A real SIGTERM, then a real SIGINT, raised at this process: the handlers + /// are installed by the call, so the signals are consumed rather than + /// killing the test binary. One test, so the two raises never overlap. + #[tokio::test] + async fn resolves_on_a_sigterm_and_on_a_sigint_raised_at_the_process() { + let stop = shutdown_signal(); + let mut stop = std::pin::pin!(stop); + assert!( + stop.as_mut().now_or_never().is_none(), + "resolved with no signal" + ); + // SAFETY: `raise` takes a signal number and touches no memory. + assert_eq!(unsafe { libc::raise(libc::SIGTERM) }, 0); + tokio::time::timeout(Duration::from_secs(5), stop) + .await + .expect("SIGTERM resolves the future"); + + let stop = shutdown_signal(); + // SAFETY: as above. + assert_eq!(unsafe { libc::raise(libc::SIGINT) }, 0); + tokio::time::timeout(Duration::from_secs(5), stop) + .await + .expect("SIGINT resolves the future"); + } + + /// A server that finishes by itself is returned from at once, and its + /// output is the caller's. + #[tokio::test] + async fn serve_bounded_returns_what_the_server_returns() { + let out = serve_bounded(std::future::pending(), async { 7 }).await; + assert_eq!(out, Some(7)); + } + + /// After `stop`, a server that never ends is given up on once [`DRAIN`] + /// has passed, and not before. + #[tokio::test(start_paused = true)] + async fn serve_bounded_gives_up_only_after_the_drain() { + let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>(); + let bounded = tokio::spawn(serve_bounded( + async { stop_rx.await.unwrap() }, + std::future::pending::<()>(), + )); + tokio::time::sleep(DRAIN * 3).await; + assert!(!bounded.is_finished(), "gave up with no stop"); + stop_tx.send(()).unwrap(); + tokio::time::sleep(DRAIN / 2).await; + assert!(!bounded.is_finished(), "gave up inside the drain"); + tokio::time::sleep(DRAIN / 2 + Duration::from_secs(1)).await; + assert!(bounded.is_finished(), "still waiting past the drain"); + assert_eq!(bounded.await.unwrap(), None); + } + + /// A handler that finishes inside the drain is waited for. + #[tokio::test(start_paused = true)] + async fn drain_waits_for_a_handler_that_finishes_in_time() { + let done = Arc::new(AtomicBool::new(false)); + let mut in_flight = InFlight::new(); + let flag = Arc::clone(&done); + in_flight.spawn(async move { + tokio::time::sleep(DRAIN / 2).await; + flag.store(true, Ordering::SeqCst); + }); + in_flight.drain().await; + assert!(done.load(Ordering::SeqCst)); + } + + /// A handler that outlives the drain is abandoned: the drain returns at + /// the bound with the handler unfinished. + #[tokio::test(start_paused = true)] + async fn drain_abandons_a_handler_that_outlives_it() { + let done = Arc::new(AtomicBool::new(false)); + let started = Arc::new(Notify::new()); + let mut in_flight = InFlight::new(); + let (flag, up) = (Arc::clone(&done), Arc::clone(&started)); + in_flight.spawn(async move { + up.notify_one(); + tokio::time::sleep(DRAIN * 5).await; + flag.store(true, Ordering::SeqCst); + }); + started.notified().await; + let before = tokio::time::Instant::now(); + in_flight.drain().await; + assert_eq!(before.elapsed(), DRAIN); + assert!(!done.load(Ordering::SeqCst)); + } +}