swarm-secret-client: add shutdown_signal, serve_bounded and InFlight
This commit is contained in:
parent
e0002095ba
commit
39c29b64e3
4 changed files with 213 additions and 0 deletions
3
Cargo.lock
generated
3
Cargo.lock
generated
|
|
@ -5278,6 +5278,7 @@ dependencies = [
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serde_norway",
|
"serde_norway",
|
||||||
"swarm-authelia-bridge-sock",
|
"swarm-authelia-bridge-sock",
|
||||||
|
"swarm-secret-client",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
|
|
@ -5438,6 +5439,7 @@ dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"axum",
|
"axum",
|
||||||
"clap",
|
"clap",
|
||||||
|
"futures-util",
|
||||||
"sha2 0.11.1",
|
"sha2 0.11.1",
|
||||||
"swarm-agent-verify",
|
"swarm-agent-verify",
|
||||||
"swarm-secret-client",
|
"swarm-secret-client",
|
||||||
|
|
@ -5599,6 +5601,7 @@ version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"backon",
|
"backon",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
|
"libc",
|
||||||
"opentelemetry",
|
"opentelemetry",
|
||||||
"opentelemetry-otlp",
|
"opentelemetry-otlp",
|
||||||
"opentelemetry_sdk",
|
"opentelemetry_sdk",
|
||||||
|
|
|
||||||
|
|
@ -47,6 +47,8 @@ workspace = true
|
||||||
# provider compiled in rustls picks it without an install, so the provider
|
# provider compiled in rustls picks it without an install, so the provider
|
||||||
# test would pass with `install_crypto_provider` doing nothing.
|
# test would pass with `install_crypto_provider` doing nothing.
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
# `raise`, to send the test process a real SIGTERM.
|
||||||
|
libc.workspace = true
|
||||||
# `InMemoryMetricExporter`, to read back what `cert_metrics` exports.
|
# `InMemoryMetricExporter`, to read back what `cert_metrics` exports.
|
||||||
opentelemetry_sdk = { workspace = true, features = ["testing"] }
|
opentelemetry_sdk = { workspace = true, features = ["testing"] }
|
||||||
rustls = { workspace = true, features = ["ring"] }
|
rustls = { workspace = true, features = ["ring"] }
|
||||||
|
|
|
||||||
|
|
@ -55,10 +55,12 @@ pub mod policy;
|
||||||
pub mod proxy;
|
pub mod proxy;
|
||||||
pub mod queue;
|
pub mod queue;
|
||||||
pub mod review_classifier;
|
pub mod review_classifier;
|
||||||
|
pub mod shutdown;
|
||||||
pub mod startup_read;
|
pub mod startup_read;
|
||||||
pub mod stored_secret;
|
pub mod stored_secret;
|
||||||
|
|
||||||
pub use client::SecretStore;
|
pub use client::SecretStore;
|
||||||
|
pub use shutdown::{DRAIN, InFlight, Shutdown, serve_bounded, shutdown_signal};
|
||||||
pub use startup_read::read_with_startup_retry;
|
pub use startup_read::read_with_startup_retry;
|
||||||
|
|
||||||
/// What can go wrong between "we have a client certificate" and "we have the
|
/// What can go wrong between "we have a client certificate" and "we have the
|
||||||
|
|
|
||||||
206
swarm-secret-client/src/shutdown.rs
Normal file
206
swarm-secret-client/src/shutdown.rs
Normal file
|
|
@ -0,0 +1,206 @@
|
||||||
|
//! Stopping a daemon when its unit is stopped.
|
||||||
|
//!
|
||||||
|
//! `systemctl stop` and a container runtime both send SIGTERM. A process with
|
||||||
|
//! no handler dies on the default action, mid-request; one that waits on
|
||||||
|
//! [`shutdown_signal`] finishes what it has in hand and exits 0. Every swarm
|
||||||
|
//! daemon's `main` shares this one definition of "told to stop" and of how
|
||||||
|
//! long it then gets ([`DRAIN`]).
|
||||||
|
|
||||||
|
use std::future::Future;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use futures_util::FutureExt as _;
|
||||||
|
use futures_util::future::{BoxFuture, Shared};
|
||||||
|
use tokio::signal::unix::{SignalKind, signal};
|
||||||
|
use tokio::task::JoinSet;
|
||||||
|
|
||||||
|
/// How long work in hand may take once the signal has arrived. Well under
|
||||||
|
/// systemd's default `TimeoutStopSec` (90 s) and a pod's 30 s grace period, so
|
||||||
|
/// the process exits 0 before either escalates to SIGKILL.
|
||||||
|
pub const DRAIN: Duration = Duration::from_secs(10);
|
||||||
|
|
||||||
|
/// What [`shutdown_signal`] returns: a future that is `Clone`, and whose clones
|
||||||
|
/// all resolve on the one signal.
|
||||||
|
pub type Shutdown = Shared<BoxFuture<'static, ()>>;
|
||||||
|
|
||||||
|
/// Resolves when the process receives SIGTERM or SIGINT.
|
||||||
|
///
|
||||||
|
/// The handlers are installed when this is called, not when the future is
|
||||||
|
/// first polled, so call it early in `main`: a signal arriving between the
|
||||||
|
/// call and the first poll is still seen, and one before the call kills the
|
||||||
|
/// process as before. Clones of the [`Shutdown`] it returns all resolve on the
|
||||||
|
/// one signal, so a server, a message loop and a bound can all wait on it.
|
||||||
|
///
|
||||||
|
/// # Panics
|
||||||
|
///
|
||||||
|
/// When the runtime is not running, or the kernel refuses a handler.
|
||||||
|
pub fn shutdown_signal() -> Shutdown {
|
||||||
|
let mut term = signal(SignalKind::terminate()).expect("installing the SIGTERM handler");
|
||||||
|
let mut int = signal(SignalKind::interrupt()).expect("installing the SIGINT handler");
|
||||||
|
async move {
|
||||||
|
tokio::select! {
|
||||||
|
_ = term.recv() => tracing::info!("SIGTERM received; shutting down"),
|
||||||
|
_ = int.recv() => tracing::info!("SIGINT received; shutting down"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.boxed()
|
||||||
|
.shared()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Runs `serving`, a server built with `with_graceful_shutdown` on a clone of
|
||||||
|
/// `stop`, and stops waiting for it [`DRAIN`] after `stop` resolves. `None`
|
||||||
|
/// means connections were still open then.
|
||||||
|
///
|
||||||
|
/// A graceful shutdown waits for every connection to close, and a streaming
|
||||||
|
/// response never does on its own; without this bound such a daemon sits out
|
||||||
|
/// the unit's stop timeout and is killed.
|
||||||
|
pub async fn serve_bounded<T>(
|
||||||
|
stop: impl Future<Output = ()>,
|
||||||
|
serving: impl Future<Output = T>,
|
||||||
|
) -> Option<T> {
|
||||||
|
tokio::pin!(serving);
|
||||||
|
tokio::select! {
|
||||||
|
out = &mut serving => Some(out),
|
||||||
|
() = async {
|
||||||
|
stop.await;
|
||||||
|
tokio::time::sleep(DRAIN).await;
|
||||||
|
} => {
|
||||||
|
tracing::warn!(drain = ?DRAIN, "still running after the drain; exiting anyway");
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The handlers a daemon has spawned for messages it already took.
|
||||||
|
///
|
||||||
|
/// After the signal the daemon takes no more, calls [`InFlight::drain`], and
|
||||||
|
/// whatever is still running then is abandoned. Nothing is acknowledged on
|
||||||
|
/// its behalf, so an unfinished message is redelivered or times out at its
|
||||||
|
/// sender exactly as it would after a kill.
|
||||||
|
#[derive(Default)]
|
||||||
|
pub struct InFlight(JoinSet<()>);
|
||||||
|
|
||||||
|
impl InFlight {
|
||||||
|
/// An empty set.
|
||||||
|
#[must_use]
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Runs `handler` on its own task. Handlers that already finished are
|
||||||
|
/// dropped first, so the set stays as small as the work in hand.
|
||||||
|
pub fn spawn(&mut self, handler: impl Future<Output = ()> + Send + 'static) {
|
||||||
|
while self.0.try_join_next().is_some() {}
|
||||||
|
self.0.spawn(handler);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Waits up to [`DRAIN`] for the handlers still running.
|
||||||
|
pub async fn drain(mut self) {
|
||||||
|
let finished =
|
||||||
|
tokio::time::timeout(DRAIN, async { while self.0.join_next().await.is_some() {} })
|
||||||
|
.await;
|
||||||
|
if finished.is_err() {
|
||||||
|
tracing::warn!(
|
||||||
|
abandoned = self.0.len(),
|
||||||
|
drain = ?DRAIN,
|
||||||
|
"handlers still running after the drain; abandoning them"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
|
||||||
|
use tokio::sync::Notify;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// A real SIGTERM, then a real SIGINT, raised at this process: the handlers
|
||||||
|
/// are installed by the call, so the signals are consumed rather than
|
||||||
|
/// killing the test binary. One test, so the two raises never overlap.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn resolves_on_a_sigterm_and_on_a_sigint_raised_at_the_process() {
|
||||||
|
let stop = shutdown_signal();
|
||||||
|
let mut stop = std::pin::pin!(stop);
|
||||||
|
assert!(
|
||||||
|
stop.as_mut().now_or_never().is_none(),
|
||||||
|
"resolved with no signal"
|
||||||
|
);
|
||||||
|
// SAFETY: `raise` takes a signal number and touches no memory.
|
||||||
|
assert_eq!(unsafe { libc::raise(libc::SIGTERM) }, 0);
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), stop)
|
||||||
|
.await
|
||||||
|
.expect("SIGTERM resolves the future");
|
||||||
|
|
||||||
|
let stop = shutdown_signal();
|
||||||
|
// SAFETY: as above.
|
||||||
|
assert_eq!(unsafe { libc::raise(libc::SIGINT) }, 0);
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), stop)
|
||||||
|
.await
|
||||||
|
.expect("SIGINT resolves the future");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A server that finishes by itself is returned from at once, and its
|
||||||
|
/// output is the caller's.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn serve_bounded_returns_what_the_server_returns() {
|
||||||
|
let out = serve_bounded(std::future::pending(), async { 7 }).await;
|
||||||
|
assert_eq!(out, Some(7));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// After `stop`, a server that never ends is given up on once [`DRAIN`]
|
||||||
|
/// has passed, and not before.
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn serve_bounded_gives_up_only_after_the_drain() {
|
||||||
|
let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>();
|
||||||
|
let bounded = tokio::spawn(serve_bounded(
|
||||||
|
async { stop_rx.await.unwrap() },
|
||||||
|
std::future::pending::<()>(),
|
||||||
|
));
|
||||||
|
tokio::time::sleep(DRAIN * 3).await;
|
||||||
|
assert!(!bounded.is_finished(), "gave up with no stop");
|
||||||
|
stop_tx.send(()).unwrap();
|
||||||
|
tokio::time::sleep(DRAIN / 2).await;
|
||||||
|
assert!(!bounded.is_finished(), "gave up inside the drain");
|
||||||
|
tokio::time::sleep(DRAIN / 2 + Duration::from_secs(1)).await;
|
||||||
|
assert!(bounded.is_finished(), "still waiting past the drain");
|
||||||
|
assert_eq!(bounded.await.unwrap(), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A handler that finishes inside the drain is waited for.
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn drain_waits_for_a_handler_that_finishes_in_time() {
|
||||||
|
let done = Arc::new(AtomicBool::new(false));
|
||||||
|
let mut in_flight = InFlight::new();
|
||||||
|
let flag = Arc::clone(&done);
|
||||||
|
in_flight.spawn(async move {
|
||||||
|
tokio::time::sleep(DRAIN / 2).await;
|
||||||
|
flag.store(true, Ordering::SeqCst);
|
||||||
|
});
|
||||||
|
in_flight.drain().await;
|
||||||
|
assert!(done.load(Ordering::SeqCst));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A handler that outlives the drain is abandoned: the drain returns at
|
||||||
|
/// the bound with the handler unfinished.
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn drain_abandons_a_handler_that_outlives_it() {
|
||||||
|
let done = Arc::new(AtomicBool::new(false));
|
||||||
|
let started = Arc::new(Notify::new());
|
||||||
|
let mut in_flight = InFlight::new();
|
||||||
|
let (flag, up) = (Arc::clone(&done), Arc::clone(&started));
|
||||||
|
in_flight.spawn(async move {
|
||||||
|
up.notify_one();
|
||||||
|
tokio::time::sleep(DRAIN * 5).await;
|
||||||
|
flag.store(true, Ordering::SeqCst);
|
||||||
|
});
|
||||||
|
started.notified().await;
|
||||||
|
let before = tokio::time::Instant::now();
|
||||||
|
in_flight.drain().await;
|
||||||
|
assert_eq!(before.elapsed(), DRAIN);
|
||||||
|
assert!(!done.load(Ordering::SeqCst));
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue