From a089da25a51694f26d5dbc5a9f4ccb31607f0b62 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 18 Nov 2026 03:28:12 +0100 Subject: [PATCH] swarm-logs-auth, swarm-metrics-auth, swarm-authelia-bridge, swarm-controller: drain http on sigterm --- swarm-authelia-bridge/Cargo.toml | 2 + swarm-authelia-bridge/src/main.rs | 18 ++++- swarm-controller/src/main.rs | 21 +++-- .../src/review_classifier/review.rs | 79 ++++++++++++++++--- swarm-logs-auth/Cargo.toml | 4 + swarm-logs-auth/src/main.rs | 75 +++++++++++++++++- swarm-metrics-auth/src/main.rs | 18 ++++- 7 files changed, 198 insertions(+), 19 deletions(-) diff --git a/swarm-authelia-bridge/Cargo.toml b/swarm-authelia-bridge/Cargo.toml index 01812e22e..226fc8ade 100644 --- a/swarm-authelia-bridge/Cargo.toml +++ b/swarm-authelia-bridge/Cargo.toml @@ -19,6 +19,8 @@ serde.workspace = true serde_json.workspace = true serde_norway = "0.9.42" swarm-authelia-bridge-sock.workspace = true +# `shutdown_signal`: the bridge drains its in-flight requests on SIGTERM. +swarm-secret-client.workspace = true tokio.workspace = true tracing.workspace = true tracing-subscriber.workspace = true diff --git a/swarm-authelia-bridge/src/main.rs b/swarm-authelia-bridge/src/main.rs index bfed6c38e..adbf7b21c 100644 --- a/swarm-authelia-bridge/src/main.rs +++ b/swarm-authelia-bridge/src/main.rs @@ -106,6 +106,7 @@ struct AppState { #[tokio::main] async fn main() -> Result<()> { + let stop = swarm_secret_client::shutdown_signal(); tracing_subscriber::fmt() .with_env_filter( tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()), @@ -130,9 +131,22 @@ async fn main() -> Result<()> { let listener = bind_socket(&socket)?; tracing::info!(socket = %socket.display(), "swarm-authelia-bridge listening"); - axum::serve(listener, app) + serve(listener, app, stop).await +} + +/// Answers on `listener` until `stop` resolves, then lets the requests in hand +/// finish, for at most the drain, and returns. +async fn serve( + listener: tokio::net::UnixListener, + app: Router, + stop: impl Future + Clone + Send + 'static, +) -> Result<()> { + let serving = axum::serve(listener, app).with_graceful_shutdown(stop.clone()); + swarm_secret_client::serve_bounded(stop, serving.into_future()) .await - .context("serving swarm-authelia-bridge") + .transpose() + .context("serving swarm-authelia-bridge")?; + Ok(()) } /// Bind the listening socket at `path`, replacing a stale one a crash left diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 2ec3e8bcb..f35a6c171 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -269,7 +269,7 @@ async fn main() -> Result<()> { reviews: Arc::default(), alerts: alerts::from_env(), }; - serve(listener, state).await + serve(listener, state, swarm_secret_client::shutdown_signal()).await } /// Registers the `bao.client_cert.*` gauges for the controller's own store @@ -286,11 +286,20 @@ fn register_cert_gauges(provider: Option<&opentelemetry_sdk::metrics::SdkMeterPr } /// Start taking the queued review requests and answering the notification -/// router's, then serve `state` on `listener`. -async fn serve(listener: tokio::net::UnixListener, state: AppState) -> Result<()> { - review_classifier::review::spawn_queue(&state); +/// router's, then serve `state` on `listener` until `stop` resolves. The +/// requests in hand finish for at most the drain; the review queue takes no +/// request after `stop`. +async fn serve( + listener: tokio::net::UnixListener, + state: AppState, + stop: impl Future + Clone + Send + 'static, +) -> Result<()> { + review_classifier::review::spawn_queue(&state, stop.clone()); request_reviewer::spawn(&state); - axum::serve(listener, build_app(state)) + let serving = axum::serve(listener, build_app(state)).with_graceful_shutdown(stop.clone()); + swarm_secret_client::serve_bounded(stop, serving.into_future()) .await - .context("serving swarm-controller") + .transpose() + .context("serving swarm-controller")?; + Ok(()) } diff --git a/swarm-controller/src/review_classifier/review.rs b/swarm-controller/src/review_classifier/review.rs index 9d14b75c5..b663ea972 100644 --- a/swarm-controller/src/review_classifier/review.rs +++ b/swarm-controller/src/review_classifier/review.rs @@ -443,7 +443,10 @@ async fn open_consumer( /// Take the queued review requests and review each, for as long as the /// controller runs, once this host has forge access. -pub(crate) fn spawn_queue(state: &AppState) { +pub(crate) fn spawn_queue( + state: &AppState, + stop: impl Future + Clone + Send + 'static, +) { let Some(client) = state.status.as_ref().map(|s| s.queue_client()) else { tracing::warn!("no swarm queue is configured on this host; requested reviews don't run"); return; @@ -466,7 +469,7 @@ pub(crate) fn spawn_queue(state: &AppState) { runs, metrics: Metrics::global(), }; - consume(&client, TIMING, &reviews, handles).await; + consume(&client, TIMING, &reviews, handles, stop).await; }); } @@ -494,8 +497,9 @@ impl Clone for Handles { } /// Take requests from [`REQUEST_STREAM`] one at a time, each once a run -/// place is free, and review each in a task of its own. Never returns; -/// dropping it stops every review it started, and the queue delivers their +/// place is free, and review each in a task of its own. Returns once `stop` +/// resolves, taking no request after it; dropping it, as the return does, stops +/// every review it started without acking, and the queue delivers their /// requests again. /// /// The limit is read from the settings before each wait for a place. A @@ -506,6 +510,7 @@ async fn consume( timing: Timing, reviews: &Arc, handles: Handles, + stop: impl Future + Clone, ) where S: FileSource + PullSource + Send + Sync + 'static, Q: RunSink + SettingsSource, @@ -519,7 +524,11 @@ async fn consume( "opening the review request queue failed; trying again" ), } - tokio::time::sleep(timing.pull_wait).await; + tokio::select! { + biased; + () = stop.clone() => return, + () = tokio::time::sleep(timing.pull_wait) => {} + } }; let mut tasks = JoinSet::new(); loop { @@ -536,8 +545,17 @@ async fn consume( the last limit read" ), } - let slot = reviews.slot().await; - let message = match next(&consumer, timing.pull_wait).await { + let slot = tokio::select! { + biased; + () = stop.clone() => return, + slot = reviews.slot() => slot, + }; + let taken = tokio::select! { + biased; + () = stop.clone() => return, + taken = next(&consumer, timing.pull_wait) => taken, + }; + let message = match taken { Ok(Some(message)) => message, Ok(None) => continue, Err(e) => { @@ -546,7 +564,11 @@ async fn consume( "taking a review request from the queue failed; trying again" ); drop(slot); - tokio::time::sleep(timing.pull_wait).await; + tokio::select! { + biased; + () = stop.clone() => return, + () = tokio::time::sleep(timing.pull_wait) => {} + } continue; } }; @@ -2032,6 +2054,15 @@ mod tests { /// A controller taking requests, its run places in `reviews`. fn consume(&self, reviews: &Arc) -> JoinHandle<()> { + self.consume_until(reviews, std::future::pending()) + } + + /// A controller that takes no request once `stop` resolves. + fn consume_until( + &self, + reviews: &Arc, + stop: impl Future + Clone + Send + 'static, + ) -> JoinHandle<()> { let client = self.client.clone(); let reviews = Arc::clone(reviews); let forge = Arc::clone(&self.rig.forge); @@ -2047,7 +2078,7 @@ mod tests { runs: store, metrics, }; - consume(&client, TEST_TIMING, &reviews, handles).await; + consume(&client, TEST_TIMING, &reviews, handles, stop).await; }) } @@ -2113,6 +2144,36 @@ mod tests { assert_eq!(q.rig.gate.reached(), 2, "no run after the ack"); } + /// After the stop the controller returns and takes no request, and the + /// review it was running stays un-acked, so the queue still holds it. + #[tokio::test] + async fn after_the_stop_no_request_is_taken_and_the_running_one_stays_unacked() { + let q = Queued::new().await; + q.request(7, "abc123").await; + let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>(); + let stop = futures_util::FutureExt::shared(async move { + let _ = stop_rx.await; + }); + let consumer = q.consume_until(&q.rig.reviews, stop); + until("the first run started", || q.rig.gate.reached() == 1).await; + + stop_tx.send(()).expect("stop"); + tokio::time::timeout(Duration::from_secs(5), consumer) + .await + .expect("consume returns after the stop") + .expect("consume did not panic"); + // A request that arrives now, and the one running, which the queue + // would redeliver after its ack wait: neither runs. + q.request(8, "abc123").await; + tokio::time::sleep(TEST_TIMING.ack_wait * 3).await; + assert_eq!(q.rig.gate.reached(), 1, "no request taken after the stop"); + assert!( + q.rig.posted().is_empty(), + "the unfinished review is not posted" + ); + assert_eq!(q.waiting().await, 2, "neither request is acked"); + } + #[tokio::test] async fn a_newer_request_for_a_pr_replaces_the_waiting_one() { let q = Queued::new().await; diff --git a/swarm-logs-auth/Cargo.toml b/swarm-logs-auth/Cargo.toml index ac742a909..ff5079ffc 100644 --- a/swarm-logs-auth/Cargo.toml +++ b/swarm-logs-auth/Cargo.toml @@ -21,3 +21,7 @@ tracing-subscriber.workspace = true [lints] workspace = true + +[dev-dependencies] +# `Shared`, so the injected stop future can be cloned like `shutdown_signal`'s. +futures-util.workspace = true diff --git a/swarm-logs-auth/src/main.rs b/swarm-logs-auth/src/main.rs index c4c40e14e..9bc40783e 100644 --- a/swarm-logs-auth/src/main.rs +++ b/swarm-logs-auth/src/main.rs @@ -195,6 +195,7 @@ async fn handle( #[tokio::main] async fn main() -> anyhow::Result<()> { + let stop = swarm_secret_client::shutdown_signal(); swarm_secret_client::install_crypto_provider(); tracing_subscriber::fmt() .with_env_filter( @@ -235,14 +236,31 @@ async fn main() -> anyhow::Result<()> { tracing::info!(socket = %args.socket.display(), "swarm-logs-auth listening"); let app = axum::Router::new().fallback(handle).with_state(responder); - axum::serve(listener, app).await.context("serving") + serve(listener, app, stop).await +} + +/// Answers on `listener` until `stop` resolves, then lets the requests in hand +/// finish, for at most the drain, and returns. +async fn serve( + listener: tokio::net::UnixListener, + app: axum::Router, + stop: impl Future + Clone + Send + 'static, +) -> anyhow::Result<()> { + let serving = axum::serve(listener, app).with_graceful_shutdown(stop.clone()); + swarm_secret_client::serve_bounded(stop, serving.into_future()) + .await + .transpose() + .context("serving")?; + Ok(()) } #[cfg(test)] mod tests { use super::*; + use futures_util::FutureExt as _; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use swarm_secret_client::queue::AgentCredential; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; /// What the mirrored grant leaf answers. #[derive(Clone, Copy)] @@ -474,4 +492,59 @@ mod tests { } assert_eq!(store.lookups(), 3); } + + /// A request that is being answered when the stop arrives is still + /// answered, and `serve` returns only afterwards. + #[tokio::test] + async fn a_request_in_flight_at_the_stop_is_answered_before_serve_returns() { + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let app = { + let (started, release) = (Arc::clone(&started), Arc::clone(&release)); + axum::Router::new().fallback(move || { + let (started, release) = (Arc::clone(&started), Arc::clone(&release)); + async move { + started.notify_one(); + release.notified().await; + "answered" + } + }) + }; + let socket = + std::env::temp_dir().join(format!("swarm-logs-auth-stop-{}.sock", std::process::id())); + let _ = std::fs::remove_file(&socket); + let listener = tokio::net::UnixListener::bind(&socket).expect("bind"); + let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>(); + let stop = async move { + let _ = stop_rx.await; + } + .shared(); + let mut serving = tokio::spawn(serve(listener, app, stop)); + + let mut conn = tokio::net::UnixStream::connect(&socket) + .await + .expect("connect"); + conn.write_all(b"GET / HTTP/1.1\r\nhost: x\r\nconnection: close\r\n\r\n") + .await + .expect("send"); + started.notified().await; + + stop_tx.send(()).expect("stop"); + tokio::time::sleep(Duration::from_millis(200)).await; + assert!( + !serving.is_finished(), + "serve returned with a request in flight" + ); + + release.notify_one(); + let mut reply = String::new(); + conn.read_to_string(&mut reply).await.expect("reply"); + assert!(reply.starts_with("HTTP/1.1 200"), "{reply}"); + assert!(reply.ends_with("answered"), "{reply}"); + let finished = tokio::time::timeout(Duration::from_secs(5), &mut serving) + .await + .expect("serve returns once the request is answered"); + finished.expect("task").expect("serve is Ok"); + let _ = std::fs::remove_file(&socket); + } } diff --git a/swarm-metrics-auth/src/main.rs b/swarm-metrics-auth/src/main.rs index fd4bc4f2b..d4845a3ba 100644 --- a/swarm-metrics-auth/src/main.rs +++ b/swarm-metrics-auth/src/main.rs @@ -282,6 +282,7 @@ async fn handle(State(responder): State>>, headers: HeaderM #[tokio::main] async fn main() -> anyhow::Result<()> { + let stop = swarm_secret_client::shutdown_signal(); swarm_secret_client::install_crypto_provider(); tracing_subscriber::fmt() .with_env_filter( @@ -322,7 +323,22 @@ async fn main() -> anyhow::Result<()> { tracing::info!(socket = %args.socket.display(), "swarm-metrics-auth listening"); let app = axum::Router::new().fallback(handle).with_state(responder); - axum::serve(listener, app).await.context("serving") + serve(listener, app, stop).await +} + +/// Answers on `listener` until `stop` resolves, then lets the requests in hand +/// finish, for at most the drain, and returns. +async fn serve( + listener: tokio::net::UnixListener, + app: axum::Router, + stop: impl Future + Clone + Send + 'static, +) -> anyhow::Result<()> { + let serving = axum::serve(listener, app).with_graceful_shutdown(stop.clone()); + swarm_secret_client::serve_bounded(stop, serving.into_future()) + .await + .transpose() + .context("serving")?; + Ok(()) } #[cfg(test)]