From a8775f1a7aa1eda63ae074ed882e22dbfc9755cb Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 18 Nov 2026 03:28:12 +0100 Subject: [PATCH] queue consumers: stop taking messages on sigterm and exit 0 --- swarm-builder/src/main.rs | 18 ++++++- swarm-claude-login/src/main.rs | 13 +++-- swarm-nats-auth/src/main.rs | 43 +++++++++------ swarm-notify-router/src/main.rs | 41 +++++++++----- swarm-notify-router/src/settle.rs | 84 +++++++++++++++++++++++------ swarm-review-classifier/src/main.rs | 9 +++- swarm-stt/src/main.rs | 9 +++- 7 files changed, 164 insertions(+), 53 deletions(-) diff --git a/swarm-builder/src/main.rs b/swarm-builder/src/main.rs index e5b45ef6d..ab42f5694 100644 --- a/swarm-builder/src/main.rs +++ b/swarm-builder/src/main.rs @@ -88,6 +88,7 @@ fn push_budget(reply_by: tokio::time::Instant) -> Duration { #[tokio::main] async fn main() -> Result<()> { + let stop = swarm_secret_client::shutdown_signal(); hive_log::init(); swarm_secret_client::install_crypto_provider(); let var = |name: &str| { @@ -174,8 +175,19 @@ async fn main() -> Result<()> { let slot = Arc::new(tokio::sync::Semaphore::new(1)); serve_meta_builds(&queue, &publisher, Store, &builder, &slot, &metrics).await?; - answer_builds(requests, publisher, Store, builder, slot, metrics).await; - bail!("the subscription to {SUBJECT} ended") + answer_builds( + requests.take_until(stop.clone()), + publisher, + Store, + builder, + slot, + metrics, + ) + .await; + if stop.peek().is_none() { + bail!("the subscription to {SUBJECT} ended"); + } + Ok(()) } /// Where a reply goes. [`Queue`] in production; tests record. @@ -260,6 +272,8 @@ async fn send_reply( } /// Answer every PR check, each in its own task, with the build's result. +/// Returns once `requests` ends; a build still running is not waited for, and +/// its requester gets no reply, as when the process is killed. async fn answer_builds( mut requests: impl Stream + Unpin, publisher: impl Publisher, diff --git a/swarm-claude-login/src/main.rs b/swarm-claude-login/src/main.rs index 3859915a3..0c58db0cd 100644 --- a/swarm-claude-login/src/main.rs +++ b/swarm-claude-login/src/main.rs @@ -100,6 +100,7 @@ impl login::CredentialSink for AgentLeaf { #[tokio::main] async fn main() -> Result<()> { + let stop = swarm_secret_client::shutdown_signal(); hive_log::init(); swarm_secret_client::install_crypto_provider(); let var = |name: &str| { @@ -138,9 +139,11 @@ async fn main() -> Result<()> { let mut requests = queue .subscribe(SUBJECT) .await - .with_context(|| format!("subscribing to {SUBJECT}"))?; + .with_context(|| format!("subscribing to {SUBJECT}"))? + .take_until(stop.clone()); tracing::info!(subject = SUBJECT, "claude login minter: awaiting logins"); + let mut in_flight = swarm_secret_client::InFlight::new(); while let Some(msg) = requests.next().await { let Some(reply_to) = msg.reply.clone() else { tracing::warn!("a login step arrived with no reply subject; dropping it"); @@ -149,7 +152,7 @@ async fn main() -> Result<()> { let queue = queue.clone(); let logins = Arc::clone(&logins); let metrics = Arc::clone(&metrics); - tokio::spawn(async move { + in_flight.spawn(async move { let reply = handle(&msg.payload, &logins, &metrics).await; match serde_json::to_vec(&reply) { Ok(body) => { @@ -161,7 +164,11 @@ async fn main() -> Result<()> { } }); } - bail!("the subscription to {SUBJECT} ended") + in_flight.drain().await; + if stop.peek().is_none() { + bail!("the subscription to {SUBJECT} ended"); + } + Ok(()) } /// Answer one step and count it. diff --git a/swarm-nats-auth/src/main.rs b/swarm-nats-auth/src/main.rs index f3fe7a639..d08c17365 100644 --- a/swarm-nats-auth/src/main.rs +++ b/swarm-nats-auth/src/main.rs @@ -182,6 +182,7 @@ struct Args { #[tokio::main] async fn main() -> anyhow::Result<()> { + let stop = swarm_secret_client::shutdown_signal(); swarm_secret_client::install_crypto_provider(); tracing_subscriber::fmt() .with_env_filter( @@ -210,14 +211,14 @@ async fn main() -> anyhow::Result<()> { ) .await } - (None, Some(args)) => serve(args).await, + (None, Some(args)) => serve(args, stop).await, (None, None) => unreachable!("clap requires the responder's flags when no verb is given"), } } /// The responder: answers every authorization request until its /// subscription ends. -async fn serve(args: Args) -> anyhow::Result<()> { +async fn serve(args: Args, stop: swarm_secret_client::Shutdown) -> anyhow::Result<()> { // The bucket is NOT a flag. `swarm_queue_client::status`'s own docs say // why: reader and writer must name the same bucket, and an option is a // way for two deployments to disagree about which one that is. This @@ -279,25 +280,33 @@ async fn serve(args: Args) -> anyhow::Result<()> { callout_xkey, issuer, }); - answer_each(requests, CONCURRENT_REQUESTS, move |msg| { - let responder = Arc::clone(&responder); - let client = client.clone(); - async move { - let Some((reply_to, reply)) = responder.answer(&msg).await else { - return; - }; - if let Err(e) = client.publish(reply_to, reply.into()).await { - tracing::warn!(error = ?e, "failed to publish auth response"); + answer_each( + requests.take_until(stop.clone()), + CONCURRENT_REQUESTS, + move |msg| { + let responder = Arc::clone(&responder); + let client = client.clone(); + async move { + let Some((reply_to, reply)) = responder.answer(&msg).await else { + return; + }; + if let Err(e) = client.publish(reply_to, reply.into()).await { + tracing::warn!(error = ?e, "failed to publish auth response"); + } } - } - }) + }, + ) .await; - anyhow::bail!("subscription to {AUTH_SUBJECT} ended") + if stop.peek().is_none() { + anyhow::bail!("subscription to {AUTH_SUBJECT} ended"); + } + Ok(()) } /// Runs `answer` on each of `requests`, at most `permits` at once. A request /// waits in `requests` until a permit is free; the reply carries `server_id` -/// and `user_nkey`, so the order answers go out in does not matter. +/// and `user_nkey`, so the order answers go out in does not matter. Once +/// `requests` ends, the answers in hand get the drain to finish. async fn answer_each( mut requests: impl futures_util::Stream + Unpin, permits: usize, @@ -307,16 +316,18 @@ async fn answer_each( Fut: std::future::Future + Send + 'static, { let permits = Arc::new(tokio::sync::Semaphore::new(permits)); + let mut in_flight = swarm_secret_client::InFlight::new(); while let Some(request) = requests.next().await { let Ok(permit) = Arc::clone(&permits).acquire_owned().await else { unreachable!("the semaphore is never closed"); }; let answering = answer(request); - tokio::spawn(async move { + in_flight.spawn(async move { answering.await; drop(permit); }); } + in_flight.drain().await; } /// Whether the server has stopped waiting for the answer to a request whose diff --git a/swarm-notify-router/src/main.rs b/swarm-notify-router/src/main.rs index fb3005b04..2d484d22b 100644 --- a/swarm-notify-router/src/main.rs +++ b/swarm-notify-router/src/main.rs @@ -35,7 +35,7 @@ use std::time::{Duration, Instant}; use anyhow::{Context as _, Result, bail}; use async_nats::jetstream::{self, consumer::pull, kv, message::PublishMessage}; -use futures_util::StreamExt as _; +use futures_util::{FutureExt as _, StreamExt as _}; use metrics::Handled; use swarm_queue_client::{ClientSecret, QueueConfig}; use swarm_secret_client::stored_secret::StoredSecret; @@ -79,6 +79,7 @@ const ROSTER_TTL: Duration = Duration::from_mins(1); #[tokio::main] async fn main() -> Result<()> { + let stop = swarm_secret_client::shutdown_signal(); hive_log::init(); swarm_secret_client::install_crypto_provider(); let var = |name: &str| { @@ -113,16 +114,26 @@ async fn main() -> Result<()> { stream = forge_events::STREAM, "notify router: routing forge events" ); - loop { - if let Err(e) = router.pull_once().await { - router.metrics.pull_failed(); - tracing::warn!( - error = format!("{e:#}"), - "notify router: pulling events failed; retrying" - ); - tokio::time::sleep(RETRY_DELAY).await; + while stop.clone().now_or_never().is_none() { + let pulled = + swarm_secret_client::serve_bounded(stop.clone(), router.pull_once(stop.clone())).await; + match pulled { + Some(Ok(())) => {} + Some(Err(e)) => { + router.metrics.pull_failed(); + tracing::warn!( + error = format!("{e:#}"), + "notify router: pulling events failed; retrying" + ); + tokio::select! { + () = stop.clone() => {} + () = tokio::time::sleep(RETRY_DELAY) => {} + } + } + None => break, } } + Ok(()) } struct Router { @@ -184,8 +195,11 @@ impl Router { }) } - /// Pull one batch and handle each event in stream order. - async fn pull_once(&mut self) -> Result<()> { + /// Pull one batch and handle each event in stream order, taking no event + /// after `stop` resolves. The one being handled finishes and is settled; + /// the rest of the batch is left un-acked and comes back after the ack + /// wait. + async fn pull_once(&mut self, stop: swarm_secret_client::Shutdown) -> Result<()> { if let Ok(info) = self.events.info().await { self.metrics .consumer(info.num_pending, info.num_ack_pending, info.num_redelivered); @@ -199,7 +213,10 @@ impl Router { .await .context("requesting a batch")?; let batch = batch.map(|m| m.map_err(|e| anyhow::anyhow!("{e}"))); - settle::batch(batch, async |m: &jetstream::Message| self.handle(m).await).await + settle::batch(batch, stop, async |m: &jetstream::Message| { + self.handle(m).await + }) + .await } /// Handle one event and count how it went. diff --git a/swarm-notify-router/src/settle.rs b/swarm-notify-router/src/settle.rs index e498c1dd2..034deb639 100644 --- a/swarm-notify-router/src/settle.rs +++ b/swarm-notify-router/src/settle.rs @@ -33,11 +33,15 @@ impl Pulled for async_nats::jetstream::Message { /// Handle each event of `batch` with `handle` and settle it. Returns the /// error of the first event that has to be retried, once the rest of the /// batch is nakked behind it. +/// +/// Once `stop` resolves no further event is taken: the one being handled is +/// settled, and the rest stay un-acked and come back after the ack wait. pub(crate) async fn batch( batch: impl Stream>, + stop: impl Future, mut handle: impl AsyncFnMut(&P) -> Result<(), Skip>, ) -> Result<()> { - let mut batch = std::pin::pin!(batch); + let mut batch = std::pin::pin!(batch.take_until(stop)); let mut failed: Option = None; while let Some(event) = batch.next().await { let event = event?; @@ -102,14 +106,18 @@ mod tests { }) }); let mut handled = Vec::new(); - let result = batch(futures_util::stream::iter(events), async |e: &Fake<'_>| { - handled.push(e.name); - if e.name == "assigned" { - Err(Skip::Retry(anyhow::anyhow!("the queue is down"))) - } else { - Ok(()) - } - }) + let result = batch( + futures_util::stream::iter(events), + std::future::pending(), + async |e: &Fake<'_>| { + handled.push(e.name); + if e.name == "assigned" { + Err(Skip::Retry(anyhow::anyhow!("the queue is down"))) + } else { + Ok(()) + } + }, + ) .await; assert!(result.is_err()); assert_eq!(handled, ["opened", "assigned"]); @@ -135,13 +143,17 @@ mod tests { settled: &settled, }) }); - let result = batch(futures_util::stream::iter(events), async |e: &Fake<'_>| { - if e.name == "bad" { - Err(Skip::Unusable("no".to_owned())) - } else { - Ok(()) - } - }) + let result = batch( + futures_util::stream::iter(events), + std::future::pending(), + async |e: &Fake<'_>| { + if e.name == "bad" { + Err(Skip::Unusable("no".to_owned())) + } else { + Ok(()) + } + }, + ) .await; assert!(result.is_ok()); assert_eq!( @@ -152,4 +164,44 @@ mod tests { ] ); } + + /// The stop arrives while `assigned` is being handled: it is still + /// settled, and `unassigned` and `closed` are neither handled nor settled, + /// so the queue redelivers them. + #[tokio::test] + async fn after_the_stop_the_event_in_hand_is_settled_and_the_rest_is_left_unacked() { + let settled = RefCell::new(Vec::new()); + let events = ["opened", "assigned", "unassigned", "closed"].map(|name| { + Ok(Fake { + name, + settled: &settled, + }) + }); + let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>(); + let mut stop_tx = Some(stop_tx); + let mut handled = Vec::new(); + let result = batch( + futures_util::stream::iter(events), + async { + let _ = stop_rx.await; + }, + async |e: &Fake<'_>| { + handled.push(e.name); + if e.name == "assigned" { + stop_tx.take().expect("sent once").send(()).expect("stop"); + } + Ok(()) + }, + ) + .await; + assert!(result.is_ok()); + assert_eq!(handled, ["opened", "assigned"]); + assert_eq!( + settled.into_inner(), + [ + ("opened", format!("{:?}", AckKind::Ack)), + ("assigned", format!("{:?}", AckKind::Ack)), + ] + ); + } } diff --git a/swarm-review-classifier/src/main.rs b/swarm-review-classifier/src/main.rs index b5cc39559..dfef53a15 100644 --- a/swarm-review-classifier/src/main.rs +++ b/swarm-review-classifier/src/main.rs @@ -47,6 +47,7 @@ fn main() -> Result<()> { } async fn serve() -> Result<()> { + let stop = swarm_secret_client::shutdown_signal(); hive_log::init(); swarm_secret_client::install_crypto_provider(); let var = |name: &str| { @@ -92,7 +93,8 @@ async fn serve() -> Result<()> { let mut requests = queue .queue_subscribe(SUBJECT, QUEUE_GROUP.to_owned()) .await - .with_context(|| format!("subscribing to {SUBJECT}"))?; + .with_context(|| format!("subscribing to {SUBJECT}"))? + .take_until(stop.clone()); tracing::info!(subject = SUBJECT, "review classifier: awaiting try-outs"); let models = std::sync::Arc::new(models); @@ -115,7 +117,10 @@ async fn serve() -> Result<()> { } }); } - bail!("the subscription to {SUBJECT} ended") + if stop.peek().is_none() { + bail!("the subscription to {SUBJECT} ended"); + } + Ok(()) } /// Decode one request, answer it and log the run. diff --git a/swarm-stt/src/main.rs b/swarm-stt/src/main.rs index 23921c291..90e2af7e6 100644 --- a/swarm-stt/src/main.rs +++ b/swarm-stt/src/main.rs @@ -115,6 +115,7 @@ async fn connect_queue(var: Var<'_>) -> Result { #[tokio::main] async fn main() -> Result<()> { + let stop = swarm_secret_client::shutdown_signal(); hive_log::init(); swarm_secret_client::install_crypto_provider(); let var = |name: &str| { @@ -151,7 +152,8 @@ async fn main() -> Result<()> { let mut requests = queue .queue_subscribe(subject_wildcard(), QUEUE_GROUP.to_owned()) .await - .context("subscribing to the request subject")?; + .context("subscribing to the request subject")? + .take_until(stop.clone()); tracing::info!( subject = subject_wildcard(), "swarm-stt: answering requests" @@ -207,7 +209,10 @@ async fn main() -> Result<()> { } }); } - bail!("the request subscription ended") + if stop.peek().is_none() { + bail!("the request subscription ended"); + } + Ok(()) } /// One agent's place among the requests in flight; dropping it frees the place.