Watch
0
0
Fork
You've already forked hyperhive
0

queue consumers: stop taking messages on sigterm and exit 0

This commit is contained in:
atlas 2026-11-18 03:28:12 +01:00 • committed by mara
commit a8775f1a7a
7 changed files with 164 additions and 53 deletions

View file

@ -88,6 +88,7 @@ fn push_budget(reply_by: tokio::time::Instant) -> Duration {
#[tokio::main] #[tokio::main]
async fn main() -> Result<()> { async fn main() -> Result<()> {
let stop = swarm_secret_client::shutdown_signal();
hive_log::init(); hive_log::init();
swarm_secret_client::install_crypto_provider(); swarm_secret_client::install_crypto_provider();
let var = |name: &str| { let var = |name: &str| {
@ -174,8 +175,19 @@ async fn main() -> Result<()> {
let slot = Arc::new(tokio::sync::Semaphore::new(1)); let slot = Arc::new(tokio::sync::Semaphore::new(1));
serve_meta_builds(&queue, &publisher, Store, &builder, &slot, &metrics).await?; serve_meta_builds(&queue, &publisher, Store, &builder, &slot, &metrics).await?;
answer_builds(requests, publisher, Store, builder, slot, metrics).await; answer_builds(
bail!("the subscription to {SUBJECT} ended") requests.take_until(stop.clone()),
publisher,
Store,
builder,
slot,
metrics,
)
.await;
if stop.peek().is_none() {
bail!("the subscription to {SUBJECT} ended");
}
Ok(())
} }
/// Where a reply goes. [`Queue`] in production; tests record. /// Where a reply goes. [`Queue`] in production; tests record.
@ -260,6 +272,8 @@ async fn send_reply(
} }
/// Answer every PR check, each in its own task, with the build's result. /// Answer every PR check, each in its own task, with the build's result.
/// Returns once `requests` ends; a build still running is not waited for, and
/// its requester gets no reply, as when the process is killed.
async fn answer_builds( async fn answer_builds(
mut requests: impl Stream<Item = Message> + Unpin, mut requests: impl Stream<Item = Message> + Unpin,
publisher: impl Publisher, publisher: impl Publisher,

View file

@ -100,6 +100,7 @@ impl login::CredentialSink for AgentLeaf {
#[tokio::main] #[tokio::main]
async fn main() -> Result<()> { async fn main() -> Result<()> {
let stop = swarm_secret_client::shutdown_signal();
hive_log::init(); hive_log::init();
swarm_secret_client::install_crypto_provider(); swarm_secret_client::install_crypto_provider();
let var = |name: &str| { let var = |name: &str| {
@ -138,9 +139,11 @@ async fn main() -> Result<()> {
let mut requests = queue let mut requests = queue
.subscribe(SUBJECT) .subscribe(SUBJECT)
.await .await
.with_context(|| format!("subscribing to {SUBJECT}"))?; .with_context(|| format!("subscribing to {SUBJECT}"))?
.take_until(stop.clone());
tracing::info!(subject = SUBJECT, "claude login minter: awaiting logins"); tracing::info!(subject = SUBJECT, "claude login minter: awaiting logins");
let mut in_flight = swarm_secret_client::InFlight::new();
while let Some(msg) = requests.next().await { while let Some(msg) = requests.next().await {
let Some(reply_to) = msg.reply.clone() else { let Some(reply_to) = msg.reply.clone() else {
tracing::warn!("a login step arrived with no reply subject; dropping it"); tracing::warn!("a login step arrived with no reply subject; dropping it");
@ -149,7 +152,7 @@ async fn main() -> Result<()> {
let queue = queue.clone(); let queue = queue.clone();
let logins = Arc::clone(&logins); let logins = Arc::clone(&logins);
let metrics = Arc::clone(&metrics); let metrics = Arc::clone(&metrics);
tokio::spawn(async move { in_flight.spawn(async move {
let reply = handle(&msg.payload, &logins, &metrics).await; let reply = handle(&msg.payload, &logins, &metrics).await;
match serde_json::to_vec(&reply) { match serde_json::to_vec(&reply) {
Ok(body) => { Ok(body) => {
@ -161,7 +164,11 @@ async fn main() -> Result<()> {
} }
}); });
} }
bail!("the subscription to {SUBJECT} ended") in_flight.drain().await;
if stop.peek().is_none() {
bail!("the subscription to {SUBJECT} ended");
}
Ok(())
} }
/// Answer one step and count it. /// Answer one step and count it.

View file

@ -182,6 +182,7 @@ struct Args {
#[tokio::main] #[tokio::main]
async fn main() -> anyhow::Result<()> { async fn main() -> anyhow::Result<()> {
let stop = swarm_secret_client::shutdown_signal();
swarm_secret_client::install_crypto_provider(); swarm_secret_client::install_crypto_provider();
tracing_subscriber::fmt() tracing_subscriber::fmt()
.with_env_filter( .with_env_filter(
@ -210,14 +211,14 @@ async fn main() -> anyhow::Result<()> {
) )
.await .await
} }
(None, Some(args)) => serve(args).await, (None, Some(args)) => serve(args, stop).await,
(None, None) => unreachable!("clap requires the responder's flags when no verb is given"), (None, None) => unreachable!("clap requires the responder's flags when no verb is given"),
} }
} }
/// The responder: answers every authorization request until its /// The responder: answers every authorization request until its
/// subscription ends. /// subscription ends.
async fn serve(args: Args) -> anyhow::Result<()> { async fn serve(args: Args, stop: swarm_secret_client::Shutdown) -> anyhow::Result<()> {
// The bucket is NOT a flag. `swarm_queue_client::status`'s own docs say // The bucket is NOT a flag. `swarm_queue_client::status`'s own docs say
// why: reader and writer must name the same bucket, and an option is a // why: reader and writer must name the same bucket, and an option is a
// way for two deployments to disagree about which one that is. This // way for two deployments to disagree about which one that is. This
@ -279,25 +280,33 @@ async fn serve(args: Args) -> anyhow::Result<()> {
callout_xkey, callout_xkey,
issuer, issuer,
}); });
answer_each(requests, CONCURRENT_REQUESTS, move |msg| { answer_each(
let responder = Arc::clone(&responder); requests.take_until(stop.clone()),
let client = client.clone(); CONCURRENT_REQUESTS,
async move { move |msg| {
let Some((reply_to, reply)) = responder.answer(&msg).await else { let responder = Arc::clone(&responder);
return; let client = client.clone();
}; async move {
if let Err(e) = client.publish(reply_to, reply.into()).await { let Some((reply_to, reply)) = responder.answer(&msg).await else {
tracing::warn!(error = ?e, "failed to publish auth response"); return;
};
if let Err(e) = client.publish(reply_to, reply.into()).await {
tracing::warn!(error = ?e, "failed to publish auth response");
}
} }
} },
}) )
.await; .await;
anyhow::bail!("subscription to {AUTH_SUBJECT} ended") if stop.peek().is_none() {
anyhow::bail!("subscription to {AUTH_SUBJECT} ended");
}
Ok(())
} }
/// Runs `answer` on each of `requests`, at most `permits` at once. A request /// Runs `answer` on each of `requests`, at most `permits` at once. A request
/// waits in `requests` until a permit is free; the reply carries `server_id` /// waits in `requests` until a permit is free; the reply carries `server_id`
/// and `user_nkey`, so the order answers go out in does not matter. /// and `user_nkey`, so the order answers go out in does not matter. Once
/// `requests` ends, the answers in hand get the drain to finish.
async fn answer_each<R, F, Fut>( async fn answer_each<R, F, Fut>(
mut requests: impl futures_util::Stream<Item = R> + Unpin, mut requests: impl futures_util::Stream<Item = R> + Unpin,
permits: usize, permits: usize,
@ -307,16 +316,18 @@ async fn answer_each<R, F, Fut>(
Fut: std::future::Future<Output = ()> + Send + 'static, Fut: std::future::Future<Output = ()> + Send + 'static,
{ {
let permits = Arc::new(tokio::sync::Semaphore::new(permits)); let permits = Arc::new(tokio::sync::Semaphore::new(permits));
let mut in_flight = swarm_secret_client::InFlight::new();
while let Some(request) = requests.next().await { while let Some(request) = requests.next().await {
let Ok(permit) = Arc::clone(&permits).acquire_owned().await else { let Ok(permit) = Arc::clone(&permits).acquire_owned().await else {
unreachable!("the semaphore is never closed"); unreachable!("the semaphore is never closed");
}; };
let answering = answer(request); let answering = answer(request);
tokio::spawn(async move { in_flight.spawn(async move {
answering.await; answering.await;
drop(permit); drop(permit);
}); });
} }
in_flight.drain().await;
} }
/// Whether the server has stopped waiting for the answer to a request whose /// Whether the server has stopped waiting for the answer to a request whose

View file

@ -35,7 +35,7 @@ use std::time::{Duration, Instant};
use anyhow::{Context as _, Result, bail}; use anyhow::{Context as _, Result, bail};
use async_nats::jetstream::{self, consumer::pull, kv, message::PublishMessage}; use async_nats::jetstream::{self, consumer::pull, kv, message::PublishMessage};
use futures_util::StreamExt as _; use futures_util::{FutureExt as _, StreamExt as _};
use metrics::Handled; use metrics::Handled;
use swarm_queue_client::{ClientSecret, QueueConfig}; use swarm_queue_client::{ClientSecret, QueueConfig};
use swarm_secret_client::stored_secret::StoredSecret; use swarm_secret_client::stored_secret::StoredSecret;
@ -79,6 +79,7 @@ const ROSTER_TTL: Duration = Duration::from_mins(1);
#[tokio::main] #[tokio::main]
async fn main() -> Result<()> { async fn main() -> Result<()> {
let stop = swarm_secret_client::shutdown_signal();
hive_log::init(); hive_log::init();
swarm_secret_client::install_crypto_provider(); swarm_secret_client::install_crypto_provider();
let var = |name: &str| { let var = |name: &str| {
@ -113,16 +114,26 @@ async fn main() -> Result<()> {
stream = forge_events::STREAM, stream = forge_events::STREAM,
"notify router: routing forge events" "notify router: routing forge events"
); );
loop { while stop.clone().now_or_never().is_none() {
if let Err(e) = router.pull_once().await { let pulled =
router.metrics.pull_failed(); swarm_secret_client::serve_bounded(stop.clone(), router.pull_once(stop.clone())).await;
tracing::warn!( match pulled {
error = format!("{e:#}"), Some(Ok(())) => {}
"notify router: pulling events failed; retrying" Some(Err(e)) => {
); router.metrics.pull_failed();
tokio::time::sleep(RETRY_DELAY).await; tracing::warn!(
error = format!("{e:#}"),
"notify router: pulling events failed; retrying"
);
tokio::select! {
() = stop.clone() => {}
() = tokio::time::sleep(RETRY_DELAY) => {}
}
}
None => break,
} }
} }
Ok(())
} }
struct Router { struct Router {
@ -184,8 +195,11 @@ impl Router {
}) })
} }
/// Pull one batch and handle each event in stream order. /// Pull one batch and handle each event in stream order, taking no event
async fn pull_once(&mut self) -> Result<()> { /// after `stop` resolves. The one being handled finishes and is settled;
/// the rest of the batch is left un-acked and comes back after the ack
/// wait.
async fn pull_once(&mut self, stop: swarm_secret_client::Shutdown) -> Result<()> {
if let Ok(info) = self.events.info().await { if let Ok(info) = self.events.info().await {
self.metrics self.metrics
.consumer(info.num_pending, info.num_ack_pending, info.num_redelivered); .consumer(info.num_pending, info.num_ack_pending, info.num_redelivered);
@ -199,7 +213,10 @@ impl Router {
.await .await
.context("requesting a batch")?; .context("requesting a batch")?;
let batch = batch.map(|m| m.map_err(|e| anyhow::anyhow!("{e}"))); let batch = batch.map(|m| m.map_err(|e| anyhow::anyhow!("{e}")));
settle::batch(batch, async |m: &jetstream::Message| self.handle(m).await).await settle::batch(batch, stop, async |m: &jetstream::Message| {
self.handle(m).await
})
.await
} }
/// Handle one event and count how it went. /// Handle one event and count how it went.

View file

@ -33,11 +33,15 @@ impl Pulled for async_nats::jetstream::Message {
/// Handle each event of `batch` with `handle` and settle it. Returns the /// Handle each event of `batch` with `handle` and settle it. Returns the
/// error of the first event that has to be retried, once the rest of the /// error of the first event that has to be retried, once the rest of the
/// batch is nakked behind it. /// batch is nakked behind it.
///
/// Once `stop` resolves no further event is taken: the one being handled is
/// settled, and the rest stay un-acked and come back after the ack wait.
pub(crate) async fn batch<P: Pulled>( pub(crate) async fn batch<P: Pulled>(
batch: impl Stream<Item = Result<P>>, batch: impl Stream<Item = Result<P>>,
stop: impl Future<Output = ()>,
mut handle: impl AsyncFnMut(&P) -> Result<(), Skip>, mut handle: impl AsyncFnMut(&P) -> Result<(), Skip>,
) -> Result<()> { ) -> Result<()> {
let mut batch = std::pin::pin!(batch); let mut batch = std::pin::pin!(batch.take_until(stop));
let mut failed: Option<anyhow::Error> = None; let mut failed: Option<anyhow::Error> = None;
while let Some(event) = batch.next().await { while let Some(event) = batch.next().await {
let event = event?; let event = event?;
@ -102,14 +106,18 @@ mod tests {
}) })
}); });
let mut handled = Vec::new(); let mut handled = Vec::new();
let result = batch(futures_util::stream::iter(events), async |e: &Fake<'_>| { let result = batch(
handled.push(e.name); futures_util::stream::iter(events),
if e.name == "assigned" { std::future::pending(),
Err(Skip::Retry(anyhow::anyhow!("the queue is down"))) async |e: &Fake<'_>| {
} else { handled.push(e.name);
Ok(()) if e.name == "assigned" {
} Err(Skip::Retry(anyhow::anyhow!("the queue is down")))
}) } else {
Ok(())
}
},
)
.await; .await;
assert!(result.is_err()); assert!(result.is_err());
assert_eq!(handled, ["opened", "assigned"]); assert_eq!(handled, ["opened", "assigned"]);
@ -135,13 +143,17 @@ mod tests {
settled: &settled, settled: &settled,
}) })
}); });
let result = batch(futures_util::stream::iter(events), async |e: &Fake<'_>| { let result = batch(
if e.name == "bad" { futures_util::stream::iter(events),
Err(Skip::Unusable("no".to_owned())) std::future::pending(),
} else { async |e: &Fake<'_>| {
Ok(()) if e.name == "bad" {
} Err(Skip::Unusable("no".to_owned()))
}) } else {
Ok(())
}
},
)
.await; .await;
assert!(result.is_ok()); assert!(result.is_ok());
assert_eq!( assert_eq!(
@ -152,4 +164,44 @@ mod tests {
] ]
); );
} }
/// The stop arrives while `assigned` is being handled: it is still
/// settled, and `unassigned` and `closed` are neither handled nor settled,
/// so the queue redelivers them.
#[tokio::test]
async fn after_the_stop_the_event_in_hand_is_settled_and_the_rest_is_left_unacked() {
let settled = RefCell::new(Vec::new());
let events = ["opened", "assigned", "unassigned", "closed"].map(|name| {
Ok(Fake {
name,
settled: &settled,
})
});
let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>();
let mut stop_tx = Some(stop_tx);
let mut handled = Vec::new();
let result = batch(
futures_util::stream::iter(events),
async {
let _ = stop_rx.await;
},
async |e: &Fake<'_>| {
handled.push(e.name);
if e.name == "assigned" {
stop_tx.take().expect("sent once").send(()).expect("stop");
}
Ok(())
},
)
.await;
assert!(result.is_ok());
assert_eq!(handled, ["opened", "assigned"]);
assert_eq!(
settled.into_inner(),
[
("opened", format!("{:?}", AckKind::Ack)),
("assigned", format!("{:?}", AckKind::Ack)),
]
);
}
} }

View file

@ -47,6 +47,7 @@ fn main() -> Result<()> {
} }
async fn serve() -> Result<()> { async fn serve() -> Result<()> {
let stop = swarm_secret_client::shutdown_signal();
hive_log::init(); hive_log::init();
swarm_secret_client::install_crypto_provider(); swarm_secret_client::install_crypto_provider();
let var = |name: &str| { let var = |name: &str| {
@ -92,7 +93,8 @@ async fn serve() -> Result<()> {
let mut requests = queue let mut requests = queue
.queue_subscribe(SUBJECT, QUEUE_GROUP.to_owned()) .queue_subscribe(SUBJECT, QUEUE_GROUP.to_owned())
.await .await
.with_context(|| format!("subscribing to {SUBJECT}"))?; .with_context(|| format!("subscribing to {SUBJECT}"))?
.take_until(stop.clone());
tracing::info!(subject = SUBJECT, "review classifier: awaiting try-outs"); tracing::info!(subject = SUBJECT, "review classifier: awaiting try-outs");
let models = std::sync::Arc::new(models); let models = std::sync::Arc::new(models);
@ -115,7 +117,10 @@ async fn serve() -> Result<()> {
} }
}); });
} }
bail!("the subscription to {SUBJECT} ended") if stop.peek().is_none() {
bail!("the subscription to {SUBJECT} ended");
}
Ok(())
} }
/// Decode one request, answer it and log the run. /// Decode one request, answer it and log the run.

View file

@ -115,6 +115,7 @@ async fn connect_queue(var: Var<'_>) -> Result<async_nats::Client> {
#[tokio::main] #[tokio::main]
async fn main() -> Result<()> { async fn main() -> Result<()> {
let stop = swarm_secret_client::shutdown_signal();
hive_log::init(); hive_log::init();
swarm_secret_client::install_crypto_provider(); swarm_secret_client::install_crypto_provider();
let var = |name: &str| { let var = |name: &str| {
@ -151,7 +152,8 @@ async fn main() -> Result<()> {
let mut requests = queue let mut requests = queue
.queue_subscribe(subject_wildcard(), QUEUE_GROUP.to_owned()) .queue_subscribe(subject_wildcard(), QUEUE_GROUP.to_owned())
.await .await
.context("subscribing to the request subject")?; .context("subscribing to the request subject")?
.take_until(stop.clone());
tracing::info!( tracing::info!(
subject = subject_wildcard(), subject = subject_wildcard(),
"swarm-stt: answering requests" "swarm-stt: answering requests"
@ -207,7 +209,10 @@ async fn main() -> Result<()> {
} }
}); });
} }
bail!("the request subscription ended") if stop.peek().is_none() {
bail!("the request subscription ended");
}
Ok(())
} }
/// One agent's place among the requests in flight; dropping it frees the place. /// One agent's place among the requests in flight; dropping it frees the place.