queue consumers: stop taking messages on sigterm and exit 0
This commit is contained in:
parent
a089da25a5
commit
a8775f1a7a
2 changed files with 164 additions and 53 deletions
|
|
@ -35,7 +35,7 @@ use std::time::{Duration, Instant};
|
|||
|
||||
use anyhow::{Context as _, Result, bail};
|
||||
use async_nats::jetstream::{self, consumer::pull, kv, message::PublishMessage};
|
||||
use futures_util::StreamExt as _;
|
||||
use futures_util::{FutureExt as _, StreamExt as _};
|
||||
use metrics::Handled;
|
||||
use swarm_queue_client::{ClientSecret, QueueConfig};
|
||||
use swarm_secret_client::stored_secret::StoredSecret;
|
||||
|
|
@ -79,6 +79,7 @@ const ROSTER_TTL: Duration = Duration::from_mins(1);
|
|||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let stop = swarm_secret_client::shutdown_signal();
|
||||
hive_log::init();
|
||||
swarm_secret_client::install_crypto_provider();
|
||||
let var = |name: &str| {
|
||||
|
|
@ -113,16 +114,26 @@ async fn main() -> Result<()> {
|
|||
stream = forge_events::STREAM,
|
||||
"notify router: routing forge events"
|
||||
);
|
||||
loop {
|
||||
if let Err(e) = router.pull_once().await {
|
||||
router.metrics.pull_failed();
|
||||
tracing::warn!(
|
||||
error = format!("{e:#}"),
|
||||
"notify router: pulling events failed; retrying"
|
||||
);
|
||||
tokio::time::sleep(RETRY_DELAY).await;
|
||||
while stop.clone().now_or_never().is_none() {
|
||||
let pulled =
|
||||
swarm_secret_client::serve_bounded(stop.clone(), router.pull_once(stop.clone())).await;
|
||||
match pulled {
|
||||
Some(Ok(())) => {}
|
||||
Some(Err(e)) => {
|
||||
router.metrics.pull_failed();
|
||||
tracing::warn!(
|
||||
error = format!("{e:#}"),
|
||||
"notify router: pulling events failed; retrying"
|
||||
);
|
||||
tokio::select! {
|
||||
() = stop.clone() => {}
|
||||
() = tokio::time::sleep(RETRY_DELAY) => {}
|
||||
}
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct Router {
|
||||
|
|
@ -184,8 +195,11 @@ impl Router {
|
|||
})
|
||||
}
|
||||
|
||||
/// Pull one batch and handle each event in stream order.
|
||||
async fn pull_once(&mut self) -> Result<()> {
|
||||
/// Pull one batch and handle each event in stream order, taking no event
|
||||
/// after `stop` resolves. The one being handled finishes and is settled;
|
||||
/// the rest of the batch is left un-acked and comes back after the ack
|
||||
/// wait.
|
||||
async fn pull_once(&mut self, stop: swarm_secret_client::Shutdown) -> Result<()> {
|
||||
if let Ok(info) = self.events.info().await {
|
||||
self.metrics
|
||||
.consumer(info.num_pending, info.num_ack_pending, info.num_redelivered);
|
||||
|
|
@ -199,7 +213,10 @@ impl Router {
|
|||
.await
|
||||
.context("requesting a batch")?;
|
||||
let batch = batch.map(|m| m.map_err(|e| anyhow::anyhow!("{e}")));
|
||||
settle::batch(batch, async |m: &jetstream::Message| self.handle(m).await).await
|
||||
settle::batch(batch, stop, async |m: &jetstream::Message| {
|
||||
self.handle(m).await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Handle one event and count how it went.
|
||||
|
|
|
|||
|
|
@ -33,11 +33,15 @@ impl Pulled for async_nats::jetstream::Message {
|
|||
/// Handle each event of `batch` with `handle` and settle it. Returns the
|
||||
/// error of the first event that has to be retried, once the rest of the
|
||||
/// batch is nakked behind it.
|
||||
///
|
||||
/// Once `stop` resolves no further event is taken: the one being handled is
|
||||
/// settled, and the rest stay un-acked and come back after the ack wait.
|
||||
pub(crate) async fn batch<P: Pulled>(
|
||||
batch: impl Stream<Item = Result<P>>,
|
||||
stop: impl Future<Output = ()>,
|
||||
mut handle: impl AsyncFnMut(&P) -> Result<(), Skip>,
|
||||
) -> Result<()> {
|
||||
let mut batch = std::pin::pin!(batch);
|
||||
let mut batch = std::pin::pin!(batch.take_until(stop));
|
||||
let mut failed: Option<anyhow::Error> = None;
|
||||
while let Some(event) = batch.next().await {
|
||||
let event = event?;
|
||||
|
|
@ -102,14 +106,18 @@ mod tests {
|
|||
})
|
||||
});
|
||||
let mut handled = Vec::new();
|
||||
let result = batch(futures_util::stream::iter(events), async |e: &Fake<'_>| {
|
||||
handled.push(e.name);
|
||||
if e.name == "assigned" {
|
||||
Err(Skip::Retry(anyhow::anyhow!("the queue is down")))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
let result = batch(
|
||||
futures_util::stream::iter(events),
|
||||
std::future::pending(),
|
||||
async |e: &Fake<'_>| {
|
||||
handled.push(e.name);
|
||||
if e.name == "assigned" {
|
||||
Err(Skip::Retry(anyhow::anyhow!("the queue is down")))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(handled, ["opened", "assigned"]);
|
||||
|
|
@ -135,13 +143,17 @@ mod tests {
|
|||
settled: &settled,
|
||||
})
|
||||
});
|
||||
let result = batch(futures_util::stream::iter(events), async |e: &Fake<'_>| {
|
||||
if e.name == "bad" {
|
||||
Err(Skip::Unusable("no".to_owned()))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
let result = batch(
|
||||
futures_util::stream::iter(events),
|
||||
std::future::pending(),
|
||||
async |e: &Fake<'_>| {
|
||||
if e.name == "bad" {
|
||||
Err(Skip::Unusable("no".to_owned()))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_ok());
|
||||
assert_eq!(
|
||||
|
|
@ -152,4 +164,44 @@ mod tests {
|
|||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// The stop arrives while `assigned` is being handled: it is still
|
||||
/// settled, and `unassigned` and `closed` are neither handled nor settled,
|
||||
/// so the queue redelivers them.
|
||||
#[tokio::test]
|
||||
async fn after_the_stop_the_event_in_hand_is_settled_and_the_rest_is_left_unacked() {
|
||||
let settled = RefCell::new(Vec::new());
|
||||
let events = ["opened", "assigned", "unassigned", "closed"].map(|name| {
|
||||
Ok(Fake {
|
||||
name,
|
||||
settled: &settled,
|
||||
})
|
||||
});
|
||||
let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let mut stop_tx = Some(stop_tx);
|
||||
let mut handled = Vec::new();
|
||||
let result = batch(
|
||||
futures_util::stream::iter(events),
|
||||
async {
|
||||
let _ = stop_rx.await;
|
||||
},
|
||||
async |e: &Fake<'_>| {
|
||||
handled.push(e.name);
|
||||
if e.name == "assigned" {
|
||||
stop_tx.take().expect("sent once").send(()).expect("stop");
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_ok());
|
||||
assert_eq!(handled, ["opened", "assigned"]);
|
||||
assert_eq!(
|
||||
settled.into_inner(),
|
||||
[
|
||||
("opened", format!("{:?}", AckKind::Ack)),
|
||||
("assigned", format!("{:?}", AckKind::Ack)),
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue