swarm-logs-auth, swarm-metrics-auth, swarm-authelia-bridge, swarm-controller: drain http on sigterm
This commit is contained in:
parent
39c29b64e3
commit
a089da25a5
1 changed files with 198 additions and 19 deletions
|
|
@ -443,7 +443,10 @@ async fn open_consumer(
|
|||
|
||||
/// Take the queued review requests and review each, for as long as the
|
||||
/// controller runs, once this host has forge access.
|
||||
pub(crate) fn spawn_queue(state: &AppState) {
|
||||
pub(crate) fn spawn_queue(
|
||||
state: &AppState,
|
||||
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||
) {
|
||||
let Some(client) = state.status.as_ref().map(|s| s.queue_client()) else {
|
||||
tracing::warn!("no swarm queue is configured on this host; requested reviews don't run");
|
||||
return;
|
||||
|
|
@ -466,7 +469,7 @@ pub(crate) fn spawn_queue(state: &AppState) {
|
|||
runs,
|
||||
metrics: Metrics::global(),
|
||||
};
|
||||
consume(&client, TIMING, &reviews, handles).await;
|
||||
consume(&client, TIMING, &reviews, handles, stop).await;
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -494,8 +497,9 @@ impl<S, Q: Clone, P> Clone for Handles<S, Q, P> {
|
|||
}
|
||||
|
||||
/// Take requests from [`REQUEST_STREAM`] one at a time, each once a run
|
||||
/// place is free, and review each in a task of its own. Never returns;
|
||||
/// dropping it stops every review it started, and the queue delivers their
|
||||
/// place is free, and review each in a task of its own. Returns once `stop`
|
||||
/// resolves, taking no request after it; dropping it, as the return does, stops
|
||||
/// every review it started without acking, and the queue delivers their
|
||||
/// requests again.
|
||||
///
|
||||
/// The limit is read from the settings before each wait for a place. A
|
||||
|
|
@ -506,6 +510,7 @@ async fn consume<S, Q, P>(
|
|||
timing: Timing,
|
||||
reviews: &Arc<Reviews>,
|
||||
handles: Handles<S, Q, P>,
|
||||
stop: impl Future<Output = ()> + Clone,
|
||||
) where
|
||||
S: FileSource + PullSource + Send + Sync + 'static,
|
||||
Q: RunSink + SettingsSource,
|
||||
|
|
@ -519,7 +524,11 @@ async fn consume<S, Q, P>(
|
|||
"opening the review request queue failed; trying again"
|
||||
),
|
||||
}
|
||||
tokio::time::sleep(timing.pull_wait).await;
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = stop.clone() => return,
|
||||
() = tokio::time::sleep(timing.pull_wait) => {}
|
||||
}
|
||||
};
|
||||
let mut tasks = JoinSet::new();
|
||||
loop {
|
||||
|
|
@ -536,8 +545,17 @@ async fn consume<S, Q, P>(
|
|||
the last limit read"
|
||||
),
|
||||
}
|
||||
let slot = reviews.slot().await;
|
||||
let message = match next(&consumer, timing.pull_wait).await {
|
||||
let slot = tokio::select! {
|
||||
biased;
|
||||
() = stop.clone() => return,
|
||||
slot = reviews.slot() => slot,
|
||||
};
|
||||
let taken = tokio::select! {
|
||||
biased;
|
||||
() = stop.clone() => return,
|
||||
taken = next(&consumer, timing.pull_wait) => taken,
|
||||
};
|
||||
let message = match taken {
|
||||
Ok(Some(message)) => message,
|
||||
Ok(None) => continue,
|
||||
Err(e) => {
|
||||
|
|
@ -546,7 +564,11 @@ async fn consume<S, Q, P>(
|
|||
"taking a review request from the queue failed; trying again"
|
||||
);
|
||||
drop(slot);
|
||||
tokio::time::sleep(timing.pull_wait).await;
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = stop.clone() => return,
|
||||
() = tokio::time::sleep(timing.pull_wait) => {}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
|
@ -2032,6 +2054,15 @@ mod tests {
|
|||
|
||||
/// A controller taking requests, its run places in `reviews`.
|
||||
fn consume(&self, reviews: &Arc<Reviews>) -> JoinHandle<()> {
|
||||
self.consume_until(reviews, std::future::pending())
|
||||
}
|
||||
|
||||
/// A controller that takes no request once `stop` resolves.
|
||||
fn consume_until(
|
||||
&self,
|
||||
reviews: &Arc<Reviews>,
|
||||
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||
) -> JoinHandle<()> {
|
||||
let client = self.client.clone();
|
||||
let reviews = Arc::clone(reviews);
|
||||
let forge = Arc::clone(&self.rig.forge);
|
||||
|
|
@ -2047,7 +2078,7 @@ mod tests {
|
|||
runs: store,
|
||||
metrics,
|
||||
};
|
||||
consume(&client, TEST_TIMING, &reviews, handles).await;
|
||||
consume(&client, TEST_TIMING, &reviews, handles, stop).await;
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -2113,6 +2144,36 @@ mod tests {
|
|||
assert_eq!(q.rig.gate.reached(), 2, "no run after the ack");
|
||||
}
|
||||
|
||||
/// After the stop the controller returns and takes no request, and the
|
||||
/// review it was running stays un-acked, so the queue still holds it.
|
||||
#[tokio::test]
|
||||
async fn after_the_stop_no_request_is_taken_and_the_running_one_stays_unacked() {
|
||||
let q = Queued::new().await;
|
||||
q.request(7, "abc123").await;
|
||||
let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let stop = futures_util::FutureExt::shared(async move {
|
||||
let _ = stop_rx.await;
|
||||
});
|
||||
let consumer = q.consume_until(&q.rig.reviews, stop);
|
||||
until("the first run started", || q.rig.gate.reached() == 1).await;
|
||||
|
||||
stop_tx.send(()).expect("stop");
|
||||
tokio::time::timeout(Duration::from_secs(5), consumer)
|
||||
.await
|
||||
.expect("consume returns after the stop")
|
||||
.expect("consume did not panic");
|
||||
// A request that arrives now, and the one running, which the queue
|
||||
// would redeliver after its ack wait: neither runs.
|
||||
q.request(8, "abc123").await;
|
||||
tokio::time::sleep(TEST_TIMING.ack_wait * 3).await;
|
||||
assert_eq!(q.rig.gate.reached(), 1, "no request taken after the stop");
|
||||
assert!(
|
||||
q.rig.posted().is_empty(),
|
||||
"the unfinished review is not posted"
|
||||
);
|
||||
assert_eq!(q.waiting().await, 2, "neither request is acked");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_newer_request_for_a_pr_replaces_the_waiting_one() {
|
||||
let q = Queued::new().await;
|
||||
|
|
|
|||
Loading…
Reference in a new issue