Watch
0
0
Fork
You've already forked hyperhive
0

swarm-logs-auth, swarm-metrics-auth, swarm-authelia-bridge, swarm-controller: drain http on sigterm

This commit is contained in:
atlas 2026-11-18 03:28:12 +01:00 • committed by mara
commit a089da25a5

View file

@ -443,7 +443,10 @@ async fn open_consumer(
/// Take the queued review requests and review each, for as long as the
/// controller runs, once this host has forge access.
pub(crate) fn spawn_queue(state: &AppState) {
pub(crate) fn spawn_queue(
state: &AppState,
stop: impl Future<Output = ()> + Clone + Send + 'static,
) {
let Some(client) = state.status.as_ref().map(|s| s.queue_client()) else {
tracing::warn!("no swarm queue is configured on this host; requested reviews don't run");
return;
@ -466,7 +469,7 @@ pub(crate) fn spawn_queue(state: &AppState) {
runs,
metrics: Metrics::global(),
};
consume(&client, TIMING, &reviews, handles).await;
consume(&client, TIMING, &reviews, handles, stop).await;
});
}
@ -494,8 +497,9 @@ impl<S, Q: Clone, P> Clone for Handles<S, Q, P> {
}
/// Take requests from [`REQUEST_STREAM`] one at a time, each once a run
/// place is free, and review each in a task of its own. Never returns;
/// dropping it stops every review it started, and the queue delivers their
/// place is free, and review each in a task of its own. Returns once `stop`
/// resolves, taking no request after it; dropping it, as the return does, stops
/// every review it started without acking, and the queue delivers their
/// requests again.
///
/// The limit is read from the settings before each wait for a place. A
@ -506,6 +510,7 @@ async fn consume<S, Q, P>(
timing: Timing,
reviews: &Arc<Reviews>,
handles: Handles<S, Q, P>,
stop: impl Future<Output = ()> + Clone,
) where
S: FileSource + PullSource + Send + Sync + 'static,
Q: RunSink + SettingsSource,
@ -519,7 +524,11 @@ async fn consume<S, Q, P>(
"opening the review request queue failed; trying again"
),
}
tokio::time::sleep(timing.pull_wait).await;
tokio::select! {
biased;
() = stop.clone() => return,
() = tokio::time::sleep(timing.pull_wait) => {}
}
};
let mut tasks = JoinSet::new();
loop {
@ -536,8 +545,17 @@ async fn consume<S, Q, P>(
the last limit read"
),
}
let slot = reviews.slot().await;
let message = match next(&consumer, timing.pull_wait).await {
let slot = tokio::select! {
biased;
() = stop.clone() => return,
slot = reviews.slot() => slot,
};
let taken = tokio::select! {
biased;
() = stop.clone() => return,
taken = next(&consumer, timing.pull_wait) => taken,
};
let message = match taken {
Ok(Some(message)) => message,
Ok(None) => continue,
Err(e) => {
@ -546,7 +564,11 @@ async fn consume<S, Q, P>(
"taking a review request from the queue failed; trying again"
);
drop(slot);
tokio::time::sleep(timing.pull_wait).await;
tokio::select! {
biased;
() = stop.clone() => return,
() = tokio::time::sleep(timing.pull_wait) => {}
}
continue;
}
};
@ -2032,6 +2054,15 @@ mod tests {
/// A controller taking requests, its run places in `reviews`.
fn consume(&self, reviews: &Arc<Reviews>) -> JoinHandle<()> {
self.consume_until(reviews, std::future::pending())
}
/// A controller that takes no request once `stop` resolves.
fn consume_until(
&self,
reviews: &Arc<Reviews>,
stop: impl Future<Output = ()> + Clone + Send + 'static,
) -> JoinHandle<()> {
let client = self.client.clone();
let reviews = Arc::clone(reviews);
let forge = Arc::clone(&self.rig.forge);
@ -2047,7 +2078,7 @@ mod tests {
runs: store,
metrics,
};
consume(&client, TEST_TIMING, &reviews, handles).await;
consume(&client, TEST_TIMING, &reviews, handles, stop).await;
})
}
@ -2113,6 +2144,36 @@ mod tests {
assert_eq!(q.rig.gate.reached(), 2, "no run after the ack");
}
/// After the stop the controller returns and takes no request, and the
/// review it was running stays un-acked, so the queue still holds it.
#[tokio::test]
async fn after_the_stop_no_request_is_taken_and_the_running_one_stays_unacked() {
let q = Queued::new().await;
q.request(7, "abc123").await;
let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>();
let stop = futures_util::FutureExt::shared(async move {
let _ = stop_rx.await;
});
let consumer = q.consume_until(&q.rig.reviews, stop);
until("the first run started", || q.rig.gate.reached() == 1).await;
stop_tx.send(()).expect("stop");
tokio::time::timeout(Duration::from_secs(5), consumer)
.await
.expect("consume returns after the stop")
.expect("consume did not panic");
// A request that arrives now, and the one running, which the queue
// would redeliver after its ack wait: neither runs.
q.request(8, "abc123").await;
tokio::time::sleep(TEST_TIMING.ack_wait * 3).await;
assert_eq!(q.rig.gate.reached(), 1, "no request taken after the stop");
assert!(
q.rig.posted().is_empty(),
"the unfinished review is not posted"
);
assert_eq!(q.waiting().await, 2, "neither request is acked");
}
#[tokio::test]
async fn a_newer_request_for_a_pr_replaces_the_waiting_one() {
let q = Queued::new().await;