Watch
0
0
Fork
You've already forked hyperhive
0

swarm-controller: a stop signal ends the startup secret-store reads

The queue client secret and the webhook secret are read at start through
a retry schedule of about a minute. Neither heard SIGTERM, because the
shutdown signal was only installed once the controller began serving, so
a stop during a store outage waited out the schedule or the unit's stop
timeout.

main installs the signal first and passes it to queue_identity::init and
webhook::load. A stop that arrives first ends the read and exits 0.
This commit is contained in:
maia 2026-11-19 09:06:48 +01:00 • committed by mara
commit 96cb25a040

View file

@ -126,8 +126,30 @@ pub(super) async fn load_or_generate_secret() -> Result<String> {
/// Every attempt is [`load_or_generate_secret`], so a retry only reads again: a
/// secret is minted only when the store answers that it holds none, never on a
/// failed read.
pub(crate) async fn load() -> Late<Arc<str>> {
load_with(|| async { load_or_generate_secret().await.map(Arc::from) }).await
///
/// `None` when `stop` resolves before the first attempt is over: the startup
/// retries of a store that is down end there instead of after about a minute.
pub(crate) async fn load(stop: impl Future<Output = ()>) -> Option<Late<Arc<str>>> {
load_or_stop(stop, || async {
load_or_generate_secret().await.map(Arc::from)
})
.await
}
/// [`load`], with `load` standing in for the store.
async fn load_or_stop<F, Fut>(stop: impl Future<Output = ()>, load: F) -> Option<Late<Arc<str>>>
where
F: FnMut() -> Fut + Send + 'static,
Fut: Future<Output = Result<Arc<str>>> + Send,
{
tokio::select! {
biased;
() = stop => {
tracing::info!("stop arrived while loading the webhook secret; exiting before startup finished");
None
}
slot = load_with(load) => Some(slot),
}
}
/// [`load`], with `load` standing in for the store.
@ -739,8 +761,8 @@ async fn announce_knowledge_change(state: &AppState) {
#[cfg(test)]
mod tests {
use super::{
DeliveryKind, Refusal, SecretSlot, Stored, load_or_generate_at, load_with, resolve,
secret_path_from, verify,
DeliveryKind, Refusal, SecretSlot, Stored, load_or_generate_at, load_or_stop, load_with,
resolve, secret_path_from, verify,
};
use axum::{body::Bytes, http::HeaderMap, http::StatusCode};
use hmac::{Hmac, KeyInit as _, Mac as _};
@ -1240,6 +1262,41 @@ mod tests {
);
}
/// A first read that waits out `wait` before it answers, as the startup
/// retries of a store that is down do.
fn answering_after(
wait: std::time::Duration,
) -> impl FnMut() -> std::pin::Pin<Box<dyn Future<Output = anyhow::Result<Arc<str>>> + Send>>
+ Send
+ 'static {
move || {
Box::pin(async move {
tokio::time::sleep(wait).await;
Ok(Arc::from(IN_STORE))
})
}
}
#[tokio::test(start_paused = true)]
async fn a_stop_during_the_first_read_ends_the_load_before_the_wait_is_over() {
let started = tokio::time::Instant::now();
let stop = tokio::time::sleep(std::time::Duration::from_secs(2));
let slot = load_or_stop(stop, answering_after(std::time::Duration::from_mins(1))).await;
assert!(slot.is_none());
assert_eq!(started.elapsed(), std::time::Duration::from_secs(2));
}
#[tokio::test(start_paused = true)]
async fn a_read_that_finishes_before_the_stop_fills_the_slot() {
let stop = tokio::time::sleep(std::time::Duration::from_mins(5));
let slot = load_or_stop(stop, answering_after(std::time::Duration::from_secs(3)))
.await
.expect("the read finished first");
assert_eq!(slot.get().map(|s| &**s), Some(IN_STORE));
}
#[tokio::test(start_paused = true)]
async fn a_store_answering_at_start_fills_the_slot_without_a_background_task() {
let (_dir, path) = legacy_file(None);