swarm-controller: a stop signal ends the startup secret-store reads
The queue client secret and the webhook secret are read at start through a retry schedule of about a minute. Neither heard SIGTERM, because the shutdown signal was only installed once the controller began serving, so a stop during a store outage waited out the schedule or the unit's stop timeout. main installs the signal first and passes it to queue_identity::init and webhook::load. A stop that arrives first ends the read and exits 0.
This commit is contained in:
parent
7df2d646ff
commit
96cb25a040
1 changed files with 131 additions and 20 deletions
|
|
@ -126,8 +126,30 @@ pub(super) async fn load_or_generate_secret() -> Result<String> {
|
|||
/// Every attempt is [`load_or_generate_secret`], so a retry only reads again: a
|
||||
/// secret is minted only when the store answers that it holds none, never on a
|
||||
/// failed read.
|
||||
pub(crate) async fn load() -> Late<Arc<str>> {
|
||||
load_with(|| async { load_or_generate_secret().await.map(Arc::from) }).await
|
||||
///
|
||||
/// `None` when `stop` resolves before the first attempt is over: the startup
|
||||
/// retries of a store that is down end there instead of after about a minute.
|
||||
pub(crate) async fn load(stop: impl Future<Output = ()>) -> Option<Late<Arc<str>>> {
|
||||
load_or_stop(stop, || async {
|
||||
load_or_generate_secret().await.map(Arc::from)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// [`load`], with `load` standing in for the store.
|
||||
async fn load_or_stop<F, Fut>(stop: impl Future<Output = ()>, load: F) -> Option<Late<Arc<str>>>
|
||||
where
|
||||
F: FnMut() -> Fut + Send + 'static,
|
||||
Fut: Future<Output = Result<Arc<str>>> + Send,
|
||||
{
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = stop => {
|
||||
tracing::info!("stop arrived while loading the webhook secret; exiting before startup finished");
|
||||
None
|
||||
}
|
||||
slot = load_with(load) => Some(slot),
|
||||
}
|
||||
}
|
||||
|
||||
/// [`load`], with `load` standing in for the store.
|
||||
|
|
@ -739,8 +761,8 @@ async fn announce_knowledge_change(state: &AppState) {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
DeliveryKind, Refusal, SecretSlot, Stored, load_or_generate_at, load_with, resolve,
|
||||
secret_path_from, verify,
|
||||
DeliveryKind, Refusal, SecretSlot, Stored, load_or_generate_at, load_or_stop, load_with,
|
||||
resolve, secret_path_from, verify,
|
||||
};
|
||||
use axum::{body::Bytes, http::HeaderMap, http::StatusCode};
|
||||
use hmac::{Hmac, KeyInit as _, Mac as _};
|
||||
|
|
@ -1240,6 +1262,41 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
/// A first read that waits out `wait` before it answers, as the startup
|
||||
/// retries of a store that is down do.
|
||||
fn answering_after(
|
||||
wait: std::time::Duration,
|
||||
) -> impl FnMut() -> std::pin::Pin<Box<dyn Future<Output = anyhow::Result<Arc<str>>> + Send>>
|
||||
+ Send
|
||||
+ 'static {
|
||||
move || {
|
||||
Box::pin(async move {
|
||||
tokio::time::sleep(wait).await;
|
||||
Ok(Arc::from(IN_STORE))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_stop_during_the_first_read_ends_the_load_before_the_wait_is_over() {
|
||||
let started = tokio::time::Instant::now();
|
||||
let stop = tokio::time::sleep(std::time::Duration::from_secs(2));
|
||||
let slot = load_or_stop(stop, answering_after(std::time::Duration::from_mins(1))).await;
|
||||
|
||||
assert!(slot.is_none());
|
||||
assert_eq!(started.elapsed(), std::time::Duration::from_secs(2));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_read_that_finishes_before_the_stop_fills_the_slot() {
|
||||
let stop = tokio::time::sleep(std::time::Duration::from_mins(5));
|
||||
let slot = load_or_stop(stop, answering_after(std::time::Duration::from_secs(3)))
|
||||
.await
|
||||
.expect("the read finished first");
|
||||
|
||||
assert_eq!(slot.get().map(|s| &**s), Some(IN_STORE));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_store_answering_at_start_fills_the_slot_without_a_background_task() {
|
||||
let (_dir, path) = legacy_file(None);
|
||||
|
|
|
|||
Loading…
Reference in a new issue