Watch
0
0
Fork
You've already forked hyperhive
0

swarm daemons exit 0 when told to stop while still waiting for the store at startup

This commit is contained in:
maia 2026-11-19 05:34:54 +01:00 • committed by mara
commit ae37c228b1
7 changed files with 133 additions and 32 deletions

View file

@ -113,11 +113,15 @@ async fn main() -> Result<()> {
system: var("SYSTEM")?, system: var("SYSTEM")?,
cache: push_target()?, cache: push_target()?,
}); });
let secret = let Some(secret) = swarm_secret_client::read_with_startup_retry_or_stop(
swarm_secret_client::read_with_startup_retry("reading the queue client secret", || { "reading the queue client secret",
client_secret(&client_id) stop.clone(),
}) || client_secret(&client_id),
.await?; )
.await?
else {
return Ok(());
};
let evaluator = archetypes::Evaluator { let evaluator = archetypes::Evaluator {
nix: builder.nix.clone(), nix: builder.nix.clone(),
hyperhive: builder.hyperhive.clone(), hyperhive: builder.hyperhive.clone(),

View file

@ -119,11 +119,15 @@ async fn main() -> Result<()> {
std::env::temp_dir(), std::env::temp_dir(),
STEP_DEADLINE, STEP_DEADLINE,
)); ));
let secret = let Some(secret) = swarm_secret_client::read_with_startup_retry_or_stop(
swarm_secret_client::read_with_startup_retry("reading the queue client secret", || { "reading the queue client secret",
client_secret(&client_id) stop.clone(),
}) || client_secret(&client_id),
.await?; )
.await?
else {
return Ok(());
};
let reread_id = client_id.clone(); let reread_id = client_id.clone();
let secret = StoredSecret::new(client_id.clone(), secret, move || { let secret = StoredSecret::new(client_id.clone(), secret, move || {
let client_id = reread_id.clone(); let client_id = reread_id.clone();

View file

@ -100,11 +100,15 @@ async fn main() -> Result<()> {
.await .await
.context("starting the health listener")?; .context("starting the health listener")?;
let client_id = var("OIDC_CLIENT_ID")?; let client_id = var("OIDC_CLIENT_ID")?;
let secret = let Some(secret) = swarm_secret_client::read_with_startup_retry_or_stop(
swarm_secret_client::read_with_startup_retry("reading the queue client secret", || { "reading the queue client secret",
client_secret(&client_id) stop.clone(),
}) || client_secret(&client_id),
.await?; )
.await?
else {
return Ok(());
};
let reread_id = client_id.clone(); let reread_id = client_id.clone();
let secret = StoredSecret::new(client_id.clone(), secret, move || { let secret = StoredSecret::new(client_id.clone(), secret, move || {
let client_id = reread_id.clone(); let client_id = reread_id.clone();

View file

@ -75,11 +75,15 @@ async fn serve() -> Result<()> {
cwd: std::env::temp_dir().join("review-classifier-acp-cwd"), cwd: std::env::temp_dir().join("review-classifier-acp-cwd"),
}, },
}; };
let secret = let Some(secret) = swarm_secret_client::read_with_startup_retry_or_stop(
swarm_secret_client::read_with_startup_retry("reading the queue client secret", || { "reading the queue client secret",
client_secret(&client_id) stop.clone(),
}) || client_secret(&client_id),
.await?; )
.await?
else {
return Ok(());
};
let reread_id = client_id.clone(); let reread_id = client_id.clone();
let secret = StoredSecret::new(client_id.clone(), secret, move || { let secret = StoredSecret::new(client_id.clone(), secret, move || {
let client_id = reread_id.clone(); let client_id = reread_id.clone();

View file

@ -62,7 +62,7 @@ pub mod stored_secret;
pub use client::SecretStore; pub use client::SecretStore;
pub use shutdown::{DRAIN, InFlight, Shutdown, serve_bounded, shutdown_signal}; pub use shutdown::{DRAIN, InFlight, Shutdown, serve_bounded, shutdown_signal};
pub use startup_read::read_with_startup_retry; pub use startup_read::{read_with_startup_retry, read_with_startup_retry_or_stop};
/// What can go wrong between "we have a client certificate" and "we have the /// What can go wrong between "we have a client certificate" and "we have the
/// credential". /// credential".

View file

@ -25,6 +25,9 @@ const RETRIES: usize = 6;
/// with the first success or the last attempt's error. Each failure is logged /// with the first success or the last attempt's error. Each failure is logged
/// as a warning naming `what`. /// as a warning naming `what`.
/// ///
/// This read doesn't hear a stop signal: a caller that already has one in hand
/// uses [`read_with_startup_retry_or_stop`].
///
/// # Errors /// # Errors
/// The error of the final attempt, once every read has failed. /// The error of the final attempt, once every read has failed.
pub async fn read_with_startup_retry<T, E, F, Fut>(what: &str, read: F) -> Result<T, E> pub async fn read_with_startup_retry<T, E, F, Fut>(what: &str, read: F) -> Result<T, E>
@ -45,6 +48,36 @@ where
.await .await
} }
/// [`read_with_startup_retry`], ended by `stop`.
///
/// When `stop` resolves, in a sleep or during an attempt, the read is dropped
/// without another attempt and the answer is `Ok(None)`. The caller exits 0, as
/// it does for the same signal later on, so `systemctl stop` or a pod delete
/// during a store outage returns at once instead of after the schedule (about a
/// minute) or the unit's stop timeout.
///
/// # Errors
/// The error of the final attempt, once every read has failed.
pub async fn read_with_startup_retry_or_stop<T, E, F, Fut>(
what: &str,
stop: impl Future<Output = ()>,
read: F,
) -> Result<Option<T>, E>
where
F: FnMut() -> Fut,
Fut: Future<Output = Result<T, E>>,
E: Display,
{
tokio::select! {
biased;
() = stop => {
tracing::info!("stop arrived while {what}; exiting before startup finished");
Ok(None)
}
read = read_with_startup_retry(what, read) => read.map(Some),
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::atomic::{AtomicUsize, Ordering};
@ -85,6 +118,48 @@ mod tests {
assert_eq!(started.elapsed(), Duration::from_secs(1 + 2 + 4)); assert_eq!(started.elapsed(), Duration::from_secs(1 + 2 + 4));
} }
/// Resolves `after` from now, like a SIGTERM that arrives mid-outage.
async fn stop_after(after: Duration) {
tokio::time::sleep(after).await;
}
#[tokio::test(start_paused = true)]
async fn a_stop_during_a_retry_wait_ends_the_read_without_another_attempt() {
// Attempts at 0 s and 1 s fail; the wait after the second runs 1 s to 3 s.
let calls = AtomicUsize::new(0);
let started = Instant::now();
let got =
read_with_startup_retry_or_stop("reading", stop_after(Duration::from_secs(2)), || {
read_after(usize::MAX, &calls)
})
.await;
assert_eq!(got, Ok(None));
assert_eq!(calls.load(Ordering::SeqCst), 2);
assert_eq!(started.elapsed(), Duration::from_secs(2));
}
#[tokio::test(start_paused = true)]
async fn a_stop_that_has_already_fired_makes_no_attempt() {
let calls = AtomicUsize::new(0);
let got = read_with_startup_retry_or_stop("reading", std::future::ready(()), || {
read_after(0, &calls)
})
.await;
assert_eq!(got, Ok(None));
assert_eq!(calls.load(Ordering::SeqCst), 0);
}
#[tokio::test(start_paused = true)]
async fn a_stop_after_the_read_succeeded_changes_nothing() {
let calls = AtomicUsize::new(0);
let got =
read_with_startup_retry_or_stop("reading", stop_after(Duration::from_mins(1)), || {
read_after(1, &calls)
})
.await;
assert_eq!(got, Ok(Some("secret")));
}
#[tokio::test(start_paused = true)] #[tokio::test(start_paused = true)]
async fn a_store_that_stays_down_fails_after_the_whole_schedule() { async fn a_store_that_stays_down_fails_after_the_whole_schedule() {
let calls = AtomicUsize::new(0); let calls = AtomicUsize::new(0);

View file

@ -87,20 +87,28 @@ fn settings(var: Var<'_>, number: Number<'_>) -> Result<Settings> {
}) })
} }
/// The queue connection, as the service's own OIDC client. /// The queue connection, as the service's own OIDC client. `None` when `stop`
async fn connect_queue(var: Var<'_>) -> Result<async_nats::Client> { /// fired while the secret was still being waited for.
async fn connect_queue(
var: Var<'_>,
stop: swarm_secret_client::Shutdown,
) -> Result<Option<async_nats::Client>> {
let client_id = var("OIDC_CLIENT_ID")?; let client_id = var("OIDC_CLIENT_ID")?;
let secret = let Some(secret) = swarm_secret_client::read_with_startup_retry_or_stop(
swarm_secret_client::read_with_startup_retry("reading the queue client secret", || { "reading the queue client secret",
client_secret(&client_id) stop.clone(),
}) || client_secret(&client_id),
.await?; )
.await?
else {
return Ok(None);
};
let reread_id = client_id.clone(); let reread_id = client_id.clone();
let secret = StoredSecret::new(client_id.clone(), secret, move || { let secret = StoredSecret::new(client_id.clone(), secret, move || {
let client_id = reread_id.clone(); let client_id = reread_id.clone();
async move { client_secret(&client_id).await.map_err(Into::into) } async move { client_secret(&client_id).await.map_err(Into::into) }
}); });
swarm_queue_client::connect(QueueConfig { let queue = swarm_queue_client::connect(QueueConfig {
url: var("NATS_URL")?, url: var("NATS_URL")?,
token_endpoint: var("OIDC_TOKEN_ENDPOINT")?, token_endpoint: var("OIDC_TOKEN_ENDPOINT")?,
client_id, client_id,
@ -109,8 +117,8 @@ async fn connect_queue(var: Var<'_>) -> Result<async_nats::Client> {
.ok() .ok()
.map(PathBuf::from), .map(PathBuf::from),
}) })
.await .await?;
.map_err(Into::into) Ok(Some(queue))
} }
/// Removes expired transcripts from `cache` every [`PURGE_EVERY`], for as long as /// Removes expired transcripts from `cache` every [`PURGE_EVERY`], for as long as
@ -155,7 +163,9 @@ async fn main() -> Result<()> {
waiting, waiting,
} = settings(&var, &number)?; } = settings(&var, &number)?;
let per_agent: Arc<Mutex<HashSet<String>>> = Arc::default(); let per_agent: Arc<Mutex<HashSet<String>>> = Arc::default();
let queue = connect_queue(&var).await?; let Some(queue) = connect_queue(&var, stop.clone()).await? else {
return Ok(());
};
swarm_queue_client::ready_when_connected(&health, &queue); swarm_queue_client::ready_when_connected(&health, &queue);
let metrics = Arc::new(Metrics::from_env()); let metrics = Arc::new(Metrics::from_env());