swarm daemons exit 0 when told to stop while still waiting for the store at startup
This commit is contained in:
parent
5dd39ba88c
commit
ae37c228b1
7 changed files with 133 additions and 32 deletions
|
|
@ -113,11 +113,15 @@ async fn main() -> Result<()> {
|
|||
system: var("SYSTEM")?,
|
||||
cache: push_target()?,
|
||||
});
|
||||
let secret =
|
||||
swarm_secret_client::read_with_startup_retry("reading the queue client secret", || {
|
||||
client_secret(&client_id)
|
||||
})
|
||||
.await?;
|
||||
let Some(secret) = swarm_secret_client::read_with_startup_retry_or_stop(
|
||||
"reading the queue client secret",
|
||||
stop.clone(),
|
||||
|| client_secret(&client_id),
|
||||
)
|
||||
.await?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let evaluator = archetypes::Evaluator {
|
||||
nix: builder.nix.clone(),
|
||||
hyperhive: builder.hyperhive.clone(),
|
||||
|
|
|
|||
|
|
@ -119,11 +119,15 @@ async fn main() -> Result<()> {
|
|||
std::env::temp_dir(),
|
||||
STEP_DEADLINE,
|
||||
));
|
||||
let secret =
|
||||
swarm_secret_client::read_with_startup_retry("reading the queue client secret", || {
|
||||
client_secret(&client_id)
|
||||
})
|
||||
.await?;
|
||||
let Some(secret) = swarm_secret_client::read_with_startup_retry_or_stop(
|
||||
"reading the queue client secret",
|
||||
stop.clone(),
|
||||
|| client_secret(&client_id),
|
||||
)
|
||||
.await?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let reread_id = client_id.clone();
|
||||
let secret = StoredSecret::new(client_id.clone(), secret, move || {
|
||||
let client_id = reread_id.clone();
|
||||
|
|
|
|||
|
|
@ -100,11 +100,15 @@ async fn main() -> Result<()> {
|
|||
.await
|
||||
.context("starting the health listener")?;
|
||||
let client_id = var("OIDC_CLIENT_ID")?;
|
||||
let secret =
|
||||
swarm_secret_client::read_with_startup_retry("reading the queue client secret", || {
|
||||
client_secret(&client_id)
|
||||
})
|
||||
.await?;
|
||||
let Some(secret) = swarm_secret_client::read_with_startup_retry_or_stop(
|
||||
"reading the queue client secret",
|
||||
stop.clone(),
|
||||
|| client_secret(&client_id),
|
||||
)
|
||||
.await?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let reread_id = client_id.clone();
|
||||
let secret = StoredSecret::new(client_id.clone(), secret, move || {
|
||||
let client_id = reread_id.clone();
|
||||
|
|
|
|||
|
|
@ -75,11 +75,15 @@ async fn serve() -> Result<()> {
|
|||
cwd: std::env::temp_dir().join("review-classifier-acp-cwd"),
|
||||
},
|
||||
};
|
||||
let secret =
|
||||
swarm_secret_client::read_with_startup_retry("reading the queue client secret", || {
|
||||
client_secret(&client_id)
|
||||
})
|
||||
.await?;
|
||||
let Some(secret) = swarm_secret_client::read_with_startup_retry_or_stop(
|
||||
"reading the queue client secret",
|
||||
stop.clone(),
|
||||
|| client_secret(&client_id),
|
||||
)
|
||||
.await?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let reread_id = client_id.clone();
|
||||
let secret = StoredSecret::new(client_id.clone(), secret, move || {
|
||||
let client_id = reread_id.clone();
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ pub mod stored_secret;
|
|||
|
||||
pub use client::SecretStore;
|
||||
pub use shutdown::{DRAIN, InFlight, Shutdown, serve_bounded, shutdown_signal};
|
||||
pub use startup_read::read_with_startup_retry;
|
||||
pub use startup_read::{read_with_startup_retry, read_with_startup_retry_or_stop};
|
||||
|
||||
/// What can go wrong between "we have a client certificate" and "we have the
|
||||
/// credential".
|
||||
|
|
|
|||
|
|
@ -25,6 +25,9 @@ const RETRIES: usize = 6;
|
|||
/// with the first success or the last attempt's error. Each failure is logged
|
||||
/// as a warning naming `what`.
|
||||
///
|
||||
/// This read doesn't hear a stop signal: a caller that already has one in hand
|
||||
/// uses [`read_with_startup_retry_or_stop`].
|
||||
///
|
||||
/// # Errors
|
||||
/// The error of the final attempt, once every read has failed.
|
||||
pub async fn read_with_startup_retry<T, E, F, Fut>(what: &str, read: F) -> Result<T, E>
|
||||
|
|
@ -45,6 +48,36 @@ where
|
|||
.await
|
||||
}
|
||||
|
||||
/// [`read_with_startup_retry`], ended by `stop`.
|
||||
///
|
||||
/// When `stop` resolves, in a sleep or during an attempt, the read is dropped
|
||||
/// without another attempt and the answer is `Ok(None)`. The caller exits 0, as
|
||||
/// it does for the same signal later on, so `systemctl stop` or a pod delete
|
||||
/// during a store outage returns at once instead of after the schedule (about a
|
||||
/// minute) or the unit's stop timeout.
|
||||
///
|
||||
/// # Errors
|
||||
/// The error of the final attempt, once every read has failed.
|
||||
pub async fn read_with_startup_retry_or_stop<T, E, F, Fut>(
|
||||
what: &str,
|
||||
stop: impl Future<Output = ()>,
|
||||
read: F,
|
||||
) -> Result<Option<T>, E>
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = Result<T, E>>,
|
||||
E: Display,
|
||||
{
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = stop => {
|
||||
tracing::info!("stop arrived while {what}; exiting before startup finished");
|
||||
Ok(None)
|
||||
}
|
||||
read = read_with_startup_retry(what, read) => read.map(Some),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
|
@ -85,6 +118,48 @@ mod tests {
|
|||
assert_eq!(started.elapsed(), Duration::from_secs(1 + 2 + 4));
|
||||
}
|
||||
|
||||
/// Resolves `after` from now, like a SIGTERM that arrives mid-outage.
|
||||
async fn stop_after(after: Duration) {
|
||||
tokio::time::sleep(after).await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_stop_during_a_retry_wait_ends_the_read_without_another_attempt() {
|
||||
// Attempts at 0 s and 1 s fail; the wait after the second runs 1 s to 3 s.
|
||||
let calls = AtomicUsize::new(0);
|
||||
let started = Instant::now();
|
||||
let got =
|
||||
read_with_startup_retry_or_stop("reading", stop_after(Duration::from_secs(2)), || {
|
||||
read_after(usize::MAX, &calls)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(got, Ok(None));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(started.elapsed(), Duration::from_secs(2));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_stop_that_has_already_fired_makes_no_attempt() {
|
||||
let calls = AtomicUsize::new(0);
|
||||
let got = read_with_startup_retry_or_stop("reading", std::future::ready(()), || {
|
||||
read_after(0, &calls)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(got, Ok(None));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_stop_after_the_read_succeeded_changes_nothing() {
|
||||
let calls = AtomicUsize::new(0);
|
||||
let got =
|
||||
read_with_startup_retry_or_stop("reading", stop_after(Duration::from_mins(1)), || {
|
||||
read_after(1, &calls)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(got, Ok(Some("secret")));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_store_that_stays_down_fails_after_the_whole_schedule() {
|
||||
let calls = AtomicUsize::new(0);
|
||||
|
|
|
|||
|
|
@ -87,20 +87,28 @@ fn settings(var: Var<'_>, number: Number<'_>) -> Result<Settings> {
|
|||
})
|
||||
}
|
||||
|
||||
/// The queue connection, as the service's own OIDC client.
|
||||
async fn connect_queue(var: Var<'_>) -> Result<async_nats::Client> {
|
||||
/// The queue connection, as the service's own OIDC client. `None` when `stop`
|
||||
/// fired while the secret was still being waited for.
|
||||
async fn connect_queue(
|
||||
var: Var<'_>,
|
||||
stop: swarm_secret_client::Shutdown,
|
||||
) -> Result<Option<async_nats::Client>> {
|
||||
let client_id = var("OIDC_CLIENT_ID")?;
|
||||
let secret =
|
||||
swarm_secret_client::read_with_startup_retry("reading the queue client secret", || {
|
||||
client_secret(&client_id)
|
||||
})
|
||||
.await?;
|
||||
let Some(secret) = swarm_secret_client::read_with_startup_retry_or_stop(
|
||||
"reading the queue client secret",
|
||||
stop.clone(),
|
||||
|| client_secret(&client_id),
|
||||
)
|
||||
.await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let reread_id = client_id.clone();
|
||||
let secret = StoredSecret::new(client_id.clone(), secret, move || {
|
||||
let client_id = reread_id.clone();
|
||||
async move { client_secret(&client_id).await.map_err(Into::into) }
|
||||
});
|
||||
swarm_queue_client::connect(QueueConfig {
|
||||
let queue = swarm_queue_client::connect(QueueConfig {
|
||||
url: var("NATS_URL")?,
|
||||
token_endpoint: var("OIDC_TOKEN_ENDPOINT")?,
|
||||
client_id,
|
||||
|
|
@ -109,8 +117,8 @@ async fn connect_queue(var: Var<'_>) -> Result<async_nats::Client> {
|
|||
.ok()
|
||||
.map(PathBuf::from),
|
||||
})
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
.await?;
|
||||
Ok(Some(queue))
|
||||
}
|
||||
|
||||
/// Removes expired transcripts from `cache` every [`PURGE_EVERY`], for as long as
|
||||
|
|
@ -155,7 +163,9 @@ async fn main() -> Result<()> {
|
|||
waiting,
|
||||
} = settings(&var, &number)?;
|
||||
let per_agent: Arc<Mutex<HashSet<String>>> = Arc::default();
|
||||
let queue = connect_queue(&var).await?;
|
||||
let Some(queue) = connect_queue(&var, stop.clone()).await? else {
|
||||
return Ok(());
|
||||
};
|
||||
swarm_queue_client::ready_when_connected(&health, &queue);
|
||||
let metrics = Arc::new(Metrics::from_env());
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue