swarm-controller: a stop signal ends the startup secret-store reads
The queue client secret and the webhook secret are read at start through a retry schedule of about a minute. Neither heard SIGTERM, because the shutdown signal was only installed once the controller began serving, so a stop during a store outage waited out the schedule or the unit's stop timeout. main installs the signal first and passes it to queue_identity::init and webhook::load. A stop that arrives first ends the read and exits 0.
This commit is contained in:
parent
7df2d646ff
commit
96cb25a040
3 changed files with 131 additions and 20 deletions
|
|
@ -111,7 +111,7 @@ use startup::{
|
|||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
hive_log::init();
|
||||
swarm_secret_client::install_crypto_provider();
|
||||
let stop = install_process_hooks();
|
||||
|
||||
let path = socket_path();
|
||||
|
||||
|
|
@ -145,7 +145,7 @@ async fn main() -> Result<()> {
|
|||
|
||||
// Before every user of the identity: the queue, the bridge and both OTLP
|
||||
// exporters read it from `queue_identity::get`.
|
||||
queue_identity::init().await?;
|
||||
exit_if_stopped(queue_identity::init(stop.clone()).await?);
|
||||
let status = connect_status_reader().await?;
|
||||
|
||||
// Same "not fatal, log and carry on" shape as the queue connect above:
|
||||
|
|
@ -202,7 +202,7 @@ async fn main() -> Result<()> {
|
|||
register_cert_gauges(jobq_metrics_provider.as_ref());
|
||||
held_deploys::register(jobq_metrics_provider.as_ref(), &jobq);
|
||||
|
||||
let webhook_secret = webhook::load().await;
|
||||
let webhook_secret = exit_if_stopped(webhook::load(stop.clone()).await);
|
||||
|
||||
// Diagnostic only — `put_matrix_account` does not call this yet (see its
|
||||
// own doc comment), so this cannot change any request's outcome. Logged
|
||||
|
|
@ -269,7 +269,21 @@ async fn main() -> Result<()> {
|
|||
reviews: Arc::default(),
|
||||
alerts: alerts::from_env(),
|
||||
};
|
||||
serve(listener, state, swarm_secret_client::shutdown_signal()).await
|
||||
serve(listener, state, stop).await
|
||||
}
|
||||
|
||||
/// The TLS provider, and the stop signal. The signal is installed first thing,
|
||||
/// so a stop that arrives while the startup reads wait on a store that is down
|
||||
/// ends them instead of being ignored until `serve`.
|
||||
fn install_process_hooks() -> swarm_secret_client::Shutdown {
|
||||
swarm_secret_client::install_crypto_provider();
|
||||
swarm_secret_client::shutdown_signal()
|
||||
}
|
||||
|
||||
/// The value a startup step produced, or exit 0 when the stop signal ended the
|
||||
/// step first: nothing is serving yet, so there is nothing to drain.
|
||||
fn exit_if_stopped<T>(started: Option<T>) -> T {
|
||||
started.unwrap_or_else(|| std::process::exit(0))
|
||||
}
|
||||
|
||||
/// Registers the `bao.client_cert.*` gauges for the controller's own store
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
//!
|
||||
//! [`init`] runs once in `main`, before anything calls [`get`].
|
||||
|
||||
use std::future::Future;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
|
|
@ -48,28 +49,44 @@ impl Coordinates {
|
|||
}
|
||||
}
|
||||
|
||||
/// Resolve the identity and hold it for [`get`].
|
||||
/// Resolve the identity and hold it for [`get`], giving up at once when `stop`
|
||||
/// resolves while the store is still being waited for.
|
||||
///
|
||||
/// `Some(())` once the identity is held; `None` when `stop` came first and the
|
||||
/// daemon should exit.
|
||||
///
|
||||
/// # Errors
|
||||
/// A half-set environment, which is a deployment bug and not an absent queue;
|
||||
/// a secret the store still would not give up after the startup retries; or a
|
||||
/// second call.
|
||||
pub async fn init() -> Result<()> {
|
||||
pub async fn init(stop: impl Future<Output = ()>) -> Result<Option<()>> {
|
||||
let identity = match coordinates_from(|name| std::env::var(name).ok())? {
|
||||
None => None,
|
||||
Some(coordinates) => {
|
||||
let secret = swarm_secret_client::read_with_startup_retry(
|
||||
"fetching the queue client secret from the swarm secret store",
|
||||
fetch_secret,
|
||||
)
|
||||
.await
|
||||
.context("fetching the queue client secret from the swarm secret store")?;
|
||||
Some(coordinates.with_secret(secret))
|
||||
}
|
||||
Some(coordinates) => match startup_secret(stop, fetch_secret).await? {
|
||||
Some(secret) => Some(coordinates.with_secret(secret)),
|
||||
None => return Ok(None),
|
||||
},
|
||||
};
|
||||
IDENTITY
|
||||
.set(identity)
|
||||
.map_err(|_| anyhow::anyhow!("the queue identity was initialised twice"))
|
||||
.map_err(|_| anyhow::anyhow!("the queue identity was initialised twice"))?;
|
||||
Ok(Some(()))
|
||||
}
|
||||
|
||||
/// The client secret, retried through a store that is still coming up, or
|
||||
/// `None` when `stop` resolves first.
|
||||
async fn startup_secret<F, Fut>(stop: impl Future<Output = ()>, fetch: F) -> Result<Option<String>>
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = Result<String>>,
|
||||
{
|
||||
swarm_secret_client::read_with_startup_retry_or_stop(
|
||||
"fetching the queue client secret from the swarm secret store",
|
||||
stop,
|
||||
fetch,
|
||||
)
|
||||
.await
|
||||
.context("fetching the queue client secret from the swarm secret store")
|
||||
}
|
||||
|
||||
/// The identity, or `None` when this deployment wired no queue up.
|
||||
|
|
@ -126,6 +143,29 @@ mod tests {
|
|||
|
||||
use super::*;
|
||||
|
||||
/// A secret read that always fails, as a store that is down does.
|
||||
async fn store_down() -> Result<String> {
|
||||
bail!("store is down")
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_stop_while_the_store_is_down_ends_the_wait() {
|
||||
let stop = tokio::time::sleep(std::time::Duration::from_secs(2));
|
||||
let started = tokio::time::Instant::now();
|
||||
let got = startup_secret(stop, store_down).await.unwrap();
|
||||
assert_eq!(got, None);
|
||||
assert_eq!(started.elapsed(), std::time::Duration::from_secs(2));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_store_that_answers_gives_the_secret_and_ignores_a_later_stop() {
|
||||
let stop = tokio::time::sleep(std::time::Duration::from_mins(1));
|
||||
let got = startup_secret(stop, || async { Ok("s3cret".to_owned()) })
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(got.as_deref(), Some("s3cret"));
|
||||
}
|
||||
|
||||
fn env(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option<String> {
|
||||
let map: HashMap<String, String> = pairs
|
||||
.iter()
|
||||
|
|
|
|||
|
|
@ -126,8 +126,30 @@ pub(super) async fn load_or_generate_secret() -> Result<String> {
|
|||
/// Every attempt is [`load_or_generate_secret`], so a retry only reads again: a
|
||||
/// secret is minted only when the store answers that it holds none, never on a
|
||||
/// failed read.
|
||||
pub(crate) async fn load() -> Late<Arc<str>> {
|
||||
load_with(|| async { load_or_generate_secret().await.map(Arc::from) }).await
|
||||
///
|
||||
/// `None` when `stop` resolves before the first attempt is over: the startup
|
||||
/// retries of a store that is down end there instead of after about a minute.
|
||||
pub(crate) async fn load(stop: impl Future<Output = ()>) -> Option<Late<Arc<str>>> {
|
||||
load_or_stop(stop, || async {
|
||||
load_or_generate_secret().await.map(Arc::from)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// [`load`], with `load` standing in for the store.
|
||||
async fn load_or_stop<F, Fut>(stop: impl Future<Output = ()>, load: F) -> Option<Late<Arc<str>>>
|
||||
where
|
||||
F: FnMut() -> Fut + Send + 'static,
|
||||
Fut: Future<Output = Result<Arc<str>>> + Send,
|
||||
{
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = stop => {
|
||||
tracing::info!("stop arrived while loading the webhook secret; exiting before startup finished");
|
||||
None
|
||||
}
|
||||
slot = load_with(load) => Some(slot),
|
||||
}
|
||||
}
|
||||
|
||||
/// [`load`], with `load` standing in for the store.
|
||||
|
|
@ -739,8 +761,8 @@ async fn announce_knowledge_change(state: &AppState) {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
DeliveryKind, Refusal, SecretSlot, Stored, load_or_generate_at, load_with, resolve,
|
||||
secret_path_from, verify,
|
||||
DeliveryKind, Refusal, SecretSlot, Stored, load_or_generate_at, load_or_stop, load_with,
|
||||
resolve, secret_path_from, verify,
|
||||
};
|
||||
use axum::{body::Bytes, http::HeaderMap, http::StatusCode};
|
||||
use hmac::{Hmac, KeyInit as _, Mac as _};
|
||||
|
|
@ -1240,6 +1262,41 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
/// A first read that waits out `wait` before it answers, as the startup
|
||||
/// retries of a store that is down do.
|
||||
fn answering_after(
|
||||
wait: std::time::Duration,
|
||||
) -> impl FnMut() -> std::pin::Pin<Box<dyn Future<Output = anyhow::Result<Arc<str>>> + Send>>
|
||||
+ Send
|
||||
+ 'static {
|
||||
move || {
|
||||
Box::pin(async move {
|
||||
tokio::time::sleep(wait).await;
|
||||
Ok(Arc::from(IN_STORE))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_stop_during_the_first_read_ends_the_load_before_the_wait_is_over() {
|
||||
let started = tokio::time::Instant::now();
|
||||
let stop = tokio::time::sleep(std::time::Duration::from_secs(2));
|
||||
let slot = load_or_stop(stop, answering_after(std::time::Duration::from_mins(1))).await;
|
||||
|
||||
assert!(slot.is_none());
|
||||
assert_eq!(started.elapsed(), std::time::Duration::from_secs(2));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_read_that_finishes_before_the_stop_fills_the_slot() {
|
||||
let stop = tokio::time::sleep(std::time::Duration::from_mins(5));
|
||||
let slot = load_or_stop(stop, answering_after(std::time::Duration::from_secs(3)))
|
||||
.await
|
||||
.expect("the read finished first");
|
||||
|
||||
assert_eq!(slot.get().map(|s| &**s), Some(IN_STORE));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_store_answering_at_start_fills_the_slot_without_a_background_task() {
|
||||
let (_dir, path) = legacy_file(None);
|
||||
|
|
|
|||
Loading…
Reference in a new issue