swarm-controller: a stop signal ends the startup secret-store reads
The queue client secret and the webhook secret are read at start through a retry schedule of about a minute. Neither heard SIGTERM, because the shutdown signal was only installed once the controller began serving, so a stop during a store outage waited out the schedule or the unit's stop timeout. main installs the signal first and passes it to queue_identity::init and webhook::load. A stop that arrives first ends the read and exits 0.
This commit is contained in:
parent
7df2d646ff
commit
96cb25a040
3 changed files with 131 additions and 20 deletions
|
|
@ -111,7 +111,7 @@ use startup::{
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<()> {
|
async fn main() -> Result<()> {
|
||||||
hive_log::init();
|
hive_log::init();
|
||||||
swarm_secret_client::install_crypto_provider();
|
let stop = install_process_hooks();
|
||||||
|
|
||||||
let path = socket_path();
|
let path = socket_path();
|
||||||
|
|
||||||
|
|
@ -145,7 +145,7 @@ async fn main() -> Result<()> {
|
||||||
|
|
||||||
// Before every user of the identity: the queue, the bridge and both OTLP
|
// Before every user of the identity: the queue, the bridge and both OTLP
|
||||||
// exporters read it from `queue_identity::get`.
|
// exporters read it from `queue_identity::get`.
|
||||||
queue_identity::init().await?;
|
exit_if_stopped(queue_identity::init(stop.clone()).await?);
|
||||||
let status = connect_status_reader().await?;
|
let status = connect_status_reader().await?;
|
||||||
|
|
||||||
// Same "not fatal, log and carry on" shape as the queue connect above:
|
// Same "not fatal, log and carry on" shape as the queue connect above:
|
||||||
|
|
@ -202,7 +202,7 @@ async fn main() -> Result<()> {
|
||||||
register_cert_gauges(jobq_metrics_provider.as_ref());
|
register_cert_gauges(jobq_metrics_provider.as_ref());
|
||||||
held_deploys::register(jobq_metrics_provider.as_ref(), &jobq);
|
held_deploys::register(jobq_metrics_provider.as_ref(), &jobq);
|
||||||
|
|
||||||
let webhook_secret = webhook::load().await;
|
let webhook_secret = exit_if_stopped(webhook::load(stop.clone()).await);
|
||||||
|
|
||||||
// Diagnostic only — `put_matrix_account` does not call this yet (see its
|
// Diagnostic only — `put_matrix_account` does not call this yet (see its
|
||||||
// own doc comment), so this cannot change any request's outcome. Logged
|
// own doc comment), so this cannot change any request's outcome. Logged
|
||||||
|
|
@ -269,7 +269,21 @@ async fn main() -> Result<()> {
|
||||||
reviews: Arc::default(),
|
reviews: Arc::default(),
|
||||||
alerts: alerts::from_env(),
|
alerts: alerts::from_env(),
|
||||||
};
|
};
|
||||||
serve(listener, state, swarm_secret_client::shutdown_signal()).await
|
serve(listener, state, stop).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The TLS provider, and the stop signal. The signal is installed first thing,
|
||||||
|
/// so a stop that arrives while the startup reads wait on a store that is down
|
||||||
|
/// ends them instead of being ignored until `serve`.
|
||||||
|
fn install_process_hooks() -> swarm_secret_client::Shutdown {
|
||||||
|
swarm_secret_client::install_crypto_provider();
|
||||||
|
swarm_secret_client::shutdown_signal()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The value a startup step produced, or exit 0 when the stop signal ended the
|
||||||
|
/// step first: nothing is serving yet, so there is nothing to drain.
|
||||||
|
fn exit_if_stopped<T>(started: Option<T>) -> T {
|
||||||
|
started.unwrap_or_else(|| std::process::exit(0))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Registers the `bao.client_cert.*` gauges for the controller's own store
|
/// Registers the `bao.client_cert.*` gauges for the controller's own store
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,7 @@
|
||||||
//!
|
//!
|
||||||
//! [`init`] runs once in `main`, before anything calls [`get`].
|
//! [`init`] runs once in `main`, before anything calls [`get`].
|
||||||
|
|
||||||
|
use std::future::Future;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::OnceLock;
|
use std::sync::OnceLock;
|
||||||
|
|
||||||
|
|
@ -48,28 +49,44 @@ impl Coordinates {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resolve the identity and hold it for [`get`].
|
/// Resolve the identity and hold it for [`get`], giving up at once when `stop`
|
||||||
|
/// resolves while the store is still being waited for.
|
||||||
|
///
|
||||||
|
/// `Some(())` once the identity is held; `None` when `stop` came first and the
|
||||||
|
/// daemon should exit.
|
||||||
///
|
///
|
||||||
/// # Errors
|
/// # Errors
|
||||||
/// A half-set environment, which is a deployment bug and not an absent queue;
|
/// A half-set environment, which is a deployment bug and not an absent queue;
|
||||||
/// a secret the store still would not give up after the startup retries; or a
|
/// a secret the store still would not give up after the startup retries; or a
|
||||||
/// second call.
|
/// second call.
|
||||||
pub async fn init() -> Result<()> {
|
pub async fn init(stop: impl Future<Output = ()>) -> Result<Option<()>> {
|
||||||
let identity = match coordinates_from(|name| std::env::var(name).ok())? {
|
let identity = match coordinates_from(|name| std::env::var(name).ok())? {
|
||||||
None => None,
|
None => None,
|
||||||
Some(coordinates) => {
|
Some(coordinates) => match startup_secret(stop, fetch_secret).await? {
|
||||||
let secret = swarm_secret_client::read_with_startup_retry(
|
Some(secret) => Some(coordinates.with_secret(secret)),
|
||||||
"fetching the queue client secret from the swarm secret store",
|
None => return Ok(None),
|
||||||
fetch_secret,
|
},
|
||||||
)
|
|
||||||
.await
|
|
||||||
.context("fetching the queue client secret from the swarm secret store")?;
|
|
||||||
Some(coordinates.with_secret(secret))
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
IDENTITY
|
IDENTITY
|
||||||
.set(identity)
|
.set(identity)
|
||||||
.map_err(|_| anyhow::anyhow!("the queue identity was initialised twice"))
|
.map_err(|_| anyhow::anyhow!("the queue identity was initialised twice"))?;
|
||||||
|
Ok(Some(()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The client secret, retried through a store that is still coming up, or
|
||||||
|
/// `None` when `stop` resolves first.
|
||||||
|
async fn startup_secret<F, Fut>(stop: impl Future<Output = ()>, fetch: F) -> Result<Option<String>>
|
||||||
|
where
|
||||||
|
F: FnMut() -> Fut,
|
||||||
|
Fut: Future<Output = Result<String>>,
|
||||||
|
{
|
||||||
|
swarm_secret_client::read_with_startup_retry_or_stop(
|
||||||
|
"fetching the queue client secret from the swarm secret store",
|
||||||
|
stop,
|
||||||
|
fetch,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.context("fetching the queue client secret from the swarm secret store")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The identity, or `None` when this deployment wired no queue up.
|
/// The identity, or `None` when this deployment wired no queue up.
|
||||||
|
|
@ -126,6 +143,29 @@ mod tests {
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
/// A secret read that always fails, as a store that is down does.
|
||||||
|
async fn store_down() -> Result<String> {
|
||||||
|
bail!("store is down")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn a_stop_while_the_store_is_down_ends_the_wait() {
|
||||||
|
let stop = tokio::time::sleep(std::time::Duration::from_secs(2));
|
||||||
|
let started = tokio::time::Instant::now();
|
||||||
|
let got = startup_secret(stop, store_down).await.unwrap();
|
||||||
|
assert_eq!(got, None);
|
||||||
|
assert_eq!(started.elapsed(), std::time::Duration::from_secs(2));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn a_store_that_answers_gives_the_secret_and_ignores_a_later_stop() {
|
||||||
|
let stop = tokio::time::sleep(std::time::Duration::from_mins(1));
|
||||||
|
let got = startup_secret(stop, || async { Ok("s3cret".to_owned()) })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(got.as_deref(), Some("s3cret"));
|
||||||
|
}
|
||||||
|
|
||||||
fn env(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option<String> {
|
fn env(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option<String> {
|
||||||
let map: HashMap<String, String> = pairs
|
let map: HashMap<String, String> = pairs
|
||||||
.iter()
|
.iter()
|
||||||
|
|
|
||||||
|
|
@ -126,8 +126,30 @@ pub(super) async fn load_or_generate_secret() -> Result<String> {
|
||||||
/// Every attempt is [`load_or_generate_secret`], so a retry only reads again: a
|
/// Every attempt is [`load_or_generate_secret`], so a retry only reads again: a
|
||||||
/// secret is minted only when the store answers that it holds none, never on a
|
/// secret is minted only when the store answers that it holds none, never on a
|
||||||
/// failed read.
|
/// failed read.
|
||||||
pub(crate) async fn load() -> Late<Arc<str>> {
|
///
|
||||||
load_with(|| async { load_or_generate_secret().await.map(Arc::from) }).await
|
/// `None` when `stop` resolves before the first attempt is over: the startup
|
||||||
|
/// retries of a store that is down end there instead of after about a minute.
|
||||||
|
pub(crate) async fn load(stop: impl Future<Output = ()>) -> Option<Late<Arc<str>>> {
|
||||||
|
load_or_stop(stop, || async {
|
||||||
|
load_or_generate_secret().await.map(Arc::from)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// [`load`], with `load` standing in for the store.
|
||||||
|
async fn load_or_stop<F, Fut>(stop: impl Future<Output = ()>, load: F) -> Option<Late<Arc<str>>>
|
||||||
|
where
|
||||||
|
F: FnMut() -> Fut + Send + 'static,
|
||||||
|
Fut: Future<Output = Result<Arc<str>>> + Send,
|
||||||
|
{
|
||||||
|
tokio::select! {
|
||||||
|
biased;
|
||||||
|
() = stop => {
|
||||||
|
tracing::info!("stop arrived while loading the webhook secret; exiting before startup finished");
|
||||||
|
None
|
||||||
|
}
|
||||||
|
slot = load_with(load) => Some(slot),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// [`load`], with `load` standing in for the store.
|
/// [`load`], with `load` standing in for the store.
|
||||||
|
|
@ -739,8 +761,8 @@ async fn announce_knowledge_change(state: &AppState) {
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
DeliveryKind, Refusal, SecretSlot, Stored, load_or_generate_at, load_with, resolve,
|
DeliveryKind, Refusal, SecretSlot, Stored, load_or_generate_at, load_or_stop, load_with,
|
||||||
secret_path_from, verify,
|
resolve, secret_path_from, verify,
|
||||||
};
|
};
|
||||||
use axum::{body::Bytes, http::HeaderMap, http::StatusCode};
|
use axum::{body::Bytes, http::HeaderMap, http::StatusCode};
|
||||||
use hmac::{Hmac, KeyInit as _, Mac as _};
|
use hmac::{Hmac, KeyInit as _, Mac as _};
|
||||||
|
|
@ -1240,6 +1262,41 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A first read that waits out `wait` before it answers, as the startup
|
||||||
|
/// retries of a store that is down do.
|
||||||
|
fn answering_after(
|
||||||
|
wait: std::time::Duration,
|
||||||
|
) -> impl FnMut() -> std::pin::Pin<Box<dyn Future<Output = anyhow::Result<Arc<str>>> + Send>>
|
||||||
|
+ Send
|
||||||
|
+ 'static {
|
||||||
|
move || {
|
||||||
|
Box::pin(async move {
|
||||||
|
tokio::time::sleep(wait).await;
|
||||||
|
Ok(Arc::from(IN_STORE))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn a_stop_during_the_first_read_ends_the_load_before_the_wait_is_over() {
|
||||||
|
let started = tokio::time::Instant::now();
|
||||||
|
let stop = tokio::time::sleep(std::time::Duration::from_secs(2));
|
||||||
|
let slot = load_or_stop(stop, answering_after(std::time::Duration::from_mins(1))).await;
|
||||||
|
|
||||||
|
assert!(slot.is_none());
|
||||||
|
assert_eq!(started.elapsed(), std::time::Duration::from_secs(2));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn a_read_that_finishes_before_the_stop_fills_the_slot() {
|
||||||
|
let stop = tokio::time::sleep(std::time::Duration::from_mins(5));
|
||||||
|
let slot = load_or_stop(stop, answering_after(std::time::Duration::from_secs(3)))
|
||||||
|
.await
|
||||||
|
.expect("the read finished first");
|
||||||
|
|
||||||
|
assert_eq!(slot.get().map(|s| &**s), Some(IN_STORE));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
async fn a_store_answering_at_start_fills_the_slot_without_a_background_task() {
|
async fn a_store_answering_at_start_fills_the_slot_without_a_background_task() {
|
||||||
let (_dir, path) = legacy_file(None);
|
let (_dir, path) = legacy_file(None);
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue