From 96cb25a0406d72b8054f4d2e01b18096def503fd Mon Sep 17 00:00:00 2001 From: maia Date: Thu, 19 Nov 2026 09:06:48 +0100 Subject: [PATCH] swarm-controller: a stop signal ends the startup secret-store reads The queue client secret and the webhook secret are read at start through a retry schedule of about a minute. Neither heard SIGTERM, because the shutdown signal was only installed once the controller began serving, so a stop during a store outage waited out the schedule or the unit's stop timeout. main installs the signal first and passes it to queue_identity::init and webhook::load. A stop that arrives first ends the read and exits 0. --- swarm-controller/src/main.rs | 22 +++++++-- swarm-controller/src/queue_identity.rs | 64 ++++++++++++++++++++----- swarm-controller/src/webhook.rs | 65 ++++++++++++++++++++++++-- 3 files changed, 131 insertions(+), 20 deletions(-) diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 741b657a3..55fccad7d 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -111,7 +111,7 @@ use startup::{ #[tokio::main] async fn main() -> Result<()> { hive_log::init(); - swarm_secret_client::install_crypto_provider(); + let stop = install_process_hooks(); let path = socket_path(); @@ -145,7 +145,7 @@ async fn main() -> Result<()> { // Before every user of the identity: the queue, the bridge and both OTLP // exporters read it from `queue_identity::get`. - queue_identity::init().await?; + exit_if_stopped(queue_identity::init(stop.clone()).await?); let status = connect_status_reader().await?; // Same "not fatal, log and carry on" shape as the queue connect above: @@ -202,7 +202,7 @@ async fn main() -> Result<()> { register_cert_gauges(jobq_metrics_provider.as_ref()); held_deploys::register(jobq_metrics_provider.as_ref(), &jobq); - let webhook_secret = webhook::load().await; + let webhook_secret = exit_if_stopped(webhook::load(stop.clone()).await); // Diagnostic only — `put_matrix_account` does not call this yet (see its // own doc comment), so this cannot change any request's outcome. Logged @@ -269,7 +269,21 @@ async fn main() -> Result<()> { reviews: Arc::default(), alerts: alerts::from_env(), }; - serve(listener, state, swarm_secret_client::shutdown_signal()).await + serve(listener, state, stop).await +} + +/// The TLS provider, and the stop signal. The signal is installed first thing, +/// so a stop that arrives while the startup reads wait on a store that is down +/// ends them instead of being ignored until `serve`. +fn install_process_hooks() -> swarm_secret_client::Shutdown { + swarm_secret_client::install_crypto_provider(); + swarm_secret_client::shutdown_signal() +} + +/// The value a startup step produced, or exit 0 when the stop signal ended the +/// step first: nothing is serving yet, so there is nothing to drain. +fn exit_if_stopped(started: Option) -> T { + started.unwrap_or_else(|| std::process::exit(0)) } /// Registers the `bao.client_cert.*` gauges for the controller's own store diff --git a/swarm-controller/src/queue_identity.rs b/swarm-controller/src/queue_identity.rs index f15ab0d8e..837f915e5 100644 --- a/swarm-controller/src/queue_identity.rs +++ b/swarm-controller/src/queue_identity.rs @@ -11,6 +11,7 @@ //! //! [`init`] runs once in `main`, before anything calls [`get`]. +use std::future::Future; use std::path::PathBuf; use std::sync::OnceLock; @@ -48,28 +49,44 @@ impl Coordinates { } } -/// Resolve the identity and hold it for [`get`]. +/// Resolve the identity and hold it for [`get`], giving up at once when `stop` +/// resolves while the store is still being waited for. +/// +/// `Some(())` once the identity is held; `None` when `stop` came first and the +/// daemon should exit. /// /// # Errors /// A half-set environment, which is a deployment bug and not an absent queue; /// a secret the store still would not give up after the startup retries; or a /// second call. -pub async fn init() -> Result<()> { +pub async fn init(stop: impl Future) -> Result> { let identity = match coordinates_from(|name| std::env::var(name).ok())? { None => None, - Some(coordinates) => { - let secret = swarm_secret_client::read_with_startup_retry( - "fetching the queue client secret from the swarm secret store", - fetch_secret, - ) - .await - .context("fetching the queue client secret from the swarm secret store")?; - Some(coordinates.with_secret(secret)) - } + Some(coordinates) => match startup_secret(stop, fetch_secret).await? { + Some(secret) => Some(coordinates.with_secret(secret)), + None => return Ok(None), + }, }; IDENTITY .set(identity) - .map_err(|_| anyhow::anyhow!("the queue identity was initialised twice")) + .map_err(|_| anyhow::anyhow!("the queue identity was initialised twice"))?; + Ok(Some(())) +} + +/// The client secret, retried through a store that is still coming up, or +/// `None` when `stop` resolves first. +async fn startup_secret(stop: impl Future, fetch: F) -> Result> +where + F: FnMut() -> Fut, + Fut: Future>, +{ + swarm_secret_client::read_with_startup_retry_or_stop( + "fetching the queue client secret from the swarm secret store", + stop, + fetch, + ) + .await + .context("fetching the queue client secret from the swarm secret store") } /// The identity, or `None` when this deployment wired no queue up. @@ -126,6 +143,29 @@ mod tests { use super::*; + /// A secret read that always fails, as a store that is down does. + async fn store_down() -> Result { + bail!("store is down") + } + + #[tokio::test(start_paused = true)] + async fn a_stop_while_the_store_is_down_ends_the_wait() { + let stop = tokio::time::sleep(std::time::Duration::from_secs(2)); + let started = tokio::time::Instant::now(); + let got = startup_secret(stop, store_down).await.unwrap(); + assert_eq!(got, None); + assert_eq!(started.elapsed(), std::time::Duration::from_secs(2)); + } + + #[tokio::test(start_paused = true)] + async fn a_store_that_answers_gives_the_secret_and_ignores_a_later_stop() { + let stop = tokio::time::sleep(std::time::Duration::from_mins(1)); + let got = startup_secret(stop, || async { Ok("s3cret".to_owned()) }) + .await + .unwrap(); + assert_eq!(got.as_deref(), Some("s3cret")); + } + fn env(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option { let map: HashMap = pairs .iter() diff --git a/swarm-controller/src/webhook.rs b/swarm-controller/src/webhook.rs index c5f328a77..fd3bae18e 100644 --- a/swarm-controller/src/webhook.rs +++ b/swarm-controller/src/webhook.rs @@ -126,8 +126,30 @@ pub(super) async fn load_or_generate_secret() -> Result { /// Every attempt is [`load_or_generate_secret`], so a retry only reads again: a /// secret is minted only when the store answers that it holds none, never on a /// failed read. -pub(crate) async fn load() -> Late> { - load_with(|| async { load_or_generate_secret().await.map(Arc::from) }).await +/// +/// `None` when `stop` resolves before the first attempt is over: the startup +/// retries of a store that is down end there instead of after about a minute. +pub(crate) async fn load(stop: impl Future) -> Option>> { + load_or_stop(stop, || async { + load_or_generate_secret().await.map(Arc::from) + }) + .await +} + +/// [`load`], with `load` standing in for the store. +async fn load_or_stop(stop: impl Future, load: F) -> Option>> +where + F: FnMut() -> Fut + Send + 'static, + Fut: Future>> + Send, +{ + tokio::select! { + biased; + () = stop => { + tracing::info!("stop arrived while loading the webhook secret; exiting before startup finished"); + None + } + slot = load_with(load) => Some(slot), + } } /// [`load`], with `load` standing in for the store. @@ -739,8 +761,8 @@ async fn announce_knowledge_change(state: &AppState) { #[cfg(test)] mod tests { use super::{ - DeliveryKind, Refusal, SecretSlot, Stored, load_or_generate_at, load_with, resolve, - secret_path_from, verify, + DeliveryKind, Refusal, SecretSlot, Stored, load_or_generate_at, load_or_stop, load_with, + resolve, secret_path_from, verify, }; use axum::{body::Bytes, http::HeaderMap, http::StatusCode}; use hmac::{Hmac, KeyInit as _, Mac as _}; @@ -1240,6 +1262,41 @@ mod tests { ); } + /// A first read that waits out `wait` before it answers, as the startup + /// retries of a store that is down do. + fn answering_after( + wait: std::time::Duration, + ) -> impl FnMut() -> std::pin::Pin>> + Send>> + + Send + + 'static { + move || { + Box::pin(async move { + tokio::time::sleep(wait).await; + Ok(Arc::from(IN_STORE)) + }) + } + } + + #[tokio::test(start_paused = true)] + async fn a_stop_during_the_first_read_ends_the_load_before_the_wait_is_over() { + let started = tokio::time::Instant::now(); + let stop = tokio::time::sleep(std::time::Duration::from_secs(2)); + let slot = load_or_stop(stop, answering_after(std::time::Duration::from_mins(1))).await; + + assert!(slot.is_none()); + assert_eq!(started.elapsed(), std::time::Duration::from_secs(2)); + } + + #[tokio::test(start_paused = true)] + async fn a_read_that_finishes_before_the_stop_fills_the_slot() { + let stop = tokio::time::sleep(std::time::Duration::from_mins(5)); + let slot = load_or_stop(stop, answering_after(std::time::Duration::from_secs(3))) + .await + .expect("the read finished first"); + + assert_eq!(slot.get().map(|s| &**s), Some(IN_STORE)); + } + #[tokio::test(start_paused = true)] async fn a_store_answering_at_start_fills_the_slot_without_a_background_task() { let (_dir, path) = legacy_file(None);