Watch
0
0
Fork
You've already forked hyperhive
0

swarm-controller: a stop signal ends the startup secret-store reads

The queue client secret and the webhook secret are read at start through
a retry schedule of about a minute. Neither heard SIGTERM, because the
shutdown signal was only installed once the controller began serving, so
a stop during a store outage waited out the schedule or the unit's stop
timeout.

main installs the signal first and passes it to queue_identity::init and
webhook::load. A stop that arrives first ends the read and exits 0.
This commit is contained in:
maia 2026-11-19 09:06:48 +01:00 • committed by mara
commit 96cb25a040
3 changed files with 131 additions and 20 deletions

View file

@ -111,7 +111,7 @@ use startup::{
#[tokio::main]
async fn main() -> Result<()> {
hive_log::init();
swarm_secret_client::install_crypto_provider();
let stop = install_process_hooks();
let path = socket_path();
@ -145,7 +145,7 @@ async fn main() -> Result<()> {
// Before every user of the identity: the queue, the bridge and both OTLP
// exporters read it from `queue_identity::get`.
queue_identity::init().await?;
exit_if_stopped(queue_identity::init(stop.clone()).await?);
let status = connect_status_reader().await?;
// Same "not fatal, log and carry on" shape as the queue connect above:
@ -202,7 +202,7 @@ async fn main() -> Result<()> {
register_cert_gauges(jobq_metrics_provider.as_ref());
held_deploys::register(jobq_metrics_provider.as_ref(), &jobq);
let webhook_secret = webhook::load().await;
let webhook_secret = exit_if_stopped(webhook::load(stop.clone()).await);
// Diagnostic only — `put_matrix_account` does not call this yet (see its
// own doc comment), so this cannot change any request's outcome. Logged
@ -269,7 +269,21 @@ async fn main() -> Result<()> {
reviews: Arc::default(),
alerts: alerts::from_env(),
};
serve(listener, state, swarm_secret_client::shutdown_signal()).await
serve(listener, state, stop).await
}
/// The TLS provider, and the stop signal. The signal is installed first thing,
/// so a stop that arrives while the startup reads wait on a store that is down
/// ends them instead of being ignored until `serve`.
fn install_process_hooks() -> swarm_secret_client::Shutdown {
swarm_secret_client::install_crypto_provider();
swarm_secret_client::shutdown_signal()
}
/// The value a startup step produced, or exit 0 when the stop signal ended the
/// step first: nothing is serving yet, so there is nothing to drain.
fn exit_if_stopped<T>(started: Option<T>) -> T {
started.unwrap_or_else(|| std::process::exit(0))
}
/// Registers the `bao.client_cert.*` gauges for the controller's own store

View file

@ -11,6 +11,7 @@
//!
//! [`init`] runs once in `main`, before anything calls [`get`].
use std::future::Future;
use std::path::PathBuf;
use std::sync::OnceLock;
@ -48,28 +49,44 @@ impl Coordinates {
}
}
/// Resolve the identity and hold it for [`get`].
/// Resolve the identity and hold it for [`get`], giving up at once when `stop`
/// resolves while the store is still being waited for.
///
/// `Some(())` once the identity is held; `None` when `stop` came first and the
/// daemon should exit.
///
/// # Errors
/// A half-set environment, which is a deployment bug and not an absent queue;
/// a secret the store still would not give up after the startup retries; or a
/// second call.
pub async fn init() -> Result<()> {
pub async fn init(stop: impl Future<Output = ()>) -> Result<Option<()>> {
let identity = match coordinates_from(|name| std::env::var(name).ok())? {
None => None,
Some(coordinates) => {
let secret = swarm_secret_client::read_with_startup_retry(
"fetching the queue client secret from the swarm secret store",
fetch_secret,
)
.await
.context("fetching the queue client secret from the swarm secret store")?;
Some(coordinates.with_secret(secret))
}
Some(coordinates) => match startup_secret(stop, fetch_secret).await? {
Some(secret) => Some(coordinates.with_secret(secret)),
None => return Ok(None),
},
};
IDENTITY
.set(identity)
.map_err(|_| anyhow::anyhow!("the queue identity was initialised twice"))
.map_err(|_| anyhow::anyhow!("the queue identity was initialised twice"))?;
Ok(Some(()))
}
/// The client secret, retried through a store that is still coming up, or
/// `None` when `stop` resolves first.
async fn startup_secret<F, Fut>(stop: impl Future<Output = ()>, fetch: F) -> Result<Option<String>>
where
F: FnMut() -> Fut,
Fut: Future<Output = Result<String>>,
{
swarm_secret_client::read_with_startup_retry_or_stop(
"fetching the queue client secret from the swarm secret store",
stop,
fetch,
)
.await
.context("fetching the queue client secret from the swarm secret store")
}
/// The identity, or `None` when this deployment wired no queue up.
@ -126,6 +143,29 @@ mod tests {
use super::*;
/// A secret read that always fails, as a store that is down does.
async fn store_down() -> Result<String> {
bail!("store is down")
}
#[tokio::test(start_paused = true)]
async fn a_stop_while_the_store_is_down_ends_the_wait() {
let stop = tokio::time::sleep(std::time::Duration::from_secs(2));
let started = tokio::time::Instant::now();
let got = startup_secret(stop, store_down).await.unwrap();
assert_eq!(got, None);
assert_eq!(started.elapsed(), std::time::Duration::from_secs(2));
}
#[tokio::test(start_paused = true)]
async fn a_store_that_answers_gives_the_secret_and_ignores_a_later_stop() {
let stop = tokio::time::sleep(std::time::Duration::from_mins(1));
let got = startup_secret(stop, || async { Ok("s3cret".to_owned()) })
.await
.unwrap();
assert_eq!(got.as_deref(), Some("s3cret"));
}
fn env(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option<String> {
let map: HashMap<String, String> = pairs
.iter()

View file

@ -126,8 +126,30 @@ pub(super) async fn load_or_generate_secret() -> Result<String> {
/// Every attempt is [`load_or_generate_secret`], so a retry only reads again: a
/// secret is minted only when the store answers that it holds none, never on a
/// failed read.
pub(crate) async fn load() -> Late<Arc<str>> {
load_with(|| async { load_or_generate_secret().await.map(Arc::from) }).await
///
/// `None` when `stop` resolves before the first attempt is over: the startup
/// retries of a store that is down end there instead of after about a minute.
pub(crate) async fn load(stop: impl Future<Output = ()>) -> Option<Late<Arc<str>>> {
load_or_stop(stop, || async {
load_or_generate_secret().await.map(Arc::from)
})
.await
}
/// [`load`], with `load` standing in for the store.
async fn load_or_stop<F, Fut>(stop: impl Future<Output = ()>, load: F) -> Option<Late<Arc<str>>>
where
F: FnMut() -> Fut + Send + 'static,
Fut: Future<Output = Result<Arc<str>>> + Send,
{
tokio::select! {
biased;
() = stop => {
tracing::info!("stop arrived while loading the webhook secret; exiting before startup finished");
None
}
slot = load_with(load) => Some(slot),
}
}
/// [`load`], with `load` standing in for the store.
@ -739,8 +761,8 @@ async fn announce_knowledge_change(state: &AppState) {
#[cfg(test)]
mod tests {
use super::{
DeliveryKind, Refusal, SecretSlot, Stored, load_or_generate_at, load_with, resolve,
secret_path_from, verify,
DeliveryKind, Refusal, SecretSlot, Stored, load_or_generate_at, load_or_stop, load_with,
resolve, secret_path_from, verify,
};
use axum::{body::Bytes, http::HeaderMap, http::StatusCode};
use hmac::{Hmac, KeyInit as _, Mac as _};
@ -1240,6 +1262,41 @@ mod tests {
);
}
/// A first read that waits out `wait` before it answers, as the startup
/// retries of a store that is down do.
fn answering_after(
wait: std::time::Duration,
) -> impl FnMut() -> std::pin::Pin<Box<dyn Future<Output = anyhow::Result<Arc<str>>> + Send>>
+ Send
+ 'static {
move || {
Box::pin(async move {
tokio::time::sleep(wait).await;
Ok(Arc::from(IN_STORE))
})
}
}
#[tokio::test(start_paused = true)]
async fn a_stop_during_the_first_read_ends_the_load_before_the_wait_is_over() {
let started = tokio::time::Instant::now();
let stop = tokio::time::sleep(std::time::Duration::from_secs(2));
let slot = load_or_stop(stop, answering_after(std::time::Duration::from_mins(1))).await;
assert!(slot.is_none());
assert_eq!(started.elapsed(), std::time::Duration::from_secs(2));
}
#[tokio::test(start_paused = true)]
async fn a_read_that_finishes_before_the_stop_fills_the_slot() {
let stop = tokio::time::sleep(std::time::Duration::from_mins(5));
let slot = load_or_stop(stop, answering_after(std::time::Duration::from_secs(3)))
.await
.expect("the read finished first");
assert_eq!(slot.get().map(|s| &**s), Some(IN_STORE));
}
#[tokio::test(start_paused = true)]
async fn a_store_answering_at_start_fills_the_slot_without_a_background_task() {
let (_dir, path) = legacy_file(None);