mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-controller: keep retrying the forge webhook secret and registration after a startup outage

Browse source 
The webhook secret slot is filled in the background, on the forge token's
capped unbounded backoff, when the store can't answer at start. The hook
registration waits for the secret and retries on the same backoff instead
of giving up after five tries.

Closes #6101
This commit is contained in:
atlas 2026-11-06 04:05:39 +01:00 • committed by mara
commit 68b38a9d44
1 changed files with 284 additions and 114 deletions
Download patch file Download diff file Expand all files Collapse all files

162
swarm-controller/src/startup.rs
View file

@ -15,7 +15,6 @@ use std::sync::{Arc, Mutex};
use anyhow::Result;
use axum::{Json, routing::get};
use backon::{ExponentialBuilder, Retryable as _};
use utoipa::OpenApi;
use utoipa_axum::{router::OpenApiRouter, routes};
@ -214,24 +213,6 @@ pub(crate) fn agent_login(
Some(agent_login::Reported::spawn(status?.queue_client()))
}
/// Same "log and carry on" shape as the queue/bridge/forge wiring in `main`.
/// A controller that cannot hold a webhook secret still serves every other
/// route; the webhook endpoint answers 503, which is the honest answer
/// rather than a silent accept. Lifted out of `main` for
/// `clippy::too_many_lines`.
pub(crate) async fn load_webhook_secret() -> Option<Arc<str>> {
match webhook::load_or_generate_secret().await {
Ok(secret) => Some(Arc::from(secret)),
Err(e) => {
tracing::warn!(
error = %format!("{e:#}"),
"webhook secret unavailable; swarm-wide forge webhooks are off"
);
None
}
}
}
/// The homeserver agents' own matrix accounts are minted on. Unset is this
/// controller creating agents with no matrix account, which the mint node
/// reports by name rather than `main` refusing to start.
@ -444,21 +425,21 @@ pub(crate) const PUBLIC_URL_ENV: &str = "SWARM_CONTROLLER_PUBLIC_URL";
/// briefly down at boot, and none of the daemon's other routes depend on a
/// hook existing.
///
/// Retried on a backoff rather than left to the next process start. Losing
/// that race is the *common* case, not an edge one — the controller and the
/// forge come up together on a rebuild, and a deploy was measured where both
/// consecutive starts got `502 Bad Gateway` from the gateway because forgejo
/// was not serving yet. Nothing schedules another start, so "the next restart
/// retries" can leave a swarm with no hooks registered for as long as the
/// daemon keeps running — and the failure is silent at both ends, because the
/// forge has nothing to report about a call that never arrived.
/// Retried on a capped backoff for as long as it takes, rather than left to
/// the next process start. Losing that race is the *common* case, not an edge
/// one — the controller and the forge come up together on a rebuild, and a
/// deploy was measured where both consecutive starts got `502 Bad Gateway`
/// from the gateway because forgejo was not serving yet. Nothing schedules
/// another start, so "the next restart retries" can leave a swarm with no
/// hooks registered for as long as the daemon keeps running — and the failure
/// is silent at both ends, because the forge has nothing to report about a
/// call that never arrived.
///
/// Runs once there is a forge client, and silently does nothing when either
/// other precondition is missing — each is a legitimate deployment shape (no
/// state directory to hold a secret, no public vhost), and each is already
/// logged where it is discovered.
fn register_swarm_webhooks(forge: Arc<forge::Client>, secret: Option<Arc<str>>) {
let Some(secret) = secret else { return };
/// Runs once there is a forge client, and silently does nothing when the
/// public URL is missing: no public vhost is a legitimate deployment shape,
/// and it is logged here. A webhook secret that is not loaded yet is waited
/// for, as a forge that is not answering is.
fn register_swarm_webhooks(forge: Arc<forge::Client>, secret: Late<Arc<str>>) {
let Ok(public_url) = std::env::var(PUBLIC_URL_ENV) else {
tracing::info!(
"{PUBLIC_URL_ENV} unset; not registering swarm-wide forge webhooks (the \
@ -467,40 +448,40 @@ fn register_swarm_webhooks(forge: Arc<forge::Client>, secret: Option<Arc<str>>)
return;
};
tokio::spawn(async move {
// A bounded schedule, not a poll loop: 15s doubling to 240s, five
// retries. It exists to outlast a forge that is slow to start, not to
// re-register periodically. Registration is idempotent, so a repeat
// costs one API call and changes nothing.
const RETRIES: usize = 5;
let mut failed = 0;
let registered = (|| forge.ensure_swarm_webhooks(&public_url, &secret))
.retry(
ExponentialBuilder::default()
.with_min_delay(std::time::Duration::from_secs(15))
.with_max_delay(std::time::Duration::from_mins(4))
.with_max_times(RETRIES),
)
.notify(|e, wait| {
failed += 1;
tracing::info!(
error = %format!("{e:#}"),
attempt = failed,
retry_in = ?wait,
"registering swarm-wide forge webhooks failed; retrying"
);
})
.await;
if let Err(e) = registered {
tracing::warn!(
error = %format!("{e:#}"),
attempts = RETRIES + 1,
"registering swarm-wide forge webhooks failed; giving up until the next start"
);
}
register_when_ready(&secret, |secret| {
let forge = Arc::clone(&forge);
let public_url = public_url.clone();
async move { forge.ensure_swarm_webhooks(&public_url, &secret).await }
})
.await;
});
}
/// How long the first retry of the registration waits.
const REGISTRATION_FIRST: std::time::Duration = std::time::Duration::from_secs(15);
/// Run `ensure` with the webhook secret until it succeeds, waiting for
/// `secret` to be filled first. Registration is idempotent, so a repeat costs
/// one API call and changes nothing.
async fn register_when_ready<F, Fut>(secret: &Late<Arc<str>>, mut ensure: F)
where
F: FnMut(Arc<str>) -> Fut,
Fut: std::future::Future<Output = Result<()>>,
{
forge::retry_unbounded("the swarm-wide forge webhooks", REGISTRATION_FIRST, || {
let secret = secret.get().cloned();
let registering = secret.map(&mut ensure);
async move {
match registering {
Some(registering) => registering.await,
None => anyhow::bail!("the webhook secret is not loaded yet"),
}
}
})
.await;
tracing::info!("registered the swarm-wide forge webhooks");
}
/// Connect to the swarm queue when this deployment wired one up, extracted
/// out of `main` purely to keep that function under clippy's line-count
/// lint — no behavior split, every comment below is unchanged from where
@ -556,7 +537,7 @@ pub(crate) async fn start_forge(
config_prs: Late<Arc<config_pr::ConfigPrCache>>,
jobq: Arc<Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>>,
hives: Vec<HiveEntry>,
webhook_secret: Option<Arc<str>>,
webhook_secret: Late<Arc<str>>,
auth: Option<Arc<auth::AuthBridge>>,
queue: Option<async_nats::Client>,
) {
@ -978,4 +959,55 @@ mod tests {
);
}
}
// Paused time: each sleep completes at once and advances the clock.
#[tokio::test(start_paused = true)]
async fn registration_waits_for_the_secret_and_then_registers_with_it() {
let secret = Late::<Arc<str>>::default();
let registered = Arc::new(Mutex::new(Vec::<String>::new()));
let filling = Arc::clone(&secret);
tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_secs(100)).await;
filling.get_or_init(|| Arc::from("the-secret"));
});
let seen = Arc::clone(&registered);
register_when_ready(&secret, move |secret| {
let seen = Arc::clone(&seen);
async move {
seen.lock().unwrap().push(secret.to_string());
Ok(())
}
})
.await;
assert_eq!(*registered.lock().unwrap(), ["the-secret"]);
}
#[tokio::test(start_paused = true)]
async fn a_forge_that_fails_past_the_old_bound_is_still_registered_with() {
let secret = Late::<Arc<str>>::default();
secret.get_or_init(|| Arc::from("the-secret"));
let start = tokio::time::Instant::now();
let mut calls = 0;
register_when_ready(&secret, |_| {
calls += 1;
let n = calls;
async move {
if n < 9 {
anyhow::bail!("502 Bad Gateway")
}
Ok(())
}
})
.await;
assert_eq!(calls, 9);
// 15, 30, 60, 120, 240, then the 300s cap three times.
assert_eq!(
start.elapsed().as_secs(),
15 + 30 + 60 + 120 + 240 + 3 * 300
);
}
}