fix(#2570): gate boot forge-provisioning behind a readiness poll

The boot provisioning pass (orgs, repos, teams, CI-runner token) all hits
the Forgejo API, but ensure_all only checked the container is *present*,
not that its HTTP is *listening*. A nixos-rebuild that restarts hive-forge
and hive-c0re together races: every ensure_* fired at a refused socket and
left a stale 'provisioning failed' banner that never cleared, since the
pass is one-shot. Poll GET /api/v1/version (unauthenticated) until it
answers, bounded at 1 minute, before provisioning; on timeout proceed
anyway so a genuinely-down forge still surfaces its real errors. Mirrors
the readiness-retry the gateway-nginx path already has.
This commit is contained in:
atlas 2026-07-19 18:41:34 +02:00 committed by mara
commit d4b9dc1ab9

View file

@ -23,6 +23,7 @@ pub use repos::{
pub use users::{core_token, ensure_user_for, provision_user_token};
use std::sync::OnceLock;
use std::time::{Duration, Instant};
use anyhow::{Context, Result};
use forgejo_api::{Auth, Forgejo};
@ -322,6 +323,46 @@ async fn ensure_all_orgs_and_repos(token: &str) {
ci_runner::ensure_ci_runner_registered(token).await;
}
/// Poll the local Forgejo API until it answers, bounded by [`READY_TIMEOUT`].
/// The whole boot provisioning below hits the API, and [`is_present`] only
/// confirms the container exists — not that Forgejo is *listening*. A
/// `nixos-rebuild` that restarts hive-forge and hive-c0re together races:
/// without this gate every `ensure_*` fires at a refused socket and leaves a
/// stale "provisioning failed" banner that never clears (the pass is one-shot).
/// Returns whether the endpoint became ready before the deadline; on timeout
/// the caller proceeds anyway so a genuinely-down forge still surfaces its real
/// errors. Uses the unauthenticated `get_version` endpoint (no token yet).
async fn wait_until_ready() -> bool {
const READY_TIMEOUT: Duration = Duration::from_mins(1);
const POLL_INTERVAL: Duration = Duration::from_millis(500);
let Ok(url) = Url::parse(forge_http_base()) else {
tracing::warn!("forge: HIVE_FORGE_URL is not a valid URL; skipping readiness wait");
return false;
};
let Ok(client) = Forgejo::new(Auth::None, url) else {
tracing::warn!("forge: could not build readiness client; skipping readiness wait");
return false;
};
let deadline = Instant::now() + READY_TIMEOUT;
let mut waited = false;
loop {
if client.get_version().await.is_ok() {
if waited {
tracing::info!("forge: endpoint ready, proceeding with provisioning");
}
return true;
}
if Instant::now() >= deadline {
tracing::warn!(
"forge: endpoint not ready after {READY_TIMEOUT:?}; provisioning anyway"
);
return false;
}
waited = true;
tokio::time::sleep(POLL_INTERVAL).await;
}
}
/// Sweep every existing container (manager + sub-agents) and ensure
/// each has a forgejo user + token, plus an `agent-configs/<name>`
/// repo mirroring its applied config. Also seeds the `core` admin
@ -334,6 +375,10 @@ pub async fn ensure_all() {
tracing::debug!("forge: hive-forge container absent, skipping user sweep");
return;
}
// The container exists, but its HTTP may not be up yet (a rebuild restarts
// hive-forge + hive-c0re together). Wait for the API to answer before the
// provisioning pass so a boot race doesn't leave stale failure banners.
wait_until_ready().await;
let core_token = match ensure_core_user_and_token().await {
Ok(t) => Some(t),
Err(e) => {