fix(#2570): gate boot forge-provisioning behind a readiness poll
The boot provisioning pass (orgs, repos, teams, CI-runner token) all hits the Forgejo API, but ensure_all only checked the container is *present*, not that its HTTP is *listening*. A nixos-rebuild that restarts hive-forge and hive-c0re together races: every ensure_* fired at a refused socket and left a stale 'provisioning failed' banner that never cleared, since the pass is one-shot. Poll GET /api/v1/version (unauthenticated) until it answers, bounded at 1 minute, before provisioning; on timeout proceed anyway so a genuinely-down forge still surfaces its real errors. Mirrors the readiness-retry the gateway-nginx path already has.
This commit is contained in:
parent
d4215c09c8
commit
d4b9dc1ab9
1 changed files with 45 additions and 0 deletions
|
|
@ -23,6 +23,7 @@ pub use repos::{
|
|||
pub use users::{core_token, ensure_user_for, provision_user_token};
|
||||
|
||||
use std::sync::OnceLock;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use forgejo_api::{Auth, Forgejo};
|
||||
|
|
@ -322,6 +323,46 @@ async fn ensure_all_orgs_and_repos(token: &str) {
|
|||
ci_runner::ensure_ci_runner_registered(token).await;
|
||||
}
|
||||
|
||||
/// Poll the local Forgejo API until it answers, bounded by [`READY_TIMEOUT`].
|
||||
/// The whole boot provisioning below hits the API, and [`is_present`] only
|
||||
/// confirms the container exists — not that Forgejo is *listening*. A
|
||||
/// `nixos-rebuild` that restarts hive-forge and hive-c0re together races:
|
||||
/// without this gate every `ensure_*` fires at a refused socket and leaves a
|
||||
/// stale "provisioning failed" banner that never clears (the pass is one-shot).
|
||||
/// Returns whether the endpoint became ready before the deadline; on timeout
|
||||
/// the caller proceeds anyway so a genuinely-down forge still surfaces its real
|
||||
/// errors. Uses the unauthenticated `get_version` endpoint (no token yet).
|
||||
async fn wait_until_ready() -> bool {
|
||||
const READY_TIMEOUT: Duration = Duration::from_mins(1);
|
||||
const POLL_INTERVAL: Duration = Duration::from_millis(500);
|
||||
let Ok(url) = Url::parse(forge_http_base()) else {
|
||||
tracing::warn!("forge: HIVE_FORGE_URL is not a valid URL; skipping readiness wait");
|
||||
return false;
|
||||
};
|
||||
let Ok(client) = Forgejo::new(Auth::None, url) else {
|
||||
tracing::warn!("forge: could not build readiness client; skipping readiness wait");
|
||||
return false;
|
||||
};
|
||||
let deadline = Instant::now() + READY_TIMEOUT;
|
||||
let mut waited = false;
|
||||
loop {
|
||||
if client.get_version().await.is_ok() {
|
||||
if waited {
|
||||
tracing::info!("forge: endpoint ready, proceeding with provisioning");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
tracing::warn!(
|
||||
"forge: endpoint not ready after {READY_TIMEOUT:?}; provisioning anyway"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
waited = true;
|
||||
tokio::time::sleep(POLL_INTERVAL).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Sweep every existing container (manager + sub-agents) and ensure
|
||||
/// each has a forgejo user + token, plus an `agent-configs/<name>`
|
||||
/// repo mirroring its applied config. Also seeds the `core` admin
|
||||
|
|
@ -334,6 +375,10 @@ pub async fn ensure_all() {
|
|||
tracing::debug!("forge: hive-forge container absent, skipping user sweep");
|
||||
return;
|
||||
}
|
||||
// The container exists, but its HTTP may not be up yet (a rebuild restarts
|
||||
// hive-forge + hive-c0re together). Wait for the API to answer before the
|
||||
// provisioning pass so a boot race doesn't leave stale failure banners.
|
||||
wait_until_ready().await;
|
||||
let core_token = match ensure_core_user_and_token().await {
|
||||
Ok(t) => Some(t),
|
||||
Err(e) => {
|
||||
|
|
|
|||
Loading…
Reference in a new issue