fix(#2570): gate boot forge-provisioning behind a readiness poll
The boot provisioning pass (orgs, repos, teams, CI-runner token) all hits the Forgejo API, but ensure_all only checked the container is *present*, not that its HTTP is *listening*. A nixos-rebuild that restarts hive-forge and hive-c0re together races: every ensure_* fired at a refused socket and left a stale 'provisioning failed' banner that never cleared, since the pass is one-shot. Poll GET /api/v1/version (unauthenticated) until it answers, bounded at 1 minute, before provisioning; on timeout proceed anyway so a genuinely-down forge still surfaces its real errors. Mirrors the readiness-retry the gateway-nginx path already has.
This commit is contained in:
parent
d4215c09c8
commit
d4b9dc1ab9
1 changed files with 45 additions and 0 deletions
|
|
@ -23,6 +23,7 @@ pub use repos::{
|
||||||
pub use users::{core_token, ensure_user_for, provision_user_token};
|
pub use users::{core_token, ensure_user_for, provision_user_token};
|
||||||
|
|
||||||
use std::sync::OnceLock;
|
use std::sync::OnceLock;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use forgejo_api::{Auth, Forgejo};
|
use forgejo_api::{Auth, Forgejo};
|
||||||
|
|
@ -322,6 +323,46 @@ async fn ensure_all_orgs_and_repos(token: &str) {
|
||||||
ci_runner::ensure_ci_runner_registered(token).await;
|
ci_runner::ensure_ci_runner_registered(token).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Poll the local Forgejo API until it answers, bounded by [`READY_TIMEOUT`].
|
||||||
|
/// The whole boot provisioning below hits the API, and [`is_present`] only
|
||||||
|
/// confirms the container exists — not that Forgejo is *listening*. A
|
||||||
|
/// `nixos-rebuild` that restarts hive-forge and hive-c0re together races:
|
||||||
|
/// without this gate every `ensure_*` fires at a refused socket and leaves a
|
||||||
|
/// stale "provisioning failed" banner that never clears (the pass is one-shot).
|
||||||
|
/// Returns whether the endpoint became ready before the deadline; on timeout
|
||||||
|
/// the caller proceeds anyway so a genuinely-down forge still surfaces its real
|
||||||
|
/// errors. Uses the unauthenticated `get_version` endpoint (no token yet).
|
||||||
|
async fn wait_until_ready() -> bool {
|
||||||
|
const READY_TIMEOUT: Duration = Duration::from_mins(1);
|
||||||
|
const POLL_INTERVAL: Duration = Duration::from_millis(500);
|
||||||
|
let Ok(url) = Url::parse(forge_http_base()) else {
|
||||||
|
tracing::warn!("forge: HIVE_FORGE_URL is not a valid URL; skipping readiness wait");
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let Ok(client) = Forgejo::new(Auth::None, url) else {
|
||||||
|
tracing::warn!("forge: could not build readiness client; skipping readiness wait");
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let deadline = Instant::now() + READY_TIMEOUT;
|
||||||
|
let mut waited = false;
|
||||||
|
loop {
|
||||||
|
if client.get_version().await.is_ok() {
|
||||||
|
if waited {
|
||||||
|
tracing::info!("forge: endpoint ready, proceeding with provisioning");
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if Instant::now() >= deadline {
|
||||||
|
tracing::warn!(
|
||||||
|
"forge: endpoint not ready after {READY_TIMEOUT:?}; provisioning anyway"
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
waited = true;
|
||||||
|
tokio::time::sleep(POLL_INTERVAL).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Sweep every existing container (manager + sub-agents) and ensure
|
/// Sweep every existing container (manager + sub-agents) and ensure
|
||||||
/// each has a forgejo user + token, plus an `agent-configs/<name>`
|
/// each has a forgejo user + token, plus an `agent-configs/<name>`
|
||||||
/// repo mirroring its applied config. Also seeds the `core` admin
|
/// repo mirroring its applied config. Also seeds the `core` admin
|
||||||
|
|
@ -334,6 +375,10 @@ pub async fn ensure_all() {
|
||||||
tracing::debug!("forge: hive-forge container absent, skipping user sweep");
|
tracing::debug!("forge: hive-forge container absent, skipping user sweep");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
// The container exists, but its HTTP may not be up yet (a rebuild restarts
|
||||||
|
// hive-forge + hive-c0re together). Wait for the API to answer before the
|
||||||
|
// provisioning pass so a boot race doesn't leave stale failure banners.
|
||||||
|
wait_until_ready().await;
|
||||||
let core_token = match ensure_core_user_and_token().await {
|
let core_token = match ensure_core_user_and_token().await {
|
||||||
Ok(t) => Some(t),
|
Ok(t) => Some(t),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue