forge: wire ensure_all() boot-warn sites to the dashboard banner
Adds set_boot_warning() to stats/warnings.rs: a one-shot variant of set_warning() for steps that run once at hive-c0re boot (no periodic retry to observe a later success and clear the banner via WarningGuard drop). It forgets the guard, leaking it for the life of the process — the banner clears on the next hive-c0re restart, which is also when a config/environment fix would take effect. Wires every remaining tracing::warn!()-and-forget site in forge::ensure_all() to also raise a boot warning: core user/token provisioning, per-org seeding, the operators-team grant per org, the core/meta repo, shared docs repo, knowledge repo (+ its local clone), both avatar uploads, and the final nixos-container list failure. Split the core_token.is_some() body of ensure_all() into a new ensure_all_orgs_and_repos() helper to stay under clippy's function-length limit. Deliberately out of scope: sync_agent() (shared with rebuild_agent(), different blast radius) and ci_runner::ensure_ci_runner_registered() (its own retry logic) — matrix::ensure_all() got the same slice-by- slice treatment.
This commit is contained in:
parent
d124dd205a
commit
f378f842ea
2 changed files with 154 additions and 50 deletions
|
|
@ -106,6 +106,16 @@ const OPERATORS_TEAM: &str = "operators";
|
||||||
/// `core/meta` (the `core` user's own namespace — no org needed).
|
/// `core/meta` (the `core` user's own namespace — no org needed).
|
||||||
const SEEDED_ORGS: &[&str] = &[CONFIG_ORG, SHARED_ORG, AGENTS_ORG];
|
const SEEDED_ORGS: &[&str] = &[CONFIG_ORG, SHARED_ORG, AGENTS_ORG];
|
||||||
|
|
||||||
|
/// Leak `s` to get a `&'static str` warning `kind` for the small, bounded
|
||||||
|
/// set of per-org boot warnings in [`ensure_all`] (one per seeded org, at
|
||||||
|
/// most a handful per process). [`crate::warnings::set_boot_warning`]
|
||||||
|
/// requires a `'static` kind so distinct orgs/repos don't clobber each
|
||||||
|
/// other's banner entry; leaking a few short strings once per boot is
|
||||||
|
/// cheap and bounded, unlike a per-request or per-loop-iteration leak.
|
||||||
|
fn static_kind(s: String) -> &'static str {
|
||||||
|
Box::leak(s.into_boxed_str())
|
||||||
|
}
|
||||||
|
|
||||||
/// Probe whether `hive-forge` exists as a nixos-container. Cheap —
|
/// Probe whether `hive-forge` exists as a nixos-container. Cheap —
|
||||||
/// `nixos-container list` is just a directory scan in /etc. Routed
|
/// `nixos-container list` is just a directory scan in /etc. Routed
|
||||||
/// through hive-priv: `nixos-container` needs root, and hive-c0re runs
|
/// through hive-priv: `nixos-container` needs root, and hive-c0re runs
|
||||||
|
|
@ -202,6 +212,104 @@ pub async fn sync_agent(name: &str, core_token: Option<&str>) {
|
||||||
// internal/knowledge is public — no per-agent collaborator grant needed.
|
// internal/knowledge is public — no per-agent collaborator grant needed.
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The `core_token.is_some()` half of [`ensure_all`]: orgs, teams, the meta
|
||||||
|
/// repo, shared/knowledge repos, avatars, and CI runner registration — every
|
||||||
|
/// step that needs an authenticated forge client. Split out purely to keep
|
||||||
|
/// `ensure_all` under clippy's function-length limit; not meant to be called
|
||||||
|
/// from anywhere else.
|
||||||
|
async fn ensure_all_orgs_and_repos(token: &str) {
|
||||||
|
for org in SEEDED_ORGS {
|
||||||
|
if let Err(e) = ensure_org(org, token).await {
|
||||||
|
tracing::warn!(%org, error = ?e, "forge: ensure_org failed");
|
||||||
|
crate::warnings::set_boot_warning(
|
||||||
|
static_kind(format!("forge_ensure_org_{org}")),
|
||||||
|
"crit",
|
||||||
|
format!("forge: org {org} provisioning failed: {e}"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Seed the operator-declared pull-mirrors (nix `forge.mirrors` +
|
||||||
|
// the CI-auto `actions/checkout`, forwarded via the
|
||||||
|
// `HYPERHIVE_FORGE_MIRRORS` env). Each ensures its own dest org, so
|
||||||
|
// this is independent of the SEEDED_ORGS loop above.
|
||||||
|
ensure_mirrors(token).await;
|
||||||
|
// Provision the operator merge-gate team (empty) inside BOTH the
|
||||||
|
// agents org and the agent-configs org so branch protection in each
|
||||||
|
// can reference it before anyone joins. Gitea teams are org-scoped —
|
||||||
|
// missing the agent-configs copy 422'd every config-repo protection
|
||||||
|
// apply, leaving those repos unprotected and letting operator-merged
|
||||||
|
// config PRs bypass the deploy pipeline. The operator adds herself as
|
||||||
|
// a member out-of-band.
|
||||||
|
for org in [AGENTS_ORG, CONFIG_ORG] {
|
||||||
|
if let Err(e) = ensure_operators_team(org, token).await {
|
||||||
|
tracing::warn!(%org, error = ?e, "forge: ensure_operators_team failed");
|
||||||
|
crate::warnings::set_boot_warning(
|
||||||
|
static_kind(format!("forge_ensure_operators_team_{org}")),
|
||||||
|
"crit",
|
||||||
|
format!("forge: operators team in {org} provisioning failed: {e}"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Meta repo lives at core/meta — pushed from git_commit in
|
||||||
|
// meta.rs on every deploy/lock-update. Make sure it exists
|
||||||
|
// before the first push hits a 404.
|
||||||
|
if let Err(e) = ensure_repo("meta", token).await {
|
||||||
|
tracing::warn!(error = ?e, "forge: ensure_repo core/meta failed");
|
||||||
|
crate::warnings::set_boot_warning(
|
||||||
|
"forge_ensure_meta_repo",
|
||||||
|
"crit",
|
||||||
|
format!("forge: core/meta repo provisioning failed: {e}"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Seed the shared docs repo. internal is already in
|
||||||
|
// SEEDED_ORGS above so the org exists; ensure the repo itself.
|
||||||
|
if let Err(e) = ensure_shared_docs_repo(token).await {
|
||||||
|
tracing::warn!(error = ?e, "forge: ensure_shared_docs_repo failed");
|
||||||
|
crate::warnings::set_boot_warning(
|
||||||
|
"forge_ensure_shared_docs_repo",
|
||||||
|
"warn",
|
||||||
|
format!("forge: shared docs repo provisioning failed: {e}"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Seed the hive-wide knowledge repo.
|
||||||
|
if let Err(e) = ensure_knowledge_repo(token).await {
|
||||||
|
tracing::warn!(error = ?e, "forge: ensure_knowledge_repo failed");
|
||||||
|
crate::warnings::set_boot_warning(
|
||||||
|
"forge_ensure_knowledge_repo",
|
||||||
|
"crit",
|
||||||
|
format!("forge: knowledge repo provisioning failed: {e}"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Clone knowledge repo locally so it can be bind-mounted into agents.
|
||||||
|
if let Err(e) = crate::knowledge::ensure_local_clone(token).await {
|
||||||
|
tracing::warn!(error = ?e, "knowledge: ensure_local_clone failed");
|
||||||
|
crate::warnings::set_boot_warning(
|
||||||
|
"forge_knowledge_local_clone",
|
||||||
|
"crit",
|
||||||
|
format!("knowledge: local clone failed: {e}"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if let Err(e) = ensure_core_avatar(token).await {
|
||||||
|
tracing::warn!(error = ?e, "forge: ensure_core_avatar failed");
|
||||||
|
crate::warnings::set_boot_warning(
|
||||||
|
"forge_ensure_core_avatar",
|
||||||
|
"warn",
|
||||||
|
format!("forge: core avatar upload failed: {e}"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if let Err(e) = ensure_config_org_avatar(token).await {
|
||||||
|
tracing::warn!(error = ?e, "forge: ensure_config_org_avatar failed");
|
||||||
|
crate::warnings::set_boot_warning(
|
||||||
|
"forge_ensure_config_org_avatar",
|
||||||
|
"warn",
|
||||||
|
format!("forge: agent-configs org avatar upload failed: {e}"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Register the hive-ci Actions runner (off the container's boot path;
|
||||||
|
// no-op when CI is disabled or the runner already holds valid creds).
|
||||||
|
ci_runner::ensure_ci_runner_registered(token).await;
|
||||||
|
}
|
||||||
|
|
||||||
/// Sweep every existing container (manager + sub-agents) and ensure
|
/// Sweep every existing container (manager + sub-agents) and ensure
|
||||||
/// each has a forgejo user + token, plus an `agent-configs/<name>`
|
/// each has a forgejo user + token, plus an `agent-configs/<name>`
|
||||||
/// repo mirroring its applied config. Also seeds the `core` admin
|
/// repo mirroring its applied config. Also seeds the `core` admin
|
||||||
|
|
@ -218,63 +326,24 @@ pub async fn ensure_all() {
|
||||||
Ok(t) => Some(t),
|
Ok(t) => Some(t),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!(error = ?e, "forge: ensure_core_user_and_token failed");
|
tracing::warn!(error = ?e, "forge: ensure_core_user_and_token failed");
|
||||||
|
crate::warnings::set_boot_warning(
|
||||||
|
"forge_ensure_core_user",
|
||||||
|
"crit",
|
||||||
|
format!("forge: core user/token provisioning failed: {e}"),
|
||||||
|
);
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if let Some(token) = core_token.as_deref() {
|
if let Some(token) = core_token.as_deref() {
|
||||||
for org in SEEDED_ORGS {
|
ensure_all_orgs_and_repos(token).await;
|
||||||
if let Err(e) = ensure_org(org, token).await {
|
|
||||||
tracing::warn!(%org, error = ?e, "forge: ensure_org failed");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Seed the operator-declared pull-mirrors (nix `forge.mirrors` +
|
|
||||||
// the CI-auto `actions/checkout`, forwarded via the
|
|
||||||
// `HYPERHIVE_FORGE_MIRRORS` env). Each ensures its own dest org, so
|
|
||||||
// this is independent of the SEEDED_ORGS loop above.
|
|
||||||
ensure_mirrors(token).await;
|
|
||||||
// Provision the operator merge-gate team (empty) inside BOTH the
|
|
||||||
// agents org and the agent-configs org so branch protection in each
|
|
||||||
// can reference it before anyone joins. Gitea teams are org-scoped —
|
|
||||||
// missing the agent-configs copy 422'd every config-repo protection
|
|
||||||
// apply, leaving those repos unprotected and letting operator-merged
|
|
||||||
// config PRs bypass the deploy pipeline. The operator adds herself as
|
|
||||||
// a member out-of-band.
|
|
||||||
for org in [AGENTS_ORG, CONFIG_ORG] {
|
|
||||||
if let Err(e) = ensure_operators_team(org, token).await {
|
|
||||||
tracing::warn!(%org, error = ?e, "forge: ensure_operators_team failed");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Meta repo lives at core/meta — pushed from git_commit in
|
|
||||||
// meta.rs on every deploy/lock-update. Make sure it exists
|
|
||||||
// before the first push hits a 404.
|
|
||||||
if let Err(e) = ensure_repo("meta", token).await {
|
|
||||||
tracing::warn!(error = ?e, "forge: ensure_repo core/meta failed");
|
|
||||||
}
|
|
||||||
// Seed the shared docs repo. internal is already in
|
|
||||||
// SEEDED_ORGS above so the org exists; ensure the repo itself.
|
|
||||||
if let Err(e) = ensure_shared_docs_repo(token).await {
|
|
||||||
tracing::warn!(error = ?e, "forge: ensure_shared_docs_repo failed");
|
|
||||||
}
|
|
||||||
// Seed the hive-wide knowledge repo.
|
|
||||||
if let Err(e) = ensure_knowledge_repo(token).await {
|
|
||||||
tracing::warn!(error = ?e, "forge: ensure_knowledge_repo failed");
|
|
||||||
}
|
|
||||||
// Clone knowledge repo locally so it can be bind-mounted into agents.
|
|
||||||
if let Err(e) = crate::knowledge::ensure_local_clone(token).await {
|
|
||||||
tracing::warn!(error = ?e, "knowledge: ensure_local_clone failed");
|
|
||||||
}
|
|
||||||
if let Err(e) = ensure_core_avatar(token).await {
|
|
||||||
tracing::warn!(error = ?e, "forge: ensure_core_avatar failed");
|
|
||||||
}
|
|
||||||
if let Err(e) = ensure_config_org_avatar(token).await {
|
|
||||||
tracing::warn!(error = ?e, "forge: ensure_config_org_avatar failed");
|
|
||||||
}
|
|
||||||
// Register the hive-ci Actions runner (off the container's boot path;
|
|
||||||
// no-op when CI is disabled or the runner already holds valid creds).
|
|
||||||
ci_runner::ensure_ci_runner_registered(token).await;
|
|
||||||
}
|
}
|
||||||
let Ok(containers) = crate::lifecycle::list().await else {
|
let Ok(containers) = crate::lifecycle::list().await else {
|
||||||
tracing::warn!("forge: nixos-container list failed; skipping user sweep");
|
tracing::warn!("forge: nixos-container list failed; skipping user sweep");
|
||||||
|
crate::warnings::set_boot_warning(
|
||||||
|
"forge_container_list",
|
||||||
|
"crit",
|
||||||
|
"forge: nixos-container list failed; per-agent forge sync skipped this boot",
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
for c in containers {
|
for c in containers {
|
||||||
|
|
|
||||||
|
|
@ -103,6 +103,23 @@ pub fn snapshot() -> Vec<ServerWarning> {
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Raise a warning for a one-shot boot-time step that has no periodic
|
||||||
|
/// retry to observe a later success and clear the banner via
|
||||||
|
/// [`WarningGuard`] drop — e.g. a step inside `forge::ensure_all()`,
|
||||||
|
/// which `tokio::spawn`s once at hive-c0re startup and never runs again
|
||||||
|
/// this process. Intentionally **leaks** the guard for the life of the
|
||||||
|
/// process: the banner clears the next time hive-c0re restarts (a fresh
|
||||||
|
/// process starts with an empty registry) and re-runs the step, which is
|
||||||
|
/// exactly when a config/environment fix would take effect anyway.
|
||||||
|
///
|
||||||
|
/// Do not use this for anything that runs periodically or can be
|
||||||
|
/// retried within the same process — hold the [`WarningGuard`] (or use
|
||||||
|
/// [`crate::stats::sweep_health::SweepHealth`]) so a later success can
|
||||||
|
/// actually clear the banner instead of waiting for a restart.
|
||||||
|
pub fn set_boot_warning(kind: &'static str, level: &'static str, message: impl Into<String>) {
|
||||||
|
std::mem::forget(set_warning(kind, level, message));
|
||||||
|
}
|
||||||
|
|
||||||
/// RAII handle for one active warning. Drop clears the `kind` from the
|
/// RAII handle for one active warning. Drop clears the `kind` from the
|
||||||
/// banner. Obtained from [`set_warning`].
|
/// banner. Obtained from [`set_warning`].
|
||||||
#[must_use = "dropping the guard immediately clears the warning; bind it for as long as the condition holds"]
|
#[must_use = "dropping the guard immediately clears the warning; bind it for as long as the condition holds"]
|
||||||
|
|
@ -237,6 +254,24 @@ mod tests {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn boot_warning_survives_without_a_held_guard() {
|
||||||
|
// Unlike `set_warning`, `set_boot_warning` returns nothing to hold —
|
||||||
|
// the whole point is that the warning outlives the call that raised
|
||||||
|
// it (no guard in scope to drop).
|
||||||
|
set_boot_warning("t_boot", "warn", "one-shot step failed");
|
||||||
|
assert_eq!(
|
||||||
|
find("t_boot").map(|w| w.message),
|
||||||
|
Some("one-shot step failed".to_owned())
|
||||||
|
);
|
||||||
|
// A later boot_warning for the same kind still just replaces the
|
||||||
|
// payload (same registry semantics as `set_warning`).
|
||||||
|
set_boot_warning("t_boot", "crit", "still failing");
|
||||||
|
let w = find("t_boot").expect("present");
|
||||||
|
assert_eq!(w.level, "crit");
|
||||||
|
assert_eq!(w.message, "still failing");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn concurrent_same_kind_resolves_to_absent_after_all_drop() {
|
fn concurrent_same_kind_resolves_to_absent_after_all_drop() {
|
||||||
// Many threads contend on ONE kind, each holding its own guard
|
// Many threads contend on ONE kind, each holding its own guard
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue