diff --git a/hive-c0re/src/forge/repos.rs b/hive-c0re/src/forge/repos.rs index 2778d070..4f0f8bd0 100644 --- a/hive-c0re/src/forge/repos.rs +++ b/hive-c0re/src/forge/repos.rs @@ -274,8 +274,60 @@ pub async fn ensure_config_repo(name: &str) -> Result<()> { &token, ) .await?; - // Protect `main` core-only, fast-forward-only (no auto force-push). - apply_config_repo_branch_protection(name, &token).await + // Protect `main` core-only, fast-forward-only (no auto force-push). A + // failure here is security-relevant (an unprotected config repo lets an + // operator-merged config PR bypass the deploy pipeline), so it's tracked + // on the dashboard banner in addition to the journal warning callers + // already log — see `record_branch_protection_result`. + let result = apply_config_repo_branch_protection(name, &token).await; + record_branch_protection_result(name, result.is_ok()); + result +} + +/// Dashboard-banner tracker for [`apply_config_repo_branch_protection`] +/// failures. The registry ([`crate::warnings::set_warning`]) only takes +/// `&'static str` kinds, so a dynamic per-agent key isn't possible — instead +/// this keeps one static `crit` warning (`"branch_protection_missing"`) whose +/// message lists every agent currently failing to protect, and clears it +/// once the set is empty. Called on every `ensure_config_repo` pass (startup +/// sweep + per-rebuild), so a fixed agent drops out of the message on its +/// next successful sweep without requiring a restart. +fn record_branch_protection_result(name: &str, ok: bool) { + use std::collections::BTreeSet; + use std::sync::{OnceLock, PoisonError}; + + use crate::warnings::{WarningGuard, set_warning}; + + static FAILING: OnceLock>> = OnceLock::new(); + static GUARD: OnceLock>> = OnceLock::new(); + + let mut failing = FAILING + .get_or_init(|| Mutex::new(BTreeSet::new())) + .lock() + .unwrap_or_else(PoisonError::into_inner); + if ok { + failing.remove(name); + } else { + failing.insert(name.to_owned()); + } + + let mut guard = GUARD + .get_or_init(|| Mutex::new(None)) + .lock() + .unwrap_or_else(PoisonError::into_inner); + if failing.is_empty() { + *guard = None; + return; + } + let names = failing.iter().cloned().collect::>().join(", "); + let message = format!( + "config-repo branch protection not applied for: {names} — \ + operator-merged config PRs for these agents could bypass the deploy pipeline" + ); + match guard.as_ref() { + Some(g) => g.update("crit", message), + None => *guard = Some(set_warning("branch_protection_missing", "crit", message)), + } } /// Ensure the `internal/docs` repo exists. Called once at startup diff --git a/hive-c0re/src/gateway_nginx.rs b/hive-c0re/src/gateway_nginx.rs index 85b0432f..69fffa9e 100644 --- a/hive-c0re/src/gateway_nginx.rs +++ b/hive-c0re/src/gateway_nginx.rs @@ -8,9 +8,11 @@ use anyhow::{Context, Result}; use std::fmt::Write as _; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Mutex, OnceLock}; use std::time::{SystemTime, UNIX_EPOCH}; use crate::priv_client; +use crate::stats::sweep_health::SweepHealth; use crate::agent_sockets; @@ -30,6 +32,15 @@ static LAST_FAILED_RELOAD: AtomicU64 = AtomicU64::new(0); /// Minimum gap between retry attempts after a reload failure (30 s). const RELOAD_RETRY_SECS: u64 = 30; +/// Dashboard-banner health tracker for the reload sweep — raises a +/// `gateway_nginx_reload` warning the first time a reload fails (routing is +/// broken for whoever depends on the change *right now*, so no debounce +/// window) and clears it the moment a reload succeeds again. +fn health() -> &'static Mutex { + static HEALTH: OnceLock> = OnceLock::new(); + HEALTH.get_or_init(|| Mutex::new(SweepHealth::new("gateway_nginx_reload", "warn", 1))) +} + /// Nginx proxy headers present in every per-agent location block. /// `$connection_upgrade` is defined in the http context by the NixOS /// nginx module when `recommendedProxySettings = true` (which the @@ -240,6 +251,9 @@ async fn reload_gateway_nginx() { tracing::debug!("gateway nginx sync succeeded"); RELOAD_PENDING.store(false, Ordering::Relaxed); LAST_FAILED_RELOAD.store(0, Ordering::Relaxed); + if let Ok(mut h) = health().lock() { + h.record_ok(); + } } Err(e) => { tracing::warn!(error = %e, "gateway nginx sync failed — will retry"); @@ -248,6 +262,20 @@ async fn reload_gateway_nginx() { .unwrap_or_default() .as_secs(); LAST_FAILED_RELOAD.store(now, Ordering::Relaxed); + if let Ok(mut h) = health().lock() { + let err = format!("{e:#}"); + h.record_err(|ctx| { + let age = ctx.since_last_ok.map_or_else( + || "no success this process".to_owned(), + |d| format!("last ok {} ago", crate::stats::sweep_health::fmt_age(d)), + ); + format!( + "gateway nginx reload failing ({} consecutive, {age}) \ + — routing changes are not taking effect: {err}", + ctx.consecutive + ) + }); + } } } }