sweep: wire gateway-nginx reload + config-repo branch-protection into warning banners
Extends the SweepHealth/warnings registry (already landed for knowledge_pull) to two more background sweeps: - gateway_nginx::reload_gateway_nginx: raises a warn-level banner immediately on the first failed reload (routing changes silently not taking effect is user-visible right now, so no debounce). - forge::repos::ensure_config_repo: raises a crit-level banner listing every agent whose config-repo branch protection is currently unapplied (security-relevant — bypasses the deploy pipeline), clearing agents out of the message as they recover. Journal warn!/error! logging is left in place; the registry adds a dashboard-visible signal on top. forge::ensure_all() and matrix::ensure_all() sweeps are deliberately left for a fast-follow.
This commit is contained in:
parent
9fdadb99c0
commit
9ea6160c94
2 changed files with 82 additions and 2 deletions
|
|
@ -274,8 +274,60 @@ pub async fn ensure_config_repo(name: &str) -> Result<()> {
|
|||
&token,
|
||||
)
|
||||
.await?;
|
||||
// Protect `main` core-only, fast-forward-only (no auto force-push).
|
||||
apply_config_repo_branch_protection(name, &token).await
|
||||
// Protect `main` core-only, fast-forward-only (no auto force-push). A
|
||||
// failure here is security-relevant (an unprotected config repo lets an
|
||||
// operator-merged config PR bypass the deploy pipeline), so it's tracked
|
||||
// on the dashboard banner in addition to the journal warning callers
|
||||
// already log — see `record_branch_protection_result`.
|
||||
let result = apply_config_repo_branch_protection(name, &token).await;
|
||||
record_branch_protection_result(name, result.is_ok());
|
||||
result
|
||||
}
|
||||
|
||||
/// Dashboard-banner tracker for [`apply_config_repo_branch_protection`]
|
||||
/// failures. The registry ([`crate::warnings::set_warning`]) only takes
|
||||
/// `&'static str` kinds, so a dynamic per-agent key isn't possible — instead
|
||||
/// this keeps one static `crit` warning (`"branch_protection_missing"`) whose
|
||||
/// message lists every agent currently failing to protect, and clears it
|
||||
/// once the set is empty. Called on every `ensure_config_repo` pass (startup
|
||||
/// sweep + per-rebuild), so a fixed agent drops out of the message on its
|
||||
/// next successful sweep without requiring a restart.
|
||||
fn record_branch_protection_result(name: &str, ok: bool) {
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::{OnceLock, PoisonError};
|
||||
|
||||
use crate::warnings::{WarningGuard, set_warning};
|
||||
|
||||
static FAILING: OnceLock<Mutex<BTreeSet<String>>> = OnceLock::new();
|
||||
static GUARD: OnceLock<Mutex<Option<WarningGuard>>> = OnceLock::new();
|
||||
|
||||
let mut failing = FAILING
|
||||
.get_or_init(|| Mutex::new(BTreeSet::new()))
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner);
|
||||
if ok {
|
||||
failing.remove(name);
|
||||
} else {
|
||||
failing.insert(name.to_owned());
|
||||
}
|
||||
|
||||
let mut guard = GUARD
|
||||
.get_or_init(|| Mutex::new(None))
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner);
|
||||
if failing.is_empty() {
|
||||
*guard = None;
|
||||
return;
|
||||
}
|
||||
let names = failing.iter().cloned().collect::<Vec<_>>().join(", ");
|
||||
let message = format!(
|
||||
"config-repo branch protection not applied for: {names} — \
|
||||
operator-merged config PRs for these agents could bypass the deploy pipeline"
|
||||
);
|
||||
match guard.as_ref() {
|
||||
Some(g) => g.update("crit", message),
|
||||
None => *guard = Some(set_warning("branch_protection_missing", "crit", message)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Ensure the `internal/docs` repo exists. Called once at startup
|
||||
|
|
|
|||
|
|
@ -8,9 +8,11 @@
|
|||
use anyhow::{Context, Result};
|
||||
use std::fmt::Write as _;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::priv_client;
|
||||
use crate::stats::sweep_health::SweepHealth;
|
||||
|
||||
use crate::agent_sockets;
|
||||
|
||||
|
|
@ -30,6 +32,15 @@ static LAST_FAILED_RELOAD: AtomicU64 = AtomicU64::new(0);
|
|||
/// Minimum gap between retry attempts after a reload failure (30 s).
|
||||
const RELOAD_RETRY_SECS: u64 = 30;
|
||||
|
||||
/// Dashboard-banner health tracker for the reload sweep — raises a
|
||||
/// `gateway_nginx_reload` warning the first time a reload fails (routing is
|
||||
/// broken for whoever depends on the change *right now*, so no debounce
|
||||
/// window) and clears it the moment a reload succeeds again.
|
||||
fn health() -> &'static Mutex<SweepHealth> {
|
||||
static HEALTH: OnceLock<Mutex<SweepHealth>> = OnceLock::new();
|
||||
HEALTH.get_or_init(|| Mutex::new(SweepHealth::new("gateway_nginx_reload", "warn", 1)))
|
||||
}
|
||||
|
||||
/// Nginx proxy headers present in every per-agent location block.
|
||||
/// `$connection_upgrade` is defined in the http context by the NixOS
|
||||
/// nginx module when `recommendedProxySettings = true` (which the
|
||||
|
|
@ -240,6 +251,9 @@ async fn reload_gateway_nginx() {
|
|||
tracing::debug!("gateway nginx sync succeeded");
|
||||
RELOAD_PENDING.store(false, Ordering::Relaxed);
|
||||
LAST_FAILED_RELOAD.store(0, Ordering::Relaxed);
|
||||
if let Ok(mut h) = health().lock() {
|
||||
h.record_ok();
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "gateway nginx sync failed — will retry");
|
||||
|
|
@ -248,6 +262,20 @@ async fn reload_gateway_nginx() {
|
|||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
LAST_FAILED_RELOAD.store(now, Ordering::Relaxed);
|
||||
if let Ok(mut h) = health().lock() {
|
||||
let err = format!("{e:#}");
|
||||
h.record_err(|ctx| {
|
||||
let age = ctx.since_last_ok.map_or_else(
|
||||
|| "no success this process".to_owned(),
|
||||
|d| format!("last ok {} ago", crate::stats::sweep_health::fmt_age(d)),
|
||||
);
|
||||
format!(
|
||||
"gateway nginx reload failing ({} consecutive, {age}) \
|
||||
— routing changes are not taking effect: {err}",
|
||||
ctx.consecutive
|
||||
)
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue