sweep: wire gateway-nginx reload + config-repo branch-protection into warning banners
Extends the SweepHealth/warnings registry (already landed for knowledge_pull) to two more background sweeps: - gateway_nginx::reload_gateway_nginx: raises a warn-level banner immediately on the first failed reload (routing changes silently not taking effect is user-visible right now, so no debounce). - forge::repos::ensure_config_repo: raises a crit-level banner listing every agent whose config-repo branch protection is currently unapplied (security-relevant — bypasses the deploy pipeline), clearing agents out of the message as they recover. Journal warn!/error! logging is left in place; the registry adds a dashboard-visible signal on top. forge::ensure_all() and matrix::ensure_all() sweeps are deliberately left for a fast-follow.
This commit is contained in:
parent
9fdadb99c0
commit
9ea6160c94
2 changed files with 82 additions and 2 deletions
|
|
@ -8,9 +8,11 @@
|
|||
use anyhow::{Context, Result};
|
||||
use std::fmt::Write as _;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::priv_client;
|
||||
use crate::stats::sweep_health::SweepHealth;
|
||||
|
||||
use crate::agent_sockets;
|
||||
|
||||
|
|
@ -30,6 +32,15 @@ static LAST_FAILED_RELOAD: AtomicU64 = AtomicU64::new(0);
|
|||
/// Minimum gap between retry attempts after a reload failure (30 s).
|
||||
const RELOAD_RETRY_SECS: u64 = 30;
|
||||
|
||||
/// Dashboard-banner health tracker for the reload sweep — raises a
|
||||
/// `gateway_nginx_reload` warning the first time a reload fails (routing is
|
||||
/// broken for whoever depends on the change *right now*, so no debounce
|
||||
/// window) and clears it the moment a reload succeeds again.
|
||||
fn health() -> &'static Mutex<SweepHealth> {
|
||||
static HEALTH: OnceLock<Mutex<SweepHealth>> = OnceLock::new();
|
||||
HEALTH.get_or_init(|| Mutex::new(SweepHealth::new("gateway_nginx_reload", "warn", 1)))
|
||||
}
|
||||
|
||||
/// Nginx proxy headers present in every per-agent location block.
|
||||
/// `$connection_upgrade` is defined in the http context by the NixOS
|
||||
/// nginx module when `recommendedProxySettings = true` (which the
|
||||
|
|
@ -240,6 +251,9 @@ async fn reload_gateway_nginx() {
|
|||
tracing::debug!("gateway nginx sync succeeded");
|
||||
RELOAD_PENDING.store(false, Ordering::Relaxed);
|
||||
LAST_FAILED_RELOAD.store(0, Ordering::Relaxed);
|
||||
if let Ok(mut h) = health().lock() {
|
||||
h.record_ok();
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "gateway nginx sync failed — will retry");
|
||||
|
|
@ -248,6 +262,20 @@ async fn reload_gateway_nginx() {
|
|||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
LAST_FAILED_RELOAD.store(now, Ordering::Relaxed);
|
||||
if let Ok(mut h) = health().lock() {
|
||||
let err = format!("{e:#}");
|
||||
h.record_err(|ctx| {
|
||||
let age = ctx.since_last_ok.map_or_else(
|
||||
|| "no success this process".to_owned(),
|
||||
|d| format!("last ok {} ago", crate::stats::sweep_health::fmt_age(d)),
|
||||
);
|
||||
format!(
|
||||
"gateway nginx reload failing ({} consecutive, {age}) \
|
||||
— routing changes are not taking effect: {err}",
|
||||
ctx.consecutive
|
||||
)
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue