sweep: wire gateway-nginx reload + config-repo branch-protection into warning banners

Extends the SweepHealth/warnings registry (already landed for
knowledge_pull) to two more background sweeps:

- gateway_nginx::reload_gateway_nginx: raises a warn-level banner
  immediately on the first failed reload (routing changes silently
  not taking effect is user-visible right now, so no debounce).
- forge::repos::ensure_config_repo: raises a crit-level banner
  listing every agent whose config-repo branch protection is
  currently unapplied (security-relevant — bypasses the deploy
  pipeline), clearing agents out of the message as they recover.

Journal warn!/error! logging is left in place; the registry adds a
dashboard-visible signal on top. forge::ensure_all() and
matrix::ensure_all() sweeps are deliberately left for a fast-follow.
This commit is contained in:
iris 2026-07-15 23:46:45 +02:00 committed by mara
commit 9ea6160c94
2 changed files with 82 additions and 2 deletions

View file

@ -8,9 +8,11 @@
use anyhow::{Context, Result};
use std::fmt::Write as _;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Mutex, OnceLock};
use std::time::{SystemTime, UNIX_EPOCH};
use crate::priv_client;
use crate::stats::sweep_health::SweepHealth;
use crate::agent_sockets;
@ -30,6 +32,15 @@ static LAST_FAILED_RELOAD: AtomicU64 = AtomicU64::new(0);
/// Minimum gap between retry attempts after a reload failure (30 s).
const RELOAD_RETRY_SECS: u64 = 30;
/// Dashboard-banner health tracker for the reload sweep — raises a
/// `gateway_nginx_reload` warning the first time a reload fails (routing is
/// broken for whoever depends on the change *right now*, so no debounce
/// window) and clears it the moment a reload succeeds again.
fn health() -> &'static Mutex<SweepHealth> {
static HEALTH: OnceLock<Mutex<SweepHealth>> = OnceLock::new();
HEALTH.get_or_init(|| Mutex::new(SweepHealth::new("gateway_nginx_reload", "warn", 1)))
}
/// Nginx proxy headers present in every per-agent location block.
/// `$connection_upgrade` is defined in the http context by the NixOS
/// nginx module when `recommendedProxySettings = true` (which the
@ -240,6 +251,9 @@ async fn reload_gateway_nginx() {
tracing::debug!("gateway nginx sync succeeded");
RELOAD_PENDING.store(false, Ordering::Relaxed);
LAST_FAILED_RELOAD.store(0, Ordering::Relaxed);
if let Ok(mut h) = health().lock() {
h.record_ok();
}
}
Err(e) => {
tracing::warn!(error = %e, "gateway nginx sync failed — will retry");
@ -248,6 +262,20 @@ async fn reload_gateway_nginx() {
.unwrap_or_default()
.as_secs();
LAST_FAILED_RELOAD.store(now, Ordering::Relaxed);
if let Ok(mut h) = health().lock() {
let err = format!("{e:#}");
h.record_err(|ctx| {
let age = ctx.since_last_ok.map_or_else(
|| "no success this process".to_owned(),
|d| format!("last ok {} ago", crate::stats::sweep_health::fmt_age(d)),
);
format!(
"gateway nginx reload failing ({} consecutive, {age}) \
routing changes are not taking effect: {err}",
ctx.consecutive
)
});
}
}
}
}