sweep: wire gateway-nginx reload + config-repo branch-protection into warning banners

Extends the SweepHealth/warnings registry (already landed for
knowledge_pull) to two more background sweeps:

- gateway_nginx::reload_gateway_nginx: raises a warn-level banner
  immediately on the first failed reload (routing changes silently
  not taking effect is user-visible right now, so no debounce).
- forge::repos::ensure_config_repo: raises a crit-level banner
  listing every agent whose config-repo branch protection is
  currently unapplied (security-relevant — bypasses the deploy
  pipeline), clearing agents out of the message as they recover.

Journal warn!/error! logging is left in place; the registry adds a
dashboard-visible signal on top. forge::ensure_all() and
matrix::ensure_all() sweeps are deliberately left for a fast-follow.
This commit is contained in:
iris 2026-07-15 23:46:45 +02:00 committed by mara
commit 9ea6160c94
2 changed files with 82 additions and 2 deletions

View file

@ -274,8 +274,60 @@ pub async fn ensure_config_repo(name: &str) -> Result<()> {
&token,
)
.await?;
// Protect `main` core-only, fast-forward-only (no auto force-push).
apply_config_repo_branch_protection(name, &token).await
// Protect `main` core-only, fast-forward-only (no auto force-push). A
// failure here is security-relevant (an unprotected config repo lets an
// operator-merged config PR bypass the deploy pipeline), so it's tracked
// on the dashboard banner in addition to the journal warning callers
// already log — see `record_branch_protection_result`.
let result = apply_config_repo_branch_protection(name, &token).await;
record_branch_protection_result(name, result.is_ok());
result
}
/// Dashboard-banner tracker for [`apply_config_repo_branch_protection`]
/// failures. The registry ([`crate::warnings::set_warning`]) only takes
/// `&'static str` kinds, so a dynamic per-agent key isn't possible — instead
/// this keeps one static `crit` warning (`"branch_protection_missing"`) whose
/// message lists every agent currently failing to protect, and clears it
/// once the set is empty. Called on every `ensure_config_repo` pass (startup
/// sweep + per-rebuild), so a fixed agent drops out of the message on its
/// next successful sweep without requiring a restart.
fn record_branch_protection_result(name: &str, ok: bool) {
use std::collections::BTreeSet;
use std::sync::{OnceLock, PoisonError};
use crate::warnings::{WarningGuard, set_warning};
static FAILING: OnceLock<Mutex<BTreeSet<String>>> = OnceLock::new();
static GUARD: OnceLock<Mutex<Option<WarningGuard>>> = OnceLock::new();
let mut failing = FAILING
.get_or_init(|| Mutex::new(BTreeSet::new()))
.lock()
.unwrap_or_else(PoisonError::into_inner);
if ok {
failing.remove(name);
} else {
failing.insert(name.to_owned());
}
let mut guard = GUARD
.get_or_init(|| Mutex::new(None))
.lock()
.unwrap_or_else(PoisonError::into_inner);
if failing.is_empty() {
*guard = None;
return;
}
let names = failing.iter().cloned().collect::<Vec<_>>().join(", ");
let message = format!(
"config-repo branch protection not applied for: {names} — \
operator-merged config PRs for these agents could bypass the deploy pipeline"
);
match guard.as_ref() {
Some(g) => g.update("crit", message),
None => *guard = Some(set_warning("branch_protection_missing", "crit", message)),
}
}
/// Ensure the `internal/docs` repo exists. Called once at startup

View file

@ -8,9 +8,11 @@
use anyhow::{Context, Result};
use std::fmt::Write as _;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Mutex, OnceLock};
use std::time::{SystemTime, UNIX_EPOCH};
use crate::priv_client;
use crate::stats::sweep_health::SweepHealth;
use crate::agent_sockets;
@ -30,6 +32,15 @@ static LAST_FAILED_RELOAD: AtomicU64 = AtomicU64::new(0);
/// Minimum gap between retry attempts after a reload failure (30 s).
const RELOAD_RETRY_SECS: u64 = 30;
/// Dashboard-banner health tracker for the reload sweep — raises a
/// `gateway_nginx_reload` warning the first time a reload fails (routing is
/// broken for whoever depends on the change *right now*, so no debounce
/// window) and clears it the moment a reload succeeds again.
fn health() -> &'static Mutex<SweepHealth> {
static HEALTH: OnceLock<Mutex<SweepHealth>> = OnceLock::new();
HEALTH.get_or_init(|| Mutex::new(SweepHealth::new("gateway_nginx_reload", "warn", 1)))
}
/// Nginx proxy headers present in every per-agent location block.
/// `$connection_upgrade` is defined in the http context by the NixOS
/// nginx module when `recommendedProxySettings = true` (which the
@ -240,6 +251,9 @@ async fn reload_gateway_nginx() {
tracing::debug!("gateway nginx sync succeeded");
RELOAD_PENDING.store(false, Ordering::Relaxed);
LAST_FAILED_RELOAD.store(0, Ordering::Relaxed);
if let Ok(mut h) = health().lock() {
h.record_ok();
}
}
Err(e) => {
tracing::warn!(error = %e, "gateway nginx sync failed — will retry");
@ -248,6 +262,20 @@ async fn reload_gateway_nginx() {
.unwrap_or_default()
.as_secs();
LAST_FAILED_RELOAD.store(now, Ordering::Relaxed);
if let Ok(mut h) = health().lock() {
let err = format!("{e:#}");
h.record_err(|ctx| {
let age = ctx.since_last_ok.map_or_else(
|| "no success this process".to_owned(),
|d| format!("last ok {} ago", crate::stats::sweep_health::fmt_age(d)),
);
format!(
"gateway nginx reload failing ({} consecutive, {age}) \
routing changes are not taking effect: {err}",
ctx.consecutive
)
});
}
}
}
}