//! Debounced sweep-failure tracking on top of the [`crate::warnings`] //! registry. //! //! Background converge sweeps (knowledge pull, matrix / forge provisioning, //! gateway nginx writes, per-agent branch protection, …) historically //! `warn!`'d to the journal and moved on, so a persistent failure was //! invisible to the operator. [`SweepHealth`] routes that failure to the //! dashboard banner instead — but *debounced*, per the routing guidance in //! the tracker issue: //! //! - a transient miss (container down mid-sweep, a network blip) shouldn't //! flap a banner on the first failure → `threshold > 1` raises only after //! that many **consecutive** misses; //! - a won't-fix-itself failure (missing team, bad config, auth rejection) //! should banner immediately → `threshold == 1`. //! //! The warning clears the moment the sweep next succeeds. A sweep owns one //! `SweepHealth`, calls [`record_ok`](SweepHealth::record_ok) on success and //! [`record_err`](SweepHealth::record_err) on failure, and the held //! [`WarningGuard`] does the rest (drop = clear). use std::time::{Duration, Instant}; use crate::warnings::{WarningGuard, set_warning}; /// Per-sweep failure context handed to the [`record_err`](SweepHealth::record_err) /// message builder so the banner text can include how long the sweep has been /// failing and since when it last worked. #[derive(Debug, Clone, Copy)] pub struct SweepFailure { /// Number of consecutive failures so far (≥ `threshold` when the banner /// is raised). pub consecutive: u32, /// Time since the last recorded success, if the sweep has ever succeeded /// in this process's lifetime. `None` = never succeeded yet. pub since_last_ok: Option, } /// A debounced link between one background sweep and the warning banner. /// Not `Sync`-shared: each sweep task owns its own instance (the underlying /// registry is the shared, thread-safe part). pub struct SweepHealth { kind: &'static str, level: &'static str, threshold: u32, misses: u32, last_ok: Option, guard: Option, } impl SweepHealth { /// Track sweep `kind` at severity `level` (`"warn"` / `"crit"`), raising /// the banner after `threshold` consecutive failures. `threshold` is /// floored at 1 (0 would banner before any failure). #[must_use] pub fn new(kind: &'static str, level: &'static str, threshold: u32) -> Self { Self { kind, level, threshold: threshold.max(1), misses: 0, last_ok: None, guard: None, } } /// Record a successful sweep: reset the consecutive-miss counter and drop /// the guard (clearing any banner entry). pub fn record_ok(&mut self) { self.misses = 0; self.last_ok = Some(Instant::now()); // Dropping the guard removes the warning from the registry. self.guard = None; } /// Record a failed sweep. Bumps the consecutive-miss counter; once it /// reaches `threshold`, raises (or refreshes) the banner warning built by /// `message`. Below the threshold this is a cheap no-op — `message` is /// only invoked when the banner is actually shown, so callers can format /// lazily. pub fn record_err(&mut self, message: impl FnOnce(SweepFailure) -> String) { self.misses = self.misses.saturating_add(1); if self.misses < self.threshold { return; } let ctx = SweepFailure { consecutive: self.misses, since_last_ok: self.last_ok.map(|t| t.elapsed()), }; let msg = message(ctx); match &self.guard { Some(g) => g.update(self.level, msg), None => self.guard = Some(set_warning(self.kind, self.level, msg)), } } /// Whether the banner warning is currently raised for this sweep. #[must_use] pub fn is_warning(&self) -> bool { self.guard.is_some() } } /// Compact human-readable age (e.g. `"45s"`, `"12m"`, `"3h"`, `"2d"`) for a /// banner "last succeeded N ago" suffix. Coarse on purpose — the banner wants /// a glanceable magnitude, not precision. #[must_use] pub fn fmt_age(d: Duration) -> String { let s = d.as_secs(); if s < 60 { format!("{s}s") } else if s < 3600 { format!("{}m", s / 60) } else if s < 86_400 { format!("{}h", s / 3600) } else { format!("{}d", s / 86_400) } } #[cfg(test)] mod tests { use super::*; use crate::warnings::snapshot; fn banner(kind: &str) -> Option { snapshot() .into_iter() .find(|w| w.kind == kind) .map(|w| w.message) } #[test] fn below_threshold_does_not_banner() { let mut h = SweepHealth::new("sh_below", "warn", 3); h.record_err(|_| "boom".to_owned()); h.record_err(|_| "boom".to_owned()); assert!(!h.is_warning()); assert!(banner("sh_below").is_none()); } #[test] fn raises_at_threshold_and_clears_on_ok() { let mut h = SweepHealth::new("sh_raise", "warn", 2); h.record_err(|c| format!("fail #{}", c.consecutive)); assert!(banner("sh_raise").is_none(), "one miss < threshold"); h.record_err(|c| format!("fail #{}", c.consecutive)); assert_eq!(banner("sh_raise").as_deref(), Some("fail #2")); assert!(h.is_warning()); h.record_ok(); assert!(!h.is_warning()); assert!(banner("sh_raise").is_none(), "success clears the banner"); } #[test] fn threshold_one_banners_immediately() { let mut h = SweepHealth::new("sh_immediate", "crit", 1); h.record_err(|_| "gone".to_owned()); assert_eq!(banner("sh_immediate").as_deref(), Some("gone")); h.record_ok(); assert!(banner("sh_immediate").is_none()); } #[test] fn success_resets_consecutive_counter() { let mut h = SweepHealth::new("sh_reset", "warn", 2); h.record_err(|_| "x".to_owned()); // miss 1 h.record_ok(); // reset h.record_err(|c| format!("c={}", c.consecutive)); // miss 1 again, < threshold assert!(banner("sh_reset").is_none(), "counter reset after ok"); h.record_err(|c| format!("c={}", c.consecutive)); // miss 2 → raise assert_eq!(banner("sh_reset").as_deref(), Some("c=2")); h.record_ok(); } #[test] fn refresh_updates_message_while_held() { let mut h = SweepHealth::new("sh_refresh", "warn", 1); h.record_err(|c| format!("miss {}", c.consecutive)); assert_eq!(banner("sh_refresh").as_deref(), Some("miss 1")); h.record_err(|c| format!("miss {}", c.consecutive)); assert_eq!(banner("sh_refresh").as_deref(), Some("miss 2")); h.record_ok(); } #[test] fn since_last_ok_is_some_after_a_success() { let mut h = SweepHealth::new("sh_age", "warn", 1); h.record_err(|c| { assert!(c.since_last_ok.is_none(), "never succeeded yet"); "first".to_owned() }); h.record_ok(); h.record_err(|c| { assert!(c.since_last_ok.is_some(), "has a last-ok timestamp now"); "again".to_owned() }); h.record_ok(); } #[test] fn fmt_age_buckets() { assert_eq!(fmt_age(Duration::from_secs(5)), "5s"); assert_eq!(fmt_age(Duration::from_secs(90)), "1m"); assert_eq!(fmt_age(Duration::from_hours(2)), "2h"); assert_eq!(fmt_age(Duration::from_secs(200_000)), "2d"); } }