crash_watch: suppress crash event on recently-cleared transient (closes #425)
This commit is contained in:
parent
8dca741f09
commit
aba1d3153e
2 changed files with 124 additions and 12 deletions
|
|
@ -67,6 +67,21 @@ pub struct Coordinator {
|
|||
/// Read by the dashboard to render a spinner; cleared when the action
|
||||
/// resolves (success or failure).
|
||||
transient: Mutex<HashMap<String, TransientState>>,
|
||||
/// Tombstone for transients that have JUST been cleared. The
|
||||
/// crash watcher polls every 10s and would race the
|
||||
/// drop-clears-immediately path of `TransientGuard`: an operator
|
||||
/// kill / restart sets `Stopping` → runs `nixos-container stop` →
|
||||
/// drop clears the transient → poll fires next tick and sees the
|
||||
/// container missing-from-running with no active transient →
|
||||
/// spurious "container stopped without an operator action"
|
||||
/// message (closes #425).
|
||||
///
|
||||
/// `clear_transient` stamps the cleared kind here with an
|
||||
/// `Instant`; `recent_transient_within(grace)` returns the set of
|
||||
/// agents whose tombstone is still inside the grace window. Crash
|
||||
/// watcher consults both this and the active map before declaring
|
||||
/// a stop deliberate.
|
||||
recent_transient: Mutex<HashMap<String, (TransientKind, std::time::Instant)>>,
|
||||
/// Unified wire-facing event channel feeding the dashboard SSE
|
||||
/// stream. Carries broker messages (mirrored from `broker.subscribe`
|
||||
/// by the forwarder task in `main.rs`) and dashboard-only mutation
|
||||
|
|
@ -211,6 +226,7 @@ impl Coordinator {
|
|||
context_window_tokens,
|
||||
agents: Mutex::new(HashMap::new()),
|
||||
transient: Mutex::new(HashMap::new()),
|
||||
recent_transient: Mutex::new(HashMap::new()),
|
||||
dashboard_events,
|
||||
event_seq: AtomicU64::new(0),
|
||||
meta_updates_active: AtomicU64::new(0),
|
||||
|
|
@ -563,8 +579,18 @@ impl Coordinator {
|
|||
}
|
||||
|
||||
pub fn clear_transient(&self, name: &str) {
|
||||
let removed = self.transient.lock().unwrap().remove(name).is_some();
|
||||
if removed {
|
||||
let removed = self.transient.lock().unwrap().remove(name);
|
||||
if let Some(state) = removed {
|
||||
// Stamp the tombstone so the crash watcher can still see
|
||||
// "operator kicked this off recently" on its next 10s poll
|
||||
// — without this, the clear-then-poll race produced a
|
||||
// spurious ContainerCrash on every operator stop/restart
|
||||
// (#425). Old entries get reaped lazily on read so the map
|
||||
// doesn't grow unbounded.
|
||||
self.recent_transient
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(name.to_owned(), (state.kind, std::time::Instant::now()));
|
||||
self.emit_dashboard_event(DashboardEvent::TransientCleared {
|
||||
seq: self.next_seq(),
|
||||
name: name.to_owned(),
|
||||
|
|
@ -572,6 +598,18 @@ impl Coordinator {
|
|||
}
|
||||
}
|
||||
|
||||
/// Set of agents whose transient was cleared within the last
|
||||
/// `grace` seconds — i.e. agents the operator just acted on,
|
||||
/// whose stop the crash watcher should NOT classify as a crash.
|
||||
/// Lazily reaps entries older than `grace` so the map stays
|
||||
/// bounded by the active agent count.
|
||||
pub fn recent_transient_within(&self, grace: std::time::Duration) -> HashMap<String, TransientKind> {
|
||||
let now = std::time::Instant::now();
|
||||
let mut map = self.recent_transient.lock().unwrap();
|
||||
map.retain(|_, (_, ts)| now.duration_since(*ts) <= grace);
|
||||
map.iter().map(|(k, (kind, _))| (k.clone(), *kind)).collect()
|
||||
}
|
||||
|
||||
/// Set a transient state and return a guard that clears it on drop.
|
||||
/// Use this from any path where the surrounding future could be
|
||||
/// cancelled or panic between set and clear (HTTP handlers, spawned
|
||||
|
|
|
|||
Loading…
Reference in a new issue