crash_watch: suppress crash event on recently-cleared transient (closes #425)

hive-c0re/src/crash_watch.rs

@@ -25,6 +25,13 @@ use crate::lifecycle::{self, AGENT_PREFIX, MANAGER_NAME};
 
 const POLL_INTERVAL: Duration = Duration::from_secs(10);
 
+/// How long an operator-initiated transient stays "recently cleared"
+/// for the purpose of suppressing crash events. Three full
+/// `POLL_INTERVAL`s gives the post-lifecycle path comfortable
+/// breathing room — the watcher will have polled at least twice
+/// inside the window even with worst-case timer skew (#425).
+const RECENT_TRANSIENT_GRACE: Duration = Duration::from_secs(30);
+
 pub fn spawn(coord: Arc<Coordinator>) {
     let mut shutdown = coord.shutdown_rx();
     tokio::spawn(async move {
@@ -92,17 +99,14 @@ pub fn spawn(coord: Arc<Coordinator>) {
 
 fn emit_crash_transitions(coord: &Coordinator, prev: &HashSet<String>, current: &HashSet<String>) {
     let transients = coord.transient_snapshot();
+    // Operator actions whose RAII guard already cleared but only just;
+    // suppresses the race where `lifecycle::kill` returns + drops the
+    // guard between two crash-watch polls (closes #425).
+    let recent = coord.recent_transient_within(RECENT_TRANSIENT_GRACE);
     for stopped in prev.difference(current) {
-        let deliberate = transients.get(stopped).is_some_and(|st| {
-            matches!(
-                st.kind,
-                TransientKind::Stopping
-                    | TransientKind::Restarting
-                    | TransientKind::Destroying
-                    | TransientKind::Rebuilding
-            )
-        });
-        if deliberate {
+        let active = transients.get(stopped).map(|st| st.kind);
+        let recently_cleared = recent.get(stopped).copied();
+        if is_deliberate_stop(active, recently_cleared) {
             continue;
         }
         tracing::warn!(agent = %stopped, "container crash detected");
@@ -113,6 +117,76 @@ fn emit_crash_transitions(coord: &Coordinator, prev: &HashSet<String>, current:
     }
 }
 
+/// Pure classifier: did the operator stop / restart / destroy /
+/// rebuild this container, or did it crash? Splits the matcher out so
+/// it has a focused unit test (#425) without needing a Coordinator
+/// fixture. `active` is the currently-set transient (if any),
+/// `recently_cleared` is one whose RAII guard dropped within the
+/// grace window.
+fn is_deliberate_stop(
+    active: Option<TransientKind>,
+    recently_cleared: Option<TransientKind>,
+) -> bool {
+    let is_op_kind = |kind: TransientKind| {
+        matches!(
+            kind,
+            TransientKind::Stopping
+                | TransientKind::Restarting
+                | TransientKind::Destroying
+                | TransientKind::Rebuilding
+        )
+    };
+    active.is_some_and(is_op_kind) || recently_cleared.is_some_and(is_op_kind)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn deliberate_when_active_transient_is_operator_kind() {
+        for kind in [
+            TransientKind::Stopping,
+            TransientKind::Restarting,
+            TransientKind::Destroying,
+            TransientKind::Rebuilding,
+        ] {
+            assert!(is_deliberate_stop(Some(kind), None), "{kind:?}");
+        }
+    }
+
+    #[test]
+    fn deliberate_when_recent_transient_is_operator_kind() {
+        // Race the #425 bug repros: lifecycle action completes + drops
+        // the guard between two polls. recent_transient catches it.
+        for kind in [
+            TransientKind::Stopping,
+            TransientKind::Restarting,
+            TransientKind::Destroying,
+            TransientKind::Rebuilding,
+        ] {
+            assert!(is_deliberate_stop(None, Some(kind)), "{kind:?}");
+        }
+    }
+
+    #[test]
+    fn not_deliberate_with_no_transient_at_all() {
+        // The real-crash case — fires the ContainerCrash event.
+        assert!(!is_deliberate_stop(None, None));
+    }
+
+    #[test]
+    fn not_deliberate_when_only_spawning_starting() {
+        // Spawning/Starting are never paired with a "stopped" transition
+        // — they're starts. If we see one alongside a stop, it's
+        // unrelated (e.g. just-started container died), still a crash.
+        for kind in [TransientKind::Spawning, TransientKind::Starting] {
+            assert!(!is_deliberate_stop(Some(kind), None), "{kind:?} active");
+            assert!(!is_deliberate_stop(None, Some(kind)), "{kind:?} recent");
+        }
+    }
+}
+
 fn emit_login_transitions(
     coord: &Coordinator,
     prev: &HashSet<String>,