hive-c0re: replace the agents_crashing banner with a crash-watcher log line + grafana alert

hive-c0re/src/stats/host_stats.rs

@@ -15,8 +15,6 @@
 //! [`refresh_disk_warning`] drives a held guard from a periodic task (see
 //! `spawn` in `main.rs`) instead of a `statvfs` per `/api/state`.
 
-use std::collections::HashMap;
-
 use serde::Serialize;
 use utoipa::ToSchema;
 
@@ -99,19 +97,20 @@ pub fn refresh_disk_warning(guard: &mut Option<crate::warnings::WarningGuard>) {
 }
 
 /// Agent-state warnings derived from the live container snapshot the
-/// dashboard already holds: agents that need a claude login, and agents
-/// that are crashing. Kept separate from [`server_warnings`] (host
-/// probes) because the caller owns the container list + crash counts;
-/// the dashboard concatenates both into one banner list.
+/// dashboard already holds: agents that need a claude login. Kept
+/// separate from [`server_warnings`] (host probes) because the caller
+/// owns the container list; the dashboard concatenates both into one
+/// banner list.
 ///
-/// `crash_counts` is `Coordinator::recent_crash_counts(window)` — agent →
-/// number of crashes inside that window — so a crash-looping agent shows
-/// its repeat count rather than a single point-in-time flap.
+/// Container crashes used to have a sibling `agents_crashing` entry here,
+/// fed by `Coordinator::recent_crash_counts` — removed in favour of a
+/// swarm-level grafana alert on the crash-watcher's own log line (see
+/// `nix/host-modules/swarm-grafana.nix`'s `hh-container-crashed` rule and
+/// `docs/scheduler/observability.md`). That ships the same signal to the
+/// operator without needing an agent (or this per-hive banner) in the
+/// loop at all.
 #[must_use]
-pub fn agent_state_warnings<S: std::hash::BuildHasher>(
-    containers: &[ContainerView],
-    crash_counts: &HashMap<String, usize, S>,
-) -> Vec<ServerWarning> {
+pub fn agent_state_warnings(containers: &[ContainerView]) -> Vec<ServerWarning> {
     let mut out = Vec::new();
 
     // `needs_login` is already running-gated in `container_view::build_all`,
@@ -137,27 +136,6 @@ pub fn agent_state_warnings<S: std::hash::BuildHasher>(
         });
     }
 
-    if !crash_counts.is_empty() {
-        let mut crashing: Vec<(&String, usize)> =
-            crash_counts.iter().map(|(a, n)| (a, *n)).collect();
-        crashing.sort_unstable_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0)));
-        let list = crashing
-            .iter()
-            .map(|(a, n)| format!("{a} (×{n})"))
-            .collect::<Vec<_>>()
-            .join(", ");
-        out.push(ServerWarning {
-            kind: "agents_crashing",
-            level: "crit",
-            message: format!(
-                "{n} agent{plural} crashing: {list} \
-                 — check the container journal (`hivectl logs <agent>`)",
-                n = crashing.len(),
-                plural = if crashing.len() == 1 { "" } else { "s" },
-            ),
-        });
-    }
-
     out
 }
 
@@ -264,13 +242,13 @@ mod tests {
     #[test]
     fn no_agent_warnings_when_all_healthy() {
         let containers = [cv("alice", false), cv("bob", false)];
-        assert!(agent_state_warnings(&containers, &HashMap::new()).is_empty());
+        assert!(agent_state_warnings(&containers).is_empty());
     }
 
     #[test]
     fn pending_logins_lists_sorted_agents() {
         let containers = [cv("zoe", true), cv("amy", false), cv("bob", true)];
-        let w = agent_state_warnings(&containers, &HashMap::new());
+        let w = agent_state_warnings(&containers);
         assert_eq!(w.len(), 1);
         assert_eq!(w[0].kind, "pending_logins");
         assert_eq!(w[0].level, "warn");
@@ -284,29 +262,7 @@ mod tests {
     #[test]
     fn singular_grammar_for_one_agent() {
         let containers = [cv("solo", true)];
-        let w = agent_state_warnings(&containers, &HashMap::new());
+        let w = agent_state_warnings(&containers);
         assert!(w[0].message.starts_with("1 agent needs claude login: solo"));
     }
-
-    #[test]
-    fn crashing_warning_is_crit_and_count_ordered() {
-        let crashes = HashMap::from([("flap".to_owned(), 5), ("blip".to_owned(), 1)]);
-        let w = agent_state_warnings(&[], &crashes);
-        assert_eq!(w.len(), 1);
-        assert_eq!(w[0].kind, "agents_crashing");
-        assert_eq!(w[0].level, "crit");
-        // higher crash count first
-        assert!(w[0].message.contains("flap (×5), blip (×1)"));
-    }
-
-    #[test]
-    fn both_warnings_coexist() {
-        let containers = [cv("a", true)];
-        let crashes = HashMap::from([("b".to_owned(), 2)]);
-        let kinds: Vec<&str> = agent_state_warnings(&containers, &crashes)
-            .iter()
-            .map(|w| w.kind)
-            .collect();
-        assert_eq!(kinds, ["pending_logins", "agents_crashing"]);
-    }
 }