//! Per-container crash and login-state watcher. Polls every managed //! container on a 10s interval. Fires `ContainerCrash`, `LoggedIn`, //! and `NeedsLogin` helper events. Event semantics and the //! `RECENT_TRANSIENT_GRACE` window: `docs/approvals.md::Helper events`. use std::collections::HashSet; use std::sync::Arc; use std::time::Duration; use crate::container_view::claude_has_session; use crate::coordinator::Coordinator; use crate::lifecycle::{self, AGENT_PREFIX}; const POLL_INTERVAL: Duration = Duration::from_secs(10); /// How long an operator-initiated transient stays "recently cleared" /// for the purpose of suppressing crash events. Three full /// `POLL_INTERVAL`s gives the post-lifecycle path comfortable /// breathing room — the watcher will have polled at least twice /// inside the window even with worst-case timer skew. const RECENT_TRANSIENT_GRACE: Duration = Duration::from_secs(30); pub fn spawn(coord: Arc) { let mut shutdown = coord.shutdown_rx(); tokio::spawn(async move { let mut prev_running: HashSet = HashSet::new(); let mut prev_logged_in: HashSet = HashSet::new(); let mut prev_sub_agents: HashSet = HashSet::new(); let mut seeded = false; loop { let raw = lifecycle::list().await.unwrap_or_default(); let mut current_running = HashSet::new(); let mut current_logged_in = HashSet::new(); let mut sub_agents: Vec = Vec::new(); for c in &raw { let Some(logical) = c.strip_prefix(AGENT_PREFIX) else { continue; }; let logical = logical.to_owned(); sub_agents.push(logical.clone()); if lifecycle::is_running(&logical).await { current_running.insert(logical.clone()); } if hive_types::Ident::parse(&logical) .is_ok_and(|id| claude_has_session(&Coordinator::agent_claude_dir(&id))) { current_logged_in.insert(logical.clone()); } } if seeded { emit_crash_transitions(&coord, &prev_running, ¤t_running); emit_login_transitions( &prev_logged_in, ¤t_logged_in, &sub_agents, &prev_sub_agents, ) .await; } // Periodic container rescan — catches state flips that // happen outside our mutation surface (operator runs // `nixos-container stop` over ssh, agent logs in via its // own web UI, etc.) so the dashboard converges within one // POLL_INTERVAL. Idempotent + cheap when nothing changed. coord.rescan_containers_and_emit().await; prev_running = current_running; prev_logged_in = current_logged_in; prev_sub_agents = sub_agents.into_iter().collect(); seeded = true; tokio::select! { () = tokio::time::sleep(POLL_INTERVAL) => {} _ = shutdown.changed() => { tracing::info!("crash watcher: shutdown signal received"); break; } } } }); } fn emit_crash_transitions(coord: &Coordinator, prev: &HashSet, current: &HashSet) { let transients = coord.transient_snapshot(); // Operator actions whose RAII guard already cleared but only just; // suppresses the race where `lifecycle::kill` returns + drops the // guard between two crash-watch polls. let recent = coord.recent_transient_within(RECENT_TRANSIENT_GRACE); for stopped in prev.difference(current) { // One source: a running queue node that declared it takes the container // down. There used to be a second — an imperative suppression guard for // operations with no node behind them — and destroy becoming a DAG // removed the last caller, so intent now has exactly one home. // `any`, not "the" pill: an agent can have several running nodes at // once (a lease-exempt build alongside a lease-holding stop), and it // only takes one of them expecting the container down for this to be // a deliberate stop rather than a crash. let active = transients .get(stopped) .map(|sts| sts.iter().any(|st| st.takes_container_down)); let recently_cleared = recent.get(stopped).copied(); if is_deliberate_stop(active, recently_cleared) { continue; } tracing::warn!(agent = %stopped, "container crash detected"); coord.record_crash(stopped); coord.notify_manager(&hive_sh4re::manager::HelperEvent::ContainerCrash { agent: stopped.clone(), note: Some("container stopped without an operator action".into()), }); } } /// Pure classifier: did an operation take this container down on purpose, or /// did it crash? Splits the matcher out so it has a focused unit test without /// needing a Coordinator fixture. `active` is the currently-set transient's /// `deliberate_stop` (if any), `recently_cleared` is one whose RAII guard /// dropped within the grace window. /// /// This reads the flag the transient's creator set and nothing else. It used to /// match a `TransientKind`, which meant a pill's *display* vocabulary decided a /// crash-alert question — so renaming or adding a label silently moved the /// alerting boundary. Whoever starts the operation knows whether the container /// is meant to go down; nothing downstream can re-derive it. fn is_deliberate_stop(active: Option, recently_cleared: Option) -> bool { active.unwrap_or(false) || recently_cleared.unwrap_or(false) } async fn emit_login_transitions( prev: &HashSet, current: &HashSet, sub_agents: &[String], prev_sub_agents: &HashSet, ) { for agent in current.difference(prev) { tracing::info!(%agent, "agent logged in"); crate::swarm_notices::notify( "core", Some(format!("logged_in:{agent}")), format!("agent '{agent}' logged in"), None, ) .await; } // Detect transitions into "needs login": an agent that was previously // logged-in goes unsigned (credentials deleted), OR a brand-new agent // appears without a session. // // prev_needs uses prev_sub_agents (the agent set from the last tick) so // that a newly-spawned agent — which does not appear in prev_sub_agents — // is absent from prev_needs even though it's not in prev_logged_in. // Without this, new agents land in both prev_needs and current_needs and // the set difference is empty, silently dropping the event. let prev_needs: HashSet<&str> = prev_sub_agents .iter() .map(String::as_str) .filter(|n| !prev.contains(*n)) .collect(); let current_needs: HashSet<&str> = sub_agents .iter() .map(String::as_str) .filter(|n| !current.contains(*n)) .collect(); for agent in current_needs.difference(&prev_needs) { tracing::info!(%agent, "agent needs login"); crate::swarm_notices::notify( "core", Some(format!("needs_login:{agent}")), format!("agent '{agent}' needs login"), None, ) .await; } } #[cfg(test)] mod tests { use super::*; #[test] fn deliberate_when_either_source_says_so() { // Active guard, and the race repro: a lifecycle action completes and // drops its guard between two polls, so only `recent` still carries it. assert!(is_deliberate_stop(Some(true), None)); assert!(is_deliberate_stop(None, Some(true))); assert!(is_deliberate_stop(Some(true), Some(true))); } #[test] fn not_deliberate_with_no_transient_at_all() { // The real-crash case — fires the ContainerCrash event. assert!(!is_deliberate_stop(None, None)); } #[test] fn not_deliberate_when_the_operation_was_bringing_the_container_up() { // The case that used to be spelled `Spawning` / `Starting`: an // operation IS in flight, but it is not one that takes the container // down, so a container that vanishes under it really did crash. // // This is why `deliberate_stop` is carried rather than inferred from // the pill — `Create` and `Start` hold the agent's lease exactly like // `Stop` does, so "has a pill" cannot answer this. assert!(!is_deliberate_stop(Some(false), None)); assert!(!is_deliberate_stop(None, Some(false))); assert!(!is_deliberate_stop(Some(false), Some(false))); } }