vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

hive-c0re: replace the agents_crashing banner with a crash-watcher log line + grafana alert

Browse source
This commit is contained in:
damocles 2026-10-12 22:21:04 +02:00 • committed by mara
commit 02831fb5e9
13 changed files with 214 additions and 224 deletions
Download patch file Download diff file Expand all files Collapse all files

80
hive-c0re/src/coordinator.rs
View file

@ -128,14 +128,6 @@ pub struct Coordinator {
/// `StopForUpdate` (`true`) leaves the tombstone reading `false`, and the
/// crash watcher then reports an intentional stop as a **crash**.
recent_transient: Mutex<HashMap<(String, String), (bool, std::time::Instant)>>,
/// Timestamps of recent unexpected container crashes, keyed by agent.
/// Fed by `crash_watch` each time it classifies a stop as a crash (so
/// a crash-looping container — which `Restart=on-failure` flips back
/// to running between polls — accumulates one entry per down-transition,
/// not just whatever its point-in-time state happens to be). Read by
/// the dashboard's `agents_crashing` banner warning via
/// `recent_crash_counts`, which prunes entries older than its window.
recent_crashes: Mutex<HashMap<String, Vec<std::time::Instant>>>,
/// Agents with a graceful stop in progress. Set by the `GracefulStop`
/// orchestration; read by `socket_server::handle_recv`, which returns
/// `Response::GracefulStop` (instead of polling the broker) while an
@ -490,7 +482,6 @@ impl Coordinator {
model_prices,
agents: Mutex::new(HashMap::new()),
recent_transient: Mutex::new(HashMap::new()),
recent_crashes: Mutex::new(HashMap::new()),
graceful_stop_pending: Mutex::new(HashSet::new()),
pause_pending: Mutex::new(HashSet::new()),
last_stopped_running: Mutex::new(None),
@ -833,7 +824,7 @@ impl Coordinator {
/// watcher reads**, for the job-queue scheduler.
///
/// 🚨 The stamp is not bookkeeping. Without it the clear-then-poll race
/// produced a spurious `ContainerCrash` on **every** operator stop/restart:
/// produced a spurious crash warning on **every** operator stop/restart:
/// the transient is gone by the time the 10s poll looks, so a deliberate
/// stop is indistinguishable from a crash. `recent_transient_within` is what
/// closes that window, which is why the clear has to be an *event* — a
@ -955,33 +946,6 @@ impl Coordinator {
fold_tombstones_by_agent(map.iter().map(|((a, _), (d, _))| (a.as_str(), *d)))
}
/// Record an unexpected crash for `agent`. Called by the crash
/// watcher whenever it classifies a container stop as a crash (not an
/// operator action). Append-only here; pruning happens lazily on read
/// in `recent_crash_counts`.
pub fn record_crash(&self, agent: &str) {
self.recent_crashes
.lock()
.unwrap()
.entry(agent.to_owned())
.or_default()
.push(std::time::Instant::now());
}
/// Per-agent count of crashes within the last `window`. Lazily reaps
/// older timestamps and drops agents with none left, so the map stays
/// bounded and only lists agents actively crashing. Powers the
/// dashboard's `agents_crashing` banner warning.
pub fn recent_crash_counts(&self, window: std::time::Duration) -> HashMap<String, usize> {
let now = std::time::Instant::now();
let mut map = self.recent_crashes.lock().unwrap();
map.retain(|_, times| {
times.retain(|ts| now.duration_since(*ts) <= window);
!times.is_empty()
});
map.iter().map(|(k, v)| (k.clone(), v.len())).collect()
}
/// Every live transient, keyed by agent.
///
/// **Derived on read, stored nowhere.** Straight off the running graph, so
@ -1036,48 +1000,6 @@ impl Coordinator {
}
}
/// Push a `HelperEvent` into the manager's inbox. Encoded as JSON in
/// `Message::body`; sender = `SYSTEM_SENDER`. The manager harness
/// recognises the sender and parses the body. Best-effort: a serde or
/// broker error is logged but does not propagate.
pub fn notify_manager(&self, event: &hive_sh4re::manager::HelperEvent) {
self.notify_agent(hive_sh4re::manager::MANAGER_AGENT, event);
}
/// Push a `HelperEvent` into an arbitrary agent's inbox. Encoded
/// the same way as `notify_manager` (sender = `SYSTEM_SENDER`,
/// body = JSON-encoded event) — e.g. `ContainerCrash`, `NeedsUpdate`.
pub fn notify_agent(&self, agent: &str, event: &hive_sh4re::manager::HelperEvent) {
self.notify_agent_from(hive_sh4re::manager::SYSTEM_SENDER, agent, event);
}
/// Same as `notify_agent` but with an explicit sender. Use this
/// when the event originates from a known agent or the operator
/// rather than the system itself, so the recipient's terminal
/// shows the right name instead of `system`.
pub fn notify_agent_from(
&self,
from: &str,
agent: &str,
event: &hive_sh4re::manager::HelperEvent,
) {
let body = match serde_json::to_string(event) {
Ok(s) => s,
Err(e) => {
tracing::warn!(error = ?e, "failed to encode helper event");
return;
}
};
if let Err(e) = self.broker.send(&hive_sh4re::inbox::Message {
from: hive_sh4re::manager::trusted_sender(from),
to: agent.to_owned(),
body,
in_reply_to: None,
}) {
tracing::warn!(error = ?e, target = %agent, "failed to push helper event");
}
}
/// Deliver `body` to every currently-registered agent except the sender,
/// appending the standard broadcast hint. Returns a list of per-agent
/// error strings for any that failed (empty = all ok).

18
hive-c0re/src/dashboard/state_snapshot.rs
View file

@ -120,13 +120,6 @@ struct TransientView {
secs: u64,
}
/// Window over which container crashes count toward the `agents_crashing`
/// banner warning. Wide enough that a crash-looping container (restarted
/// by `Restart=on-failure` every few seconds) keeps the warning lit
/// between flaps, short enough that a single recovered crash clears within
/// minutes.
const CRASH_WARNING_WINDOW: std::time::Duration = std::time::Duration::from_mins(10);
/// Cold-load snapshot of the whole dashboard.
///
/// Includes the roster, tombstones, job queue, meta inputs, and more. Live
@ -195,15 +188,12 @@ pub(super) async fn api_state(
let tombstones = build_tombstone_views(&state.coord, &containers);
let port_conflicts = build_port_conflicts(&containers);
// Banner warnings: host probes (disk) + agent-state (pending logins,
// crashing agents). Built before the response struct because the
// agent-state producer borrows `containers`, which moves in below.
// Banner warnings: host probes (disk) + agent-state (pending logins).
// Built before the response struct because the agent-state producer
// borrows `containers`, which moves in below.
let server_warnings = {
let mut w = crate::host_stats::server_warnings();
w.extend(crate::host_stats::agent_state_warnings(
&containers,
&state.coord.recent_crash_counts(CRASH_WARNING_WINDOW),
));
w.extend(crate::host_stats::agent_state_warnings(&containers));
w
};

6
hive-c0re/src/main.rs
View file

@ -416,9 +416,9 @@ async fn cmd_serve(
// build_logs.sqlite vacuum: c0re-side (single db). Failures kept
// 30d, successes 24h — see `build_logs::vacuum` for the rule.
crate::build_logs::spawn_vacuum(&coord);
// Container crash watcher: emits HelperEvent::ContainerCrash
// when a previously-running container goes away without an
// operator-initiated transient state.
// Container crash watcher: warn!()s "container crash detected" when a
// previously-running container goes away without an operator-initiated
// transient state — a swarm-level grafana alert matches that line.
crash_watch::spawn(coord.clone());
// Agent-sockets marker poll: re-fires `agent_sockets::write`
// and `gateway_nginx::write` every 10s so the JSON and nginx

74
hive-c0re/src/stats/host_stats.rs
View file

@ -15,8 +15,6 @@
//! [`refresh_disk_warning`] drives a held guard from a periodic task (see
//! `spawn` in `main.rs`) instead of a `statvfs` per `/api/state`.
use std::collections::HashMap;
use serde::Serialize;
use utoipa::ToSchema;
@ -99,19 +97,20 @@ pub fn refresh_disk_warning(guard: &mut Option<crate::warnings::WarningGuard>) {
}
/// Agent-state warnings derived from the live container snapshot the
/// dashboard already holds: agents that need a claude login, and agents
/// that are crashing. Kept separate from [`server_warnings`] (host
/// probes) because the caller owns the container list + crash counts;
/// the dashboard concatenates both into one banner list.
/// dashboard already holds: agents that need a claude login. Kept
/// separate from [`server_warnings`] (host probes) because the caller
/// owns the container list; the dashboard concatenates both into one
/// banner list.
///
/// `crash_counts` is `Coordinator::recent_crash_counts(window)` — agent →
/// number of crashes inside that window — so a crash-looping agent shows
/// its repeat count rather than a single point-in-time flap.
/// Container crashes used to have a sibling `agents_crashing` entry here,
/// fed by `Coordinator::recent_crash_counts` — removed in favour of a
/// swarm-level grafana alert on the crash-watcher's own log line (see
/// `nix/host-modules/swarm-grafana.nix`'s `hh-container-crashed` rule and
/// `docs/scheduler/observability.md`). That ships the same signal to the
/// operator without needing an agent (or this per-hive banner) in the
/// loop at all.
#[must_use]
pub fn agent_state_warnings<S: std::hash::BuildHasher>(
containers: &[ContainerView],
crash_counts: &HashMap<String, usize, S>,
) -> Vec<ServerWarning> {
pub fn agent_state_warnings(containers: &[ContainerView]) -> Vec<ServerWarning> {
let mut out = Vec::new();
// `needs_login` is already running-gated in `container_view::build_all`,
@ -137,27 +136,6 @@ pub fn agent_state_warnings<S: std::hash::BuildHasher>(
});
}
if !crash_counts.is_empty() {
let mut crashing: Vec<(&String, usize)> =
crash_counts.iter().map(|(a, n)| (a, *n)).collect();
crashing.sort_unstable_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0)));
let list = crashing
.iter()
.map(|(a, n)| format!("{a} (×{n})"))
.collect::<Vec<_>>()
.join(", ");
out.push(ServerWarning {
kind: "agents_crashing",
level: "crit",
message: format!(
"{n} agent{plural} crashing: {list} \
— check the container journal (`hivectl logs <agent>`)",
n = crashing.len(),
plural = if crashing.len() == 1 { "" } else { "s" },
),
});
}
out
}
@ -264,13 +242,13 @@ mod tests {
#[test]
fn no_agent_warnings_when_all_healthy() {
let containers = [cv("alice", false), cv("bob", false)];
assert!(agent_state_warnings(&containers, &HashMap::new()).is_empty());
assert!(agent_state_warnings(&containers).is_empty());
}
#[test]
fn pending_logins_lists_sorted_agents() {
let containers = [cv("zoe", true), cv("amy", false), cv("bob", true)];
let w = agent_state_warnings(&containers, &HashMap::new());
let w = agent_state_warnings(&containers);
assert_eq!(w.len(), 1);
assert_eq!(w[0].kind, "pending_logins");
assert_eq!(w[0].level, "warn");
@ -284,29 +262,7 @@ mod tests {
#[test]
fn singular_grammar_for_one_agent() {
let containers = [cv("solo", true)];
let w = agent_state_warnings(&containers, &HashMap::new());
let w = agent_state_warnings(&containers);
assert!(w[0].message.starts_with("1 agent needs claude login: solo"));
}
#[test]
fn crashing_warning_is_crit_and_count_ordered() {
let crashes = HashMap::from([("flap".to_owned(), 5), ("blip".to_owned(), 1)]);
let w = agent_state_warnings(&[], &crashes);
assert_eq!(w.len(), 1);
assert_eq!(w[0].kind, "agents_crashing");
assert_eq!(w[0].level, "crit");
// higher crash count first
assert!(w[0].message.contains("flap (×5), blip (×1)"));
}
#[test]
fn both_warnings_coexist() {
let containers = [cv("a", true)];
let crashes = HashMap::from([("b".to_owned(), 2)]);
let kinds: Vec<&str> = agent_state_warnings(&containers, &crashes)
.iter()
.map(|w| w.kind)
.collect();
assert_eq!(kinds, ["pending_logins", "agents_crashing"]);
}
}

23
hive-c0re/src/workers/crash_watch.rs
View file

@ -1,7 +1,10 @@
//! Per-container crash and login-state watcher. Polls every managed
//! container on a 10s interval. Fires `ContainerCrash`, `LoggedIn`,
//! and `NeedsLogin` helper events. Event semantics and the
//! `RECENT_TRANSIENT_GRACE` window: `docs/agent-lifecycle/config-changes.md::Helper events`.
//! container on a 10s interval. A classified crash only `tracing::warn!`s
//! (`"container crash detected"`) — a swarm-level grafana alert matches
//! that line directly; see the module doc on [`emit_crash_transitions`].
//! Login-state transitions (`needs_login`/`logged_in`) publish onto
//! `swarm_notices` instead. The `RECENT_TRANSIENT_GRACE` window:
//! `docs/agent-lifecycle/config-changes.md::Helper events`.
use std::collections::HashSet;
use std::sync::Arc;
@ -143,12 +146,12 @@ fn emit_crash_transitions(coord: &Coordinator, prev: &HashSet<String>, current:
if is_deliberate_stop(active, recently_cleared) {
continue;
}
// The one place this fires: a swarm-level grafana alert
// (`hh-container-crashed` in `nix/host-modules/swarm-grafana.nix`)
// matches this exact message against the log store and pages the
// operator directly — no agent (and no per-hive dashboard banner)
// in the loop. See `docs/scheduler/observability.md`.
tracing::warn!(agent = %stopped, "container crash detected");
coord.record_crash(stopped);
coord.notify_manager(&hive_sh4re::manager::HelperEvent::ContainerCrash {
agent: stopped.clone(),
note: Some("container stopped without an operator action".into()),
});
}
}
@ -230,7 +233,7 @@ mod tests {
/// Control: a real drop from running to absent is a *readable*, empty
/// list — it must still be processed (and so still reach
/// `emit_crash_transitions`, which fires the crash event), unlike a
/// `emit_crash_transitions`, which warns the crash), unlike a
/// read failure.
#[test]
fn readable_empty_list_still_processes() {
@ -261,7 +264,7 @@ mod tests {
#[test]
fn not_deliberate_with_no_transient_at_all() {
// The real-crash case — fires the ContainerCrash event.
// The real-crash case — warns "container crash detected".
assert!(!is_deliberate_stop(None, None));
}