vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

hive-c0re: replace the agents_crashing banner with a crash-watcher log line + grafana alert

Browse source
This commit is contained in:
damocles 2026-10-12 22:21:04 +02:00 • committed by mara
commit 02831fb5e9
13 changed files with 214 additions and 224 deletions
Download patch file Download diff file Expand all files Collapse all files

80
hive-c0re/src/coordinator.rs
View file

@ -128,14 +128,6 @@ pub struct Coordinator {
/// `StopForUpdate` (`true`) leaves the tombstone reading `false`, and the
/// crash watcher then reports an intentional stop as a **crash**.
recent_transient: Mutex<HashMap<(String, String), (bool, std::time::Instant)>>,
/// Timestamps of recent unexpected container crashes, keyed by agent.
/// Fed by `crash_watch` each time it classifies a stop as a crash (so
/// a crash-looping container — which `Restart=on-failure` flips back
/// to running between polls — accumulates one entry per down-transition,
/// not just whatever its point-in-time state happens to be). Read by
/// the dashboard's `agents_crashing` banner warning via
/// `recent_crash_counts`, which prunes entries older than its window.
recent_crashes: Mutex<HashMap<String, Vec<std::time::Instant>>>,
/// Agents with a graceful stop in progress. Set by the `GracefulStop`
/// orchestration; read by `socket_server::handle_recv`, which returns
/// `Response::GracefulStop` (instead of polling the broker) while an
@ -490,7 +482,6 @@ impl Coordinator {
model_prices,
agents: Mutex::new(HashMap::new()),
recent_transient: Mutex::new(HashMap::new()),
recent_crashes: Mutex::new(HashMap::new()),
graceful_stop_pending: Mutex::new(HashSet::new()),
pause_pending: Mutex::new(HashSet::new()),
last_stopped_running: Mutex::new(None),
@ -833,7 +824,7 @@ impl Coordinator {
/// watcher reads**, for the job-queue scheduler.
///
/// 🚨 The stamp is not bookkeeping. Without it the clear-then-poll race
/// produced a spurious `ContainerCrash` on **every** operator stop/restart:
/// produced a spurious crash warning on **every** operator stop/restart:
/// the transient is gone by the time the 10s poll looks, so a deliberate
/// stop is indistinguishable from a crash. `recent_transient_within` is what
/// closes that window, which is why the clear has to be an *event* — a
@ -955,33 +946,6 @@ impl Coordinator {
fold_tombstones_by_agent(map.iter().map(|((a, _), (d, _))| (a.as_str(), *d)))
}
/// Record an unexpected crash for `agent`. Called by the crash
/// watcher whenever it classifies a container stop as a crash (not an
/// operator action). Append-only here; pruning happens lazily on read
/// in `recent_crash_counts`.
pub fn record_crash(&self, agent: &str) {
self.recent_crashes
.lock()
.unwrap()
.entry(agent.to_owned())
.or_default()
.push(std::time::Instant::now());
}
/// Per-agent count of crashes within the last `window`. Lazily reaps
/// older timestamps and drops agents with none left, so the map stays
/// bounded and only lists agents actively crashing. Powers the
/// dashboard's `agents_crashing` banner warning.
pub fn recent_crash_counts(&self, window: std::time::Duration) -> HashMap<String, usize> {
let now = std::time::Instant::now();
let mut map = self.recent_crashes.lock().unwrap();
map.retain(|_, times| {
times.retain(|ts| now.duration_since(*ts) <= window);
!times.is_empty()
});
map.iter().map(|(k, v)| (k.clone(), v.len())).collect()
}
/// Every live transient, keyed by agent.
///
/// **Derived on read, stored nowhere.** Straight off the running graph, so
@ -1036,48 +1000,6 @@ impl Coordinator {
}
}
/// Push a `HelperEvent` into the manager's inbox. Encoded as JSON in
/// `Message::body`; sender = `SYSTEM_SENDER`. The manager harness
/// recognises the sender and parses the body. Best-effort: a serde or
/// broker error is logged but does not propagate.
pub fn notify_manager(&self, event: &hive_sh4re::manager::HelperEvent) {
self.notify_agent(hive_sh4re::manager::MANAGER_AGENT, event);
}
/// Push a `HelperEvent` into an arbitrary agent's inbox. Encoded
/// the same way as `notify_manager` (sender = `SYSTEM_SENDER`,
/// body = JSON-encoded event) — e.g. `ContainerCrash`, `NeedsUpdate`.
pub fn notify_agent(&self, agent: &str, event: &hive_sh4re::manager::HelperEvent) {
self.notify_agent_from(hive_sh4re::manager::SYSTEM_SENDER, agent, event);
}
/// Same as `notify_agent` but with an explicit sender. Use this
/// when the event originates from a known agent or the operator
/// rather than the system itself, so the recipient's terminal
/// shows the right name instead of `system`.
pub fn notify_agent_from(
&self,
from: &str,
agent: &str,
event: &hive_sh4re::manager::HelperEvent,
) {
let body = match serde_json::to_string(event) {
Ok(s) => s,
Err(e) => {
tracing::warn!(error = ?e, "failed to encode helper event");
return;
}
};
if let Err(e) = self.broker.send(&hive_sh4re::inbox::Message {
from: hive_sh4re::manager::trusted_sender(from),
to: agent.to_owned(),
body,
in_reply_to: None,
}) {
tracing::warn!(error = ?e, target = %agent, "failed to push helper event");
}
}
/// Deliver `body` to every currently-registered agent except the sender,
/// appending the standard broadcast hint. Returns a list of per-agent
/// error strings for any that failed (empty = all ok).