58c3cd853b
new hive_c0re::crash_watch task polls every 10s, builds the set of currently-running containers, and on running→stopped transitions checks the transient snapshot: if no Stopping / Restarting / Destroying / Rebuilding flag is set, the container exited unexpectedly and we fire HelperEvent::ContainerCrash into the manager's inbox so it can react (typically: start it again). first poll is a seeding pass — no events on harness startup. dbus subscription would be lower-latency but polling is honest and debuggable, and a 10s delay on crash detection is fine for our scale. manager prompt + approvals doc updated to advertise the new event variant. todo drops the entry (and the journald-viewer entry that already shipped).
72 lines
2.9 KiB
Rust
72 lines
2.9 KiB
Rust
//! Container crash watcher. Polls every managed container's running
|
|
//! state on a fixed interval; when a previously-running container is
|
|
//! suddenly stopped AND no operator-initiated transient (`Stopping`,
|
|
//! `Restarting`, `Destroying`) was set, fire `HelperEvent::ContainerCrash`
|
|
//! into the manager's inbox. The manager can then react — usually
|
|
//! a `start` or a config rebuild.
|
|
//!
|
|
//! D-Bus subscription would be lower-latency, but polling is far
|
|
//! simpler and the failure modes are honest (a crash discovered 10s
|
|
//! late is fine for our scale).
|
|
|
|
use std::collections::HashSet;
|
|
use std::sync::Arc;
|
|
use std::time::Duration;
|
|
|
|
use crate::coordinator::{Coordinator, TransientKind};
|
|
use crate::lifecycle::{self, AGENT_PREFIX, MANAGER_NAME};
|
|
|
|
const POLL_INTERVAL: Duration = Duration::from_secs(10);
|
|
|
|
pub fn spawn(coord: Arc<Coordinator>) {
|
|
tokio::spawn(async move {
|
|
// Seed the running-set from the first poll so we don't emit a
|
|
// crash for every agent on startup. First tick fills it; only
|
|
// running→stopped transitions across subsequent ticks count.
|
|
let mut prev_running: HashSet<String> = HashSet::new();
|
|
let mut seeded = false;
|
|
loop {
|
|
let raw = lifecycle::list().await.unwrap_or_default();
|
|
let mut current_running = HashSet::new();
|
|
for c in &raw {
|
|
let logical = if c == MANAGER_NAME {
|
|
MANAGER_NAME.to_owned()
|
|
} else if let Some(n) = c.strip_prefix(AGENT_PREFIX) {
|
|
n.to_owned()
|
|
} else {
|
|
continue;
|
|
};
|
|
if lifecycle::is_running(&logical).await {
|
|
current_running.insert(logical);
|
|
}
|
|
}
|
|
|
|
if seeded {
|
|
let transients = coord.transient_snapshot();
|
|
for stopped in prev_running.difference(¤t_running) {
|
|
let deliberate = transients.get(stopped).is_some_and(|st| {
|
|
matches!(
|
|
st.kind,
|
|
TransientKind::Stopping
|
|
| TransientKind::Restarting
|
|
| TransientKind::Destroying
|
|
| TransientKind::Rebuilding
|
|
)
|
|
});
|
|
if deliberate {
|
|
continue;
|
|
}
|
|
tracing::warn!(agent = %stopped, "container crash detected");
|
|
coord.notify_manager(&hive_sh4re::HelperEvent::ContainerCrash {
|
|
agent: stopped.clone(),
|
|
note: Some("container stopped without an operator action".into()),
|
|
});
|
|
}
|
|
}
|
|
prev_running = current_running;
|
|
seeded = true;
|
|
|
|
tokio::time::sleep(POLL_INTERVAL).await;
|
|
}
|
|
});
|
|
}
|