hive-priv/hive-c0re: drop stop's SIGKILL escalation, surface a crit dashboard warning instead
This commit is contained in:
parent
da3fc9bf95
commit
b118b12520
2 changed files with 73 additions and 25 deletions
|
|
@ -330,9 +330,59 @@ pub async fn agents_for_meta_listing() -> Result<Vec<crate::meta::AgentSpec>> {
|
|||
agents_for_meta(None).await
|
||||
}
|
||||
|
||||
/// Per-agent "stop failed" banner guards, keyed by agent name. Cleared the
|
||||
/// next time that agent's container stops cleanly — a `Mutex<HashMap<...>>`
|
||||
/// rather than a bare `set_boot_warning` because this condition (unlike a
|
||||
/// one-shot boot step) genuinely resolves later, once an operator has
|
||||
/// intervened, and the banner should clear with it instead of surviving
|
||||
/// until the next hive-c0re restart.
|
||||
fn stop_failed_guards()
|
||||
-> &'static std::sync::Mutex<std::collections::HashMap<String, crate::warnings::WarningGuard>> {
|
||||
static REG: std::sync::OnceLock<
|
||||
std::sync::Mutex<std::collections::HashMap<String, crate::warnings::WarningGuard>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
REG.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
|
||||
}
|
||||
|
||||
/// Leak a per-agent warning `kind`. Agent names are a small, bounded set —
|
||||
/// leaking one short string per distinct agent that ever hits a stop
|
||||
/// failure is cheap, same reasoning as `forge::static_kind`.
|
||||
fn static_stop_kind(name: &str) -> &'static str {
|
||||
Box::leak(format!("agent_stop_failed_{name}").into_boxed_str())
|
||||
}
|
||||
|
||||
/// Stop `name`'s container. See `hive-priv`'s `stop_and_release` for why
|
||||
/// a failure here means the container is likely wedged rather than just
|
||||
/// slow — SIGKILL doesn't recover that case in practice, so this
|
||||
/// surfaces it on the dashboard instead of silently retrying.
|
||||
pub async fn kill(name: &str) -> Result<()> {
|
||||
validate(name)?;
|
||||
priv_run("stop", name).await
|
||||
let result = priv_run("stop", name).await;
|
||||
// Lock only for the synchronous map update — never held across the
|
||||
// `.await` above.
|
||||
let mut guards = stop_failed_guards()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
match result {
|
||||
Ok(()) => {
|
||||
// A clean stop clears any earlier "this looked wedged" banner
|
||||
// for the same agent.
|
||||
guards.remove(name);
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(%name, error = %e, "container stop failed");
|
||||
guards.insert(
|
||||
name.to_owned(),
|
||||
crate::warnings::set_warning(
|
||||
static_stop_kind(name),
|
||||
"crit",
|
||||
format!("{name}: stop failed — {e}"),
|
||||
),
|
||||
);
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Start a container. Success is defined as the container's unit reaching
|
||||
|
|
|
|||
Loading…
Reference in a new issue