hive-c0re: keep is_running unbounded, bound only the periodic status reads
Bounding is_running turned a hung systemctl into "not running" for callers that record or act on the answer: the broad-stop restore set dropped the agent, start_many could rebuild-and-restart a running agent, reconcile could seed Offline. is_running is back to the unbounded call it was. The periodic workers read through the bounded running_state instead: crash_watch (unchanged), and the swarm_agent_status publisher via the new container_view::read_agent_status_bounded, which skips the agent for that sweep when the unit state is unreadable. The GetAgentMeta socket handler keeps the unbounded read_agent_status_live.
This commit is contained in:
parent
3cb7be29cf
commit
7214b4c22f
3 changed files with 44 additions and 14 deletions
|
|
@ -312,8 +312,8 @@ fn read_harness_flags(name: &hive_types::Ident) -> (bool, bool, bool) {
|
|||
///
|
||||
/// NB: a caller building `AgentMeta` for a *stopped* container must clear
|
||||
/// the result — the on-disk status is a stale snapshot from before the
|
||||
/// stop. [`read_agent_status_live`] applies that rule, and is what the
|
||||
/// socket server and the swarm status reader actually call.
|
||||
/// stop. [`read_agent_status_live`] and [`read_agent_status_bounded`] apply
|
||||
/// that rule, for the socket server and the swarm status reader respectively.
|
||||
fn read_agent_status(name: &hive_types::Ident) -> (Option<String>, Option<i64>) {
|
||||
let path = Coordinator::agent_notes_dir(name).join("hyperhive-status");
|
||||
let meta = std::fs::metadata(&path).ok();
|
||||
|
|
@ -357,7 +357,24 @@ fn read_agent_status(name: &hive_types::Ident) -> (Option<String>, Option<i64>)
|
|||
pub async fn read_agent_status_live(
|
||||
name: &hive_types::Ident,
|
||||
) -> (Option<String>, Option<i64>, bool) {
|
||||
if !lifecycle::is_running(name.as_str()).await {
|
||||
status_if_running(name, lifecycle::is_running(name.as_str()).await)
|
||||
}
|
||||
|
||||
/// [`read_agent_status_live`] on the bounded [`lifecycle::running_state`],
|
||||
/// for periodic workers; `None` when the unit state is unreadable.
|
||||
pub async fn read_agent_status_bounded(
|
||||
name: &hive_types::Ident,
|
||||
) -> Option<(Option<String>, Option<i64>, bool)> {
|
||||
lifecycle::running_state(name.as_str())
|
||||
.await
|
||||
.map(|running| status_if_running(name, running))
|
||||
}
|
||||
|
||||
fn status_if_running(
|
||||
name: &hive_types::Ident,
|
||||
running: bool,
|
||||
) -> (Option<String>, Option<i64>, bool) {
|
||||
if !running {
|
||||
return (None, None, false);
|
||||
}
|
||||
let (text, set_at) = read_agent_status(name);
|
||||
|
|
|
|||
|
|
@ -627,19 +627,28 @@ pub async fn unit_state(name: &str) -> UnitState {
|
|||
}
|
||||
|
||||
/// True when the container's systemd unit is active. Used by the dashboard
|
||||
/// to gate stop/restart buttons. A state that could not be read (see
|
||||
/// [`running_state`]) is `false`.
|
||||
/// to gate stop/restart buttons.
|
||||
///
|
||||
/// Kept boolean on purpose: nearly every caller is reconcile/power logic
|
||||
/// asking "is it up? if not, start it", and that question has two answers.
|
||||
/// A caller that needs to tell *failed* from *stopped* wants
|
||||
/// [`unit_state`] instead.
|
||||
///
|
||||
/// Unbounded: its callers record or act on the answer, so a hung `systemctl`
|
||||
/// must not read as "not running". Periodic workers use [`running_state`].
|
||||
pub async fn is_running(name: &str) -> bool {
|
||||
running_state(name).await == Some(true)
|
||||
let container = container_name(name);
|
||||
let unit = format!("container@{container}.service");
|
||||
Command::new("systemctl")
|
||||
.args(["is-active", "--quiet", &unit])
|
||||
.status()
|
||||
.await
|
||||
.is_ok_and(|s| s.success())
|
||||
}
|
||||
|
||||
/// Whether the container's systemd unit is active; `None` when `systemctl`
|
||||
/// could not be run or did not answer within [`IS_ACTIVE_TIMEOUT`].
|
||||
/// Whether the container's systemd unit is active, for periodic workers that
|
||||
/// must not stall on a hung `systemctl`; `None` when it could not be run or
|
||||
/// did not answer within [`IS_ACTIVE_TIMEOUT`].
|
||||
pub async fn running_state(name: &str) -> Option<bool> {
|
||||
let container = container_name(name);
|
||||
let unit = format!("container@{container}.service");
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@
|
|||
//! `swarm_queue_client::agent_status`'s module doc for why hive status and
|
||||
//! agent status live in separate buckets. Same direction, though: a hive
|
||||
//! **offers** what it already knows about its own agents (via
|
||||
//! [`crate::container_view::read_agent_status_live`], the same read
|
||||
//! `GetAgentMeta` serves to the dashboard), and the swarm controller never
|
||||
//! reaches down to collect it.
|
||||
//! [`crate::container_view::read_agent_status_bounded`], the bounded form of
|
||||
//! the read `GetAgentMeta` serves to the dashboard), and the swarm
|
||||
//! controller never reaches down to collect it.
|
||||
//!
|
||||
//! **One publish per agent, not one document for the hive.** Each agent
|
||||
//! gets its own key (`swarm_queue_client::agent_status::key`), so one
|
||||
|
|
@ -39,7 +39,7 @@ const FAILURES_BEFORE_BANNER: u32 = 3;
|
|||
///
|
||||
/// Takes no `Coordinator` handle, unlike its sibling: this task only reads
|
||||
/// agent state that already exists on disk / in the container runtime
|
||||
/// (`lifecycle::agents_for_meta_listing`, `container_view::read_agent_status_live`),
|
||||
/// (`lifecycle::agents_for_meta_listing`, `container_view::read_agent_status_bounded`),
|
||||
/// it never touches the job queue or broker the way `swarm_status::spawn`'s
|
||||
/// deploy-event listener does.
|
||||
pub fn spawn(mut shutdown: tokio::sync::watch::Receiver<bool>) {
|
||||
|
|
@ -121,8 +121,12 @@ async fn publish_all(client: &async_nats::Client, hive: &str) -> Result<usize> {
|
|||
tracing::warn!(agent = %spec.name, "agent status: name failed Ident::parse, skipping");
|
||||
continue;
|
||||
};
|
||||
let (status_text, status_set_at, running) =
|
||||
crate::container_view::read_agent_status_live(&ident).await;
|
||||
let Some((status_text, status_set_at, running)) =
|
||||
crate::container_view::read_agent_status_bounded(&ident).await
|
||||
else {
|
||||
tracing::warn!(agent = %spec.name, "agent status: unit state unreadable, skipping");
|
||||
continue;
|
||||
};
|
||||
let payload = swarm_queue_client::agent_status::AgentStatus {
|
||||
status_text,
|
||||
status_set_at,
|
||||
|
|
|
|||
Loading…
Reference in a new issue