diff --git a/hive-c0re/src/container_view.rs b/hive-c0re/src/container_view.rs index 53ba20d9..89f64e29 100644 --- a/hive-c0re/src/container_view.rs +++ b/hive-c0re/src/container_view.rs @@ -312,8 +312,8 @@ fn read_harness_flags(name: &hive_types::Ident) -> (bool, bool, bool) { /// /// NB: a caller building `AgentMeta` for a *stopped* container must clear /// the result — the on-disk status is a stale snapshot from before the -/// stop. [`read_agent_status_live`] applies that rule, and is what the -/// socket server and the swarm status reader actually call. +/// stop. [`read_agent_status_live`] and [`read_agent_status_bounded`] apply +/// that rule, for the socket server and the swarm status reader respectively. fn read_agent_status(name: &hive_types::Ident) -> (Option, Option) { let path = Coordinator::agent_notes_dir(name).join("hyperhive-status"); let meta = std::fs::metadata(&path).ok(); @@ -357,7 +357,24 @@ fn read_agent_status(name: &hive_types::Ident) -> (Option, Option) pub async fn read_agent_status_live( name: &hive_types::Ident, ) -> (Option, Option, bool) { - if !lifecycle::is_running(name.as_str()).await { + status_if_running(name, lifecycle::is_running(name.as_str()).await) +} + +/// [`read_agent_status_live`] on the bounded [`lifecycle::running_state`], +/// for periodic workers; `None` when the unit state is unreadable. +pub async fn read_agent_status_bounded( + name: &hive_types::Ident, +) -> Option<(Option, Option, bool)> { + lifecycle::running_state(name.as_str()) + .await + .map(|running| status_if_running(name, running)) +} + +fn status_if_running( + name: &hive_types::Ident, + running: bool, +) -> (Option, Option, bool) { + if !running { return (None, None, false); } let (text, set_at) = read_agent_status(name); diff --git a/hive-c0re/src/lifecycle/mod.rs b/hive-c0re/src/lifecycle/mod.rs index ca78d8fc..4328b8fa 100644 --- a/hive-c0re/src/lifecycle/mod.rs +++ b/hive-c0re/src/lifecycle/mod.rs @@ -627,19 +627,28 @@ pub async fn unit_state(name: &str) -> UnitState { } /// True when the container's systemd unit is active. Used by the dashboard -/// to gate stop/restart buttons. A state that could not be read (see -/// [`running_state`]) is `false`. +/// to gate stop/restart buttons. /// /// Kept boolean on purpose: nearly every caller is reconcile/power logic /// asking "is it up? if not, start it", and that question has two answers. /// A caller that needs to tell *failed* from *stopped* wants /// [`unit_state`] instead. +/// +/// Unbounded: its callers record or act on the answer, so a hung `systemctl` +/// must not read as "not running". Periodic workers use [`running_state`]. pub async fn is_running(name: &str) -> bool { - running_state(name).await == Some(true) + let container = container_name(name); + let unit = format!("container@{container}.service"); + Command::new("systemctl") + .args(["is-active", "--quiet", &unit]) + .status() + .await + .is_ok_and(|s| s.success()) } -/// Whether the container's systemd unit is active; `None` when `systemctl` -/// could not be run or did not answer within [`IS_ACTIVE_TIMEOUT`]. +/// Whether the container's systemd unit is active, for periodic workers that +/// must not stall on a hung `systemctl`; `None` when it could not be run or +/// did not answer within [`IS_ACTIVE_TIMEOUT`]. pub async fn running_state(name: &str) -> Option { let container = container_name(name); let unit = format!("container@{container}.service"); diff --git a/hive-c0re/src/swarm_agent_status.rs b/hive-c0re/src/swarm_agent_status.rs index 54dd1777..7ba8f140 100644 --- a/hive-c0re/src/swarm_agent_status.rs +++ b/hive-c0re/src/swarm_agent_status.rs @@ -4,9 +4,9 @@ //! `swarm_queue_client::agent_status`'s module doc for why hive status and //! agent status live in separate buckets. Same direction, though: a hive //! **offers** what it already knows about its own agents (via -//! [`crate::container_view::read_agent_status_live`], the same read -//! `GetAgentMeta` serves to the dashboard), and the swarm controller never -//! reaches down to collect it. +//! [`crate::container_view::read_agent_status_bounded`], the bounded form of +//! the read `GetAgentMeta` serves to the dashboard), and the swarm +//! controller never reaches down to collect it. //! //! **One publish per agent, not one document for the hive.** Each agent //! gets its own key (`swarm_queue_client::agent_status::key`), so one @@ -39,7 +39,7 @@ const FAILURES_BEFORE_BANNER: u32 = 3; /// /// Takes no `Coordinator` handle, unlike its sibling: this task only reads /// agent state that already exists on disk / in the container runtime -/// (`lifecycle::agents_for_meta_listing`, `container_view::read_agent_status_live`), +/// (`lifecycle::agents_for_meta_listing`, `container_view::read_agent_status_bounded`), /// it never touches the job queue or broker the way `swarm_status::spawn`'s /// deploy-event listener does. pub fn spawn(mut shutdown: tokio::sync::watch::Receiver) { @@ -121,8 +121,12 @@ async fn publish_all(client: &async_nats::Client, hive: &str) -> Result { tracing::warn!(agent = %spec.name, "agent status: name failed Ident::parse, skipping"); continue; }; - let (status_text, status_set_at, running) = - crate::container_view::read_agent_status_live(&ident).await; + let Some((status_text, status_set_at, running)) = + crate::container_view::read_agent_status_bounded(&ident).await + else { + tracing::warn!(agent = %spec.name, "agent status: unit state unreadable, skipping"); + continue; + }; let payload = swarm_queue_client::agent_status::AgentStatus { status_text, status_set_at,