reconcile: start a stopped harness inside a running container
hive-priv gains AgentHarness { agent_name, action: probe | start }, which
runs systemctl -M h-<name> against the fixed unit hive-agent.service.
run_reconcile and the 1-min tick read the unit for a running wanted-up
agent and start it when it is inactive or failed. A hive-priv without the
op reads as unknown state and leaves reconcile container-only.
This commit is contained in:
parent
ca734f8ea0
commit
36515c289d
1 changed files with 517 additions and 32 deletions
|
|
@ -604,6 +604,80 @@ pub async fn running_state(name: &str) -> Option<bool> {
|
|||
.map(|out| out.status.success())
|
||||
}
|
||||
|
||||
/// Whether an agent's harness unit (`hive-agent.service`) is up inside its
|
||||
/// running container.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HarnessState {
|
||||
/// Active, or on its way: activating, deactivating or reloading.
|
||||
Present,
|
||||
/// Loaded and `inactive` or `failed`: nothing is running it and nothing
|
||||
/// is about to.
|
||||
Stopped,
|
||||
/// Not readable: hive-priv is older than the op, the container's systemd
|
||||
/// did not answer, or the unit is not loaded.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// Read the harness unit's state out of `AgentHarness` probe output
|
||||
/// (`LoadState=` and `ActiveState=` lines).
|
||||
fn parse_harness_state(out: &str) -> HarnessState {
|
||||
let field = |key: &str| {
|
||||
out.lines()
|
||||
.find_map(|l| l.strip_prefix(key).and_then(|v| v.strip_prefix('=')))
|
||||
.map(str::trim)
|
||||
};
|
||||
match (field("LoadState"), field("ActiveState")) {
|
||||
(Some("loaded"), Some("inactive" | "failed")) => HarnessState::Stopped,
|
||||
(Some("loaded"), Some("active" | "activating" | "deactivating" | "reloading")) => {
|
||||
HarnessState::Present
|
||||
}
|
||||
_ => HarnessState::Unknown,
|
||||
}
|
||||
}
|
||||
|
||||
/// The state of `name`'s harness unit, asked of hive-priv. Bounded by
|
||||
/// [`IS_ACTIVE_TIMEOUT`], because the tick calls it per agent inline.
|
||||
///
|
||||
/// Any failure reads as [`HarnessState::Unknown`], so reconcile keeps its
|
||||
/// container-only behaviour: a hive-priv that predates the op answers an
|
||||
/// error, and that is logged at `warn` the first time and `debug` after.
|
||||
pub async fn harness_state(name: &str) -> HarnessState {
|
||||
let probe = crate::priv_client::agent_harness(name, hive_priv_sock::HarnessAction::Probe);
|
||||
if let Ok(answer) = tokio::time::timeout(IS_ACTIVE_TIMEOUT, probe).await {
|
||||
harness_of_answer(name, answer)
|
||||
} else {
|
||||
tracing::debug!(%name, "reading the agent's harness state timed out");
|
||||
HarnessState::Unknown
|
||||
}
|
||||
}
|
||||
|
||||
/// The state a probe answer means; an error is [`HarnessState::Unknown`].
|
||||
fn harness_of_answer(name: &str, answer: Result<String>) -> HarnessState {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
static WARNED: AtomicBool = AtomicBool::new(false);
|
||||
match answer {
|
||||
Ok(out) => parse_harness_state(&out),
|
||||
Err(e) if !WARNED.swap(true, Ordering::Relaxed) => {
|
||||
tracing::warn!(%name, error = ?e, "cannot read the agent's harness state; reconcile only watches the container (logged once)");
|
||||
HarnessState::Unknown
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(%name, error = ?e, "cannot read the agent's harness state");
|
||||
HarnessState::Unknown
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Queue a start of `name`'s harness unit inside its running container.
|
||||
///
|
||||
/// # Errors
|
||||
/// hive-priv is unreachable, refuses, or `systemctl` fails.
|
||||
pub async fn start_harness(name: &str) -> Result<()> {
|
||||
crate::priv_client::agent_harness(name, hive_priv_sock::HarnessAction::Start)
|
||||
.await
|
||||
.map(drop)
|
||||
}
|
||||
|
||||
/// How long one `systemctl is-active` may run before the unit's state counts
|
||||
/// as unreadable. Periodic workers make this call per agent inline.
|
||||
const IS_ACTIVE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
|
|
|
|||
Loading…
Reference in a new issue