reconcile: start a stopped harness inside a running container
hive-priv gains AgentHarness { agent_name, action: probe | start }, which
runs systemctl -M h-<name> against the fixed unit hive-agent.service.
run_reconcile and the 1-min tick read the unit for a running wanted-up
agent and start it when it is inactive or failed. A hive-priv without the
op reads as unknown state and leaves reconcile container-only.
This commit is contained in:
parent
ca734f8ea0
commit
36515c289d
2 changed files with 517 additions and 32 deletions
|
|
@ -604,6 +604,80 @@ pub async fn running_state(name: &str) -> Option<bool> {
|
|||
.map(|out| out.status.success())
|
||||
}
|
||||
|
||||
/// Whether an agent's harness unit (`hive-agent.service`) is up inside its
|
||||
/// running container.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HarnessState {
|
||||
/// Active, or on its way: activating, deactivating or reloading.
|
||||
Present,
|
||||
/// Loaded and `inactive` or `failed`: nothing is running it and nothing
|
||||
/// is about to.
|
||||
Stopped,
|
||||
/// Not readable: hive-priv is older than the op, the container's systemd
|
||||
/// did not answer, or the unit is not loaded.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// Read the harness unit's state out of `AgentHarness` probe output
|
||||
/// (`LoadState=` and `ActiveState=` lines).
|
||||
fn parse_harness_state(out: &str) -> HarnessState {
|
||||
let field = |key: &str| {
|
||||
out.lines()
|
||||
.find_map(|l| l.strip_prefix(key).and_then(|v| v.strip_prefix('=')))
|
||||
.map(str::trim)
|
||||
};
|
||||
match (field("LoadState"), field("ActiveState")) {
|
||||
(Some("loaded"), Some("inactive" | "failed")) => HarnessState::Stopped,
|
||||
(Some("loaded"), Some("active" | "activating" | "deactivating" | "reloading")) => {
|
||||
HarnessState::Present
|
||||
}
|
||||
_ => HarnessState::Unknown,
|
||||
}
|
||||
}
|
||||
|
||||
/// The state of `name`'s harness unit, asked of hive-priv. Bounded by
|
||||
/// [`IS_ACTIVE_TIMEOUT`], because the tick calls it per agent inline.
|
||||
///
|
||||
/// Any failure reads as [`HarnessState::Unknown`], so reconcile keeps its
|
||||
/// container-only behaviour: a hive-priv that predates the op answers an
|
||||
/// error, and that is logged at `warn` the first time and `debug` after.
|
||||
pub async fn harness_state(name: &str) -> HarnessState {
|
||||
let probe = crate::priv_client::agent_harness(name, hive_priv_sock::HarnessAction::Probe);
|
||||
if let Ok(answer) = tokio::time::timeout(IS_ACTIVE_TIMEOUT, probe).await {
|
||||
harness_of_answer(name, answer)
|
||||
} else {
|
||||
tracing::debug!(%name, "reading the agent's harness state timed out");
|
||||
HarnessState::Unknown
|
||||
}
|
||||
}
|
||||
|
||||
/// The state a probe answer means; an error is [`HarnessState::Unknown`].
|
||||
fn harness_of_answer(name: &str, answer: Result<String>) -> HarnessState {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
static WARNED: AtomicBool = AtomicBool::new(false);
|
||||
match answer {
|
||||
Ok(out) => parse_harness_state(&out),
|
||||
Err(e) if !WARNED.swap(true, Ordering::Relaxed) => {
|
||||
tracing::warn!(%name, error = ?e, "cannot read the agent's harness state; reconcile only watches the container (logged once)");
|
||||
HarnessState::Unknown
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(%name, error = ?e, "cannot read the agent's harness state");
|
||||
HarnessState::Unknown
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Queue a start of `name`'s harness unit inside its running container.
|
||||
///
|
||||
/// # Errors
|
||||
/// hive-priv is unreachable, refuses, or `systemctl` fails.
|
||||
pub async fn start_harness(name: &str) -> Result<()> {
|
||||
crate::priv_client::agent_harness(name, hive_priv_sock::HarnessAction::Start)
|
||||
.await
|
||||
.map(drop)
|
||||
}
|
||||
|
||||
/// How long one `systemctl is-active` may run before the unit's state counts
|
||||
/// as unreadable. Periodic workers make this call per agent inline.
|
||||
const IS_ACTIVE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
|
|
|
|||
|
|
@ -445,3 +445,45 @@ fn stop_kind_survives_a_clean_stop_between_failures() {
|
|||
guards.remove("stopkind-c");
|
||||
assert!(std::ptr::eq(first, failed(&mut guards)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn harness_state_reads_load_and_active_state() {
|
||||
let probe = |load: &str, active: &str| {
|
||||
parse_harness_state(&format!("LoadState={load}\nActiveState={active}\n"))
|
||||
};
|
||||
for active in ["inactive", "failed"] {
|
||||
assert_eq!(probe("loaded", active), HarnessState::Stopped, "{active}");
|
||||
}
|
||||
for active in ["active", "activating", "deactivating", "reloading"] {
|
||||
assert_eq!(probe("loaded", active), HarnessState::Present, "{active}");
|
||||
}
|
||||
// A unit that is not loaded, or output that is not a probe, is not
|
||||
// evidence the harness is down.
|
||||
assert_eq!(probe("not-found", "inactive"), HarnessState::Unknown);
|
||||
assert_eq!(parse_harness_state(""), HarnessState::Unknown);
|
||||
assert_eq!(parse_harness_state("garbage"), HarnessState::Unknown);
|
||||
}
|
||||
|
||||
/// A hive-priv that predates the op answers an error; that reads as unknown
|
||||
/// state, which leaves reconcile with its container-only behaviour.
|
||||
#[test]
|
||||
fn an_unreadable_harness_is_unknown_never_stopped() {
|
||||
let old_priv = anyhow::anyhow!("parse request: unknown variant `agent_harness`");
|
||||
assert_eq!(
|
||||
harness_of_answer("alice", Err(old_priv)),
|
||||
HarnessState::Unknown
|
||||
);
|
||||
assert_eq!(
|
||||
harness_of_answer("alice", Err(anyhow::anyhow!("again"))),
|
||||
HarnessState::Unknown,
|
||||
"later errors read the same"
|
||||
);
|
||||
assert_eq!(
|
||||
harness_of_answer(
|
||||
"alice",
|
||||
Ok("LoadState=loaded\nActiveState=inactive\n".to_owned())
|
||||
),
|
||||
HarnessState::Stopped,
|
||||
"control: a real answer is read"
|
||||
);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue