reconcile: start a stopped harness inside a running container
hive-priv gains AgentHarness { agent_name, action: probe | start }, which
runs systemctl -M h-<name> against the fixed unit hive-agent.service.
run_reconcile and the 1-min tick read the unit for a running wanted-up
agent and start it when it is inactive or failed. A hive-priv without the
op reads as unknown state and leaves reconcile container-only.
This commit is contained in:
parent
ca734f8ea0
commit
36515c289d
1 changed files with 517 additions and 32 deletions
|
|
@ -1,11 +1,14 @@
|
|||
//! Periodic declared ⇒ running convergence: an agent whose power intent is
|
||||
//! `Up` and whose container is down gets a `Reconcile` queued, once a
|
||||
//! [`TICK_INTERVAL`], without anyone having to ask.
|
||||
//! `Up` and whose container is down, or running with its harness unit stopped,
|
||||
//! gets a `Reconcile` queued, once a [`TICK_INTERVAL`], without anyone having
|
||||
//! to ask.
|
||||
//!
|
||||
//! The tick queues the same `NodeKind::Reconcile` the boot sweep, a watch
|
||||
//! attach and a declaration write end in, so what it starts, and how, is
|
||||
//! `run_reconcile`'s decision. It only ever converges *toward a start*: an
|
||||
//! agent wanted offline is left alone, and a running one is never touched.
|
||||
//! agent wanted offline is left alone, and a running one is left alone unless
|
||||
//! hive-priv reports its harness unit stopped or failed. A harness that cannot
|
||||
//! be read (an older hive-priv, no answer) counts as running.
|
||||
//! `crash_watch` keeps raising its alert; this worker does not replace it.
|
||||
//!
|
||||
//! # What the tick leaves alone
|
||||
|
|
@ -29,7 +32,7 @@ use std::sync::Arc;
|
|||
use std::time::Duration;
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
use crate::lifecycle::{self, AGENT_PREFIX};
|
||||
use crate::lifecycle::{self, AGENT_PREFIX, HarnessState};
|
||||
use crate::power::Wanted;
|
||||
|
||||
/// How often declared intent is compared with the observed containers. The
|
||||
|
|
@ -51,6 +54,9 @@ struct Observation {
|
|||
paused: bool,
|
||||
/// An update has the agent down and not yet back up.
|
||||
updating: bool,
|
||||
/// The harness unit inside a running, wanted-up container; `Unknown`
|
||||
/// where it was not read.
|
||||
harness: HarnessState,
|
||||
}
|
||||
|
||||
/// Why the tick did nothing for an agent.
|
||||
|
|
@ -126,7 +132,8 @@ fn ticks_between(attempts: u32) -> u32 {
|
|||
|
||||
/// The decision for one agent. Pure but for the backoff counters it advances.
|
||||
fn step(backoffs: &mut Backoffs, agent: &str, seen: Observation) -> Step {
|
||||
if seen.running == Some(true) || seen.wanted != Some(Wanted::Up) {
|
||||
let harness_down = seen.running == Some(true) && seen.harness == HarnessState::Stopped;
|
||||
if (seen.running == Some(true) && !harness_down) || seen.wanted != Some(Wanted::Up) {
|
||||
backoffs.settle(agent);
|
||||
return Step::Leave(Leave::Converged);
|
||||
}
|
||||
|
|
@ -170,11 +177,20 @@ async fn tick(coord: &Arc<Coordinator>, backoffs: &mut Backoffs) {
|
|||
let Some(name) = container.strip_prefix(AGENT_PREFIX) else {
|
||||
continue;
|
||||
};
|
||||
let wanted = coord.power.get(name);
|
||||
let running = lifecycle::running_state(name).await;
|
||||
let updating = coord.is_updating(name);
|
||||
let harness = if running == Some(true) && wanted == Some(Wanted::Up) && !updating {
|
||||
lifecycle::harness_state(name).await
|
||||
} else {
|
||||
HarnessState::Unknown
|
||||
};
|
||||
let seen = Observation {
|
||||
wanted: coord.power.get(name),
|
||||
running: lifecycle::running_state(name).await,
|
||||
wanted,
|
||||
running,
|
||||
paused: hive_types::Ident::parse(name).is_ok_and(|id| Coordinator::is_paused(&id)),
|
||||
updating: coord.is_updating(name),
|
||||
updating,
|
||||
harness,
|
||||
};
|
||||
if step(backoffs, name, seen) != Step::Restart {
|
||||
continue;
|
||||
|
|
@ -184,7 +200,11 @@ async fn tick(coord: &Arc<Coordinator>, backoffs: &mut Backoffs) {
|
|||
let attempt = backoffs.record_attempt(name);
|
||||
tracing::info!(
|
||||
agent = %name,
|
||||
observed = "stopped",
|
||||
observed = if seen.running == Some(true) {
|
||||
"harness stopped"
|
||||
} else {
|
||||
"stopped"
|
||||
},
|
||||
wanted = "up",
|
||||
attempt,
|
||||
"reconcile tick: restarting agent"
|
||||
|
|
@ -214,6 +234,7 @@ mod tests {
|
|||
running: Some(false),
|
||||
paused: false,
|
||||
updating: false,
|
||||
harness: HarnessState::Unknown,
|
||||
};
|
||||
|
||||
#[test]
|
||||
|
|
@ -284,6 +305,56 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
/// A running, wanted-up agent whose harness unit is stopped is restarted;
|
||||
/// every other reading of a running agent leaves it alone.
|
||||
#[test]
|
||||
fn a_running_agent_with_a_stopped_harness_is_restarted() {
|
||||
let running = |harness| Observation {
|
||||
running: Some(true),
|
||||
harness,
|
||||
..DOWN
|
||||
};
|
||||
assert_eq!(
|
||||
step(
|
||||
&mut Backoffs::default(),
|
||||
"a",
|
||||
running(HarnessState::Stopped)
|
||||
),
|
||||
Step::Restart
|
||||
);
|
||||
for harness in [HarnessState::Present, HarnessState::Unknown] {
|
||||
assert_eq!(
|
||||
step(&mut Backoffs::default(), "a", running(harness)),
|
||||
Step::Leave(Leave::Converged),
|
||||
"{harness:?}"
|
||||
);
|
||||
}
|
||||
let wanted_offline = Observation {
|
||||
wanted: Some(Wanted::Offline),
|
||||
..running(HarnessState::Stopped)
|
||||
};
|
||||
assert_eq!(
|
||||
step(&mut Backoffs::default(), "a", wanted_offline),
|
||||
Step::Leave(Leave::Converged)
|
||||
);
|
||||
let mid_update = Observation {
|
||||
updating: true,
|
||||
..running(HarnessState::Stopped)
|
||||
};
|
||||
assert_eq!(
|
||||
step(&mut Backoffs::default(), "a", mid_update),
|
||||
Step::Leave(Leave::Held)
|
||||
);
|
||||
let mut backoffs = Backoffs::default();
|
||||
backoffs.record_attempt("a");
|
||||
backoffs.record_attempt("a");
|
||||
assert_eq!(
|
||||
step(&mut backoffs, "a", running(HarnessState::Stopped)),
|
||||
Step::Leave(Leave::BackingOff),
|
||||
"a harness that stays down backs off like a container"
|
||||
);
|
||||
}
|
||||
|
||||
/// Ticks between restarts of an agent that never comes up: 1, 2, 4, 8,
|
||||
/// then the 15-tick cap, and it stays there.
|
||||
#[test]
|
||||
|
|
|
|||
Loading…
Reference in a new issue