hive-priv gains AgentHarness { agent_name, action: probe | start }, which
runs systemctl -M h-<name> against the fixed unit hive-agent.service.
run_reconcile and the 1-min tick read the unit for a running wanted-up
agent and start it when it is inactive or failed. A hive-priv without the
op reads as unknown state and leaves reconcile container-only.
442 lines
15 KiB
Rust
442 lines
15 KiB
Rust
//! Periodic declared ⇒ running convergence: an agent whose power intent is
|
|
//! `Up` and whose container is down, or running with its harness unit stopped,
|
|
//! gets a `Reconcile` queued, once a [`TICK_INTERVAL`], without anyone having
|
|
//! to ask.
|
|
//!
|
|
//! The tick queues the same `NodeKind::Reconcile` the boot sweep, a watch
|
|
//! attach and a declaration write end in, so what it starts, and how, is
|
|
//! `run_reconcile`'s decision. It only ever converges *toward a start*: an
|
|
//! agent wanted offline is left alone, and a running one is left alone unless
|
|
//! hive-priv reports its harness unit stopped or failed. A harness that cannot
|
|
//! be read (an older hive-priv, no answer) counts as running.
|
|
//! `crash_watch` keeps raising its alert; this worker does not replace it.
|
|
//!
|
|
//! # What the tick leaves alone
|
|
//!
|
|
//! - an agent with no known intent, or one wanted offline (the operator's stop
|
|
//! stands);
|
|
//! - a paused agent, and one whose container state could not be read in time;
|
|
//! - an agent any queued or running node already targets — an update, create,
|
|
//! destroy, power op or earlier reconcile (see
|
|
//! [`crate::job_queue::JobQueue::insert_reconcile_if_idle`]).
|
|
//!
|
|
//! # Backoff
|
|
//!
|
|
//! An agent that is still down on the tick after a restart was queued for it
|
|
//! has failed to come up. Restarts of one agent are spaced `1, 2, 4, …` ticks
|
|
//! apart, capped at [`MAX_BACKOFF`]. Seeing the agent running, or no longer
|
|
//! wanted up, forgets its history.
|
|
|
|
use std::collections::HashMap;
|
|
use std::sync::Arc;
|
|
use std::time::Duration;
|
|
|
|
use crate::coordinator::Coordinator;
|
|
use crate::lifecycle::{self, AGENT_PREFIX, HarnessState};
|
|
use crate::power::Wanted;
|
|
|
|
/// How often declared intent is compared with the observed containers. The
|
|
/// first tick fires one interval after boot: the boot sweep owns startup.
|
|
const TICK_INTERVAL: Duration = Duration::from_mins(1);
|
|
|
|
/// The longest wait between two restarts of one agent.
|
|
const MAX_BACKOFF: Duration = Duration::from_mins(15);
|
|
|
|
/// What the tick can see of one agent.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
struct Observation {
|
|
/// The power intent this process holds; `None` before anything set or
|
|
/// seeded one.
|
|
wanted: Option<Wanted>,
|
|
/// Whether the container's unit is active; `None` when `systemctl` did not
|
|
/// answer in time.
|
|
running: Option<bool>,
|
|
paused: bool,
|
|
/// An update has the agent down and not yet back up.
|
|
updating: bool,
|
|
/// The harness unit inside a running, wanted-up container; `Unknown`
|
|
/// where it was not read.
|
|
harness: HarnessState,
|
|
}
|
|
|
|
/// Why the tick did nothing for an agent.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
enum Leave {
|
|
/// Running, or wanted offline, or no intent: nothing to converge.
|
|
Converged,
|
|
/// Paused, updating, or the container state is unreadable.
|
|
Held,
|
|
/// A restart was queued recently and the agent has not come up; waiting
|
|
/// out the backoff.
|
|
BackingOff,
|
|
}
|
|
|
|
/// What one tick does for one agent.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
enum Step {
|
|
Leave(Leave),
|
|
/// Queue a `Reconcile` (unless the agent turns out to be busy).
|
|
Restart,
|
|
}
|
|
|
|
/// Per-agent restart history, in ticks.
|
|
#[derive(Debug, Default)]
|
|
struct Backoffs {
|
|
agents: HashMap<String, Backoff>,
|
|
}
|
|
|
|
#[derive(Debug, Default)]
|
|
struct Backoff {
|
|
/// Restarts queued since the agent was last seen converged.
|
|
attempts: u32,
|
|
/// Ticks still to skip before the next restart.
|
|
skip: u32,
|
|
}
|
|
|
|
impl Backoffs {
|
|
/// Forget `agent`: it is running, or no longer wanted up.
|
|
fn settle(&mut self, agent: &str) {
|
|
self.agents.remove(agent);
|
|
}
|
|
|
|
/// Whether a restart may be queued for `agent` this tick. A refusal uses
|
|
/// up one skipped tick.
|
|
fn ready(&mut self, agent: &str) -> bool {
|
|
match self.agents.get_mut(agent) {
|
|
Some(b) if b.skip > 0 => {
|
|
b.skip -= 1;
|
|
false
|
|
}
|
|
_ => true,
|
|
}
|
|
}
|
|
|
|
/// Record that a restart was queued; returns how many this makes.
|
|
fn record_attempt(&mut self, agent: &str) -> u32 {
|
|
let b = self.agents.entry(agent.to_owned()).or_default();
|
|
b.attempts = b.attempts.saturating_add(1);
|
|
b.skip = ticks_between(b.attempts).saturating_sub(1);
|
|
b.attempts
|
|
}
|
|
}
|
|
|
|
/// Ticks between the `attempts`-th restart and the next: `2^(attempts-1)`,
|
|
/// capped at [`MAX_BACKOFF`].
|
|
fn ticks_between(attempts: u32) -> u32 {
|
|
let cap = u32::try_from(MAX_BACKOFF.as_secs() / TICK_INTERVAL.as_secs()).unwrap_or(u32::MAX);
|
|
1_u32
|
|
.checked_shl(attempts.saturating_sub(1))
|
|
.unwrap_or(u32::MAX)
|
|
.min(cap)
|
|
}
|
|
|
|
/// The decision for one agent. Pure but for the backoff counters it advances.
|
|
fn step(backoffs: &mut Backoffs, agent: &str, seen: Observation) -> Step {
|
|
let harness_down = seen.running == Some(true) && seen.harness == HarnessState::Stopped;
|
|
if (seen.running == Some(true) && !harness_down) || seen.wanted != Some(Wanted::Up) {
|
|
backoffs.settle(agent);
|
|
return Step::Leave(Leave::Converged);
|
|
}
|
|
if seen.running.is_none() || seen.paused || seen.updating {
|
|
return Step::Leave(Leave::Held);
|
|
}
|
|
if !backoffs.ready(agent) {
|
|
return Step::Leave(Leave::BackingOff);
|
|
}
|
|
Step::Restart
|
|
}
|
|
|
|
/// Run the tick until shutdown.
|
|
pub fn spawn(coord: Arc<Coordinator>) {
|
|
let mut shutdown = coord.shutdown_rx();
|
|
tokio::spawn(async move {
|
|
let mut backoffs = Backoffs::default();
|
|
loop {
|
|
tokio::select! {
|
|
() = tokio::time::sleep(TICK_INTERVAL) => {}
|
|
_ = shutdown.changed() => {
|
|
tracing::info!("reconcile tick: shutdown signal received");
|
|
return;
|
|
}
|
|
}
|
|
tick(&coord, &mut backoffs).await;
|
|
}
|
|
});
|
|
}
|
|
|
|
/// One pass over every agent container.
|
|
async fn tick(coord: &Arc<Coordinator>, backoffs: &mut Backoffs) {
|
|
let containers = match lifecycle::list().await {
|
|
Ok(c) => c,
|
|
Err(e) => {
|
|
tracing::warn!(error = ?e, "reconcile tick: container list unreadable — skipping this tick");
|
|
return;
|
|
}
|
|
};
|
|
for container in &containers {
|
|
let Some(name) = container.strip_prefix(AGENT_PREFIX) else {
|
|
continue;
|
|
};
|
|
let wanted = coord.power.get(name);
|
|
let running = lifecycle::running_state(name).await;
|
|
let updating = coord.is_updating(name);
|
|
let harness = if running == Some(true) && wanted == Some(Wanted::Up) && !updating {
|
|
lifecycle::harness_state(name).await
|
|
} else {
|
|
HarnessState::Unknown
|
|
};
|
|
let seen = Observation {
|
|
wanted,
|
|
running,
|
|
paused: hive_types::Ident::parse(name).is_ok_and(|id| Coordinator::is_paused(&id)),
|
|
updating,
|
|
harness,
|
|
};
|
|
if step(backoffs, name, seen) != Step::Restart {
|
|
continue;
|
|
}
|
|
match coord.job_queue.insert_reconcile_if_idle(name) {
|
|
Ok(Some(_)) => {
|
|
let attempt = backoffs.record_attempt(name);
|
|
tracing::info!(
|
|
agent = %name,
|
|
observed = if seen.running == Some(true) {
|
|
"harness stopped"
|
|
} else {
|
|
"stopped"
|
|
},
|
|
wanted = "up",
|
|
attempt,
|
|
"reconcile tick: restarting agent"
|
|
);
|
|
crate::otel_metrics::count_reconcile_restart();
|
|
coord.emit_rebuild_queue_snapshot();
|
|
}
|
|
// Work targeting the agent is in flight; it converges the agent or
|
|
// decides not to.
|
|
Ok(None) => {
|
|
tracing::debug!(agent = %name, "reconcile tick: agent is busy; leaving it");
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(agent = %name, error = ?e, "reconcile tick: queueing the reconcile failed");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
/// A wanted-up agent whose container is down, nothing holding it back.
|
|
const DOWN: Observation = Observation {
|
|
wanted: Some(Wanted::Up),
|
|
running: Some(false),
|
|
paused: false,
|
|
updating: false,
|
|
harness: HarnessState::Unknown,
|
|
};
|
|
|
|
#[test]
|
|
fn a_wanted_up_stopped_agent_is_restarted() {
|
|
assert_eq!(step(&mut Backoffs::default(), "a", DOWN), Step::Restart);
|
|
}
|
|
|
|
/// Each deviation from [`DOWN`] that must leave the agent alone, against
|
|
/// the control above that the same agent is restarted without it.
|
|
#[test]
|
|
fn the_tick_leaves_alone_what_is_not_a_plain_down_agent() {
|
|
let cases = [
|
|
(
|
|
"wanted offline",
|
|
Observation {
|
|
wanted: Some(Wanted::Offline),
|
|
..DOWN
|
|
},
|
|
Leave::Converged,
|
|
),
|
|
(
|
|
"no intent yet",
|
|
Observation {
|
|
wanted: None,
|
|
..DOWN
|
|
},
|
|
Leave::Converged,
|
|
),
|
|
(
|
|
"already running",
|
|
Observation {
|
|
running: Some(true),
|
|
..DOWN
|
|
},
|
|
Leave::Converged,
|
|
),
|
|
(
|
|
"paused",
|
|
Observation {
|
|
paused: true,
|
|
..DOWN
|
|
},
|
|
Leave::Held,
|
|
),
|
|
(
|
|
"mid-update",
|
|
Observation {
|
|
updating: true,
|
|
..DOWN
|
|
},
|
|
Leave::Held,
|
|
),
|
|
(
|
|
"container state unreadable",
|
|
Observation {
|
|
running: None,
|
|
..DOWN
|
|
},
|
|
Leave::Held,
|
|
),
|
|
];
|
|
for (what, seen, why) in cases {
|
|
assert_eq!(
|
|
step(&mut Backoffs::default(), "a", seen),
|
|
Step::Leave(why),
|
|
"{what}"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// A running, wanted-up agent whose harness unit is stopped is restarted;
|
|
/// every other reading of a running agent leaves it alone.
|
|
#[test]
|
|
fn a_running_agent_with_a_stopped_harness_is_restarted() {
|
|
let running = |harness| Observation {
|
|
running: Some(true),
|
|
harness,
|
|
..DOWN
|
|
};
|
|
assert_eq!(
|
|
step(
|
|
&mut Backoffs::default(),
|
|
"a",
|
|
running(HarnessState::Stopped)
|
|
),
|
|
Step::Restart
|
|
);
|
|
for harness in [HarnessState::Present, HarnessState::Unknown] {
|
|
assert_eq!(
|
|
step(&mut Backoffs::default(), "a", running(harness)),
|
|
Step::Leave(Leave::Converged),
|
|
"{harness:?}"
|
|
);
|
|
}
|
|
let wanted_offline = Observation {
|
|
wanted: Some(Wanted::Offline),
|
|
..running(HarnessState::Stopped)
|
|
};
|
|
assert_eq!(
|
|
step(&mut Backoffs::default(), "a", wanted_offline),
|
|
Step::Leave(Leave::Converged)
|
|
);
|
|
let mid_update = Observation {
|
|
updating: true,
|
|
..running(HarnessState::Stopped)
|
|
};
|
|
assert_eq!(
|
|
step(&mut Backoffs::default(), "a", mid_update),
|
|
Step::Leave(Leave::Held)
|
|
);
|
|
let mut backoffs = Backoffs::default();
|
|
backoffs.record_attempt("a");
|
|
backoffs.record_attempt("a");
|
|
assert_eq!(
|
|
step(&mut backoffs, "a", running(HarnessState::Stopped)),
|
|
Step::Leave(Leave::BackingOff),
|
|
"a harness that stays down backs off like a container"
|
|
);
|
|
}
|
|
|
|
/// Ticks between restarts of an agent that never comes up: 1, 2, 4, 8,
|
|
/// then the 15-tick cap, and it stays there.
|
|
#[test]
|
|
fn restarts_of_a_failing_agent_back_off_to_the_cap() {
|
|
let mut backoffs = Backoffs::default();
|
|
let mut gaps = Vec::new();
|
|
let mut since = 0_u32;
|
|
for _ in 0..80 {
|
|
since += 1;
|
|
if step(&mut backoffs, "a", DOWN) == Step::Restart {
|
|
backoffs.record_attempt("a");
|
|
gaps.push(since);
|
|
since = 0;
|
|
}
|
|
}
|
|
assert_eq!(gaps[..6], [1, 1, 2, 4, 8, 15]);
|
|
assert!(gaps[5..].iter().all(|g| *g == 15), "{gaps:?}");
|
|
}
|
|
|
|
#[test]
|
|
fn the_wait_is_a_backoff_not_a_refusal() {
|
|
let mut backoffs = Backoffs::default();
|
|
backoffs.record_attempt("a");
|
|
backoffs.record_attempt("a");
|
|
assert_eq!(
|
|
step(&mut backoffs, "a", DOWN),
|
|
Step::Leave(Leave::BackingOff)
|
|
);
|
|
assert_eq!(step(&mut backoffs, "a", DOWN), Step::Restart);
|
|
// Another agent's history is not this one's.
|
|
assert_eq!(step(&mut backoffs, "b", DOWN), Step::Restart);
|
|
}
|
|
|
|
#[test]
|
|
fn seeing_the_agent_running_resets_its_backoff() {
|
|
let mut backoffs = Backoffs::default();
|
|
for _ in 0..4 {
|
|
backoffs.record_attempt("a");
|
|
}
|
|
assert_eq!(
|
|
step(&mut backoffs, "a", DOWN),
|
|
Step::Leave(Leave::BackingOff),
|
|
"control: the history holds the agent back"
|
|
);
|
|
let up = Observation {
|
|
running: Some(true),
|
|
..DOWN
|
|
};
|
|
assert_eq!(step(&mut backoffs, "a", up), Step::Leave(Leave::Converged));
|
|
assert_eq!(step(&mut backoffs, "a", DOWN), Step::Restart);
|
|
assert_eq!(backoffs.record_attempt("a"), 1, "the count starts over");
|
|
}
|
|
|
|
#[test]
|
|
fn an_operator_stop_resets_the_backoff() {
|
|
let mut backoffs = Backoffs::default();
|
|
for _ in 0..4 {
|
|
backoffs.record_attempt("a");
|
|
}
|
|
let stopped = Observation {
|
|
wanted: Some(Wanted::Offline),
|
|
..DOWN
|
|
};
|
|
step(&mut backoffs, "a", stopped);
|
|
assert_eq!(step(&mut backoffs, "a", DOWN), Step::Restart);
|
|
}
|
|
|
|
/// A refused restart (the agent turned out busy) does not count as an
|
|
/// attempt, so it does not lengthen the next wait.
|
|
#[test]
|
|
fn only_a_recorded_attempt_lengthens_the_backoff() {
|
|
let mut backoffs = Backoffs::default();
|
|
assert_eq!(step(&mut backoffs, "a", DOWN), Step::Restart);
|
|
assert_eq!(step(&mut backoffs, "a", DOWN), Step::Restart);
|
|
assert_eq!(backoffs.record_attempt("a"), 1);
|
|
}
|
|
|
|
#[test]
|
|
fn the_cap_is_reached_without_overflow() {
|
|
assert_eq!(ticks_between(1), 1);
|
|
assert_eq!(ticks_between(4), 8);
|
|
assert_eq!(ticks_between(5), 15);
|
|
assert_eq!(ticks_between(u32::MAX), 15);
|
|
}
|
|
}
|