//! Periodic declared ⇒ running convergence: an agent whose power intent is //! `Up` and whose container is down, or running with its harness unit stopped, //! gets a `Reconcile` queued, once a [`TICK_INTERVAL`], without anyone having //! to ask. //! //! The tick queues the same `NodeKind::Reconcile` the boot sweep, a watch //! attach and a declaration write end in, so what it starts, and how, is //! `run_reconcile`'s decision. It only ever converges *toward a start*: an //! agent wanted offline is left alone, and a running one is left alone unless //! hive-priv reports its harness unit stopped or failed. A harness that cannot //! be read (an older hive-priv, no answer) counts as running. //! `crash_watch` keeps raising its alert; this worker does not replace it. //! //! # What the tick leaves alone //! //! - an agent with no known intent, or one wanted offline (the operator's stop //! stands); //! - a paused agent, and one whose container state could not be read in time; //! - an agent any queued or running node already targets — an update, create, //! destroy, power op or earlier reconcile (see //! [`crate::job_queue::JobQueue::insert_reconcile_if_idle`]). //! //! # Backoff //! //! An agent that is still down on the tick after a restart was queued for it //! has failed to come up. Restarts of one agent are spaced `1, 2, 4, …` ticks //! apart, capped at [`MAX_BACKOFF`]. Seeing the agent running, or no longer //! wanted up, forgets its history. use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; use crate::coordinator::Coordinator; use crate::lifecycle::{self, AGENT_PREFIX, HarnessState}; use crate::power::Wanted; /// How often declared intent is compared with the observed containers. The /// first tick fires one interval after boot: the boot sweep owns startup. const TICK_INTERVAL: Duration = Duration::from_mins(1); /// The longest wait between two restarts of one agent. const MAX_BACKOFF: Duration = Duration::from_mins(15); /// What the tick can see of one agent. #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct Observation { /// The power intent this process holds; `None` before anything set or /// seeded one. wanted: Option, /// Whether the container's unit is active; `None` when `systemctl` did not /// answer in time. running: Option, paused: bool, /// An update has the agent down and not yet back up. updating: bool, /// The harness unit inside a running, wanted-up container; `Unknown` /// where it was not read. harness: HarnessState, } /// Why the tick did nothing for an agent. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Leave { /// Running, or wanted offline, or no intent: nothing to converge. Converged, /// Paused, updating, or the container state is unreadable. Held, /// A restart was queued recently and the agent has not come up; waiting /// out the backoff. BackingOff, } /// What one tick does for one agent. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Step { Leave(Leave), /// Queue a `Reconcile` (unless the agent turns out to be busy). Restart, } /// Per-agent restart history, in ticks. #[derive(Debug, Default)] struct Backoffs { agents: HashMap, } #[derive(Debug, Default)] struct Backoff { /// Restarts queued since the agent was last seen converged. attempts: u32, /// Ticks still to skip before the next restart. skip: u32, } impl Backoffs { /// Forget `agent`: it is running, or no longer wanted up. fn settle(&mut self, agent: &str) { self.agents.remove(agent); } /// Whether a restart may be queued for `agent` this tick. A refusal uses /// up one skipped tick. fn ready(&mut self, agent: &str) -> bool { match self.agents.get_mut(agent) { Some(b) if b.skip > 0 => { b.skip -= 1; false } _ => true, } } /// Record that a restart was queued; returns how many this makes. fn record_attempt(&mut self, agent: &str) -> u32 { let b = self.agents.entry(agent.to_owned()).or_default(); b.attempts = b.attempts.saturating_add(1); b.skip = ticks_between(b.attempts).saturating_sub(1); b.attempts } } /// Ticks between the `attempts`-th restart and the next: `2^(attempts-1)`, /// capped at [`MAX_BACKOFF`]. fn ticks_between(attempts: u32) -> u32 { let cap = u32::try_from(MAX_BACKOFF.as_secs() / TICK_INTERVAL.as_secs()).unwrap_or(u32::MAX); 1_u32 .checked_shl(attempts.saturating_sub(1)) .unwrap_or(u32::MAX) .min(cap) } /// The decision for one agent. Pure but for the backoff counters it advances. fn step(backoffs: &mut Backoffs, agent: &str, seen: Observation) -> Step { let harness_down = seen.running == Some(true) && seen.harness == HarnessState::Stopped; if (seen.running == Some(true) && !harness_down) || seen.wanted != Some(Wanted::Up) { backoffs.settle(agent); return Step::Leave(Leave::Converged); } if seen.running.is_none() || seen.paused || seen.updating { return Step::Leave(Leave::Held); } if !backoffs.ready(agent) { return Step::Leave(Leave::BackingOff); } Step::Restart } /// Run the tick until shutdown. pub fn spawn(coord: Arc) { let mut shutdown = coord.shutdown_rx(); tokio::spawn(async move { let mut backoffs = Backoffs::default(); loop { tokio::select! { () = tokio::time::sleep(TICK_INTERVAL) => {} _ = shutdown.changed() => { tracing::info!("reconcile tick: shutdown signal received"); return; } } tick(&coord, &mut backoffs).await; } }); } /// One pass over every agent container. async fn tick(coord: &Arc, backoffs: &mut Backoffs) { let containers = match lifecycle::list().await { Ok(c) => c, Err(e) => { tracing::warn!(error = ?e, "reconcile tick: container list unreadable — skipping this tick"); return; } }; for container in &containers { let Some(name) = container.strip_prefix(AGENT_PREFIX) else { continue; }; let wanted = coord.power.get(name); let running = lifecycle::running_state(name).await; let updating = coord.is_updating(name); let harness = if running == Some(true) && wanted == Some(Wanted::Up) && !updating { lifecycle::harness_state(name).await } else { HarnessState::Unknown }; let seen = Observation { wanted, running, paused: hive_types::Ident::parse(name).is_ok_and(|id| Coordinator::is_paused(&id)), updating, harness, }; if step(backoffs, name, seen) != Step::Restart { continue; } match coord.job_queue.insert_reconcile_if_idle(name) { Ok(Some(_)) => { let attempt = backoffs.record_attempt(name); tracing::info!( agent = %name, observed = if seen.running == Some(true) { "harness stopped" } else { "stopped" }, wanted = "up", attempt, "reconcile tick: restarting agent" ); crate::otel_metrics::count_reconcile_restart(); coord.emit_rebuild_queue_snapshot(); } // Work targeting the agent is in flight; it converges the agent or // decides not to. Ok(None) => { tracing::debug!(agent = %name, "reconcile tick: agent is busy; leaving it"); } Err(e) => { tracing::warn!(agent = %name, error = ?e, "reconcile tick: queueing the reconcile failed"); } } } } #[cfg(test)] mod tests { use super::*; /// A wanted-up agent whose container is down, nothing holding it back. const DOWN: Observation = Observation { wanted: Some(Wanted::Up), running: Some(false), paused: false, updating: false, harness: HarnessState::Unknown, }; #[test] fn a_wanted_up_stopped_agent_is_restarted() { assert_eq!(step(&mut Backoffs::default(), "a", DOWN), Step::Restart); } /// Each deviation from [`DOWN`] that must leave the agent alone, against /// the control above that the same agent is restarted without it. #[test] fn the_tick_leaves_alone_what_is_not_a_plain_down_agent() { let cases = [ ( "wanted offline", Observation { wanted: Some(Wanted::Offline), ..DOWN }, Leave::Converged, ), ( "no intent yet", Observation { wanted: None, ..DOWN }, Leave::Converged, ), ( "already running", Observation { running: Some(true), ..DOWN }, Leave::Converged, ), ( "paused", Observation { paused: true, ..DOWN }, Leave::Held, ), ( "mid-update", Observation { updating: true, ..DOWN }, Leave::Held, ), ( "container state unreadable", Observation { running: None, ..DOWN }, Leave::Held, ), ]; for (what, seen, why) in cases { assert_eq!( step(&mut Backoffs::default(), "a", seen), Step::Leave(why), "{what}" ); } } /// A running, wanted-up agent whose harness unit is stopped is restarted; /// every other reading of a running agent leaves it alone. #[test] fn a_running_agent_with_a_stopped_harness_is_restarted() { let running = |harness| Observation { running: Some(true), harness, ..DOWN }; assert_eq!( step( &mut Backoffs::default(), "a", running(HarnessState::Stopped) ), Step::Restart ); for harness in [HarnessState::Present, HarnessState::Unknown] { assert_eq!( step(&mut Backoffs::default(), "a", running(harness)), Step::Leave(Leave::Converged), "{harness:?}" ); } let wanted_offline = Observation { wanted: Some(Wanted::Offline), ..running(HarnessState::Stopped) }; assert_eq!( step(&mut Backoffs::default(), "a", wanted_offline), Step::Leave(Leave::Converged) ); let mid_update = Observation { updating: true, ..running(HarnessState::Stopped) }; assert_eq!( step(&mut Backoffs::default(), "a", mid_update), Step::Leave(Leave::Held) ); let mut backoffs = Backoffs::default(); backoffs.record_attempt("a"); backoffs.record_attempt("a"); assert_eq!( step(&mut backoffs, "a", running(HarnessState::Stopped)), Step::Leave(Leave::BackingOff), "a harness that stays down backs off like a container" ); } /// Ticks between restarts of an agent that never comes up: 1, 2, 4, 8, /// then the 15-tick cap, and it stays there. #[test] fn restarts_of_a_failing_agent_back_off_to_the_cap() { let mut backoffs = Backoffs::default(); let mut gaps = Vec::new(); let mut since = 0_u32; for _ in 0..80 { since += 1; if step(&mut backoffs, "a", DOWN) == Step::Restart { backoffs.record_attempt("a"); gaps.push(since); since = 0; } } assert_eq!(gaps[..6], [1, 1, 2, 4, 8, 15]); assert!(gaps[5..].iter().all(|g| *g == 15), "{gaps:?}"); } #[test] fn the_wait_is_a_backoff_not_a_refusal() { let mut backoffs = Backoffs::default(); backoffs.record_attempt("a"); backoffs.record_attempt("a"); assert_eq!( step(&mut backoffs, "a", DOWN), Step::Leave(Leave::BackingOff) ); assert_eq!(step(&mut backoffs, "a", DOWN), Step::Restart); // Another agent's history is not this one's. assert_eq!(step(&mut backoffs, "b", DOWN), Step::Restart); } #[test] fn seeing_the_agent_running_resets_its_backoff() { let mut backoffs = Backoffs::default(); for _ in 0..4 { backoffs.record_attempt("a"); } assert_eq!( step(&mut backoffs, "a", DOWN), Step::Leave(Leave::BackingOff), "control: the history holds the agent back" ); let up = Observation { running: Some(true), ..DOWN }; assert_eq!(step(&mut backoffs, "a", up), Step::Leave(Leave::Converged)); assert_eq!(step(&mut backoffs, "a", DOWN), Step::Restart); assert_eq!(backoffs.record_attempt("a"), 1, "the count starts over"); } #[test] fn an_operator_stop_resets_the_backoff() { let mut backoffs = Backoffs::default(); for _ in 0..4 { backoffs.record_attempt("a"); } let stopped = Observation { wanted: Some(Wanted::Offline), ..DOWN }; step(&mut backoffs, "a", stopped); assert_eq!(step(&mut backoffs, "a", DOWN), Step::Restart); } /// A refused restart (the agent turned out busy) does not count as an /// attempt, so it does not lengthen the next wait. #[test] fn only_a_recorded_attempt_lengthens_the_backoff() { let mut backoffs = Backoffs::default(); assert_eq!(step(&mut backoffs, "a", DOWN), Step::Restart); assert_eq!(step(&mut backoffs, "a", DOWN), Step::Restart); assert_eq!(backoffs.record_attempt("a"), 1); } #[test] fn the_cap_is_reached_without_overflow() { assert_eq!(ticks_between(1), 1); assert_eq!(ticks_between(4), 8); assert_eq!(ticks_between(5), 15); assert_eq!(ticks_between(u32::MAX), 15); } }