hive-c0re: reconcile declared agents periodically, with backoff
This commit is contained in:
parent
7468aa4cff
commit
b8c07a5492
2 changed files with 516 additions and 9 deletions
|
|
@ -1,5 +1,5 @@
|
|||
//! Background tasks and periodic sweeps: crash/login watcher, boot-time
|
||||
//! auto-update reconcile, the agent-sockets.json writer loop, the MCP socket listener
|
||||
//! auto-update reconcile, the periodic declared ⇒ running reconcile tick, the agent-sockets.json writer loop, the MCP socket listener
|
||||
//! reconcile loop, knowledge-repo sync, the swarm wanted-state pull, and the
|
||||
//! swarm resource-cap and permissions reads, and the squid users file.
|
||||
//! Each submodule is re-exported
|
||||
|
|
@ -11,6 +11,7 @@ pub mod crash_watch;
|
|||
pub mod knowledge;
|
||||
pub mod mcp_sockets;
|
||||
pub mod permissions_import;
|
||||
pub mod reconcile_tick;
|
||||
pub mod squid_allow;
|
||||
pub mod squid_deny;
|
||||
pub mod squid_users;
|
||||
|
|
|
|||
371
hive-c0re/src/workers/reconcile_tick.rs
Normal file
371
hive-c0re/src/workers/reconcile_tick.rs
Normal file
|
|
@ -0,0 +1,371 @@
|
|||
//! Periodic declared ⇒ running convergence: an agent whose power intent is
|
||||
//! `Up` and whose container is down gets a `Reconcile` queued, once a
|
||||
//! [`TICK_INTERVAL`], without anyone having to ask.
|
||||
//!
|
||||
//! The tick queues the same `NodeKind::Reconcile` the boot sweep, a watch
|
||||
//! attach and a declaration write end in, so what it starts, and how, is
|
||||
//! `run_reconcile`'s decision. It only ever converges *toward a start*: an
|
||||
//! agent wanted offline is left alone, and a running one is never touched.
|
||||
//! `crash_watch` keeps raising its alert; this worker does not replace it.
|
||||
//!
|
||||
//! # What the tick leaves alone
|
||||
//!
|
||||
//! - an agent with no known intent, or one wanted offline (the operator's stop
|
||||
//! stands);
|
||||
//! - a paused agent, and one whose container state could not be read in time;
|
||||
//! - an agent any queued or running node already targets — an update, create,
|
||||
//! destroy, power op or earlier reconcile (see
|
||||
//! [`JobQueue::insert_reconcile_if_idle`]).
|
||||
//!
|
||||
//! # Backoff
|
||||
//!
|
||||
//! An agent that is still down on the tick after a restart was queued for it
|
||||
//! has failed to come up. Restarts of one agent are spaced `1, 2, 4, …` ticks
|
||||
//! apart, capped at [`MAX_BACKOFF`]. Seeing the agent running, or no longer
|
||||
//! wanted up, forgets its history.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
use crate::lifecycle::{self, AGENT_PREFIX};
|
||||
use crate::power::Wanted;
|
||||
|
||||
/// How often declared intent is compared with the observed containers. The
|
||||
/// first tick fires one interval after boot: the boot sweep owns startup.
|
||||
const TICK_INTERVAL: Duration = Duration::from_mins(1);
|
||||
|
||||
/// The longest wait between two restarts of one agent.
|
||||
const MAX_BACKOFF: Duration = Duration::from_mins(15);
|
||||
|
||||
/// What the tick can see of one agent.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
struct Observation {
|
||||
/// The power intent this process holds; `None` before anything set or
|
||||
/// seeded one.
|
||||
wanted: Option<Wanted>,
|
||||
/// Whether the container's unit is active; `None` when `systemctl` did not
|
||||
/// answer in time.
|
||||
running: Option<bool>,
|
||||
paused: bool,
|
||||
/// An update has the agent down and not yet back up.
|
||||
updating: bool,
|
||||
}
|
||||
|
||||
/// Why the tick did nothing for an agent.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum Leave {
|
||||
/// Running, or wanted offline, or no intent: nothing to converge.
|
||||
Converged,
|
||||
/// Paused, updating, or the container state is unreadable.
|
||||
Held,
|
||||
/// A restart was queued recently and the agent has not come up; waiting
|
||||
/// out the backoff.
|
||||
BackingOff,
|
||||
}
|
||||
|
||||
/// What one tick does for one agent.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum Step {
|
||||
Leave(Leave),
|
||||
/// Queue a `Reconcile` (unless the agent turns out to be busy).
|
||||
Restart,
|
||||
}
|
||||
|
||||
/// Per-agent restart history, in ticks.
|
||||
#[derive(Debug, Default)]
|
||||
struct Backoffs {
|
||||
agents: HashMap<String, Backoff>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Backoff {
|
||||
/// Restarts queued since the agent was last seen converged.
|
||||
attempts: u32,
|
||||
/// Ticks still to skip before the next restart.
|
||||
skip: u32,
|
||||
}
|
||||
|
||||
impl Backoffs {
|
||||
/// Forget `agent`: it is running, or no longer wanted up.
|
||||
fn settle(&mut self, agent: &str) {
|
||||
self.agents.remove(agent);
|
||||
}
|
||||
|
||||
/// Whether a restart may be queued for `agent` this tick. A refusal uses
|
||||
/// up one skipped tick.
|
||||
fn ready(&mut self, agent: &str) -> bool {
|
||||
match self.agents.get_mut(agent) {
|
||||
Some(b) if b.skip > 0 => {
|
||||
b.skip -= 1;
|
||||
false
|
||||
}
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Record that a restart was queued; returns how many this makes.
|
||||
fn record_attempt(&mut self, agent: &str) -> u32 {
|
||||
let b = self.agents.entry(agent.to_owned()).or_default();
|
||||
b.attempts = b.attempts.saturating_add(1);
|
||||
b.skip = ticks_between(b.attempts).saturating_sub(1);
|
||||
b.attempts
|
||||
}
|
||||
}
|
||||
|
||||
/// Ticks between the `attempts`-th restart and the next: `2^(attempts-1)`,
|
||||
/// capped at [`MAX_BACKOFF`].
|
||||
fn ticks_between(attempts: u32) -> u32 {
|
||||
let cap = u32::try_from(MAX_BACKOFF.as_secs() / TICK_INTERVAL.as_secs()).unwrap_or(u32::MAX);
|
||||
1_u32
|
||||
.checked_shl(attempts.saturating_sub(1))
|
||||
.unwrap_or(u32::MAX)
|
||||
.min(cap)
|
||||
}
|
||||
|
||||
/// The decision for one agent. Pure but for the backoff counters it advances.
|
||||
fn step(backoffs: &mut Backoffs, agent: &str, seen: Observation) -> Step {
|
||||
if seen.running == Some(true) || seen.wanted != Some(Wanted::Up) {
|
||||
backoffs.settle(agent);
|
||||
return Step::Leave(Leave::Converged);
|
||||
}
|
||||
if seen.running.is_none() || seen.paused || seen.updating {
|
||||
return Step::Leave(Leave::Held);
|
||||
}
|
||||
if !backoffs.ready(agent) {
|
||||
return Step::Leave(Leave::BackingOff);
|
||||
}
|
||||
Step::Restart
|
||||
}
|
||||
|
||||
/// Run the tick until shutdown.
|
||||
pub fn spawn(coord: Arc<Coordinator>) {
|
||||
let mut shutdown = coord.shutdown_rx();
|
||||
tokio::spawn(async move {
|
||||
let mut backoffs = Backoffs::default();
|
||||
loop {
|
||||
tokio::select! {
|
||||
() = tokio::time::sleep(TICK_INTERVAL) => {}
|
||||
_ = shutdown.changed() => {
|
||||
tracing::info!("reconcile tick: shutdown signal received");
|
||||
return;
|
||||
}
|
||||
}
|
||||
tick(&coord, &mut backoffs).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// One pass over every agent container.
|
||||
async fn tick(coord: &Arc<Coordinator>, backoffs: &mut Backoffs) {
|
||||
let containers = match lifecycle::list().await {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "reconcile tick: container list unreadable — skipping this tick");
|
||||
return;
|
||||
}
|
||||
};
|
||||
for container in &containers {
|
||||
let Some(name) = container.strip_prefix(AGENT_PREFIX) else {
|
||||
continue;
|
||||
};
|
||||
let seen = Observation {
|
||||
wanted: coord.power.get(name),
|
||||
running: lifecycle::running_state(name).await,
|
||||
paused: hive_types::Ident::parse(name).is_ok_and(|id| Coordinator::is_paused(&id)),
|
||||
updating: coord.is_updating(name),
|
||||
};
|
||||
if step(backoffs, name, seen) != Step::Restart {
|
||||
continue;
|
||||
}
|
||||
match coord.job_queue.insert_reconcile_if_idle(name) {
|
||||
Ok(Some(_)) => {
|
||||
let attempt = backoffs.record_attempt(name);
|
||||
tracing::info!(
|
||||
agent = %name,
|
||||
observed = "stopped",
|
||||
wanted = "up",
|
||||
attempt,
|
||||
"reconcile tick: restarting agent"
|
||||
);
|
||||
crate::otel_metrics::count_reconcile_restart();
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
// Work targeting the agent is in flight; it converges the agent or
|
||||
// decides not to.
|
||||
Ok(None) => {
|
||||
tracing::debug!(agent = %name, "reconcile tick: agent is busy; leaving it");
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(agent = %name, error = ?e, "reconcile tick: queueing the reconcile failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A wanted-up agent whose container is down, nothing holding it back.
|
||||
const DOWN: Observation = Observation {
|
||||
wanted: Some(Wanted::Up),
|
||||
running: Some(false),
|
||||
paused: false,
|
||||
updating: false,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn a_wanted_up_stopped_agent_is_restarted() {
|
||||
assert_eq!(step(&mut Backoffs::default(), "a", DOWN), Step::Restart);
|
||||
}
|
||||
|
||||
/// Each deviation from [`DOWN`] that must leave the agent alone, against
|
||||
/// the control above that the same agent is restarted without it.
|
||||
#[test]
|
||||
fn the_tick_leaves_alone_what_is_not_a_plain_down_agent() {
|
||||
let cases = [
|
||||
(
|
||||
"wanted offline",
|
||||
Observation {
|
||||
wanted: Some(Wanted::Offline),
|
||||
..DOWN
|
||||
},
|
||||
Leave::Converged,
|
||||
),
|
||||
(
|
||||
"no intent yet",
|
||||
Observation {
|
||||
wanted: None,
|
||||
..DOWN
|
||||
},
|
||||
Leave::Converged,
|
||||
),
|
||||
(
|
||||
"already running",
|
||||
Observation {
|
||||
running: Some(true),
|
||||
..DOWN
|
||||
},
|
||||
Leave::Converged,
|
||||
),
|
||||
(
|
||||
"paused",
|
||||
Observation {
|
||||
paused: true,
|
||||
..DOWN
|
||||
},
|
||||
Leave::Held,
|
||||
),
|
||||
(
|
||||
"mid-update",
|
||||
Observation {
|
||||
updating: true,
|
||||
..DOWN
|
||||
},
|
||||
Leave::Held,
|
||||
),
|
||||
(
|
||||
"container state unreadable",
|
||||
Observation {
|
||||
running: None,
|
||||
..DOWN
|
||||
},
|
||||
Leave::Held,
|
||||
),
|
||||
];
|
||||
for (what, seen, why) in cases {
|
||||
assert_eq!(
|
||||
step(&mut Backoffs::default(), "a", seen),
|
||||
Step::Leave(why),
|
||||
"{what}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Ticks between restarts of an agent that never comes up: 1, 2, 4, 8,
|
||||
/// then the 15-tick cap, and it stays there.
|
||||
#[test]
|
||||
fn restarts_of_a_failing_agent_back_off_to_the_cap() {
|
||||
let mut backoffs = Backoffs::default();
|
||||
let mut gaps = Vec::new();
|
||||
let mut since = 0_u32;
|
||||
for _ in 0..80 {
|
||||
since += 1;
|
||||
if step(&mut backoffs, "a", DOWN) == Step::Restart {
|
||||
backoffs.record_attempt("a");
|
||||
gaps.push(since);
|
||||
since = 0;
|
||||
}
|
||||
}
|
||||
assert_eq!(gaps[..6], [1, 1, 2, 4, 8, 15]);
|
||||
assert!(gaps[5..].iter().all(|g| *g == 15), "{gaps:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_wait_is_a_backoff_not_a_refusal() {
|
||||
let mut backoffs = Backoffs::default();
|
||||
backoffs.record_attempt("a");
|
||||
backoffs.record_attempt("a");
|
||||
assert_eq!(
|
||||
step(&mut backoffs, "a", DOWN),
|
||||
Step::Leave(Leave::BackingOff)
|
||||
);
|
||||
assert_eq!(step(&mut backoffs, "a", DOWN), Step::Restart);
|
||||
// Another agent's history is not this one's.
|
||||
assert_eq!(step(&mut backoffs, "b", DOWN), Step::Restart);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seeing_the_agent_running_resets_its_backoff() {
|
||||
let mut backoffs = Backoffs::default();
|
||||
for _ in 0..4 {
|
||||
backoffs.record_attempt("a");
|
||||
}
|
||||
assert_eq!(
|
||||
step(&mut backoffs, "a", DOWN),
|
||||
Step::Leave(Leave::BackingOff),
|
||||
"control: the history holds the agent back"
|
||||
);
|
||||
let up = Observation {
|
||||
running: Some(true),
|
||||
..DOWN
|
||||
};
|
||||
assert_eq!(step(&mut backoffs, "a", up), Step::Leave(Leave::Converged));
|
||||
assert_eq!(step(&mut backoffs, "a", DOWN), Step::Restart);
|
||||
assert_eq!(backoffs.record_attempt("a"), 1, "the count starts over");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_operator_stop_resets_the_backoff() {
|
||||
let mut backoffs = Backoffs::default();
|
||||
for _ in 0..4 {
|
||||
backoffs.record_attempt("a");
|
||||
}
|
||||
let stopped = Observation {
|
||||
wanted: Some(Wanted::Offline),
|
||||
..DOWN
|
||||
};
|
||||
step(&mut backoffs, "a", stopped);
|
||||
assert_eq!(step(&mut backoffs, "a", DOWN), Step::Restart);
|
||||
}
|
||||
|
||||
/// A refused restart (the agent turned out busy) does not count as an
|
||||
/// attempt, so it does not lengthen the next wait.
|
||||
#[test]
|
||||
fn only_a_recorded_attempt_lengthens_the_backoff() {
|
||||
let mut backoffs = Backoffs::default();
|
||||
assert_eq!(step(&mut backoffs, "a", DOWN), Step::Restart);
|
||||
assert_eq!(step(&mut backoffs, "a", DOWN), Step::Restart);
|
||||
assert_eq!(backoffs.record_attempt("a"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_cap_is_reached_without_overflow() {
|
||||
assert_eq!(ticks_between(1), 1);
|
||||
assert_eq!(ticks_between(4), 8);
|
||||
assert_eq!(ticks_between(5), 15);
|
||||
assert_eq!(ticks_between(u32::MAX), 15);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue