reconcile tick: create a placed agent whose deploy event was missed
The swarm's deploy event is core NATS. The wanted-state declaration lands before the build, so its create fails on no record, and only the event re-drives it; a hive that missed the event had nothing to retry. wanted.rs now remembers which agents the last declaration places with a container. Each tick, a placed agent with no container whose record says the builder pushed its closure gets its first deploy through JobQueue::insert_first_deploy_if_idle, which queues nothing while any node targets the agent. Record reads and creates back off like restarts, in a history of their own.
This commit is contained in:
parent
887ea26c56
commit
ad30b7ea31
5 changed files with 439 additions and 17 deletions
|
|
@ -98,6 +98,14 @@ pub struct RunningTransient {
|
|||
/// satisfy).
|
||||
type Sched = Scheduler<NodeKind, Resource>;
|
||||
|
||||
/// Whether any queued or running node targets `agent`.
|
||||
fn agent_busy(sched: &Sched, agent: &str) -> bool {
|
||||
sched
|
||||
.graph()
|
||||
.nodes()
|
||||
.any(|n| !n.state.is_terminal() && n.payload.agent() == agent)
|
||||
}
|
||||
|
||||
/// The queue. Lives on `Coordinator` (one per hive-c0re process); a single
|
||||
/// scheduler task ([`scheduler::run_worker`]) drives it.
|
||||
pub struct JobQueue {
|
||||
|
|
@ -224,6 +232,27 @@ impl JobQueue {
|
|||
Ok(Some(named))
|
||||
}
|
||||
|
||||
/// Queue `agent`'s first deploy unless any queued or running node targets
|
||||
/// it, by the same rule as [`Self::insert_reconcile_if_idle`].
|
||||
///
|
||||
/// `None` means the agent is busy: a first deploy, or any other work on it,
|
||||
/// is in flight. The check and the insert share one lock.
|
||||
///
|
||||
/// # Errors
|
||||
/// Propagates a graph-insert error.
|
||||
pub fn insert_first_deploy_if_idle(&self, agent: &str) -> anyhow::Result<Option<Vec<NodeId>>> {
|
||||
let mut inner = self.lock();
|
||||
if agent_busy(&inner, agent) {
|
||||
return Ok(None);
|
||||
}
|
||||
let named = inner
|
||||
.insert_job(None, |b| templates::first_deploy(b, agent))
|
||||
.map_err(|e| anyhow::anyhow!("job_queue: graph insert failed: {e}"))?;
|
||||
drop(inner);
|
||||
self.notify.notify_one();
|
||||
Ok(Some(named))
|
||||
}
|
||||
|
||||
/// Queue a lone `Reconcile` for `agent` unless any queued or running node
|
||||
/// already targets it.
|
||||
///
|
||||
|
|
@ -238,11 +267,7 @@ impl JobQueue {
|
|||
/// Propagates a graph-insert error.
|
||||
pub fn insert_reconcile_if_idle(&self, agent: &str) -> anyhow::Result<Option<Vec<NodeId>>> {
|
||||
let mut inner = self.lock();
|
||||
let busy = inner
|
||||
.graph()
|
||||
.nodes()
|
||||
.any(|n| !n.state.is_terminal() && n.payload.agent() == agent);
|
||||
if busy {
|
||||
if agent_busy(&inner, agent) {
|
||||
return Ok(None);
|
||||
}
|
||||
let targets = [(agent.to_owned(), false)];
|
||||
|
|
|
|||
|
|
@ -1541,6 +1541,39 @@ fn a_first_deploy_is_not_queued_twice_for_one_agent() {
|
|||
);
|
||||
}
|
||||
|
||||
/// The reconcile tick's create: not behind any live work on the agent, a
|
||||
/// rebuild included, and a finished node does not count.
|
||||
#[test]
|
||||
fn a_first_deploy_if_idle_waits_for_any_live_work_on_the_agent() {
|
||||
let q = JobQueue::new(1);
|
||||
insert(&q, |b| {
|
||||
let _ = rebuild(b, "busy");
|
||||
});
|
||||
assert!(
|
||||
q.insert_first_deploy_if_idle("busy")
|
||||
.expect("insert")
|
||||
.is_none(),
|
||||
"a live rebuild blocks it"
|
||||
);
|
||||
let first = q
|
||||
.insert_first_deploy_if_idle("newbie")
|
||||
.expect("insert")
|
||||
.expect("control: an idle agent gets its first deploy");
|
||||
assert!(
|
||||
q.insert_first_deploy_if_idle("newbie")
|
||||
.expect("insert")
|
||||
.is_none(),
|
||||
"the live first deploy blocks another"
|
||||
);
|
||||
assert!(q.cancel(first[0].get()), "cancel the queued first deploy");
|
||||
assert!(
|
||||
q.insert_first_deploy_if_idle("newbie")
|
||||
.expect("insert")
|
||||
.is_some(),
|
||||
"a terminal node does not count as busy"
|
||||
);
|
||||
}
|
||||
|
||||
/// The auto-update pass queues no second rebuild for an agent that has one in
|
||||
/// flight; another agent's rebuild does not count.
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -191,9 +191,9 @@ pub fn spawn(
|
|||
/// daemon pulls at startup regardless, and the webhook it replaced was lost
|
||||
/// identically when a hive was down.
|
||||
///
|
||||
/// The deploy event's second path is [`crate::workers::wanted`], which reads
|
||||
/// the whole declared set at boot and creates the agents this hive lacks — so
|
||||
/// a missed **first** deploy repairs itself. A missed **rebuild** does not:
|
||||
/// A missed **first** deploy repairs itself: [`crate::workers::wanted`] creates
|
||||
/// a declared agent this hive lacks at boot, [`crate::workers::reconcile_tick`]
|
||||
/// once its build is pushed. A missed **rebuild** does not:
|
||||
/// the declaration names agents, not revisions, so that falls to the boot
|
||||
/// reconcile noticing drift. Still not `JetStream`: durability on one subject
|
||||
/// looks like a fix while the desired state lives only in a message.
|
||||
|
|
@ -373,6 +373,18 @@ pub(crate) fn queue_first_deploy(
|
|||
coord.job_queue.insert_first_deploy_once(agent)
|
||||
}
|
||||
|
||||
/// [`queue_first_deploy`] for the reconcile tick, which acts on what it saw
|
||||
/// before this call: `Ok(None)` when any queued or running node targets the
|
||||
/// agent, not only a first deploy.
|
||||
pub(crate) fn queue_first_deploy_if_idle(
|
||||
power: &crate::power::PowerStore,
|
||||
queue: &crate::job_queue::JobQueue,
|
||||
agent: &str,
|
||||
) -> Result<Option<Vec<hive_jobq::NodeId>>> {
|
||||
power.set(agent, crate::power::Wanted::Up);
|
||||
queue.insert_first_deploy_if_idle(agent)
|
||||
}
|
||||
|
||||
/// The snapshot a hive offers: its readiness, and what it can do (`swarm_wire::hive_status`).
|
||||
#[derive(serde::Serialize)]
|
||||
struct Snapshot {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
//! Periodic declared ⇒ running convergence: an agent whose power intent is
|
||||
//! `Up` and whose container is down, or running with its harness unit stopped,
|
||||
//! gets a `Reconcile` queued, once a [`TICK_INTERVAL`], without anyone having
|
||||
//! to ask.
|
||||
//! to ask. A declared agent with no container gets its first deploy instead,
|
||||
//! once its build is pushed ([`create_placed`]).
|
||||
//!
|
||||
//! The tick queues the same `NodeKind::Reconcile` the boot sweep, a watch
|
||||
//! attach and a declaration write end in, so what it starts, and how, is
|
||||
|
|
@ -27,13 +28,14 @@
|
|||
//! apart, capped at [`MAX_BACKOFF`]. Seeing the agent running, or no longer
|
||||
//! wanted up, forgets its history.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
use crate::job_queue::JobQueue;
|
||||
use crate::lifecycle::{self, AGENT_PREFIX, HarnessState};
|
||||
use crate::power::Wanted;
|
||||
use crate::power::{PowerStore, Wanted};
|
||||
|
||||
/// How often declared intent is compared with the observed containers. The
|
||||
/// first tick fires one interval after boot: the boot sweep owns startup.
|
||||
|
|
@ -146,11 +148,87 @@ fn step(backoffs: &mut Backoffs, agent: &str, seen: Observation) -> Step {
|
|||
Step::Restart
|
||||
}
|
||||
|
||||
/// Whether the swarm has a build an agent's container can be created from.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
enum Build {
|
||||
/// The builder recorded a build and pushed its closure to the cache.
|
||||
Pushed,
|
||||
/// Nothing to create from yet, and why.
|
||||
Waiting(String),
|
||||
}
|
||||
|
||||
/// `agent`'s build, from its record in `agent-closures`.
|
||||
async fn build_of(agent: &str) -> Build {
|
||||
match crate::attic::closure::record(agent).await {
|
||||
Ok(Some(record)) if record.cached => Build::Pushed,
|
||||
Ok(Some(_)) => Build::Waiting("the builder did not push its closure".to_owned()),
|
||||
Ok(None) => Build::Waiting("the swarm has recorded no build of it".to_owned()),
|
||||
Err(e) => Build::Waiting(format!("its build record is unreadable: {e:#}")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Queue the first deploy of each `placed` agent with no container, once
|
||||
/// `build` says the swarm pushed a build of it. Returns whether any was queued.
|
||||
/// The deploy event normally queues that create; it is core NATS, so a hive can
|
||||
/// miss it.
|
||||
///
|
||||
/// A create queued, or a build not pushed yet, counts as an attempt in
|
||||
/// `backoffs`, spaced like restarts; an agent busy with other work does not.
|
||||
/// An agent with a container, or no longer placed, loses its history.
|
||||
///
|
||||
/// `present` is read before `build` is awaited: a create finishing in that
|
||||
/// window is not seen, and the create queued here fails, since
|
||||
/// `nixos-container create` refuses an existing container.
|
||||
async fn create_placed(
|
||||
queue: &JobQueue,
|
||||
power: &PowerStore,
|
||||
backoffs: &mut Backoffs,
|
||||
placed: &[String],
|
||||
present: &BTreeSet<&str>,
|
||||
build: impl AsyncFn(&str) -> Build,
|
||||
) -> bool {
|
||||
backoffs
|
||||
.agents
|
||||
.retain(|agent, _| placed.contains(agent) && !present.contains(agent.as_str()));
|
||||
let mut queued = false;
|
||||
for agent in placed {
|
||||
if present.contains(agent.as_str()) || !backoffs.ready(agent) {
|
||||
continue;
|
||||
}
|
||||
if let Build::Waiting(reason) = build(agent).await {
|
||||
let attempt = backoffs.record_attempt(agent);
|
||||
tracing::info!(
|
||||
%agent, observed = "no container", %reason, attempt,
|
||||
"reconcile tick: placed agent has no build to create from yet"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
match crate::swarm_status::queue_first_deploy_if_idle(power, queue, agent) {
|
||||
Ok(Some(_)) => {
|
||||
let attempt = backoffs.record_attempt(agent);
|
||||
tracing::info!(
|
||||
%agent, observed = "no container", wanted = "up", attempt,
|
||||
"reconcile tick: creating placed agent"
|
||||
);
|
||||
queued = true;
|
||||
}
|
||||
Ok(None) => {
|
||||
tracing::debug!(%agent, "reconcile tick: placed agent is busy; leaving it");
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(%agent, error = ?e, "reconcile tick: queueing the create failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
queued
|
||||
}
|
||||
|
||||
/// Run the tick until shutdown.
|
||||
pub fn spawn(coord: Arc<Coordinator>) {
|
||||
let mut shutdown = coord.shutdown_rx();
|
||||
tokio::spawn(async move {
|
||||
let mut backoffs = Backoffs::default();
|
||||
let mut creates = Backoffs::default();
|
||||
loop {
|
||||
tokio::select! {
|
||||
() = tokio::time::sleep(TICK_INTERVAL) => {}
|
||||
|
|
@ -159,13 +237,14 @@ pub fn spawn(coord: Arc<Coordinator>) {
|
|||
return;
|
||||
}
|
||||
}
|
||||
tick(&coord, &mut backoffs).await;
|
||||
tick(&coord, &mut backoffs, &mut creates).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// One pass over every agent container.
|
||||
async fn tick(coord: &Arc<Coordinator>, backoffs: &mut Backoffs) {
|
||||
/// One pass over the placed agents with no container, then over every agent
|
||||
/// container.
|
||||
async fn tick(coord: &Arc<Coordinator>, backoffs: &mut Backoffs, creates: &mut Backoffs) {
|
||||
let containers = match lifecycle::list().await {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
|
|
@ -173,6 +252,23 @@ async fn tick(coord: &Arc<Coordinator>, backoffs: &mut Backoffs) {
|
|||
return;
|
||||
}
|
||||
};
|
||||
let present: BTreeSet<&str> = containers
|
||||
.iter()
|
||||
.filter_map(|c| c.strip_prefix(AGENT_PREFIX))
|
||||
.collect();
|
||||
let placed = crate::workers::wanted::placed();
|
||||
if create_placed(
|
||||
&coord.job_queue,
|
||||
&coord.power,
|
||||
creates,
|
||||
&placed,
|
||||
&present,
|
||||
build_of,
|
||||
)
|
||||
.await
|
||||
{
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
for container in &containers {
|
||||
let Some(name) = container.strip_prefix(AGENT_PREFIX) else {
|
||||
continue;
|
||||
|
|
@ -439,4 +535,192 @@ mod tests {
|
|||
assert_eq!(ticks_between(5), 15);
|
||||
assert_eq!(ticks_between(u32::MAX), 15);
|
||||
}
|
||||
|
||||
/// The first-deploy nodes queued for `agent`.
|
||||
fn provisions(queue: &JobQueue, agent: &str) -> usize {
|
||||
let sched = queue.sched().lock().expect("job_queue mutex poisoned");
|
||||
sched
|
||||
.graph()
|
||||
.nodes()
|
||||
.filter(|n| <&str>::from(&n.payload) == "provision" && n.payload.agent() == agent)
|
||||
.count()
|
||||
}
|
||||
|
||||
/// One create pass over `placed` with nothing present, every build reading
|
||||
/// as `build`; returns whether it queued anything and how many builds it read.
|
||||
async fn pass(
|
||||
queue: &JobQueue,
|
||||
power: &PowerStore,
|
||||
backoffs: &mut Backoffs,
|
||||
placed: &[&str],
|
||||
present: &[&str],
|
||||
build: &Build,
|
||||
) -> (bool, usize) {
|
||||
let placed: Vec<String> = placed.iter().map(|a| (*a).to_owned()).collect();
|
||||
let present: BTreeSet<&str> = present.iter().copied().collect();
|
||||
let reads = std::sync::atomic::AtomicUsize::new(0);
|
||||
let queued = create_placed(
|
||||
queue,
|
||||
power,
|
||||
backoffs,
|
||||
&placed,
|
||||
&present,
|
||||
async |_: &str| {
|
||||
reads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
build.clone()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
(queued, reads.into_inner())
|
||||
}
|
||||
|
||||
fn waiting() -> Build {
|
||||
Build::Waiting("the swarm has recorded no build of it".to_owned())
|
||||
}
|
||||
|
||||
/// A placed agent with no container and a pushed build — the deploy event
|
||||
/// that should have created it was missed — gets its first deploy, with
|
||||
/// its intent seeded `Up` for the tail `Reconcile`.
|
||||
#[tokio::test]
|
||||
async fn a_placed_agent_with_a_pushed_build_and_no_container_is_created() {
|
||||
let (queue, power) = (JobQueue::new(1), PowerStore::default());
|
||||
let mut backoffs = Backoffs::default();
|
||||
let (queued, _) = pass(
|
||||
&queue,
|
||||
&power,
|
||||
&mut backoffs,
|
||||
&["newbie"],
|
||||
&[],
|
||||
&Build::Pushed,
|
||||
)
|
||||
.await;
|
||||
assert!(queued);
|
||||
assert_eq!(provisions(&queue, "newbie"), 1);
|
||||
assert_eq!(power.get("newbie"), Some(Wanted::Up));
|
||||
}
|
||||
|
||||
/// No pushed build yet: nothing is queued, and the next reads of the
|
||||
/// record are spaced out like restarts.
|
||||
#[tokio::test]
|
||||
async fn a_placed_agent_without_a_pushed_build_backs_off_and_is_not_created() {
|
||||
let (queue, power) = (JobQueue::new(1), PowerStore::default());
|
||||
let mut backoffs = Backoffs::default();
|
||||
let build = waiting();
|
||||
let (queued, reads) = pass(&queue, &power, &mut backoffs, &["newbie"], &[], &build).await;
|
||||
assert!(!queued);
|
||||
assert_eq!(reads, 1, "control: the record was read");
|
||||
assert_eq!(provisions(&queue, "newbie"), 0);
|
||||
// Second attempt one tick later, then a skipped tick before the third.
|
||||
assert_eq!(
|
||||
pass(&queue, &power, &mut backoffs, &["newbie"], &[], &build).await,
|
||||
(false, 1)
|
||||
);
|
||||
assert_eq!(
|
||||
pass(&queue, &power, &mut backoffs, &["newbie"], &[], &build).await,
|
||||
(false, 0),
|
||||
"backing off: the record is not read"
|
||||
);
|
||||
assert_eq!(
|
||||
pass(
|
||||
&queue,
|
||||
&power,
|
||||
&mut backoffs,
|
||||
&["newbie"],
|
||||
&[],
|
||||
&Build::Pushed
|
||||
)
|
||||
.await,
|
||||
(true, 1),
|
||||
"the build lands; the next attempt creates"
|
||||
);
|
||||
assert_eq!(provisions(&queue, "newbie"), 1);
|
||||
}
|
||||
|
||||
/// Any live node targeting the agent — the deploy event's own create, or a
|
||||
/// create the tick queued earlier — means no second create.
|
||||
#[tokio::test]
|
||||
async fn a_placed_agent_with_live_work_is_not_created_again() {
|
||||
let (queue, power) = (JobQueue::new(1), PowerStore::default());
|
||||
let mut backoffs = Backoffs::default();
|
||||
assert!(
|
||||
queue
|
||||
.insert_first_deploy_once("newbie")
|
||||
.expect("insert")
|
||||
.is_some(),
|
||||
"the deploy event's create"
|
||||
);
|
||||
let (queued, reads) = pass(
|
||||
&queue,
|
||||
&power,
|
||||
&mut backoffs,
|
||||
&["newbie"],
|
||||
&[],
|
||||
&Build::Pushed,
|
||||
)
|
||||
.await;
|
||||
assert!(!queued);
|
||||
assert_eq!(reads, 1, "control: it got as far as the insert");
|
||||
assert_eq!(provisions(&queue, "newbie"), 1, "still the one create");
|
||||
assert_eq!(
|
||||
pass(
|
||||
&queue,
|
||||
&power,
|
||||
&mut backoffs,
|
||||
&["newbie"],
|
||||
&[],
|
||||
&Build::Pushed
|
||||
)
|
||||
.await,
|
||||
(false, 1),
|
||||
"a refused create is not an attempt, so the next tick tries again"
|
||||
);
|
||||
assert_eq!(provisions(&queue, "newbie"), 1);
|
||||
}
|
||||
|
||||
/// An agent with a container is left to the restart path, and one the
|
||||
/// declaration does not place is never looked at.
|
||||
#[tokio::test]
|
||||
async fn a_present_or_unplaced_agent_is_not_created() {
|
||||
let (queue, power) = (JobQueue::new(1), PowerStore::default());
|
||||
let mut backoffs = Backoffs::default();
|
||||
let (queued, reads) = pass(
|
||||
&queue,
|
||||
&power,
|
||||
&mut backoffs,
|
||||
&["here"],
|
||||
&["here", "local"],
|
||||
&Build::Pushed,
|
||||
)
|
||||
.await;
|
||||
assert!(!queued);
|
||||
assert_eq!(reads, 0);
|
||||
assert_eq!(provisions(&queue, "here") + provisions(&queue, "local"), 0);
|
||||
assert_eq!(power.get("here"), None);
|
||||
}
|
||||
|
||||
/// The create history is dropped once the agent has a container, or is no
|
||||
/// longer placed, so a later absence starts from the first attempt.
|
||||
#[tokio::test]
|
||||
async fn a_create_history_ends_with_a_container_or_the_placement() {
|
||||
let (queue, power) = (JobQueue::new(1), PowerStore::default());
|
||||
let build = waiting();
|
||||
for present in [&["newbie"][..], &[]] {
|
||||
let mut backoffs = Backoffs::default();
|
||||
for _ in 0..4 {
|
||||
pass(&queue, &power, &mut backoffs, &["newbie"], &[], &build).await;
|
||||
}
|
||||
assert_eq!(
|
||||
pass(&queue, &power, &mut backoffs, &["newbie"], &[], &build).await,
|
||||
(false, 0),
|
||||
"control: the history holds the agent back"
|
||||
);
|
||||
let placed: &[&str] = if present.is_empty() { &[] } else { &["newbie"] };
|
||||
pass(&queue, &power, &mut backoffs, placed, present, &build).await;
|
||||
assert_eq!(
|
||||
pass(&queue, &power, &mut backoffs, &["newbie"], &[], &build).await,
|
||||
(false, 1),
|
||||
"the record is read again at once"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,6 +52,39 @@ const FIRST_RETRY_DELAY: Duration = Duration::from_secs(5);
|
|||
/// The longest wait between two watch sessions.
|
||||
const MAX_RETRY_DELAY: Duration = Duration::from_mins(5);
|
||||
|
||||
/// The agents the declaration this hive last read places here with a container
|
||||
/// ([`placed_in`]); empty before the first read, and while there is no
|
||||
/// readable declaration.
|
||||
static PLACED: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(Vec::new());
|
||||
|
||||
/// Record the declaration just read, or that there is none to act on.
|
||||
fn remember(declared: Option<&HiveWanted>) {
|
||||
*PLACED
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner) = placed_in(declared);
|
||||
}
|
||||
|
||||
/// The agents the last declaration this hive read wants it to have a container
|
||||
/// for: the ones [`converge`] deploys when they are absent.
|
||||
#[must_use]
|
||||
pub fn placed() -> Vec<String> {
|
||||
PLACED
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn placed_in(declared: Option<&HiveWanted>) -> Vec<String> {
|
||||
declared.map_or_else(Vec::new, |declared| {
|
||||
declared
|
||||
.agents
|
||||
.iter()
|
||||
.filter(|(_, decl)| decide(decl.state, false, None) == Converge::Deploy)
|
||||
.map(|(agent, _)| agent.clone())
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
|
||||
/// Read this hive's declaration and converge the agents it names.
|
||||
///
|
||||
/// `Ok(())` covers three quiet cases that are **not** an empty declaration: no
|
||||
|
|
@ -99,6 +132,7 @@ async fn pull_with(
|
|||
.context("opening this hive's wanted-state bucket")?
|
||||
else {
|
||||
tracing::info!(%hive, "wanted state: no bucket yet; nothing has been declared");
|
||||
remember(None);
|
||||
return Ok(());
|
||||
};
|
||||
let Some(raw) = store
|
||||
|
|
@ -107,10 +141,12 @@ async fn pull_with(
|
|||
.context("reading this hive's wanted state")?
|
||||
else {
|
||||
tracing::info!(%hive, "wanted state: no key for this hive; nothing has been declared");
|
||||
remember(None);
|
||||
return Ok(());
|
||||
};
|
||||
let declared: HiveWanted =
|
||||
serde_json::from_slice(&raw).context("decoding this hive's wanted state")?;
|
||||
let declared: HiveWanted = serde_json::from_slice(&raw)
|
||||
.inspect_err(|_| remember(None))
|
||||
.context("decoding this hive's wanted state")?;
|
||||
|
||||
converge(coord, &declared).await
|
||||
}
|
||||
|
|
@ -296,6 +332,7 @@ async fn apply_entry(
|
|||
) {
|
||||
if !carries_a_declaration(entry.operation) {
|
||||
tracing::info!(%hive, "wanted state: declaration withdrawn; nothing to converge");
|
||||
remember(None);
|
||||
return;
|
||||
}
|
||||
let declared: HiveWanted = match serde_json::from_slice(&entry.value) {
|
||||
|
|
@ -305,6 +342,7 @@ async fn apply_entry(
|
|||
// watch must keep running to pick up the next, possibly good, value.
|
||||
Err(e) => {
|
||||
tracing::warn!(%hive, error = %e, "wanted state: undecodable declaration");
|
||||
remember(None);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
|
@ -330,6 +368,7 @@ fn carries_a_declaration(operation: async_nats::jetstream::kv::Operation) -> boo
|
|||
|
||||
/// Queue whatever the declaration asks for and this hive is not already doing.
|
||||
async fn converge(coord: &Arc<Coordinator>, declared: &HiveWanted) -> Result<()> {
|
||||
remember(Some(declared));
|
||||
// Fail closed, for the reason the deploy event's own arm gives: without
|
||||
// the list, "this hive does not have that agent" cannot be told from
|
||||
// "this hive cannot see its agents", and acting on the guess creates a
|
||||
|
|
@ -586,7 +625,9 @@ fn decide_pause(state: AgentState, present: bool, currently_paused: bool) -> Pau
|
|||
mod tests {
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use super::{Converge, PauseConverge, Plan, carries_a_declaration, decide, decide_pause, plan};
|
||||
use super::{
|
||||
Converge, PauseConverge, Plan, carries_a_declaration, decide, decide_pause, placed_in, plan,
|
||||
};
|
||||
use crate::power::Wanted;
|
||||
use swarm_wire::wanted::{AgentState, AgentWanted, HiveWanted};
|
||||
|
||||
|
|
@ -698,6 +739,20 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
/// The agents the reconcile tick may create are exactly the ones `converge`
|
||||
/// deploys when absent; no declaration places none.
|
||||
#[test]
|
||||
fn the_placed_agents_are_the_ones_converge_deploys_when_absent() {
|
||||
let declared = declaration(&[
|
||||
("up", AgentState::Up),
|
||||
("paused", AgentState::Paused),
|
||||
("offline", AgentState::Offline),
|
||||
("destroyed", AgentState::Destroyed),
|
||||
]);
|
||||
assert_eq!(placed_in(Some(&declared)), ["paused", "up"]);
|
||||
assert!(placed_in(None).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_hive_that_already_agrees_queues_nothing() {
|
||||
assert_eq!(
|
||||
|
|
|
|||
Loading…
Reference in a new issue