From ad30b7ea313480577e6391a0523d52e68f2bc63f Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 19 Nov 2026 11:54:39 +0100 Subject: [PATCH] reconcile tick: create a placed agent whose deploy event was missed The swarm's deploy event is core NATS. The wanted-state declaration lands before the build, so its create fails on no record, and only the event re-drives it; a hive that missed the event had nothing to retry. wanted.rs now remembers which agents the last declaration places with a container. Each tick, a placed agent with no container whose record says the builder pushed its closure gets its first deploy through JobQueue::insert_first_deploy_if_idle, which queues nothing while any node targets the agent. Record reads and creates back off like restarts, in a history of their own. --- .../4-implementation/scheduler/coordinator.md | 13 + hive-c0re/src/job_queue/mod.rs | 35 ++- hive-c0re/src/job_queue/tests.rs | 33 ++ hive-c0re/src/swarm_status.rs | 18 +- hive-c0re/src/workers/reconcile_tick.rs | 296 +++++++++++++++++- hive-c0re/src/workers/wanted.rs | 61 +++- 6 files changed, 439 insertions(+), 17 deletions(-) diff --git a/docs/4-implementation/scheduler/coordinator.md b/docs/4-implementation/scheduler/coordinator.md index e1383807e..fe2b7d7cc 100644 --- a/docs/4-implementation/scheduler/coordinator.md +++ b/docs/4-implementation/scheduler/coordinator.md @@ -352,6 +352,19 @@ the agent, `observed=stopped` (`harness stopped` for a running container), `want `hyperhive.hive.reconcile_tick.restarts`. `crash_watch` keeps raising its alert; the tick only converges. +The tick also creates a placed agent that has no container: one the swarm's +declaration wants up or paused on this hive (`wanted::placed`) and that +`nixos-container list` doesn't show. The swarm's deploy event normally queues +that create, but it travels over core NATS, so a hive can miss it. Once the +agent's record in `agent-closures` says the builder pushed its closure, the tick +queues the first deploy through `JobQueue::insert_first_deploy_if_idle`, which +queues nothing while any node targets the agent, and sets the agent's intent to +`Up` as the event does. A missing or unpushed build counts as an attempt, like a +queued create, so one agent's record reads and creates follow the restart +spacing, in a history of their own. Each create logs `reconcile tick: creating +placed agent`; each wait logs `reconcile tick: placed agent has no build to +create from yet` with the reason. + ## Meta flake Agents build from the swarm's `internal/meta` flake, which swarm-controller diff --git a/hive-c0re/src/job_queue/mod.rs b/hive-c0re/src/job_queue/mod.rs index 607526654..bc8bd1fec 100644 --- a/hive-c0re/src/job_queue/mod.rs +++ b/hive-c0re/src/job_queue/mod.rs @@ -98,6 +98,14 @@ pub struct RunningTransient { /// satisfy). type Sched = Scheduler; +/// Whether any queued or running node targets `agent`. +fn agent_busy(sched: &Sched, agent: &str) -> bool { + sched + .graph() + .nodes() + .any(|n| !n.state.is_terminal() && n.payload.agent() == agent) +} + /// The queue. Lives on `Coordinator` (one per hive-c0re process); a single /// scheduler task ([`scheduler::run_worker`]) drives it. pub struct JobQueue { @@ -224,6 +232,27 @@ impl JobQueue { Ok(Some(named)) } + /// Queue `agent`'s first deploy unless any queued or running node targets + /// it, by the same rule as [`Self::insert_reconcile_if_idle`]. + /// + /// `None` means the agent is busy: a first deploy, or any other work on it, + /// is in flight. The check and the insert share one lock. + /// + /// # Errors + /// Propagates a graph-insert error. + pub fn insert_first_deploy_if_idle(&self, agent: &str) -> anyhow::Result>> { + let mut inner = self.lock(); + if agent_busy(&inner, agent) { + return Ok(None); + } + let named = inner + .insert_job(None, |b| templates::first_deploy(b, agent)) + .map_err(|e| anyhow::anyhow!("job_queue: graph insert failed: {e}"))?; + drop(inner); + self.notify.notify_one(); + Ok(Some(named)) + } + /// Queue a lone `Reconcile` for `agent` unless any queued or running node /// already targets it. /// @@ -238,11 +267,7 @@ impl JobQueue { /// Propagates a graph-insert error. pub fn insert_reconcile_if_idle(&self, agent: &str) -> anyhow::Result>> { let mut inner = self.lock(); - let busy = inner - .graph() - .nodes() - .any(|n| !n.state.is_terminal() && n.payload.agent() == agent); - if busy { + if agent_busy(&inner, agent) { return Ok(None); } let targets = [(agent.to_owned(), false)]; diff --git a/hive-c0re/src/job_queue/tests.rs b/hive-c0re/src/job_queue/tests.rs index 4c2625b17..522e21aa3 100644 --- a/hive-c0re/src/job_queue/tests.rs +++ b/hive-c0re/src/job_queue/tests.rs @@ -1541,6 +1541,39 @@ fn a_first_deploy_is_not_queued_twice_for_one_agent() { ); } +/// The reconcile tick's create: not behind any live work on the agent, a +/// rebuild included, and a finished node does not count. +#[test] +fn a_first_deploy_if_idle_waits_for_any_live_work_on_the_agent() { + let q = JobQueue::new(1); + insert(&q, |b| { + let _ = rebuild(b, "busy"); + }); + assert!( + q.insert_first_deploy_if_idle("busy") + .expect("insert") + .is_none(), + "a live rebuild blocks it" + ); + let first = q + .insert_first_deploy_if_idle("newbie") + .expect("insert") + .expect("control: an idle agent gets its first deploy"); + assert!( + q.insert_first_deploy_if_idle("newbie") + .expect("insert") + .is_none(), + "the live first deploy blocks another" + ); + assert!(q.cancel(first[0].get()), "cancel the queued first deploy"); + assert!( + q.insert_first_deploy_if_idle("newbie") + .expect("insert") + .is_some(), + "a terminal node does not count as busy" + ); +} + /// The auto-update pass queues no second rebuild for an agent that has one in /// flight; another agent's rebuild does not count. #[test] diff --git a/hive-c0re/src/swarm_status.rs b/hive-c0re/src/swarm_status.rs index ca46ae106..8573211d4 100644 --- a/hive-c0re/src/swarm_status.rs +++ b/hive-c0re/src/swarm_status.rs @@ -191,9 +191,9 @@ pub fn spawn( /// daemon pulls at startup regardless, and the webhook it replaced was lost /// identically when a hive was down. /// -/// The deploy event's second path is [`crate::workers::wanted`], which reads -/// the whole declared set at boot and creates the agents this hive lacks — so -/// a missed **first** deploy repairs itself. A missed **rebuild** does not: +/// A missed **first** deploy repairs itself: [`crate::workers::wanted`] creates +/// a declared agent this hive lacks at boot, [`crate::workers::reconcile_tick`] +/// once its build is pushed. A missed **rebuild** does not: /// the declaration names agents, not revisions, so that falls to the boot /// reconcile noticing drift. Still not `JetStream`: durability on one subject /// looks like a fix while the desired state lives only in a message. @@ -373,6 +373,18 @@ pub(crate) fn queue_first_deploy( coord.job_queue.insert_first_deploy_once(agent) } +/// [`queue_first_deploy`] for the reconcile tick, which acts on what it saw +/// before this call: `Ok(None)` when any queued or running node targets the +/// agent, not only a first deploy. +pub(crate) fn queue_first_deploy_if_idle( + power: &crate::power::PowerStore, + queue: &crate::job_queue::JobQueue, + agent: &str, +) -> Result>> { + power.set(agent, crate::power::Wanted::Up); + queue.insert_first_deploy_if_idle(agent) +} + /// The snapshot a hive offers: its readiness, and what it can do (`swarm_wire::hive_status`). #[derive(serde::Serialize)] struct Snapshot { diff --git a/hive-c0re/src/workers/reconcile_tick.rs b/hive-c0re/src/workers/reconcile_tick.rs index 6013534a4..860daac40 100644 --- a/hive-c0re/src/workers/reconcile_tick.rs +++ b/hive-c0re/src/workers/reconcile_tick.rs @@ -1,7 +1,8 @@ //! Periodic declared ⇒ running convergence: an agent whose power intent is //! `Up` and whose container is down, or running with its harness unit stopped, //! gets a `Reconcile` queued, once a [`TICK_INTERVAL`], without anyone having -//! to ask. +//! to ask. A declared agent with no container gets its first deploy instead, +//! once its build is pushed ([`create_placed`]). //! //! The tick queues the same `NodeKind::Reconcile` the boot sweep, a watch //! attach and a declaration write end in, so what it starts, and how, is @@ -27,13 +28,14 @@ //! apart, capped at [`MAX_BACKOFF`]. Seeing the agent running, or no longer //! wanted up, forgets its history. -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::sync::Arc; use std::time::Duration; use crate::coordinator::Coordinator; +use crate::job_queue::JobQueue; use crate::lifecycle::{self, AGENT_PREFIX, HarnessState}; -use crate::power::Wanted; +use crate::power::{PowerStore, Wanted}; /// How often declared intent is compared with the observed containers. The /// first tick fires one interval after boot: the boot sweep owns startup. @@ -146,11 +148,87 @@ fn step(backoffs: &mut Backoffs, agent: &str, seen: Observation) -> Step { Step::Restart } +/// Whether the swarm has a build an agent's container can be created from. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Build { + /// The builder recorded a build and pushed its closure to the cache. + Pushed, + /// Nothing to create from yet, and why. + Waiting(String), +} + +/// `agent`'s build, from its record in `agent-closures`. +async fn build_of(agent: &str) -> Build { + match crate::attic::closure::record(agent).await { + Ok(Some(record)) if record.cached => Build::Pushed, + Ok(Some(_)) => Build::Waiting("the builder did not push its closure".to_owned()), + Ok(None) => Build::Waiting("the swarm has recorded no build of it".to_owned()), + Err(e) => Build::Waiting(format!("its build record is unreadable: {e:#}")), + } +} + +/// Queue the first deploy of each `placed` agent with no container, once +/// `build` says the swarm pushed a build of it. Returns whether any was queued. +/// The deploy event normally queues that create; it is core NATS, so a hive can +/// miss it. +/// +/// A create queued, or a build not pushed yet, counts as an attempt in +/// `backoffs`, spaced like restarts; an agent busy with other work does not. +/// An agent with a container, or no longer placed, loses its history. +/// +/// `present` is read before `build` is awaited: a create finishing in that +/// window is not seen, and the create queued here fails, since +/// `nixos-container create` refuses an existing container. +async fn create_placed( + queue: &JobQueue, + power: &PowerStore, + backoffs: &mut Backoffs, + placed: &[String], + present: &BTreeSet<&str>, + build: impl AsyncFn(&str) -> Build, +) -> bool { + backoffs + .agents + .retain(|agent, _| placed.contains(agent) && !present.contains(agent.as_str())); + let mut queued = false; + for agent in placed { + if present.contains(agent.as_str()) || !backoffs.ready(agent) { + continue; + } + if let Build::Waiting(reason) = build(agent).await { + let attempt = backoffs.record_attempt(agent); + tracing::info!( + %agent, observed = "no container", %reason, attempt, + "reconcile tick: placed agent has no build to create from yet" + ); + continue; + } + match crate::swarm_status::queue_first_deploy_if_idle(power, queue, agent) { + Ok(Some(_)) => { + let attempt = backoffs.record_attempt(agent); + tracing::info!( + %agent, observed = "no container", wanted = "up", attempt, + "reconcile tick: creating placed agent" + ); + queued = true; + } + Ok(None) => { + tracing::debug!(%agent, "reconcile tick: placed agent is busy; leaving it"); + } + Err(e) => { + tracing::warn!(%agent, error = ?e, "reconcile tick: queueing the create failed"); + } + } + } + queued +} + /// Run the tick until shutdown. pub fn spawn(coord: Arc) { let mut shutdown = coord.shutdown_rx(); tokio::spawn(async move { let mut backoffs = Backoffs::default(); + let mut creates = Backoffs::default(); loop { tokio::select! { () = tokio::time::sleep(TICK_INTERVAL) => {} @@ -159,13 +237,14 @@ pub fn spawn(coord: Arc) { return; } } - tick(&coord, &mut backoffs).await; + tick(&coord, &mut backoffs, &mut creates).await; } }); } -/// One pass over every agent container. -async fn tick(coord: &Arc, backoffs: &mut Backoffs) { +/// One pass over the placed agents with no container, then over every agent +/// container. +async fn tick(coord: &Arc, backoffs: &mut Backoffs, creates: &mut Backoffs) { let containers = match lifecycle::list().await { Ok(c) => c, Err(e) => { @@ -173,6 +252,23 @@ async fn tick(coord: &Arc, backoffs: &mut Backoffs) { return; } }; + let present: BTreeSet<&str> = containers + .iter() + .filter_map(|c| c.strip_prefix(AGENT_PREFIX)) + .collect(); + let placed = crate::workers::wanted::placed(); + if create_placed( + &coord.job_queue, + &coord.power, + creates, + &placed, + &present, + build_of, + ) + .await + { + coord.emit_rebuild_queue_snapshot(); + } for container in &containers { let Some(name) = container.strip_prefix(AGENT_PREFIX) else { continue; @@ -439,4 +535,192 @@ mod tests { assert_eq!(ticks_between(5), 15); assert_eq!(ticks_between(u32::MAX), 15); } + + /// The first-deploy nodes queued for `agent`. + fn provisions(queue: &JobQueue, agent: &str) -> usize { + let sched = queue.sched().lock().expect("job_queue mutex poisoned"); + sched + .graph() + .nodes() + .filter(|n| <&str>::from(&n.payload) == "provision" && n.payload.agent() == agent) + .count() + } + + /// One create pass over `placed` with nothing present, every build reading + /// as `build`; returns whether it queued anything and how many builds it read. + async fn pass( + queue: &JobQueue, + power: &PowerStore, + backoffs: &mut Backoffs, + placed: &[&str], + present: &[&str], + build: &Build, + ) -> (bool, usize) { + let placed: Vec = placed.iter().map(|a| (*a).to_owned()).collect(); + let present: BTreeSet<&str> = present.iter().copied().collect(); + let reads = std::sync::atomic::AtomicUsize::new(0); + let queued = create_placed( + queue, + power, + backoffs, + &placed, + &present, + async |_: &str| { + reads.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + build.clone() + }, + ) + .await; + (queued, reads.into_inner()) + } + + fn waiting() -> Build { + Build::Waiting("the swarm has recorded no build of it".to_owned()) + } + + /// A placed agent with no container and a pushed build — the deploy event + /// that should have created it was missed — gets its first deploy, with + /// its intent seeded `Up` for the tail `Reconcile`. + #[tokio::test] + async fn a_placed_agent_with_a_pushed_build_and_no_container_is_created() { + let (queue, power) = (JobQueue::new(1), PowerStore::default()); + let mut backoffs = Backoffs::default(); + let (queued, _) = pass( + &queue, + &power, + &mut backoffs, + &["newbie"], + &[], + &Build::Pushed, + ) + .await; + assert!(queued); + assert_eq!(provisions(&queue, "newbie"), 1); + assert_eq!(power.get("newbie"), Some(Wanted::Up)); + } + + /// No pushed build yet: nothing is queued, and the next reads of the + /// record are spaced out like restarts. + #[tokio::test] + async fn a_placed_agent_without_a_pushed_build_backs_off_and_is_not_created() { + let (queue, power) = (JobQueue::new(1), PowerStore::default()); + let mut backoffs = Backoffs::default(); + let build = waiting(); + let (queued, reads) = pass(&queue, &power, &mut backoffs, &["newbie"], &[], &build).await; + assert!(!queued); + assert_eq!(reads, 1, "control: the record was read"); + assert_eq!(provisions(&queue, "newbie"), 0); + // Second attempt one tick later, then a skipped tick before the third. + assert_eq!( + pass(&queue, &power, &mut backoffs, &["newbie"], &[], &build).await, + (false, 1) + ); + assert_eq!( + pass(&queue, &power, &mut backoffs, &["newbie"], &[], &build).await, + (false, 0), + "backing off: the record is not read" + ); + assert_eq!( + pass( + &queue, + &power, + &mut backoffs, + &["newbie"], + &[], + &Build::Pushed + ) + .await, + (true, 1), + "the build lands; the next attempt creates" + ); + assert_eq!(provisions(&queue, "newbie"), 1); + } + + /// Any live node targeting the agent — the deploy event's own create, or a + /// create the tick queued earlier — means no second create. + #[tokio::test] + async fn a_placed_agent_with_live_work_is_not_created_again() { + let (queue, power) = (JobQueue::new(1), PowerStore::default()); + let mut backoffs = Backoffs::default(); + assert!( + queue + .insert_first_deploy_once("newbie") + .expect("insert") + .is_some(), + "the deploy event's create" + ); + let (queued, reads) = pass( + &queue, + &power, + &mut backoffs, + &["newbie"], + &[], + &Build::Pushed, + ) + .await; + assert!(!queued); + assert_eq!(reads, 1, "control: it got as far as the insert"); + assert_eq!(provisions(&queue, "newbie"), 1, "still the one create"); + assert_eq!( + pass( + &queue, + &power, + &mut backoffs, + &["newbie"], + &[], + &Build::Pushed + ) + .await, + (false, 1), + "a refused create is not an attempt, so the next tick tries again" + ); + assert_eq!(provisions(&queue, "newbie"), 1); + } + + /// An agent with a container is left to the restart path, and one the + /// declaration does not place is never looked at. + #[tokio::test] + async fn a_present_or_unplaced_agent_is_not_created() { + let (queue, power) = (JobQueue::new(1), PowerStore::default()); + let mut backoffs = Backoffs::default(); + let (queued, reads) = pass( + &queue, + &power, + &mut backoffs, + &["here"], + &["here", "local"], + &Build::Pushed, + ) + .await; + assert!(!queued); + assert_eq!(reads, 0); + assert_eq!(provisions(&queue, "here") + provisions(&queue, "local"), 0); + assert_eq!(power.get("here"), None); + } + + /// The create history is dropped once the agent has a container, or is no + /// longer placed, so a later absence starts from the first attempt. + #[tokio::test] + async fn a_create_history_ends_with_a_container_or_the_placement() { + let (queue, power) = (JobQueue::new(1), PowerStore::default()); + let build = waiting(); + for present in [&["newbie"][..], &[]] { + let mut backoffs = Backoffs::default(); + for _ in 0..4 { + pass(&queue, &power, &mut backoffs, &["newbie"], &[], &build).await; + } + assert_eq!( + pass(&queue, &power, &mut backoffs, &["newbie"], &[], &build).await, + (false, 0), + "control: the history holds the agent back" + ); + let placed: &[&str] = if present.is_empty() { &[] } else { &["newbie"] }; + pass(&queue, &power, &mut backoffs, placed, present, &build).await; + assert_eq!( + pass(&queue, &power, &mut backoffs, &["newbie"], &[], &build).await, + (false, 1), + "the record is read again at once" + ); + } + } } diff --git a/hive-c0re/src/workers/wanted.rs b/hive-c0re/src/workers/wanted.rs index 47517766a..ba7abeb04 100644 --- a/hive-c0re/src/workers/wanted.rs +++ b/hive-c0re/src/workers/wanted.rs @@ -52,6 +52,39 @@ const FIRST_RETRY_DELAY: Duration = Duration::from_secs(5); /// The longest wait between two watch sessions. const MAX_RETRY_DELAY: Duration = Duration::from_mins(5); +/// The agents the declaration this hive last read places here with a container +/// ([`placed_in`]); empty before the first read, and while there is no +/// readable declaration. +static PLACED: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); + +/// Record the declaration just read, or that there is none to act on. +fn remember(declared: Option<&HiveWanted>) { + *PLACED + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = placed_in(declared); +} + +/// The agents the last declaration this hive read wants it to have a container +/// for: the ones [`converge`] deploys when they are absent. +#[must_use] +pub fn placed() -> Vec { + PLACED + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() +} + +fn placed_in(declared: Option<&HiveWanted>) -> Vec { + declared.map_or_else(Vec::new, |declared| { + declared + .agents + .iter() + .filter(|(_, decl)| decide(decl.state, false, None) == Converge::Deploy) + .map(|(agent, _)| agent.clone()) + .collect() + }) +} + /// Read this hive's declaration and converge the agents it names. /// /// `Ok(())` covers three quiet cases that are **not** an empty declaration: no @@ -99,6 +132,7 @@ async fn pull_with( .context("opening this hive's wanted-state bucket")? else { tracing::info!(%hive, "wanted state: no bucket yet; nothing has been declared"); + remember(None); return Ok(()); }; let Some(raw) = store @@ -107,10 +141,12 @@ async fn pull_with( .context("reading this hive's wanted state")? else { tracing::info!(%hive, "wanted state: no key for this hive; nothing has been declared"); + remember(None); return Ok(()); }; - let declared: HiveWanted = - serde_json::from_slice(&raw).context("decoding this hive's wanted state")?; + let declared: HiveWanted = serde_json::from_slice(&raw) + .inspect_err(|_| remember(None)) + .context("decoding this hive's wanted state")?; converge(coord, &declared).await } @@ -296,6 +332,7 @@ async fn apply_entry( ) { if !carries_a_declaration(entry.operation) { tracing::info!(%hive, "wanted state: declaration withdrawn; nothing to converge"); + remember(None); return; } let declared: HiveWanted = match serde_json::from_slice(&entry.value) { @@ -305,6 +342,7 @@ async fn apply_entry( // watch must keep running to pick up the next, possibly good, value. Err(e) => { tracing::warn!(%hive, error = %e, "wanted state: undecodable declaration"); + remember(None); return; } }; @@ -330,6 +368,7 @@ fn carries_a_declaration(operation: async_nats::jetstream::kv::Operation) -> boo /// Queue whatever the declaration asks for and this hive is not already doing. async fn converge(coord: &Arc, declared: &HiveWanted) -> Result<()> { + remember(Some(declared)); // Fail closed, for the reason the deploy event's own arm gives: without // the list, "this hive does not have that agent" cannot be told from // "this hive cannot see its agents", and acting on the guess creates a @@ -586,7 +625,9 @@ fn decide_pause(state: AgentState, present: bool, currently_paused: bool) -> Pau mod tests { use std::collections::{BTreeMap, BTreeSet}; - use super::{Converge, PauseConverge, Plan, carries_a_declaration, decide, decide_pause, plan}; + use super::{ + Converge, PauseConverge, Plan, carries_a_declaration, decide, decide_pause, placed_in, plan, + }; use crate::power::Wanted; use swarm_wire::wanted::{AgentState, AgentWanted, HiveWanted}; @@ -698,6 +739,20 @@ mod tests { ); } + /// The agents the reconcile tick may create are exactly the ones `converge` + /// deploys when absent; no declaration places none. + #[test] + fn the_placed_agents_are_the_ones_converge_deploys_when_absent() { + let declared = declaration(&[ + ("up", AgentState::Up), + ("paused", AgentState::Paused), + ("offline", AgentState::Offline), + ("destroyed", AgentState::Destroyed), + ]); + assert_eq!(placed_in(Some(&declared)), ["paused", "up"]); + assert!(placed_in(None).is_empty()); + } + #[test] fn a_hive_that_already_agrees_queues_nothing() { assert_eq!(