From 12e797e7e3c2d1920b5ace0cfa8c9abfc16c224c Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 19 Nov 2026 00:48:25 +0100 Subject: [PATCH] auto-update: a rebuild is live until its whole subtree is terminal rebuild_live checked only MetaSync, which finishes in 0.2 s, so the next 5-min pass queued a second rebuild while the first still held the stop, swap and start. It now covers MetaSync, AgentWindow, Prebuild, StopForUpdate, Swap and RebuildBookkeeping. AgentWindow is also the brace of a pause window, so an open pause skips that agent's pass too; the docs say so and a test pins it. --- .../4-implementation/scheduler/coordinator.md | 8 +++--- hive-c0re/src/job_queue/mod.rs | 8 +++--- hive-c0re/src/job_queue/model.rs | 18 +++++++++++++ hive-c0re/src/job_queue/scheduler.rs | 27 +++++++++++++++++++ hive-c0re/src/job_queue/tests.rs | 12 +++++++++ 5 files changed, 66 insertions(+), 7 deletions(-) diff --git a/docs/4-implementation/scheduler/coordinator.md b/docs/4-implementation/scheduler/coordinator.md index f91f14c5d..e1383807e 100644 --- a/docs/4-implementation/scheduler/coordinator.md +++ b/docs/4-implementation/scheduler/coordinator.md @@ -312,9 +312,11 @@ store path the agent's `agent-closures` record declares) and persisted Booting with every agent on its declared closure performs no rebuild — only reconciles. After the boot pass, `auto_update::run` repeats the rebuild half -every five minutes. It skips an agent with a rebuild waiting or running -(`JobQueue::rebuild_live`) and one whose declared store path this process -already attempted, so a closure that fails to deploy gets one try per process. +every five minutes. It skips an agent with a rebuild or a pause window waiting or +running (`JobQueue::rebuild_live`: any node from `MetaSync` to +`RebuildBookkeeping`, or an `AgentWindow`, still live) and one whose declared store path this +process already attempted, so a closure that fails to deploy gets one try per +process. The boot log line records the rebuild / deferred / up-to-date counts so the operator sees at a glance how much work the boot triggered. `auto_update::run` diff --git a/hive-c0re/src/job_queue/mod.rs b/hive-c0re/src/job_queue/mod.rs index 75b2ad21b..607526654 100644 --- a/hive-c0re/src/job_queue/mod.rs +++ b/hive-c0re/src/job_queue/mod.rs @@ -188,13 +188,13 @@ impl JobQueue { Ok(named) } - /// Whether a rebuild of `agent` is queued or running: its `MetaSync` is not - /// terminal yet. + /// Whether a rebuild of `agent` (or a pause window for it) is queued or + /// running: any node matching [`model::NodeKind::is_rebuild_step`] is not + /// terminal yet. The tail `Reconcile` is outside that set. #[must_use] pub fn rebuild_live(&self, agent: &str) -> bool { self.lock().graph().nodes().any(|n| { - !n.state.is_terminal() - && matches!(&n.payload, model::NodeKind::MetaSync { agent: a } if a == agent) + !n.state.is_terminal() && n.payload.is_rebuild_step() && n.payload.agent() == agent }) } diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs index 4e1b5c3cb..b7e3110dd 100644 --- a/hive-c0re/src/job_queue/model.rs +++ b/hive-c0re/src/job_queue/model.rs @@ -161,6 +161,24 @@ impl NodeKind { } } + /// Whether this node is a rebuild step: the preamble, the `AgentWindow` + /// brace and the steps under it. `AgentWindow` is also the brace of a pause + /// window (`templates::pause_quiesce`), so a pause counts as well. The tail + /// `Reconcile` and the quiesce nodes are not listed, because power DAGs + /// declare them too. + #[must_use] + pub fn is_rebuild_step(&self) -> bool { + matches!( + self, + NodeKind::MetaSync { .. } + | NodeKind::AgentWindow { .. } + | NodeKind::Prebuild { .. } + | NodeKind::StopForUpdate { .. } + | NodeKind::Swap { .. } + | NodeKind::RebuildBookkeeping { .. } + ) + } + /// Whether running this node is *expected* to take the agent's container /// down. Feeds `RunningTransient::takes_container_down`, which the crash watcher /// reads to tell an intentional stop from a crash. diff --git a/hive-c0re/src/job_queue/scheduler.rs b/hive-c0re/src/job_queue/scheduler.rs index 975b901e7..a3fd60659 100644 --- a/hive-c0re/src/job_queue/scheduler.rs +++ b/hive-c0re/src/job_queue/scheduler.rs @@ -268,4 +268,31 @@ mod tests { assert_eq!(running(&q), ["knowledge_pull"]); second.await; } + + /// A rebuild stays live for its agent after its `MetaSync` finishes, until + /// the swap steps behind it are done too. + #[tokio::test] + async fn a_rebuild_is_live_until_its_swap_steps_finish() { + let q = JobQueue::new(1); + q.insert_job(|b| templates::rebuild(b, "alice")) + .expect("insert"); + assert!(q.rebuild_live("alice"), "control: queued rebuild is live"); + + let meta_sync = claim(&q).expect("MetaSync is the first runnable node"); + assert_eq!(running(&q), ["meta_sync"]); + meta_sync.await; + assert!( + q.rebuild_live("alice"), + "MetaSync is done but the swap steps are pending" + ); + assert!(!q.rebuild_live("bob"), "another agent is not affected"); + + while let Some(node) = claim(&q) { + node.await; + } + assert!( + !q.rebuild_live("alice"), + "every node of the rebuild is terminal" + ); + } } diff --git a/hive-c0re/src/job_queue/tests.rs b/hive-c0re/src/job_queue/tests.rs index 489476bdc..4c2625b17 100644 --- a/hive-c0re/src/job_queue/tests.rs +++ b/hive-c0re/src/job_queue/tests.rs @@ -1556,6 +1556,18 @@ fn a_rebuild_is_live_for_its_own_agent_only() { assert!(!q.rebuild_live("bob")); } +/// An open pause window declares an `AgentWindow`, so it also holds the agent +/// live for the auto-update pass. +#[test] +fn a_pause_window_counts_as_a_live_rebuild() { + let q = JobQueue::new(1); + assert!(!q.rebuild_live("alice"), "control: nothing queued yet"); + q.insert_job(|b| vec![templates::pause_quiesce(b, "alice").guid()]) + .expect("insert"); + assert!(q.rebuild_live("alice")); + assert!(!q.rebuild_live("bob")); +} + /// The periodic reconcile tick queues one lone `Reconcile` on the agent's lease /// and nothing for an agent any live node already targets — whichever kind /// of work it is — while other agents and finished work do not block it.