restart preserves wanted intent instead of forcing all agents up (#2540)

docs/coordinator.md

@@ -75,21 +75,24 @@ container build:
 
 ### Every operation as a DAG
 
-The power ops write the durable `wanted` intent via a head `SetWanted`
+The `stop` / `start` power ops write the durable `wanted` intent via a head
-node (not a pre-submit side effect) — it holds the agent lease, so
+`SetWanted` node (not a pre-submit side effect) — it holds the agent lease,
-intent-write + reconcile is atomic per-agent. The hive-wide power ops —
+so intent-write + reconcile is atomic per-agent. `restart` is the exception:
-`restart`, `stop`, and `start` — take an agent *list*: a hive-wide `hivectl
+it writes *no* intent (no `SetWanted` head) — it bounces the container and
-restart` / `stop` / `start` is ONE DAG with a per-agent subgraph each
+lets the tail `Reconcile` converge to the agent's existing `wanted`, so a
-(independent roots, run concurrently on their own leases), not N separate
+deliberately-stopped agent is not forced back up by a hive-wide restart. The
-DAGs.
+hive-wide power ops — `restart`, `stop`, and `start` — take an agent *list*:
+a hive-wide `hivectl restart` / `stop` / `start` is ONE DAG with a per-agent
+subgraph each (independent roots, run concurrently on their own leases), not
+N separate DAGs.
 
 **These are built dynamically from each agent's live running state** (an
 async `lifecycle::is_running` read), so they live in `job_queue/submit.rs`,
-not the pure/sync `templates.rs`. Per-agent shape rule: the head `SetWanted`
+not the pure/sync `templates.rs`. Per-agent shape rule: `stop`/`start` carry
-(intent) and the tail `Reconcile` (convergence guarantee — cheap, noops when
+a head `SetWanted` (intent) — `restart` does not; the tail `Reconcile`
-already converged) are ALWAYS present; only the *mechanical* nodes
+(convergence guarantee — cheap, noops when already converged) is ALWAYS
-(`Signal`/`Drain`/`StopForUpdate`) are state-conditional — skipped for a
+present; only the *mechanical* nodes (`Signal`/`Drain`/`StopForUpdate`) are
-*down* agent (nothing to quiesce/stop). Keeping `Reconcile` in every shape
+state-conditional — skipped for a *down* agent (nothing to quiesce/stop). Keeping `Reconcile` in every shape
 closes the TOCTOU window: if an agent flips state between the `is_running`
 read and node exec, the tail `Reconcile` still converges it in-DAG (with
 `StopForUpdate`-noop as the backstop) — no reliance on an external reconcile
@@ -100,8 +103,8 @@ agent's subgraph is a rebuild-then-start).
 rebuild(a):        Prebuild(a) → StopForUpdate(a) → Swap(a) →(after-ok) PostSwap(a) →(after-any) Reconcile(a)
 stop(a..):     online a: SetWanted(a,Off) → [Signal→Drain→ if graceful] Reconcile(a)
                offline a: SetWanted(a,Off) → Reconcile(a)                    (N subgraphs, 1 DAG)
-restart(a..):  online a: SetWanted(a,Up) → [Signal→Drain→ if graceful] StopForUpdate(a) → Reconcile(a)
+restart(a..):  online a: [Signal→Drain→ if graceful] StopForUpdate(a) → Reconcile(a)  (no SetWanted)
-               offline a: SetWanted(a,Up) → Reconcile(a)  (nothing to stop — it's a start)
+               offline a: Reconcile(a)  (nothing to stop; Reconcile converges to existing wanted)
 start(a..):    a: SetWanted(a,Up) → Reconcile(a)   (down+stale ⇒ SetWanted(a,Up) → «rebuild subgraph»)
 spawn(a):          [wanted=Up at approve]  Create(a) → WriteDropin(a) → Reconcile(a)
 perm-change(a):    WritePermFile(a) → «rebuild subgraph»

hive-c0re/src/job_queue/submit.rs

@@ -9,9 +9,11 @@
 //! (`templates::{node, after_ok, rebuild_nodes}`), and concatenate them into
 //! ONE DAG (independent per-agent roots, concurrent on their own leases).
 //!
-//! Dynamic shape rule: the head `SetWanted(w)` (durable intent) and the tail
+//! Dynamic shape rule: `stop`/`start` carry a head `SetWanted(w)` (durable
-//! `Reconcile` (the convergence guarantee — cheap, noops when already
+//! intent write) — `restart` does NOT (it bounces the container but leaves
-//! converged) are ALWAYS present; only the *mechanical* nodes
+//! `wanted` untouched, so a deliberately-stopped agent isn't forced up). The
+//! tail `Reconcile` (the convergence guarantee — cheap, noops when already
+//! converged) is ALWAYS present; only the *mechanical* nodes
 //! (`Signal`/`Drain`/`StopForUpdate`) are state-conditional (skipped for a
 //! down agent — nothing to quiesce/stop). Keeping `Reconcile` in every shape
 //! closes the TOCTOU window: if an agent flips state between the `is_running`
@@ -85,26 +87,32 @@ fn start_chain(agent: &str, running: bool, stale: bool) -> Vec<NodeSpec> {
     n
 }
 
-/// One agent's **restart** subgraph. `SetWanted(Up)` head + `Reconcile`
+/// One agent's **restart** subgraph. Restart NEVER rewrites `wanted`
-/// tail; the stop portion (`Signal → Drain` when graceful, then
+/// intent (no `SetWanted` head, unlike stop/start): it bounces the
-/// `StopForUpdate`) only when the agent is running — a restart of a down
+/// container and lets the tail `Reconcile` converge to the agent's
-/// agent is really a start (`SetWanted(Up) → Reconcile`).
+/// EXISTING intent, so a deliberately-stopped (`wanted = Off`) agent is
+/// not forced back up by a hive-wide restart. A running agent gets the
+/// mechanical stop (`Signal → Drain` when graceful, then `StopForUpdate`)
+/// before `Reconcile`; a down agent gets just `Reconcile`, which
+/// converges to intent — a stopped (`wanted = Off`) agent stays stopped,
+/// a crashed (`wanted = Up`) agent comes back up.
 fn restart_chain(agent: &str, graceful: bool, running: bool) -> Vec<NodeSpec> {
-    let mut n = vec![node(agent, NodeKind::SetWanted { up: true }, Vec::new())];
+    if !running {
-    if running {
+        // Nothing to bounce — a lone Reconcile converges to intent.
-        let mut prev = 0u32;
+        return vec![node(agent, NodeKind::Reconcile, Vec::new())];
-        if graceful {
-            n.push(node(agent, NodeKind::Signal, after_ok(prev)));
-            prev += 1;
-            n.push(node(agent, NodeKind::Drain, after_ok(prev)));
-            prev += 1;
-        }
-        n.push(node(agent, NodeKind::StopForUpdate, after_ok(prev)));
-        prev += 1;
-        n.push(node(agent, NodeKind::Reconcile, after_ok(prev)));
-    } else {
-        n.push(node(agent, NodeKind::Reconcile, after_ok(0)));
     }
+    // Running: mechanical stop then Reconcile. The first stop node is the
+    // subgraph root (no SetWanted head) and acquires the agent lease.
+    let mut n = Vec::new();
+    if graceful {
+        n.push(node(agent, NodeKind::Signal, Vec::new()));
+        n.push(node(agent, NodeKind::Drain, after_ok(0)));
+        n.push(node(agent, NodeKind::StopForUpdate, after_ok(1)));
+    } else {
+        n.push(node(agent, NodeKind::StopForUpdate, Vec::new()));
+    }
+    let stop_idx = u32::try_from(n.len() - 1).unwrap_or(0);
+    n.push(node(agent, NodeKind::Reconcile, after_ok(stop_idx)));
     n
 }
 
@@ -258,9 +266,11 @@ pub async fn graceful_restart(
 /// Restart `agents` (one or many) in a **single** DAG — one per-agent
 /// subgraph each, built dynamically from live running state and run
 /// concurrently on their own leases. A running agent gets the stop→reconcile
-/// chain (`graceful` prepends signal→drain); a down agent gets just
+/// chain (`graceful` prepends signal→drain); a down agent gets just a lone
-/// `SetWanted(Up) → Reconcile` (there's nothing to stop). The whole
+/// `Reconcile` (nothing to stop). Restart never writes `wanted`, so the
-/// hive-wide `hivectl restart` / `restart-all` is one DAG.
+/// tail `Reconcile` converges each agent to its EXISTING intent — a
+/// deliberately-stopped agent stays down. The whole hive-wide
+/// `hivectl restart` / `restart-all` is one DAG.
 pub async fn restart_many(
     coord: &Arc<Coordinator>,
     agents: &[String],

hive-c0re/src/job_queue/templates.rs

@@ -11,9 +11,11 @@
 //! assembled dynamically in `submit.rs` out of the shared pure primitives
 //! this module exports (`node`, `after_ok`, `rebuild_nodes`) — one
 //! independent per-agent subgraph each, concurrent on its own lease, ONE
-//! DAG for the whole hive-wide op. Power ops write the durable `wanted`
+//! DAG for the whole hive-wide op. `stop`/`start` write the durable `wanted`
 //! intent via a head `SetWanted(w)` node (holding the agent lease, so
-//! intent+reconcile is atomic per-agent).
+//! intent+reconcile is atomic per-agent); `restart` writes no intent — it
+//! bounces the container and lets the tail `Reconcile` converge to the
+//! agent's existing `wanted`.
 //!
 //! ```text
 //! rebuild(a):        Prebuild(a) → StopForUpdate(a) → Swap(a) →(ok) PostSwap(a) →(any) Reconcile(a)

hive-c0re/src/job_queue/tests.rs

@@ -18,9 +18,9 @@ fn rebuild(agent: &str, reason: &str) -> DagSpec {
 }
 
 /// Restart DAG spec with every agent treated as **running** — the online
-/// shape (`SetWanted → [Signal→Drain→] StopForUpdate → Reconcile`) most
+/// shape (`[Signal→Drain→] StopForUpdate → Reconcile`, no `SetWanted` head)
-/// queue-mechanics tests assume. Mirrors the pre-dynamic `templates::restart`
+/// most queue-mechanics tests assume. Mirrors the pre-dynamic
-/// (which is now the state-aware `submit::restart_spec`).
+/// `templates::restart` (which is now the state-aware `submit::restart_spec`).
 fn restart_online(agents: &[&str], graceful: bool, reason: &str) -> DagSpec {
     let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
     submit::restart_spec(&targets, graceful, Source::Manual, reason.to_owned())
@@ -229,27 +229,23 @@ fn lease_serializes_two_lifecycle_dags_for_same_agent() {
             None,
         ),
     );
-    // Restart's head SetWanted takes the lease; stop's Reconcile must
+    // Restart's first node (StopForUpdate) takes the lease; stop's
-    // wait even though slots are free.
+    // Reconcile must wait even though slots are free.
     let first = claim_one(&q);
     assert_eq!(first.dag_id, restart);
-    assert_eq!(first.kind.as_str(), "set_wanted");
+    assert_eq!(first.kind.as_str(), "stop_for_update");
     assert!(first.lease_acquired);
     q.complete_node(restart, first.node_id, Ok(()));
-    // Same DAG keeps the lease through StopForUpdate then Reconcile.
+    // Same DAG keeps the lease through the tail Reconcile.
     let second = claim_one(&q);
     assert_eq!(second.dag_id, restart);
-    assert_eq!(second.kind.as_str(), "stop_for_update");
+    assert_eq!(second.kind.as_str(), "reconcile");
     assert!(!second.lease_acquired, "lease already held by this DAG");
     q.complete_node(restart, second.node_id, Ok(()));
-    let third = claim_one(&q);
-    assert_eq!(third.dag_id, restart);
-    assert_eq!(third.kind.as_str(), "reconcile");
-    q.complete_node(restart, third.node_id, Ok(()));
     // Restart terminal → lease released → stop's Reconcile runs.
-    let fourth = claim_one(&q);
+    let third = claim_one(&q);
-    assert_eq!(fourth.dag_id, stop);
+    assert_eq!(third.dag_id, stop);
-    q.complete_node(stop, fourth.node_id, Ok(()));
+    q.complete_node(stop, third.node_id, Ok(()));
     assert_eq!(state_of(&q, restart), State::Done);
     assert_eq!(state_of(&q, stop), State::Done);
 }
@@ -314,9 +310,9 @@ fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() {
     );
     // A hive-wide restart is ONE DAG, not one-per-agent.
     assert_eq!(q.snapshot().len(), 1);
-    // Each agent's subgraph head (SetWanted) is a root, so both are
+    // Each agent's subgraph head (StopForUpdate, since both are running) is
-    // claimable at once — each takes its OWN agent's lease (no contention
+    // a root, so both are claimable at once — each takes its OWN agent's
-    // across distinct agents), all inside the single DAG.
+    // lease (no contention across distinct agents), all inside the single DAG.
     let claims = q.claim_ready();
     assert!(claims.iter().all(|c| c.dag_id == id));
     let mut heads: Vec<(&str, &str, bool)> = claims
@@ -327,8 +323,8 @@ fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() {
     assert_eq!(
         heads,
         vec![
-            ("agent-a", "set_wanted", true),
+            ("agent-a", "stop_for_update", true),
-            ("agent-b", "set_wanted", true),
+            ("agent-b", "stop_for_update", true),
         ],
         "both per-agent subgraphs start concurrently, each acquiring its own lease"
     );
@@ -470,8 +466,9 @@ fn offline_agents_skip_mechanical_nodes_but_keep_reconcile() {
             "stop down".to_owned(),
         ),
     );
-    // Offline restart → SetWanted(Up) → Reconcile (no StopForUpdate): a
+    // Offline restart → a lone Reconcile (no SetWanted, no StopForUpdate):
-    // restart of a down agent is really a start.
+    // nothing to bounce, and restart never rewrites intent, so the tail
+    // Reconcile converges the down agent to its existing `wanted`.
     let restart = submit(
         &q,
         submit::restart_spec(
@@ -498,8 +495,8 @@ fn offline_agents_skip_mechanical_nodes_but_keep_reconcile() {
     );
     assert_eq!(
         shape(restart),
-        vec!["set_wanted".to_owned(), "reconcile".to_owned()],
+        vec!["reconcile".to_owned()],
-        "offline restart skips StopForUpdate, keeps Reconcile (it's a start)"
+        "offline restart is a lone Reconcile (no SetWanted head, nothing to stop)"
     );
 }
 
@@ -749,11 +746,8 @@ fn cancel_refuses_running_dag() {
 fn terminal_dag_reported_exactly_once_and_lease_released() {
     let q = JobQueue::new(1);
     let id = submit(&q, restart_online(&["agent-a"], false, "r"));
-    // restart = SetWanted → StopForUpdate → Reconcile; not terminal until
+    // restart = StopForUpdate → Reconcile; not terminal until the last
-    // the last node completes.
+    // node completes.
-    let set_wanted = claim_one(&q);
-    q.complete_node(id, set_wanted.node_id, Ok(()));
-    assert!(q.drain_terminal().is_empty(), "dag not terminal yet");
     let stop = claim_one(&q);
     q.complete_node(id, stop.node_id, Ok(()));
     assert!(q.drain_terminal().is_empty(), "dag not terminal yet");