fix(#2398): graceful restart as one atomic DAG, not compose-and-await

mara's review on #2436: no submit-await-submit composition, even
server-side. Adds Template::GracefulRestart (Signal -> Drain ->
StopForUpdate -> Reconcile, wanted=Up) mirroring how Restart already
does StopForUpdate -> Reconcile, plus submit::graceful_restart and
templates::graceful_restart. handle_restart_scoped now submits exactly
one DAG per agent up front for both the graceful and non-graceful
case -- no await_dags in the loop anymore.

docs/coordinator.md

@@ -78,6 +78,7 @@ container build:
 rebuild(a):        Prebuild(a) → StopForUpdate(a) → Swap(a) →(after-any) Reconcile(a)
 graceful-stop(a):  [wanted=Offline]  Signal(a) → Drain(a) → Reconcile(a)
 restart(a):        [wanted=Up]       StopForUpdate(a) → Reconcile(a)
+graceful-restart(a): [wanted=Up]     Signal(a) → Drain(a) → StopForUpdate(a) → Reconcile(a)
 start(a):          [wanted=Up]       Reconcile(a)     (stale rev ⇒ upgraded to rebuild)
 stop(a):           [wanted=Offline]  Reconcile(a)
 spawn(a):          [wanted=Up]  Create(a) → WriteDropin(a) → Reconcile(a)

hive-c0re/src/bin/hivectl.rs

@@ -1643,10 +1643,12 @@ async fn start(socket: &Path, scope: hive_host_sock::LifecycleScope, no_wait: bo
 }
 
 /// Restart — one `RestartScoped` daemon call, server-side DAG-based (see
-/// issue tracker "dagify hivectl commands"). Each targeted agent rides one
+/// issue tracker "dagify hivectl commands"). Each targeted agent rides
-/// atomic `Restart` DAG (or, with `--graceful`, a server-awaited
+/// exactly one atomic DAG queued up front — `Restart` (mechanical stop +
-/// graceful-stop DAG followed by a start DAG); infra containers restart
+/// reconcile), or `GracefulRestart` with `--graceful` (signal → drain →
-/// synchronously. Unlike the old client-side stop-then-start compose, a
+/// mechanical stop → reconcile); infra containers restart synchronously.
+/// No "submit one DAG, wait for it, submit another" composition on either
+/// side of the wire: unlike the old client-side stop-then-start compose, a
 /// dropped `hivectl` connection mid-restart no longer leaves an agent
 /// stopped with no automatic follow-up — the daemon owns the whole
 /// sequence once this call is made.

hive-c0re/src/job_queue/exec.rs

@@ -507,7 +507,11 @@ pub(super) async fn on_dag_terminal(coord: &Arc<Coordinator>, terminal: &Termina
     if terminal.state == State::Cancelled
         && matches!(
             terminal.template,
-            Template::Start | Template::Stop | Template::GracefulStop | Template::Restart
+            Template::Start
+                | Template::Stop
+                | Template::GracefulStop
+                | Template::Restart
+                | Template::GracefulRestart
         )
     {
         let running = crate::lifecycle::is_running(&terminal.agent).await;

hive-c0re/src/job_queue/submit.rs

@@ -100,6 +100,19 @@ pub fn graceful_stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reas
     submit_and_emit(coord, templates::graceful_stop(agent, source, reason))
 }
 
+/// Graceful restart: persist `wanted = Up`, then signal → drain →
+/// mechanical stop → reconcile (starts it back up) — one atomic DAG,
+/// no client-side "await the stop DAG then submit a start DAG" split.
+pub fn graceful_restart(
+    coord: &Arc<Coordinator>,
+    agent: &str,
+    source: Source,
+    reason: String,
+) -> u64 {
+    set_wanted(coord, agent, Wanted::Up);
+    submit_and_emit(coord, templates::graceful_restart(agent, source, reason))
+}
+
 /// Perm change: commit the JSON file(s) then rebuild.
 pub fn perm_change(
     coord: &Arc<Coordinator>,

hive-c0re/src/job_queue/templates.rs

@@ -7,6 +7,7 @@
 //! rebuild(a):        Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a)
 //! graceful-stop(a):  [wanted=Offline]  Signal(a) → Drain(a) → Reconcile(a)
 //! restart(a):        [wanted=Up]       StopForUpdate(a) → Reconcile(a)
+//! graceful-restart(a): [wanted=Up]     Signal(a) → Drain(a) → StopForUpdate(a) → Reconcile(a)
 //! start(a):          [wanted=Up]       Reconcile(a)
 //! stop(a):           [wanted=Offline]  Reconcile(a)
 //! spawn(a):          [wanted=Up]  Provision(a) → Create(a) → WriteDropin(a) → Reconcile(a)
@@ -168,6 +169,46 @@ pub fn restart(agent: &str, source: Source, reason: String) -> DagSpec {
     }
 }
 
+/// Graceful restart: signal → drain → mechanical stop → converge to
+/// `wanted` — the caller writes `wanted = Up` first, same as `restart`.
+/// One atomic DAG start to finish (no client- or server-side "submit
+/// one DAG, await it, submit the next" composition): the `Drain` node
+/// is the same bounded harness-checkpoint wait `graceful_stop` uses,
+/// then `StopForUpdate` (mechanical, ignores `wanted`) and the tail
+/// `Reconcile` (converges to `wanted = Up`, i.e. starts it back up)
+/// chain exactly like `restart`'s tail.
+pub fn graceful_restart(agent: &str, source: Source, reason: String) -> DagSpec {
+    DagSpec {
+        template: Template::GracefulRestart,
+        agent: agent.to_owned(),
+        source,
+        reason,
+        parent_id: None,
+        approval_id: None,
+        inputs: Vec::new(),
+        perm_payload: None,
+        transient: Some(TransientKind::Restarting),
+        nodes: vec![
+            NodeSpec {
+                kind: NodeKind::Signal,
+                deps: Vec::new(),
+            },
+            NodeSpec {
+                kind: NodeKind::Drain,
+                deps: after_ok(0),
+            },
+            NodeSpec {
+                kind: NodeKind::StopForUpdate,
+                deps: after_ok(1),
+            },
+            NodeSpec {
+                kind: NodeKind::Reconcile,
+                deps: after_ok(2),
+            },
+        ],
+    }
+}
+
 /// Single-`Reconcile` DAG: `Start` / `Stop` (caller writes `wanted`
 /// first) and the boot-time `Reconcile` converge (wanted untouched).
 pub fn reconcile_only(

hive-c0re/src/server.rs

@@ -657,39 +657,30 @@ async fn handle_restart_scoped(
     let mut errors: Vec<String> = Vec::new();
     let mut queued: Vec<u64> = Vec::new();
 
-    if graceful {
+    // One atomic DAG per agent, submitted up front — `Restart`
-        let mut stop_ids = Vec::new();
+    // (mechanical stop + reconcile) or, with `--graceful`,
-        for agent in &agents {
+    // `GracefulRestart` (signal → drain → mechanical stop → reconcile).
-            stop_ids.push(crate::job_queue::submit::graceful_stop(
+    // No client- or server-side "submit a stop DAG, await it, then
+    // submit a start DAG" composition: that window is exactly the
+    // dropped-connection gap this DAG-based path exists to close.
+    for agent in &agents {
+        let id = if graceful {
+            crate::job_queue::submit::graceful_restart(
                 coord,
                 agent,
                 crate::job_queue::Source::Manual,
                 "manual via hivectl restart --graceful".to_owned(),
-            ));
+            )
-        }
+        } else {
-        // Graceful drains can take a while; bound the wait generously —
+            crate::job_queue::submit::restart(
-        // same shape as `handle_stop`'s hard-stop-before-infra wait, just
-        // a longer ceiling since drains are minutes, not seconds.
-        await_dags(coord, &stop_ids, std::time::Duration::from_mins(10)).await;
-        for agent in &agents {
-            queued.push(crate::job_queue::submit::start(
-                coord,
-                agent,
-                crate::job_queue::Source::Manual,
-                "manual via hivectl restart --graceful".to_owned(),
-            ));
-            ok_items.push(agent.clone());
-        }
-    } else {
-        for agent in &agents {
-            queued.push(crate::job_queue::submit::restart(
                 coord,
                 agent,
                 crate::job_queue::Source::Manual,
                 "manual restart via hivectl restart".to_owned(),
-            ));
+            )
-            ok_items.push(agent.clone());
+        };
-        }
+        queued.push(id);
+        ok_items.push(agent.clone());
     }
 
     for &container in &infra {

hive-host-sock/src/lib.rs

@@ -40,16 +40,21 @@ pub enum HostRequest {
     /// container list and restarts each one.
     RestartAll,
     /// Restart containers hive-wide (`hivectl restart`), scoped like
-    /// `Stop`/`Start`. Each targeted agent rides exactly one DAG server-side
+    /// `Stop`/`Start`. Each targeted agent rides exactly one DAG
-    /// (the `Restart` template, hard stop + reconcile — or, when `graceful`
+    /// server-side — the `Restart` template (mechanical stop + reconcile),
-    /// is set, a submitted graceful-stop DAG the server itself waits out
+    /// or, when `graceful` is set, the `GracefulRestart` template (signal →
-    /// before submitting the start DAG) rather than the old client-side
+    /// drain → mechanical stop → reconcile) — rather than the old
-    /// stop-then-start composition: a dropped `hivectl` connection mid-way
+    /// client-side stop-then-start composition (and, briefly, a server-side
-    /// used to leave the agent stopped with no automatic follow-up, since
+    /// "submit stop DAG, await it, submit start DAG" composition): a
-    /// nothing durable remembered "finish the restart" once the CLI process
+    /// dropped `hivectl` connection mid-way, or a crash between the two
-    /// was gone. Infra containers have no lease/DAG and restart
+    /// submits, used to leave the agent stopped with no automatic
-    /// synchronously (stop then start), same as before. Scope semantics
+    /// follow-up, since nothing durable remembered "finish the restart"
-    /// match `Stop`/`Start` (all-false = everything).
+    /// once the calling process/turn was gone. `GracefulRestart` closes
+    /// that gap the same way `Restart` already does — one DAG, queued up
+    /// front, that owns the whole sequence. Infra containers have no
+    /// lease/DAG and restart synchronously (stop then start), same as
+    /// before. Scope semantics match `Stop`/`Start` (all-false =
+    /// everything).
     RestartScoped {
         #[serde(default)]
         scope: LifecycleScope,

hive-sh4re/src/jobs.rs

@@ -29,6 +29,11 @@ pub enum Template {
     StartupSweep,
     /// Mechanical stop + converge to `wanted = Up` (a restart).
     Restart,
+    /// Signal → drain → mechanical stop → converge to `wanted = Up` — a
+    /// graceful restart as one atomic DAG (drains the harness before the
+    /// stop, same as `GracefulStop`, but then reconciles back up instead
+    /// of staying down).
+    GracefulRestart,
     /// Perm-file commit followed by the rebuild subgraph.
     PermChange,
     /// Quiesce the harness, drain, then stop (`wanted = Offline`).
@@ -56,6 +61,7 @@ impl Template {
             Template::Destroy => "destroy",
             Template::StartupSweep => "startup_sweep",
             Template::Restart => "restart",
+            Template::GracefulRestart => "graceful_restart",
             Template::PermChange => "perm_change",
             Template::GracefulStop => "graceful_stop",
             Template::Start => "start",