From 4cfa0401541475425817d1ccd7fd0ac542bfa731 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 14 Jul 2026 19:57:50 +0200 Subject: [PATCH 1/3] feat(#2398): dagify hivectl restart (RestartScoped) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hivectl restart --agent NAME previously composed stop() then start() as two separate client-side daemon calls glued by CLI-process control flow — not one DAG, and a dropped hivectl connection mid-restart (ssh drop, Ctrl-C) left the agent stopped with no automatic follow-up. mara flagged this as the first target for the 'dagify hivectl commands' issue. New HostRequest::RestartScoped{scope, graceful} handles it server-side: each targeted agent now rides exactly one atomic Restart-template DAG (same one hivectl agents restart / restart-all already use) in the common non-graceful case. --graceful has no single-DAG template yet, so it submits the graceful-stop DAGs, awaits them server-side, then submits the start DAGs — still one daemon call end to end, just not yet a single DAG (noted as a follow-up). Infra containers restart synchronously as before (no lease/DAG concept for them). CLI-side restart() now just makes the one call + waits, same output shape as before via render_lifecycle. --- hive-c0re/src/bin/hivectl.rs | 29 ++++++++----- hive-c0re/src/server.rs | 82 ++++++++++++++++++++++++++++++++++++ hive-host-sock/src/lib.rs | 17 ++++++++ 3 files changed, 118 insertions(+), 10 deletions(-) diff --git a/hive-c0re/src/bin/hivectl.rs b/hive-c0re/src/bin/hivectl.rs index e2b7a029..b912976e 100644 --- a/hive-c0re/src/bin/hivectl.rs +++ b/hive-c0re/src/bin/hivectl.rs @@ -1642,22 +1642,31 @@ async fn start(socket: &Path, scope: hive_host_sock::LifecycleScope, no_wait: bo rendered } -/// Restart = `stop` then `start` over the same scope, composed client-side -/// from the two daemon ops (no dedicated wire op). The stop phase honours -/// `--graceful`; if it reports a failure (`stop` returns `Err`) the `?` -/// short-circuits before the start phase, so a half-stopped hive isn't -/// blindly started over — the operator sees the stop errors and can recover. +/// Restart — one `RestartScoped` daemon call, server-side DAG-based (see +/// issue tracker "dagify hivectl commands"). Each targeted agent rides one +/// atomic `Restart` DAG (or, with `--graceful`, a server-awaited +/// graceful-stop DAG followed by a start DAG); infra containers restart +/// synchronously. Unlike the old client-side stop-then-start compose, a +/// dropped `hivectl` connection mid-restart no longer leaves an agent +/// stopped with no automatic follow-up — the daemon owns the whole +/// sequence once this call is made. /// -/// No `--no-wait` here on purpose: the stop DAGs must complete before -/// the start submits, otherwise the start's `wanted = Up` write would -/// land before the queued stops execute and turn them into noops. +/// No `--no-wait` here on purpose, same as before: the operator wants to +/// see the restart actually land, not just get queued. async fn restart( socket: &Path, scope: hive_host_sock::LifecycleScope, graceful: bool, ) -> Result<()> { - stop(socket, scope.clone(), graceful, false).await?; - start(socket, scope, false).await + let resp = hive_c0re::client::request( + socket, + hive_host_sock::HostRequest::RestartScoped { scope, graceful }, + ) + .await + .with_context(|| format!("connect to daemon socket {}", socket.display()))?; + let rendered = render_lifecycle(&resp, "restart queued"); + wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), false).await?; + rendered } /// A [`LifecycleScope`](hive_host_sock::LifecycleScope) targeting exactly one diff --git a/hive-c0re/src/server.rs b/hive-c0re/src/server.rs index 008359da..e9fae489 100644 --- a/hive-c0re/src/server.rs +++ b/hive-c0re/src/server.rs @@ -94,6 +94,9 @@ async fn dispatch(req: &HostRequest, coord: Arc) -> HostResponse { HostRequest::Kill { name } => submit_single(&coord, name, Verb::Kill), HostRequest::Restart { name } => submit_single(&coord, name, Verb::Restart), HostRequest::RestartAll => handle_restart_all(&coord).await?, + HostRequest::RestartScoped { scope, graceful } => { + handle_restart_scoped(&coord, scope, *graceful).await? + } HostRequest::Stop { scope, graceful } => { // Resolve the scope to explicit container names at the entry // point, then operate on names — never pass the bare "all @@ -631,6 +634,85 @@ async fn handle_start( Ok(resp) } +/// Restart containers hive-wide (`hivectl restart`) — the DAG-based +/// sibling of [`handle_stop`]/[`handle_start`], replacing the old +/// client-side stop-then-start composition (issue tracker "dagify hivectl +/// commands"). Each targeted agent gets exactly one atomic `Restart` +/// template DAG in the common (non-graceful) case, so it survives a +/// dropped `hivectl` connection the same way `handle_restart_all` already +/// does. `graceful` has no single-DAG template yet, so it falls back to +/// submitting the graceful-stop DAGs, waiting them out server-side, then +/// submitting the start DAGs — still one daemon call end to end, unlike +/// the old CLI-side compose. Infra containers have no lease/DAG and +/// restart synchronously (stop then start). +async fn handle_restart_scoped( + coord: &Arc, + scope: &LifecycleScope, + graceful: bool, +) -> Result { + tracing::info!(?scope, graceful, "restart"); + let agents = scoped_agents(scope).await?; + let infra = scoped_infra(scope); + let mut ok_items: Vec = Vec::new(); + let mut errors: Vec = Vec::new(); + let mut queued: Vec = Vec::new(); + + if graceful { + let mut stop_ids = Vec::new(); + for agent in &agents { + stop_ids.push(crate::job_queue::submit::graceful_stop( + coord, + agent, + crate::job_queue::Source::Manual, + "manual via hivectl restart --graceful".to_owned(), + )); + } + // Graceful drains can take a while; bound the wait generously — + // same shape as `handle_stop`'s hard-stop-before-infra wait, just + // a longer ceiling since drains are minutes, not seconds. + await_dags(coord, &stop_ids, std::time::Duration::from_mins(10)).await; + for agent in &agents { + queued.push(crate::job_queue::submit::start( + coord, + agent, + crate::job_queue::Source::Manual, + "manual via hivectl restart --graceful".to_owned(), + )); + ok_items.push(agent.clone()); + } + } else { + for agent in &agents { + queued.push(crate::job_queue::submit::restart( + coord, + agent, + crate::job_queue::Source::Manual, + "manual restart via hivectl restart".to_owned(), + )); + ok_items.push(agent.clone()); + } + } + + for &container in &infra { + let name = container.unit_name(); + let res = async { + crate::priv_client::control_infra_container(container, InfraAction::Stop).await?; + crate::priv_client::control_infra_container(container, InfraAction::Start).await + } + .await; + match res { + Ok(()) => ok_items.push(name.to_owned()), + Err(e) => { + tracing::warn!(%name, error = ?e, "restart: infra restart failed"); + errors.push(format!("{name}: {e:#}")); + } + } + } + + let mut resp = finish_lifecycle(ok_items, &errors); + resp.queued_dags = Some(queued); + Ok(resp) +} + /// Resolve which sub-agent logical names a scope targets: every live /// container (from `lifecycle::list`) when `agents` is set or the scope is /// "everything", plus any explicit `agent_names`. Returns de-duplicated diff --git a/hive-host-sock/src/lib.rs b/hive-host-sock/src/lib.rs index 9d7ed40c..96c0dee2 100644 --- a/hive-host-sock/src/lib.rs +++ b/hive-host-sock/src/lib.rs @@ -39,6 +39,23 @@ pub enum HostRequest { /// wrapper for `hivectl agents restart-all`; iterates the live /// container list and restarts each one. RestartAll, + /// Restart containers hive-wide (`hivectl restart`), scoped like + /// `Stop`/`Start`. Each targeted agent rides exactly one DAG server-side + /// (the `Restart` template, hard stop + reconcile — or, when `graceful` + /// is set, a submitted graceful-stop DAG the server itself waits out + /// before submitting the start DAG) rather than the old client-side + /// stop-then-start composition: a dropped `hivectl` connection mid-way + /// used to leave the agent stopped with no automatic follow-up, since + /// nothing durable remembered "finish the restart" once the CLI process + /// was gone. Infra containers have no lease/DAG and restart + /// synchronously (stop then start), same as before. Scope semantics + /// match `Stop`/`Start` (all-false = everything). + RestartScoped { + #[serde(default)] + scope: LifecycleScope, + #[serde(default)] + graceful: bool, + }, /// Apply pending config to a managed container. Rebuild { name: String }, /// List managed containers. From 901ab6a779a409ec78772ef4431cbe31397cf2da Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 14 Jul 2026 20:21:06 +0200 Subject: [PATCH 2/3] fix(#2398): graceful restart as one atomic DAG, not compose-and-await mara's review on #2436: no submit-await-submit composition, even server-side. Adds Template::GracefulRestart (Signal -> Drain -> StopForUpdate -> Reconcile, wanted=Up) mirroring how Restart already does StopForUpdate -> Reconcile, plus submit::graceful_restart and templates::graceful_restart. handle_restart_scoped now submits exactly one DAG per agent up front for both the graceful and non-graceful case -- no await_dags in the loop anymore. --- docs/coordinator.md | 1 + hive-c0re/src/bin/hivectl.rs | 10 ++++--- hive-c0re/src/job_queue/exec.rs | 6 +++- hive-c0re/src/job_queue/submit.rs | 13 +++++++++ hive-c0re/src/job_queue/templates.rs | 41 ++++++++++++++++++++++++++++ hive-c0re/src/server.rs | 41 +++++++++++----------------- hive-host-sock/src/lib.rs | 25 ++++++++++------- hive-sh4re/src/jobs.rs | 6 ++++ 8 files changed, 103 insertions(+), 40 deletions(-) diff --git a/docs/coordinator.md b/docs/coordinator.md index 4c83785a..1ac89697 100644 --- a/docs/coordinator.md +++ b/docs/coordinator.md @@ -78,6 +78,7 @@ container build: rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(after-any) Reconcile(a) graceful-stop(a): [wanted=Offline] Signal(a) → Drain(a) → Reconcile(a) restart(a): [wanted=Up] StopForUpdate(a) → Reconcile(a) +graceful-restart(a): [wanted=Up] Signal(a) → Drain(a) → StopForUpdate(a) → Reconcile(a) start(a): [wanted=Up] Reconcile(a) (stale rev ⇒ upgraded to rebuild) stop(a): [wanted=Offline] Reconcile(a) spawn(a): [wanted=Up] Create(a) → WriteDropin(a) → Reconcile(a) diff --git a/hive-c0re/src/bin/hivectl.rs b/hive-c0re/src/bin/hivectl.rs index b912976e..143e0657 100644 --- a/hive-c0re/src/bin/hivectl.rs +++ b/hive-c0re/src/bin/hivectl.rs @@ -1643,10 +1643,12 @@ async fn start(socket: &Path, scope: hive_host_sock::LifecycleScope, no_wait: bo } /// Restart — one `RestartScoped` daemon call, server-side DAG-based (see -/// issue tracker "dagify hivectl commands"). Each targeted agent rides one -/// atomic `Restart` DAG (or, with `--graceful`, a server-awaited -/// graceful-stop DAG followed by a start DAG); infra containers restart -/// synchronously. Unlike the old client-side stop-then-start compose, a +/// issue tracker "dagify hivectl commands"). Each targeted agent rides +/// exactly one atomic DAG queued up front — `Restart` (mechanical stop + +/// reconcile), or `GracefulRestart` with `--graceful` (signal → drain → +/// mechanical stop → reconcile); infra containers restart synchronously. +/// No "submit one DAG, wait for it, submit another" composition on either +/// side of the wire: unlike the old client-side stop-then-start compose, a /// dropped `hivectl` connection mid-restart no longer leaves an agent /// stopped with no automatic follow-up — the daemon owns the whole /// sequence once this call is made. diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index 09d8c812..8ccbc6d9 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -507,7 +507,11 @@ pub(super) async fn on_dag_terminal(coord: &Arc, terminal: &Termina if terminal.state == State::Cancelled && matches!( terminal.template, - Template::Start | Template::Stop | Template::GracefulStop | Template::Restart + Template::Start + | Template::Stop + | Template::GracefulStop + | Template::Restart + | Template::GracefulRestart ) { let running = crate::lifecycle::is_running(&terminal.agent).await; diff --git a/hive-c0re/src/job_queue/submit.rs b/hive-c0re/src/job_queue/submit.rs index 073c3671..0e5bc4b4 100644 --- a/hive-c0re/src/job_queue/submit.rs +++ b/hive-c0re/src/job_queue/submit.rs @@ -100,6 +100,19 @@ pub fn graceful_stop(coord: &Arc, agent: &str, source: Source, reas submit_and_emit(coord, templates::graceful_stop(agent, source, reason)) } +/// Graceful restart: persist `wanted = Up`, then signal → drain → +/// mechanical stop → reconcile (starts it back up) — one atomic DAG, +/// no client-side "await the stop DAG then submit a start DAG" split. +pub fn graceful_restart( + coord: &Arc, + agent: &str, + source: Source, + reason: String, +) -> u64 { + set_wanted(coord, agent, Wanted::Up); + submit_and_emit(coord, templates::graceful_restart(agent, source, reason)) +} + /// Perm change: commit the JSON file(s) then rebuild. pub fn perm_change( coord: &Arc, diff --git a/hive-c0re/src/job_queue/templates.rs b/hive-c0re/src/job_queue/templates.rs index 586e1d01..29ffa8d3 100644 --- a/hive-c0re/src/job_queue/templates.rs +++ b/hive-c0re/src/job_queue/templates.rs @@ -7,6 +7,7 @@ //! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a) //! graceful-stop(a): [wanted=Offline] Signal(a) → Drain(a) → Reconcile(a) //! restart(a): [wanted=Up] StopForUpdate(a) → Reconcile(a) +//! graceful-restart(a): [wanted=Up] Signal(a) → Drain(a) → StopForUpdate(a) → Reconcile(a) //! start(a): [wanted=Up] Reconcile(a) //! stop(a): [wanted=Offline] Reconcile(a) //! spawn(a): [wanted=Up] Provision(a) → Create(a) → WriteDropin(a) → Reconcile(a) @@ -168,6 +169,46 @@ pub fn restart(agent: &str, source: Source, reason: String) -> DagSpec { } } +/// Graceful restart: signal → drain → mechanical stop → converge to +/// `wanted` — the caller writes `wanted = Up` first, same as `restart`. +/// One atomic DAG start to finish (no client- or server-side "submit +/// one DAG, await it, submit the next" composition): the `Drain` node +/// is the same bounded harness-checkpoint wait `graceful_stop` uses, +/// then `StopForUpdate` (mechanical, ignores `wanted`) and the tail +/// `Reconcile` (converges to `wanted = Up`, i.e. starts it back up) +/// chain exactly like `restart`'s tail. +pub fn graceful_restart(agent: &str, source: Source, reason: String) -> DagSpec { + DagSpec { + template: Template::GracefulRestart, + agent: agent.to_owned(), + source, + reason, + parent_id: None, + approval_id: None, + inputs: Vec::new(), + perm_payload: None, + transient: Some(TransientKind::Restarting), + nodes: vec![ + NodeSpec { + kind: NodeKind::Signal, + deps: Vec::new(), + }, + NodeSpec { + kind: NodeKind::Drain, + deps: after_ok(0), + }, + NodeSpec { + kind: NodeKind::StopForUpdate, + deps: after_ok(1), + }, + NodeSpec { + kind: NodeKind::Reconcile, + deps: after_ok(2), + }, + ], + } +} + /// Single-`Reconcile` DAG: `Start` / `Stop` (caller writes `wanted` /// first) and the boot-time `Reconcile` converge (wanted untouched). pub fn reconcile_only( diff --git a/hive-c0re/src/server.rs b/hive-c0re/src/server.rs index e9fae489..2204a5f4 100644 --- a/hive-c0re/src/server.rs +++ b/hive-c0re/src/server.rs @@ -657,39 +657,30 @@ async fn handle_restart_scoped( let mut errors: Vec = Vec::new(); let mut queued: Vec = Vec::new(); - if graceful { - let mut stop_ids = Vec::new(); - for agent in &agents { - stop_ids.push(crate::job_queue::submit::graceful_stop( + // One atomic DAG per agent, submitted up front — `Restart` + // (mechanical stop + reconcile) or, with `--graceful`, + // `GracefulRestart` (signal → drain → mechanical stop → reconcile). + // No client- or server-side "submit a stop DAG, await it, then + // submit a start DAG" composition: that window is exactly the + // dropped-connection gap this DAG-based path exists to close. + for agent in &agents { + let id = if graceful { + crate::job_queue::submit::graceful_restart( coord, agent, crate::job_queue::Source::Manual, "manual via hivectl restart --graceful".to_owned(), - )); - } - // Graceful drains can take a while; bound the wait generously — - // same shape as `handle_stop`'s hard-stop-before-infra wait, just - // a longer ceiling since drains are minutes, not seconds. - await_dags(coord, &stop_ids, std::time::Duration::from_mins(10)).await; - for agent in &agents { - queued.push(crate::job_queue::submit::start( - coord, - agent, - crate::job_queue::Source::Manual, - "manual via hivectl restart --graceful".to_owned(), - )); - ok_items.push(agent.clone()); - } - } else { - for agent in &agents { - queued.push(crate::job_queue::submit::restart( + ) + } else { + crate::job_queue::submit::restart( coord, agent, crate::job_queue::Source::Manual, "manual restart via hivectl restart".to_owned(), - )); - ok_items.push(agent.clone()); - } + ) + }; + queued.push(id); + ok_items.push(agent.clone()); } for &container in &infra { diff --git a/hive-host-sock/src/lib.rs b/hive-host-sock/src/lib.rs index 96c0dee2..b55581fc 100644 --- a/hive-host-sock/src/lib.rs +++ b/hive-host-sock/src/lib.rs @@ -40,16 +40,21 @@ pub enum HostRequest { /// container list and restarts each one. RestartAll, /// Restart containers hive-wide (`hivectl restart`), scoped like - /// `Stop`/`Start`. Each targeted agent rides exactly one DAG server-side - /// (the `Restart` template, hard stop + reconcile — or, when `graceful` - /// is set, a submitted graceful-stop DAG the server itself waits out - /// before submitting the start DAG) rather than the old client-side - /// stop-then-start composition: a dropped `hivectl` connection mid-way - /// used to leave the agent stopped with no automatic follow-up, since - /// nothing durable remembered "finish the restart" once the CLI process - /// was gone. Infra containers have no lease/DAG and restart - /// synchronously (stop then start), same as before. Scope semantics - /// match `Stop`/`Start` (all-false = everything). + /// `Stop`/`Start`. Each targeted agent rides exactly one DAG + /// server-side — the `Restart` template (mechanical stop + reconcile), + /// or, when `graceful` is set, the `GracefulRestart` template (signal → + /// drain → mechanical stop → reconcile) — rather than the old + /// client-side stop-then-start composition (and, briefly, a server-side + /// "submit stop DAG, await it, submit start DAG" composition): a + /// dropped `hivectl` connection mid-way, or a crash between the two + /// submits, used to leave the agent stopped with no automatic + /// follow-up, since nothing durable remembered "finish the restart" + /// once the calling process/turn was gone. `GracefulRestart` closes + /// that gap the same way `Restart` already does — one DAG, queued up + /// front, that owns the whole sequence. Infra containers have no + /// lease/DAG and restart synchronously (stop then start), same as + /// before. Scope semantics match `Stop`/`Start` (all-false = + /// everything). RestartScoped { #[serde(default)] scope: LifecycleScope, diff --git a/hive-sh4re/src/jobs.rs b/hive-sh4re/src/jobs.rs index 71ea49cc..b576f3c2 100644 --- a/hive-sh4re/src/jobs.rs +++ b/hive-sh4re/src/jobs.rs @@ -29,6 +29,11 @@ pub enum Template { StartupSweep, /// Mechanical stop + converge to `wanted = Up` (a restart). Restart, + /// Signal → drain → mechanical stop → converge to `wanted = Up` — a + /// graceful restart as one atomic DAG (drains the harness before the + /// stop, same as `GracefulStop`, but then reconciles back up instead + /// of staying down). + GracefulRestart, /// Perm-file commit followed by the rebuild subgraph. PermChange, /// Quiesce the harness, drain, then stop (`wanted = Offline`). @@ -56,6 +61,7 @@ impl Template { Template::Destroy => "destroy", Template::StartupSweep => "startup_sweep", Template::Restart => "restart", + Template::GracefulRestart => "graceful_restart", Template::PermChange => "perm_change", Template::GracefulStop => "graceful_stop", Template::Start => "start", From 407965b6e149883fbadfc881a6d56c40a5224c35 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 14 Jul 2026 20:26:59 +0200 Subject: [PATCH 3/3] fix(#2398): correct stale doc on handle_restart_scoped argus caught it: the function-level /// comment still described the old submit-await-submit graceful approach after the code moved to one atomic GracefulRestart DAG. --- hive-c0re/src/server.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/hive-c0re/src/server.rs b/hive-c0re/src/server.rs index 2204a5f4..5e88d8a1 100644 --- a/hive-c0re/src/server.rs +++ b/hive-c0re/src/server.rs @@ -637,14 +637,14 @@ async fn handle_start( /// Restart containers hive-wide (`hivectl restart`) — the DAG-based /// sibling of [`handle_stop`]/[`handle_start`], replacing the old /// client-side stop-then-start composition (issue tracker "dagify hivectl -/// commands"). Each targeted agent gets exactly one atomic `Restart` -/// template DAG in the common (non-graceful) case, so it survives a -/// dropped `hivectl` connection the same way `handle_restart_all` already -/// does. `graceful` has no single-DAG template yet, so it falls back to -/// submitting the graceful-stop DAGs, waiting them out server-side, then -/// submitting the start DAGs — still one daemon call end to end, unlike -/// the old CLI-side compose. Infra containers have no lease/DAG and -/// restart synchronously (stop then start). +/// commands"). Each targeted agent gets exactly one atomic DAG submitted +/// up front: the `Restart` template (mechanical stop + reconcile) in the +/// common case, or `GracefulRestart` (signal → drain → mechanical stop → +/// reconcile) with `graceful` set — no "submit a DAG, wait for it, submit +/// another" composition on either path, so a dropped `hivectl` connection +/// never strands an agent, the same way `handle_restart_all` already +/// avoids it. Infra containers have no lease/DAG and restart +/// synchronously (stop then start). async fn handle_restart_scoped( coord: &Arc, scope: &LifecycleScope,