diff --git a/docs/tools/hivectl-cli.md b/docs/tools/hivectl-cli.md index 1695597d..7aeacc30 100644 --- a/docs/tools/hivectl-cli.md +++ b/docs/tools/hivectl-cli.md @@ -309,9 +309,7 @@ Stop containers hive-wide in one operator action. Bare `hivectl stop` stops **ev * `--forge` — The forge container (`hive-forge`) * `--gateway` — The gateway container (`hive-gateway`) * `--matrix` — The matrix container (`hive-matrix`) -* `--graceful` — Gracefully quiesce each agent (finish the current turn, drain the inbox) before stopping, instead of a hard stop. - - NOTE: not yet effective — the per-agent quiesce is still being implemented (see the graceful-agent-stop tracker), so today this falls through to a hard stop. The flag is accepted now so the wire/CLI shape is stable when the quiesce lands. +* `--graceful` — Gracefully quiesce each agent before stopping, instead of a hard stop. Each agent is enqueued as a `GracefulStop` on the rebuild queue: the harness is signalled, runs one stop-checkpoint turn to flush durable `/state`, drains, then the container is stopped (bounded by a 3-min timeout that falls back to a hard stop). Applies to agents only diff --git a/hive-c0re/src/bin/hivectl.rs b/hive-c0re/src/bin/hivectl.rs index 5d76abc7..8f0529a9 100644 --- a/hive-c0re/src/bin/hivectl.rs +++ b/hive-c0re/src/bin/hivectl.rs @@ -116,13 +116,12 @@ enum Cmd { Stop { #[command(flatten)] scope: ScopeArgs, - /// Gracefully quiesce each agent (finish the current turn, drain - /// the inbox) before stopping, instead of a hard stop. - /// - /// NOTE: not yet effective — the per-agent quiesce is still being - /// implemented (see the graceful-agent-stop tracker), so today - /// this falls through to a hard stop. The flag is accepted now so - /// the wire/CLI shape is stable when the quiesce lands. + /// Gracefully quiesce each agent before stopping, instead of a + /// hard stop. Each agent is enqueued as a `GracefulStop` on the + /// rebuild queue: the harness is signalled, runs one + /// stop-checkpoint turn to flush durable `/state`, drains, then + /// the container is stopped (bounded by a 3-min timeout that + /// falls back to a hard stop). Applies to agents only. #[arg(long)] graceful: bool, }, diff --git a/hive-c0re/src/server.rs b/hive-c0re/src/server.rs index 2e2a041a..10ce93f7 100644 --- a/hive-c0re/src/server.rs +++ b/hive-c0re/src/server.rs @@ -101,7 +101,7 @@ async fn dispatch(req: &HostRequest, coord: Arc) -> HostResponse { // graceful-stop queue, to re-expand it). let agents = scoped_agents(scope).await?; let infra = scoped_infra(scope); - handle_stop(&agents, &infra, *graceful).await? + handle_stop(&coord, &agents, &infra, *graceful).await? } HostRequest::Start { scope } => { let agents = scoped_agents(scope).await?; @@ -220,17 +220,40 @@ async fn handle_restart_all() -> Result { /// aggregated rather than aborting on the first error, mirroring /// `handle_restart_all`. Callers resolve the [`LifecycleScope`] to these /// explicit name lists up front — this never sees the "all" flag. -async fn handle_stop(agents: &[String], infra: &[&str], graceful: bool) -> Result { +/// +/// A `graceful` stop enqueues a `QueueKind::GracefulStop` per agent (signal the +/// harness, run one stop-checkpoint turn, drain, then container stop, with a +/// timeout fallback to a hard stop), mirroring the dashboard `?graceful=1` +/// path. `graceful` applies to agents only - infra containers have no harness +/// turn loop, so they're always hard-stopped. +async fn handle_stop( + coord: &Arc, + agents: &[String], + infra: &[&str], + graceful: bool, +) -> Result { tracing::info!(?agents, ?infra, graceful, "stop"); let mut ok_items: Vec = Vec::new(); let mut errors: Vec = Vec::new(); + let mut enqueued_graceful = false; for agent in agents { - // TODO(graceful agent stop): when `graceful`, run the per-agent - // quiesce (turn-end → drain → reject new messages) before the kill. - // The flag is threaded through the wire now; the quiesce itself - // lands with the graceful-agent-stop work. - let _ = graceful; + if graceful { + // Graceful stop: enqueue the quiesce orchestration rather than a + // hard kill. Serialised through the rebuild queue so it can't race + // an in-flight rebuild for the same agent, and its per-step + // progress surfaces on the queue snapshot + build log. + coord.rebuild_queue.enqueue( + crate::rebuild_queue::QueueKind::GracefulStop, + agent.clone(), + crate::rebuild_queue::QueueSource::Manual, + "manual via hivectl graceful stop".to_owned(), + None, + ); + ok_items.push(agent.clone()); + enqueued_graceful = true; + continue; + } match lifecycle::kill(agent).await { Ok(()) => ok_items.push(agent.clone()), Err(e) => { @@ -239,6 +262,9 @@ async fn handle_stop(agents: &[String], infra: &[&str], graceful: bool) -> Resul } } } + if enqueued_graceful { + coord.emit_rebuild_queue_snapshot(); + } for &container in infra { match crate::priv_client::control_infra_container(container, InfraAction::Stop).await {