hive-c0re: wire hivectl --graceful to enqueue GracefulStop

This commit is contained in:
damocles 2026-06-19 11:57:37 +02:00
commit 1b81ad423e
3 changed files with 40 additions and 17 deletions

View file

@ -309,9 +309,7 @@ Stop containers hive-wide in one operator action. Bare `hivectl stop` stops **ev
* `--forge` — The forge container (`hive-forge`)
* `--gateway` — The gateway container (`hive-gateway`)
* `--matrix` — The matrix container (`hive-matrix`)
* `--graceful` — Gracefully quiesce each agent (finish the current turn, drain the inbox) before stopping, instead of a hard stop.
NOTE: not yet effective — the per-agent quiesce is still being implemented (see the graceful-agent-stop tracker), so today this falls through to a hard stop. The flag is accepted now so the wire/CLI shape is stable when the quiesce lands.
* `--graceful` — Gracefully quiesce each agent before stopping, instead of a hard stop. Each agent is enqueued as a `GracefulStop` on the rebuild queue: the harness is signalled, runs one stop-checkpoint turn to flush durable `/state`, drains, then the container is stopped (bounded by a 3-min timeout that falls back to a hard stop). Applies to agents only

View file

@ -116,13 +116,12 @@ enum Cmd {
Stop {
#[command(flatten)]
scope: ScopeArgs,
/// Gracefully quiesce each agent (finish the current turn, drain
/// the inbox) before stopping, instead of a hard stop.
///
/// NOTE: not yet effective — the per-agent quiesce is still being
/// implemented (see the graceful-agent-stop tracker), so today
/// this falls through to a hard stop. The flag is accepted now so
/// the wire/CLI shape is stable when the quiesce lands.
/// Gracefully quiesce each agent before stopping, instead of a
/// hard stop. Each agent is enqueued as a `GracefulStop` on the
/// rebuild queue: the harness is signalled, runs one
/// stop-checkpoint turn to flush durable `/state`, drains, then
/// the container is stopped (bounded by a 3-min timeout that
/// falls back to a hard stop). Applies to agents only.
#[arg(long)]
graceful: bool,
},

View file

@ -101,7 +101,7 @@ async fn dispatch(req: &HostRequest, coord: Arc<Coordinator>) -> HostResponse {
// graceful-stop queue, to re-expand it).
let agents = scoped_agents(scope).await?;
let infra = scoped_infra(scope);
handle_stop(&agents, &infra, *graceful).await?
handle_stop(&coord, &agents, &infra, *graceful).await?
}
HostRequest::Start { scope } => {
let agents = scoped_agents(scope).await?;
@ -220,17 +220,40 @@ async fn handle_restart_all() -> Result<HostResponse> {
/// aggregated rather than aborting on the first error, mirroring
/// `handle_restart_all`. Callers resolve the [`LifecycleScope`] to these
/// explicit name lists up front — this never sees the "all" flag.
async fn handle_stop(agents: &[String], infra: &[&str], graceful: bool) -> Result<HostResponse> {
///
/// A `graceful` stop enqueues a `QueueKind::GracefulStop` per agent (signal the
/// harness, run one stop-checkpoint turn, drain, then container stop, with a
/// timeout fallback to a hard stop), mirroring the dashboard `?graceful=1`
/// path. `graceful` applies to agents only - infra containers have no harness
/// turn loop, so they're always hard-stopped.
async fn handle_stop(
coord: &Arc<Coordinator>,
agents: &[String],
infra: &[&str],
graceful: bool,
) -> Result<HostResponse> {
tracing::info!(?agents, ?infra, graceful, "stop");
let mut ok_items: Vec<String> = Vec::new();
let mut errors: Vec<String> = Vec::new();
let mut enqueued_graceful = false;
for agent in agents {
// TODO(graceful agent stop): when `graceful`, run the per-agent
// quiesce (turn-end → drain → reject new messages) before the kill.
// The flag is threaded through the wire now; the quiesce itself
// lands with the graceful-agent-stop work.
let _ = graceful;
if graceful {
// Graceful stop: enqueue the quiesce orchestration rather than a
// hard kill. Serialised through the rebuild queue so it can't race
// an in-flight rebuild for the same agent, and its per-step
// progress surfaces on the queue snapshot + build log.
coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::GracefulStop,
agent.clone(),
crate::rebuild_queue::QueueSource::Manual,
"manual via hivectl graceful stop".to_owned(),
None,
);
ok_items.push(agent.clone());
enqueued_graceful = true;
continue;
}
match lifecycle::kill(agent).await {
Ok(()) => ok_items.push(agent.clone()),
Err(e) => {
@ -239,6 +262,9 @@ async fn handle_stop(agents: &[String], infra: &[&str], graceful: bool) -> Resul
}
}
}
if enqueued_graceful {
coord.emit_rebuild_queue_snapshot();
}
for &container in infra {
match crate::priv_client::control_infra_container(container, InfraAction::Stop).await {