hive-c0re: wire hivectl --graceful to enqueue GracefulStop

This commit is contained in:
damocles 2026-06-19 11:57:37 +02:00
commit 1b81ad423e
3 changed files with 40 additions and 17 deletions

View file

@ -309,9 +309,7 @@ Stop containers hive-wide in one operator action. Bare `hivectl stop` stops **ev
* `--forge` — The forge container (`hive-forge`) * `--forge` — The forge container (`hive-forge`)
* `--gateway` — The gateway container (`hive-gateway`) * `--gateway` — The gateway container (`hive-gateway`)
* `--matrix` — The matrix container (`hive-matrix`) * `--matrix` — The matrix container (`hive-matrix`)
* `--graceful` — Gracefully quiesce each agent (finish the current turn, drain the inbox) before stopping, instead of a hard stop. * `--graceful` — Gracefully quiesce each agent before stopping, instead of a hard stop. Each agent is enqueued as a `GracefulStop` on the rebuild queue: the harness is signalled, runs one stop-checkpoint turn to flush durable `/state`, drains, then the container is stopped (bounded by a 3-min timeout that falls back to a hard stop). Applies to agents only
NOTE: not yet effective — the per-agent quiesce is still being implemented (see the graceful-agent-stop tracker), so today this falls through to a hard stop. The flag is accepted now so the wire/CLI shape is stable when the quiesce lands.

View file

@ -116,13 +116,12 @@ enum Cmd {
Stop { Stop {
#[command(flatten)] #[command(flatten)]
scope: ScopeArgs, scope: ScopeArgs,
/// Gracefully quiesce each agent (finish the current turn, drain /// Gracefully quiesce each agent before stopping, instead of a
/// the inbox) before stopping, instead of a hard stop. /// hard stop. Each agent is enqueued as a `GracefulStop` on the
/// /// rebuild queue: the harness is signalled, runs one
/// NOTE: not yet effective — the per-agent quiesce is still being /// stop-checkpoint turn to flush durable `/state`, drains, then
/// implemented (see the graceful-agent-stop tracker), so today /// the container is stopped (bounded by a 3-min timeout that
/// this falls through to a hard stop. The flag is accepted now so /// falls back to a hard stop). Applies to agents only.
/// the wire/CLI shape is stable when the quiesce lands.
#[arg(long)] #[arg(long)]
graceful: bool, graceful: bool,
}, },

View file

@ -101,7 +101,7 @@ async fn dispatch(req: &HostRequest, coord: Arc<Coordinator>) -> HostResponse {
// graceful-stop queue, to re-expand it). // graceful-stop queue, to re-expand it).
let agents = scoped_agents(scope).await?; let agents = scoped_agents(scope).await?;
let infra = scoped_infra(scope); let infra = scoped_infra(scope);
handle_stop(&agents, &infra, *graceful).await? handle_stop(&coord, &agents, &infra, *graceful).await?
} }
HostRequest::Start { scope } => { HostRequest::Start { scope } => {
let agents = scoped_agents(scope).await?; let agents = scoped_agents(scope).await?;
@ -220,17 +220,40 @@ async fn handle_restart_all() -> Result<HostResponse> {
/// aggregated rather than aborting on the first error, mirroring /// aggregated rather than aborting on the first error, mirroring
/// `handle_restart_all`. Callers resolve the [`LifecycleScope`] to these /// `handle_restart_all`. Callers resolve the [`LifecycleScope`] to these
/// explicit name lists up front — this never sees the "all" flag. /// explicit name lists up front — this never sees the "all" flag.
async fn handle_stop(agents: &[String], infra: &[&str], graceful: bool) -> Result<HostResponse> { ///
/// A `graceful` stop enqueues a `QueueKind::GracefulStop` per agent (signal the
/// harness, run one stop-checkpoint turn, drain, then container stop, with a
/// timeout fallback to a hard stop), mirroring the dashboard `?graceful=1`
/// path. `graceful` applies to agents only - infra containers have no harness
/// turn loop, so they're always hard-stopped.
async fn handle_stop(
coord: &Arc<Coordinator>,
agents: &[String],
infra: &[&str],
graceful: bool,
) -> Result<HostResponse> {
tracing::info!(?agents, ?infra, graceful, "stop"); tracing::info!(?agents, ?infra, graceful, "stop");
let mut ok_items: Vec<String> = Vec::new(); let mut ok_items: Vec<String> = Vec::new();
let mut errors: Vec<String> = Vec::new(); let mut errors: Vec<String> = Vec::new();
let mut enqueued_graceful = false;
for agent in agents { for agent in agents {
// TODO(graceful agent stop): when `graceful`, run the per-agent if graceful {
// quiesce (turn-end → drain → reject new messages) before the kill. // Graceful stop: enqueue the quiesce orchestration rather than a
// The flag is threaded through the wire now; the quiesce itself // hard kill. Serialised through the rebuild queue so it can't race
// lands with the graceful-agent-stop work. // an in-flight rebuild for the same agent, and its per-step
let _ = graceful; // progress surfaces on the queue snapshot + build log.
coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::GracefulStop,
agent.clone(),
crate::rebuild_queue::QueueSource::Manual,
"manual via hivectl graceful stop".to_owned(),
None,
);
ok_items.push(agent.clone());
enqueued_graceful = true;
continue;
}
match lifecycle::kill(agent).await { match lifecycle::kill(agent).await {
Ok(()) => ok_items.push(agent.clone()), Ok(()) => ok_items.push(agent.clone()),
Err(e) => { Err(e) => {
@ -239,6 +262,9 @@ async fn handle_stop(agents: &[String], infra: &[&str], graceful: bool) -> Resul
} }
} }
} }
if enqueued_graceful {
coord.emit_rebuild_queue_snapshot();
}
for &container in infra { for &container in infra {
match crate::priv_client::control_infra_container(container, InfraAction::Stop).await { match crate::priv_client::control_infra_container(container, InfraAction::Stop).await {