hive-c0re: wire hivectl --graceful to enqueue GracefulStop
This commit is contained in:
parent
cba02fc47e
commit
1b81ad423e
3 changed files with 40 additions and 17 deletions
|
|
@ -309,9 +309,7 @@ Stop containers hive-wide in one operator action. Bare `hivectl stop` stops **ev
|
|||
* `--forge` — The forge container (`hive-forge`)
|
||||
* `--gateway` — The gateway container (`hive-gateway`)
|
||||
* `--matrix` — The matrix container (`hive-matrix`)
|
||||
* `--graceful` — Gracefully quiesce each agent (finish the current turn, drain the inbox) before stopping, instead of a hard stop.
|
||||
|
||||
NOTE: not yet effective — the per-agent quiesce is still being implemented (see the graceful-agent-stop tracker), so today this falls through to a hard stop. The flag is accepted now so the wire/CLI shape is stable when the quiesce lands.
|
||||
* `--graceful` — Gracefully quiesce each agent before stopping, instead of a hard stop. Each agent is enqueued as a `GracefulStop` on the rebuild queue: the harness is signalled, runs one stop-checkpoint turn to flush durable `/state`, drains, then the container is stopped (bounded by a 3-min timeout that falls back to a hard stop). Applies to agents only
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -116,13 +116,12 @@ enum Cmd {
|
|||
Stop {
|
||||
#[command(flatten)]
|
||||
scope: ScopeArgs,
|
||||
/// Gracefully quiesce each agent (finish the current turn, drain
|
||||
/// the inbox) before stopping, instead of a hard stop.
|
||||
///
|
||||
/// NOTE: not yet effective — the per-agent quiesce is still being
|
||||
/// implemented (see the graceful-agent-stop tracker), so today
|
||||
/// this falls through to a hard stop. The flag is accepted now so
|
||||
/// the wire/CLI shape is stable when the quiesce lands.
|
||||
/// Gracefully quiesce each agent before stopping, instead of a
|
||||
/// hard stop. Each agent is enqueued as a `GracefulStop` on the
|
||||
/// rebuild queue: the harness is signalled, runs one
|
||||
/// stop-checkpoint turn to flush durable `/state`, drains, then
|
||||
/// the container is stopped (bounded by a 3-min timeout that
|
||||
/// falls back to a hard stop). Applies to agents only.
|
||||
#[arg(long)]
|
||||
graceful: bool,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ async fn dispatch(req: &HostRequest, coord: Arc<Coordinator>) -> HostResponse {
|
|||
// graceful-stop queue, to re-expand it).
|
||||
let agents = scoped_agents(scope).await?;
|
||||
let infra = scoped_infra(scope);
|
||||
handle_stop(&agents, &infra, *graceful).await?
|
||||
handle_stop(&coord, &agents, &infra, *graceful).await?
|
||||
}
|
||||
HostRequest::Start { scope } => {
|
||||
let agents = scoped_agents(scope).await?;
|
||||
|
|
@ -220,17 +220,40 @@ async fn handle_restart_all() -> Result<HostResponse> {
|
|||
/// aggregated rather than aborting on the first error, mirroring
|
||||
/// `handle_restart_all`. Callers resolve the [`LifecycleScope`] to these
|
||||
/// explicit name lists up front — this never sees the "all" flag.
|
||||
async fn handle_stop(agents: &[String], infra: &[&str], graceful: bool) -> Result<HostResponse> {
|
||||
///
|
||||
/// A `graceful` stop enqueues a `QueueKind::GracefulStop` per agent (signal the
|
||||
/// harness, run one stop-checkpoint turn, drain, then container stop, with a
|
||||
/// timeout fallback to a hard stop), mirroring the dashboard `?graceful=1`
|
||||
/// path. `graceful` applies to agents only - infra containers have no harness
|
||||
/// turn loop, so they're always hard-stopped.
|
||||
async fn handle_stop(
|
||||
coord: &Arc<Coordinator>,
|
||||
agents: &[String],
|
||||
infra: &[&str],
|
||||
graceful: bool,
|
||||
) -> Result<HostResponse> {
|
||||
tracing::info!(?agents, ?infra, graceful, "stop");
|
||||
let mut ok_items: Vec<String> = Vec::new();
|
||||
let mut errors: Vec<String> = Vec::new();
|
||||
let mut enqueued_graceful = false;
|
||||
|
||||
for agent in agents {
|
||||
// TODO(graceful agent stop): when `graceful`, run the per-agent
|
||||
// quiesce (turn-end → drain → reject new messages) before the kill.
|
||||
// The flag is threaded through the wire now; the quiesce itself
|
||||
// lands with the graceful-agent-stop work.
|
||||
let _ = graceful;
|
||||
if graceful {
|
||||
// Graceful stop: enqueue the quiesce orchestration rather than a
|
||||
// hard kill. Serialised through the rebuild queue so it can't race
|
||||
// an in-flight rebuild for the same agent, and its per-step
|
||||
// progress surfaces on the queue snapshot + build log.
|
||||
coord.rebuild_queue.enqueue(
|
||||
crate::rebuild_queue::QueueKind::GracefulStop,
|
||||
agent.clone(),
|
||||
crate::rebuild_queue::QueueSource::Manual,
|
||||
"manual via hivectl graceful stop".to_owned(),
|
||||
None,
|
||||
);
|
||||
ok_items.push(agent.clone());
|
||||
enqueued_graceful = true;
|
||||
continue;
|
||||
}
|
||||
match lifecycle::kill(agent).await {
|
||||
Ok(()) => ok_items.push(agent.clone()),
|
||||
Err(e) => {
|
||||
|
|
@ -239,6 +262,9 @@ async fn handle_stop(agents: &[String], infra: &[&str], graceful: bool) -> Resul
|
|||
}
|
||||
}
|
||||
}
|
||||
if enqueued_graceful {
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
|
||||
for &container in infra {
|
||||
match crate::priv_client::control_infra_container(container, InfraAction::Stop).await {
|
||||
|
|
|
|||
Loading…
Reference in a new issue