hive-c0re: wire hivectl --graceful to enqueue GracefulStop
This commit is contained in:
parent
cba02fc47e
commit
1b81ad423e
3 changed files with 40 additions and 17 deletions
|
|
@ -309,9 +309,7 @@ Stop containers hive-wide in one operator action. Bare `hivectl stop` stops **ev
|
||||||
* `--forge` — The forge container (`hive-forge`)
|
* `--forge` — The forge container (`hive-forge`)
|
||||||
* `--gateway` — The gateway container (`hive-gateway`)
|
* `--gateway` — The gateway container (`hive-gateway`)
|
||||||
* `--matrix` — The matrix container (`hive-matrix`)
|
* `--matrix` — The matrix container (`hive-matrix`)
|
||||||
* `--graceful` — Gracefully quiesce each agent (finish the current turn, drain the inbox) before stopping, instead of a hard stop.
|
* `--graceful` — Gracefully quiesce each agent before stopping, instead of a hard stop. Each agent is enqueued as a `GracefulStop` on the rebuild queue: the harness is signalled, runs one stop-checkpoint turn to flush durable `/state`, drains, then the container is stopped (bounded by a 3-min timeout that falls back to a hard stop). Applies to agents only
|
||||||
|
|
||||||
NOTE: not yet effective — the per-agent quiesce is still being implemented (see the graceful-agent-stop tracker), so today this falls through to a hard stop. The flag is accepted now so the wire/CLI shape is stable when the quiesce lands.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -116,13 +116,12 @@ enum Cmd {
|
||||||
Stop {
|
Stop {
|
||||||
#[command(flatten)]
|
#[command(flatten)]
|
||||||
scope: ScopeArgs,
|
scope: ScopeArgs,
|
||||||
/// Gracefully quiesce each agent (finish the current turn, drain
|
/// Gracefully quiesce each agent before stopping, instead of a
|
||||||
/// the inbox) before stopping, instead of a hard stop.
|
/// hard stop. Each agent is enqueued as a `GracefulStop` on the
|
||||||
///
|
/// rebuild queue: the harness is signalled, runs one
|
||||||
/// NOTE: not yet effective — the per-agent quiesce is still being
|
/// stop-checkpoint turn to flush durable `/state`, drains, then
|
||||||
/// implemented (see the graceful-agent-stop tracker), so today
|
/// the container is stopped (bounded by a 3-min timeout that
|
||||||
/// this falls through to a hard stop. The flag is accepted now so
|
/// falls back to a hard stop). Applies to agents only.
|
||||||
/// the wire/CLI shape is stable when the quiesce lands.
|
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
graceful: bool,
|
graceful: bool,
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -101,7 +101,7 @@ async fn dispatch(req: &HostRequest, coord: Arc<Coordinator>) -> HostResponse {
|
||||||
// graceful-stop queue, to re-expand it).
|
// graceful-stop queue, to re-expand it).
|
||||||
let agents = scoped_agents(scope).await?;
|
let agents = scoped_agents(scope).await?;
|
||||||
let infra = scoped_infra(scope);
|
let infra = scoped_infra(scope);
|
||||||
handle_stop(&agents, &infra, *graceful).await?
|
handle_stop(&coord, &agents, &infra, *graceful).await?
|
||||||
}
|
}
|
||||||
HostRequest::Start { scope } => {
|
HostRequest::Start { scope } => {
|
||||||
let agents = scoped_agents(scope).await?;
|
let agents = scoped_agents(scope).await?;
|
||||||
|
|
@ -220,17 +220,40 @@ async fn handle_restart_all() -> Result<HostResponse> {
|
||||||
/// aggregated rather than aborting on the first error, mirroring
|
/// aggregated rather than aborting on the first error, mirroring
|
||||||
/// `handle_restart_all`. Callers resolve the [`LifecycleScope`] to these
|
/// `handle_restart_all`. Callers resolve the [`LifecycleScope`] to these
|
||||||
/// explicit name lists up front — this never sees the "all" flag.
|
/// explicit name lists up front — this never sees the "all" flag.
|
||||||
async fn handle_stop(agents: &[String], infra: &[&str], graceful: bool) -> Result<HostResponse> {
|
///
|
||||||
|
/// A `graceful` stop enqueues a `QueueKind::GracefulStop` per agent (signal the
|
||||||
|
/// harness, run one stop-checkpoint turn, drain, then container stop, with a
|
||||||
|
/// timeout fallback to a hard stop), mirroring the dashboard `?graceful=1`
|
||||||
|
/// path. `graceful` applies to agents only - infra containers have no harness
|
||||||
|
/// turn loop, so they're always hard-stopped.
|
||||||
|
async fn handle_stop(
|
||||||
|
coord: &Arc<Coordinator>,
|
||||||
|
agents: &[String],
|
||||||
|
infra: &[&str],
|
||||||
|
graceful: bool,
|
||||||
|
) -> Result<HostResponse> {
|
||||||
tracing::info!(?agents, ?infra, graceful, "stop");
|
tracing::info!(?agents, ?infra, graceful, "stop");
|
||||||
let mut ok_items: Vec<String> = Vec::new();
|
let mut ok_items: Vec<String> = Vec::new();
|
||||||
let mut errors: Vec<String> = Vec::new();
|
let mut errors: Vec<String> = Vec::new();
|
||||||
|
let mut enqueued_graceful = false;
|
||||||
|
|
||||||
for agent in agents {
|
for agent in agents {
|
||||||
// TODO(graceful agent stop): when `graceful`, run the per-agent
|
if graceful {
|
||||||
// quiesce (turn-end → drain → reject new messages) before the kill.
|
// Graceful stop: enqueue the quiesce orchestration rather than a
|
||||||
// The flag is threaded through the wire now; the quiesce itself
|
// hard kill. Serialised through the rebuild queue so it can't race
|
||||||
// lands with the graceful-agent-stop work.
|
// an in-flight rebuild for the same agent, and its per-step
|
||||||
let _ = graceful;
|
// progress surfaces on the queue snapshot + build log.
|
||||||
|
coord.rebuild_queue.enqueue(
|
||||||
|
crate::rebuild_queue::QueueKind::GracefulStop,
|
||||||
|
agent.clone(),
|
||||||
|
crate::rebuild_queue::QueueSource::Manual,
|
||||||
|
"manual via hivectl graceful stop".to_owned(),
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
ok_items.push(agent.clone());
|
||||||
|
enqueued_graceful = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
match lifecycle::kill(agent).await {
|
match lifecycle::kill(agent).await {
|
||||||
Ok(()) => ok_items.push(agent.clone()),
|
Ok(()) => ok_items.push(agent.clone()),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|
@ -239,6 +262,9 @@ async fn handle_stop(agents: &[String], infra: &[&str], graceful: bool) -> Resul
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if enqueued_graceful {
|
||||||
|
coord.emit_rebuild_queue_snapshot();
|
||||||
|
}
|
||||||
|
|
||||||
for &container in infra {
|
for &container in infra {
|
||||||
match crate::priv_client::control_infra_container(container, InfraAction::Stop).await {
|
match crate::priv_client::control_infra_container(container, InfraAction::Stop).await {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue