feat(#2398): dagify hivectl restart (RestartScoped)
hivectl restart --agent NAME previously composed stop() then start() as
two separate client-side daemon calls glued by CLI-process control flow
— not one DAG, and a dropped hivectl connection mid-restart (ssh drop,
Ctrl-C) left the agent stopped with no automatic follow-up. mara flagged
this as the first target for the 'dagify hivectl commands' issue.
New HostRequest::RestartScoped{scope, graceful} handles it server-side:
each targeted agent now rides exactly one atomic Restart-template DAG
(same one hivectl agents restart / restart-all already use) in the
common non-graceful case. --graceful has no single-DAG template yet, so
it submits the graceful-stop DAGs, awaits them server-side, then submits
the start DAGs — still one daemon call end to end, just not yet a single
DAG (noted as a follow-up). Infra containers restart synchronously as
before (no lease/DAG concept for them).
CLI-side restart() now just makes the one call + waits, same output
shape as before via render_lifecycle.
This commit is contained in:
parent
5853ce2c8d
commit
4cfa040154
3 changed files with 118 additions and 10 deletions
|
|
@ -1642,22 +1642,31 @@ async fn start(socket: &Path, scope: hive_host_sock::LifecycleScope, no_wait: bo
|
||||||
rendered
|
rendered
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Restart = `stop` then `start` over the same scope, composed client-side
|
/// Restart — one `RestartScoped` daemon call, server-side DAG-based (see
|
||||||
/// from the two daemon ops (no dedicated wire op). The stop phase honours
|
/// issue tracker "dagify hivectl commands"). Each targeted agent rides one
|
||||||
/// `--graceful`; if it reports a failure (`stop` returns `Err`) the `?`
|
/// atomic `Restart` DAG (or, with `--graceful`, a server-awaited
|
||||||
/// short-circuits before the start phase, so a half-stopped hive isn't
|
/// graceful-stop DAG followed by a start DAG); infra containers restart
|
||||||
/// blindly started over — the operator sees the stop errors and can recover.
|
/// synchronously. Unlike the old client-side stop-then-start compose, a
|
||||||
|
/// dropped `hivectl` connection mid-restart no longer leaves an agent
|
||||||
|
/// stopped with no automatic follow-up — the daemon owns the whole
|
||||||
|
/// sequence once this call is made.
|
||||||
///
|
///
|
||||||
/// No `--no-wait` here on purpose: the stop DAGs must complete before
|
/// No `--no-wait` here on purpose, same as before: the operator wants to
|
||||||
/// the start submits, otherwise the start's `wanted = Up` write would
|
/// see the restart actually land, not just get queued.
|
||||||
/// land before the queued stops execute and turn them into noops.
|
|
||||||
async fn restart(
|
async fn restart(
|
||||||
socket: &Path,
|
socket: &Path,
|
||||||
scope: hive_host_sock::LifecycleScope,
|
scope: hive_host_sock::LifecycleScope,
|
||||||
graceful: bool,
|
graceful: bool,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
stop(socket, scope.clone(), graceful, false).await?;
|
let resp = hive_c0re::client::request(
|
||||||
start(socket, scope, false).await
|
socket,
|
||||||
|
hive_host_sock::HostRequest::RestartScoped { scope, graceful },
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.with_context(|| format!("connect to daemon socket {}", socket.display()))?;
|
||||||
|
let rendered = render_lifecycle(&resp, "restart queued");
|
||||||
|
wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), false).await?;
|
||||||
|
rendered
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A [`LifecycleScope`](hive_host_sock::LifecycleScope) targeting exactly one
|
/// A [`LifecycleScope`](hive_host_sock::LifecycleScope) targeting exactly one
|
||||||
|
|
|
||||||
|
|
@ -94,6 +94,9 @@ async fn dispatch(req: &HostRequest, coord: Arc<Coordinator>) -> HostResponse {
|
||||||
HostRequest::Kill { name } => submit_single(&coord, name, Verb::Kill),
|
HostRequest::Kill { name } => submit_single(&coord, name, Verb::Kill),
|
||||||
HostRequest::Restart { name } => submit_single(&coord, name, Verb::Restart),
|
HostRequest::Restart { name } => submit_single(&coord, name, Verb::Restart),
|
||||||
HostRequest::RestartAll => handle_restart_all(&coord).await?,
|
HostRequest::RestartAll => handle_restart_all(&coord).await?,
|
||||||
|
HostRequest::RestartScoped { scope, graceful } => {
|
||||||
|
handle_restart_scoped(&coord, scope, *graceful).await?
|
||||||
|
}
|
||||||
HostRequest::Stop { scope, graceful } => {
|
HostRequest::Stop { scope, graceful } => {
|
||||||
// Resolve the scope to explicit container names at the entry
|
// Resolve the scope to explicit container names at the entry
|
||||||
// point, then operate on names — never pass the bare "all
|
// point, then operate on names — never pass the bare "all
|
||||||
|
|
@ -631,6 +634,85 @@ async fn handle_start(
|
||||||
Ok(resp)
|
Ok(resp)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Restart containers hive-wide (`hivectl restart`) — the DAG-based
|
||||||
|
/// sibling of [`handle_stop`]/[`handle_start`], replacing the old
|
||||||
|
/// client-side stop-then-start composition (issue tracker "dagify hivectl
|
||||||
|
/// commands"). Each targeted agent gets exactly one atomic `Restart`
|
||||||
|
/// template DAG in the common (non-graceful) case, so it survives a
|
||||||
|
/// dropped `hivectl` connection the same way `handle_restart_all` already
|
||||||
|
/// does. `graceful` has no single-DAG template yet, so it falls back to
|
||||||
|
/// submitting the graceful-stop DAGs, waiting them out server-side, then
|
||||||
|
/// submitting the start DAGs — still one daemon call end to end, unlike
|
||||||
|
/// the old CLI-side compose. Infra containers have no lease/DAG and
|
||||||
|
/// restart synchronously (stop then start).
|
||||||
|
async fn handle_restart_scoped(
|
||||||
|
coord: &Arc<Coordinator>,
|
||||||
|
scope: &LifecycleScope,
|
||||||
|
graceful: bool,
|
||||||
|
) -> Result<HostResponse> {
|
||||||
|
tracing::info!(?scope, graceful, "restart");
|
||||||
|
let agents = scoped_agents(scope).await?;
|
||||||
|
let infra = scoped_infra(scope);
|
||||||
|
let mut ok_items: Vec<String> = Vec::new();
|
||||||
|
let mut errors: Vec<String> = Vec::new();
|
||||||
|
let mut queued: Vec<u64> = Vec::new();
|
||||||
|
|
||||||
|
if graceful {
|
||||||
|
let mut stop_ids = Vec::new();
|
||||||
|
for agent in &agents {
|
||||||
|
stop_ids.push(crate::job_queue::submit::graceful_stop(
|
||||||
|
coord,
|
||||||
|
agent,
|
||||||
|
crate::job_queue::Source::Manual,
|
||||||
|
"manual via hivectl restart --graceful".to_owned(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// Graceful drains can take a while; bound the wait generously —
|
||||||
|
// same shape as `handle_stop`'s hard-stop-before-infra wait, just
|
||||||
|
// a longer ceiling since drains are minutes, not seconds.
|
||||||
|
await_dags(coord, &stop_ids, std::time::Duration::from_mins(10)).await;
|
||||||
|
for agent in &agents {
|
||||||
|
queued.push(crate::job_queue::submit::start(
|
||||||
|
coord,
|
||||||
|
agent,
|
||||||
|
crate::job_queue::Source::Manual,
|
||||||
|
"manual via hivectl restart --graceful".to_owned(),
|
||||||
|
));
|
||||||
|
ok_items.push(agent.clone());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for agent in &agents {
|
||||||
|
queued.push(crate::job_queue::submit::restart(
|
||||||
|
coord,
|
||||||
|
agent,
|
||||||
|
crate::job_queue::Source::Manual,
|
||||||
|
"manual restart via hivectl restart".to_owned(),
|
||||||
|
));
|
||||||
|
ok_items.push(agent.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for &container in &infra {
|
||||||
|
let name = container.unit_name();
|
||||||
|
let res = async {
|
||||||
|
crate::priv_client::control_infra_container(container, InfraAction::Stop).await?;
|
||||||
|
crate::priv_client::control_infra_container(container, InfraAction::Start).await
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
match res {
|
||||||
|
Ok(()) => ok_items.push(name.to_owned()),
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(%name, error = ?e, "restart: infra restart failed");
|
||||||
|
errors.push(format!("{name}: {e:#}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut resp = finish_lifecycle(ok_items, &errors);
|
||||||
|
resp.queued_dags = Some(queued);
|
||||||
|
Ok(resp)
|
||||||
|
}
|
||||||
|
|
||||||
/// Resolve which sub-agent logical names a scope targets: every live
|
/// Resolve which sub-agent logical names a scope targets: every live
|
||||||
/// container (from `lifecycle::list`) when `agents` is set or the scope is
|
/// container (from `lifecycle::list`) when `agents` is set or the scope is
|
||||||
/// "everything", plus any explicit `agent_names`. Returns de-duplicated
|
/// "everything", plus any explicit `agent_names`. Returns de-duplicated
|
||||||
|
|
|
||||||
|
|
@ -39,6 +39,23 @@ pub enum HostRequest {
|
||||||
/// wrapper for `hivectl agents restart-all`; iterates the live
|
/// wrapper for `hivectl agents restart-all`; iterates the live
|
||||||
/// container list and restarts each one.
|
/// container list and restarts each one.
|
||||||
RestartAll,
|
RestartAll,
|
||||||
|
/// Restart containers hive-wide (`hivectl restart`), scoped like
|
||||||
|
/// `Stop`/`Start`. Each targeted agent rides exactly one DAG server-side
|
||||||
|
/// (the `Restart` template, hard stop + reconcile — or, when `graceful`
|
||||||
|
/// is set, a submitted graceful-stop DAG the server itself waits out
|
||||||
|
/// before submitting the start DAG) rather than the old client-side
|
||||||
|
/// stop-then-start composition: a dropped `hivectl` connection mid-way
|
||||||
|
/// used to leave the agent stopped with no automatic follow-up, since
|
||||||
|
/// nothing durable remembered "finish the restart" once the CLI process
|
||||||
|
/// was gone. Infra containers have no lease/DAG and restart
|
||||||
|
/// synchronously (stop then start), same as before. Scope semantics
|
||||||
|
/// match `Stop`/`Start` (all-false = everything).
|
||||||
|
RestartScoped {
|
||||||
|
#[serde(default)]
|
||||||
|
scope: LifecycleScope,
|
||||||
|
#[serde(default)]
|
||||||
|
graceful: bool,
|
||||||
|
},
|
||||||
/// Apply pending config to a managed container.
|
/// Apply pending config to a managed container.
|
||||||
Rebuild { name: String },
|
Rebuild { name: String },
|
||||||
/// List managed containers.
|
/// List managed containers.
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue