fix: close second review round on the queue-routed CLI
- subvol upgrade waits for the queued stop DAG before migrating (was snapshotting + swapping state under a live bind mount) and for the restart job after - history trim gets a 5-min grace for fresh terminals so broad stop/start waits can't miss a failed DAG evicted by the per-template cap (cap still applies past the grace) - restart-all returns its DAG ids so hivectl actually waits - hard stops await their agent DAGs (bounded) before infra goes down, restoring the agents-before-infra invariant - hivectl wait uses node-level terminality so the after-any recovery reconcile is watched to completion; infra render errors no longer skip watching already-queued agent DAGs - fold hive-bash-mcp's last local now_unix into wire_time
This commit is contained in:
parent
0e4b5a1120
commit
2486251b32
5 changed files with 106 additions and 26 deletions
|
|
@ -291,19 +291,22 @@ async fn handle_restart_all(coord: &Arc<Coordinator>) -> Result<HostResponse> {
|
|||
tracing::info!("restart-all");
|
||||
let agents = lifecycle::list().await?;
|
||||
let mut ok_agents: Vec<String> = Vec::new();
|
||||
let mut queued: Vec<u64> = Vec::new();
|
||||
for agent in &agents {
|
||||
let Some(logical) = agent.strip_prefix(lifecycle::AGENT_PREFIX) else {
|
||||
continue;
|
||||
};
|
||||
crate::job_queue::submit::restart(
|
||||
queued.push(crate::job_queue::submit::restart(
|
||||
coord,
|
||||
logical,
|
||||
crate::job_queue::Source::Manual,
|
||||
"manual restart via hivectl restart-all".to_owned(),
|
||||
);
|
||||
));
|
||||
ok_agents.push(logical.to_owned());
|
||||
}
|
||||
Ok(HostResponse::list(ok_agents))
|
||||
let mut resp = HostResponse::list(ok_agents);
|
||||
resp.queued_dags = Some(queued);
|
||||
Ok(resp)
|
||||
}
|
||||
|
||||
/// Stop the given `agents` (resolved logical names) then `infra` containers
|
||||
|
|
@ -356,6 +359,14 @@ async fn handle_stop(
|
|||
ok_items.push(agent.clone());
|
||||
}
|
||||
|
||||
// Agents go down before infra so they're not mid-request against a
|
||||
// forge/matrix that's already gone. Hard stops are quick kills —
|
||||
// await their DAGs (bounded) before touching infra. Graceful stops
|
||||
// keep the immediate return (drains take minutes and the
|
||||
// agents-then-infra race pre-existed there).
|
||||
if !graceful && !infra.is_empty() {
|
||||
await_dags(coord, &queued, std::time::Duration::from_mins(2)).await;
|
||||
}
|
||||
for &container in infra {
|
||||
let name = container.unit_name();
|
||||
match crate::priv_client::control_infra_container(container, InfraAction::Stop).await {
|
||||
|
|
@ -372,6 +383,28 @@ async fn handle_stop(
|
|||
Ok(resp)
|
||||
}
|
||||
|
||||
/// Best-effort server-side wait for a set of DAGs to settle terminal,
|
||||
/// bounded by `timeout` — used to preserve ordering invariants inside
|
||||
/// one request (agent stops before infra stops) without trusting the
|
||||
/// client to wait.
|
||||
async fn await_dags(coord: &Arc<Coordinator>, ids: &[u64], timeout: std::time::Duration) {
|
||||
let deadline = std::time::Instant::now() + timeout;
|
||||
loop {
|
||||
let snap = coord.job_queue.snapshot();
|
||||
let pending = ids
|
||||
.iter()
|
||||
.any(|id| snap.iter().any(|d| d.id == *id && !d.state.is_terminal()));
|
||||
if !pending {
|
||||
return;
|
||||
}
|
||||
if std::time::Instant::now() >= deadline {
|
||||
tracing::warn!(?ids, "await_dags: timed out; proceeding");
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Start the given `infra` containers then `agents` (`hivectl start`) — the
|
||||
/// inverse of [`handle_stop`]. Infra comes up before agents so the agents
|
||||
/// find forge/matrix/gateway ready. Per-target failures aggregated. Callers
|
||||
|
|
|
|||
Loading…
Reference in a new issue