fix: close second review round on the queue-routed CLI

- subvol upgrade waits for the queued stop DAG before migrating (was
  snapshotting + swapping state under a live bind mount) and for the
  restart job after
- history trim gets a 5-min grace for fresh terminals so broad
  stop/start waits can't miss a failed DAG evicted by the per-template
  cap (cap still applies past the grace)
- restart-all returns its DAG ids so hivectl actually waits
- hard stops await their agent DAGs (bounded) before infra goes down,
  restoring the agents-before-infra invariant
- hivectl wait uses node-level terminality so the after-any recovery
  reconcile is watched to completion; infra render errors no longer
  skip watching already-queued agent DAGs
- fold hive-bash-mcp's last local now_unix into wire_time
This commit is contained in:
müde 2026-07-06 22:57:28 +02:00
commit 2486251b32
5 changed files with 106 additions and 26 deletions

View file

@ -291,19 +291,22 @@ async fn handle_restart_all(coord: &Arc<Coordinator>) -> Result<HostResponse> {
tracing::info!("restart-all");
let agents = lifecycle::list().await?;
let mut ok_agents: Vec<String> = Vec::new();
let mut queued: Vec<u64> = Vec::new();
for agent in &agents {
let Some(logical) = agent.strip_prefix(lifecycle::AGENT_PREFIX) else {
continue;
};
crate::job_queue::submit::restart(
queued.push(crate::job_queue::submit::restart(
coord,
logical,
crate::job_queue::Source::Manual,
"manual restart via hivectl restart-all".to_owned(),
);
));
ok_agents.push(logical.to_owned());
}
Ok(HostResponse::list(ok_agents))
let mut resp = HostResponse::list(ok_agents);
resp.queued_dags = Some(queued);
Ok(resp)
}
/// Stop the given `agents` (resolved logical names) then `infra` containers
@ -356,6 +359,14 @@ async fn handle_stop(
ok_items.push(agent.clone());
}
// Agents go down before infra so they're not mid-request against a
// forge/matrix that's already gone. Hard stops are quick kills —
// await their DAGs (bounded) before touching infra. Graceful stops
// keep the immediate return (drains take minutes and the
// agents-then-infra race pre-existed there).
if !graceful && !infra.is_empty() {
await_dags(coord, &queued, std::time::Duration::from_mins(2)).await;
}
for &container in infra {
let name = container.unit_name();
match crate::priv_client::control_infra_container(container, InfraAction::Stop).await {
@ -372,6 +383,28 @@ async fn handle_stop(
Ok(resp)
}
/// Best-effort server-side wait for a set of DAGs to settle terminal,
/// bounded by `timeout` — used to preserve ordering invariants inside
/// one request (agent stops before infra stops) without trusting the
/// client to wait.
async fn await_dags(coord: &Arc<Coordinator>, ids: &[u64], timeout: std::time::Duration) {
let deadline = std::time::Instant::now() + timeout;
loop {
let snap = coord.job_queue.snapshot();
let pending = ids
.iter()
.any(|id| snap.iter().any(|d| d.id == *id && !d.state.is_terminal()));
if !pending {
return;
}
if std::time::Instant::now() >= deadline {
tracing::warn!(?ids, "await_dags: timed out; proceeding");
return;
}
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
}
}
/// Start the given `infra` containers then `agents` (`hivectl start`) — the
/// inverse of [`handle_stop`]. Infra comes up before agents so the agents
/// find forge/matrix/gateway ready. Per-target failures aggregated. Callers