fix: close second review round on the queue-routed CLI

- subvol upgrade waits for the queued stop DAG before migrating (was
  snapshotting + swapping state under a live bind mount) and for the
  restart job after
- history trim gets a 5-min grace for fresh terminals so broad
  stop/start waits can't miss a failed DAG evicted by the per-template
  cap (cap still applies past the grace)
- restart-all returns its DAG ids so hivectl actually waits
- hard stops await their agent DAGs (bounded) before infra goes down,
  restoring the agents-before-infra invariant
- hivectl wait uses node-level terminality so the after-any recovery
  reconcile is watched to completion; infra render errors no longer
  skip watching already-queued agent DAGs
- fold hive-bash-mcp's last local now_unix into wire_time
This commit is contained in:
müde 2026-07-06 22:57:28 +02:00
commit 2486251b32
5 changed files with 106 additions and 26 deletions

View file

@ -1482,12 +1482,15 @@ async fn wait_for_dags(socket: &Path, ids: Vec<u64>, no_wait: bool) -> Result<()
println!("{line}");
last.insert(d.id, line);
}
match d.state {
hive_sh4re::jobs::State::Failed => {
failed.push(format!("{} {}", d.kind.as_str(), d.agent));
}
hive_sh4re::jobs::State::Done | hive_sh4re::jobs::State::Cancelled => {}
_ => all_terminal = false,
// Node-level terminality, not the roll-up: a DAG rolls
// up `failed` the moment one node fails while its
// after-any recovery node (rebuild's tail Reconcile)
// may still be running — keep watching so the operator
// sees whether the agent came back.
if !d.nodes.iter().all(|n| n.state.is_terminal()) {
all_terminal = false;
} else if d.state == hive_sh4re::jobs::State::Failed {
failed.push(format!("{} {}", d.kind.as_str(), d.agent));
}
}
if all_terminal {
@ -1651,16 +1654,21 @@ async fn stop(
hive_c0re::client::request(socket, hive_sh4re::HostRequest::Stop { scope, graceful })
.await
.with_context(|| format!("connect to daemon socket {}", socket.display()))?;
render_lifecycle(&resp, "stop queued")?;
wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), no_wait).await
// Render first, but even when an infra failure makes it bail,
// watch the already-queued agent DAGs before surfacing the error —
// they run regardless.
let rendered = render_lifecycle(&resp, "stop queued");
wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), no_wait).await?;
rendered
}
async fn start(socket: &Path, scope: hive_sh4re::LifecycleScope, no_wait: bool) -> Result<()> {
let resp = hive_c0re::client::request(socket, hive_sh4re::HostRequest::Start { scope })
.await
.with_context(|| format!("connect to daemon socket {}", socket.display()))?;
render_lifecycle(&resp, "start queued")?;
wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), no_wait).await
let rendered = render_lifecycle(&resp, "start queued");
wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), no_wait).await?;
rendered
}
/// Restart = `stop` then `start` over the same scope, composed client-side
@ -1723,6 +1731,12 @@ async fn subvol_upgrade(socket: &Path, name: &str, yes: bool) -> Result<()> {
stop_resp.error.as_deref().unwrap_or("unknown error")
);
}
// The stop is a queued DAG now — the migration below snapshots +
// swaps the state dir and MUST NOT run under a live bind mount, so
// wait for the stop to actually execute before touching anything.
wait_for_dags(socket, stop_resp.queued_dags.unwrap_or_default(), false)
.await
.with_context(|| format!("waiting for {name} to stop before the migration"))?;
println!("migrating {name} state dir to a btrfs subvolume…");
let upgrade = hive_c0re::priv_client::upgrade_agent_subvolume(name).await;
@ -1762,6 +1776,14 @@ async fn subvol_upgrade(socket: &Path, name: &str, yes: bool) -> Result<()> {
start_resp.error.as_deref().unwrap_or("unknown error")
);
}
wait_for_dags(socket, start_resp.queued_dags.unwrap_or_default(), false)
.await
.with_context(|| {
format!(
"{name} migrated to a btrfs subvolume, but its restart job failed — run \
`hivectl start --agent {name}` to retry"
)
})?;
println!("upgraded {name} to a btrfs subvolume and restarted it");
Ok(())
}