fix: close second review round on the queue-routed CLI
- subvol upgrade waits for the queued stop DAG before migrating (was snapshotting + swapping state under a live bind mount) and for the restart job after - history trim gets a 5-min grace for fresh terminals so broad stop/start waits can't miss a failed DAG evicted by the per-template cap (cap still applies past the grace) - restart-all returns its DAG ids so hivectl actually waits - hard stops await their agent DAGs (bounded) before infra goes down, restoring the agents-before-infra invariant - hivectl wait uses node-level terminality so the after-any recovery reconcile is watched to completion; infra render errors no longer skip watching already-queued agent DAGs - fold hive-bash-mcp's last local now_unix into wire_time
This commit is contained in:
parent
0e4b5a1120
commit
2486251b32
5 changed files with 106 additions and 26 deletions
|
|
@ -1482,12 +1482,15 @@ async fn wait_for_dags(socket: &Path, ids: Vec<u64>, no_wait: bool) -> Result<()
|
|||
println!("{line}");
|
||||
last.insert(d.id, line);
|
||||
}
|
||||
match d.state {
|
||||
hive_sh4re::jobs::State::Failed => {
|
||||
failed.push(format!("{} {}", d.kind.as_str(), d.agent));
|
||||
}
|
||||
hive_sh4re::jobs::State::Done | hive_sh4re::jobs::State::Cancelled => {}
|
||||
_ => all_terminal = false,
|
||||
// Node-level terminality, not the roll-up: a DAG rolls
|
||||
// up `failed` the moment one node fails while its
|
||||
// after-any recovery node (rebuild's tail Reconcile)
|
||||
// may still be running — keep watching so the operator
|
||||
// sees whether the agent came back.
|
||||
if !d.nodes.iter().all(|n| n.state.is_terminal()) {
|
||||
all_terminal = false;
|
||||
} else if d.state == hive_sh4re::jobs::State::Failed {
|
||||
failed.push(format!("{} {}", d.kind.as_str(), d.agent));
|
||||
}
|
||||
}
|
||||
if all_terminal {
|
||||
|
|
@ -1651,16 +1654,21 @@ async fn stop(
|
|||
hive_c0re::client::request(socket, hive_sh4re::HostRequest::Stop { scope, graceful })
|
||||
.await
|
||||
.with_context(|| format!("connect to daemon socket {}", socket.display()))?;
|
||||
render_lifecycle(&resp, "stop queued")?;
|
||||
wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), no_wait).await
|
||||
// Render first, but even when an infra failure makes it bail,
|
||||
// watch the already-queued agent DAGs before surfacing the error —
|
||||
// they run regardless.
|
||||
let rendered = render_lifecycle(&resp, "stop queued");
|
||||
wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), no_wait).await?;
|
||||
rendered
|
||||
}
|
||||
|
||||
async fn start(socket: &Path, scope: hive_sh4re::LifecycleScope, no_wait: bool) -> Result<()> {
|
||||
let resp = hive_c0re::client::request(socket, hive_sh4re::HostRequest::Start { scope })
|
||||
.await
|
||||
.with_context(|| format!("connect to daemon socket {}", socket.display()))?;
|
||||
render_lifecycle(&resp, "start queued")?;
|
||||
wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), no_wait).await
|
||||
let rendered = render_lifecycle(&resp, "start queued");
|
||||
wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), no_wait).await?;
|
||||
rendered
|
||||
}
|
||||
|
||||
/// Restart = `stop` then `start` over the same scope, composed client-side
|
||||
|
|
@ -1723,6 +1731,12 @@ async fn subvol_upgrade(socket: &Path, name: &str, yes: bool) -> Result<()> {
|
|||
stop_resp.error.as_deref().unwrap_or("unknown error")
|
||||
);
|
||||
}
|
||||
// The stop is a queued DAG now — the migration below snapshots +
|
||||
// swaps the state dir and MUST NOT run under a live bind mount, so
|
||||
// wait for the stop to actually execute before touching anything.
|
||||
wait_for_dags(socket, stop_resp.queued_dags.unwrap_or_default(), false)
|
||||
.await
|
||||
.with_context(|| format!("waiting for {name} to stop before the migration"))?;
|
||||
|
||||
println!("migrating {name} state dir to a btrfs subvolume…");
|
||||
let upgrade = hive_c0re::priv_client::upgrade_agent_subvolume(name).await;
|
||||
|
|
@ -1762,6 +1776,14 @@ async fn subvol_upgrade(socket: &Path, name: &str, yes: bool) -> Result<()> {
|
|||
start_resp.error.as_deref().unwrap_or("unknown error")
|
||||
);
|
||||
}
|
||||
wait_for_dags(socket, start_resp.queued_dags.unwrap_or_default(), false)
|
||||
.await
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"{name} migrated to a btrfs subvolume, but its restart job failed — run \
|
||||
`hivectl start --agent {name}` to retry"
|
||||
)
|
||||
})?;
|
||||
println!("upgraded {name} to a btrfs subvolume and restarted it");
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue