fix: close second review round on the queue-routed CLI

- subvol upgrade waits for the queued stop DAG before migrating (was
  snapshotting + swapping state under a live bind mount) and for the
  restart job after
- history trim gets a 5-min grace for fresh terminals so broad
  stop/start waits can't miss a failed DAG evicted by the per-template
  cap (cap still applies past the grace)
- restart-all returns its DAG ids so hivectl actually waits
- hard stops await their agent DAGs (bounded) before infra goes down,
  restoring the agents-before-infra invariant
- hivectl wait uses node-level terminality so the after-any recovery
  reconcile is watched to completion; infra render errors no longer
  skip watching already-queued agent DAGs
- fold hive-bash-mcp's last local now_unix into wire_time
This commit is contained in:
müde 2026-07-06 22:57:28 +02:00
commit 2486251b32
5 changed files with 106 additions and 26 deletions

View file

@ -39,6 +39,14 @@ pub use model::{
/// per template in the snapshot, matching the old per-kind history cap.
const MAX_HISTORY_PER_TEMPLATE: usize = 5;
/// Terminal DAGs younger than this are exempt from the per-template
/// history cap. A broad `hivectl stop`/`start` submits many
/// same-template DAGs that can all settle within one poll interval —
/// without the grace, the cap would evict some before the ~1s
/// `QueueDag` poller ever observes their terminal state, silently
/// swallowing failures.
const HISTORY_GRACE_SECS: i64 = 300;
/// Cap on stored node error strings.
const MAX_ERROR_LEN: usize = 2_000;
@ -399,7 +407,7 @@ impl JobQueue {
for agent in freed {
inner.leases.remove(&agent);
}
Self::trim_history(inner);
Self::trim_history(inner, now_unix() - HISTORY_GRACE_SECS);
}
/// Cancel a DAG that hasn't started yet (roll-up `Queued`): every
@ -540,11 +548,12 @@ impl JobQueue {
}
/// Keep only the newest `MAX_HISTORY_PER_TEMPLATE` terminal DAGs
/// per template. Live DAGs are never evicted — and neither is a
/// terminal parent that still has live children (a fan-out parent
/// is terminal the moment its `MetaLock` completes; evicting it
/// while cascade rebuilds run would orphan their dashboard group).
fn trim_history(inner: &mut Inner) {
/// per template. Never evicted: live DAGs; terminal parents with
/// live children (a fan-out parent is terminal the moment its
/// `MetaLock` completes — evicting it while cascade rebuilds run
/// would orphan their dashboard group); and terminal DAGs that
/// finished after `grace_cutoff` (see [`HISTORY_GRACE_SECS`]).
fn trim_history(inner: &mut Inner, grace_cutoff: i64) {
let live_parents: std::collections::HashSet<u64> = inner
.dags
.iter()
@ -560,6 +569,10 @@ impl JobQueue {
if !d.is_terminal() || live_parents.contains(&d.id) {
return true;
}
let finished = d.nodes.iter().filter_map(|n| n.finished_at).max();
if finished.is_none_or(|t| t > grace_cutoff) {
return true;
}
let n = counts.entry(d.template).or_insert(0);
*n += 1;
*n <= MAX_HISTORY_PER_TEMPLATE
@ -568,4 +581,12 @@ impl JobQueue {
.collect();
inner.dags = kept.into_iter().rev().collect();
}
/// Test hook: trim with the grace window disabled, so eviction
/// behavior is assertable without aging real timestamps.
#[cfg(test)]
pub(crate) fn trim_ignoring_grace(&self) {
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
Self::trim_history(&mut inner, i64::MAX);
}
}

View file

@ -804,6 +804,16 @@ fn history_evicts_old_terminals_per_template() {
let c = claim_one(&q);
q.complete_node(id, c.node_id, Ok(()));
}
// Fresh terminals are inside the grace window: nothing evicts yet,
// so a ~1s QueueDag poller can still observe every terminal state
// (a broad stop/start settles many same-template DAGs at once).
assert_eq!(
q.snapshot().len(),
8,
"grace window protects fresh terminals"
);
// Past the grace window the per-template cap applies.
q.trim_ignoring_grace();
assert_eq!(q.snapshot().len(), 5, "per-template history cap");
assert_eq!(q.live_count(), 0);
}