fix: close second review round on the queue-routed CLI
- subvol upgrade waits for the queued stop DAG before migrating (was snapshotting + swapping state under a live bind mount) and for the restart job after - history trim gets a 5-min grace for fresh terminals so broad stop/start waits can't miss a failed DAG evicted by the per-template cap (cap still applies past the grace) - restart-all returns its DAG ids so hivectl actually waits - hard stops await their agent DAGs (bounded) before infra goes down, restoring the agents-before-infra invariant - hivectl wait uses node-level terminality so the after-any recovery reconcile is watched to completion; infra render errors no longer skip watching already-queued agent DAGs - fold hive-bash-mcp's last local now_unix into wire_time
This commit is contained in:
parent
0e4b5a1120
commit
2486251b32
5 changed files with 106 additions and 26 deletions
|
|
@ -39,6 +39,14 @@ pub use model::{
|
|||
/// per template in the snapshot, matching the old per-kind history cap.
|
||||
const MAX_HISTORY_PER_TEMPLATE: usize = 5;
|
||||
|
||||
/// Terminal DAGs younger than this are exempt from the per-template
|
||||
/// history cap. A broad `hivectl stop`/`start` submits many
|
||||
/// same-template DAGs that can all settle within one poll interval —
|
||||
/// without the grace, the cap would evict some before the ~1s
|
||||
/// `QueueDag` poller ever observes their terminal state, silently
|
||||
/// swallowing failures.
|
||||
const HISTORY_GRACE_SECS: i64 = 300;
|
||||
|
||||
/// Cap on stored node error strings.
|
||||
const MAX_ERROR_LEN: usize = 2_000;
|
||||
|
||||
|
|
@ -399,7 +407,7 @@ impl JobQueue {
|
|||
for agent in freed {
|
||||
inner.leases.remove(&agent);
|
||||
}
|
||||
Self::trim_history(inner);
|
||||
Self::trim_history(inner, now_unix() - HISTORY_GRACE_SECS);
|
||||
}
|
||||
|
||||
/// Cancel a DAG that hasn't started yet (roll-up `Queued`): every
|
||||
|
|
@ -540,11 +548,12 @@ impl JobQueue {
|
|||
}
|
||||
|
||||
/// Keep only the newest `MAX_HISTORY_PER_TEMPLATE` terminal DAGs
|
||||
/// per template. Live DAGs are never evicted — and neither is a
|
||||
/// terminal parent that still has live children (a fan-out parent
|
||||
/// is terminal the moment its `MetaLock` completes; evicting it
|
||||
/// while cascade rebuilds run would orphan their dashboard group).
|
||||
fn trim_history(inner: &mut Inner) {
|
||||
/// per template. Never evicted: live DAGs; terminal parents with
|
||||
/// live children (a fan-out parent is terminal the moment its
|
||||
/// `MetaLock` completes — evicting it while cascade rebuilds run
|
||||
/// would orphan their dashboard group); and terminal DAGs that
|
||||
/// finished after `grace_cutoff` (see [`HISTORY_GRACE_SECS`]).
|
||||
fn trim_history(inner: &mut Inner, grace_cutoff: i64) {
|
||||
let live_parents: std::collections::HashSet<u64> = inner
|
||||
.dags
|
||||
.iter()
|
||||
|
|
@ -560,6 +569,10 @@ impl JobQueue {
|
|||
if !d.is_terminal() || live_parents.contains(&d.id) {
|
||||
return true;
|
||||
}
|
||||
let finished = d.nodes.iter().filter_map(|n| n.finished_at).max();
|
||||
if finished.is_none_or(|t| t > grace_cutoff) {
|
||||
return true;
|
||||
}
|
||||
let n = counts.entry(d.template).or_insert(0);
|
||||
*n += 1;
|
||||
*n <= MAX_HISTORY_PER_TEMPLATE
|
||||
|
|
@ -568,4 +581,12 @@ impl JobQueue {
|
|||
.collect();
|
||||
inner.dags = kept.into_iter().rev().collect();
|
||||
}
|
||||
|
||||
/// Test hook: trim with the grace window disabled, so eviction
|
||||
/// behavior is assertable without aging real timestamps.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn trim_ignoring_grace(&self) {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
Self::trim_history(&mut inner, i64::MAX);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -804,6 +804,16 @@ fn history_evicts_old_terminals_per_template() {
|
|||
let c = claim_one(&q);
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
// Fresh terminals are inside the grace window: nothing evicts yet,
|
||||
// so a ~1s QueueDag poller can still observe every terminal state
|
||||
// (a broad stop/start settles many same-template DAGs at once).
|
||||
assert_eq!(
|
||||
q.snapshot().len(),
|
||||
8,
|
||||
"grace window protects fresh terminals"
|
||||
);
|
||||
// Past the grace window the per-template cap applies.
|
||||
q.trim_ignoring_grace();
|
||||
assert_eq!(q.snapshot().len(), 5, "per-template history cap");
|
||||
assert_eq!(q.live_count(), 0);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue