fix(hive-c0re): close review findings on the job-DAG queue
- deploy-window gate (meta::exclusive) + path-limited meta commits: a perm/lock/topology commit can no longer sweep an ApprovalDeploy's staged flake.lock and neuter abort_deploy (regression test included) - cancel surfaces now buffer terminal roll-ups the scheduler drains, so a queued approval DAG cancelled by the operator resolves its approval instead of dangling, and cancelled power ops revert their wanted flip to the observed state - hivectl restart / restart-all ride the queue (lease serialization, transient guard) and restart sets wanted=Up like the old kill+start - exactly one Rebuilt event per rebuild DAG, emitted at terminal - StopForUpdate pre-seeds a missing agent_power row from the pre-stop observation so a rebuild can't strand an unknown agent offline - history trim keeps terminal fan-out parents with live children - audit_log back on db::open; swarm.js badge for reconcile DAGs
This commit is contained in:
parent
58e86a3adf
commit
084e12503c
12 changed files with 448 additions and 160 deletions
|
|
@ -38,6 +38,11 @@ pub async fn run_worker(coord: Arc<Coordinator>) {
|
|||
// DAG id → transient guard held for the lease window.
|
||||
let mut transients: HashMap<u64, crate::coordinator::TransientGuard> = HashMap::new();
|
||||
loop {
|
||||
// Terminal roll-ups can appear without a node completion —
|
||||
// the cancel surfaces settle DAGs directly and wake this loop
|
||||
// via notify — so drain on every iteration, not just inside
|
||||
// handle_completion.
|
||||
process_terminals(&coord, &mut transients).await;
|
||||
let claims = coord.job_queue.claim_ready();
|
||||
if !claims.is_empty() {
|
||||
for claim in claims {
|
||||
|
|
@ -109,20 +114,28 @@ async fn handle_completion(
|
|||
(Err(msg), Vec::new())
|
||||
}
|
||||
};
|
||||
let report = coord
|
||||
coord
|
||||
.job_queue
|
||||
.complete_node(claim.dag_id, claim.node_id, queue_result);
|
||||
if !fanout.is_empty() {
|
||||
let specs = fanout_specs(&claim, fanout);
|
||||
coord.job_queue.append_children(specs);
|
||||
}
|
||||
for terminal in report.terminal {
|
||||
// Drop the lease-window transient guard, then let the hook
|
||||
// fire approval resolution / failure events.
|
||||
process_terminals(coord, transients).await;
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
|
||||
/// Drain buffered terminal roll-ups: drop each DAG's lease-window
|
||||
/// transient guard, then run the terminal hook (approval resolution,
|
||||
/// `Rebuilt` events, cancelled-power-op intent revert).
|
||||
async fn process_terminals(
|
||||
coord: &Arc<Coordinator>,
|
||||
transients: &mut HashMap<u64, crate::coordinator::TransientGuard>,
|
||||
) {
|
||||
for terminal in coord.job_queue.drain_terminal() {
|
||||
transients.remove(&terminal.dag_id);
|
||||
exec::on_dag_terminal(coord, &terminal).await;
|
||||
}
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
|
||||
/// Child `Rebuild` specs for a completed `MetaLock` fan-out, grouped
|
||||
|
|
|
|||
Loading…
Reference in a new issue