fix(hive-c0re): close review findings on the job-DAG queue

- deploy-window gate (meta::exclusive) + path-limited meta commits:
  a perm/lock/topology commit can no longer sweep an ApprovalDeploy's
  staged flake.lock and neuter abort_deploy (regression test included)
- cancel surfaces now buffer terminal roll-ups the scheduler drains,
  so a queued approval DAG cancelled by the operator resolves its
  approval instead of dangling, and cancelled power ops revert their
  wanted flip to the observed state
- hivectl restart / restart-all ride the queue (lease serialization,
  transient guard) and restart sets wanted=Up like the old kill+start
- exactly one Rebuilt event per rebuild DAG, emitted at terminal
- StopForUpdate pre-seeds a missing agent_power row from the pre-stop
  observation so a rebuild can't strand an unknown agent offline
- history trim keeps terminal fan-out parents with live children
- audit_log back on db::open; swarm.js badge for reconcile DAGs
This commit is contained in:
müde 2026-07-06 21:44:43 +02:00
commit 084e12503c
12 changed files with 448 additions and 160 deletions

View file

@ -76,15 +76,6 @@ pub struct TerminalDag {
pub error: Option<String>,
}
/// Report from [`JobQueue::complete_node`].
#[derive(Debug, Default)]
pub struct CompletionReport {
/// DAGs that became terminal as a result of this completion
/// (the completed node's own DAG, plus none others — but kept as a
/// Vec so cancel paths can reuse the same settle plumbing).
pub terminal: Vec<TerminalDag>,
}
#[derive(Debug, Default)]
struct Inner {
dags: VecDeque<Dag>,
@ -93,6 +84,12 @@ struct Inner {
slots_used: usize,
/// agent → dag id currently holding that agent's lifecycle lease.
leases: HashMap<String, u64>,
/// Terminal roll-ups not yet consumed by the scheduler
/// ([`JobQueue::drain_terminal`]). Fed by every path that settles
/// state — node completion AND the cancel surfaces — so the
/// terminal hooks (approval resolution, intent revert, transient
/// release) fire exactly once per DAG no matter how it ended.
pending_terminal: Vec<TerminalDag>,
}
/// The queue. Lives on `Coordinator` (one per hive-c0re process); a
@ -324,13 +321,9 @@ impl JobQueue {
/// Mark a claimed node terminal, release its build slot, cascade
/// cancellations, and settle terminal DAGs (lease release + history
/// trim). `error` is stored (truncated) when `result` is `Err`.
pub fn complete_node(
&self,
dag_id: u64,
node_id: NodeId,
result: Result<(), String>,
) -> CompletionReport {
/// trim; the terminal roll-up lands in the [`Self::drain_terminal`]
/// buffer). `error` is stored (truncated) when `result` is `Err`.
pub fn complete_node(&self, dag_id: u64, node_id: NodeId, result: Result<(), String>) {
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
if let Some(dag) = inner.dags.iter_mut().find(|d| d.id == dag_id)
&& let Some(node) = dag.node_mut(node_id)
@ -360,21 +353,27 @@ impl JobQueue {
inner.slots_used = inner.slots_used.saturating_sub(1);
}
}
let report = Self::settle(&mut inner);
Self::settle(&mut inner);
drop(inner);
self.notify.notify_one();
report
}
/// Take the terminal roll-ups accumulated since the last drain.
/// The scheduler calls this after every wakeup and runs the
/// terminal hooks on each entry.
pub fn drain_terminal(&self) -> Vec<TerminalDag> {
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
std::mem::take(&mut inner.pending_terminal)
}
/// Propagate cancellations, release the leases of newly-terminal
/// DAGs, and trim history. Each terminal DAG is reported exactly
/// once (the `terminal_reported` flag) so the scheduler's hooks —
/// approval resolution, transient-guard release — fire once per
/// DAG.
fn settle(inner: &mut Inner) -> CompletionReport {
/// DAGs, buffer each terminal roll-up exactly once (the
/// `terminal_reported` flag) for [`Self::drain_terminal`], and trim
/// history.
fn settle(inner: &mut Inner) {
Self::propagate_cancellations(inner);
let mut report = CompletionReport::default();
let mut freed: Vec<String> = Vec::new();
let mut reports: Vec<TerminalDag> = Vec::new();
for dag in &mut inner.dags {
if !dag.is_terminal() || dag.terminal_reported {
continue;
@ -383,7 +382,7 @@ impl JobQueue {
if inner.leases.get(dag.agent.as_str()) == Some(&dag.id) {
freed.push(dag.agent.clone());
}
report.terminal.push(TerminalDag {
reports.push(TerminalDag {
dag_id: dag.id,
template: dag.template,
agent: dag.agent.clone(),
@ -392,11 +391,11 @@ impl JobQueue {
error: dag.first_error().map(str::to_owned),
});
}
inner.pending_terminal.append(&mut reports);
for agent in freed {
inner.leases.remove(&agent);
}
Self::trim_history(inner);
report
}
/// Cancel a DAG that hasn't started yet (roll-up `Queued`): every
@ -416,7 +415,10 @@ impl JobQueue {
n.state = State::Cancelled;
n.finished_at = Some(now);
}
let _ = Self::settle(&mut inner);
// Settle buffers the terminal roll-up; the notify wakes the
// scheduler, which drains it and fires the terminal hooks
// (approval resolution, power-intent revert).
Self::settle(&mut inner);
drop(inner);
self.notify.notify_one();
true
@ -438,7 +440,7 @@ impl JobQueue {
}
}
if count > 0 {
let _ = Self::settle(&mut inner);
Self::settle(&mut inner);
drop(inner);
self.notify.notify_one();
}
@ -534,15 +536,24 @@ impl JobQueue {
}
/// Keep only the newest `MAX_HISTORY_PER_TEMPLATE` terminal DAGs
/// per template; live DAGs are never evicted.
/// per template. Live DAGs are never evicted — and neither is a
/// terminal parent that still has live children (a fan-out parent
/// is terminal the moment its `MetaLock` completes; evicting it
/// while cascade rebuilds run would orphan their dashboard group).
fn trim_history(inner: &mut Inner) {
let live_parents: std::collections::HashSet<u64> = inner
.dags
.iter()
.filter(|d| !d.is_terminal())
.filter_map(|d| d.parent_id)
.collect();
let mut counts: HashMap<Template, usize> = HashMap::new();
let kept: Vec<Dag> = inner
.dags
.iter()
.rev()
.filter(|d| {
if !d.is_terminal() {
if !d.is_terminal() || live_parents.contains(&d.id) {
return true;
}
let n = counts.entry(d.template).or_insert(0);