fix(hive-c0re): close review findings on the job-DAG queue
- deploy-window gate (meta::exclusive) + path-limited meta commits: a perm/lock/topology commit can no longer sweep an ApprovalDeploy's staged flake.lock and neuter abort_deploy (regression test included) - cancel surfaces now buffer terminal roll-ups the scheduler drains, so a queued approval DAG cancelled by the operator resolves its approval instead of dangling, and cancelled power ops revert their wanted flip to the observed state - hivectl restart / restart-all ride the queue (lease serialization, transient guard) and restart sets wanted=Up like the old kill+start - exactly one Rebuilt event per rebuild DAG, emitted at terminal - StopForUpdate pre-seeds a missing agent_power row from the pre-stop observation so a rebuild can't strand an unknown agent offline - history trim keeps terminal fan-out parents with live children - audit_log back on db::open; swarm.js badge for reconcile DAGs
This commit is contained in:
parent
58e86a3adf
commit
084e12503c
12 changed files with 448 additions and 160 deletions
|
|
@ -76,15 +76,6 @@ pub struct TerminalDag {
|
|||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Report from [`JobQueue::complete_node`].
|
||||
#[derive(Debug, Default)]
|
||||
pub struct CompletionReport {
|
||||
/// DAGs that became terminal as a result of this completion
|
||||
/// (the completed node's own DAG, plus none others — but kept as a
|
||||
/// Vec so cancel paths can reuse the same settle plumbing).
|
||||
pub terminal: Vec<TerminalDag>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Inner {
|
||||
dags: VecDeque<Dag>,
|
||||
|
|
@ -93,6 +84,12 @@ struct Inner {
|
|||
slots_used: usize,
|
||||
/// agent → dag id currently holding that agent's lifecycle lease.
|
||||
leases: HashMap<String, u64>,
|
||||
/// Terminal roll-ups not yet consumed by the scheduler
|
||||
/// ([`JobQueue::drain_terminal`]). Fed by every path that settles
|
||||
/// state — node completion AND the cancel surfaces — so the
|
||||
/// terminal hooks (approval resolution, intent revert, transient
|
||||
/// release) fire exactly once per DAG no matter how it ended.
|
||||
pending_terminal: Vec<TerminalDag>,
|
||||
}
|
||||
|
||||
/// The queue. Lives on `Coordinator` (one per hive-c0re process); a
|
||||
|
|
@ -324,13 +321,9 @@ impl JobQueue {
|
|||
|
||||
/// Mark a claimed node terminal, release its build slot, cascade
|
||||
/// cancellations, and settle terminal DAGs (lease release + history
|
||||
/// trim). `error` is stored (truncated) when `result` is `Err`.
|
||||
pub fn complete_node(
|
||||
&self,
|
||||
dag_id: u64,
|
||||
node_id: NodeId,
|
||||
result: Result<(), String>,
|
||||
) -> CompletionReport {
|
||||
/// trim; the terminal roll-up lands in the [`Self::drain_terminal`]
|
||||
/// buffer). `error` is stored (truncated) when `result` is `Err`.
|
||||
pub fn complete_node(&self, dag_id: u64, node_id: NodeId, result: Result<(), String>) {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
if let Some(dag) = inner.dags.iter_mut().find(|d| d.id == dag_id)
|
||||
&& let Some(node) = dag.node_mut(node_id)
|
||||
|
|
@ -360,21 +353,27 @@ impl JobQueue {
|
|||
inner.slots_used = inner.slots_used.saturating_sub(1);
|
||||
}
|
||||
}
|
||||
let report = Self::settle(&mut inner);
|
||||
Self::settle(&mut inner);
|
||||
drop(inner);
|
||||
self.notify.notify_one();
|
||||
report
|
||||
}
|
||||
|
||||
/// Take the terminal roll-ups accumulated since the last drain.
|
||||
/// The scheduler calls this after every wakeup and runs the
|
||||
/// terminal hooks on each entry.
|
||||
pub fn drain_terminal(&self) -> Vec<TerminalDag> {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
std::mem::take(&mut inner.pending_terminal)
|
||||
}
|
||||
|
||||
/// Propagate cancellations, release the leases of newly-terminal
|
||||
/// DAGs, and trim history. Each terminal DAG is reported exactly
|
||||
/// once (the `terminal_reported` flag) so the scheduler's hooks —
|
||||
/// approval resolution, transient-guard release — fire once per
|
||||
/// DAG.
|
||||
fn settle(inner: &mut Inner) -> CompletionReport {
|
||||
/// DAGs, buffer each terminal roll-up exactly once (the
|
||||
/// `terminal_reported` flag) for [`Self::drain_terminal`], and trim
|
||||
/// history.
|
||||
fn settle(inner: &mut Inner) {
|
||||
Self::propagate_cancellations(inner);
|
||||
let mut report = CompletionReport::default();
|
||||
let mut freed: Vec<String> = Vec::new();
|
||||
let mut reports: Vec<TerminalDag> = Vec::new();
|
||||
for dag in &mut inner.dags {
|
||||
if !dag.is_terminal() || dag.terminal_reported {
|
||||
continue;
|
||||
|
|
@ -383,7 +382,7 @@ impl JobQueue {
|
|||
if inner.leases.get(dag.agent.as_str()) == Some(&dag.id) {
|
||||
freed.push(dag.agent.clone());
|
||||
}
|
||||
report.terminal.push(TerminalDag {
|
||||
reports.push(TerminalDag {
|
||||
dag_id: dag.id,
|
||||
template: dag.template,
|
||||
agent: dag.agent.clone(),
|
||||
|
|
@ -392,11 +391,11 @@ impl JobQueue {
|
|||
error: dag.first_error().map(str::to_owned),
|
||||
});
|
||||
}
|
||||
inner.pending_terminal.append(&mut reports);
|
||||
for agent in freed {
|
||||
inner.leases.remove(&agent);
|
||||
}
|
||||
Self::trim_history(inner);
|
||||
report
|
||||
}
|
||||
|
||||
/// Cancel a DAG that hasn't started yet (roll-up `Queued`): every
|
||||
|
|
@ -416,7 +415,10 @@ impl JobQueue {
|
|||
n.state = State::Cancelled;
|
||||
n.finished_at = Some(now);
|
||||
}
|
||||
let _ = Self::settle(&mut inner);
|
||||
// Settle buffers the terminal roll-up; the notify wakes the
|
||||
// scheduler, which drains it and fires the terminal hooks
|
||||
// (approval resolution, power-intent revert).
|
||||
Self::settle(&mut inner);
|
||||
drop(inner);
|
||||
self.notify.notify_one();
|
||||
true
|
||||
|
|
@ -438,7 +440,7 @@ impl JobQueue {
|
|||
}
|
||||
}
|
||||
if count > 0 {
|
||||
let _ = Self::settle(&mut inner);
|
||||
Self::settle(&mut inner);
|
||||
drop(inner);
|
||||
self.notify.notify_one();
|
||||
}
|
||||
|
|
@ -534,15 +536,24 @@ impl JobQueue {
|
|||
}
|
||||
|
||||
/// Keep only the newest `MAX_HISTORY_PER_TEMPLATE` terminal DAGs
|
||||
/// per template; live DAGs are never evicted.
|
||||
/// per template. Live DAGs are never evicted — and neither is a
|
||||
/// terminal parent that still has live children (a fan-out parent
|
||||
/// is terminal the moment its `MetaLock` completes; evicting it
|
||||
/// while cascade rebuilds run would orphan their dashboard group).
|
||||
fn trim_history(inner: &mut Inner) {
|
||||
let live_parents: std::collections::HashSet<u64> = inner
|
||||
.dags
|
||||
.iter()
|
||||
.filter(|d| !d.is_terminal())
|
||||
.filter_map(|d| d.parent_id)
|
||||
.collect();
|
||||
let mut counts: HashMap<Template, usize> = HashMap::new();
|
||||
let kept: Vec<Dag> = inner
|
||||
.dags
|
||||
.iter()
|
||||
.rev()
|
||||
.filter(|d| {
|
||||
if !d.is_terminal() {
|
||||
if !d.is_terminal() || live_parents.contains(&d.id) {
|
||||
return true;
|
||||
}
|
||||
let n = counts.entry(d.template).or_insert(0);
|
||||
|
|
|
|||
Loading…
Reference in a new issue