fix(hive-c0re): close review findings on the job-DAG queue

- deploy-window gate (meta::exclusive) + path-limited meta commits:
  a perm/lock/topology commit can no longer sweep an ApprovalDeploy's
  staged flake.lock and neuter abort_deploy (regression test included)
- cancel surfaces now buffer terminal roll-ups the scheduler drains,
  so a queued approval DAG cancelled by the operator resolves its
  approval instead of dangling, and cancelled power ops revert their
  wanted flip to the observed state
- hivectl restart / restart-all ride the queue (lease serialization,
  transient guard) and restart sets wanted=Up like the old kill+start
- exactly one Rebuilt event per rebuild DAG, emitted at terminal
- StopForUpdate pre-seeds a missing agent_power row from the pre-stop
  observation so a rebuild can't strand an unknown agent offline
- history trim keeps terminal fan-out parents with live children
- audit_log back on db::open; swarm.js badge for reconcile DAGs
This commit is contained in:
müde 2026-07-06 21:44:43 +02:00
commit 084e12503c
12 changed files with 448 additions and 160 deletions

View file

@ -626,13 +626,15 @@ fn terminal_dag_reported_exactly_once_and_lease_released() {
templates::restart("agent-a", Source::Manual, "r".to_owned()),
);
let stop = claim_one(&q);
let r1 = q.complete_node(id, stop.node_id, Ok(()));
assert!(r1.terminal.is_empty(), "dag not terminal yet");
q.complete_node(id, stop.node_id, Ok(()));
assert!(q.drain_terminal().is_empty(), "dag not terminal yet");
let rec = claim_one(&q);
let r2 = q.complete_node(id, rec.node_id, Ok(()));
assert_eq!(r2.terminal.len(), 1);
assert_eq!(r2.terminal[0].dag_id, id);
assert_eq!(r2.terminal[0].state, State::Done);
q.complete_node(id, rec.node_id, Ok(()));
let reports = q.drain_terminal();
assert_eq!(reports.len(), 1);
assert_eq!(reports[0].dag_id, id);
assert_eq!(reports[0].state, State::Done);
assert!(q.drain_terminal().is_empty(), "reported exactly once");
// Lease released: a new DAG for the agent can claim immediately.
let next = submit(
&q,
@ -649,20 +651,105 @@ fn terminal_dag_reported_exactly_once_and_lease_released() {
assert!(c.lease_acquired);
}
/// A DAG cancelled while fully queued must still surface a terminal
/// roll-up for the scheduler's hooks — otherwise a queued approval
/// DAG cancelled by the operator would dangle its approval forever.
#[test]
fn cancelled_dag_reports_terminal() {
fn cancelled_dag_reports_terminal_once() {
let q = JobQueue::new(1);
let id = submit(&q, rebuild("agent-a", "r"));
let id = submit(
&q,
templates::approval_deploy("agent-a", 7, "approval #7".to_owned()),
);
assert!(q.cancel(id));
// The cancel path settles internally; a subsequent completion
// report must not re-report it. Verify via a second dag's cycle.
let reports = q.drain_terminal();
assert_eq!(reports.len(), 1);
assert_eq!(reports[0].dag_id, id);
assert_eq!(reports[0].state, State::Cancelled);
assert_eq!(reports[0].approval_id, Some(7));
// Never re-reported by later activity.
let other = submit(&q, rebuild("agent-b", "r"));
let c = claim_one(&q);
assert_eq!(c.dag_id, other);
let report = q.complete_node(other, c.node_id, Err("boom".to_owned()));
// agent-b's dag isn't terminal (reconcile still pending) and
// agent-a's was already reported by cancel → nothing here.
assert!(report.terminal.iter().all(|t| t.dag_id != id));
q.complete_node(other, c.node_id, Err("boom".to_owned()));
assert!(q.drain_terminal().iter().all(|t| t.dag_id != id));
}
#[test]
fn cancel_children_reports_terminals() {
let q = JobQueue::new(1);
let meta = submit(
&q,
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
);
let _lock = claim_one(&q);
let child = submit(
&q,
templates::rebuild(
"agent-a",
Source::MetaUpdate,
"cascade".to_owned(),
Some(meta),
false,
),
);
assert_eq!(q.cancel_children(meta), 1);
let reports = q.drain_terminal();
assert_eq!(reports.len(), 1);
assert_eq!(reports[0].dag_id, child);
assert_eq!(reports[0].state, State::Cancelled);
}
/// History trim must not evict a terminal fan-out parent while its
/// children are still live — the dashboard groups children under it.
#[test]
fn trim_keeps_terminal_parent_with_live_children() {
let q = JobQueue::new(1);
// Pin agent-x's lease with a running stop DAG so the child below
// stays fully queued while we churn history.
let pin = submit(
&q,
templates::reconcile_only(
Template::Stop,
"agent-x",
Source::Manual,
"lease pin".to_owned(),
None,
),
);
let pin_claim = claim_one(&q);
assert_eq!(pin_claim.dag_id, pin);
// Terminal fan-out parent + a lease-blocked child under it.
let meta = submit(
&q,
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
);
let lock = claim_one(&q);
q.complete_node(meta, lock.node_id, Ok(()));
let mut child_spec = templates::restart("agent-x", Source::MetaUpdate, "cascade".to_owned());
child_spec.parent_id = Some(meta);
let child = submit(&q, child_spec);
// Churn > MAX_HISTORY_PER_TEMPLATE terminal meta_update DAGs.
for i in 0..7 {
let id = submit(
&q,
templates::meta_update(
vec![format!("input-{i}")],
Source::Manual,
"churn".to_owned(),
None,
),
);
let c = claim_one(&q);
assert_eq!(c.dag_id, id, "child is lease-blocked; churn claims freely");
q.complete_node(id, c.node_id, Ok(()));
}
let snap = q.snapshot();
assert!(
snap.iter().any(|d| d.id == meta),
"terminal parent with live child must survive trim"
);
assert!(snap.iter().any(|d| d.id == child));
}
// ---- steps, build logs, history ----