fix(hive-c0re): close review findings on the job-DAG queue
- deploy-window gate (meta::exclusive) + path-limited meta commits: a perm/lock/topology commit can no longer sweep an ApprovalDeploy's staged flake.lock and neuter abort_deploy (regression test included) - cancel surfaces now buffer terminal roll-ups the scheduler drains, so a queued approval DAG cancelled by the operator resolves its approval instead of dangling, and cancelled power ops revert their wanted flip to the observed state - hivectl restart / restart-all ride the queue (lease serialization, transient guard) and restart sets wanted=Up like the old kill+start - exactly one Rebuilt event per rebuild DAG, emitted at terminal - StopForUpdate pre-seeds a missing agent_power row from the pre-stop observation so a rebuild can't strand an unknown agent offline - history trim keeps terminal fan-out parents with live children - audit_log back on db::open; swarm.js badge for reconcile DAGs
This commit is contained in:
parent
58e86a3adf
commit
084e12503c
12 changed files with 448 additions and 160 deletions
|
|
@ -626,13 +626,15 @@ fn terminal_dag_reported_exactly_once_and_lease_released() {
|
|||
templates::restart("agent-a", Source::Manual, "r".to_owned()),
|
||||
);
|
||||
let stop = claim_one(&q);
|
||||
let r1 = q.complete_node(id, stop.node_id, Ok(()));
|
||||
assert!(r1.terminal.is_empty(), "dag not terminal yet");
|
||||
q.complete_node(id, stop.node_id, Ok(()));
|
||||
assert!(q.drain_terminal().is_empty(), "dag not terminal yet");
|
||||
let rec = claim_one(&q);
|
||||
let r2 = q.complete_node(id, rec.node_id, Ok(()));
|
||||
assert_eq!(r2.terminal.len(), 1);
|
||||
assert_eq!(r2.terminal[0].dag_id, id);
|
||||
assert_eq!(r2.terminal[0].state, State::Done);
|
||||
q.complete_node(id, rec.node_id, Ok(()));
|
||||
let reports = q.drain_terminal();
|
||||
assert_eq!(reports.len(), 1);
|
||||
assert_eq!(reports[0].dag_id, id);
|
||||
assert_eq!(reports[0].state, State::Done);
|
||||
assert!(q.drain_terminal().is_empty(), "reported exactly once");
|
||||
// Lease released: a new DAG for the agent can claim immediately.
|
||||
let next = submit(
|
||||
&q,
|
||||
|
|
@ -649,20 +651,105 @@ fn terminal_dag_reported_exactly_once_and_lease_released() {
|
|||
assert!(c.lease_acquired);
|
||||
}
|
||||
|
||||
/// A DAG cancelled while fully queued must still surface a terminal
|
||||
/// roll-up for the scheduler's hooks — otherwise a queued approval
|
||||
/// DAG cancelled by the operator would dangle its approval forever.
|
||||
#[test]
|
||||
fn cancelled_dag_reports_terminal() {
|
||||
fn cancelled_dag_reports_terminal_once() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::approval_deploy("agent-a", 7, "approval #7".to_owned()),
|
||||
);
|
||||
assert!(q.cancel(id));
|
||||
// The cancel path settles internally; a subsequent completion
|
||||
// report must not re-report it. Verify via a second dag's cycle.
|
||||
let reports = q.drain_terminal();
|
||||
assert_eq!(reports.len(), 1);
|
||||
assert_eq!(reports[0].dag_id, id);
|
||||
assert_eq!(reports[0].state, State::Cancelled);
|
||||
assert_eq!(reports[0].approval_id, Some(7));
|
||||
// Never re-reported by later activity.
|
||||
let other = submit(&q, rebuild("agent-b", "r"));
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.dag_id, other);
|
||||
let report = q.complete_node(other, c.node_id, Err("boom".to_owned()));
|
||||
// agent-b's dag isn't terminal (reconcile still pending) and
|
||||
// agent-a's was already reported by cancel → nothing here.
|
||||
assert!(report.terminal.iter().all(|t| t.dag_id != id));
|
||||
q.complete_node(other, c.node_id, Err("boom".to_owned()));
|
||||
assert!(q.drain_terminal().iter().all(|t| t.dag_id != id));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_children_reports_terminals() {
|
||||
let q = JobQueue::new(1);
|
||||
let meta = submit(
|
||||
&q,
|
||||
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
|
||||
);
|
||||
let _lock = claim_one(&q);
|
||||
let child = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"agent-a",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
);
|
||||
assert_eq!(q.cancel_children(meta), 1);
|
||||
let reports = q.drain_terminal();
|
||||
assert_eq!(reports.len(), 1);
|
||||
assert_eq!(reports[0].dag_id, child);
|
||||
assert_eq!(reports[0].state, State::Cancelled);
|
||||
}
|
||||
|
||||
/// History trim must not evict a terminal fan-out parent while its
|
||||
/// children are still live — the dashboard groups children under it.
|
||||
#[test]
|
||||
fn trim_keeps_terminal_parent_with_live_children() {
|
||||
let q = JobQueue::new(1);
|
||||
// Pin agent-x's lease with a running stop DAG so the child below
|
||||
// stays fully queued while we churn history.
|
||||
let pin = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
Template::Stop,
|
||||
"agent-x",
|
||||
Source::Manual,
|
||||
"lease pin".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
let pin_claim = claim_one(&q);
|
||||
assert_eq!(pin_claim.dag_id, pin);
|
||||
// Terminal fan-out parent + a lease-blocked child under it.
|
||||
let meta = submit(
|
||||
&q,
|
||||
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
|
||||
);
|
||||
let lock = claim_one(&q);
|
||||
q.complete_node(meta, lock.node_id, Ok(()));
|
||||
let mut child_spec = templates::restart("agent-x", Source::MetaUpdate, "cascade".to_owned());
|
||||
child_spec.parent_id = Some(meta);
|
||||
let child = submit(&q, child_spec);
|
||||
// Churn > MAX_HISTORY_PER_TEMPLATE terminal meta_update DAGs.
|
||||
for i in 0..7 {
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::meta_update(
|
||||
vec![format!("input-{i}")],
|
||||
Source::Manual,
|
||||
"churn".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.dag_id, id, "child is lease-blocked; churn claims freely");
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
let snap = q.snapshot();
|
||||
assert!(
|
||||
snap.iter().any(|d| d.id == meta),
|
||||
"terminal parent with live child must survive trim"
|
||||
);
|
||||
assert!(snap.iter().any(|d| d.id == child));
|
||||
}
|
||||
|
||||
// ---- steps, build logs, history ----
|
||||
|
|
|
|||
Loading…
Reference in a new issue