feat(hive-c0re): replace rebuild queue with generic job-DAG queue
jobs are now DAGs of primitive nodes (prebuild, stop-for-update, swap, reconcile, signal, drain, ...) driven by one scheduler with N build slots + per-agent lifecycle leases. per-agent power intent (wanted up/offline) is durable in agent_power.sqlite; Reconcile nodes converge observed state to it. kills the graceful-stop watcher thread, the deferred-start follow-up, and the cascade pre-enqueue (fan-out on MetaLock completion instead). tracker: #2166
This commit is contained in:
parent
79a3993def
commit
7946e03fde
25 changed files with 3673 additions and 2731 deletions
823
hive-c0re/src/job_queue/tests.rs
Normal file
823
hive-c0re/src/job_queue/tests.rs
Normal file
|
|
@ -0,0 +1,823 @@
|
|||
//! Queue-core unit tests: dedup, cycle rejection, resource
|
||||
//! serialization (build slots / per-agent leases), lease-exempt
|
||||
//! overlap, FIFO fairness, cancel semantics, `AfterAny` failure
|
||||
//! routing, fan-out, and history retention. All synchronous — the
|
||||
//! scheduler's async loop is a thin claim/complete pump over the same
|
||||
//! methods exercised here.
|
||||
|
||||
use super::model::{Dep, DepWhen, NodeKind, NodeSpec};
|
||||
use super::*;
|
||||
|
||||
fn submit(q: &JobQueue, spec: DagSpec) -> u64 {
|
||||
q.submit(spec).expect("valid spec")
|
||||
}
|
||||
|
||||
fn rebuild(agent: &str, reason: &str) -> DagSpec {
|
||||
templates::rebuild(agent, Source::Manual, reason.to_owned(), None, true)
|
||||
}
|
||||
|
||||
/// Claim helper asserting exactly one node comes back.
|
||||
fn claim_one(q: &JobQueue) -> Claim {
|
||||
let mut claims = q.claim_ready();
|
||||
assert_eq!(
|
||||
claims.len(),
|
||||
1,
|
||||
"expected exactly one claim, got {claims:?}"
|
||||
);
|
||||
claims.pop().expect("one claim")
|
||||
}
|
||||
|
||||
fn state_of(q: &JobQueue, dag_id: u64) -> State {
|
||||
q.snapshot()
|
||||
.iter()
|
||||
.find(|d| d.id == dag_id)
|
||||
.expect("dag present")
|
||||
.state
|
||||
}
|
||||
|
||||
// ---- submit / dedup ----
|
||||
|
||||
#[test]
|
||||
fn submit_assigns_distinct_ids() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "first"));
|
||||
let b = submit(&q, rebuild("agent-b", "second"));
|
||||
assert_ne!(a, b);
|
||||
assert_eq!(q.snapshot().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedup_pending_same_template_and_agent() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "first"));
|
||||
let b = submit(&q, rebuild("agent-a", "auto sweep"));
|
||||
assert_eq!(a, b, "dedup should return existing id");
|
||||
let snap = q.snapshot();
|
||||
assert_eq!(snap.len(), 1);
|
||||
assert!(snap[0].reason.contains("first"));
|
||||
assert!(snap[0].reason.contains("auto sweep"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedup_does_not_apply_across_templates_or_agents() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "r"));
|
||||
let b = submit(&q, rebuild("agent-b", "r"));
|
||||
let c = submit(
|
||||
&q,
|
||||
templates::restart("agent-a", Source::Manual, "r".to_owned()),
|
||||
);
|
||||
assert_ne!(a, b);
|
||||
assert_ne!(a, c);
|
||||
assert_eq!(q.snapshot().len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedup_skips_running_dags() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "first"));
|
||||
let claim = claim_one(&q); // Prebuild running
|
||||
assert_eq!(claim.dag_id, a);
|
||||
// While the original runs, re-submit is legitimate new work.
|
||||
let again = submit(&q, rebuild("agent-a", "config bumped during build"));
|
||||
assert_ne!(a, again);
|
||||
assert_eq!(q.snapshot().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_update_dedup_matches_inputs() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(
|
||||
&q,
|
||||
templates::meta_update(
|
||||
vec!["nixpkgs".to_owned()],
|
||||
Source::Manual,
|
||||
"first".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
let b = submit(
|
||||
&q,
|
||||
templates::meta_update(
|
||||
vec!["nixpkgs".to_owned()],
|
||||
Source::Manual,
|
||||
"duplicate click".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
assert_eq!(a, b, "identical-inputs meta-updates should dedup");
|
||||
let c = submit(
|
||||
&q,
|
||||
templates::meta_update(
|
||||
vec!["agent-bitburner/bitburner-agent".to_owned()],
|
||||
Source::Manual,
|
||||
"bump agent".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
assert_ne!(a, c, "different-inputs meta-updates must NOT dedup");
|
||||
assert_eq!(q.snapshot().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn approval_dags_dedup_only_on_matching_id() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(
|
||||
&q,
|
||||
templates::approval_deploy("agent-a", 1, "approval #1".to_owned()),
|
||||
);
|
||||
let b = submit(
|
||||
&q,
|
||||
templates::approval_deploy("agent-a", 2, "approval #2".to_owned()),
|
||||
);
|
||||
assert_ne!(a, b, "distinct approvals must not collapse");
|
||||
// Rapid double-click on the same approval IS a single op.
|
||||
let c = submit(
|
||||
&q,
|
||||
templates::approval_deploy("agent-a", 1, "approval #1 (dup)".to_owned()),
|
||||
);
|
||||
assert_eq!(a, c);
|
||||
assert_eq!(q.snapshot().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn perm_change_dedup_respects_perm_type() {
|
||||
let q = JobQueue::new(1);
|
||||
let groups = templates::perm_change(
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"groups".to_owned(),
|
||||
PermPayload::ToolGroups { groups: vec![] },
|
||||
);
|
||||
let caps = templates::perm_change(
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"caps".to_owned(),
|
||||
PermPayload::Capabilities { caps: vec![] },
|
||||
);
|
||||
let a = submit(&q, groups.clone());
|
||||
let b = submit(&q, caps);
|
||||
assert_ne!(a, b, "tool-groups vs capabilities must not collapse");
|
||||
let c = submit(&q, groups);
|
||||
assert_eq!(a, c, "same perm type dedups");
|
||||
}
|
||||
|
||||
/// A `MetaUpdate` cascade `Rebuild` (with `parent_id = Some(meta_id)`)
|
||||
/// must NOT dedup into a queued `Rebuild` with a different
|
||||
/// `parent_id` (e.g. from a startup sweep) — without the guard the
|
||||
/// cascade child would be swallowed and the agent never rebuilt
|
||||
/// against the post-bump meta.
|
||||
#[test]
|
||||
fn dedup_respects_parent_id() {
|
||||
let q = JobQueue::new(1);
|
||||
let sweep = submit(&q, templates::startup_sweep("boot".to_owned(), vec![]));
|
||||
let sweep_child = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"alice",
|
||||
Source::StartupSweep,
|
||||
"startup sweep".to_owned(),
|
||||
Some(sweep),
|
||||
true,
|
||||
),
|
||||
);
|
||||
let meta = submit(
|
||||
&q,
|
||||
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
|
||||
);
|
||||
let cascade_child = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"alice",
|
||||
Source::MetaUpdate,
|
||||
"meta-update cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
);
|
||||
assert_ne!(sweep_child, cascade_child);
|
||||
let rebuilds = q
|
||||
.snapshot()
|
||||
.iter()
|
||||
.filter(|d| d.kind == Template::Rebuild && d.agent == "alice")
|
||||
.count();
|
||||
assert_eq!(rebuilds, 2, "both rebuilds must be present");
|
||||
}
|
||||
|
||||
// ---- cycle rejection ----
|
||||
|
||||
#[test]
|
||||
fn cyclic_dag_is_rejected_at_submit() {
|
||||
let q = JobQueue::new(1);
|
||||
let mut spec = rebuild("agent-a", "cyclic");
|
||||
// 0 → 1 → 0 cycle.
|
||||
spec.nodes = vec![
|
||||
NodeSpec {
|
||||
kind: NodeKind::StopForUpdate,
|
||||
deps: vec![Dep {
|
||||
on: 1,
|
||||
when: DepWhen::AfterOk,
|
||||
}],
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: vec![Dep {
|
||||
on: 0,
|
||||
when: DepWhen::AfterOk,
|
||||
}],
|
||||
},
|
||||
];
|
||||
assert!(q.submit(spec).is_err(), "cyclic spec must be refused");
|
||||
assert!(q.snapshot().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dep_is_rejected_at_submit() {
|
||||
let q = JobQueue::new(1);
|
||||
let mut spec = rebuild("agent-a", "bad dep");
|
||||
spec.nodes = vec![NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: vec![Dep {
|
||||
on: 9,
|
||||
when: DepWhen::AfterOk,
|
||||
}],
|
||||
}];
|
||||
assert!(q.submit(spec).is_err());
|
||||
}
|
||||
|
||||
// ---- dependency order within a DAG ----
|
||||
|
||||
#[test]
|
||||
fn rebuild_chain_claims_in_dep_order() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
for expected in ["prebuild", "stop_for_update", "swap", "reconcile"] {
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.dag_id, id);
|
||||
assert_eq!(c.kind.as_str(), expected);
|
||||
assert!(
|
||||
q.claim_ready().is_empty(),
|
||||
"chain must serialize: nothing ready while {expected} runs"
|
||||
);
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
assert_eq!(state_of(&q, id), State::Done);
|
||||
}
|
||||
|
||||
// ---- build slots ----
|
||||
|
||||
#[test]
|
||||
fn build_slot_serializes_nix_heavy_nodes() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "r"));
|
||||
let b = submit(&q, rebuild("agent-b", "r"));
|
||||
let first = claim_one(&q); // a's Prebuild takes the only slot
|
||||
assert_eq!(first.dag_id, a);
|
||||
assert_eq!(first.kind.as_str(), "prebuild");
|
||||
q.complete_node(a, first.node_id, Ok(()));
|
||||
// With the slot free again, FIFO gives... a's StopForUpdate is
|
||||
// slot-free (lease) and b's Prebuild takes the slot — both run.
|
||||
let claims = q.claim_ready();
|
||||
let kinds: Vec<(u64, &str)> = claims.iter().map(|c| (c.dag_id, c.kind.as_str())).collect();
|
||||
assert!(kinds.contains(&(a, "stop_for_update")));
|
||||
assert!(kinds.contains(&(b, "prebuild")));
|
||||
assert_eq!(claims.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_build_slots_run_two_prebuilds() {
|
||||
let q = JobQueue::new(2);
|
||||
submit(&q, rebuild("agent-a", "r"));
|
||||
submit(&q, rebuild("agent-b", "r"));
|
||||
let claims = q.claim_ready();
|
||||
assert_eq!(claims.len(), 2, "two slots → two concurrent prebuilds");
|
||||
assert!(claims.iter().all(|c| c.kind.as_str() == "prebuild"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fifo_fairness_for_the_slot() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "r"));
|
||||
let b = submit(&q, rebuild("agent-b", "r"));
|
||||
let c = submit(&q, rebuild("agent-c", "r"));
|
||||
let first = claim_one(&q);
|
||||
assert_eq!(first.dag_id, a, "submit order wins the slot");
|
||||
q.complete_node(a, first.node_id, Ok(()));
|
||||
let next: Vec<u64> = q.claim_ready().iter().map(|cl| cl.dag_id).collect();
|
||||
assert!(next.contains(&b), "b's prebuild before c's");
|
||||
assert!(!next.contains(&c));
|
||||
}
|
||||
|
||||
// ---- per-agent lease ----
|
||||
|
||||
#[test]
|
||||
fn lease_serializes_two_lifecycle_dags_for_same_agent() {
|
||||
let q = JobQueue::new(4);
|
||||
let restart = submit(
|
||||
&q,
|
||||
templates::restart("agent-a", Source::Manual, "restart".to_owned()),
|
||||
);
|
||||
let stop = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
Template::Stop,
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"stop".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
// Restart's StopForUpdate acquires the lease; stop's Reconcile
|
||||
// must wait even though slots are free.
|
||||
let first = claim_one(&q);
|
||||
assert_eq!(first.dag_id, restart);
|
||||
assert!(first.lease_acquired);
|
||||
q.complete_node(restart, first.node_id, Ok(()));
|
||||
// Same DAG keeps the lease for its Reconcile.
|
||||
let second = claim_one(&q);
|
||||
assert_eq!(second.dag_id, restart);
|
||||
assert!(!second.lease_acquired, "lease already held by this DAG");
|
||||
q.complete_node(restart, second.node_id, Ok(()));
|
||||
// Restart terminal → lease released → stop's Reconcile runs.
|
||||
let third = claim_one(&q);
|
||||
assert_eq!(third.dag_id, stop);
|
||||
q.complete_node(stop, third.node_id, Ok(()));
|
||||
assert_eq!(state_of(&q, restart), State::Done);
|
||||
assert_eq!(state_of(&q, stop), State::Done);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lease_exempt_prebuild_overlaps_other_dag_on_same_agent() {
|
||||
let q = JobQueue::new(2);
|
||||
submit(&q, rebuild("agent-a", "rebuild"));
|
||||
let stop = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
Template::Stop,
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"stop".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
// Prebuild is lease-exempt: the stop's Reconcile takes the lease
|
||||
// and runs concurrently with the rebuild's out-of-band nix build.
|
||||
let claims = q.claim_ready();
|
||||
let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect();
|
||||
assert!(kinds.contains(&"prebuild"));
|
||||
assert!(kinds.contains(&"reconcile"));
|
||||
// But the rebuild's StopForUpdate must then wait for the stop DAG
|
||||
// to finish (lease).
|
||||
let prebuild = claims
|
||||
.iter()
|
||||
.find(|c| c.kind.as_str() == "prebuild")
|
||||
.expect("prebuild claim")
|
||||
.clone();
|
||||
q.complete_node(prebuild.dag_id, prebuild.node_id, Ok(()));
|
||||
assert!(
|
||||
q.claim_ready().is_empty(),
|
||||
"StopForUpdate blocked while stop DAG holds the lease"
|
||||
);
|
||||
let reconcile = claims
|
||||
.iter()
|
||||
.find(|c| c.kind.as_str() == "reconcile")
|
||||
.expect("reconcile claim")
|
||||
.clone();
|
||||
q.complete_node(stop, reconcile.node_id, Ok(()));
|
||||
let next = claim_one(&q);
|
||||
assert_eq!(next.kind.as_str(), "stop_for_update");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agents_do_not_contend_on_each_others_leases() {
|
||||
let q = JobQueue::new(4);
|
||||
submit(
|
||||
&q,
|
||||
templates::restart("agent-a", Source::Manual, "r".to_owned()),
|
||||
);
|
||||
submit(
|
||||
&q,
|
||||
templates::restart("agent-b", Source::Manual, "r".to_owned()),
|
||||
);
|
||||
let claims = q.claim_ready();
|
||||
assert_eq!(claims.len(), 2, "different agents run concurrently");
|
||||
}
|
||||
|
||||
// ---- failure: cancel-downstream + AfterAny ----
|
||||
|
||||
#[test]
|
||||
fn failed_node_cancels_downstream_but_afterany_reconcile_runs() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
let prebuild = claim_one(&q);
|
||||
q.complete_node(id, prebuild.node_id, Err("nix build exploded".to_owned()));
|
||||
// StopForUpdate + Swap are cancelled (AfterOk on a failed chain);
|
||||
// the AfterAny Reconcile still runs once Swap is terminal.
|
||||
let reconcile = claim_one(&q);
|
||||
assert_eq!(reconcile.kind.as_str(), "reconcile");
|
||||
q.complete_node(id, reconcile.node_id, Ok(()));
|
||||
let snap = q.snapshot();
|
||||
let dag = snap.iter().find(|d| d.id == id).expect("dag");
|
||||
assert_eq!(dag.state, State::Failed, "roll-up failed");
|
||||
let by_kind = |k: &str| {
|
||||
dag.nodes
|
||||
.iter()
|
||||
.find(|n| n.kind == k)
|
||||
.expect("node present")
|
||||
.state
|
||||
};
|
||||
assert_eq!(by_kind("prebuild"), State::Failed);
|
||||
assert_eq!(by_kind("stop_for_update"), State::Cancelled);
|
||||
assert_eq!(by_kind("swap"), State::Cancelled);
|
||||
assert_eq!(by_kind("reconcile"), State::Done);
|
||||
assert_eq!(
|
||||
dag.nodes
|
||||
.iter()
|
||||
.find(|n| n.kind == "prebuild")
|
||||
.and_then(|n| n.error.as_deref()),
|
||||
Some("nix build exploded")
|
||||
);
|
||||
}
|
||||
|
||||
/// The swap-failure recovery: `Swap` fails → the `AfterAny` edge still
|
||||
/// runs `Reconcile`, which brings a wanted-up agent back on its old
|
||||
/// config.
|
||||
#[test]
|
||||
fn swap_failure_still_runs_reconcile() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
for _ in 0..2 {
|
||||
let c = claim_one(&q);
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
let swap = claim_one(&q);
|
||||
assert_eq!(swap.kind.as_str(), "swap");
|
||||
q.complete_node(id, swap.node_id, Err("update failed".to_owned()));
|
||||
let reconcile = claim_one(&q);
|
||||
assert_eq!(reconcile.kind.as_str(), "reconcile");
|
||||
q.complete_node(id, reconcile.node_id, Ok(()));
|
||||
assert_eq!(state_of(&q, id), State::Failed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_reconcile_marks_dag_failed() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
Template::Start,
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"start".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
let c = claim_one(&q);
|
||||
q.complete_node(id, c.node_id, Err("start failed".to_owned()));
|
||||
assert_eq!(state_of(&q, id), State::Failed);
|
||||
}
|
||||
|
||||
// ---- cancel ----
|
||||
|
||||
#[test]
|
||||
fn cancel_clears_queued_dag() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
assert!(q.cancel(id));
|
||||
assert_eq!(state_of(&q, id), State::Cancelled);
|
||||
assert!(q.claim_ready().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_refuses_running_dag() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
let _ = claim_one(&q);
|
||||
assert!(!q.cancel(id));
|
||||
assert_eq!(state_of(&q, id), State::Running);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_children_marks_queued_children_only() {
|
||||
let q = JobQueue::new(1);
|
||||
let meta = submit(
|
||||
&q,
|
||||
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
|
||||
);
|
||||
// Parent's MetaLock is running while children exist.
|
||||
let lock = claim_one(&q);
|
||||
assert_eq!(lock.dag_id, meta);
|
||||
let child_a = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"agent-a",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
);
|
||||
let child_b = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"agent-b",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
);
|
||||
let unrelated = submit(&q, rebuild("agent-c", "operator queued"));
|
||||
// MetaLock holds the single build slot, so both children (and the
|
||||
// unrelated rebuild) are still fully queued here.
|
||||
let cancelled = q.cancel_children(meta);
|
||||
assert_eq!(cancelled, 2);
|
||||
assert_eq!(state_of(&q, child_a), State::Cancelled);
|
||||
assert_eq!(state_of(&q, child_b), State::Cancelled);
|
||||
assert_eq!(state_of(&q, unrelated), State::Queued);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_children_skips_running_child() {
|
||||
let q = JobQueue::new(2);
|
||||
let meta = submit(
|
||||
&q,
|
||||
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
|
||||
);
|
||||
let lock = claim_one(&q);
|
||||
let running_child = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"agent-a",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
);
|
||||
let queued_child = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"agent-b",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
);
|
||||
// Second slot lets running_child's prebuild start.
|
||||
let child_claim = claim_one(&q);
|
||||
assert_eq!(child_claim.dag_id, running_child);
|
||||
let n = q.cancel_children(meta);
|
||||
assert_eq!(n, 1);
|
||||
assert_eq!(state_of(&q, running_child), State::Running);
|
||||
assert_eq!(state_of(&q, queued_child), State::Cancelled);
|
||||
q.complete_node(meta, lock.node_id, Ok(()));
|
||||
}
|
||||
|
||||
// ---- fan-out ----
|
||||
|
||||
#[test]
|
||||
fn append_children_sets_parent_and_dedups() {
|
||||
let q = JobQueue::new(1);
|
||||
let meta = submit(
|
||||
&q,
|
||||
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
|
||||
);
|
||||
let specs = vec![
|
||||
templates::rebuild(
|
||||
"alice",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
templates::rebuild(
|
||||
"bob",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
// Duplicate — must coalesce into the first alice child.
|
||||
templates::rebuild(
|
||||
"alice",
|
||||
Source::MetaUpdate,
|
||||
"cascade again".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
];
|
||||
let ids = q.append_children(specs);
|
||||
assert_eq!(ids.len(), 3);
|
||||
assert_eq!(ids[0], ids[2], "duplicate child dedups");
|
||||
let snap = q.snapshot();
|
||||
let children: Vec<_> = snap.iter().filter(|d| d.parent_id == Some(meta)).collect();
|
||||
assert_eq!(children.len(), 2);
|
||||
}
|
||||
|
||||
// ---- terminal reporting + lease release ----
|
||||
|
||||
#[test]
|
||||
fn terminal_dag_reported_exactly_once_and_lease_released() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::restart("agent-a", Source::Manual, "r".to_owned()),
|
||||
);
|
||||
let stop = claim_one(&q);
|
||||
let r1 = q.complete_node(id, stop.node_id, Ok(()));
|
||||
assert!(r1.terminal.is_empty(), "dag not terminal yet");
|
||||
let rec = claim_one(&q);
|
||||
let r2 = q.complete_node(id, rec.node_id, Ok(()));
|
||||
assert_eq!(r2.terminal.len(), 1);
|
||||
assert_eq!(r2.terminal[0].dag_id, id);
|
||||
assert_eq!(r2.terminal[0].state, State::Done);
|
||||
// Lease released: a new DAG for the agent can claim immediately.
|
||||
let next = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
Template::Stop,
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"stop".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.dag_id, next);
|
||||
assert!(c.lease_acquired);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancelled_dag_reports_terminal() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
assert!(q.cancel(id));
|
||||
// The cancel path settles internally; a subsequent completion
|
||||
// report must not re-report it. Verify via a second dag's cycle.
|
||||
let other = submit(&q, rebuild("agent-b", "r"));
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.dag_id, other);
|
||||
let report = q.complete_node(other, c.node_id, Err("boom".to_owned()));
|
||||
// agent-b's dag isn't terminal (reconcile still pending) and
|
||||
// agent-a's was already reported by cancel → nothing here.
|
||||
assert!(report.terminal.iter().all(|t| t.dag_id != id));
|
||||
}
|
||||
|
||||
// ---- steps, build logs, history ----
|
||||
|
||||
#[test]
|
||||
fn set_step_only_on_running_and_signals_change() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
assert!(!q.set_step(id, 0, "too early"), "queued node refuses step");
|
||||
let c = claim_one(&q);
|
||||
assert!(q.set_step(id, c.node_id, "nix build"));
|
||||
assert!(
|
||||
!q.set_step(id, c.node_id, "nix build"),
|
||||
"same label → false"
|
||||
);
|
||||
assert!(q.set_step(id, c.node_id, "next phase"));
|
||||
assert!(q.set_step_running(id, "via running lookup"));
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
let snap = q.snapshot();
|
||||
let node = &snap.iter().find(|d| d.id == id).expect("dag").nodes[0];
|
||||
assert_eq!(node.step, None, "step cleared on completion");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_build_log_id_links_running_node() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
assert!(!q.set_build_log_id(id, 0, 41), "queued node refuses log id");
|
||||
let c = claim_one(&q);
|
||||
assert!(q.set_build_log_id(id, c.node_id, 42));
|
||||
assert!(q.set_build_log_id_running(id, 43));
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
let snap = q.snapshot();
|
||||
let node = &snap.iter().find(|d| d.id == id).expect("dag").nodes[0];
|
||||
assert_eq!(node.build_log_id, Some(43), "log id survives completion");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn history_evicts_old_terminals_per_template() {
|
||||
let q = JobQueue::new(1);
|
||||
for i in 0..8 {
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
Template::Start,
|
||||
&format!("agent-{i}"),
|
||||
Source::Manual,
|
||||
"start".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
let c = claim_one(&q);
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
assert_eq!(q.snapshot().len(), 5, "per-template history cap");
|
||||
assert_eq!(q.live_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_is_truncated() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
let c = claim_one(&q);
|
||||
q.complete_node(id, c.node_id, Err("x".repeat(5000)));
|
||||
let snap = q.snapshot();
|
||||
let err = snap.iter().find(|d| d.id == id).expect("dag").nodes[0]
|
||||
.error
|
||||
.clone()
|
||||
.expect("error stored");
|
||||
assert!(err.chars().count() <= 2001, "truncated + ellipsis");
|
||||
assert!(err.ends_with('…'));
|
||||
}
|
||||
|
||||
// ---- template shapes ----
|
||||
|
||||
#[test]
|
||||
fn graceful_stop_shape_signal_drain_reconcile() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::graceful_stop("agent-a", Source::Manual, "graceful".to_owned()),
|
||||
);
|
||||
for expected in ["signal", "drain", "reconcile"] {
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.kind.as_str(), expected);
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
assert_eq!(state_of(&q, id), State::Done);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn graceful_signal_and_drain_hold_no_build_slot() {
|
||||
// A whole-hive graceful stop overlaps every drain even at
|
||||
// buildSlots = 1 while a rebuild hogs the slot.
|
||||
let q = JobQueue::new(1);
|
||||
submit(&q, rebuild("builder", "slot hog"));
|
||||
submit(
|
||||
&q,
|
||||
templates::graceful_stop("agent-a", Source::Manual, "g".to_owned()),
|
||||
);
|
||||
submit(
|
||||
&q,
|
||||
templates::graceful_stop("agent-b", Source::Manual, "g".to_owned()),
|
||||
);
|
||||
let claims = q.claim_ready();
|
||||
let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect();
|
||||
assert_eq!(
|
||||
kinds,
|
||||
vec!["prebuild", "signal", "signal"],
|
||||
"both agents' signals fire while the slot is held"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spawn_shape_create_dropin_reconcile() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::spawn("newbie", 7, "approval #7 spawn".to_owned()),
|
||||
);
|
||||
for expected in ["create", "write_dropin", "reconcile"] {
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.kind.as_str(), expected);
|
||||
assert_eq!(c.approval_id, Some(7));
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
let report_terminal = state_of(&q, id);
|
||||
assert_eq!(report_terminal, State::Done);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn perm_change_shape_prefixes_rebuild_chain() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::perm_change(
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"perm".to_owned(),
|
||||
PermPayload::Combined {
|
||||
groups: Some(vec![]),
|
||||
caps: None,
|
||||
},
|
||||
),
|
||||
);
|
||||
for expected in [
|
||||
"write_perm_file",
|
||||
"prebuild",
|
||||
"stop_for_update",
|
||||
"swap",
|
||||
"reconcile",
|
||||
] {
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.kind.as_str(), expected);
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
assert_eq!(state_of(&q, id), State::Done);
|
||||
}
|
||||
Loading…
Reference in a new issue