vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

jobq: templates swallow the DagSpec layer

Browse source 
DagSpec described the graph the templates were about to build, one layer
below the templates themselves. Per #2972 the templates should be that
unit, so the spec type is gone and every declarer writes onto the job
builder directly.

- delete DagSpec<F> and its hand-written Debug impl
- submit(source, reason, declare: impl FnOnce(&Job)) replaces the
  pre-built-spec signature; submit_and_emit follows
- all six templates take &Job; the Source is now the caller's to pass,
  which spawn and approval_deploy previously hardcoded while the other
  four did not
- power_dag dissolves into stop_nodes/start_nodes/restart_nodes, which
  borrow their targets instead of owning them

315 tests pass unchanged.
This commit is contained in:
atlas 2026-08-03 01:23:40 +02:00 • committed by mara
commit 6899f574f6
7 changed files with 347 additions and 550 deletions
Download patch file Download diff file Expand all files Collapse all files

356
hive-c0re/src/job_queue/tests.rs
View file

@ -17,40 +17,36 @@
use super::model::NodeKind;
use super::*;
fn submit<F: FnOnce(&Job)>(q: &JobQueue, spec: DagSpec<F>) -> u64 {
q.submit(spec).expect("valid spec")
/// Submit a declared shape with the metadata every mechanics test uses.
/// `Source::Manual` because none of these exercise provenance — the tests that
/// do name their own source at the call site.
fn submit(q: &JobQueue, reason: &str, declare: impl FnOnce(&Job)) -> u64 {
q.submit(Source::Manual, reason.to_owned(), declare)
.expect("valid shape")
}
fn ident(s: &str) -> hive_types::Ident {
hive_types::Ident::parse(s).expect("valid test ident")
}
fn rebuild(agent: &str, reason: &str) -> DagSpec<impl FnOnce(&Job) + use<>> {
templates::rebuild(agent, Source::Manual, reason.to_owned(), true)
fn rebuild(b: &Job, agent: &str) {
templates::rebuild(b, agent, true);
}
/// Restart DAG spec with every agent treated as **running** — the online
/// Restart shape with every agent treated as **running** — the online
/// shape (`[Signal→Drain→] StopForUpdate → Reconcile`, no `SetWanted` head)
/// most queue-mechanics tests assume. Mirrors the pre-dynamic
/// `templates::restart` (which is now the state-aware `submit::restart_spec`).
fn restart_online(
agents: &[&str],
graceful: bool,
reason: &str,
) -> DagSpec<impl FnOnce(&Job) + use<>> {
/// `templates::restart` (which is now the state-aware `submit::restart_nodes`).
fn restart_online(b: &Job, agents: &[&str], graceful: bool) {
let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
submit::restart_spec(&targets, graceful, Source::Manual, reason.to_owned())
submit::restart_nodes(b, &targets, graceful);
}
/// Stop DAG spec with every agent treated as **running** — the online shape
/// Stop shape with every agent treated as **running** — the online shape
/// (`SetWanted → [Signal→Drain→](graceful) Reconcile`).
fn stop_online(
agents: &[&str],
graceful: bool,
reason: &str,
) -> DagSpec<impl FnOnce(&Job) + use<>> {
fn stop_online(b: &Job, agents: &[&str], graceful: bool) {
let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
submit::stop_spec(&targets, graceful, Source::Manual, reason.to_owned())
submit::stop_nodes(b, &targets, graceful);
}
// `Claimed` / `ClaimReady` / `CompleteNode` lived here: a claim snapshot type
@ -303,9 +299,9 @@ fn state_of(q: &JobQueue, dag_id: u64) -> State {
#[test]
fn submit_assigns_distinct_ids() {
let q = JobQueue::new(1);
let a = submit(&q, rebuild("agent-a", "first"));
let b = submit(&q, rebuild("agent-b", "second"));
assert_ne!(a, b);
let first = submit(&q, "first", |b| rebuild(b, "agent-a"));
let second = submit(&q, "second", |b| rebuild(b, "agent-b"));
assert_ne!(first, second);
assert_eq!(q.snapshot().len(), 2);
}
@ -317,20 +313,20 @@ fn submit_assigns_distinct_ids() {
#[test]
fn identical_resubmit_is_a_distinct_dag() {
let q = JobQueue::new(1);
let a = submit(&q, rebuild("agent-a", "first"));
let b = submit(&q, rebuild("agent-a", "again"));
assert_ne!(a, b, "no dedup: identical resubmit is a new DAG");
let first = submit(&q, "first", |b| rebuild(b, "agent-a"));
let resubmit = submit(&q, "again", |b| rebuild(b, "agent-a"));
assert_ne!(first, resubmit, "no dedup: identical resubmit is a new DAG");
assert_eq!(q.snapshot().len(), 2);
}
#[test]
fn distinct_submits_never_collapse() {
let q = JobQueue::new(1);
let a = submit(&q, rebuild("agent-a", "r"));
let b = submit(&q, rebuild("agent-b", "r"));
let c = submit(&q, restart_online(&["agent-a"], false, "r"));
assert_ne!(a, b);
assert_ne!(a, c);
let rebuild_a = submit(&q, "r", |b| rebuild(b, "agent-a"));
let rebuild_b = submit(&q, "r", |b| rebuild(b, "agent-b"));
let restart_a = submit(&q, "r", |b| restart_online(b, &["agent-a"], false));
assert_ne!(rebuild_a, rebuild_b);
assert_ne!(rebuild_a, restart_a);
assert_eq!(q.snapshot().len(), 3);
}
@ -346,8 +342,8 @@ fn resubmit_while_running_is_new_dag() {
// swallowed" is the scenario people worry about, and a reader looking for
// it should find it.
let q = JobQueue::new(1);
let a = submit(&q, rebuild("agent-a", "first"));
let again = submit(&q, rebuild("agent-a", "config bumped during build"));
let a = submit(&q, "first", |b| rebuild(b, "agent-a"));
let again = submit(&q, "config bumped during build", |b| rebuild(b, "agent-a"));
assert_ne!(a, again);
assert_eq!(q.snapshot().len(), 2);
}
@ -383,7 +379,7 @@ fn rebuild_chain_is_declared_serial() {
// logic"). Both axes are asserted below because a template can break either
// one independently.
let q = JobQueue::new(1);
let id = submit(&q, rebuild("agent-a", "r"));
let id = submit(&q, "r", |b| rebuild(b, "agent-a"));
assert_eq!(
declared_shape(&q, id),
vec![
@ -429,24 +425,19 @@ fn rebuild_chain_is_declared_serial() {
#[test]
fn graceful_rebuild_chain_drains_before_stopping() {
let q = JobQueue::new(1);
let id = submit(
&q,
DagSpec {
source: Source::AutoUpdate,
reason: "sweep".to_owned(),
declare: Box::new(|b: &Job| {
templates::rebuild_nodes(
b,
"agent-a",
templates::RebuildOpts {
relock: true,
graceful: true,
},
None,
);
}),
},
);
let id = q
.submit(Source::AutoUpdate, "sweep".to_owned(), |b: &Job| {
templates::rebuild_nodes(
b,
"agent-a",
templates::RebuildOpts {
relock: true,
graceful: true,
},
None,
);
})
.expect("valid shape");
assert_eq!(
declared_shape(&q, id)
.iter()
@ -477,24 +468,17 @@ fn non_graceful_rebuild_has_no_signal_or_drain() {
// job keeps its nodes to itself and inserts them, so what it built is
// observable where it matters — in what the scheduler runs.
let q = JobQueue::new(1);
let id = submit(
&q,
DagSpec {
source: Source::Manual,
reason: "manual".to_owned(),
declare: Box::new(|b: &Job| {
templates::rebuild_nodes(
b,
"agent-a",
templates::RebuildOpts {
relock: true,
graceful: false,
},
None,
);
}),
},
);
let id = submit(&q, "manual", |b| {
templates::rebuild_nodes(
b,
"agent-a",
templates::RebuildOpts {
relock: true,
graceful: false,
},
None,
);
});
assert_eq!(
declared_shape(&q, id)
.iter()
@ -573,7 +557,7 @@ fn rebuild_chain_declares_the_slot_where_the_nix_work_is() {
// a resource unit is held for the acquirer's whole subtree, so the slot
// `Prebuild` takes covers `StopForUpdate` → `Swap` → `PostSwap` beneath it.
let q = JobQueue::new(1);
let id = submit(&q, rebuild("agent-a", "r"));
let id = submit(&q, "r", |b| rebuild(b, "agent-a"));
let res = |kind: &str| declared_resources(&q, node_of(&q, id, kind));
let agent = || Resource::Agent("agent-a".to_owned());
@ -610,10 +594,9 @@ fn rebuild_chain_declares_the_slot_where_the_nix_work_is() {
#[test]
fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() {
let q = JobQueue::new(4);
let id = submit(
&q,
restart_online(&["agent-a", "agent-b"], false, "hive-wide"),
);
let id = submit(&q, "hive-wide", |b| {
restart_online(b, &["agent-a", "agent-b"], false);
});
// A hive-wide restart is ONE DAG, not one-per-agent.
assert_eq!(q.snapshot().len(), 1);
// Each agent's subgraph head (StopForUpdate, since both are running) is a
@ -647,10 +630,9 @@ fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() {
#[test]
fn multi_agent_stop_is_one_dag_with_concurrent_per_agent_subgraphs() {
let q = JobQueue::new(4);
let id = submit(
&q,
stop_online(&["agent-a", "agent-b"], false, "hive-wide stop"),
);
let id = submit(&q, "hive-wide stop", |b| {
stop_online(b, &["agent-a", "agent-b"], false);
});
// A hive-wide stop is ONE DAG, not one-per-agent.
assert_eq!(q.snapshot().len(), 1);
// Same declared story as the restart case above: each agent's subgraph head
@ -679,19 +661,17 @@ fn multi_agent_stop_is_one_dag_with_concurrent_per_agent_subgraphs() {
#[test]
fn multi_agent_start_one_dag_folds_per_agent_stale_rebuild() {
let q = JobQueue::new(4);
let id = submit(
&q,
// fresh: offline + not stale → SetWanted → Reconcile.
// stale: offline + stale → SetWanted → «rebuild subgraph».
submit::start_spec(
// fresh: offline + not stale → SetWanted → Reconcile.
// stale: offline + stale → SetWanted → «rebuild subgraph».
let id = submit(&q, "hive-wide start", |b| {
submit::start_nodes(
b,
&[
("fresh".to_owned(), false, false),
("stale".to_owned(), false, true),
],
Source::Manual,
"hive-wide start".to_owned(),
),
);
);
});
// One DAG spanning both agents.
assert_eq!(q.snapshot().len(), 1);
// The fold is a *declared* difference, readable the moment submit returns:
@ -730,27 +710,15 @@ fn offline_agents_skip_mechanical_nodes_but_keep_reconcile() {
// read and node exec.
let q = JobQueue::new(4);
// Offline graceful stop → SetWanted(Off) → Reconcile (no Signal/Drain).
let stop = submit(
&q,
submit::stop_spec(
&[("down".to_owned(), false)],
true,
Source::Manual,
"stop down".to_owned(),
),
);
let stop = submit(&q, "stop down", |b| {
submit::stop_nodes(b, &[("down".to_owned(), false)], true);
});
// Offline restart → a lone Reconcile (no SetWanted, no StopForUpdate):
// nothing to bounce, and restart never rewrites intent, so the tail
// Reconcile converges the down agent to its existing `wanted`.
let restart = submit(
&q,
submit::restart_spec(
&[("down2".to_owned(), false)],
true,
Source::Manual,
"restart down".to_owned(),
),
);
let restart = submit(&q, "restart down", |b| {
submit::restart_nodes(b, &[("down2".to_owned(), false)], true);
});
let shape = |id: u64| -> Vec<String> {
q.snapshot()
.iter()
@ -784,21 +752,16 @@ fn boot_sweep_nodes_declare_their_own_resources() {
// meta commit inside another node's staged deploy window. Nothing failed to
// compile; only an exhaustive caller list would have caught it.
let q = JobQueue::new(4);
let id = submit(
&q,
DagSpec {
source: Source::AutoUpdate,
reason: "boot".to_owned(),
declare: Box::new(|b: &Job| {
crate::workers::auto_update::boot_nodes(
b,
true,
vec!["stale-agent".to_owned()],
vec!["drifted-agent".to_owned()],
);
}),
},
);
let id = q
.submit(Source::AutoUpdate, "boot".to_owned(), |b: &Job| {
crate::workers::auto_update::boot_nodes(
b,
true,
vec!["stale-agent".to_owned()],
vec!["drifted-agent".to_owned()],
);
})
.expect("valid shape");
let mut lock = declared_resources(&q, node_of(&q, id, "meta_lock"));
lock.sort_by_key(|r| format!("{r:?}"));
@ -816,7 +779,9 @@ fn boot_sweep_nodes_declare_their_own_resources() {
}
/// Crash-watch suppression for a cascade rebuild, which the deleted half of
/// `meta_update_grows_cascade_in_dag` used to assert via `DagSpec::transient`.
/// `meta_update_grows_cascade_in_dag` used to assert via a DAG-level
/// `transient` field on the submit-time spec (both the field and the spec type
/// are gone).
///
/// The property is unchanged — a container going down under a rebuild must not
/// read as a crash — but it is no longer a DAG-level declaration: each node
@ -897,7 +862,7 @@ fn rebuild_reconcile_waits_for_the_whole_build_subtree() {
// easy thing to break — someone flattening the chain would keep every edge
// and still lose the guarantee.
let q = JobQueue::new(1);
let id = submit(&q, rebuild("agent-a", "r"));
let id = submit(&q, "r", |b| rebuild(b, "agent-a"));
let shape = declared_shape(&q, id);
let parent_of = |kind: &str| {
shape
@ -969,21 +934,14 @@ fn rebuild_reconcile_waits_for_the_whole_build_subtree() {
#[test]
fn a_fanned_out_mechanical_node_declares_its_agent_lease() {
let q = JobQueue::new(4);
let id = submit(
&q,
DagSpec {
source: Source::Manual,
reason: "fan-out".to_owned(),
declare: Box::new(|b: &Job| {
templates::fanned_out_mechanical(
b,
NodeKind::Start {
agent: "agent-a".to_owned(),
},
);
}),
},
);
let id = submit(&q, "fan-out", |b| {
templates::fanned_out_mechanical(
b,
NodeKind::Start {
agent: "agent-a".to_owned(),
},
);
});
assert_eq!(declared_shape(&q, id), vec![row("start", None, &[])]);
assert_eq!(
declared_resources(&q, node_of(&q, id, "start")),
@ -1010,23 +968,18 @@ fn a_fanned_out_mechanical_node_declares_its_agent_lease() {
fn a_meta_lock_grows_one_rebuild_subgraph_per_agent() {
let q = JobQueue::new(4);
let agents = vec!["alice".to_owned(), "bob".to_owned()];
let id = submit(
&q,
DagSpec {
source: Source::AutoUpdate,
reason: "sweep".to_owned(),
declare: Box::new(move |b: &Job| {
templates::grown_rebuilds(
b,
&agents,
templates::RebuildOpts {
relock: true,
graceful: true,
},
);
}),
},
);
let id = q
.submit(Source::AutoUpdate, "sweep".to_owned(), |b: &Job| {
templates::grown_rebuilds(
b,
&agents,
templates::RebuildOpts {
relock: true,
graceful: true,
},
);
})
.expect("valid shape");
// One chain per agent, each an independent group root — so the two rebuild
// concurrently, each on its own lease.
@ -1056,7 +1009,7 @@ fn a_meta_lock_grows_one_rebuild_subgraph_per_agent() {
#[test]
fn cancel_clears_queued_dag() {
let q = JobQueue::new(1);
let id = submit(&q, rebuild("agent-a", "r"));
let id = submit(&q, "r", |b| rebuild(b, "agent-a"));
assert!(q.cancel(id), "fully-queued dag cancels");
// The operator sees `Cancelled` the moment the cancel returns — the spared
// tail is still `Pending`, and a DAG must not read `Queued` back to the
@ -1085,7 +1038,9 @@ fn cancel_clears_queued_dag() {
#[test]
fn cancel_drops_one_agents_branch_leaving_the_rest() {
let q = JobQueue::new(2);
let id = submit(&q, restart_online(&["agent-a", "agent-b"], false, "r"));
let id = submit(&q, "r", |b| {
restart_online(b, &["agent-a", "agent-b"], false);
});
// Per-agent subgraphs are independent roots; find agent-a's.
let snap = q.snapshot();
let dag = snap.iter().find(|d| d.id == id).expect("dag in snapshot");
@ -1152,28 +1107,21 @@ fn cancelled_power_op_runs_no_compensating_node() {
let case = format!("graceful={graceful} running={running}");
let q = JobQueue::new(1);
let id = submit(
&q,
submit::restart_spec(&targets, graceful, Source::Manual, "bounce".to_owned()),
);
let id = submit(&q, "bounce", |b| {
submit::restart_nodes(b, &targets, graceful);
});
assert_cancels_clean(&q, id, false, &format!("restart {case}"));
let q = JobQueue::new(1);
let id = submit(
&q,
submit::stop_spec(&targets, graceful, Source::Manual, "stop".to_owned()),
);
let id = submit(&q, "stop", |b| {
submit::stop_nodes(b, &targets, graceful);
});
assert_cancels_clean(&q, id, true, &format!("stop {case}"));
let q = JobQueue::new(1);
let id = submit(
&q,
submit::start_spec(
&[("agent-a".to_owned(), running, false)],
Source::Manual,
"start".to_owned(),
),
);
let id = submit(&q, "start", |b| {
submit::start_nodes(b, &[("agent-a".to_owned(), running, false)]);
});
assert_cancels_clean(&q, id, true, &format!("start {case}"));
}
}
@ -1192,10 +1140,9 @@ fn cancelled_power_op_runs_no_compensating_node() {
#[test]
fn cancelled_dag_still_runs_its_approval_tail() {
let q = JobQueue::new(1);
let id = submit(
&q,
templates::approval_deploy("agent-a", 7, "approval #7".to_owned()),
);
let id = submit(&q, "approval #7", |b| {
templates::approval_deploy(b, "agent-a", 7);
});
assert!(q.cancel(id), "fully-queued dag cancels");
// The `Cancelled` tail is the only node whose edge accepts a dropped
// dependency, so it is the only one `cancel` spares — and *which* tail
@ -1216,7 +1163,7 @@ fn cancelled_dag_still_runs_its_approval_tail() {
assert_eq!(state_of(&q, id), State::Cancelled);
// An unrelated DAG landing in the same graph doesn't disturb this one's
// roll-up — the snapshot is per-DAG, not a global state machine.
let _other = submit(&q, rebuild("agent-b", "r"));
let _other = submit(&q, "r", |b| rebuild(b, "agent-b"));
assert_eq!(state_of(&q, id), State::Cancelled);
}
@ -1230,10 +1177,9 @@ fn cancelled_dag_still_runs_its_approval_tail() {
#[test]
fn deploy_dag_runs_phases_in_order_and_tails_a_failed_apply() {
let q = JobQueue::new(1);
let id = submit(
&q,
templates::approval_deploy("agent-a", 7, "approval #7".to_owned()),
);
let id = submit(&q, "approval #7", |b| {
templates::approval_deploy(b, "agent-a", 7);
});
assert_eq!(
declared_shape(&q, id),
@ -1289,14 +1235,9 @@ fn deploy_apply_grows_rebuild_subgraph_and_finalizes_after_it() {
// children run (`a_completing_node_grows_the_work_it_declared`,
// `parent_parks_in_finishing_until_children_roll_up`).
let q = JobQueue::new(1);
let id = submit(
&q,
DagSpec {
source: Source::Manual,
reason: "deploy graft".to_owned(),
declare: Box::new(|b: &Job| templates::deploy_rebuild_nodes(b, "agent-a", 11)),
},
);
let id = submit(&q, "deploy graft", |b| {
templates::deploy_rebuild_nodes(b, "agent-a", 11);
});
assert_eq!(
declared_shape(&q, id),
@ -1461,7 +1402,7 @@ fn error_truncation_cuts_on_a_char_boundary() {
#[test]
fn graceful_stop_shape_signal_drain_reconcile() {
let q = JobQueue::new(1);
let id = submit(&q, stop_online(&["agent-a"], true, "graceful"));
let id = submit(&q, "graceful", |b| stop_online(b, &["agent-a"], true));
assert_eq!(
declared_shape(&q, id),
vec![
@ -1491,10 +1432,9 @@ fn graceful_stop_shape_signal_drain_reconcile() {
#[test]
fn spawn_shape_provision_create_dropin_reconcile() {
let q = JobQueue::new(1);
let id = submit(
&q,
templates::spawn("newbie", 7, "approval #7 spawn".to_owned()),
);
let id = submit(&q, "approval #7 spawn", |b| {
templates::spawn(b, "newbie", 7);
});
assert_eq!(
declared_shape(&q, id),
vec![
@ -1517,18 +1457,16 @@ fn spawn_shape_provision_create_dropin_reconcile() {
#[test]
fn perm_change_shape_prefixes_rebuild_chain() {
let q = JobQueue::new(1);
let id = submit(
&q,
let id = submit(&q, "perm", |b| {
templates::perm_change(
b,
"agent-a",
Source::Manual,
"perm".to_owned(),
PermPayload::Combined {
groups: Some(vec![]),
caps: None,
},
),
);
);
});
assert_eq!(
declared_shape(&q, id)
.iter()
@ -1557,14 +1495,9 @@ fn reparent_shape_is_a_lone_agentless_meta_window_node() {
// `MetaLock`, and it must declare the meta window — a topology commit
// must not land inside another node's staged deploy window.
let q = JobQueue::new(1);
let id = submit(
&q,
templates::reparent(
vec![(ident("alice"), Some(ident("bob")))],
Source::Manual,
"set-parent".to_owned(),
),
);
let id = submit(&q, "set-parent", |b| {
templates::reparent(b, vec![(ident("alice"), Some(ident("bob")))]);
});
assert_eq!(
declared_shape(&q, id),
vec![row("reparent", None, &[])],
@ -1586,10 +1519,9 @@ fn reparent_bulk_shape_carries_every_move_on_one_node() {
// request is the reason a single node was chosen in the first place.
let moves = vec![(ident("alice"), Some(ident("bob"))), (ident("carol"), None)];
let q = JobQueue::new(1);
let id = submit(
&q,
templates::reparent(moves.clone(), Source::Manual, "set-parent-bulk".to_owned()),
);
let id = submit(&q, "set-parent-bulk", |b| {
templates::reparent(b, moves.clone());
});
assert_eq!(
declared_shape(&q, id),
vec![row("reparent", None, &[])],