vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

restart preserves wanted intent instead of forcing all agents up (#2540)

Browse source
This commit is contained in:
damocles 2026-07-16 20:05:27 +02:00 • committed by mara
commit 5bb5a88aa0
4 changed files with 77 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

52
hive-c0re/src/job_queue/tests.rs
View file

@ -18,9 +18,9 @@ fn rebuild(agent: &str, reason: &str) -> DagSpec {
}
/// Restart DAG spec with every agent treated as **running** — the online
/// shape (`SetWanted → [Signal→Drain→] StopForUpdate → Reconcile`) most
/// queue-mechanics tests assume. Mirrors the pre-dynamic `templates::restart`
/// (which is now the state-aware `submit::restart_spec`).
/// shape (`[Signal→Drain→] StopForUpdate → Reconcile`, no `SetWanted` head)
/// most queue-mechanics tests assume. Mirrors the pre-dynamic
/// `templates::restart` (which is now the state-aware `submit::restart_spec`).
fn restart_online(agents: &[&str], graceful: bool, reason: &str) -> DagSpec {
let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
submit::restart_spec(&targets, graceful, Source::Manual, reason.to_owned())
@ -229,27 +229,23 @@ fn lease_serializes_two_lifecycle_dags_for_same_agent() {
None,
),
);
// Restart's head SetWanted takes the lease; stop's Reconcile must
// wait even though slots are free.
// Restart's first node (StopForUpdate) takes the lease; stop's
// Reconcile must wait even though slots are free.
let first = claim_one(&q);
assert_eq!(first.dag_id, restart);
assert_eq!(first.kind.as_str(), "set_wanted");
assert_eq!(first.kind.as_str(), "stop_for_update");
assert!(first.lease_acquired);
q.complete_node(restart, first.node_id, Ok(()));
// Same DAG keeps the lease through StopForUpdate then Reconcile.
// Same DAG keeps the lease through the tail Reconcile.
let second = claim_one(&q);
assert_eq!(second.dag_id, restart);
assert_eq!(second.kind.as_str(), "stop_for_update");
assert_eq!(second.kind.as_str(), "reconcile");
assert!(!second.lease_acquired, "lease already held by this DAG");
q.complete_node(restart, second.node_id, Ok(()));
let third = claim_one(&q);
assert_eq!(third.dag_id, restart);
assert_eq!(third.kind.as_str(), "reconcile");
q.complete_node(restart, third.node_id, Ok(()));
// Restart terminal → lease released → stop's Reconcile runs.
let fourth = claim_one(&q);
assert_eq!(fourth.dag_id, stop);
q.complete_node(stop, fourth.node_id, Ok(()));
let third = claim_one(&q);
assert_eq!(third.dag_id, stop);
q.complete_node(stop, third.node_id, Ok(()));
assert_eq!(state_of(&q, restart), State::Done);
assert_eq!(state_of(&q, stop), State::Done);
}
@ -314,9 +310,9 @@ fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() {
);
// A hive-wide restart is ONE DAG, not one-per-agent.
assert_eq!(q.snapshot().len(), 1);
// Each agent's subgraph head (SetWanted) is a root, so both are
// claimable at once — each takes its OWN agent's lease (no contention
// across distinct agents), all inside the single DAG.
// Each agent's subgraph head (StopForUpdate, since both are running) is
// a root, so both are claimable at once — each takes its OWN agent's
// lease (no contention across distinct agents), all inside the single DAG.
let claims = q.claim_ready();
assert!(claims.iter().all(|c| c.dag_id == id));
let mut heads: Vec<(&str, &str, bool)> = claims
@ -327,8 +323,8 @@ fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() {
assert_eq!(
heads,
vec![
("agent-a", "set_wanted", true),
("agent-b", "set_wanted", true),
("agent-a", "stop_for_update", true),
("agent-b", "stop_for_update", true),
],
"both per-agent subgraphs start concurrently, each acquiring its own lease"
);
@ -470,8 +466,9 @@ fn offline_agents_skip_mechanical_nodes_but_keep_reconcile() {
"stop down".to_owned(),
),
);
// Offline restart → SetWanted(Up) → Reconcile (no StopForUpdate): a
// restart of a down agent is really a start.
// Offline restart → a lone Reconcile (no SetWanted, no StopForUpdate):
// nothing to bounce, and restart never rewrites intent, so the tail
// Reconcile converges the down agent to its existing `wanted`.
let restart = submit(
&q,
submit::restart_spec(
@ -498,8 +495,8 @@ fn offline_agents_skip_mechanical_nodes_but_keep_reconcile() {
);
assert_eq!(
shape(restart),
vec!["set_wanted".to_owned(), "reconcile".to_owned()],
"offline restart skips StopForUpdate, keeps Reconcile (it's a start)"
vec!["reconcile".to_owned()],
"offline restart is a lone Reconcile (no SetWanted head, nothing to stop)"
);
}
@ -749,11 +746,8 @@ fn cancel_refuses_running_dag() {
fn terminal_dag_reported_exactly_once_and_lease_released() {
let q = JobQueue::new(1);
let id = submit(&q, restart_online(&["agent-a"], false, "r"));
// restart = SetWanted → StopForUpdate → Reconcile; not terminal until
// the last node completes.
let set_wanted = claim_one(&q);
q.complete_node(id, set_wanted.node_id, Ok(()));
assert!(q.drain_terminal().is_empty(), "dag not terminal yet");
// restart = StopForUpdate → Reconcile; not terminal until the last
// node completes.
let stop = claim_one(&q);
q.complete_node(id, stop.node_id, Ok(()));
assert!(q.drain_terminal().is_empty(), "dag not terminal yet");