diff --git a/docs/coordinator.md b/docs/coordinator.md index 8670e92a..7b0d6cff 100644 --- a/docs/coordinator.md +++ b/docs/coordinator.md @@ -76,12 +76,15 @@ container build: The power ops write the durable `wanted` intent via a head `SetWanted` node (not a pre-submit side effect) — it holds the agent lease, so -intent-write + reconcile is atomic per-agent. +intent-write + reconcile is atomic per-agent. `restart` takes an agent +*list*: a hive-wide `hivectl restart` / `restart-all` is ONE DAG with a +per-agent restart subgraph each (independent roots, run concurrently on +their own leases), not N separate DAGs. ```text rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(after-any) Reconcile(a) graceful-stop(a): SetWanted(a,Off) → Signal(a) → Drain(a) → Reconcile(a) -restart(a): SetWanted(a,Up) → StopForUpdate(a) → Reconcile(a) +restart(a..): per agent: SetWanted(a,Up) → StopForUpdate(a) → Reconcile(a) (N subgraphs, 1 DAG) graceful-restart(a): SetWanted(a,Up) → Signal(a) → Drain(a) → StopForUpdate(a) → Reconcile(a) start(a): SetWanted(a,Up) → Reconcile(a) (stale rev ⇒ stale-start below) stale-start(a): SetWanted(a,Up) → «rebuild subgraph» (prebuild noops — agent is down) diff --git a/hive-c0re/src/job_queue/tests.rs b/hive-c0re/src/job_queue/tests.rs index df26da4e..9a691ce8 100644 --- a/hive-c0re/src/job_queue/tests.rs +++ b/hive-c0re/src/job_queue/tests.rs @@ -67,7 +67,12 @@ fn distinct_submits_never_collapse() { let b = submit(&q, rebuild("agent-b", "r")); let c = submit( &q, - templates::restart("agent-a", Source::Manual, "r".to_owned()), + templates::restart( + &["agent-a".to_owned()], + false, + Source::Manual, + "r".to_owned(), + ), ); assert_ne!(a, b); assert_ne!(a, c); @@ -200,7 +205,12 @@ fn lease_serializes_two_lifecycle_dags_for_same_agent() { let q = JobQueue::new(4); let restart = submit( &q, - templates::restart("agent-a", Source::Manual, "restart".to_owned()), + templates::restart( + &["agent-a".to_owned()], + false, + Source::Manual, + "restart".to_owned(), + ), ); let stop = submit( &q, @@ -284,16 +294,60 @@ fn agents_do_not_contend_on_each_others_leases() { let q = JobQueue::new(4); submit( &q, - templates::restart("agent-a", Source::Manual, "r".to_owned()), + templates::restart( + &["agent-a".to_owned()], + false, + Source::Manual, + "r".to_owned(), + ), ); submit( &q, - templates::restart("agent-b", Source::Manual, "r".to_owned()), + templates::restart( + &["agent-b".to_owned()], + false, + Source::Manual, + "r".to_owned(), + ), ); let claims = q.claim_ready(); assert_eq!(claims.len(), 2, "different agents run concurrently"); } +#[test] +fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() { + let q = JobQueue::new(4); + let id = submit( + &q, + templates::restart( + &["agent-a".to_owned(), "agent-b".to_owned()], + false, + Source::Manual, + "hive-wide".to_owned(), + ), + ); + // A hive-wide restart is ONE DAG, not one-per-agent. + assert_eq!(q.snapshot().len(), 1); + // Each agent's subgraph head (SetWanted) is a root, so both are + // claimable at once — each takes its OWN agent's lease (no contention + // across distinct agents), all inside the single DAG. + let claims = q.claim_ready(); + assert!(claims.iter().all(|c| c.dag_id == id)); + let mut heads: Vec<(&str, &str, bool)> = claims + .iter() + .map(|c| (c.agent.as_str(), c.kind.as_str(), c.lease_acquired)) + .collect(); + heads.sort_unstable(); + assert_eq!( + heads, + vec![ + ("agent-a", "set_wanted", true), + ("agent-b", "set_wanted", true), + ], + "both per-agent subgraphs start concurrently, each acquiring its own lease" + ); +} + // ---- failure: cancel-downstream + AfterAny ---- #[test] @@ -514,7 +568,12 @@ fn terminal_dag_reported_exactly_once_and_lease_released() { let q = JobQueue::new(1); let id = submit( &q, - templates::restart("agent-a", Source::Manual, "r".to_owned()), + templates::restart( + &["agent-a".to_owned()], + false, + Source::Manual, + "r".to_owned(), + ), ); // restart = SetWanted → StopForUpdate → Reconcile; not terminal until // the last node completes. @@ -622,7 +681,12 @@ fn trim_keeps_terminal_parent_with_live_children() { ); let lock = claim_one(&q); q.complete_node(meta, lock.node_id, Ok(())); - let mut child_spec = templates::restart("agent-x", Source::MetaUpdate, "cascade".to_owned()); + let mut child_spec = templates::restart( + &["agent-x".to_owned()], + false, + Source::MetaUpdate, + "cascade".to_owned(), + ); child_spec.parent_id = Some(meta); let child = submit(&q, child_spec); // Churn > MAX_HISTORY_PER_TEMPLATE terminal meta_update DAGs.