test+docs(#2448): multi-agent restart test + doc shapes

- multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs:
  two agents restart in one DAG; both SetWanted heads are roots, each
  acquiring its own agent's lease concurrently. Existing single-agent
  shape/lease tests updated for the templates::restart(&[..], graceful)
  signature.
- coordinator.md + templates.rs module doc: restart takes an agent list;
  a hive-wide restart is one DAG with N per-agent subgraphs.
This commit is contained in:
atlas 2026-07-14 23:00:21 +02:00 committed by mara
commit c8bd0f7180
2 changed files with 75 additions and 8 deletions

View file

@ -76,12 +76,15 @@ container build:
The power ops write the durable `wanted` intent via a head `SetWanted`
node (not a pre-submit side effect) — it holds the agent lease, so
intent-write + reconcile is atomic per-agent.
intent-write + reconcile is atomic per-agent. `restart` takes an agent
*list*: a hive-wide `hivectl restart` / `restart-all` is ONE DAG with a
per-agent restart subgraph each (independent roots, run concurrently on
their own leases), not N separate DAGs.
```text
rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(after-any) Reconcile(a)
graceful-stop(a): SetWanted(a,Off) → Signal(a) → Drain(a) → Reconcile(a)
restart(a): SetWanted(a,Up) → StopForUpdate(a) → Reconcile(a)
restart(a..): per agent: SetWanted(a,Up) → StopForUpdate(a) → Reconcile(a) (N subgraphs, 1 DAG)
graceful-restart(a): SetWanted(a,Up) → Signal(a) → Drain(a) → StopForUpdate(a) → Reconcile(a)
start(a): SetWanted(a,Up) → Reconcile(a) (stale rev ⇒ stale-start below)
stale-start(a): SetWanted(a,Up) → «rebuild subgraph» (prebuild noops — agent is down)

View file

@ -67,7 +67,12 @@ fn distinct_submits_never_collapse() {
let b = submit(&q, rebuild("agent-b", "r"));
let c = submit(
&q,
templates::restart("agent-a", Source::Manual, "r".to_owned()),
templates::restart(
&["agent-a".to_owned()],
false,
Source::Manual,
"r".to_owned(),
),
);
assert_ne!(a, b);
assert_ne!(a, c);
@ -200,7 +205,12 @@ fn lease_serializes_two_lifecycle_dags_for_same_agent() {
let q = JobQueue::new(4);
let restart = submit(
&q,
templates::restart("agent-a", Source::Manual, "restart".to_owned()),
templates::restart(
&["agent-a".to_owned()],
false,
Source::Manual,
"restart".to_owned(),
),
);
let stop = submit(
&q,
@ -284,16 +294,60 @@ fn agents_do_not_contend_on_each_others_leases() {
let q = JobQueue::new(4);
submit(
&q,
templates::restart("agent-a", Source::Manual, "r".to_owned()),
templates::restart(
&["agent-a".to_owned()],
false,
Source::Manual,
"r".to_owned(),
),
);
submit(
&q,
templates::restart("agent-b", Source::Manual, "r".to_owned()),
templates::restart(
&["agent-b".to_owned()],
false,
Source::Manual,
"r".to_owned(),
),
);
let claims = q.claim_ready();
assert_eq!(claims.len(), 2, "different agents run concurrently");
}
#[test]
fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() {
let q = JobQueue::new(4);
let id = submit(
&q,
templates::restart(
&["agent-a".to_owned(), "agent-b".to_owned()],
false,
Source::Manual,
"hive-wide".to_owned(),
),
);
// A hive-wide restart is ONE DAG, not one-per-agent.
assert_eq!(q.snapshot().len(), 1);
// Each agent's subgraph head (SetWanted) is a root, so both are
// claimable at once — each takes its OWN agent's lease (no contention
// across distinct agents), all inside the single DAG.
let claims = q.claim_ready();
assert!(claims.iter().all(|c| c.dag_id == id));
let mut heads: Vec<(&str, &str, bool)> = claims
.iter()
.map(|c| (c.agent.as_str(), c.kind.as_str(), c.lease_acquired))
.collect();
heads.sort_unstable();
assert_eq!(
heads,
vec![
("agent-a", "set_wanted", true),
("agent-b", "set_wanted", true),
],
"both per-agent subgraphs start concurrently, each acquiring its own lease"
);
}
// ---- failure: cancel-downstream + AfterAny ----
#[test]
@ -514,7 +568,12 @@ fn terminal_dag_reported_exactly_once_and_lease_released() {
let q = JobQueue::new(1);
let id = submit(
&q,
templates::restart("agent-a", Source::Manual, "r".to_owned()),
templates::restart(
&["agent-a".to_owned()],
false,
Source::Manual,
"r".to_owned(),
),
);
// restart = SetWanted → StopForUpdate → Reconcile; not terminal until
// the last node completes.
@ -622,7 +681,12 @@ fn trim_keeps_terminal_parent_with_live_children() {
);
let lock = claim_one(&q);
q.complete_node(meta, lock.node_id, Ok(()));
let mut child_spec = templates::restart("agent-x", Source::MetaUpdate, "cascade".to_owned());
let mut child_spec = templates::restart(
&["agent-x".to_owned()],
false,
Source::MetaUpdate,
"cascade".to_owned(),
);
child_spec.parent_id = Some(meta);
let child = submit(&q, child_spec);
// Churn > MAX_HISTORY_PER_TEMPLATE terminal meta_update DAGs.