test+docs(#2448): multi-agent restart test + doc shapes
- multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs: two agents restart in one DAG; both SetWanted heads are roots, each acquiring its own agent's lease concurrently. Existing single-agent shape/lease tests updated for the templates::restart(&[..], graceful) signature. - coordinator.md + templates.rs module doc: restart takes an agent list; a hive-wide restart is one DAG with N per-agent subgraphs.
This commit is contained in:
parent
1739716fa2
commit
c8bd0f7180
2 changed files with 75 additions and 8 deletions
|
|
@ -76,12 +76,15 @@ container build:
|
|||
|
||||
The power ops write the durable `wanted` intent via a head `SetWanted`
|
||||
node (not a pre-submit side effect) — it holds the agent lease, so
|
||||
intent-write + reconcile is atomic per-agent.
|
||||
intent-write + reconcile is atomic per-agent. `restart` takes an agent
|
||||
*list*: a hive-wide `hivectl restart` / `restart-all` is ONE DAG with a
|
||||
per-agent restart subgraph each (independent roots, run concurrently on
|
||||
their own leases), not N separate DAGs.
|
||||
|
||||
```text
|
||||
rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(after-any) Reconcile(a)
|
||||
graceful-stop(a): SetWanted(a,Off) → Signal(a) → Drain(a) → Reconcile(a)
|
||||
restart(a): SetWanted(a,Up) → StopForUpdate(a) → Reconcile(a)
|
||||
restart(a..): per agent: SetWanted(a,Up) → StopForUpdate(a) → Reconcile(a) (N subgraphs, 1 DAG)
|
||||
graceful-restart(a): SetWanted(a,Up) → Signal(a) → Drain(a) → StopForUpdate(a) → Reconcile(a)
|
||||
start(a): SetWanted(a,Up) → Reconcile(a) (stale rev ⇒ stale-start below)
|
||||
stale-start(a): SetWanted(a,Up) → «rebuild subgraph» (prebuild noops — agent is down)
|
||||
|
|
|
|||
|
|
@ -67,7 +67,12 @@ fn distinct_submits_never_collapse() {
|
|||
let b = submit(&q, rebuild("agent-b", "r"));
|
||||
let c = submit(
|
||||
&q,
|
||||
templates::restart("agent-a", Source::Manual, "r".to_owned()),
|
||||
templates::restart(
|
||||
&["agent-a".to_owned()],
|
||||
false,
|
||||
Source::Manual,
|
||||
"r".to_owned(),
|
||||
),
|
||||
);
|
||||
assert_ne!(a, b);
|
||||
assert_ne!(a, c);
|
||||
|
|
@ -200,7 +205,12 @@ fn lease_serializes_two_lifecycle_dags_for_same_agent() {
|
|||
let q = JobQueue::new(4);
|
||||
let restart = submit(
|
||||
&q,
|
||||
templates::restart("agent-a", Source::Manual, "restart".to_owned()),
|
||||
templates::restart(
|
||||
&["agent-a".to_owned()],
|
||||
false,
|
||||
Source::Manual,
|
||||
"restart".to_owned(),
|
||||
),
|
||||
);
|
||||
let stop = submit(
|
||||
&q,
|
||||
|
|
@ -284,16 +294,60 @@ fn agents_do_not_contend_on_each_others_leases() {
|
|||
let q = JobQueue::new(4);
|
||||
submit(
|
||||
&q,
|
||||
templates::restart("agent-a", Source::Manual, "r".to_owned()),
|
||||
templates::restart(
|
||||
&["agent-a".to_owned()],
|
||||
false,
|
||||
Source::Manual,
|
||||
"r".to_owned(),
|
||||
),
|
||||
);
|
||||
submit(
|
||||
&q,
|
||||
templates::restart("agent-b", Source::Manual, "r".to_owned()),
|
||||
templates::restart(
|
||||
&["agent-b".to_owned()],
|
||||
false,
|
||||
Source::Manual,
|
||||
"r".to_owned(),
|
||||
),
|
||||
);
|
||||
let claims = q.claim_ready();
|
||||
assert_eq!(claims.len(), 2, "different agents run concurrently");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() {
|
||||
let q = JobQueue::new(4);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::restart(
|
||||
&["agent-a".to_owned(), "agent-b".to_owned()],
|
||||
false,
|
||||
Source::Manual,
|
||||
"hive-wide".to_owned(),
|
||||
),
|
||||
);
|
||||
// A hive-wide restart is ONE DAG, not one-per-agent.
|
||||
assert_eq!(q.snapshot().len(), 1);
|
||||
// Each agent's subgraph head (SetWanted) is a root, so both are
|
||||
// claimable at once — each takes its OWN agent's lease (no contention
|
||||
// across distinct agents), all inside the single DAG.
|
||||
let claims = q.claim_ready();
|
||||
assert!(claims.iter().all(|c| c.dag_id == id));
|
||||
let mut heads: Vec<(&str, &str, bool)> = claims
|
||||
.iter()
|
||||
.map(|c| (c.agent.as_str(), c.kind.as_str(), c.lease_acquired))
|
||||
.collect();
|
||||
heads.sort_unstable();
|
||||
assert_eq!(
|
||||
heads,
|
||||
vec![
|
||||
("agent-a", "set_wanted", true),
|
||||
("agent-b", "set_wanted", true),
|
||||
],
|
||||
"both per-agent subgraphs start concurrently, each acquiring its own lease"
|
||||
);
|
||||
}
|
||||
|
||||
// ---- failure: cancel-downstream + AfterAny ----
|
||||
|
||||
#[test]
|
||||
|
|
@ -514,7 +568,12 @@ fn terminal_dag_reported_exactly_once_and_lease_released() {
|
|||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::restart("agent-a", Source::Manual, "r".to_owned()),
|
||||
templates::restart(
|
||||
&["agent-a".to_owned()],
|
||||
false,
|
||||
Source::Manual,
|
||||
"r".to_owned(),
|
||||
),
|
||||
);
|
||||
// restart = SetWanted → StopForUpdate → Reconcile; not terminal until
|
||||
// the last node completes.
|
||||
|
|
@ -622,7 +681,12 @@ fn trim_keeps_terminal_parent_with_live_children() {
|
|||
);
|
||||
let lock = claim_one(&q);
|
||||
q.complete_node(meta, lock.node_id, Ok(()));
|
||||
let mut child_spec = templates::restart("agent-x", Source::MetaUpdate, "cascade".to_owned());
|
||||
let mut child_spec = templates::restart(
|
||||
&["agent-x".to_owned()],
|
||||
false,
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
);
|
||||
child_spec.parent_id = Some(meta);
|
||||
let child = submit(&q, child_spec);
|
||||
// Churn > MAX_HISTORY_PER_TEMPLATE terminal meta_update DAGs.
|
||||
|
|
|
|||
Loading…
Reference in a new issue