refactor(#2439): build hive-wide stop/start/restart DAGs dynamically
Hive-wide `stop` / `start` / `restart` emit ONE DAG with a per-agent subgraph each (concurrent on their own leases) instead of N DAGs — and each subgraph is now built dynamically from the agent's live running state rather than a fixed template shape: - online agent: the full stop→reconcile (restart: stop-for-update→reconcile) chain; `graceful` prepends signal→drain. - offline agent: just `SetWanted → Reconcile` (nothing to quiesce/stop; a restart of a down agent is really a start). The head `SetWanted` (intent) and tail `Reconcile` (convergence guarantee) are always present; only the mechanical `Signal`/`Drain`/`StopForUpdate` nodes are state-conditional. Keeping `Reconcile` in every shape closes the TOCTOU window — a race-up between the `is_running` read and node exec is still converged in-DAG (with `StopForUpdate`-noop as the backstop) — with no reliance on an external reconcile sweep. The state-aware assembly needs an async `is_running` read, so it moves out of the pure/sync `templates.rs` into `submit.rs`, layered as pure `*_chain(running)` → pure `*_spec(targets)` (the unit-test seam) → async `*_many` (reads live state + submits). `templates.rs` keeps only the shared pure primitives (`node`/`after_ok`/`rebuild_nodes`). Callers await the now-async submit fns (server, dashboard, socket_server). Tests exercise both the online and offline shapes via the pure `*_spec` seam. docs/coordinator.md shapes updated.
This commit is contained in:
atlas
parent
3797177e7f
commit
860484a193
8 changed files with 574 additions and 335 deletions
|
|
@ -16,6 +16,22 @@ fn rebuild(agent: &str, reason: &str) -> DagSpec {
|
|||
templates::rebuild(agent, Source::Manual, reason.to_owned(), None, true)
|
||||
}
|
||||
|
||||
/// Restart DAG spec with every agent treated as **running** — the online
|
||||
/// shape (`SetWanted → [Signal→Drain→] StopForUpdate → Reconcile`) most
|
||||
/// queue-mechanics tests assume. Mirrors the pre-dynamic `templates::restart`
|
||||
/// (which is now the state-aware `submit::restart_spec`).
|
||||
fn restart_online(agents: &[&str], graceful: bool, reason: &str) -> DagSpec {
|
||||
let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
|
||||
submit::restart_spec(&targets, graceful, Source::Manual, reason.to_owned())
|
||||
}
|
||||
|
||||
/// Stop DAG spec with every agent treated as **running** — the online shape
|
||||
/// (`SetWanted → [Signal→Drain→](graceful) Reconcile`).
|
||||
fn stop_online(agents: &[&str], graceful: bool, reason: &str) -> DagSpec {
|
||||
let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
|
||||
submit::stop_spec(&targets, graceful, Source::Manual, reason.to_owned())
|
||||
}
|
||||
|
||||
/// Claim helper asserting exactly one node comes back.
|
||||
fn claim_one(q: &JobQueue) -> Claim {
|
||||
let mut claims = q.claim_ready();
|
||||
|
|
@ -65,15 +81,7 @@ fn distinct_submits_never_collapse() {
|
|||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "r"));
|
||||
let b = submit(&q, rebuild("agent-b", "r"));
|
||||
let c = submit(
|
||||
&q,
|
||||
templates::restart(
|
||||
&["agent-a".to_owned()],
|
||||
false,
|
||||
Source::Manual,
|
||||
"r".to_owned(),
|
||||
),
|
||||
);
|
||||
let c = submit(&q, restart_online(&["agent-a"], false, "r"));
|
||||
assert_ne!(a, b);
|
||||
assert_ne!(a, c);
|
||||
assert_eq!(q.snapshot().len(), 3);
|
||||
|
|
@ -203,15 +211,7 @@ fn fifo_fairness_for_the_slot() {
|
|||
#[test]
|
||||
fn lease_serializes_two_lifecycle_dags_for_same_agent() {
|
||||
let q = JobQueue::new(4);
|
||||
let restart = submit(
|
||||
&q,
|
||||
templates::restart(
|
||||
&["agent-a".to_owned()],
|
||||
false,
|
||||
Source::Manual,
|
||||
"restart".to_owned(),
|
||||
),
|
||||
);
|
||||
let restart = submit(&q, restart_online(&["agent-a"], false, "restart"));
|
||||
let stop = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
|
|
@ -292,24 +292,8 @@ fn lease_exempt_prebuild_overlaps_other_dag_on_same_agent() {
|
|||
#[test]
|
||||
fn agents_do_not_contend_on_each_others_leases() {
|
||||
let q = JobQueue::new(4);
|
||||
submit(
|
||||
&q,
|
||||
templates::restart(
|
||||
&["agent-a".to_owned()],
|
||||
false,
|
||||
Source::Manual,
|
||||
"r".to_owned(),
|
||||
),
|
||||
);
|
||||
submit(
|
||||
&q,
|
||||
templates::restart(
|
||||
&["agent-b".to_owned()],
|
||||
false,
|
||||
Source::Manual,
|
||||
"r".to_owned(),
|
||||
),
|
||||
);
|
||||
submit(&q, restart_online(&["agent-a"], false, "r"));
|
||||
submit(&q, restart_online(&["agent-b"], false, "r"));
|
||||
let claims = q.claim_ready();
|
||||
assert_eq!(claims.len(), 2, "different agents run concurrently");
|
||||
}
|
||||
|
|
@ -319,12 +303,7 @@ fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() {
|
|||
let q = JobQueue::new(4);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::restart(
|
||||
&["agent-a".to_owned(), "agent-b".to_owned()],
|
||||
false,
|
||||
Source::Manual,
|
||||
"hive-wide".to_owned(),
|
||||
),
|
||||
restart_online(&["agent-a", "agent-b"], false, "hive-wide"),
|
||||
);
|
||||
// A hive-wide restart is ONE DAG, not one-per-agent.
|
||||
assert_eq!(q.snapshot().len(), 1);
|
||||
|
|
@ -348,6 +327,129 @@ fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_agent_stop_is_one_dag_with_concurrent_per_agent_subgraphs() {
|
||||
let q = JobQueue::new(4);
|
||||
let id = submit(
|
||||
&q,
|
||||
stop_online(&["agent-a", "agent-b"], false, "hive-wide stop"),
|
||||
);
|
||||
// A hive-wide stop is ONE DAG, not one-per-agent.
|
||||
assert_eq!(q.snapshot().len(), 1);
|
||||
let claims = q.claim_ready();
|
||||
assert!(claims.iter().all(|c| c.dag_id == id));
|
||||
let mut heads: Vec<(&str, &str, bool)> = claims
|
||||
.iter()
|
||||
.map(|c| (c.agent.as_str(), c.kind.as_str(), c.lease_acquired))
|
||||
.collect();
|
||||
heads.sort_unstable();
|
||||
assert_eq!(
|
||||
heads,
|
||||
vec![
|
||||
("agent-a", "set_wanted", true),
|
||||
("agent-b", "set_wanted", true),
|
||||
],
|
||||
"both per-agent stop subgraphs start concurrently, each on its own lease"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_agent_start_one_dag_folds_per_agent_stale_rebuild() {
|
||||
let q = JobQueue::new(4);
|
||||
let id = submit(
|
||||
&q,
|
||||
// fresh: offline + not stale → SetWanted → Reconcile.
|
||||
// stale: offline + stale → SetWanted → «rebuild subgraph».
|
||||
submit::start_spec(
|
||||
&[
|
||||
("fresh".to_owned(), false, false),
|
||||
("stale".to_owned(), false, true),
|
||||
],
|
||||
Source::Manual,
|
||||
"hive-wide start".to_owned(),
|
||||
),
|
||||
);
|
||||
// One DAG spanning both agents.
|
||||
assert_eq!(q.snapshot().len(), 1);
|
||||
// Both subgraph heads (SetWanted(Up)) are roots — claimable at once,
|
||||
// each acquiring its own agent lease.
|
||||
let heads = q.claim_ready();
|
||||
assert!(
|
||||
heads
|
||||
.iter()
|
||||
.all(|c| c.dag_id == id && c.kind.as_str() == "set_wanted")
|
||||
);
|
||||
let mut head_agents: Vec<&str> = heads.iter().map(|c| c.agent.as_str()).collect();
|
||||
head_agents.sort_unstable();
|
||||
assert_eq!(head_agents, vec!["fresh", "stale"]);
|
||||
// Complete both heads; the fresh agent then reconciles directly while
|
||||
// the stale agent's subgraph is the rebuild chain (prebuild first).
|
||||
for c in &heads {
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
let next = q.claim_ready();
|
||||
let mut kinds: Vec<(&str, &str)> = next
|
||||
.iter()
|
||||
.map(|c| (c.agent.as_str(), c.kind.as_str()))
|
||||
.collect();
|
||||
kinds.sort_unstable();
|
||||
assert_eq!(
|
||||
kinds,
|
||||
vec![("fresh", "reconcile"), ("stale", "prebuild")],
|
||||
"fresh agent starts directly; stale agent rebuilds first, all in one DAG"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn offline_agents_skip_mechanical_nodes_but_keep_reconcile() {
|
||||
// The dynamic build skips Signal/Drain/StopForUpdate for a down agent
|
||||
// (nothing to quiesce/stop) but ALWAYS keeps the Reconcile tail — the
|
||||
// convergence guarantee that catches a race-up between the is_running
|
||||
// read and node exec.
|
||||
let q = JobQueue::new(4);
|
||||
// Offline graceful stop → SetWanted(Off) → Reconcile (no Signal/Drain).
|
||||
let stop = submit(
|
||||
&q,
|
||||
submit::stop_spec(
|
||||
&[("down".to_owned(), false)],
|
||||
true,
|
||||
Source::Manual,
|
||||
"stop down".to_owned(),
|
||||
),
|
||||
);
|
||||
// Offline restart → SetWanted(Up) → Reconcile (no StopForUpdate): a
|
||||
// restart of a down agent is really a start.
|
||||
let restart = submit(
|
||||
&q,
|
||||
submit::restart_spec(
|
||||
&[("down2".to_owned(), false)],
|
||||
true,
|
||||
Source::Manual,
|
||||
"restart down".to_owned(),
|
||||
),
|
||||
);
|
||||
let shape = |id: u64| -> Vec<String> {
|
||||
q.snapshot()
|
||||
.iter()
|
||||
.find(|d| d.id == id)
|
||||
.expect("dag")
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|n| n.kind.clone())
|
||||
.collect()
|
||||
};
|
||||
assert_eq!(
|
||||
shape(stop),
|
||||
vec!["set_wanted".to_owned(), "reconcile".to_owned()],
|
||||
"offline graceful stop skips the signal/drain quiesce, keeps Reconcile"
|
||||
);
|
||||
assert_eq!(
|
||||
shape(restart),
|
||||
vec!["set_wanted".to_owned(), "reconcile".to_owned()],
|
||||
"offline restart skips StopForUpdate, keeps Reconcile (it's a start)"
|
||||
);
|
||||
}
|
||||
|
||||
// ---- failure: cancel-downstream + AfterAny ----
|
||||
|
||||
#[test]
|
||||
|
|
@ -566,15 +668,7 @@ fn append_children_sets_parent() {
|
|||
#[test]
|
||||
fn terminal_dag_reported_exactly_once_and_lease_released() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::restart(
|
||||
&["agent-a".to_owned()],
|
||||
false,
|
||||
Source::Manual,
|
||||
"r".to_owned(),
|
||||
),
|
||||
);
|
||||
let id = submit(&q, restart_online(&["agent-a"], false, "r"));
|
||||
// restart = SetWanted → StopForUpdate → Reconcile; not terminal until
|
||||
// the last node completes.
|
||||
let set_wanted = claim_one(&q);
|
||||
|
|
@ -681,8 +775,8 @@ fn trim_keeps_terminal_parent_with_live_children() {
|
|||
);
|
||||
let lock = claim_one(&q);
|
||||
q.complete_node(meta, lock.node_id, Ok(()));
|
||||
let mut child_spec = templates::restart(
|
||||
&["agent-x".to_owned()],
|
||||
let mut child_spec = submit::restart_spec(
|
||||
&[("agent-x".to_owned(), true)],
|
||||
false,
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
|
|
@ -798,10 +892,7 @@ fn error_is_truncated() {
|
|||
#[test]
|
||||
fn graceful_stop_shape_signal_drain_reconcile() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::graceful_stop("agent-a", Source::Manual, "graceful".to_owned()),
|
||||
);
|
||||
let id = submit(&q, stop_online(&["agent-a"], true, "graceful"));
|
||||
for expected in ["set_wanted", "signal", "drain", "reconcile"] {
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.kind.as_str(), expected);
|
||||
|
|
@ -816,14 +907,8 @@ fn graceful_signal_and_drain_hold_no_build_slot() {
|
|||
// buildSlots = 1 while a rebuild hogs the slot.
|
||||
let q = JobQueue::new(1);
|
||||
submit(&q, rebuild("builder", "slot hog"));
|
||||
submit(
|
||||
&q,
|
||||
templates::graceful_stop("agent-a", Source::Manual, "g".to_owned()),
|
||||
);
|
||||
submit(
|
||||
&q,
|
||||
templates::graceful_stop("agent-b", Source::Manual, "g".to_owned()),
|
||||
);
|
||||
submit(&q, stop_online(&["agent-a"], true, "g"));
|
||||
submit(&q, stop_online(&["agent-b"], true, "g"));
|
||||
let claims = q.claim_ready();
|
||||
let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect();
|
||||
assert_eq!(
|
||||
|
|
|
|||
Loading…
Reference in a new issue