refactor(job-queue): build DAGs by naming nodes, not counting them
Every template built a `Vec<NodeSpec>` whose edges and parents were positional indices into that vector, so a shape was expressed as arithmetic: `base + 1`, `stop_root + 2`, `sfu + 1`, and a `reconcile_index()` helper that read the emitted vector's length to find out where its own last node had landed. `concat_subgraphs` existed solely to rebase one per-agent subgraph's indices onto another's. Templates now declare into a `hive_jobq::JobBuilder` and hold the handles they get back, so an edge names the node it waits on. The arithmetic is gone, and with it: - `NodeSpec` and the job-queue's own index-based `Dep`. - `insert_group`'s index resolution — it wraps `Scheduler::insert_job`. - `concat_subgraphs` — per-agent chains share one builder and each keeps its own root, so independence is structural rather than computed. - `reconcile_index` and `dep_index`. - `templates::validate` and its petgraph toposort. It rejected dangling deps and cycles; both are now unrepresentable, since a handle only exists for an already-declared node and every edge therefore points backwards. (petgraph stays in the tree for `agent_config::topology`.) `NodeOutput.append_subgraph` becomes `Vec<Job>`: an executor cannot reach the queue, so it hands back declarations and the scheduler inserts them under its own lock. That is what the in-DAG growth path always wanted — a transferable declaration, not a vector of specs. Resource declaration is unchanged in behaviour: the `templates::node` helper applies `NodeKind::resource_deps()` at the construction site, so every node still declares what its kind needs. Moving that declaration to the call sites is #2818's job; this leaves it one place to delete. Three tests went with the guard they covered — they hand-built malformed specs out of indices, which is the representation that made those shapes possible. Two more now read a DAG's shape off the queue rather than out of a spec vector, which is where it is observable. The remaining 45 job-queue tests are unchanged and still pass: lease serialization, roll-up, cancel-cascade, in-DAG growth and per-agent concurrency all behave as before.
This commit is contained in:
atlas
mara
parent
9be7731c5e
commit
e7c3cf5a3d
9 changed files with 528 additions and 696 deletions
|
|
@ -6,8 +6,8 @@
|
|||
//! state, which needs an async `lifecycle::is_running` read that a pure/sync
|
||||
//! template can't do. So these fns are async — they read each agent's state,
|
||||
//! assemble a per-agent subgraph out of the shared pure primitives
|
||||
//! (`templates::{node, after_ok, rebuild_nodes}`), and concatenate them into
|
||||
//! ONE DAG (independent per-agent roots, concurrent on their own leases).
|
||||
//! (`templates::{node, rebuild_nodes}`), all declaring into ONE job
|
||||
//! (independent per-agent roots, concurrent on their own leases).
|
||||
//!
|
||||
//! Dynamic shape rule: `stop`/`start` carry a head `SetWanted(w)` (durable
|
||||
//! intent write) — `restart` does NOT (it bounces the container but leaves
|
||||
|
|
@ -24,9 +24,9 @@
|
|||
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::model::{DagSpec, Dep, NodeKind, NodeSpec};
|
||||
use super::templates::{RebuildOpts, after_ok, child, node, rebuild_nodes};
|
||||
use super::{Source, templates};
|
||||
use super::model::{DagSpec, NodeKind};
|
||||
use super::templates::{RebuildOpts, node, rebuild_nodes};
|
||||
use super::{Job, Source, templates};
|
||||
use crate::coordinator::Coordinator;
|
||||
use crate::lifecycle;
|
||||
|
||||
|
|
@ -51,71 +51,75 @@ pub fn rebuild(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: St
|
|||
// The pure per-agent chain builders below take `running` (and `stale`)
|
||||
// explicitly so they stay pure + unit-testable without a live container;
|
||||
// the async `*_many` fns read the real state via `lifecycle::is_running`
|
||||
// then hand it in. Each chain uses LOCAL (0-based) deps; `concat_subgraphs`
|
||||
// rebases them into one DAG.
|
||||
// then hand it in. Each chain declares into the shared job it is handed, and
|
||||
// names the nodes it depends on — so there is nothing to rebase.
|
||||
|
||||
/// One agent's **stop** subgraph. `SetWanted(Off)` head + `Reconcile` tail
|
||||
/// always; the graceful `Signal → Drain` quiesce only when the agent is
|
||||
/// actually running (nothing to drain on a down container). The `Reconcile`
|
||||
/// stays even for a down agent so a race-up between the state read and exec
|
||||
/// is still stopped in-DAG.
|
||||
fn stop_chain(agent: &str, graceful: bool, running: bool) -> Vec<NodeSpec> {
|
||||
fn stop_chain(b: &Job, agent: &str, graceful: bool, running: bool) {
|
||||
// `SetWanted` is the group root and owns the agent lease; the mechanical
|
||||
// steps are its children (borrow the lease, run once it reaches `Finishing`,
|
||||
// dep-ordered among themselves).
|
||||
let a = || agent.to_owned();
|
||||
let mut n = vec![node(
|
||||
let wanted = node(
|
||||
b,
|
||||
NodeKind::SetWanted {
|
||||
agent: a(),
|
||||
up: false,
|
||||
},
|
||||
Vec::new(),
|
||||
)];
|
||||
);
|
||||
// Declaration order is dependency order: the quiesce steps come first so
|
||||
// the `Reconcile` that waits on them can name them.
|
||||
if graceful && running {
|
||||
n.push(child(0, NodeKind::Signal { agent: a() }, Vec::new()));
|
||||
n.push(child(0, NodeKind::Drain { agent: a() }, after_ok(1)));
|
||||
n.push(child(0, NodeKind::Reconcile { agent: a() }, after_ok(2)));
|
||||
let signal = node(b, NodeKind::Signal { agent: a() }).part_of(wanted);
|
||||
let drain = node(b, NodeKind::Drain { agent: a() })
|
||||
.part_of(wanted)
|
||||
.after_ok(signal);
|
||||
let _ = node(b, NodeKind::Reconcile { agent: a() })
|
||||
.part_of(wanted)
|
||||
.after_ok(drain);
|
||||
} else {
|
||||
n.push(child(0, NodeKind::Reconcile { agent: a() }, Vec::new()));
|
||||
let _ = node(b, NodeKind::Reconcile { agent: a() }).part_of(wanted);
|
||||
}
|
||||
n
|
||||
}
|
||||
|
||||
/// One agent's **start** subgraph. `SetWanted(Up)` head; a down + stale-rev
|
||||
/// agent gets the rebuild subgraph (its tail `Reconcile` starts it on
|
||||
/// current derivations), otherwise a plain `Reconcile` (which starts a down
|
||||
/// agent and noops an already-running one).
|
||||
fn start_chain(agent: &str, running: bool, stale: bool) -> Vec<NodeSpec> {
|
||||
let mut n = vec![node(
|
||||
fn start_chain(b: &Job, agent: &str, running: bool, stale: bool) {
|
||||
let wanted = node(
|
||||
b,
|
||||
NodeKind::SetWanted {
|
||||
agent: agent.to_owned(),
|
||||
up: true,
|
||||
},
|
||||
Vec::new(),
|
||||
)];
|
||||
);
|
||||
if !running && stale {
|
||||
// Rebuild subtree after the SetWanted head (base = 1, so the rebuild's
|
||||
// `MetaSync` root deps `after_ok(0)` = the head). `MetaSync`,
|
||||
// Rebuild subtree chained behind the `SetWanted` head. `MetaSync`,
|
||||
// `Prebuild` + `Reconcile` are their own group roots (top-level, per
|
||||
// `rebuild_nodes`).
|
||||
n.extend(rebuild_nodes(
|
||||
rebuild_nodes(
|
||||
b,
|
||||
agent,
|
||||
RebuildOpts {
|
||||
relock: true,
|
||||
graceful: false,
|
||||
},
|
||||
1,
|
||||
));
|
||||
Some(wanted),
|
||||
);
|
||||
} else {
|
||||
n.push(child(
|
||||
0,
|
||||
let _ = node(
|
||||
b,
|
||||
NodeKind::Reconcile {
|
||||
agent: agent.to_owned(),
|
||||
},
|
||||
Vec::new(),
|
||||
));
|
||||
)
|
||||
.part_of(wanted);
|
||||
}
|
||||
n
|
||||
}
|
||||
|
||||
/// One agent's **restart** subgraph. Restart NEVER rewrites `wanted`
|
||||
|
|
@ -127,82 +131,49 @@ fn start_chain(agent: &str, running: bool, stale: bool) -> Vec<NodeSpec> {
|
|||
/// before `Reconcile`; a down agent gets just `Reconcile`, which
|
||||
/// converges to intent — a stopped (`wanted = Off`) agent stays stopped,
|
||||
/// a crashed (`wanted = Up`) agent comes back up.
|
||||
fn restart_chain(agent: &str, graceful: bool, running: bool) -> Vec<NodeSpec> {
|
||||
fn restart_chain(b: &Job, agent: &str, graceful: bool, running: bool) {
|
||||
let a = || agent.to_owned();
|
||||
if !running {
|
||||
// Nothing to bounce — a lone Reconcile converges to intent.
|
||||
return vec![node(NodeKind::Reconcile { agent: a() }, Vec::new())];
|
||||
let _ = node(b, NodeKind::Reconcile { agent: a() });
|
||||
return;
|
||||
}
|
||||
// Running: mechanical stop then Reconcile. The first stop node is the group
|
||||
// ROOT (no SetWanted head) and owns the agent lease; the rest are its
|
||||
// children (borrow the lease, dep-ordered), so the bounce holds one
|
||||
// continuous lease and `Reconcile` cancel-cascades if a stop step fails.
|
||||
let mut n = vec![if graceful {
|
||||
node(NodeKind::Signal { agent: a() }, Vec::new())
|
||||
} else {
|
||||
node(NodeKind::StopForUpdate { agent: a() }, Vec::new())
|
||||
}];
|
||||
//
|
||||
// `Reconcile` gates on the last mechanical step. For a non-graceful bounce
|
||||
// that step *is* the root, and the parent gate already orders it — a child
|
||||
// must NOT dep on its own parent (dep-scope), so it takes no sibling edge.
|
||||
if graceful {
|
||||
n.push(child(0, NodeKind::Drain { agent: a() }, Vec::new()));
|
||||
n.push(child(
|
||||
0,
|
||||
NodeKind::StopForUpdate { agent: a() },
|
||||
after_ok(1),
|
||||
));
|
||||
}
|
||||
// `Reconcile` gates on the last mechanical step. When the only step is the
|
||||
// root itself (non-graceful, `StopForUpdate` == index 0), the parent gate
|
||||
// already orders `Reconcile` after it — a child must NOT dep on its own
|
||||
// parent (dep-scope). So the sibling dep is added only for a graceful
|
||||
// bounce, where the last step is a sibling child.
|
||||
let deps = if n.len() > 1 {
|
||||
after_ok(u64::try_from(n.len() - 1).unwrap_or(0))
|
||||
let signal = node(b, NodeKind::Signal { agent: a() });
|
||||
let drain = node(b, NodeKind::Drain { agent: a() }).part_of(signal);
|
||||
let stop = node(b, NodeKind::StopForUpdate { agent: a() })
|
||||
.part_of(signal)
|
||||
.after_ok(drain);
|
||||
let _ = node(b, NodeKind::Reconcile { agent: a() })
|
||||
.part_of(signal)
|
||||
.after_ok(stop);
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
n.push(child(0, NodeKind::Reconcile { agent: a() }, deps));
|
||||
n
|
||||
}
|
||||
|
||||
/// Concatenate per-agent subgraphs (each with LOCAL 0-based deps) into one
|
||||
/// node list, rebasing each subgraph's internal deps by its offset. A
|
||||
/// subgraph root (empty deps — the `SetWanted` head) stays a root, so the
|
||||
/// per-agent subgraphs are independent and run concurrently, each on its
|
||||
/// own lease.
|
||||
fn concat_subgraphs(chains: Vec<Vec<NodeSpec>>) -> Vec<NodeSpec> {
|
||||
let mut out: Vec<NodeSpec> = Vec::new();
|
||||
for chain in chains {
|
||||
let base = u64::try_from(out.len()).unwrap_or(u64::MAX);
|
||||
for spec in chain {
|
||||
let deps = spec
|
||||
.deps
|
||||
.into_iter()
|
||||
.map(|d| Dep {
|
||||
on: base + d.on,
|
||||
when: d.when,
|
||||
})
|
||||
.collect();
|
||||
out.push(NodeSpec {
|
||||
kind: spec.kind,
|
||||
deps,
|
||||
// Rebase the structural parent by the same offset (a subgraph
|
||||
// root keeps `parent = None`, so the per-agent groups stay
|
||||
// independent + concurrent).
|
||||
parent: spec.parent.map(|p| base + p),
|
||||
});
|
||||
}
|
||||
let stop = node(b, NodeKind::StopForUpdate { agent: a() });
|
||||
let _ = node(b, NodeKind::Reconcile { agent: a() }).part_of(stop);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Wrap assembled power-op `nodes` in a `DagSpec`. No tail node: a power op's
|
||||
/// Wrap the per-agent subgraphs in a `DagSpec`. No tail node: a power op's
|
||||
/// effect is its nodes (`SetWanted` + `Reconcile`), with nothing left to do once
|
||||
/// they settle.
|
||||
fn power_dag(source: Source, reason: String, nodes: Vec<NodeSpec>) -> DagSpec {
|
||||
///
|
||||
/// There is no concatenation step: every chain declares into the same builder
|
||||
/// and each keeps its own root, so the per-agent subgraphs are independent and
|
||||
/// run concurrently, each on its own lease. Rebasing one subgraph's indices
|
||||
/// onto another's used to be a function.
|
||||
fn power_dag(source: Source, reason: String, job: Job) -> DagSpec {
|
||||
DagSpec {
|
||||
source,
|
||||
reason,
|
||||
nodes,
|
||||
job,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -218,11 +189,11 @@ pub(crate) fn stop_spec(
|
|||
source: Source,
|
||||
reason: String,
|
||||
) -> DagSpec {
|
||||
let chains = targets
|
||||
.iter()
|
||||
.map(|(agent, running)| stop_chain(agent, graceful, *running))
|
||||
.collect();
|
||||
power_dag(source, reason, concat_subgraphs(chains))
|
||||
let job = Job::new();
|
||||
for (agent, running) in targets {
|
||||
stop_chain(&job, agent, graceful, *running);
|
||||
}
|
||||
power_dag(source, reason, job)
|
||||
}
|
||||
|
||||
/// Assemble the start DAG from explicit `(agent, running, stale)` targets.
|
||||
|
|
@ -236,11 +207,11 @@ pub(crate) fn start_spec(
|
|||
source: Source,
|
||||
reason: String,
|
||||
) -> DagSpec {
|
||||
let chains = targets
|
||||
.iter()
|
||||
.map(|(agent, running, stale)| start_chain(agent, *running, *stale))
|
||||
.collect();
|
||||
power_dag(source, reason, concat_subgraphs(chains))
|
||||
let job = Job::new();
|
||||
for (agent, running, stale) in targets {
|
||||
start_chain(&job, agent, *running, *stale);
|
||||
}
|
||||
power_dag(source, reason, job)
|
||||
}
|
||||
|
||||
/// Assemble the restart DAG from explicit `(agent, running)` targets.
|
||||
|
|
@ -250,11 +221,11 @@ pub(crate) fn restart_spec(
|
|||
source: Source,
|
||||
reason: String,
|
||||
) -> DagSpec {
|
||||
let chains = targets
|
||||
.iter()
|
||||
.map(|(agent, running)| restart_chain(agent, graceful, *running))
|
||||
.collect();
|
||||
power_dag(source, reason, concat_subgraphs(chains))
|
||||
let job = Job::new();
|
||||
for (agent, running) in targets {
|
||||
restart_chain(&job, agent, graceful, *running);
|
||||
}
|
||||
power_dag(source, reason, job)
|
||||
}
|
||||
|
||||
/// Restart a single agent. Thin wrapper over [`restart_many`].
|
||||
|
|
|
|||
Loading…
Reference in a new issue