//! Build a job's sub-DAG by **naming** nodes instead of counting them. //! //! [`JobBuilder::node`] hands back a handle, obtainable no other way; edges are //! handle → handle. So a caller names the node it wants to depend on rather //! than computing where that node landed, and there is no positional index to //! get wrong. //! //! **An insertion API, not a spec factory.** A builder is only ever handed to a //! closure by the single insertion entry point //! ([`crate::scheduler::Scheduler::insert_job`]), which inserts the declared //! nodes and returns the ids the job asked for. It cannot be constructed, held //! or inserted from outside this crate, and there is no intermediate //! node-description type to keep in sync with [`crate::Graph::insert`]'s signature — //! so a job has no representation that can be passed around instead of being //! inserted. //! //! **Payload-agnostic.** Generic over the same `N` and `R` as [`crate::Graph`]: the //! builder knows nothing about what a node *does*, only how nodes relate. //! //! # Declaration order //! //! Nodes are inserted in declaration order, so a node must be declared *after* //! everything it references. That is not a limitation the builder invents: node //! ids are minted by the graph at insert time, so a forward edge has nothing to //! point at. Declaring one is a [`BuildError::ForwardEdge`] rather than a //! silent reorder — a builder that sorted for you would quietly accept a shape //! the graph itself cannot express. use std::cell::RefCell; use std::collections::HashMap; use crate::{Dep, DepWhen, NodeId, TerminalState}; /// An opaque identity for a node **within the job being built**. /// /// Obtainable only from [`JobBuilder::node`], so it can only ever name a node /// that exists. Deliberately not a [`NodeId`]: those are minted by the graph at /// insert time and are meaningful system-wide, while this is a builder-local /// label that stops existing once the job is inserted. /// /// It is a v4 uuid because it has to survive leaving its builder. A [`NodeRef`] /// converts into a bare `NodeGuid`, dropping the borrow that tied it to the /// builder — so a handle *can* be carried into a different job (captured by an /// inner closure, say). Under a per-builder counter the two jobs' `0`s would be /// equal and that handle would silently address an unrelated node; drawn at /// random per node it simply isn't found, and the insert fails by name. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct NodeGuid(uuid::Uuid); /// Why a job could not be inserted. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum BuildError { /// A node depends on one declared later. See the module docs on /// declaration order. #[error( "node {node:?} depends on {dep:?}, which is declared later — a dependency \ must be declared before the node that names it" )] ForwardEdge { /// The node carrying the edge. node: NodeGuid, /// The not-yet-declared target. dep: NodeGuid, }, /// A node is grouped under a parent declared later. Same cause as /// [`BuildError::ForwardEdge`]. #[error( "node {node:?} is grouped under {parent:?}, which is declared later — a \ parent must be declared before its children" )] ForwardParent { /// The child node. node: NodeGuid, /// The not-yet-declared parent. parent: NodeGuid, }, /// A handle named in the closure's return value belongs to a different /// job. Same cause as a foreign handle on an edge: a [`NodeGuid`] outlives /// the borrow that tied it to its builder, so one can be carried here. #[error("handle {node:?} names no node in this job")] UnknownNode { /// The handle that resolved to nothing. node: NodeGuid, }, /// An edge leaves the depender's own parent group — including an edge onto /// its **own parent**, which would deadlock: a parent parks in /// `Finishing` until its children settle, so a child waiting on it could /// never run. The graph enforces the same rule /// (`GraphError::DepOutsideParent`); this catches it before any insert. #[error( "node {node:?} depends on {dep:?}, which is outside its parent group — \ an edge must stay within the depender's own group" )] DepOutsideGroup { /// The node carrying the edge. node: NodeGuid, /// The out-of-group target. dep: NodeGuid, }, /// An edge that no outcome can satisfy (an empty [`DepWhen`]), so the node /// could never become runnable. Refused at declaration rather than /// inserted — the graph only catches this when validating a *deserialized* /// graph, so without this check it would insert and silently never run. #[error("node {node:?} has an edge on {dep:?} that no outcome can satisfy")] UnsatisfiableDep { /// The node carrying the edge. node: NodeGuid, /// The target of the unsatisfiable edge. dep: NodeGuid, }, } /// Reject a job whose own declarations don't hold up — **before anything is /// inserted**, so no failure can leave a partial job behind. /// /// This covers *every* rejection [`crate::Graph::insert`] can raise for a /// builder-produced node, which is what makes the insert loop below infallible /// in practice: /// /// | graph rejection | why it cannot reach the graph | /// |---|---| /// | `UnknownParent` / `UnknownDep` | a handle only exists if this job declared it, and the ids are minted here | /// | `DepOutsideParent` | the grouping rule is re-checked here against the job's own parent chains | /// /// It also rejects an **empty [`DepWhen`]**, which the graph currently only /// catches on the deserialize path (`Graph::validate`) — so a node that nothing /// could ever satisfy is refused at declaration instead of being inserted and /// silently never running. /// /// `root_parent` matters for the grouping rule: a node that declared no parent /// hangs there, so what counts as "inside the group" depends on whether the job /// is being attached under a container or at the top level. /// /// # Errors /// [`BuildError::ForwardEdge`] / [`BuildError::ForwardParent`] for a reference /// to a later node, [`BuildError::UnknownNode`] for a requested handle this job /// never declared, [`BuildError::DepOutsideGroup`] for an edge leaving the /// depender's parent group, [`BuildError::UnsatisfiableDep`] for an empty edge. fn check_job_shape( pending: &[Pending], wanted: &[NodeGuid], root_parent: Option, ) -> Result<(), BuildError> { let mut declared: std::collections::HashSet = std::collections::HashSet::new(); // Declared parent per node, for walking a job-internal parent chain. let mut parent_of: HashMap> = HashMap::new(); for node in pending { for (dep, when) in &node.deps { if !declared.contains(dep) { return Err(BuildError::ForwardEdge { node: node.guid, dep: *dep, }); } if when.is_empty() { return Err(BuildError::UnsatisfiableDep { node: node.guid, dep: *dep, }); } if !dep_in_group(&parent_of, node.parent, *dep, root_parent) { return Err(BuildError::DepOutsideGroup { node: node.guid, dep: *dep, }); } } if let Some(parent) = node.parent && !declared.contains(&parent) { return Err(BuildError::ForwardParent { node: node.guid, parent, }); } declared.insert(node.guid); parent_of.insert(node.guid, node.parent); } for guid in wanted { if !declared.contains(guid) { return Err(BuildError::UnknownNode { node: *guid }); } } Ok(()) } /// The graph's grouping rule (`Graph::dep_target_in_group`) decided against the /// job's own declarations, before any node exists. /// /// A depender whose declared parent is `Some(q)` may only name a **proper /// descendant of `q`** — never `q` itself, which would deadlock (a parent parks /// in `Finishing` until its children settle). A depender that declared no parent /// hangs under `root_parent`: /// - `root_parent = Some(_)` — every node in this job is somewhere under it, so /// any job-internal target is in-group. /// - `root_parent = None` — the depender is top-level, so the target must be /// top-level too, i.e. it must also have declared no parent. fn dep_in_group( parent_of: &HashMap>, node_parent: Option, dep: NodeGuid, root_parent: Option, ) -> bool { match node_parent { // Walk the target's declared chain looking for the depender's parent. // Starts at the target's *parent*, so the target is never its own // ancestor — matching `Graph::is_descendant`. Some(group) => { let mut cur = parent_of.get(&dep).copied().flatten(); while let Some(p) = cur { if p == group { return true; } cur = parent_of.get(&p).copied().flatten(); } false } None if root_parent.is_some() => true, None => parent_of.get(&dep).copied().flatten().is_none(), } } /// One node as the builder holds it: edges and parent still name *handles*, so /// nothing here depends on ids the graph has not minted yet. #[derive(Debug)] struct Pending { guid: NodeGuid, payload: N, deps: Vec<(NodeGuid, DepWhen)>, resources: Vec<(R, u32)>, parent: Option, } /// Accumulates a job's nodes, handing back a handle for each. /// /// Sub-builders take `&JobBuilder` and **return their handles**, which is what /// removes the base-offset parameter a positional API needs: a caller names the /// node it wants rather than computing where that node landed. #[derive(Debug)] pub struct JobBuilder { nodes: RefCell>>, } // Hand-written rather than derived: `#[derive(Default)]` would demand // `N: Default, R: Default`, which has nothing to do with an empty builder. impl Default for JobBuilder { fn default() -> Self { Self { nodes: RefCell::new(Vec::new()), } } } impl JobBuilder { /// A fresh, empty builder. /// /// **Crate-private, and that is the API.** A builder is only ever handed to /// a closure by the single insertion entry point /// ([`crate::scheduler::Scheduler::insert_job`]), which inserts the declared /// nodes and returns the ids. Nothing job-shaped is constructible or /// carryable outside this crate — otherwise it is a spec factory again, /// just with a builder's name on it. pub(crate) fn new() -> Self { Self::default() } /// Whether nothing has been declared yet — for a caller deciding whether an /// insertion is worth taking a lock for. #[must_use] pub fn is_empty(&self) -> bool { self.nodes.borrow().is_empty() } /// Add a node carrying `payload`, with no edges, resources, or parent yet. /// /// The returned handle is where those are declared; it is [`Copy`], so it /// can be named as a dependency as many times as needed. pub fn node(&self, payload: N) -> NodeRef<'_, N, R> { let guid = NodeGuid(uuid::Uuid::new_v4()); self.nodes.borrow_mut().push(Pending { guid, payload, deps: Vec::new(), resources: Vec::new(), parent: None, }); NodeRef { builder: self, guid, } } /// Insert every declared node into `graph`, in declaration order, and /// return the id each handle's node was minted as. /// /// `root_parent` is the group's attachment point: a node that declared no /// [`NodeRef::part_of`] hangs there rather than at the top level. That is /// what makes a job a *self-contained sub-DAG* — the whole thing goes under /// one container node, or under the runtime node that emitted it, and the /// job's own declarations stay relative. /// /// The returned map is how a caller finds out what its handles became: hold /// the handles it cares about, then look them up here. Handles that were /// never looked up cost nothing. /// /// # Errors /// /// A [`BuildError`] if the job's own declarations don't hold up — see /// [`check_job_shape`], which decides every one of them **before** the /// first insert. The sink itself is infallible: by the time it runs, the /// job is known-good, so no node can be rejected half-way through. pub(crate) fn insert_with( self, root_parent: Option, wanted: &[NodeGuid], mut insert: impl FnMut(N, Vec>, Option) -> NodeId, ) -> Result, BuildError> { let pending = self.nodes.into_inner(); check_job_shape(&pending, wanted, root_parent)?; let mut ids: HashMap = HashMap::new(); for node in pending { let parent = match node.parent { None => root_parent, Some(p) => Some(ids[&p]), }; let mut deps: Vec> = Vec::with_capacity(node.deps.len()); for (on, when) in node.deps { deps.push(Dep::Node { id: ids[&on], when }); } deps.extend( node.resources .into_iter() .map(|(name, count)| Dep::Resource { name, count }), ); // Infallible by construction: `check_job_shape` above decided every // rejection the graph could raise, so there is nothing left here to // abandon a half-inserted job on. let id = insert(node.payload, deps, parent); ids.insert(node.guid, id); } Ok(wanted.iter().map(|g| ids[g]).collect()) } /// Apply `f` to the node named by `guid`. fn with_node(&self, guid: NodeGuid, f: impl FnOnce(&mut Pending)) { let mut nodes = self.nodes.borrow_mut(); if let Some(n) = nodes.iter_mut().find(|n| n.guid == guid) { f(n); } } } /// A handle to one declared node, and the surface its edges are declared on. /// /// [`Copy`] because it is only a builder borrow plus a handle: naming a node as /// a dependency must not consume the ability to name it again. #[derive(Debug)] pub struct NodeRef<'a, N, R> { builder: &'a JobBuilder, guid: NodeGuid, } // Hand-written for the same reason as `Default` above: deriving would demand // `N: Copy, R: Copy`, but a handle copies a borrow and an integer. impl Clone for NodeRef<'_, N, R> { fn clone(&self) -> Self { *self } } impl Copy for NodeRef<'_, N, R> {} impl From> for NodeGuid { fn from(n: NodeRef<'_, N, R>) -> Self { n.guid } } impl From<&NodeRef<'_, N, R>> for NodeGuid { fn from(n: &NodeRef<'_, N, R>) -> Self { n.guid } } impl NodeRef<'_, N, R> { /// This node's handle, for callers that want to hold the identity without /// the builder borrow (e.g. to look the id up after /// [`JobBuilder::insert_into`]). #[must_use] pub fn guid(self) -> NodeGuid { self.guid } /// Run only after `on` succeeds — the common chain link. /// /// Chained against several nodes it means *all* of them succeeded, and this /// node is ruled out ([`TerminalState::Skipped`]) the moment one doesn't — /// which is the signal [`NodeRef::on_elimination_of`] waits for. #[must_use] pub fn after_ok(self, on: impl Into) -> Self { self.edge(on.into(), DepWhen::AFTER_OK) } /// Run once `on` is terminal, however it ended. The cleanup edge: a node /// that must run whether or not the work it follows succeeded. #[must_use] pub fn after_any(self, on: impl Into) -> Self { self.edge(on.into(), DepWhen::AFTER_ANY) } /// Run only on the listed outcomes of `on` — the general form of /// [`NodeRef::after_ok`] / [`NodeRef::after_any`], for the /// one-node-per-outcome shape a job with a row to resolve uses. #[must_use] pub fn on_outcome(self, on: impl Into, outcomes: &[TerminalState]) -> Self { self.edge(on.into(), DepWhen::of(outcomes)) } /// Run only if `on` was **ruled out** — i.e. its own `after_ok` edges did /// not hold. /// /// This is how *"any one of several nodes failed"* gets expressed at all. /// Dependency edges are conjunctive, so that condition cannot be written /// directly; the composition that expresses it is a pair — point the /// success branch at every root with [`NodeRef::after_ok`], then hang the /// failure branch off *that* node's elimination. Exactly one of the pair /// ever runs. /// /// It accepts [`TerminalState::Skipped`] and **not** /// [`TerminalState::Cancelled`] on purpose: if the whole job was dropped /// before it started, the success branch is `Cancelled` directly and this /// branch is ruled out too — a job nobody ran reports nothing. #[must_use] pub fn on_elimination_of(self, on: impl Into) -> Self { self.edge(on.into(), DepWhen::of(&[TerminalState::Skipped])) } /// Hold one unit of the named resource while this node and its subtree run. /// /// Declared on every node that needs it, even when an ancestor already /// holds the same resource — the ancestor's grant is re-used (a re-entrant /// borrow), so declaring it costs nothing and makes the node's needs /// readable where the node is written. #[must_use] pub fn needs(self, name: R) -> Self { self.needs_units(name, 1) } /// [`NodeRef::needs`] with an explicit unit count, for a resource whose /// capacity is a budget rather than a mutex. #[must_use] pub fn needs_units(self, name: R, count: u32) -> Self { self.builder .with_node(self.guid, |n| n.resources.push((name, count))); self } /// Group this node under `parent`. /// /// The child runs once its parent reaches [`crate::State::Finishing`] (the /// parent gate), so it must not also depend on that parent — dep-scope /// validation rejects that, and it would deadlock. A group root owns /// whatever resource it declares for its whole subtree. #[must_use] pub fn part_of(self, parent: impl Into) -> Self { let parent = parent.into(); self.builder .with_node(self.guid, |n| n.parent = Some(parent)); self } /// Record one dependency edge and hand the handle back for chaining. fn edge(self, on: NodeGuid, when: DepWhen) -> Self { self.builder .with_node(self.guid, |n| n.deps.push((on, when))); self } } #[cfg(test)] mod tests { use super::BuildError; use crate::resources::ResourceTable; use crate::scheduler::Scheduler; use crate::{Dep, DepWhen, Graph, NodeId}; /// A scheduler over a graph whose payload is a name and whose resources are /// strings — the only way in, since insertion is a scheduler operation. fn sched() -> Scheduler<&'static str, &'static str> { Scheduler::new(Graph::new(), ResourceTable::new()) } fn deps_of(g: &Scheduler<&'static str, &'static str>, id: NodeId) -> Vec> { g.graph().node(id).expect("node present").deps.clone() } /// The point of the handle layer: an edge declared against a *handle* comes /// out addressing the id that node was actually minted as. #[test] fn edges_resolve_to_minted_ids() { let mut g = sched(); let ids = g .insert_job(None, |b| { let first = b.node("a"); let second = b.node("b").after_ok(first); vec![first.guid(), second.guid()] }) .expect("insert"); let [first, second] = ids[..] else { panic!("two ids back, in the order asked for") }; assert_eq!( deps_of(&g, second), vec![Dep::Node { id: first, when: DepWhen::AFTER_OK }] ); assert!(deps_of(&g, first).is_empty()); } #[test] fn parent_resolves_to_a_minted_id() { let mut g = sched(); let ids = g .insert_job(None, |b| { let root = b.node("a"); let child = b.node("b").part_of(root); vec![root.guid(), child.guid()] }) .expect("insert"); let [root, child] = ids[..] else { panic!("two ids back") }; assert_eq!(g.graph().node(root).expect("root").parent, None); assert_eq!(g.graph().node(child).expect("child").parent, Some(root)); } /// A handle is `Copy`, so naming the same node as a dependency twice must /// not consume it — the fan-out every composite job needs. #[test] fn one_handle_can_be_depended_on_twice() { let mut g = sched(); let ids = g .insert_job(None, |b| { let shared = b.node("a"); let ok = b.node("b").after_ok(shared); let any = b.node("c").after_any(shared); vec![shared.guid(), ok.guid(), any.guid()] }) .expect("insert"); let [shared, ok, any] = ids[..] else { panic!("three ids back") }; assert_eq!( deps_of(&g, ok), vec![Dep::Node { id: shared, when: DepWhen::AFTER_OK }] ); assert_eq!( deps_of(&g, any), vec![Dep::Node { id: shared, when: DepWhen::AFTER_ANY }] ); } /// Resource deps ride along with the node deps, in one insert. #[test] fn resources_become_resource_deps() { let mut g = sched(); let ids = g .insert_job(None, |b| { vec![ b.node("a") .needs("agent/atlas") .needs_units("build", 2) .guid(), ] }) .expect("insert"); let [only] = ids[..] else { panic!("one id back") }; assert_eq!( deps_of(&g, only), vec![ Dep::Resource { name: "agent/atlas", count: 1 }, Dep::Resource { name: "build", count: 2 }, ] ); } /// A handle stays live across later `node()` calls, so it is possible to /// aim an edge *backwards* in declaration order. The graph mints ids as it /// inserts, so there is nothing for that edge to point at — say so instead /// of quietly reordering. #[test] fn a_forward_edge_is_rejected_by_name() { let mut g = sched(); let mut named = None; let err = g .insert_job(None, |b| { let first = b.node("a"); let second = b.node("b"); let _ = first.after_any(second); named = Some((first.guid(), second.guid())); // The insert fails, so nothing comes back to ask for — the // handles under test travel out through `named` instead. Vec::new() }) .expect_err("forward edge"); let (first, second) = named.expect("declared"); assert_eq!( err, BuildError::ForwardEdge { node: first, dep: second } ); } #[test] fn a_forward_parent_is_rejected_by_name() { let mut g = sched(); let mut named = None; let err = g .insert_job(None, |b| { let child = b.node("a"); let parent = b.node("b"); let _ = child.part_of(parent); named = Some((child.guid(), parent.guid())); Vec::new() }) .expect_err("forward parent"); let (child, parent) = named.expect("declared"); assert_eq!( err, BuildError::ForwardParent { node: child, parent } ); } /// A handle belongs to the job that issued it. Carrying one into a second /// job — an inner closure capturing an outer handle, say — must not /// silently address whichever node happens to sit at the same position. /// /// This is what a random [`NodeGuid`] buys: with a plain per-builder /// counter both jobs' first handles would be equal, and the edge below /// would resolve, wrongly, to the second job's own node. #[test] fn a_handle_from_another_job_is_not_silently_resolved() { let mut g = sched(); let mut foreign = None; g.insert_job(None, |b| { foreign = Some(b.node("first job").guid()); Vec::new() }) .expect("first job inserts"); let foreign = foreign.expect("declared"); let err = g .insert_job(None, |b| { let _ = b.node("second job").after_ok(foreign); Vec::new() }) .expect_err("foreign handle"); assert!( matches!(err, BuildError::ForwardEdge { dep, .. } if dep == foreign), "{err:?}" ); } /// The grouping rule is enforced **before the first insert**, so a job that /// breaks it leaves the graph untouched rather than half-built. /// /// This used to assert `BuildError::Graph(_)` — i.e. that the graph's own /// rejection surfaced through the builder. It does not reach the graph any /// more: the same rule is now decided from the job's own declarations, and /// the stronger property (nothing was inserted) is what this pins. #[test] fn an_out_of_group_dep_is_refused_before_anything_is_inserted() { let mut g = sched(); let err = g .insert_job(None, |b| { let root = b.node("root"); // A child may not depend on its own parent: the parent gate // already orders them, and the edge would deadlock. let _ = b.node("child").part_of(root).after_ok(root); Vec::new() }) .expect_err("out-of-group dep"); assert!(matches!(err, BuildError::DepOutsideGroup { .. }), "{err:?}"); assert_eq!(g.graph().nodes().count(), 0, "nothing may have landed"); } /// An edge no outcome can satisfy is refused at declaration. The graph only /// catches this when validating a deserialized graph, so without the /// pre-pass such a node would insert cleanly and then never become /// runnable — a silent hang rather than an error. #[test] fn an_edge_no_outcome_can_satisfy_is_refused() { let mut g = sched(); let err = g .insert_job(None, |b| { let first = b.node("a"); let _ = b.node("b").on_outcome(first, &[]); Vec::new() }) .expect_err("unsatisfiable edge"); assert!( matches!(err, BuildError::UnsatisfiableDep { .. }), "{err:?}" ); assert_eq!(g.graph().nodes().count(), 0, "nothing may have landed"); } /// A job's own roots hang under the group's attachment point, while a node /// that named a parent inside the job keeps it. This is what lets a /// template be written without knowing the container it will live under. #[test] fn root_parent_adopts_only_the_jobs_own_roots() { let mut g = sched(); let container = g.append("container", Vec::new(), None).expect("container"); let ids = g .insert_job(Some(container), |b| { let root = b.node("root"); let child = b.node("child").part_of(root); vec![root.guid(), child.guid()] }) .expect("insert"); let [root, child] = ids[..] else { panic!("two ids back") }; assert_eq!(g.graph().node(root).expect("root").parent, Some(container)); assert_eq!(g.graph().node(child).expect("child").parent, Some(root)); } /// A handle that names no node in *this* job is refused rather than /// silently dropped from the returned ids — the return is positional, so a /// short vector would misalign every id after it. #[test] fn asking_for_a_foreign_handle_is_an_error() { let mut g = sched(); let mut foreign = None; g.insert_job(None, |b| { foreign = Some(b.node("first job").guid()); Vec::new() }) .expect("first job inserts"); let foreign = foreign.expect("declared"); let err = g .insert_job(None, |b| { let _ = b.node("second job"); vec![foreign] }) .expect_err("foreign handle asked for"); assert_eq!(err, BuildError::UnknownNode { node: foreign }); } #[test] fn an_empty_builder_inserts_nothing() { let mut g = sched(); let ids = g.insert_job(None, |_| Vec::new()).expect("insert"); assert!(ids.is_empty()); assert_eq!(g.graph().nodes().count(), 0); } }