//! `hive-jobq` — a persistent job-DAG scheduler, extracted from hive-c0re's //! in-tree `job_queue` as a domain-agnostic library. //! //! # Model (v2) //! //! One **persistent graph** for the whole system, not a DAG per job. Enqueuing //! inserts a self-contained sub-DAG of nodes and returns their ids; the //! scheduler runs a continuous loop, starting every node whose [`Dep`]s are //! satisfied: //! //! - **Resource** deps are named counting semaphores over a caller-chosen //! type `R`: `build-slot` (cap N), `agent/` (cap 1), or any name //! (cap 1, created on use). A node acquires *all* its resource deps //! atomically at start (all-or-nothing) — no hold-and-wait, no deadlock. //! - **Node** deps wait on another node per [`DepWhen`]: `AfterOk` needs //! success (a failed dep cancels the dependent), `AfterAny` only terminal. //! //! A node carries two independent axes: its [`Dep`]s (ordering + resource //! needs) and its [`Node::parent`] (structural grouping) — the parent chain, //! not the [`Dep::Node`] edges, is what the [`scheduler`] consults for resource //! re-entrancy. A [`NodeId`] is opaque, stable, and monotonic (persisted). The //! payload `N` is generic so the library stays container-agnostic. //! //! A resource unit is held for the acquiring node + its whole [`Node::parent`] //! subtree; a node needing a resource an ancestor holds re-uses that grant (a //! re-entrant borrow, one branch at a time). Single-threaded — the scheduler //! owns the resource table and mutates it directly. See [`scheduler`]. pub mod resources; pub mod scheduler; use chrono::{DateTime, Utc}; /// Opaque, stable, monotonic node identifier. /// /// Assigned by the [`Graph`] on insert and persisted, so it is stable across /// restarts. /// /// The inner field is crate-private: an id can only originate from the graph's /// monotonic counter (or deserialization of a persisted graph), never be /// fabricated by a caller — that is what makes it opaque. #[derive( Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize, serde::Deserialize, )] pub struct NodeId(pub(crate) u64); impl NodeId { /// The underlying monotonic value, for carrying this id across a boundary /// that cannot hold the opaque `NodeId` type — e.g. serializing it onto a /// wire protocol. The inverse (fabricating a `NodeId` from a raw value) /// stays impossible by construction: an id only ever originates from the /// graph's counter, which is what makes it opaque. #[must_use] pub fn get(self) -> u64 { self.0 } } /// When a [`Dep::Node`] edge is satisfied — the strong/weak distinction the /// current queue carries as `DepWhen`, load-bearing for failure safety. #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum DepWhen { /// The dependency must reach [`State::Done`]. This is the default chain /// edge: if the dependency *fails*, the dependent must not run and is /// cancelled ([`State::Cancelled`]) down the chain — e.g. a failed /// `Prebuild` must not let `StopForUpdate` stop a healthy container. AfterOk, /// The dependency need only be terminal — success or failure both satisfy /// it. For steps that must converge regardless, e.g. `Reconcile` running /// even when the preceding `Swap` failed. AfterAny, } impl DepWhen { /// Whether a dependency in `dep_state` satisfies this edge. #[must_use] pub fn satisfied_by(self, dep_state: State) -> bool { match self { DepWhen::AfterOk => dep_state == State::Done, DepWhen::AfterAny => dep_state.is_terminal(), } } } /// One dependency of a node. A node becomes runnable once every [`Dep::Node`] /// edge it names is satisfied (per its [`DepWhen`]) *and* every [`Dep::Resource`] /// it names can be acquired (all of them, atomically). #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum Dep { /// Depend on another node. Whether a *failed* dependency satisfies the edge /// is decided by `when`: `AfterOk` requires success (and cancels this node /// if the dep fails), `AfterAny` only requires the dep to be terminal. Node { /// The node depended on. id: NodeId, /// Strong (`AfterOk`) vs weak (`AfterAny`). when: DepWhen, }, /// Need `count` units of a named resource to run. Declared on every node /// that needs it, even when a [`Node::parent`]-ancestor already holds it. /// Acquired atomically with the node's other resource deps at start; the /// acquired unit is held for the acquirer's whole subtree (released only /// once the acquirer and all its sub-nodes are terminal). A node whose /// parent-ancestor already holds this resource re-uses that grant (a /// re-entrant borrow) instead of taking a fresh unit. Resource { /// The resource to acquire. name: R, /// How many units to hold (usually 1). count: u32, }, } /// A node's lifecycle state. #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum State { /// Waiting on dependencies (node or resource). Pending, /// Dependencies satisfied, resources held, currently executing its own logic. Running, /// Own logic finished successfully, but the node is *not yet terminal*: it /// waits here until all its sub-nodes ([`Node::parent`] children) are /// terminal, then rolls up to [`State::Done`] (every child `Done`) or /// [`State::Failed`] (any child `Failed`/`Cancelled`). A node with no /// children never rests here — it goes straight to a terminal state. Finishing, /// Completed successfully — own logic done *and* every sub-node `Done`. Done, /// Completed unsuccessfully — own logic failed, or a sub-node did. Failed, /// Never ran: an `AfterOk` dependency failed, so this node (and the rest of /// its strong-dependent chain) is cancelled rather than run. Cancelled, } impl State { /// A node is *terminal* once it has finished — successfully, unsuccessfully, /// or cancelled — which is when its resources are released and dependents /// are re-evaluated. #[must_use] pub fn is_terminal(self) -> bool { matches!(self, State::Done | State::Failed | State::Cancelled) } } /// Wall-clock UTC now — the source for node lifecycle timestamps /// ([`Node::started_at`] / [`Node::finished_at`]). The graph stamps its own /// timestamps rather than threading a clock through every call, so a node's /// timing is self-contained. Derived from `SystemTime` (the workspace `chrono` /// carries no `clock` feature, matching `hive_sh4re::wire_time`), truncated to /// whole seconds; a pre-epoch or out-of-range clock clamps to the epoch. fn now_utc() -> DateTime { let secs = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map_or(0, |d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX)); DateTime::::from_timestamp(secs, 0).unwrap_or_default() } /// A node in the graph, carrying a caller-defined payload `N`. /// /// The library schedules over `Node`s and resources without interpreting the /// payload; the caller supplies `N` (its own node kind) and a runner to execute /// a claimed node. A node carries two independent axes: its [`Dep`]s (ordering + /// resource needs) and its [`Node::parent`] (structural grouping), both set by /// the caller/submit layer. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct Node { /// Stable identity, assigned on insert. pub id: NodeId, /// Structural grouping: the node this one is a sub-node of, or `None` for a /// group root. Independent of [`Node::deps`] — grouping is *not* ordering. /// The [`scheduler`] uses the parent chain to decide resource re-entrancy: a /// node needing a resource a parent-ancestor holds re-uses that grant rather /// than acquiring a fresh unit, and a held unit stays reserved for the /// acquirer's whole subtree. A node's sub-nodes run *after* its own logic. pub parent: Option, /// Caller-defined payload (the node's kind / work description). pub payload: N, /// What must hold before this node runs (other nodes + resources). pub deps: Vec>, /// Lifecycle state. pub state: State, /// UTC instant the node entered [`State::Running`] (`None` until it starts; /// a cancelled node never ran, so it stays `None`). Stamped by the graph. pub started_at: Option>, /// UTC instant the node reached a terminal state (`Done` / `Failed` / /// `Cancelled`). `None` while non-terminal. Stamped by the graph. pub finished_at: Option>, /// Failure reason for a `Failed` node, supplied by the runner via /// [`scheduler::Outcome::Failed`]. `None` unless this node's own logic /// failed (a node that rolled up `Failed` from a child, or was cancelled, /// carries no error of its own). pub error: Option, } /// An error from inserting into or loading a [`Graph`] with a dangling id. /// /// A [`NodeId`] is only meaningful against the graph that minted it, so both /// entry points — [`Graph::insert`] and deserialization — reject references to /// nodes the graph does not contain. That is what lets internal iteration trust /// every id the graph holds. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum GraphError { /// A node's dependency named an id not present in the graph. #[error("dependency references unknown node {0:?}")] UnknownDep(NodeId), /// A node's `parent` named an id not present in the graph. #[error("parent references unknown node {0:?}")] UnknownParent(NodeId), /// A node's [`Dep::Node`] edge points outside its own parent group — the /// target must be a proper descendant of the depender's `parent` (a sibling /// or a sibling's sub-node), never the parent itself or a node in another /// group. Top-level nodes may only depend on top-level nodes. #[error("dependency {dep:?} is outside the depender's parent group {parent:?}")] DepOutsideParent { /// The out-of-group dependency target. dep: NodeId, /// The depender's parent (the group the target had to be inside). parent: Option, }, /// A loaded graph's `next_id` counter is not past the largest existing id, /// so the next minted id would collide with one already in the graph. #[error("next_id {next_id} must exceed the largest existing node id {max_id}")] NextIdTooSmall { /// The persisted counter value. next_id: u64, /// The largest id already present. max_id: u64, }, } /// The single persistent graph of all nodes. /// /// New jobs are inserted as sub-DAGs of nodes; the scheduler walks this graph /// filling open slots. Completed nodes are retained (no pruning in v1). #[derive(Debug, serde::Serialize, serde::Deserialize)] #[serde( try_from = "GraphData", bound( serialize = "N: serde::Serialize, R: serde::Serialize", deserialize = "N: serde::Deserialize<'de>, R: serde::Deserialize<'de>" ) )] pub struct Graph { nodes: Vec>, next_id: u64, } // Deserialization target: the raw fields, turned into a `Graph` by the `TryFrom` // below — which runs [`Graph::validate`], so a loaded graph can never carry a // dangling id reference (Serialize does not validate; Deserialize always does). #[derive(serde::Deserialize)] #[serde(bound(deserialize = "N: serde::Deserialize<'de>, R: serde::Deserialize<'de>"))] struct GraphData { nodes: Vec>, next_id: u64, } impl TryFrom> for Graph { type Error = GraphError; fn try_from(data: GraphData) -> Result { let graph = Graph { nodes: data.nodes, next_id: data.next_id, }; graph.validate()?; Ok(graph) } } // A `derive(Default)` would wrongly require `N: Default` (an empty graph holds // no payload); an empty `Vec>` needs no such bound, so impl it directly. impl Default for Graph { fn default() -> Self { Self::new() } } impl Graph { /// An empty graph. #[must_use] pub fn new() -> Self { Self { nodes: Vec::new(), next_id: 0, } } /// Mint the next stable node id. fn mint_id(&mut self) -> NodeId { let id = NodeId(self.next_id); self.next_id += 1; id } /// Insert a node with the given payload, deps, and `parent`, returning its /// freshly-minted id. The node starts [`State::Pending`]. /// /// Every [`Dep::Node`] id and the `parent` id (when `Some`) must already /// resolve to a node in the graph — an id is only meaningful against the /// graph that minted it, so a dangling reference is rejected here rather than /// surfacing as a broken edge later. /// /// # Errors /// Returns [`GraphError::UnknownDep`] / [`GraphError::UnknownParent`] for a /// dangling dependency or parent id, or [`GraphError::DepOutsideParent`] if a /// `Dep::Node` edge points outside the node's own parent group. pub fn insert( &mut self, payload: N, deps: Vec>, parent: Option, ) -> Result { if let Some(p) = parent && self.node(p).is_none() { return Err(GraphError::UnknownParent(p)); } for dep in &deps { if let Dep::Node { id, .. } = dep { if self.node(*id).is_none() { return Err(GraphError::UnknownDep(*id)); } if !self.dep_target_in_group(parent, *id) { return Err(GraphError::DepOutsideParent { dep: *id, parent }); } } } let id = self.mint_id(); self.nodes.push(Node { id, parent, payload, deps, state: State::Pending, started_at: None, finished_at: None, error: None, }); Ok(id) } /// Borrow a node by id. #[must_use] pub fn node(&self, id: NodeId) -> Option<&Node> { self.nodes.iter().find(|n| n.id == id) } /// Every node in the graph, in insertion order. The scheduler iterates /// this to find runnable pending nodes. pub fn nodes(&self) -> impl Iterator> { self.nodes.iter() } /// Whether `ancestor` lies on `node`'s [`Node::parent`] chain (i.e. `node` is /// in `ancestor`'s subtree). `node` is not its own ancestor. fn is_descendant(&self, node: NodeId, ancestor: NodeId) -> bool { let mut cur = self.node(node).and_then(|n| n.parent); while let Some(p) = cur { if p == ancestor { return true; } cur = self.node(p).and_then(|n| n.parent); } false } /// Whether a node whose parent is `node_parent` may depend on `target` — the /// grouping rule: a [`Dep::Node`] edge must stay inside the depender's own /// parent group. `target` must be a proper descendant of `node_parent` (a /// sibling or a sibling's sub-node), never the parent itself (which would /// deadlock: the parent stays [`State::Finishing`] until its children finish, /// so a child that waited on the parent could never run). Top-level nodes /// (`parent == None`) may only depend on other top-level nodes. fn dep_target_in_group(&self, node_parent: Option, target: NodeId) -> bool { match node_parent { Some(p) => self.is_descendant(target, p), None => self.node(target).is_some_and(|n| n.parent.is_none()), } } /// Set a node's lifecycle state, returning `false` for an unknown id. The /// scheduler drives every state transition — nothing else mutates state, /// which is what keeps the resource guards + terminality in sync. This is /// also where the node's lifecycle timestamps are stamped: `started_at` on /// the first transition to [`State::Running`], `finished_at` on the first /// transition to a terminal state (`Done` / `Failed` / `Cancelled`). pub(crate) fn set_state(&mut self, id: NodeId, state: State) -> bool { if let Some(node) = self.nodes.iter_mut().find(|n| n.id == id) { node.state = state; if state == State::Running { if node.started_at.is_none() { node.started_at = Some(now_utc()); } } else if state.is_terminal() && node.finished_at.is_none() { node.finished_at = Some(now_utc()); } true } else { false } } /// Record a node's failure reason ([`Node::error`]). No-op for an unknown /// id. Called by the scheduler on an [`scheduler::Outcome::Failed`] before /// the terminal state transition. pub(crate) fn set_error(&mut self, id: NodeId, error: String) { if let Some(node) = self.nodes.iter_mut().find(|n| n.id == id) { node.error = Some(error); } } /// Check that every id the graph holds resolves: every [`Dep::Node`] id /// names a node present in the graph, and `next_id` is past the largest /// existing id. Deserialization runs this, so a loaded graph is internally /// consistent and internal iteration can trust its ids. /// /// # Errors /// Returns [`GraphError`] on a dangling dependency reference, or a `next_id` /// that would remint an id already in the graph. pub fn validate(&self) -> Result<(), GraphError> { for node in &self.nodes { if let Some(p) = node.parent && self.node(p).is_none() { return Err(GraphError::UnknownParent(p)); } for dep in &node.deps { if let Dep::Node { id, .. } = dep { if self.node(*id).is_none() { return Err(GraphError::UnknownDep(*id)); } if !self.dep_target_in_group(node.parent, *id) { return Err(GraphError::DepOutsideParent { dep: *id, parent: node.parent, }); } } } } if let Some(max_id) = self.nodes.iter().map(|n| n.id.0).max() && self.next_id <= max_id { return Err(GraphError::NextIdTooSmall { next_id: self.next_id, max_id, }); } Ok(()) } } #[cfg(test)] mod tests { use super::*; #[test] fn insert_mints_stable_monotonic_ids() { let mut g: Graph<&str, String> = Graph::new(); let a = g.insert("sweep", vec![], None).unwrap(); let b = g .insert( "update", vec![Dep::Node { id: a, when: DepWhen::AfterOk, }], None, ) .unwrap(); assert_eq!(a, NodeId(0)); assert_eq!(b, NodeId(1)); // The dep edge references the earlier node; ids are stable + monotonic. assert!(matches!( g.node(b).unwrap().deps.first(), Some(Dep::Node { id, .. }) if *id == a )); } #[test] fn state_terminality() { assert!(State::Done.is_terminal()); assert!(State::Failed.is_terminal()); assert!(State::Cancelled.is_terminal()); assert!(!State::Pending.is_terminal()); assert!(!State::Running.is_terminal()); // Finishing (logic done, children still running) is NOT terminal. assert!(!State::Finishing.is_terminal()); } #[test] fn after_ok_needs_success_after_any_needs_terminal() { // AfterOk: only Done satisfies; a Failed/Cancelled dep does NOT (the // dependent must be cancelled, not run). assert!(DepWhen::AfterOk.satisfied_by(State::Done)); assert!(!DepWhen::AfterOk.satisfied_by(State::Failed)); assert!(!DepWhen::AfterOk.satisfied_by(State::Cancelled)); assert!(!DepWhen::AfterOk.satisfied_by(State::Running)); // AfterAny: any terminal state satisfies. assert!(DepWhen::AfterAny.satisfied_by(State::Done)); assert!(DepWhen::AfterAny.satisfied_by(State::Failed)); assert!(DepWhen::AfterAny.satisfied_by(State::Cancelled)); assert!(!DepWhen::AfterAny.satisfied_by(State::Pending)); // Finishing satisfies neither — a dependent waits until the node rolls // up to a terminal state (all its sub-nodes done). assert!(!DepWhen::AfterOk.satisfied_by(State::Finishing)); assert!(!DepWhen::AfterAny.satisfied_by(State::Finishing)); } #[test] fn insert_rejects_unknown_dep() { let mut g: Graph<&str, String> = Graph::new(); let bogus = NodeId(42); let deps = vec![Dep::Node { id: bogus, when: DepWhen::AfterOk, }]; assert_eq!( g.insert("x", deps, None).unwrap_err(), GraphError::UnknownDep(bogus) ); } #[test] fn insert_rejects_unknown_parent() { let mut g: Graph<&str, String> = Graph::new(); let bogus = NodeId(7); assert_eq!( g.insert("x", vec![], Some(bogus)).unwrap_err(), GraphError::UnknownParent(bogus) ); // A resolvable parent is accepted and recorded. let a = g.insert("a", vec![], None).unwrap(); let b = g.insert("b", vec![], Some(a)).unwrap(); assert_eq!(g.node(b).unwrap().parent, Some(a)); } #[test] fn valid_graph_round_trips_through_serde() { let mut g: Graph = Graph::new(); // `a` (top-level) and `b` (top-level, depends on its sibling `a`), plus // `c` — a sub-node of `a` (grouping, no dep on its parent). let a = g.insert("a".to_owned(), vec![], None).unwrap(); g.insert( "b".to_owned(), vec![Dep::Node { id: a, when: DepWhen::AfterAny, }], None, ) .unwrap(); let c = g.insert("c".to_owned(), vec![], Some(a)).unwrap(); let json = serde_json::to_string(&g).unwrap(); let back: Graph = serde_json::from_str(&json).unwrap(); assert!(back.validate().is_ok()); assert_eq!(back.node(a).unwrap().payload, "a"); assert_eq!(back.node(c).unwrap().parent, Some(a)); } #[test] fn insert_rejects_dep_on_parent_and_cross_group() { let mut g: Graph<&str, String> = Graph::new(); let root = g.insert("root", vec![], None).unwrap(); // A child cannot depend on its own parent (would deadlock under the // roll-up model — the parent stays `Finishing` awaiting its children). let on_parent = vec![Dep::Node { id: root, when: DepWhen::AfterOk, }]; assert_eq!( g.insert("child", on_parent, Some(root)).unwrap_err(), GraphError::DepOutsideParent { dep: root, parent: Some(root), } ); // A sibling dep IS allowed: two children of `root`, the second on the first. let c1 = g.insert("c1", vec![], Some(root)).unwrap(); let c2 = g .insert("c2", vec![after_ok_dep(c1)], Some(root)) .expect("sibling dep is in-group"); assert_eq!(g.node(c2).unwrap().parent, Some(root)); // But a node in another group cannot be depended on across the boundary. let other = g.insert("other", vec![], None).unwrap(); assert_eq!( g.insert("x", vec![after_ok_dep(other)], Some(root)) .unwrap_err(), GraphError::DepOutsideParent { dep: other, parent: Some(root), } ); } fn after_ok_dep(on: NodeId) -> Dep { Dep::Node { id: on, when: DepWhen::AfterOk, } } #[test] fn deserialize_rejects_a_dangling_dependency() { // Build a graph whose only node depends on a non-existent id, serialize // it (Serialize does not validate), and confirm deserialize rejects it. let bad = Graph:: { nodes: vec![Node { id: NodeId(0), parent: None, payload: "x".to_owned(), deps: vec![Dep::Node { id: NodeId(99), when: DepWhen::AfterOk, }], state: State::Pending, started_at: None, finished_at: None, error: None, }], next_id: 1, }; let json = serde_json::to_string(&bad).unwrap(); let err = serde_json::from_str::>(&json).unwrap_err(); assert!(err.to_string().contains("unknown node")); } #[test] fn validate_rejects_next_id_that_would_remint() { let bad = Graph::<&str, String> { nodes: vec![Node { id: NodeId(5), parent: None, payload: "x", deps: vec![], state: State::Pending, started_at: None, finished_at: None, error: None, }], next_id: 3, }; assert_eq!( bad.validate().unwrap_err(), GraphError::NextIdTooSmall { next_id: 3, max_id: 5, } ); } }