feat(#2591): port hive-c0re job_queue onto the hive-jobq crate
Replace the in-tree scheduler with the domain-agnostic hive-jobq crate (merged in #2615): parent-axis grouping + borrow/subtree-reservation resource model + roll-up completion (State::Finishing). Host adaptation: - NodeSpec gains an explicit `parent` axis; templates declare grouping + sibling ordering directly (deps order execution, parent groups a subtree whose resource the descendants borrow). - Rebuild is a nested two-root subtree: Prebuild (root, owns the build slot for the whole subtree, lease-exempt) -> StopForUpdate (child, owns the agent lease) -> Swap/PostSwap (children, borrow both); Reconcile is a separate top-level root (AfterAny Prebuild) so it survives the cancel- cascade of any failed step (recovery-start invariant) and converges to the persisted `wanted` on a fresh lease. This is the multi-root correction to the single-root-chain sketch: node0=root broke lease- exemption (hoisting the lease onto Prebuild) and recovery-reconcile (root failure cancels all children). - Spawn / perm-change / power-ops (stop/start/restart) group-rooted the same way; per-agent power-op subgraphs stay independent roots so a multi-agent DAG runs them concurrently, each on its own lease. - insert_group honours the explicit parent axis (no lease hoisting); the DAG terminal node deps AfterAny on every group root and runs once the whole op rolls up. Drop the old Graph::add_dep terminal wiring. 36/36 job_queue tests, full hive-c0re suite green, clippy --all-targets.
This commit is contained in:
atlas
mara
parent
e58457da17
commit
a5c321a1a0
14 changed files with 1111 additions and 893 deletions
|
|
@ -11,7 +11,7 @@
|
|||
//! DAG can span agents). See `docs/coordinator.md::Job queue` for the
|
||||
//! full design.
|
||||
|
||||
pub use hive_sh4re::jobs::{DagView, NodeId, NodeView, PermPayload, Source, State, Template};
|
||||
pub use hive_sh4re::jobs::{DagView, NodeId, PermPayload, Source, State, Template};
|
||||
use serde::Serialize;
|
||||
|
||||
/// When a dependency edge is considered satisfied.
|
||||
|
|
@ -139,6 +139,21 @@ pub enum NodeKind {
|
|||
/// `Prebuild`, but that's a no-op there — the agent is down, so prebuild
|
||||
/// is skipped.)
|
||||
SetWanted { up: bool },
|
||||
/// Per-DAG terminal hook (approval-driven DAGs — spawn / opaque deploy):
|
||||
/// resolve the DAG's approval row from the rolled-up outcome. Appended once
|
||||
/// with a weak (`AfterAny`) edge on the DAG's tails, so it runs exactly when
|
||||
/// the DAG has settled (any outcome, including a cancel before starting).
|
||||
/// Build-slot- and lease-exempt; always runs (weak edge ⇒ never cascaded).
|
||||
ResolveApproval,
|
||||
/// Per-DAG terminal hook (rebuild / perm-change DAGs): emit one `Rebuilt`
|
||||
/// manager event per targeted agent — `ok` on `Done`, `!ok` on `Failed`,
|
||||
/// none on cancel. Appended weak-dep on the tails; slot/lease-exempt.
|
||||
EmitRebuilt,
|
||||
/// Per-DAG terminal hook (power-op DAGs): on a *cancelled* DAG, revert each
|
||||
/// agent's `wanted` intent to its observed state — the operator's cancel
|
||||
/// means "don't do it". Noop on any non-cancelled outcome. Appended weak-dep
|
||||
/// on the tails; slot/lease-exempt.
|
||||
RevertIntent,
|
||||
}
|
||||
|
||||
impl NodeKind {
|
||||
|
|
@ -161,6 +176,9 @@ impl NodeKind {
|
|||
NodeKind::WritePermFile => "write_perm_file",
|
||||
NodeKind::ApprovalDeploy => "approval_deploy",
|
||||
NodeKind::SetWanted { .. } => "set_wanted",
|
||||
NodeKind::ResolveApproval => "resolve_approval",
|
||||
NodeKind::EmitRebuilt => "emit_rebuilt",
|
||||
NodeKind::RevertIntent => "revert_intent",
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -201,39 +219,23 @@ impl NodeKind {
|
|||
}
|
||||
}
|
||||
|
||||
/// One schedulable unit inside a DAG.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Node {
|
||||
pub id: NodeId,
|
||||
/// The agent this node's work targets. Per-node so a single DAG can
|
||||
/// span agents (e.g. a hive-wide restart); the lifecycle lease is
|
||||
/// acquired against *this* agent (still globally exclusive per agent
|
||||
/// across all DAGs). `"hyperhive"` for meta-level nodes.
|
||||
pub agent: String,
|
||||
pub kind: NodeKind,
|
||||
pub deps: Vec<Dep>,
|
||||
pub state: State,
|
||||
/// Live sub-label while `Running` (kept for parity with the old
|
||||
/// per-entry `step`).
|
||||
pub step: Option<String>,
|
||||
/// Row id of the `build_logs` entry this node opened (`Prebuild` /
|
||||
/// `Swap` / `ApprovalDeploy`), for the dashboard's live-stream link.
|
||||
pub build_log_id: Option<i64>,
|
||||
pub started_at: Option<i64>,
|
||||
pub finished_at: Option<i64>,
|
||||
/// Populated when `state == Failed` (truncated by the queue).
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Submit-time spec for one node.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NodeSpec {
|
||||
/// The agent this node targets (see [`Node::agent`]). Built by the
|
||||
/// `templates.rs` `node` helper, which stamps the template's agent
|
||||
/// onto every node.
|
||||
/// The agent this node's work targets. Built by the `templates.rs` `node`
|
||||
/// helper, which stamps the template's agent onto every node.
|
||||
pub agent: String,
|
||||
pub kind: NodeKind,
|
||||
pub deps: Vec<Dep>,
|
||||
/// The **structural parent** axis — the spec-local index of this node's
|
||||
/// group parent, or `None` for a top-level (group-root) node. Independent
|
||||
/// of `deps`: `deps` order execution, `parent` groups nodes into a subtree
|
||||
/// whose resource the whole subtree borrows (the agent lease is owned by a
|
||||
/// group root and re-entered by its descendants for continuity). A child
|
||||
/// runs once its parent reaches `Finishing` (the parent gate), so a child
|
||||
/// never `deps` on its own parent (that would deadlock — dep-scope
|
||||
/// validation rejects it).
|
||||
pub parent: Option<u64>,
|
||||
}
|
||||
|
||||
/// Submit-time spec for a whole DAG. Built by `templates.rs`; validated
|
||||
|
|
@ -258,140 +260,3 @@ pub struct DagSpec {
|
|||
pub transient: Option<crate::coordinator::TransientKind>,
|
||||
pub nodes: Vec<NodeSpec>,
|
||||
}
|
||||
|
||||
/// A live DAG in the queue. No DAG-level `agent`: agent is per-[`Node`],
|
||||
/// so a DAG can span agents. Per-agent leasing is derived from the
|
||||
/// nodes' agents.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Dag {
|
||||
pub id: u64,
|
||||
pub template: Template,
|
||||
pub source: Source,
|
||||
pub reason: String,
|
||||
pub approval_id: Option<i64>,
|
||||
pub inputs: Vec<String>,
|
||||
pub perm_payload: Option<PermPayload>,
|
||||
pub transient: Option<crate::coordinator::TransientKind>,
|
||||
pub created_at: i64,
|
||||
pub nodes: Vec<Node>,
|
||||
/// Terminal roll-up already reported to the scheduler's hooks
|
||||
/// (approval resolution, transient release). Internal bookkeeping,
|
||||
/// never serialized.
|
||||
pub terminal_reported: bool,
|
||||
}
|
||||
|
||||
impl Dag {
|
||||
/// Roll-up state: `Failed` if any node failed; else `Running` if
|
||||
/// any running; else `Queued` if any queued; else `Cancelled` if
|
||||
/// any cancelled; else `Done`.
|
||||
pub fn rollup(&self) -> State {
|
||||
let mut any_cancelled = false;
|
||||
let mut any_queued = false;
|
||||
let mut any_running = false;
|
||||
for n in &self.nodes {
|
||||
match n.state {
|
||||
State::Failed => return State::Failed,
|
||||
State::Running => any_running = true,
|
||||
State::Queued => any_queued = true,
|
||||
State::Cancelled => any_cancelled = true,
|
||||
State::Done => {}
|
||||
}
|
||||
}
|
||||
if any_running {
|
||||
State::Running
|
||||
} else if any_queued {
|
||||
State::Queued
|
||||
} else if any_cancelled {
|
||||
State::Cancelled
|
||||
} else {
|
||||
State::Done
|
||||
}
|
||||
}
|
||||
|
||||
/// True when every node is terminal.
|
||||
pub fn is_terminal(&self) -> bool {
|
||||
self.nodes.iter().all(|n| n.state.is_terminal())
|
||||
}
|
||||
|
||||
/// True when no live (non-terminal) node of this DAG still targets
|
||||
/// `agent` — i.e. that agent's subgraph within the DAG has settled.
|
||||
/// Used to release an agent's lifecycle lease the moment its own
|
||||
/// work is done, rather than waiting for the whole DAG to terminate.
|
||||
/// Vacuously true for an agent the DAG has no node for; callers gate
|
||||
/// on actually holding that agent's lease first.
|
||||
pub fn agent_subgraph_terminal(&self, agent: &str) -> bool {
|
||||
self.nodes
|
||||
.iter()
|
||||
.filter(|n| n.agent == agent)
|
||||
.all(|n| n.state.is_terminal())
|
||||
}
|
||||
|
||||
/// First failed node's error, for the roll-up `error` field.
|
||||
pub fn first_error(&self) -> Option<&str> {
|
||||
self.nodes
|
||||
.iter()
|
||||
.find(|n| n.state == State::Failed)
|
||||
.and_then(|n| n.error.as_deref())
|
||||
}
|
||||
|
||||
pub fn node(&self, id: NodeId) -> Option<&Node> {
|
||||
self.nodes.iter().find(|n| n.id == id)
|
||||
}
|
||||
|
||||
pub fn node_mut(&mut self, id: NodeId) -> Option<&mut Node> {
|
||||
self.nodes.iter_mut().find(|n| n.id == id)
|
||||
}
|
||||
|
||||
/// Distinct agents this DAG's nodes target, in first-seen order.
|
||||
/// Used for per-agent lease release and the terminal cancel-revert —
|
||||
/// a single-agent DAG yields one, a multi-agent DAG yields several.
|
||||
pub fn agents(&self) -> Vec<String> {
|
||||
let mut seen: Vec<String> = Vec::new();
|
||||
for n in &self.nodes {
|
||||
if !seen.iter().any(|a| a == &n.agent) {
|
||||
seen.push(n.agent.clone());
|
||||
}
|
||||
}
|
||||
seen
|
||||
}
|
||||
}
|
||||
|
||||
impl Dag {
|
||||
pub fn view(&self) -> DagView {
|
||||
let started_at = self.nodes.iter().filter_map(|n| n.started_at).min();
|
||||
let finished_at = if self.is_terminal() {
|
||||
self.nodes.iter().filter_map(|n| n.finished_at).max()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
DagView {
|
||||
id: self.id,
|
||||
kind: self.template,
|
||||
state: self.rollup(),
|
||||
source: self.source,
|
||||
reason: self.reason.clone(),
|
||||
enqueued_at: self.created_at,
|
||||
started_at,
|
||||
finished_at,
|
||||
inputs: self.inputs.clone(),
|
||||
approval_id: self.approval_id,
|
||||
perm_payload: self.perm_payload.clone(),
|
||||
nodes: self
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|n| NodeView {
|
||||
id: n.id,
|
||||
agent: n.agent.clone(),
|
||||
kind: n.kind.as_str().to_owned(),
|
||||
deps: n.deps.iter().map(|d| d.on).collect(),
|
||||
state: n.state,
|
||||
step: n.step.clone(),
|
||||
build_log_id: n.build_log_id,
|
||||
started_at: n.started_at,
|
||||
finished_at: n.finished_at,
|
||||
error: n.error.clone(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue