feat(#2591): port hive-c0re job_queue onto the hive-jobq crate

Replace the in-tree scheduler with the domain-agnostic hive-jobq crate
(merged in #2615): parent-axis grouping + borrow/subtree-reservation
resource model + roll-up completion (State::Finishing).

Host adaptation:
- NodeSpec gains an explicit `parent` axis; templates declare grouping +
  sibling ordering directly (deps order execution, parent groups a subtree
  whose resource the descendants borrow).
- Rebuild is a nested two-root subtree: Prebuild (root, owns the build slot
  for the whole subtree, lease-exempt) -> StopForUpdate (child, owns the
  agent lease) -> Swap/PostSwap (children, borrow both); Reconcile is a
  separate top-level root (AfterAny Prebuild) so it survives the cancel-
  cascade of any failed step (recovery-start invariant) and converges to
  the persisted `wanted` on a fresh lease. This is the multi-root
  correction to the single-root-chain sketch: node0=root broke lease-
  exemption (hoisting the lease onto Prebuild) and recovery-reconcile
  (root failure cancels all children).
- Spawn / perm-change / power-ops (stop/start/restart) group-rooted the
  same way; per-agent power-op subgraphs stay independent roots so a
  multi-agent DAG runs them concurrently, each on its own lease.
- insert_group honours the explicit parent axis (no lease hoisting); the
  DAG terminal node deps AfterAny on every group root and runs once the
  whole op rolls up. Drop the old Graph::add_dep terminal wiring.

36/36 job_queue tests, full hive-c0re suite green, clippy --all-targets.
This commit is contained in:
atlas 2026-07-20 21:46:08 +02:00 committed by mara
commit a5c321a1a0
14 changed files with 1111 additions and 893 deletions

View file

@ -25,7 +25,7 @@
use std::sync::Arc;
use super::model::{DagSpec, Dep, NodeKind, NodeSpec, Template};
use super::templates::{after_ok, node, rebuild_nodes};
use super::templates::{after_ok, child, node, rebuild_nodes};
use super::{Source, templates};
use crate::coordinator::{Coordinator, TransientKind};
use crate::lifecycle;
@ -60,13 +60,16 @@ pub fn rebuild(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: St
/// stays even for a down agent so a race-up between the state read and exec
/// is still stopped in-DAG.
fn stop_chain(agent: &str, graceful: bool, running: bool) -> Vec<NodeSpec> {
// `SetWanted` is the group root and owns the agent lease; the mechanical
// steps are its children (borrow the lease, run once it reaches `Finishing`,
// dep-ordered among themselves).
let mut n = vec![node(agent, NodeKind::SetWanted { up: false }, Vec::new())];
if graceful && running {
n.push(node(agent, NodeKind::Signal, after_ok(0)));
n.push(node(agent, NodeKind::Drain, after_ok(1)));
n.push(node(agent, NodeKind::Reconcile, after_ok(2)));
n.push(child(0, agent, NodeKind::Signal, Vec::new()));
n.push(child(0, agent, NodeKind::Drain, after_ok(1)));
n.push(child(0, agent, NodeKind::Reconcile, after_ok(2)));
} else {
n.push(node(agent, NodeKind::Reconcile, after_ok(0)));
n.push(child(0, agent, NodeKind::Reconcile, Vec::new()));
}
n
}
@ -78,11 +81,12 @@ fn stop_chain(agent: &str, graceful: bool, running: bool) -> Vec<NodeSpec> {
fn start_chain(agent: &str, running: bool, stale: bool) -> Vec<NodeSpec> {
let mut n = vec![node(agent, NodeKind::SetWanted { up: true }, Vec::new())];
if !running && stale {
// Rebuild subgraph rooted at the SetWanted head (base = 1, so
// `Prebuild` deps `after_ok(0)` = the head).
// Rebuild subtree after the SetWanted head (base = 1, so the rebuild's
// `Prebuild` root deps `after_ok(0)` = the head). `Prebuild` +
// `Reconcile` are their own group roots (top-level, per `rebuild_nodes`).
n.extend(rebuild_nodes(agent, true, 1));
} else {
n.push(node(agent, NodeKind::Reconcile, after_ok(0)));
n.push(child(0, agent, NodeKind::Reconcile, Vec::new()));
}
n
}
@ -101,18 +105,30 @@ fn restart_chain(agent: &str, graceful: bool, running: bool) -> Vec<NodeSpec> {
// Nothing to bounce — a lone Reconcile converges to intent.
return vec![node(agent, NodeKind::Reconcile, Vec::new())];
}
// Running: mechanical stop then Reconcile. The first stop node is the
// subgraph root (no SetWanted head) and acquires the agent lease.
let mut n = Vec::new();
if graceful {
n.push(node(agent, NodeKind::Signal, Vec::new()));
n.push(node(agent, NodeKind::Drain, after_ok(0)));
n.push(node(agent, NodeKind::StopForUpdate, after_ok(1)));
// Running: mechanical stop then Reconcile. The first stop node is the group
// ROOT (no SetWanted head) and owns the agent lease; the rest are its
// children (borrow the lease, dep-ordered), so the bounce holds one
// continuous lease and `Reconcile` cancel-cascades if a stop step fails.
let mut n = vec![if graceful {
node(agent, NodeKind::Signal, Vec::new())
} else {
n.push(node(agent, NodeKind::StopForUpdate, Vec::new()));
node(agent, NodeKind::StopForUpdate, Vec::new())
}];
if graceful {
n.push(child(0, agent, NodeKind::Drain, Vec::new()));
n.push(child(0, agent, NodeKind::StopForUpdate, after_ok(1)));
}
let stop_idx = u32::try_from(n.len() - 1).unwrap_or(0);
n.push(node(agent, NodeKind::Reconcile, after_ok(stop_idx)));
// `Reconcile` gates on the last mechanical step. When the only step is the
// root itself (non-graceful, `StopForUpdate` == index 0), the parent gate
// already orders `Reconcile` after it — a child must NOT dep on its own
// parent (dep-scope). So the sibling dep is added only for a graceful
// bounce, where the last step is a sibling child.
let deps = if n.len() > 1 {
after_ok(u64::try_from(n.len() - 1).unwrap_or(0))
} else {
Vec::new()
};
n.push(child(0, agent, NodeKind::Reconcile, deps));
n
}
@ -124,7 +140,7 @@ fn restart_chain(agent: &str, graceful: bool, running: bool) -> Vec<NodeSpec> {
fn concat_subgraphs(chains: Vec<Vec<NodeSpec>>) -> Vec<NodeSpec> {
let mut out: Vec<NodeSpec> = Vec::new();
for chain in chains {
let base = u32::try_from(out.len()).unwrap_or(u32::MAX);
let base = u64::try_from(out.len()).unwrap_or(u64::MAX);
for spec in chain {
let deps = spec
.deps
@ -138,6 +154,10 @@ fn concat_subgraphs(chains: Vec<Vec<NodeSpec>>) -> Vec<NodeSpec> {
agent: spec.agent,
kind: spec.kind,
deps,
// Rebase the structural parent by the same offset (a subgraph
// root keeps `parent = None`, so the per-agent groups stay
// independent + concurrent).
parent: spec.parent.map(|p| base + p),
});
}
}