feat(#2591): port hive-c0re job_queue onto the hive-jobq crate

Replace the in-tree scheduler with the domain-agnostic hive-jobq crate
(merged in #2615): parent-axis grouping + borrow/subtree-reservation
resource model + roll-up completion (State::Finishing).

Host adaptation:
- NodeSpec gains an explicit `parent` axis; templates declare grouping +
  sibling ordering directly (deps order execution, parent groups a subtree
  whose resource the descendants borrow).
- Rebuild is a nested two-root subtree: Prebuild (root, owns the build slot
  for the whole subtree, lease-exempt) -> StopForUpdate (child, owns the
  agent lease) -> Swap/PostSwap (children, borrow both); Reconcile is a
  separate top-level root (AfterAny Prebuild) so it survives the cancel-
  cascade of any failed step (recovery-start invariant) and converges to
  the persisted `wanted` on a fresh lease. This is the multi-root
  correction to the single-root-chain sketch: node0=root broke lease-
  exemption (hoisting the lease onto Prebuild) and recovery-reconcile
  (root failure cancels all children).
- Spawn / perm-change / power-ops (stop/start/restart) group-rooted the
  same way; per-agent power-op subgraphs stay independent roots so a
  multi-agent DAG runs them concurrently, each on its own lease.
- insert_group honours the explicit parent axis (no lease hoisting); the
  DAG terminal node deps AfterAny on every group root and runs once the
  whole op rolls up. Drop the old Graph::add_dep terminal wiring.

36/36 job_queue tests, full hive-c0re suite green, clippy --all-targets.
This commit is contained in:
atlas 2026-07-20 21:46:08 +02:00 committed by mara
commit a5c321a1a0
14 changed files with 1111 additions and 893 deletions

View file

@ -35,34 +35,58 @@ use crate::coordinator::TransientKind;
/// After-ok edge on the previous node — the common chain link. Shared with
/// the async power-op builders in `submit.rs` (which assemble per-agent
/// chains dynamically from live container state).
pub(crate) fn after_ok(on: u32) -> Vec<Dep> {
pub(crate) fn after_ok(on: u64) -> Vec<Dep> {
vec![Dep {
on,
when: DepWhen::AfterOk,
}]
}
/// Build one node targeting `agent`. The single place a node's agent is
/// stamped. Shared with `submit.rs`'s dynamic power-op builders.
/// Build one **top-level (group-root)** node targeting `agent` — `parent =
/// None`. The single place a node's agent is stamped. Shared with `submit.rs`'s
/// dynamic power-op builders. A root owns whatever resource it declares for its
/// whole subtree; its descendants borrow it (agent-lease / build-slot
/// continuity). Ordering vs other nodes is `deps`; grouping is `parent`.
pub(crate) fn node(agent: &str, kind: NodeKind, deps: Vec<Dep>) -> NodeSpec {
NodeSpec {
agent: agent.to_owned(),
kind,
deps,
parent: None,
}
}
/// The rebuild node chain. `PostSwap` carries the swap's Ok-only
/// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan) and deps
/// `Swap` with `AfterOk`. `Reconcile` then deps on `PostSwap` with
/// `AfterAny`: it must run even when the swap failed, so a previously-up
/// agent comes back on its old config (today's recovery-start). On swap
/// failure the `AfterOk` `PostSwap` is cancel-cascaded to a terminal state,
/// which still satisfies `Reconcile`'s `AfterAny` edge — the only `AfterAny`
/// edge in v1. Pointing `Reconcile` at `PostSwap` (not `Swap`) also
/// serializes the tail ahead of the reconcile, so there's no double
/// rescan/kick race.
pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u32) -> Vec<NodeSpec> {
/// Build a **child** node whose structural parent is spec-index `parent`. The
/// child runs once its parent reaches `Finishing` (the parent gate), so it must
/// NOT `deps` on `parent` (dep-scope validation rejects a dep on one's own
/// parent). `deps` here order the child against its *siblings* only.
pub(crate) fn child(parent: u64, agent: &str, kind: NodeKind, deps: Vec<Dep>) -> NodeSpec {
NodeSpec {
agent: agent.to_owned(),
kind,
deps,
parent: Some(parent),
}
}
/// The rebuild node subtree (nested, two group roots). `base` is the spec index
/// of the first node (`Prebuild`). Structure:
/// - `Prebuild` (base+0, **root**): owns the build slot for the whole subtree.
/// Lease-exempt — the nix build overlaps other DAGs on the same agent.
/// - `StopForUpdate` (base+1, child of `Prebuild`): owns the agent lease. Runs
/// once `Prebuild` reaches `Finishing` (parent gate).
/// - `Swap` (base+2, child of `StopForUpdate`): borrows the agent lease from its
/// parent and the build slot from grand-ancestor `Prebuild` — both continuous.
/// - `PostSwap` (base+3, child of `StopForUpdate`): the swap's Ok-only
/// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan), `AfterOk`
/// its sibling `Swap`.
/// - `Reconcile` (base+4, **root**): `AfterAny` `Prebuild`, which rolls up
/// terminal only once its whole mechanical subtree (SFU→Swap→PostSwap) has
/// settled — so `Reconcile` runs after the swap regardless of outcome, and as
/// a top-level root it survives the cancel-cascade of a failed `Prebuild`
/// (recovery-start invariant). It takes a fresh lease; the tiny gap is
/// harmless — `Reconcile` converges to the persisted `wanted` idempotently.
pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u64) -> Vec<NodeSpec> {
vec![
node(
agent,
@ -73,14 +97,14 @@ pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u32) -> Vec<NodeSpe
after_ok(base - 1)
},
),
node(agent, NodeKind::StopForUpdate, after_ok(base)),
node(agent, NodeKind::Swap, after_ok(base + 1)),
node(agent, NodeKind::PostSwap, after_ok(base + 2)),
child(base, agent, NodeKind::StopForUpdate, Vec::new()),
child(base + 1, agent, NodeKind::Swap, Vec::new()),
child(base + 1, agent, NodeKind::PostSwap, after_ok(base + 2)),
node(
agent,
NodeKind::Reconcile,
vec![Dep {
on: base + 3,
on: base,
when: DepWhen::AfterAny,
}],
),
@ -149,7 +173,12 @@ pub fn reconcile_only(
/// First-deploy spawn (approval-driven): `Provision` (proposed/applied
/// repos, state subvolume, meta registration) then `Create`
/// (`nixos-container create`), drop-in write, then `Reconcile` starts
/// the container (`wanted = Up` written at approve time).
/// the container (`wanted = Up` written at approve time). All-or-nothing:
/// `Provision` (lease-exempt, precedes the container) is the group root;
/// `Create` (child) owns the agent lease; `WriteDropin` + `Reconcile`
/// (children of `Create`) borrow it. A failure cancel-cascades the rest —
/// unlike rebuild there's no recovery-reconcile (nothing to converge if the
/// container was never created).
pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
DagSpec {
template: Template::Spawn,
@ -161,9 +190,9 @@ pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
transient: Some(TransientKind::Spawning),
nodes: vec![
node(agent, NodeKind::Provision, Vec::new()),
node(agent, NodeKind::Create, after_ok(0)),
node(agent, NodeKind::WriteDropin, after_ok(1)),
node(agent, NodeKind::Reconcile, after_ok(2)),
child(0, agent, NodeKind::Create, Vec::new()),
child(1, agent, NodeKind::WriteDropin, Vec::new()),
child(1, agent, NodeKind::Reconcile, after_ok(2)),
],
}
}
@ -241,7 +270,7 @@ pub fn validate(spec: &DagSpec) -> Result<()> {
.collect();
for (i, node) in spec.nodes.iter().enumerate() {
for dep in &node.deps {
let Some(&dep_idx) = idx.get(dep.on as usize) else {
let Some(&dep_idx) = usize::try_from(dep.on).ok().and_then(|i| idx.get(i)) else {
bail!(
"dag spec {:?} node {i} depends on unknown node {}",
spec.template,