feat(#2591): port hive-c0re job_queue onto the hive-jobq crate
Replace the in-tree scheduler with the domain-agnostic hive-jobq crate (merged in #2615): parent-axis grouping + borrow/subtree-reservation resource model + roll-up completion (State::Finishing). Host adaptation: - NodeSpec gains an explicit `parent` axis; templates declare grouping + sibling ordering directly (deps order execution, parent groups a subtree whose resource the descendants borrow). - Rebuild is a nested two-root subtree: Prebuild (root, owns the build slot for the whole subtree, lease-exempt) -> StopForUpdate (child, owns the agent lease) -> Swap/PostSwap (children, borrow both); Reconcile is a separate top-level root (AfterAny Prebuild) so it survives the cancel- cascade of any failed step (recovery-start invariant) and converges to the persisted `wanted` on a fresh lease. This is the multi-root correction to the single-root-chain sketch: node0=root broke lease- exemption (hoisting the lease onto Prebuild) and recovery-reconcile (root failure cancels all children). - Spawn / perm-change / power-ops (stop/start/restart) group-rooted the same way; per-agent power-op subgraphs stay independent roots so a multi-agent DAG runs them concurrently, each on its own lease. - insert_group honours the explicit parent axis (no lease hoisting); the DAG terminal node deps AfterAny on every group root and runs once the whole op rolls up. Drop the old Graph::add_dep terminal wiring. 36/36 job_queue tests, full hive-c0re suite green, clippy --all-targets.
This commit is contained in:
parent
e58457da17
commit
a5c321a1a0
14 changed files with 1111 additions and 893 deletions
|
|
@ -35,34 +35,58 @@ use crate::coordinator::TransientKind;
|
|||
/// After-ok edge on the previous node — the common chain link. Shared with
|
||||
/// the async power-op builders in `submit.rs` (which assemble per-agent
|
||||
/// chains dynamically from live container state).
|
||||
pub(crate) fn after_ok(on: u32) -> Vec<Dep> {
|
||||
pub(crate) fn after_ok(on: u64) -> Vec<Dep> {
|
||||
vec![Dep {
|
||||
on,
|
||||
when: DepWhen::AfterOk,
|
||||
}]
|
||||
}
|
||||
|
||||
/// Build one node targeting `agent`. The single place a node's agent is
|
||||
/// stamped. Shared with `submit.rs`'s dynamic power-op builders.
|
||||
/// Build one **top-level (group-root)** node targeting `agent` — `parent =
|
||||
/// None`. The single place a node's agent is stamped. Shared with `submit.rs`'s
|
||||
/// dynamic power-op builders. A root owns whatever resource it declares for its
|
||||
/// whole subtree; its descendants borrow it (agent-lease / build-slot
|
||||
/// continuity). Ordering vs other nodes is `deps`; grouping is `parent`.
|
||||
pub(crate) fn node(agent: &str, kind: NodeKind, deps: Vec<Dep>) -> NodeSpec {
|
||||
NodeSpec {
|
||||
agent: agent.to_owned(),
|
||||
kind,
|
||||
deps,
|
||||
parent: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The rebuild node chain. `PostSwap` carries the swap's Ok-only
|
||||
/// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan) and deps
|
||||
/// `Swap` with `AfterOk`. `Reconcile` then deps on `PostSwap` with
|
||||
/// `AfterAny`: it must run even when the swap failed, so a previously-up
|
||||
/// agent comes back on its old config (today's recovery-start). On swap
|
||||
/// failure the `AfterOk` `PostSwap` is cancel-cascaded to a terminal state,
|
||||
/// which still satisfies `Reconcile`'s `AfterAny` edge — the only `AfterAny`
|
||||
/// edge in v1. Pointing `Reconcile` at `PostSwap` (not `Swap`) also
|
||||
/// serializes the tail ahead of the reconcile, so there's no double
|
||||
/// rescan/kick race.
|
||||
pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u32) -> Vec<NodeSpec> {
|
||||
/// Build a **child** node whose structural parent is spec-index `parent`. The
|
||||
/// child runs once its parent reaches `Finishing` (the parent gate), so it must
|
||||
/// NOT `deps` on `parent` (dep-scope validation rejects a dep on one's own
|
||||
/// parent). `deps` here order the child against its *siblings* only.
|
||||
pub(crate) fn child(parent: u64, agent: &str, kind: NodeKind, deps: Vec<Dep>) -> NodeSpec {
|
||||
NodeSpec {
|
||||
agent: agent.to_owned(),
|
||||
kind,
|
||||
deps,
|
||||
parent: Some(parent),
|
||||
}
|
||||
}
|
||||
|
||||
/// The rebuild node subtree (nested, two group roots). `base` is the spec index
|
||||
/// of the first node (`Prebuild`). Structure:
|
||||
/// - `Prebuild` (base+0, **root**): owns the build slot for the whole subtree.
|
||||
/// Lease-exempt — the nix build overlaps other DAGs on the same agent.
|
||||
/// - `StopForUpdate` (base+1, child of `Prebuild`): owns the agent lease. Runs
|
||||
/// once `Prebuild` reaches `Finishing` (parent gate).
|
||||
/// - `Swap` (base+2, child of `StopForUpdate`): borrows the agent lease from its
|
||||
/// parent and the build slot from grand-ancestor `Prebuild` — both continuous.
|
||||
/// - `PostSwap` (base+3, child of `StopForUpdate`): the swap's Ok-only
|
||||
/// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan), `AfterOk`
|
||||
/// its sibling `Swap`.
|
||||
/// - `Reconcile` (base+4, **root**): `AfterAny` `Prebuild`, which rolls up
|
||||
/// terminal only once its whole mechanical subtree (SFU→Swap→PostSwap) has
|
||||
/// settled — so `Reconcile` runs after the swap regardless of outcome, and as
|
||||
/// a top-level root it survives the cancel-cascade of a failed `Prebuild`
|
||||
/// (recovery-start invariant). It takes a fresh lease; the tiny gap is
|
||||
/// harmless — `Reconcile` converges to the persisted `wanted` idempotently.
|
||||
pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u64) -> Vec<NodeSpec> {
|
||||
vec![
|
||||
node(
|
||||
agent,
|
||||
|
|
@ -73,14 +97,14 @@ pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u32) -> Vec<NodeSpe
|
|||
after_ok(base - 1)
|
||||
},
|
||||
),
|
||||
node(agent, NodeKind::StopForUpdate, after_ok(base)),
|
||||
node(agent, NodeKind::Swap, after_ok(base + 1)),
|
||||
node(agent, NodeKind::PostSwap, after_ok(base + 2)),
|
||||
child(base, agent, NodeKind::StopForUpdate, Vec::new()),
|
||||
child(base + 1, agent, NodeKind::Swap, Vec::new()),
|
||||
child(base + 1, agent, NodeKind::PostSwap, after_ok(base + 2)),
|
||||
node(
|
||||
agent,
|
||||
NodeKind::Reconcile,
|
||||
vec![Dep {
|
||||
on: base + 3,
|
||||
on: base,
|
||||
when: DepWhen::AfterAny,
|
||||
}],
|
||||
),
|
||||
|
|
@ -149,7 +173,12 @@ pub fn reconcile_only(
|
|||
/// First-deploy spawn (approval-driven): `Provision` (proposed/applied
|
||||
/// repos, state subvolume, meta registration) then `Create`
|
||||
/// (`nixos-container create`), drop-in write, then `Reconcile` starts
|
||||
/// the container (`wanted = Up` written at approve time).
|
||||
/// the container (`wanted = Up` written at approve time). All-or-nothing:
|
||||
/// `Provision` (lease-exempt, precedes the container) is the group root;
|
||||
/// `Create` (child) owns the agent lease; `WriteDropin` + `Reconcile`
|
||||
/// (children of `Create`) borrow it. A failure cancel-cascades the rest —
|
||||
/// unlike rebuild there's no recovery-reconcile (nothing to converge if the
|
||||
/// container was never created).
|
||||
pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::Spawn,
|
||||
|
|
@ -161,9 +190,9 @@ pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
|
|||
transient: Some(TransientKind::Spawning),
|
||||
nodes: vec![
|
||||
node(agent, NodeKind::Provision, Vec::new()),
|
||||
node(agent, NodeKind::Create, after_ok(0)),
|
||||
node(agent, NodeKind::WriteDropin, after_ok(1)),
|
||||
node(agent, NodeKind::Reconcile, after_ok(2)),
|
||||
child(0, agent, NodeKind::Create, Vec::new()),
|
||||
child(1, agent, NodeKind::WriteDropin, Vec::new()),
|
||||
child(1, agent, NodeKind::Reconcile, after_ok(2)),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
|
@ -241,7 +270,7 @@ pub fn validate(spec: &DagSpec) -> Result<()> {
|
|||
.collect();
|
||||
for (i, node) in spec.nodes.iter().enumerate() {
|
||||
for dep in &node.deps {
|
||||
let Some(&dep_idx) = idx.get(dep.on as usize) else {
|
||||
let Some(&dep_idx) = usize::try_from(dep.on).ok().and_then(|i| idx.get(i)) else {
|
||||
bail!(
|
||||
"dag spec {:?} node {i} depends on unknown node {}",
|
||||
spec.template,
|
||||
|
|
|
|||
Loading…
Reference in a new issue