feat(hive-c0re): replace rebuild queue with generic job-DAG queue
jobs are now DAGs of primitive nodes (prebuild, stop-for-update, swap, reconcile, signal, drain, ...) driven by one scheduler with N build slots + per-agent lifecycle leases. per-agent power intent (wanted up/offline) is durable in agent_power.sqlite; Reconcile nodes converge observed state to it. kills the graceful-stop watcher thread, the deferred-start follow-up, and the cascade pre-enqueue (fan-out on MetaLock completion instead). tracker: #2166
This commit is contained in:
müde
parent
79a3993def
commit
7946e03fde
25 changed files with 3673 additions and 2731 deletions
333
hive-c0re/src/job_queue/templates.rs
Normal file
333
hive-c0re/src/job_queue/templates.rs
Normal file
|
|
@ -0,0 +1,333 @@
|
|||
//! DAG shape builders — every operation as a template over the shared
|
||||
//! node primitives — plus submit-time cycle validation (petgraph is
|
||||
//! confined to this validation; the runtime store stays the plain
|
||||
//! `Vec<Node>` + `deps`).
|
||||
//!
|
||||
//! ```text
|
||||
//! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a)
|
||||
//! graceful-stop(a): [wanted=Offline] Signal(a) → Drain(a) → Reconcile(a)
|
||||
//! restart(a): StopForUpdate(a) → Reconcile(a) (wanted unchanged)
|
||||
//! start(a): [wanted=Up] Reconcile(a)
|
||||
//! stop(a): [wanted=Offline] Reconcile(a)
|
||||
//! spawn(a): [wanted=Up] Create(a) → WriteDropin(a) → Reconcile(a)
|
||||
//! perm-change(a): WritePermFile(a) → «rebuild subgraph»
|
||||
//! meta-update(inp): MetaLock(inp) → «fan-out rebuild(a) per affected a»
|
||||
//! startup sweep: MetaLock(hyperhive, non-fatal) → «fan-out rebuild(stale a)»
|
||||
//! ```
|
||||
|
||||
use anyhow::{Result, bail};
|
||||
|
||||
use super::model::{DagSpec, Dep, DepWhen, NodeKind, NodeSpec, PermPayload, Source, Template};
|
||||
use crate::coordinator::TransientKind;
|
||||
|
||||
/// After-ok edge on the previous node — the common chain link.
|
||||
fn after_ok(on: u32) -> Vec<Dep> {
|
||||
vec![Dep {
|
||||
on,
|
||||
when: DepWhen::AfterOk,
|
||||
}]
|
||||
}
|
||||
|
||||
/// The rebuild node chain. `Reconcile` deps on `Swap` with `AfterAny`:
|
||||
/// it must run even when the profile swap failed, so a previously-up
|
||||
/// agent comes back on its old config (today's recovery-start). This
|
||||
/// is the only `AfterAny` edge in v1.
|
||||
fn rebuild_nodes(relock: bool, base: u32) -> Vec<NodeSpec> {
|
||||
vec![
|
||||
NodeSpec {
|
||||
kind: NodeKind::Prebuild { relock },
|
||||
deps: if base == 0 {
|
||||
Vec::new()
|
||||
} else {
|
||||
after_ok(base - 1)
|
||||
},
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::StopForUpdate,
|
||||
deps: after_ok(base),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Swap,
|
||||
deps: after_ok(base + 1),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: vec![Dep {
|
||||
on: base + 2,
|
||||
when: DepWhen::AfterAny,
|
||||
}],
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
/// One uniform rebuild shape — no `was_running` branch. `StopForUpdate`
|
||||
/// noops when already down; the tail `Reconcile` auto-noops the start
|
||||
/// when `wanted = Offline` (a rebuild of a deliberately-stopped agent
|
||||
/// leaves it stopped). `relock = false` only for meta-update cascade
|
||||
/// children.
|
||||
pub fn rebuild(
|
||||
agent: &str,
|
||||
source: Source,
|
||||
reason: String,
|
||||
parent_id: Option<u64>,
|
||||
relock: bool,
|
||||
) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::Rebuild,
|
||||
agent: agent.to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: Some(TransientKind::Rebuilding),
|
||||
nodes: rebuild_nodes(relock, 0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Approval-driven deploy (`ApplyCommit` / `MergeConfigPr`): the whole
|
||||
/// two-phase pipeline stays one opaque node in v1 (design doc §9) —
|
||||
/// wire-visible as a `rebuild` card like today.
|
||||
pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::Rebuild,
|
||||
agent: agent.to_owned(),
|
||||
source: Source::Approval,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: Some(approval_id),
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: Some(TransientKind::Rebuilding),
|
||||
nodes: vec![NodeSpec {
|
||||
kind: NodeKind::ApprovalDeploy,
|
||||
deps: Vec::new(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
/// Graceful stop: cheap `Signal` fires immediately (no build slot), the
|
||||
/// `Drain` awaits the harness checkpoint (bounded), and the tail
|
||||
/// `Reconcile` performs the actual container stop — the caller sets
|
||||
/// `wanted = Offline` at submit time. A whole-hive graceful stop
|
||||
/// therefore signals every agent up front and overlaps every drain,
|
||||
/// replacing the old detached-watcher thread structurally.
|
||||
pub fn graceful_stop(agent: &str, source: Source, reason: String) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::GracefulStop,
|
||||
agent: agent.to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: Some(TransientKind::Stopping),
|
||||
nodes: vec![
|
||||
NodeSpec {
|
||||
kind: NodeKind::Signal,
|
||||
deps: Vec::new(),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Drain,
|
||||
deps: after_ok(0),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: after_ok(1),
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/// Restart: mechanical stop, then converge back to `wanted`
|
||||
/// (unchanged) — a stop + start for a wanted-up agent.
|
||||
pub fn restart(agent: &str, source: Source, reason: String) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::Restart,
|
||||
agent: agent.to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: Some(TransientKind::Restarting),
|
||||
nodes: vec![
|
||||
NodeSpec {
|
||||
kind: NodeKind::StopForUpdate,
|
||||
deps: Vec::new(),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: after_ok(0),
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/// Single-`Reconcile` DAG: `Start` / `Stop` (caller writes `wanted`
|
||||
/// first) and the boot-time `Reconcile` converge (wanted untouched).
|
||||
pub fn reconcile_only(
|
||||
template: Template,
|
||||
agent: &str,
|
||||
source: Source,
|
||||
reason: String,
|
||||
transient: Option<TransientKind>,
|
||||
) -> DagSpec {
|
||||
DagSpec {
|
||||
template,
|
||||
agent: agent.to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient,
|
||||
nodes: vec![NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: Vec::new(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
/// First-deploy spawn (approval-driven): pre-start provisioning +
|
||||
/// `nixos-container create`, drop-in write, then `Reconcile` starts the
|
||||
/// container (`wanted = Up` written at approve time).
|
||||
pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::Spawn,
|
||||
agent: agent.to_owned(),
|
||||
source: Source::Approval,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: Some(approval_id),
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: Some(TransientKind::Spawning),
|
||||
nodes: vec![
|
||||
NodeSpec {
|
||||
kind: NodeKind::Create,
|
||||
deps: Vec::new(),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::WriteDropin,
|
||||
deps: after_ok(0),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: after_ok(1),
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/// Perm change: commit the JSON file(s), then the rebuild subgraph so
|
||||
/// the updated `HIVE_TOOL_GROUPS` / `HIVE_CAPABILITIES` env var takes
|
||||
/// effect in the container.
|
||||
pub fn perm_change(agent: &str, source: Source, reason: String, payload: PermPayload) -> DagSpec {
|
||||
let mut nodes = vec![NodeSpec {
|
||||
kind: NodeKind::WritePermFile,
|
||||
deps: Vec::new(),
|
||||
}];
|
||||
nodes.extend(rebuild_nodes(true, 1));
|
||||
DagSpec {
|
||||
template: Template::PermChange,
|
||||
agent: agent.to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: Some(payload),
|
||||
transient: Some(TransientKind::Rebuilding),
|
||||
nodes,
|
||||
}
|
||||
}
|
||||
|
||||
/// Meta-input lock bump. Child `Rebuild` DAGs fan out on completion —
|
||||
/// appended *after* the bump lands so their prebuilds run against the
|
||||
/// post-bump lock (and so a failed bump simply fans out nothing,
|
||||
/// replacing the old pre-enqueue + `cancel_children` dance).
|
||||
pub fn meta_update(
|
||||
inputs: Vec<String>,
|
||||
source: Source,
|
||||
reason: String,
|
||||
approval_id: Option<i64>,
|
||||
) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::MetaUpdate,
|
||||
agent: "hyperhive".to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id,
|
||||
inputs,
|
||||
perm_payload: None,
|
||||
transient: None,
|
||||
nodes: vec![NodeSpec {
|
||||
kind: NodeKind::MetaLock {
|
||||
sweep: false,
|
||||
fanout: None,
|
||||
},
|
||||
deps: Vec::new(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
/// Boot-time sweep parent: bump meta's hyperhive input (non-fatal),
|
||||
/// then fan out `Rebuild` children for the precomputed stale agent
|
||||
/// list (topology-sorted by the caller).
|
||||
pub fn startup_sweep(reason: String, stale_agents: Vec<String>) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::StartupSweep,
|
||||
agent: "hyperhive".to_owned(),
|
||||
source: Source::AutoUpdate,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: None,
|
||||
nodes: vec![NodeSpec {
|
||||
kind: NodeKind::MetaLock {
|
||||
sweep: true,
|
||||
fanout: Some(stale_agents),
|
||||
},
|
||||
deps: Vec::new(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate a spec before it enters the queue: node ids are dense
|
||||
/// (index = id), deps reference existing nodes, and the dep graph is
|
||||
/// acyclic (petgraph `toposort`). Rejecting cycles here fixes the old
|
||||
/// queue's documented "circular dep silently deadlocks forever" caveat.
|
||||
pub fn validate(spec: &DagSpec) -> Result<()> {
|
||||
if spec.nodes.is_empty() {
|
||||
bail!("dag spec {:?} has no nodes", spec.template);
|
||||
}
|
||||
let n = spec.nodes.len();
|
||||
let mut graph = petgraph::graph::DiGraph::<u32, ()>::new();
|
||||
let idx: Vec<_> = (0..n)
|
||||
.map(|i| graph.add_node(u32::try_from(i).unwrap_or(u32::MAX)))
|
||||
.collect();
|
||||
for (i, node) in spec.nodes.iter().enumerate() {
|
||||
for dep in &node.deps {
|
||||
let Some(&dep_idx) = idx.get(dep.on as usize) else {
|
||||
bail!(
|
||||
"dag spec {:?} node {i} depends on unknown node {}",
|
||||
spec.template,
|
||||
dep.on
|
||||
);
|
||||
};
|
||||
graph.add_edge(dep_idx, idx[i], ());
|
||||
}
|
||||
}
|
||||
if petgraph::algo::toposort(&graph, None).is_err() {
|
||||
bail!("dag spec {:?} contains a dependency cycle", spec.template);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Loading…
Reference in a new issue