feat(hive-c0re): replace rebuild queue with generic job-DAG queue
jobs are now DAGs of primitive nodes (prebuild, stop-for-update, swap, reconcile, signal, drain, ...) driven by one scheduler with N build slots + per-agent lifecycle leases. per-agent power intent (wanted up/offline) is durable in agent_power.sqlite; Reconcile nodes converge observed state to it. kills the graceful-stop watcher thread, the deferred-start follow-up, and the cascade pre-enqueue (fan-out on MetaLock completion instead). tracker: #2166
This commit is contained in:
müde
parent
79a3993def
commit
7946e03fde
25 changed files with 3673 additions and 2731 deletions
433
hive-c0re/src/job_queue/exec.rs
Normal file
433
hive-c0re/src/job_queue/exec.rs
Normal file
|
|
@ -0,0 +1,433 @@
|
|||
//! Node executors — one async fn per [`NodeKind`], each a thin wrapper
|
||||
//! over existing `lifecycle.rs` / `meta.rs` / `actions.rs` code. Node
|
||||
//! executors keep their own internal error handling where it exists
|
||||
//! today (cold-start fallback inside `Reconcile`, non-fatal boot-time
|
||||
//! lock bump inside the sweep `MetaLock`, warn-only forge sync in the
|
||||
//! `Swap` tail); DAG-level failure handling is cancel-downstream in
|
||||
//! the queue.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
|
||||
use super::model::{NodeKind, State, Template};
|
||||
use super::{Claim, TerminalDag};
|
||||
use crate::coordinator::Coordinator;
|
||||
use crate::power::{ReconcileAction, reconcile_action};
|
||||
|
||||
/// Max time `Drain` waits for the harness to run its stop-checkpoint
|
||||
/// turn before falling back to the hard stop. Generous — a checkpoint
|
||||
/// turn can take a while — but bounded so a wedged agent never blocks
|
||||
/// the stop indefinitely. Drains hold no build slot, so a whole-hive
|
||||
/// graceful stop overlaps every agent's drain instead of serialising
|
||||
/// N × this timeout.
|
||||
pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(3);
|
||||
|
||||
/// Extra signal an executor hands back to the scheduler alongside
|
||||
/// success.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct NodeOutput {
|
||||
/// Agents to fan child `Rebuild` DAGs out for (`MetaLock` only).
|
||||
pub fanout: Vec<String>,
|
||||
}
|
||||
|
||||
/// Step-label + build-log sink for one claimed node.
|
||||
struct Ctx<'a> {
|
||||
coord: &'a Arc<Coordinator>,
|
||||
dag_id: u64,
|
||||
node_id: super::NodeId,
|
||||
}
|
||||
|
||||
impl Ctx<'_> {
|
||||
fn step(&self, step: &str) {
|
||||
if self
|
||||
.coord
|
||||
.job_queue
|
||||
.set_step(self.dag_id, self.node_id, step)
|
||||
{
|
||||
self.coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
}
|
||||
|
||||
fn build_log(&self, log_id: i64) {
|
||||
if self
|
||||
.coord
|
||||
.job_queue
|
||||
.set_build_log_id(self.dag_id, self.node_id, log_id)
|
||||
{
|
||||
self.coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Run one claimed node to completion. Called from a task the
|
||||
/// scheduler spawns per claim; the `Result` (stringified) becomes the
|
||||
/// node's terminal state.
|
||||
pub(super) async fn run_node(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
|
||||
let ctx = Ctx {
|
||||
coord,
|
||||
dag_id: claim.dag_id,
|
||||
node_id: claim.node_id,
|
||||
};
|
||||
match &claim.kind {
|
||||
NodeKind::Prebuild { relock } => run_prebuild(coord, claim, &ctx, *relock).await,
|
||||
NodeKind::Swap => run_swap(coord, claim, &ctx).await,
|
||||
NodeKind::Create => run_create(coord, claim, &ctx).await,
|
||||
NodeKind::MetaLock { sweep, fanout } => {
|
||||
run_meta_lock(coord, claim, &ctx, *sweep, fanout.clone()).await
|
||||
}
|
||||
NodeKind::Reconcile => run_reconcile(coord, claim, &ctx).await,
|
||||
NodeKind::StopForUpdate => run_stop_for_update(coord, claim, &ctx).await,
|
||||
NodeKind::Signal => Ok(run_signal(coord, claim, &ctx)),
|
||||
NodeKind::Drain => run_drain(coord, claim, &ctx).await,
|
||||
NodeKind::WriteDropin => run_write_dropin(coord, claim).await,
|
||||
NodeKind::WritePermFile => run_write_perm_file(coord, claim, &ctx).await,
|
||||
NodeKind::ApprovalDeploy => run_approval_deploy(coord, claim).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Out-of-band toplevel build while the container keeps serving: meta
|
||||
/// sync + optional per-agent relock, then warm
|
||||
/// `system.build.toplevel` so the later `Swap` hits cache and skips
|
||||
/// straight to the profile-swap.
|
||||
async fn run_prebuild(
|
||||
coord: &Arc<Coordinator>,
|
||||
claim: &Claim,
|
||||
ctx: &Ctx<'_>,
|
||||
relock: bool,
|
||||
) -> Result<NodeOutput> {
|
||||
let name = &claim.agent;
|
||||
let agent_dir = coord
|
||||
.ensure_runtime(name)
|
||||
.with_context(|| format!("ensure_runtime {name}"))?;
|
||||
let hive = coord.hive_env();
|
||||
let paths = Coordinator::agent_paths(name, agent_dir);
|
||||
crate::lifecycle::prepare_rebuild_dirs(name, &paths).await?;
|
||||
// Idempotent meta sync so a manual rebuild can also recover from a
|
||||
// divergent meta repo; then bump just this agent's input. `relock =
|
||||
// false` only for meta-update cascade children, where re-locking
|
||||
// would revert the bump the cascade just committed.
|
||||
let agents = crate::lifecycle::agents_for_meta_listing().await?;
|
||||
crate::meta::sync_agents(&hive, &agents).await?;
|
||||
if relock {
|
||||
crate::meta::lock_update_for_rebuild(name).await?;
|
||||
}
|
||||
ctx.step("nix build");
|
||||
let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display());
|
||||
crate::lifecycle::prebuild_toplevel(name, &flake_ref, &|log_id| ctx.build_log(log_id)).await?;
|
||||
Ok(NodeOutput::default())
|
||||
}
|
||||
|
||||
/// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb),
|
||||
/// `nixos-container update`, then the post-rebuild bookkeeping tail
|
||||
/// (rev marker, `Rebuilt` event, forge/matrix sync, kick, rescan).
|
||||
/// The recovery-start on failure is NOT here — the DAG's tail
|
||||
/// `Reconcile` runs after this node terminal ok *or* fail.
|
||||
async fn run_swap(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
|
||||
let name = &claim.agent;
|
||||
let agent_dir = coord.ensure_runtime(name)?;
|
||||
let hive = coord.hive_env();
|
||||
let paths = Coordinator::agent_paths(name, agent_dir);
|
||||
let result =
|
||||
crate::lifecycle::swap_update(name, &hive, &paths, &|step| ctx.step(step), &|log_id| {
|
||||
ctx.build_log(log_id)
|
||||
})
|
||||
.await;
|
||||
match &result {
|
||||
Ok(()) => {
|
||||
if let Some(rev) = crate::auto_update::current_flake_rev(&coord.hyperhive_flake)
|
||||
&& let Err(e) = std::fs::write(crate::auto_update::rev_marker_path(name), rev)
|
||||
{
|
||||
tracing::warn!(%name, error = ?e, "write rev marker failed");
|
||||
}
|
||||
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
|
||||
agent: name.clone(),
|
||||
ok: true,
|
||||
note: None,
|
||||
sha: None,
|
||||
tag: None,
|
||||
});
|
||||
ctx.step("forge sync");
|
||||
// Full forge + matrix sync on every successful rebuild so
|
||||
// the rebuild path is equivalent to the startup sweep:
|
||||
// tokens, config-repo mirror, meta access all recover
|
||||
// without a hive-c0re restart.
|
||||
crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await;
|
||||
crate::matrix::sync_agent_standalone(name).await;
|
||||
// Wake the agent on its next turn so claude sees a "you
|
||||
// were rebuilt" hint; rescan so dashboards drop the
|
||||
// "needs update" chip; lock bump → meta-inputs re-render.
|
||||
coord.kick_agent(name, "container rebuilt");
|
||||
coord.rescan_containers_and_emit().await;
|
||||
crate::dashboard::emit_meta_inputs_snapshot(coord);
|
||||
}
|
||||
Err(_) => {
|
||||
// The `Rebuilt { ok: false }` manager event fires once per
|
||||
// DAG from the terminal hook (any node may be the one that
|
||||
// failed); here only refresh the observed state.
|
||||
coord.rescan_containers_and_emit().await;
|
||||
}
|
||||
}
|
||||
result.map(|()| NodeOutput::default())
|
||||
}
|
||||
|
||||
/// First-spawn provisioning + `nixos-container create` (atomic
|
||||
/// build+create — no prebuild needed).
|
||||
async fn run_create(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
|
||||
let name = &claim.agent;
|
||||
let agent_dir = coord.ensure_runtime(name)?;
|
||||
let hive = coord.hive_env();
|
||||
let paths = Coordinator::agent_paths(name, agent_dir);
|
||||
ctx.step("nixos-container create");
|
||||
crate::lifecycle::create_container(name, &hive, &paths).await?;
|
||||
Ok(NodeOutput::default())
|
||||
}
|
||||
|
||||
/// Meta flake lock bump. Boot-sweep flavour is non-fatal (a failed
|
||||
/// bump must not cancel the fan-out rebuilds — they proceed against
|
||||
/// the current lock, exactly like today's sweep); the meta-update
|
||||
/// flavour propagates errors, and a failed bump fans out nothing.
|
||||
async fn run_meta_lock(
|
||||
coord: &Arc<Coordinator>,
|
||||
claim: &Claim,
|
||||
ctx: &Ctx<'_>,
|
||||
sweep: bool,
|
||||
fanout: Option<Vec<String>>,
|
||||
) -> Result<NodeOutput> {
|
||||
if sweep {
|
||||
ctx.step("nix flake update hyperhive");
|
||||
if let Err(e) = crate::meta::lock_update_hyperhive().await {
|
||||
tracing::warn!(error = ?e, "startup sweep: meta lock_update_hyperhive failed");
|
||||
}
|
||||
return Ok(NodeOutput {
|
||||
fanout: fanout.unwrap_or_default(),
|
||||
});
|
||||
}
|
||||
let _progress = coord.meta_update_guard();
|
||||
ctx.step("nix flake update");
|
||||
crate::meta::lock_update(&claim.inputs).await?;
|
||||
// Lock file changed — meta-inputs panel re-renders.
|
||||
crate::dashboard::emit_meta_inputs_snapshot(coord);
|
||||
let cascade = match fanout {
|
||||
Some(list) => list,
|
||||
None => meta_update_cascade_agents(&claim.inputs).await,
|
||||
};
|
||||
Ok(NodeOutput { fanout: cascade })
|
||||
}
|
||||
|
||||
/// Idempotent power converge: `wanted` (durable intent) vs observed.
|
||||
async fn run_reconcile(
|
||||
coord: &Arc<Coordinator>,
|
||||
claim: &Claim,
|
||||
ctx: &Ctx<'_>,
|
||||
) -> Result<NodeOutput> {
|
||||
let name = &claim.agent;
|
||||
let running = crate::lifecycle::is_running(name).await;
|
||||
let wanted = coord.power.get_or_seed(name, running)?;
|
||||
match reconcile_action(wanted, running) {
|
||||
ReconcileAction::Start => {
|
||||
// Node-local transient only when the DAG holds none (the
|
||||
// boot-reconcile template); lease-window guards otherwise
|
||||
// already cover this node.
|
||||
let _guard = claim
|
||||
.transient
|
||||
.is_none()
|
||||
.then(|| coord.transient_guard(name, crate::coordinator::TransientKind::Starting));
|
||||
ctx.step("nixos-container start");
|
||||
crate::lifecycle::start_with_fallback(name).await?;
|
||||
coord.kick_agent(name, "container started");
|
||||
coord.rescan_containers_and_emit().await;
|
||||
}
|
||||
ReconcileAction::Stop => {
|
||||
let _guard = claim
|
||||
.transient
|
||||
.is_none()
|
||||
.then(|| coord.transient_guard(name, crate::coordinator::TransientKind::Stopping));
|
||||
ctx.step("nixos-container stop");
|
||||
crate::lifecycle::kill(name).await?;
|
||||
coord.unregister_agent(name);
|
||||
coord.notify_manager(&hive_sh4re::HelperEvent::Killed {
|
||||
agent: name.clone(),
|
||||
});
|
||||
coord.rescan_containers_and_emit().await;
|
||||
}
|
||||
ReconcileAction::Noop => {
|
||||
tracing::debug!(%name, wanted = wanted.as_str(), running, "reconcile: noop");
|
||||
}
|
||||
}
|
||||
Ok(NodeOutput::default())
|
||||
}
|
||||
|
||||
/// Mechanical stop for the profile swap. Never touches `wanted`; noop
|
||||
/// when already stopped.
|
||||
async fn run_stop_for_update(
|
||||
coord: &Arc<Coordinator>,
|
||||
claim: &Claim,
|
||||
ctx: &Ctx<'_>,
|
||||
) -> Result<NodeOutput> {
|
||||
let name = &claim.agent;
|
||||
if crate::lifecycle::is_running(name).await {
|
||||
ctx.step("nixos-container stop");
|
||||
crate::lifecycle::kill(name).await?;
|
||||
coord.rescan_containers_and_emit().await;
|
||||
}
|
||||
Ok(NodeOutput::default())
|
||||
}
|
||||
|
||||
/// Set the graceful fence + kick so the harness sees it promptly and
|
||||
/// runs its one stop-checkpoint turn.
|
||||
fn run_signal(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> NodeOutput {
|
||||
ctx.step("graceful stop: signalling agent");
|
||||
coord.mark_graceful_stop(&claim.agent);
|
||||
coord.kick_agent(&claim.agent, "graceful stop requested");
|
||||
NodeOutput::default()
|
||||
}
|
||||
|
||||
/// Await the harness clearing the fence (`GracefulStopComplete`) or
|
||||
/// the timeout — either way the downstream `Reconcile` proceeds with
|
||||
/// the actual stop.
|
||||
async fn run_drain(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
|
||||
let name = &claim.agent;
|
||||
ctx.step("graceful stop: draining");
|
||||
let deadline = std::time::Instant::now() + GRACEFUL_STOP_TIMEOUT;
|
||||
while coord.is_graceful_stop_pending(name) {
|
||||
if std::time::Instant::now() >= deadline {
|
||||
tracing::warn!(agent = %name, "graceful stop: drain timed out — hard-stopping");
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||
}
|
||||
coord.clear_graceful_stop(name);
|
||||
Ok(NodeOutput::default())
|
||||
}
|
||||
|
||||
/// `set_nspawn_flags` + `set_resource_limits` + daemon-reload.
|
||||
async fn run_write_dropin(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
|
||||
let name = &claim.agent;
|
||||
let agent_dir = coord.ensure_runtime(name)?;
|
||||
let hive = coord.hive_env();
|
||||
let paths = Coordinator::agent_paths(name, agent_dir);
|
||||
crate::lifecycle::write_dropins(name, &hive, &paths).await?;
|
||||
Ok(NodeOutput::default())
|
||||
}
|
||||
|
||||
/// Write + commit the perm file(s) (fused under `META_LOCK` so the
|
||||
/// working tree is never left dirty), then emit the P3RM1SS10NS-tab
|
||||
/// snapshots so the dashboard reflects the new assignment.
|
||||
async fn run_write_perm_file(
|
||||
coord: &Arc<Coordinator>,
|
||||
claim: &Claim,
|
||||
ctx: &Ctx<'_>,
|
||||
) -> Result<NodeOutput> {
|
||||
use super::model::PermPayload;
|
||||
let name = &claim.agent;
|
||||
ctx.step("writing + committing perm file");
|
||||
match &claim.perm_payload {
|
||||
Some(PermPayload::ToolGroups { groups }) => {
|
||||
crate::meta::commit_tool_groups(name, groups)
|
||||
.await
|
||||
.with_context(|| format!("commit tool-groups for {name}"))?;
|
||||
coord.emit_tool_groups_snapshot();
|
||||
}
|
||||
Some(PermPayload::Capabilities { caps }) => {
|
||||
crate::meta::commit_capabilities(name, caps)
|
||||
.await
|
||||
.with_context(|| format!("commit capabilities for {name}"))?;
|
||||
coord.emit_capabilities_snapshot();
|
||||
}
|
||||
Some(PermPayload::Combined { groups, caps }) => {
|
||||
crate::meta::commit_perms(name, groups.as_deref(), caps.as_deref())
|
||||
.await
|
||||
.with_context(|| format!("commit perms for {name}"))?;
|
||||
if groups.is_some() {
|
||||
coord.emit_tool_groups_snapshot();
|
||||
}
|
||||
if caps.is_some() {
|
||||
coord.emit_capabilities_snapshot();
|
||||
}
|
||||
}
|
||||
None => anyhow::bail!(
|
||||
"perm_change dag {} for {name} is missing perm_payload",
|
||||
claim.dag_id
|
||||
),
|
||||
}
|
||||
Ok(NodeOutput::default())
|
||||
}
|
||||
|
||||
/// Opaque approval deploy pipeline: `ApplyCommit` and `MergeConfigPr`
|
||||
/// both end in a container rebuild; branch on the approval row's kind
|
||||
/// (the authoritative source). The two-phase prepare/finalize/abort
|
||||
/// meta deploy — and the approval resolution — stay inside
|
||||
/// `actions.rs` in v1 (design doc §9).
|
||||
async fn run_approval_deploy(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
|
||||
let approval_id = claim
|
||||
.approval_id
|
||||
.with_context(|| format!("approval_deploy dag {} has no approval_id", claim.dag_id))?;
|
||||
let kind = coord
|
||||
.approvals
|
||||
.get(approval_id)
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|a| a.kind);
|
||||
let result = if kind == Some(hive_sh4re::ApprovalKind::MergeConfigPr) {
|
||||
crate::actions::run_approval_merge_config_pr(coord, Some(claim.dag_id), approval_id).await
|
||||
} else {
|
||||
crate::actions::run_approval_apply_commit(coord, Some(claim.dag_id), approval_id).await
|
||||
};
|
||||
result.map(|()| NodeOutput::default())
|
||||
}
|
||||
|
||||
/// Terminal-roll-up hook, fired exactly once per DAG. Approval DAGs
|
||||
/// resolve their approval row (except the opaque deploy pipeline,
|
||||
/// which resolves inside its node); non-approval rebuild-shaped DAGs
|
||||
/// surface the `Rebuilt { ok: false }` manager event on failure —
|
||||
/// success fires from the `Swap` tail, matching today's timing.
|
||||
pub(super) async fn on_dag_terminal(coord: &Arc<Coordinator>, terminal: &TerminalDag) {
|
||||
if terminal.approval_id.is_some() {
|
||||
crate::actions::resolve_approval_dag(coord, terminal).await;
|
||||
return;
|
||||
}
|
||||
if matches!(terminal.template, Template::Rebuild | Template::PermChange)
|
||||
&& terminal.state == State::Failed
|
||||
{
|
||||
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
|
||||
agent: terminal.agent.clone(),
|
||||
ok: false,
|
||||
note: terminal.error.clone(),
|
||||
sha: None,
|
||||
tag: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute which agents a `nix flake update <inputs>` on the meta
|
||||
/// flake affects — the fan-out set for `MetaUpdate` DAGs. Empty
|
||||
/// `inputs` or any input under `hyperhive` → every container;
|
||||
/// otherwise just the agents named by `agent-<name>` inputs.
|
||||
/// Topology-sorted so parents rebuild before their children.
|
||||
pub async fn meta_update_cascade_agents(inputs: &[String]) -> Vec<String> {
|
||||
let touched_hyperhive = inputs
|
||||
.iter()
|
||||
.any(|i| i == "hyperhive" || i.starts_with("hyperhive/"));
|
||||
let touched_agents: Vec<String> = inputs
|
||||
.iter()
|
||||
.filter_map(|i| i.strip_prefix("agent-"))
|
||||
.map(|rest| rest.split('/').next().unwrap_or(rest).to_owned())
|
||||
.collect();
|
||||
let mut names = if touched_hyperhive || inputs.is_empty() {
|
||||
crate::lifecycle::list()
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter_map(|c| {
|
||||
c.strip_prefix(crate::lifecycle::AGENT_PREFIX)
|
||||
.map(str::to_owned)
|
||||
})
|
||||
.collect()
|
||||
} else {
|
||||
touched_agents
|
||||
};
|
||||
let topo = crate::topology::read();
|
||||
crate::auto_update::topology_sort(&mut names, &topo);
|
||||
names
|
||||
}
|
||||
565
hive-c0re/src/job_queue/mod.rs
Normal file
565
hive-c0re/src/job_queue/mod.rs
Normal file
|
|
@ -0,0 +1,565 @@
|
|||
//! Generic job-DAG queue + desired-state reconciliation — replaces the
|
||||
//! old flat `rebuild_queue`. Jobs are nodes in per-request DAGs (see
|
||||
//! [`templates`]); the special cases (graceful-stop watcher thread,
|
||||
//! deferred-start follow-up, meta-update cascade) collapse into DAG
|
||||
//! *shapes* over a shared set of primitive nodes ([`model::NodeKind`]).
|
||||
//!
|
||||
//! Concurrency is gated by two resource classes:
|
||||
//! 1. **Build slots** — N permits (`services.hyperhive.c0re.buildSlots`,
|
||||
//! default 1) held by nix-heavy nodes for the node's duration.
|
||||
//! 2. **Per-agent lifecycle lease** — DAG-scoped: acquired before the
|
||||
//! DAG's first container-affecting node runs, held until the DAG is
|
||||
//! terminal, so two lifecycle DAGs for one agent never interleave
|
||||
//! their container ops.
|
||||
//!
|
||||
//! The meta *repo* is serialized by `meta::META_LOCK` inside the
|
||||
//! executors themselves. Per-agent power *intent* (`wanted`) lives in
|
||||
//! the durable [`crate::power`] store; the DAGs are the reconcile
|
||||
//! mechanism. Design + rationale: `docs/coordinator.md::Job queue`.
|
||||
|
||||
pub mod exec;
|
||||
pub mod model;
|
||||
pub mod scheduler;
|
||||
pub mod submit;
|
||||
pub mod templates;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use tokio::sync::Notify;
|
||||
|
||||
pub use model::{
|
||||
Dag, DagSpec, DagView, DepWhen, Node, NodeId, NodeKind, PermPayload, Source, State, Template,
|
||||
};
|
||||
|
||||
/// How many terminal DAGs (`Done` / `Failed` / `Cancelled`) to retain
|
||||
/// per template in the snapshot, matching the old per-kind history cap.
|
||||
const MAX_HISTORY_PER_TEMPLATE: usize = 5;
|
||||
|
||||
/// Cap on stored node error strings.
|
||||
const MAX_ERROR_LEN: usize = 2_000;
|
||||
|
||||
/// A node claimed for execution — everything the executor needs,
|
||||
/// snapshotted at claim time.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Claim {
|
||||
pub dag_id: u64,
|
||||
pub node_id: NodeId,
|
||||
pub kind: NodeKind,
|
||||
pub agent: String,
|
||||
pub template: Template,
|
||||
pub source: Source,
|
||||
pub approval_id: Option<i64>,
|
||||
pub inputs: Vec<String>,
|
||||
pub perm_payload: Option<PermPayload>,
|
||||
/// True when claiming this node acquired the DAG's agent lease —
|
||||
/// the scheduler creates the DAG-scoped transient guard on this
|
||||
/// edge.
|
||||
pub lease_acquired: bool,
|
||||
/// Transient pill kind for the lease window (from the spec).
|
||||
pub transient: Option<crate::coordinator::TransientKind>,
|
||||
}
|
||||
|
||||
/// Summary of a DAG that just reached its terminal roll-up state —
|
||||
/// input to the approval-resolution hook and the lease/transient
|
||||
/// release.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TerminalDag {
|
||||
pub dag_id: u64,
|
||||
pub template: Template,
|
||||
pub agent: String,
|
||||
pub approval_id: Option<i64>,
|
||||
pub state: State,
|
||||
/// First failed node's error when `state == Failed`.
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Report from [`JobQueue::complete_node`].
|
||||
#[derive(Debug, Default)]
|
||||
pub struct CompletionReport {
|
||||
/// DAGs that became terminal as a result of this completion
|
||||
/// (the completed node's own DAG, plus none others — but kept as a
|
||||
/// Vec so cancel paths can reuse the same settle plumbing).
|
||||
pub terminal: Vec<TerminalDag>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Inner {
|
||||
dags: VecDeque<Dag>,
|
||||
next_id: u64,
|
||||
build_slots: usize,
|
||||
slots_used: usize,
|
||||
/// agent → dag id currently holding that agent's lifecycle lease.
|
||||
leases: HashMap<String, u64>,
|
||||
}
|
||||
|
||||
/// The queue. Lives on `Coordinator` (one per hive-c0re process); a
|
||||
/// single scheduler task ([`scheduler::run_worker`]) drives it —
|
||||
/// concurrency comes from the build-slot count, not multiple workers.
|
||||
#[derive(Debug)]
|
||||
pub struct JobQueue {
|
||||
inner: Mutex<Inner>,
|
||||
/// Wakes the scheduler when something new arrives or state changed.
|
||||
pub(crate) notify: Notify,
|
||||
}
|
||||
|
||||
impl Default for JobQueue {
|
||||
fn default() -> Self {
|
||||
Self::new(1)
|
||||
}
|
||||
}
|
||||
|
||||
impl JobQueue {
|
||||
pub fn new(build_slots: usize) -> Self {
|
||||
Self {
|
||||
inner: Mutex::new(Inner {
|
||||
build_slots: build_slots.max(1),
|
||||
..Inner::default()
|
||||
}),
|
||||
notify: Notify::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Submit a DAG. Validates the spec (cycle rejection) and dedups
|
||||
/// against non-started DAGs; returns the DAG id (newly-allocated,
|
||||
/// or the existing DAG's id with the new reason appended).
|
||||
///
|
||||
/// Dedup: a DAG whose roll-up is still `Queued` (no node started)
|
||||
/// with the same `(template, agent, parent_id, approval_id)` — plus
|
||||
/// `inputs` for `MetaUpdate` and the perm-type discriminant for
|
||||
/// `PermChange` — swallows the repeat. `parent_id` is part of the
|
||||
/// key so a meta-update cascade rebuild never collapses into a
|
||||
/// standalone or sweep rebuild. Running / terminal DAGs never
|
||||
/// dedup — operators are free to re-queue.
|
||||
pub fn submit(&self, spec: DagSpec) -> anyhow::Result<u64> {
|
||||
templates::validate(&spec)?;
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
if let Some(existing) = Self::dedup_target(&mut inner, &spec) {
|
||||
if !existing.reason.contains(&spec.reason) {
|
||||
use std::fmt::Write as _;
|
||||
let _ = write!(existing.reason, "\nalso requested by: {}", spec.reason);
|
||||
}
|
||||
return Ok(existing.id);
|
||||
}
|
||||
let id = Self::push_dag(&mut inner, spec);
|
||||
drop(inner);
|
||||
self.notify.notify_one();
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Append fan-out children under a parent DAG (meta-update / sweep
|
||||
/// cascade). Applies the same dedup as [`Self::submit`]; returns
|
||||
/// the child ids actually created or coalesced into.
|
||||
pub fn append_children(&self, specs: Vec<DagSpec>) -> Vec<u64> {
|
||||
let mut ids = Vec::with_capacity(specs.len());
|
||||
for spec in specs {
|
||||
match self.submit(spec) {
|
||||
Ok(id) => ids.push(id),
|
||||
Err(e) => tracing::error!(error = ?e, "job_queue: invalid fan-out child spec"),
|
||||
}
|
||||
}
|
||||
ids
|
||||
}
|
||||
|
||||
fn dedup_target<'a>(inner: &'a mut Inner, spec: &DagSpec) -> Option<&'a mut Dag> {
|
||||
inner.dags.iter_mut().find(|d| {
|
||||
d.rollup() == State::Queued
|
||||
&& d.template == spec.template
|
||||
&& d.agent == spec.agent
|
||||
&& d.parent_id == spec.parent_id
|
||||
&& d.approval_id == spec.approval_id
|
||||
&& (d.template != Template::MetaUpdate || d.inputs == spec.inputs)
|
||||
&& PermPayload::same_type(d.perm_payload.as_ref(), spec.perm_payload.as_ref())
|
||||
})
|
||||
}
|
||||
|
||||
fn push_dag(inner: &mut Inner, spec: DagSpec) -> u64 {
|
||||
inner.next_id += 1;
|
||||
let id = inner.next_id;
|
||||
let nodes = spec
|
||||
.nodes
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, n)| Node {
|
||||
id: u32::try_from(i).unwrap_or(u32::MAX),
|
||||
kind: n.kind,
|
||||
deps: n.deps,
|
||||
state: State::Queued,
|
||||
step: None,
|
||||
build_log_id: None,
|
||||
started_at: None,
|
||||
finished_at: None,
|
||||
error: None,
|
||||
})
|
||||
.collect();
|
||||
inner.dags.push_back(Dag {
|
||||
id,
|
||||
template: spec.template,
|
||||
agent: spec.agent,
|
||||
source: spec.source,
|
||||
reason: spec.reason,
|
||||
parent_id: spec.parent_id,
|
||||
approval_id: spec.approval_id,
|
||||
inputs: spec.inputs,
|
||||
perm_payload: spec.perm_payload,
|
||||
transient: spec.transient,
|
||||
created_at: now_unix(),
|
||||
nodes,
|
||||
terminal_reported: false,
|
||||
});
|
||||
id
|
||||
}
|
||||
|
||||
/// Claim every currently-ready node, acquiring resources, and mark
|
||||
/// them `Running`. A node is ready when it's `Queued`, every dep is
|
||||
/// satisfied (`AfterOk`: dep `Done`; `AfterAny`: dep terminal), and
|
||||
/// its resources are free (build slot; agent lease free or already
|
||||
/// held by this DAG). Iteration is in DAG-submit order, so
|
||||
/// simultaneously-ready nodes compete FIFO — bulk operations drain
|
||||
/// predictably.
|
||||
pub fn claim_ready(&self) -> Vec<Claim> {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
Self::propagate_cancellations(&mut inner);
|
||||
let mut claims = Vec::new();
|
||||
let inner = &mut *inner;
|
||||
for di in 0..inner.dags.len() {
|
||||
// Split-borrow dance: deps are checked against the same
|
||||
// DAG's other nodes, so snapshot the states first.
|
||||
let dag = &inner.dags[di];
|
||||
let dag_id = dag.id;
|
||||
let ready_ids: Vec<NodeId> = dag
|
||||
.nodes
|
||||
.iter()
|
||||
.filter(|n| n.state == State::Queued && Self::deps_satisfied(dag, n))
|
||||
.map(|n| n.id)
|
||||
.collect();
|
||||
for node_id in ready_ids {
|
||||
let dag = &inner.dags[di];
|
||||
let node = dag.node(node_id).expect("node id from same dag");
|
||||
let needs_slot = node.kind.needs_build_slot();
|
||||
if needs_slot && inner.slots_used >= inner.build_slots {
|
||||
continue;
|
||||
}
|
||||
let mut lease_acquired = false;
|
||||
if node.kind.needs_lease() {
|
||||
match inner.leases.get(dag.agent.as_str()) {
|
||||
Some(&holder) if holder != dag_id => continue,
|
||||
Some(_) => {}
|
||||
None => {
|
||||
inner.leases.insert(dag.agent.clone(), dag_id);
|
||||
lease_acquired = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if needs_slot {
|
||||
inner.slots_used += 1;
|
||||
}
|
||||
let dag = &mut inner.dags[di];
|
||||
let claim = Claim {
|
||||
dag_id,
|
||||
node_id,
|
||||
kind: dag.node(node_id).expect("node").kind.clone(),
|
||||
agent: dag.agent.clone(),
|
||||
template: dag.template,
|
||||
source: dag.source,
|
||||
approval_id: dag.approval_id,
|
||||
inputs: dag.inputs.clone(),
|
||||
perm_payload: dag.perm_payload.clone(),
|
||||
lease_acquired,
|
||||
transient: dag.transient,
|
||||
};
|
||||
let node = dag.node_mut(node_id).expect("node");
|
||||
node.state = State::Running;
|
||||
node.started_at = Some(now_unix());
|
||||
claims.push(claim);
|
||||
}
|
||||
}
|
||||
claims
|
||||
}
|
||||
|
||||
fn deps_satisfied(dag: &Dag, node: &Node) -> bool {
|
||||
node.deps.iter().all(|dep| {
|
||||
dag.node(dep.on).is_some_and(|d| match dep.when {
|
||||
DepWhen::AfterOk => d.state == State::Done,
|
||||
DepWhen::AfterAny => d.state.is_terminal(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Cancel-downstream: a `Queued` node with an `AfterOk` dep that
|
||||
/// `Failed` / `Cancelled` becomes `Cancelled` itself. Loops to a
|
||||
/// fixpoint so the cancellation cascades through chains.
|
||||
fn propagate_cancellations(inner: &mut Inner) {
|
||||
for dag in &mut inner.dags {
|
||||
loop {
|
||||
let doomed: Vec<NodeId> = dag
|
||||
.nodes
|
||||
.iter()
|
||||
.filter(|n| {
|
||||
n.state == State::Queued
|
||||
&& n.deps.iter().any(|dep| {
|
||||
dep.when == DepWhen::AfterOk
|
||||
&& dag.node(dep.on).is_some_and(|d| {
|
||||
matches!(d.state, State::Failed | State::Cancelled)
|
||||
})
|
||||
})
|
||||
})
|
||||
.map(|n| n.id)
|
||||
.collect();
|
||||
if doomed.is_empty() {
|
||||
break;
|
||||
}
|
||||
let now = now_unix();
|
||||
for id in doomed {
|
||||
if let Some(n) = dag.node_mut(id) {
|
||||
n.state = State::Cancelled;
|
||||
n.finished_at = Some(now);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark a claimed node terminal, release its build slot, cascade
|
||||
/// cancellations, and settle terminal DAGs (lease release + history
|
||||
/// trim). `error` is stored (truncated) when `result` is `Err`.
|
||||
pub fn complete_node(
|
||||
&self,
|
||||
dag_id: u64,
|
||||
node_id: NodeId,
|
||||
result: Result<(), String>,
|
||||
) -> CompletionReport {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
if let Some(dag) = inner.dags.iter_mut().find(|d| d.id == dag_id)
|
||||
&& let Some(node) = dag.node_mut(node_id)
|
||||
&& node.state == State::Running
|
||||
{
|
||||
let needs_slot = node.kind.needs_build_slot();
|
||||
node.finished_at = Some(now_unix());
|
||||
node.step = None;
|
||||
match result {
|
||||
Ok(()) => node.state = State::Done,
|
||||
Err(e) => {
|
||||
node.state = State::Failed;
|
||||
let mut msg = e;
|
||||
if msg.len() > MAX_ERROR_LEN {
|
||||
msg.truncate(
|
||||
(0..=MAX_ERROR_LEN)
|
||||
.rev()
|
||||
.find(|i| msg.is_char_boundary(*i))
|
||||
.unwrap_or(0),
|
||||
);
|
||||
msg.push('…');
|
||||
}
|
||||
node.error = Some(msg);
|
||||
}
|
||||
}
|
||||
if needs_slot {
|
||||
inner.slots_used = inner.slots_used.saturating_sub(1);
|
||||
}
|
||||
}
|
||||
let report = Self::settle(&mut inner);
|
||||
drop(inner);
|
||||
self.notify.notify_one();
|
||||
report
|
||||
}
|
||||
|
||||
/// Propagate cancellations, release the leases of newly-terminal
|
||||
/// DAGs, and trim history. Each terminal DAG is reported exactly
|
||||
/// once (the `terminal_reported` flag) so the scheduler's hooks —
|
||||
/// approval resolution, transient-guard release — fire once per
|
||||
/// DAG.
|
||||
fn settle(inner: &mut Inner) -> CompletionReport {
|
||||
Self::propagate_cancellations(inner);
|
||||
let mut report = CompletionReport::default();
|
||||
let mut freed: Vec<String> = Vec::new();
|
||||
for dag in &mut inner.dags {
|
||||
if !dag.is_terminal() || dag.terminal_reported {
|
||||
continue;
|
||||
}
|
||||
dag.terminal_reported = true;
|
||||
if inner.leases.get(dag.agent.as_str()) == Some(&dag.id) {
|
||||
freed.push(dag.agent.clone());
|
||||
}
|
||||
report.terminal.push(TerminalDag {
|
||||
dag_id: dag.id,
|
||||
template: dag.template,
|
||||
agent: dag.agent.clone(),
|
||||
approval_id: dag.approval_id,
|
||||
state: dag.rollup(),
|
||||
error: dag.first_error().map(str::to_owned),
|
||||
});
|
||||
}
|
||||
for agent in freed {
|
||||
inner.leases.remove(&agent);
|
||||
}
|
||||
Self::trim_history(inner);
|
||||
report
|
||||
}
|
||||
|
||||
/// Cancel a DAG that hasn't started yet (roll-up `Queued`): every
|
||||
/// node flips to `Cancelled`. No-op (returns `false`) once any node
|
||||
/// is running or terminal — an in-flight nix build isn't
|
||||
/// interruptible, matching the old queue's rule.
|
||||
pub fn cancel(&self, dag_id: u64) -> bool {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
let Some(dag) = inner.dags.iter_mut().find(|d| d.id == dag_id) else {
|
||||
return false;
|
||||
};
|
||||
if dag.rollup() != State::Queued {
|
||||
return false;
|
||||
}
|
||||
let now = now_unix();
|
||||
for n in &mut dag.nodes {
|
||||
n.state = State::Cancelled;
|
||||
n.finished_at = Some(now);
|
||||
}
|
||||
let _ = Self::settle(&mut inner);
|
||||
drop(inner);
|
||||
self.notify.notify_one();
|
||||
true
|
||||
}
|
||||
|
||||
/// Cancel every still-fully-queued child DAG of `parent`. Running
|
||||
/// children are left alone. Returns the count of cancelled DAGs.
|
||||
pub fn cancel_children(&self, parent: u64) -> usize {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
let now = now_unix();
|
||||
let mut count = 0;
|
||||
for dag in &mut inner.dags {
|
||||
if dag.parent_id == Some(parent) && dag.rollup() == State::Queued {
|
||||
for n in &mut dag.nodes {
|
||||
n.state = State::Cancelled;
|
||||
n.finished_at = Some(now);
|
||||
}
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
if count > 0 {
|
||||
let _ = Self::settle(&mut inner);
|
||||
drop(inner);
|
||||
self.notify.notify_one();
|
||||
}
|
||||
count
|
||||
}
|
||||
|
||||
/// Set the step label on a `Running` node. Returns `true` when the
|
||||
/// label actually changed (callers emit a snapshot only then).
|
||||
pub fn set_step(&self, dag_id: u64, node_id: NodeId, step: &str) -> bool {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
let Some(node) = inner
|
||||
.dags
|
||||
.iter_mut()
|
||||
.find(|d| d.id == dag_id)
|
||||
.and_then(|d| d.node_mut(node_id))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if node.state != State::Running || node.step.as_deref() == Some(step) {
|
||||
return false;
|
||||
}
|
||||
node.step = Some(step.to_owned());
|
||||
true
|
||||
}
|
||||
|
||||
/// Set the step label on the DAG's currently-running node —
|
||||
/// compatibility surface for the opaque approval pipeline, whose
|
||||
/// callbacks only know the DAG id. Single-node approval DAGs make
|
||||
/// this exact.
|
||||
pub fn set_step_running(&self, dag_id: u64, step: &str) -> bool {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
let Some(node) = inner
|
||||
.dags
|
||||
.iter_mut()
|
||||
.find(|d| d.id == dag_id)
|
||||
.and_then(|d| d.nodes.iter_mut().find(|n| n.state == State::Running))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if node.step.as_deref() == Some(step) {
|
||||
return false;
|
||||
}
|
||||
node.step = Some(step.to_owned());
|
||||
true
|
||||
}
|
||||
|
||||
/// Link a `build_logs` row to a specific `Running` node.
|
||||
pub fn set_build_log_id(&self, dag_id: u64, node_id: NodeId, log_id: i64) -> bool {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
let Some(node) = inner
|
||||
.dags
|
||||
.iter_mut()
|
||||
.find(|d| d.id == dag_id)
|
||||
.and_then(|d| d.node_mut(node_id))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if node.state != State::Running {
|
||||
return false;
|
||||
}
|
||||
node.build_log_id = Some(log_id);
|
||||
true
|
||||
}
|
||||
|
||||
/// Link a `build_logs` row to the DAG's currently-running node —
|
||||
/// DAG-id-only compatibility surface (approval pipeline callbacks).
|
||||
pub fn set_build_log_id_running(&self, dag_id: u64, log_id: i64) -> bool {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
let Some(node) = inner
|
||||
.dags
|
||||
.iter_mut()
|
||||
.find(|d| d.id == dag_id)
|
||||
.and_then(|d| d.nodes.iter_mut().find(|n| n.state == State::Running))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
node.build_log_id = Some(log_id);
|
||||
true
|
||||
}
|
||||
|
||||
/// Snapshot every DAG for `/api/state` + `RebuildQueueChanged`.
|
||||
pub fn snapshot(&self) -> Vec<DagView> {
|
||||
let inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
inner.dags.iter().map(Dag::view).collect()
|
||||
}
|
||||
|
||||
/// Number of live (non-terminal) DAGs — used by tests and
|
||||
/// diagnostics.
|
||||
#[cfg(test)]
|
||||
pub fn live_count(&self) -> usize {
|
||||
let inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
inner.dags.iter().filter(|d| !d.is_terminal()).count()
|
||||
}
|
||||
|
||||
/// Keep only the newest `MAX_HISTORY_PER_TEMPLATE` terminal DAGs
|
||||
/// per template; live DAGs are never evicted.
|
||||
fn trim_history(inner: &mut Inner) {
|
||||
let mut counts: HashMap<Template, usize> = HashMap::new();
|
||||
let kept: Vec<Dag> = inner
|
||||
.dags
|
||||
.iter()
|
||||
.rev()
|
||||
.filter(|d| {
|
||||
if !d.is_terminal() {
|
||||
return true;
|
||||
}
|
||||
let n = counts.entry(d.template).or_insert(0);
|
||||
*n += 1;
|
||||
*n <= MAX_HISTORY_PER_TEMPLATE
|
||||
})
|
||||
.cloned()
|
||||
.collect();
|
||||
inner.dags = kept.into_iter().rev().collect();
|
||||
}
|
||||
}
|
||||
|
||||
/// Current unix timestamp in seconds.
|
||||
fn now_unix() -> i64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|d| i64::try_from(d.as_secs()).ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
520
hive-c0re/src/job_queue/model.rs
Normal file
520
hive-c0re/src/job_queue/model.rs
Normal file
|
|
@ -0,0 +1,520 @@
|
|||
//! Data model for the generic job-DAG queue: templates (what a DAG
|
||||
//! *means*), node kinds (the primitive operations), dependency edges,
|
||||
//! states, and the wire-facing `DagView` / `NodeView` snapshot shapes.
|
||||
//!
|
||||
//! Two levels: the **DAG** is the unit of dedup / cancel /
|
||||
//! approval-resolution and the dashboard group; the **node** is the
|
||||
//! unit of scheduling / execution / build-log / step label. See
|
||||
//! `docs/coordinator.md::Job queue` for the full design.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// What a DAG *means* — the request-level shape. Wire strings match the
|
||||
/// old `QueueKind` values (serialized as the `kind` field on `DagView`)
|
||||
/// so the dashboard's glyph map keeps working.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Template {
|
||||
/// Rebuild one agent's container: `Prebuild → StopForUpdate → Swap
|
||||
/// → Reconcile` (the tail `Reconcile` runs after `Swap` terminal,
|
||||
/// ok *or* fail — the recovery-start).
|
||||
Rebuild,
|
||||
/// Bump meta flake locks: one `MetaLock` node; child `Rebuild`
|
||||
/// DAGs fan out on completion for every affected agent.
|
||||
MetaUpdate,
|
||||
/// First-deploy spawn (approval-driven): `Create → WriteDropin →
|
||||
/// Reconcile`.
|
||||
Spawn,
|
||||
/// Reserved for a future destroy integration — kept so the wire
|
||||
/// shape doesn't need to change later.
|
||||
#[allow(dead_code, reason = "wire shape — routed by a future PR")]
|
||||
Destroy,
|
||||
/// Boot-time config sweep: one `MetaLock` (hyperhive input,
|
||||
/// non-fatal) node; stale agents' `Rebuild` DAGs fan out on
|
||||
/// completion.
|
||||
StartupSweep,
|
||||
/// `StopForUpdate → Reconcile` — stop + converge back to `wanted`
|
||||
/// (unchanged), i.e. a restart for a wanted-up agent.
|
||||
Restart,
|
||||
/// `WritePermFile → Prebuild → StopForUpdate → Swap → Reconcile` —
|
||||
/// perm-file commit followed by the rebuild subgraph.
|
||||
PermChange,
|
||||
/// `Signal → Drain → Reconcile` with `wanted` set to `Offline` at
|
||||
/// submit time: quiesce the harness, await the drain (bounded),
|
||||
/// then the tail `Reconcile` performs the actual container stop.
|
||||
GracefulStop,
|
||||
/// Single `Reconcile` with `wanted` set to `Up` at submit time.
|
||||
Start,
|
||||
/// Single `Reconcile` with `wanted` set to `Offline` at submit time.
|
||||
Stop,
|
||||
/// Single `Reconcile` with `wanted` untouched — boot-time converge
|
||||
/// of observed state to the persisted intent.
|
||||
Reconcile,
|
||||
}
|
||||
|
||||
impl Template {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Template::Rebuild => "rebuild",
|
||||
Template::MetaUpdate => "meta_update",
|
||||
Template::Spawn => "spawn",
|
||||
Template::Destroy => "destroy",
|
||||
Template::StartupSweep => "startup_sweep",
|
||||
Template::Restart => "restart",
|
||||
Template::PermChange => "perm_change",
|
||||
Template::GracefulStop => "graceful_stop",
|
||||
Template::Start => "start",
|
||||
Template::Stop => "stop",
|
||||
Template::Reconcile => "reconcile",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Where the submit request originated. Same variants + wire strings
|
||||
/// as the old `QueueSource` — drives the "why" chip on the dashboard.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Source {
|
||||
/// Operator action (dashboard button, CLI, manager tool).
|
||||
Manual,
|
||||
/// Cascade child of a `MetaUpdate` DAG's fan-out; `parent_id`
|
||||
/// points back at the originating meta-update.
|
||||
MetaUpdate,
|
||||
/// Boot-time submission (sweep parent, boot reconciles).
|
||||
AutoUpdate,
|
||||
/// Cascade child of a `StartupSweep` DAG's fan-out.
|
||||
StartupSweep,
|
||||
/// Crash recovery path (future use).
|
||||
#[allow(dead_code, reason = "wire shape — used by a future feature")]
|
||||
CrashRecover,
|
||||
/// Operator approved a pending `Approval` row; `approval_id` on
|
||||
/// the DAG points back at the source row.
|
||||
Approval,
|
||||
}
|
||||
|
||||
impl Source {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Source::Manual => "manual",
|
||||
Source::MetaUpdate => "meta_update",
|
||||
Source::AutoUpdate => "auto_update",
|
||||
Source::StartupSweep => "startup_sweep",
|
||||
Source::CrashRecover => "crash_recover",
|
||||
Source::Approval => "approval",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Lifecycle state of a node — and, rolled up, of a DAG. Same wire
|
||||
/// strings as the old `QueueState`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum State {
|
||||
Queued,
|
||||
Running,
|
||||
Done,
|
||||
Failed,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
impl State {
|
||||
pub fn is_terminal(self) -> bool {
|
||||
matches!(self, State::Done | State::Failed | State::Cancelled)
|
||||
}
|
||||
}
|
||||
|
||||
/// Kind-specific payload for `Template::PermChange` DAGs. Carried on
|
||||
/// the DAG (not the node) so dedup can compare the perm *type*
|
||||
/// discriminant. Identical to the old `rebuild_queue::PermPayload`.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum PermPayload {
|
||||
/// Set the tool groups for one agent (`tool-groups.json`).
|
||||
ToolGroups { groups: Vec<String> },
|
||||
/// Set the capabilities for one agent (`capabilities.json`).
|
||||
Capabilities { caps: Vec<String> },
|
||||
/// Set both perm-types in one entry — the batch
|
||||
/// `POST /api/permissions` path. `None` leaves that file untouched;
|
||||
/// the worker commits whichever are present in one git commit,
|
||||
/// then rebuilds once.
|
||||
Combined {
|
||||
groups: Option<Vec<String>>,
|
||||
caps: Option<Vec<String>>,
|
||||
},
|
||||
}
|
||||
|
||||
impl PermPayload {
|
||||
/// Dedup compares the perm *type*, not the value — a tool-groups
|
||||
/// change and a capabilities change for the same agent are
|
||||
/// distinct operations that must not collapse.
|
||||
pub fn same_type(a: Option<&PermPayload>, b: Option<&PermPayload>) -> bool {
|
||||
matches!(
|
||||
(a, b),
|
||||
(
|
||||
Some(PermPayload::ToolGroups { .. }),
|
||||
Some(PermPayload::ToolGroups { .. })
|
||||
) | (
|
||||
Some(PermPayload::Capabilities { .. }),
|
||||
Some(PermPayload::Capabilities { .. })
|
||||
) | (
|
||||
Some(PermPayload::Combined { .. }),
|
||||
Some(PermPayload::Combined { .. })
|
||||
) | (None, None)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Node id, unique within its DAG (dense small ints assigned by the
|
||||
/// template builders).
|
||||
pub type NodeId = u32;
|
||||
|
||||
/// When a dependency edge is considered satisfied.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum DepWhen {
|
||||
/// Dep must reach `Done`. A `Failed` / `Cancelled` dep cancels this
|
||||
/// node (cancel-downstream).
|
||||
AfterOk,
|
||||
/// Dep must merely reach a terminal state (ok *or* fail). Used only
|
||||
/// by `rebuild`'s tail `Reconcile` so the recovery-start runs even
|
||||
/// when `Swap` failed.
|
||||
AfterAny,
|
||||
}
|
||||
|
||||
/// A dependency edge (intra-DAG only — cross-DAG ordering comes from
|
||||
/// the per-agent lease + dedup, never from edges between DAGs).
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
pub struct Dep {
|
||||
pub on: NodeId,
|
||||
pub when: DepWhen,
|
||||
}
|
||||
|
||||
/// The primitive operations — each kind maps to one executor fn in
|
||||
/// `exec.rs`, a thin wrapper over existing `lifecycle.rs` / `meta.rs`
|
||||
/// code. Concurrency is gated by two resource classes (see
|
||||
/// [`NodeKind::needs_build_slot`] / [`NodeKind::needs_lease`]); the
|
||||
/// meta *repo* is serialized by `meta::META_LOCK` inside the wrapped
|
||||
/// functions themselves, which is why there is no `GitCommit` node —
|
||||
/// a standalone commit node would open a dirty-working-tree window
|
||||
/// between nodes that the fused `meta.rs` ops deliberately close.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum NodeKind {
|
||||
/// Out-of-band toplevel build while the container keeps serving:
|
||||
/// meta `sync_agents`, optional per-agent relock, then
|
||||
/// `lifecycle::prebuild_toplevel`. `relock = false` only for
|
||||
/// meta-update cascade rebuilds (re-locking would revert the bump
|
||||
/// the cascade just committed).
|
||||
Prebuild { relock: bool },
|
||||
/// `nixos-container update` profile-swap (requires the container
|
||||
/// stopped). Re-applies nspawn flags + resource limits first —
|
||||
/// rebuild is the reconcile verb — and carries the post-rebuild
|
||||
/// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan).
|
||||
Swap,
|
||||
/// First-spawn `nixos-container create` plus the pre-create
|
||||
/// provisioning (proposed/applied repos, state subvolume, meta
|
||||
/// registration).
|
||||
Create,
|
||||
/// Meta flake lock bump. `sweep = false`: `meta::lock_update`
|
||||
/// (commit fused, under `META_LOCK`) with the DAG's `inputs`;
|
||||
/// `sweep = true`: `meta::lock_update_hyperhive`, *non-fatal* (a
|
||||
/// failed boot-time bump must not cancel the fan-out rebuilds).
|
||||
/// On success the scheduler appends child `Rebuild` DAGs: the
|
||||
/// precomputed `fanout` list when present (boot sweep), else the
|
||||
/// post-bump affected set (`meta_update_cascade_agents`).
|
||||
MetaLock {
|
||||
sweep: bool,
|
||||
fanout: Option<Vec<String>>,
|
||||
},
|
||||
/// Idempotent power converge: read `wanted` + observed state;
|
||||
/// start if `Up` & down (with cold-start fallback), stop if
|
||||
/// `Offline` & up, else noop.
|
||||
Reconcile,
|
||||
/// Mechanical `nixos-container stop` for the profile swap. Never
|
||||
/// touches `wanted`. Noop if already stopped.
|
||||
StopForUpdate,
|
||||
/// Set the graceful-stop fence + kick the harness so it runs one
|
||||
/// stop-checkpoint turn.
|
||||
Signal,
|
||||
/// Await the harness clearing the fence, bounded by
|
||||
/// `GRACEFUL_STOP_TIMEOUT`. Resolves ok either way — the
|
||||
/// downstream `Reconcile` performs the actual stop.
|
||||
Drain,
|
||||
/// `set_nspawn_flags` + `set_resource_limits` + daemon-reload.
|
||||
WriteDropin,
|
||||
/// Commit `tool-groups.json` / `capabilities.json` per the DAG's
|
||||
/// `perm_payload` (commit fused under `META_LOCK`).
|
||||
WritePermFile,
|
||||
/// Opaque approval deploy pipeline (`ApplyCommit` /
|
||||
/// `MergeConfigPr`): the two-phase prepare/finalize/abort meta
|
||||
/// deploy stays inside `actions.rs` in v1 — deliberately not
|
||||
/// modeled as scheduler nodes (see the design doc §9).
|
||||
ApprovalDeploy,
|
||||
}
|
||||
|
||||
impl NodeKind {
|
||||
/// Wire string for `NodeView.kind`.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
NodeKind::Prebuild { .. } => "prebuild",
|
||||
NodeKind::Swap => "swap",
|
||||
NodeKind::Create => "create",
|
||||
NodeKind::MetaLock { .. } => "meta_lock",
|
||||
NodeKind::Reconcile => "reconcile",
|
||||
NodeKind::StopForUpdate => "stop_for_update",
|
||||
NodeKind::Signal => "signal",
|
||||
NodeKind::Drain => "drain",
|
||||
NodeKind::WriteDropin => "write_dropin",
|
||||
NodeKind::WritePermFile => "write_perm_file",
|
||||
NodeKind::ApprovalDeploy => "approval_deploy",
|
||||
}
|
||||
}
|
||||
|
||||
/// Nix-heavy kinds hold one of the `buildSlots` semaphore permits
|
||||
/// for the node's duration.
|
||||
pub fn needs_build_slot(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
NodeKind::Prebuild { .. }
|
||||
| NodeKind::Swap
|
||||
| NodeKind::Create
|
||||
| NodeKind::MetaLock { .. }
|
||||
| NodeKind::ApprovalDeploy
|
||||
)
|
||||
}
|
||||
|
||||
/// Container-affecting kinds require the DAG to hold the agent's
|
||||
/// lifecycle lease (acquired at the first such node, held until the
|
||||
/// DAG is terminal). Lease-exempt kinds (`Prebuild`, `MetaLock`,
|
||||
/// `WritePermFile`) touch the store / meta repo, not the running
|
||||
/// container — which is exactly why a `Prebuild` can overlap
|
||||
/// another DAG's work on the same agent.
|
||||
pub fn needs_lease(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
NodeKind::Swap
|
||||
| NodeKind::Create
|
||||
| NodeKind::Reconcile
|
||||
| NodeKind::StopForUpdate
|
||||
| NodeKind::Signal
|
||||
| NodeKind::Drain
|
||||
| NodeKind::WriteDropin
|
||||
| NodeKind::ApprovalDeploy
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// One schedulable unit inside a DAG.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Node {
|
||||
pub id: NodeId,
|
||||
pub kind: NodeKind,
|
||||
pub deps: Vec<Dep>,
|
||||
pub state: State,
|
||||
/// Live sub-label while `Running` (kept for parity with the old
|
||||
/// per-entry `step`).
|
||||
pub step: Option<String>,
|
||||
/// Row id of the `build_logs` entry this node opened (`Prebuild` /
|
||||
/// `Swap` / `ApprovalDeploy`), for the dashboard's live-stream link.
|
||||
pub build_log_id: Option<i64>,
|
||||
pub started_at: Option<i64>,
|
||||
pub finished_at: Option<i64>,
|
||||
/// Populated when `state == Failed` (truncated by the queue).
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Submit-time spec for one node.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NodeSpec {
|
||||
pub kind: NodeKind,
|
||||
pub deps: Vec<Dep>,
|
||||
}
|
||||
|
||||
/// Submit-time spec for a whole DAG. Built by `templates.rs`; validated
|
||||
/// (cycle rejection) and dedup'd by `JobQueue::submit`.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DagSpec {
|
||||
pub template: Template,
|
||||
/// Primary target agent, or `"hyperhive"` for meta-level DAGs.
|
||||
pub agent: String,
|
||||
pub source: Source,
|
||||
/// Free-form "why"; dedup appends "also requested by …" lines.
|
||||
pub reason: String,
|
||||
/// Cascade grouping (meta-update / sweep children).
|
||||
pub parent_id: Option<u64>,
|
||||
/// Fires the approval-resolution hook on DAG terminal.
|
||||
pub approval_id: Option<i64>,
|
||||
/// `MetaUpdate`-only: the inputs to bump (also part of the dedup
|
||||
/// key for that template). Display copy lives on the DAG.
|
||||
pub inputs: Vec<String>,
|
||||
/// `PermChange`-only payload.
|
||||
pub perm_payload: Option<PermPayload>,
|
||||
/// Dashboard transient pill (and crash-watch suppression) held for
|
||||
/// the lease window — from lease acquisition to DAG terminal.
|
||||
pub transient: Option<crate::coordinator::TransientKind>,
|
||||
pub nodes: Vec<NodeSpec>,
|
||||
}
|
||||
|
||||
/// A live DAG in the queue.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Dag {
|
||||
pub id: u64,
|
||||
pub template: Template,
|
||||
pub agent: String,
|
||||
pub source: Source,
|
||||
pub reason: String,
|
||||
pub parent_id: Option<u64>,
|
||||
pub approval_id: Option<i64>,
|
||||
pub inputs: Vec<String>,
|
||||
pub perm_payload: Option<PermPayload>,
|
||||
pub transient: Option<crate::coordinator::TransientKind>,
|
||||
pub created_at: i64,
|
||||
pub nodes: Vec<Node>,
|
||||
/// Terminal roll-up already reported to the scheduler's hooks
|
||||
/// (approval resolution, transient release). Internal bookkeeping,
|
||||
/// never serialized.
|
||||
pub terminal_reported: bool,
|
||||
}
|
||||
|
||||
impl Dag {
|
||||
/// Roll-up state: `Failed` if any node failed; else `Running` if
|
||||
/// any running; else `Queued` if any queued; else `Cancelled` if
|
||||
/// any cancelled; else `Done`.
|
||||
pub fn rollup(&self) -> State {
|
||||
let mut any_cancelled = false;
|
||||
let mut any_queued = false;
|
||||
let mut any_running = false;
|
||||
for n in &self.nodes {
|
||||
match n.state {
|
||||
State::Failed => return State::Failed,
|
||||
State::Running => any_running = true,
|
||||
State::Queued => any_queued = true,
|
||||
State::Cancelled => any_cancelled = true,
|
||||
State::Done => {}
|
||||
}
|
||||
}
|
||||
if any_running {
|
||||
State::Running
|
||||
} else if any_queued {
|
||||
State::Queued
|
||||
} else if any_cancelled {
|
||||
State::Cancelled
|
||||
} else {
|
||||
State::Done
|
||||
}
|
||||
}
|
||||
|
||||
/// True when every node is terminal.
|
||||
pub fn is_terminal(&self) -> bool {
|
||||
self.nodes.iter().all(|n| n.state.is_terminal())
|
||||
}
|
||||
|
||||
/// First failed node's error, for the roll-up `error` field.
|
||||
pub fn first_error(&self) -> Option<&str> {
|
||||
self.nodes
|
||||
.iter()
|
||||
.find(|n| n.state == State::Failed)
|
||||
.and_then(|n| n.error.as_deref())
|
||||
}
|
||||
|
||||
pub fn node(&self, id: NodeId) -> Option<&Node> {
|
||||
self.nodes.iter().find(|n| n.id == id)
|
||||
}
|
||||
|
||||
pub fn node_mut(&mut self, id: NodeId) -> Option<&mut Node> {
|
||||
self.nodes.iter_mut().find(|n| n.id == id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Wire shape of one node inside a `DagView`.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct NodeView {
|
||||
pub id: NodeId,
|
||||
/// Flattened `NodeKind` tag ("prebuild", "swap", …).
|
||||
pub kind: &'static str,
|
||||
/// Ids of the nodes this one waits for.
|
||||
pub deps: Vec<NodeId>,
|
||||
pub state: State,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub step: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub build_log_id: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub finished_at: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Wire shape of a DAG, serialized onto `RebuildQueueChanged` and the
|
||||
/// `/api/state` snapshot. DAG-level fields mirror the old `QueueEntry`
|
||||
/// names (`kind` = template string, roll-up `state`); everything
|
||||
/// per-node — step labels, build-log links, errors, timestamps —
|
||||
/// appears exactly once, inside `nodes`.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct DagView {
|
||||
pub id: u64,
|
||||
pub agent: String,
|
||||
/// Template wire string — same values the old `kind` field used.
|
||||
pub kind: Template,
|
||||
/// Roll-up state (see [`Dag::rollup`]).
|
||||
pub state: State,
|
||||
pub source: Source,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub parent_id: Option<u64>,
|
||||
pub reason: String,
|
||||
pub enqueued_at: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub finished_at: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub inputs: Vec<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub approval_id: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub perm_payload: Option<PermPayload>,
|
||||
pub nodes: Vec<NodeView>,
|
||||
}
|
||||
|
||||
impl Dag {
|
||||
pub fn view(&self) -> DagView {
|
||||
let started_at = self.nodes.iter().filter_map(|n| n.started_at).min();
|
||||
let finished_at = if self.is_terminal() {
|
||||
self.nodes.iter().filter_map(|n| n.finished_at).max()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
DagView {
|
||||
id: self.id,
|
||||
agent: self.agent.clone(),
|
||||
kind: self.template,
|
||||
state: self.rollup(),
|
||||
source: self.source,
|
||||
parent_id: self.parent_id,
|
||||
reason: self.reason.clone(),
|
||||
enqueued_at: self.created_at,
|
||||
started_at,
|
||||
finished_at,
|
||||
inputs: self.inputs.clone(),
|
||||
approval_id: self.approval_id,
|
||||
perm_payload: self.perm_payload.clone(),
|
||||
nodes: self
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|n| NodeView {
|
||||
id: n.id,
|
||||
kind: n.kind.as_str(),
|
||||
deps: n.deps.iter().map(|d| d.on).collect(),
|
||||
state: n.state,
|
||||
step: n.step.clone(),
|
||||
build_log_id: n.build_log_id,
|
||||
started_at: n.started_at,
|
||||
finished_at: n.finished_at,
|
||||
error: n.error.clone(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
150
hive-c0re/src/job_queue/scheduler.rs
Normal file
150
hive-c0re/src/job_queue/scheduler.rs
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
//! The single scheduler task that drives all DAGs: claim every ready
|
||||
//! node (as many as the build slots / leases allow), spawn one
|
||||
//! executor task per claim, and on any completion re-evaluate.
|
||||
//! Concurrency comes from the build-slot count, not multiple workers.
|
||||
//!
|
||||
//! Also owns the two DAG-lifetime side channels the sync queue core
|
||||
//! can't hold itself:
|
||||
//! - the per-DAG transient guard (dashboard pill + crash-watch
|
||||
//! suppression), created when a DAG acquires its agent lease and
|
||||
//! dropped when the DAG settles terminal;
|
||||
//! - the `MetaLock` fan-out: appending child `Rebuild` DAGs once the
|
||||
//! lock bump lands, so children build against the post-bump lock
|
||||
//! (and a failed bump fans out nothing — replacing the old
|
||||
//! pre-enqueue + cancel-children dance).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::exec::{self, NodeOutput};
|
||||
use super::{Claim, Source, Template, templates};
|
||||
use crate::coordinator::Coordinator;
|
||||
|
||||
struct NodeDone {
|
||||
claim: Claim,
|
||||
result: anyhow::Result<NodeOutput>,
|
||||
}
|
||||
|
||||
/// Scheduler loop. Spawned once at hive-c0re startup from `main.rs`.
|
||||
///
|
||||
/// Shutdown semantics: subscribes to `coord.shutdown_rx()`. On a true
|
||||
/// signal the loop exits immediately; already-running node tasks ride
|
||||
/// the runtime down with the process, and pending `Queued` DAGs are
|
||||
/// dropped — desired state is re-derived on next boot (boot sweep +
|
||||
/// reconcile), so the in-memory queue is deliberately not durable.
|
||||
pub async fn run_worker(coord: Arc<Coordinator>) {
|
||||
let mut shutdown = coord.shutdown_rx();
|
||||
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<NodeDone>();
|
||||
// DAG id → transient guard held for the lease window.
|
||||
let mut transients: HashMap<u64, crate::coordinator::TransientGuard> = HashMap::new();
|
||||
loop {
|
||||
let claims = coord.job_queue.claim_ready();
|
||||
if !claims.is_empty() {
|
||||
for claim in claims {
|
||||
if claim.lease_acquired
|
||||
&& let Some(kind) = claim.transient
|
||||
{
|
||||
transients.insert(claim.dag_id, coord.transient_guard(&claim.agent, kind));
|
||||
}
|
||||
tracing::info!(
|
||||
dag = claim.dag_id,
|
||||
node = claim.node_id,
|
||||
kind = claim.kind.as_str(),
|
||||
agent = %claim.agent,
|
||||
template = claim.template.as_str(),
|
||||
"job_queue: node running"
|
||||
);
|
||||
let coord = Arc::clone(&coord);
|
||||
let tx = tx.clone();
|
||||
tokio::spawn(async move {
|
||||
let result = exec::run_node(&coord, &claim).await;
|
||||
// Send failure = scheduler gone (shutdown); drop.
|
||||
let _ = tx.send(NodeDone { claim, result });
|
||||
});
|
||||
}
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
continue;
|
||||
}
|
||||
tokio::select! {
|
||||
biased;
|
||||
res = shutdown.changed() => {
|
||||
if res.is_err() || *shutdown.borrow() {
|
||||
tracing::info!("job_queue: scheduler exiting on shutdown");
|
||||
return;
|
||||
}
|
||||
}
|
||||
Some(done) = rx.recv() => {
|
||||
handle_completion(&coord, &mut transients, done).await;
|
||||
}
|
||||
() = coord.job_queue.notify.notified() => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_completion(
|
||||
coord: &Arc<Coordinator>,
|
||||
transients: &mut HashMap<u64, crate::coordinator::TransientGuard>,
|
||||
done: NodeDone,
|
||||
) {
|
||||
let NodeDone { claim, result } = done;
|
||||
let (queue_result, fanout) = match result {
|
||||
Ok(output) => {
|
||||
tracing::info!(
|
||||
dag = claim.dag_id,
|
||||
node = claim.node_id,
|
||||
"job_queue: node done"
|
||||
);
|
||||
(Ok(()), output.fanout)
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = format!("{e:#}");
|
||||
tracing::warn!(
|
||||
dag = claim.dag_id,
|
||||
node = claim.node_id,
|
||||
kind = claim.kind.as_str(),
|
||||
agent = %claim.agent,
|
||||
error = %msg,
|
||||
"job_queue: node failed"
|
||||
);
|
||||
(Err(msg), Vec::new())
|
||||
}
|
||||
};
|
||||
let report = coord
|
||||
.job_queue
|
||||
.complete_node(claim.dag_id, claim.node_id, queue_result);
|
||||
if !fanout.is_empty() {
|
||||
let specs = fanout_specs(&claim, fanout);
|
||||
coord.job_queue.append_children(specs);
|
||||
}
|
||||
for terminal in report.terminal {
|
||||
// Drop the lease-window transient guard, then let the hook
|
||||
// fire approval resolution / failure events.
|
||||
transients.remove(&terminal.dag_id);
|
||||
exec::on_dag_terminal(coord, &terminal).await;
|
||||
}
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
|
||||
/// Child `Rebuild` specs for a completed `MetaLock` fan-out, grouped
|
||||
/// under the parent via `parent_id`. Meta-update children skip the
|
||||
/// per-agent relock (it would revert the bump the parent just
|
||||
/// committed); sweep children relock like a manual rebuild.
|
||||
fn fanout_specs(claim: &Claim, agents: Vec<String>) -> Vec<super::DagSpec> {
|
||||
let sweep = claim.template == Template::StartupSweep;
|
||||
let (source, relock) = if sweep {
|
||||
(Source::StartupSweep, true)
|
||||
} else {
|
||||
(Source::MetaUpdate, false)
|
||||
};
|
||||
let reason = if sweep {
|
||||
"startup sweep".to_owned()
|
||||
} else if let Some(approval_id) = claim.approval_id {
|
||||
format!("approval #{approval_id} meta input cascade")
|
||||
} else {
|
||||
"meta-update cascade".to_owned()
|
||||
};
|
||||
agents
|
||||
.into_iter()
|
||||
.map(|agent| templates::rebuild(&agent, source, reason.clone(), Some(claim.dag_id), relock))
|
||||
.collect()
|
||||
}
|
||||
121
hive-c0re/src/job_queue/submit.rs
Normal file
121
hive-c0re/src/job_queue/submit.rs
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
//! Request-level submit API — the surface the dashboard POST handlers,
|
||||
//! the MCP socket handlers, and `hivectl` paths call. Owns the
|
||||
//! submit-time side effects the DAG templates deliberately don't:
|
||||
//! writing the durable `wanted` power intent (synchronously,
|
||||
//! last-writer-wins) before the DAG whose `Reconcile` reads it, and
|
||||
//! upgrading a stale start to a full rebuild. Every helper emits a
|
||||
//! fresh queue snapshot so the dashboard shows the new DAG immediately.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::{Source, Template, templates};
|
||||
use crate::coordinator::Coordinator;
|
||||
use crate::power::Wanted;
|
||||
|
||||
fn submit_and_emit(coord: &Arc<Coordinator>, spec: super::DagSpec) -> u64 {
|
||||
let id = coord
|
||||
.job_queue
|
||||
.submit(spec)
|
||||
.expect("template-built dag specs are acyclic");
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
id
|
||||
}
|
||||
|
||||
fn set_wanted(coord: &Arc<Coordinator>, agent: &str, wanted: Wanted) {
|
||||
if let Err(e) = coord.power.set(agent, wanted) {
|
||||
tracing::warn!(%agent, wanted = wanted.as_str(), error = ?e, "agent_power: set failed");
|
||||
}
|
||||
}
|
||||
|
||||
/// Manual/approval-independent rebuild (always relocks the agent's
|
||||
/// meta input — cascade children are built by the scheduler's fan-out
|
||||
/// instead of this surface).
|
||||
pub fn rebuild(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||
submit_and_emit(coord, templates::rebuild(agent, source, reason, None, true))
|
||||
}
|
||||
|
||||
/// Restart: mechanical stop + converge back to `wanted` (unchanged).
|
||||
pub fn restart(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||
submit_and_emit(coord, templates::restart(agent, source, reason))
|
||||
}
|
||||
|
||||
/// Start: persist `wanted = Up`, then reconcile. A stale rev marker
|
||||
/// upgrades the start to a full rebuild (whose tail `Reconcile` does
|
||||
/// the start) so the container always comes up on current derivations
|
||||
/// — the old fast-lane `run_start` upgrade, moved to submit time.
|
||||
pub fn start(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||
set_wanted(coord, agent, Wanted::Up);
|
||||
let stored = std::fs::read_to_string(crate::auto_update::rev_marker_path(agent)).ok();
|
||||
let stale = crate::auto_update::current_flake_rev(&coord.hyperhive_flake)
|
||||
.is_some_and(|rev| stored.as_deref() != Some(rev.as_str()));
|
||||
if stale {
|
||||
tracing::info!(%agent, "start: rev stale — upgrading to rebuild+start");
|
||||
return submit_and_emit(
|
||||
coord,
|
||||
templates::rebuild(
|
||||
agent,
|
||||
source,
|
||||
format!("{reason} (stale — rebuild+start)"),
|
||||
None,
|
||||
true,
|
||||
),
|
||||
);
|
||||
}
|
||||
submit_and_emit(
|
||||
coord,
|
||||
templates::reconcile_only(
|
||||
Template::Start,
|
||||
agent,
|
||||
source,
|
||||
reason,
|
||||
Some(crate::coordinator::TransientKind::Starting),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
/// Hard stop: persist `wanted = Offline`, then reconcile (kill +
|
||||
/// unregister + `Killed` event).
|
||||
pub fn stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||
set_wanted(coord, agent, Wanted::Offline);
|
||||
submit_and_emit(
|
||||
coord,
|
||||
templates::reconcile_only(
|
||||
Template::Stop,
|
||||
agent,
|
||||
source,
|
||||
reason,
|
||||
Some(crate::coordinator::TransientKind::Stopping),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
/// Graceful stop: persist `wanted = Offline`, then signal → drain →
|
||||
/// reconcile (the actual stop).
|
||||
pub fn graceful_stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||
set_wanted(coord, agent, Wanted::Offline);
|
||||
submit_and_emit(coord, templates::graceful_stop(agent, source, reason))
|
||||
}
|
||||
|
||||
/// Perm change: commit the JSON file(s) then rebuild.
|
||||
pub fn perm_change(
|
||||
coord: &Arc<Coordinator>,
|
||||
agent: &str,
|
||||
source: Source,
|
||||
reason: String,
|
||||
payload: super::PermPayload,
|
||||
) -> u64 {
|
||||
submit_and_emit(
|
||||
coord,
|
||||
templates::perm_change(agent, source, reason, payload),
|
||||
)
|
||||
}
|
||||
|
||||
/// Meta-input lock bump; cascade rebuilds fan out on completion.
|
||||
pub fn meta_update(
|
||||
coord: &Arc<Coordinator>,
|
||||
inputs: Vec<String>,
|
||||
source: Source,
|
||||
reason: String,
|
||||
) -> u64 {
|
||||
submit_and_emit(coord, templates::meta_update(inputs, source, reason, None))
|
||||
}
|
||||
333
hive-c0re/src/job_queue/templates.rs
Normal file
333
hive-c0re/src/job_queue/templates.rs
Normal file
|
|
@ -0,0 +1,333 @@
|
|||
//! DAG shape builders — every operation as a template over the shared
|
||||
//! node primitives — plus submit-time cycle validation (petgraph is
|
||||
//! confined to this validation; the runtime store stays the plain
|
||||
//! `Vec<Node>` + `deps`).
|
||||
//!
|
||||
//! ```text
|
||||
//! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a)
|
||||
//! graceful-stop(a): [wanted=Offline] Signal(a) → Drain(a) → Reconcile(a)
|
||||
//! restart(a): StopForUpdate(a) → Reconcile(a) (wanted unchanged)
|
||||
//! start(a): [wanted=Up] Reconcile(a)
|
||||
//! stop(a): [wanted=Offline] Reconcile(a)
|
||||
//! spawn(a): [wanted=Up] Create(a) → WriteDropin(a) → Reconcile(a)
|
||||
//! perm-change(a): WritePermFile(a) → «rebuild subgraph»
|
||||
//! meta-update(inp): MetaLock(inp) → «fan-out rebuild(a) per affected a»
|
||||
//! startup sweep: MetaLock(hyperhive, non-fatal) → «fan-out rebuild(stale a)»
|
||||
//! ```
|
||||
|
||||
use anyhow::{Result, bail};
|
||||
|
||||
use super::model::{DagSpec, Dep, DepWhen, NodeKind, NodeSpec, PermPayload, Source, Template};
|
||||
use crate::coordinator::TransientKind;
|
||||
|
||||
/// After-ok edge on the previous node — the common chain link.
|
||||
fn after_ok(on: u32) -> Vec<Dep> {
|
||||
vec![Dep {
|
||||
on,
|
||||
when: DepWhen::AfterOk,
|
||||
}]
|
||||
}
|
||||
|
||||
/// The rebuild node chain. `Reconcile` deps on `Swap` with `AfterAny`:
|
||||
/// it must run even when the profile swap failed, so a previously-up
|
||||
/// agent comes back on its old config (today's recovery-start). This
|
||||
/// is the only `AfterAny` edge in v1.
|
||||
fn rebuild_nodes(relock: bool, base: u32) -> Vec<NodeSpec> {
|
||||
vec![
|
||||
NodeSpec {
|
||||
kind: NodeKind::Prebuild { relock },
|
||||
deps: if base == 0 {
|
||||
Vec::new()
|
||||
} else {
|
||||
after_ok(base - 1)
|
||||
},
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::StopForUpdate,
|
||||
deps: after_ok(base),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Swap,
|
||||
deps: after_ok(base + 1),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: vec![Dep {
|
||||
on: base + 2,
|
||||
when: DepWhen::AfterAny,
|
||||
}],
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
/// One uniform rebuild shape — no `was_running` branch. `StopForUpdate`
|
||||
/// noops when already down; the tail `Reconcile` auto-noops the start
|
||||
/// when `wanted = Offline` (a rebuild of a deliberately-stopped agent
|
||||
/// leaves it stopped). `relock = false` only for meta-update cascade
|
||||
/// children.
|
||||
pub fn rebuild(
|
||||
agent: &str,
|
||||
source: Source,
|
||||
reason: String,
|
||||
parent_id: Option<u64>,
|
||||
relock: bool,
|
||||
) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::Rebuild,
|
||||
agent: agent.to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: Some(TransientKind::Rebuilding),
|
||||
nodes: rebuild_nodes(relock, 0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Approval-driven deploy (`ApplyCommit` / `MergeConfigPr`): the whole
|
||||
/// two-phase pipeline stays one opaque node in v1 (design doc §9) —
|
||||
/// wire-visible as a `rebuild` card like today.
|
||||
pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::Rebuild,
|
||||
agent: agent.to_owned(),
|
||||
source: Source::Approval,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: Some(approval_id),
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: Some(TransientKind::Rebuilding),
|
||||
nodes: vec![NodeSpec {
|
||||
kind: NodeKind::ApprovalDeploy,
|
||||
deps: Vec::new(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
/// Graceful stop: cheap `Signal` fires immediately (no build slot), the
|
||||
/// `Drain` awaits the harness checkpoint (bounded), and the tail
|
||||
/// `Reconcile` performs the actual container stop — the caller sets
|
||||
/// `wanted = Offline` at submit time. A whole-hive graceful stop
|
||||
/// therefore signals every agent up front and overlaps every drain,
|
||||
/// replacing the old detached-watcher thread structurally.
|
||||
pub fn graceful_stop(agent: &str, source: Source, reason: String) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::GracefulStop,
|
||||
agent: agent.to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: Some(TransientKind::Stopping),
|
||||
nodes: vec![
|
||||
NodeSpec {
|
||||
kind: NodeKind::Signal,
|
||||
deps: Vec::new(),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Drain,
|
||||
deps: after_ok(0),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: after_ok(1),
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/// Restart: mechanical stop, then converge back to `wanted`
|
||||
/// (unchanged) — a stop + start for a wanted-up agent.
|
||||
pub fn restart(agent: &str, source: Source, reason: String) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::Restart,
|
||||
agent: agent.to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: Some(TransientKind::Restarting),
|
||||
nodes: vec![
|
||||
NodeSpec {
|
||||
kind: NodeKind::StopForUpdate,
|
||||
deps: Vec::new(),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: after_ok(0),
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/// Single-`Reconcile` DAG: `Start` / `Stop` (caller writes `wanted`
|
||||
/// first) and the boot-time `Reconcile` converge (wanted untouched).
|
||||
pub fn reconcile_only(
|
||||
template: Template,
|
||||
agent: &str,
|
||||
source: Source,
|
||||
reason: String,
|
||||
transient: Option<TransientKind>,
|
||||
) -> DagSpec {
|
||||
DagSpec {
|
||||
template,
|
||||
agent: agent.to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient,
|
||||
nodes: vec![NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: Vec::new(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
/// First-deploy spawn (approval-driven): pre-start provisioning +
|
||||
/// `nixos-container create`, drop-in write, then `Reconcile` starts the
|
||||
/// container (`wanted = Up` written at approve time).
|
||||
pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::Spawn,
|
||||
agent: agent.to_owned(),
|
||||
source: Source::Approval,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: Some(approval_id),
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: Some(TransientKind::Spawning),
|
||||
nodes: vec![
|
||||
NodeSpec {
|
||||
kind: NodeKind::Create,
|
||||
deps: Vec::new(),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::WriteDropin,
|
||||
deps: after_ok(0),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: after_ok(1),
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/// Perm change: commit the JSON file(s), then the rebuild subgraph so
|
||||
/// the updated `HIVE_TOOL_GROUPS` / `HIVE_CAPABILITIES` env var takes
|
||||
/// effect in the container.
|
||||
pub fn perm_change(agent: &str, source: Source, reason: String, payload: PermPayload) -> DagSpec {
|
||||
let mut nodes = vec![NodeSpec {
|
||||
kind: NodeKind::WritePermFile,
|
||||
deps: Vec::new(),
|
||||
}];
|
||||
nodes.extend(rebuild_nodes(true, 1));
|
||||
DagSpec {
|
||||
template: Template::PermChange,
|
||||
agent: agent.to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: Some(payload),
|
||||
transient: Some(TransientKind::Rebuilding),
|
||||
nodes,
|
||||
}
|
||||
}
|
||||
|
||||
/// Meta-input lock bump. Child `Rebuild` DAGs fan out on completion —
|
||||
/// appended *after* the bump lands so their prebuilds run against the
|
||||
/// post-bump lock (and so a failed bump simply fans out nothing,
|
||||
/// replacing the old pre-enqueue + `cancel_children` dance).
|
||||
pub fn meta_update(
|
||||
inputs: Vec<String>,
|
||||
source: Source,
|
||||
reason: String,
|
||||
approval_id: Option<i64>,
|
||||
) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::MetaUpdate,
|
||||
agent: "hyperhive".to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id,
|
||||
inputs,
|
||||
perm_payload: None,
|
||||
transient: None,
|
||||
nodes: vec![NodeSpec {
|
||||
kind: NodeKind::MetaLock {
|
||||
sweep: false,
|
||||
fanout: None,
|
||||
},
|
||||
deps: Vec::new(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
/// Boot-time sweep parent: bump meta's hyperhive input (non-fatal),
|
||||
/// then fan out `Rebuild` children for the precomputed stale agent
|
||||
/// list (topology-sorted by the caller).
|
||||
pub fn startup_sweep(reason: String, stale_agents: Vec<String>) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::StartupSweep,
|
||||
agent: "hyperhive".to_owned(),
|
||||
source: Source::AutoUpdate,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: None,
|
||||
nodes: vec![NodeSpec {
|
||||
kind: NodeKind::MetaLock {
|
||||
sweep: true,
|
||||
fanout: Some(stale_agents),
|
||||
},
|
||||
deps: Vec::new(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate a spec before it enters the queue: node ids are dense
|
||||
/// (index = id), deps reference existing nodes, and the dep graph is
|
||||
/// acyclic (petgraph `toposort`). Rejecting cycles here fixes the old
|
||||
/// queue's documented "circular dep silently deadlocks forever" caveat.
|
||||
pub fn validate(spec: &DagSpec) -> Result<()> {
|
||||
if spec.nodes.is_empty() {
|
||||
bail!("dag spec {:?} has no nodes", spec.template);
|
||||
}
|
||||
let n = spec.nodes.len();
|
||||
let mut graph = petgraph::graph::DiGraph::<u32, ()>::new();
|
||||
let idx: Vec<_> = (0..n)
|
||||
.map(|i| graph.add_node(u32::try_from(i).unwrap_or(u32::MAX)))
|
||||
.collect();
|
||||
for (i, node) in spec.nodes.iter().enumerate() {
|
||||
for dep in &node.deps {
|
||||
let Some(&dep_idx) = idx.get(dep.on as usize) else {
|
||||
bail!(
|
||||
"dag spec {:?} node {i} depends on unknown node {}",
|
||||
spec.template,
|
||||
dep.on
|
||||
);
|
||||
};
|
||||
graph.add_edge(dep_idx, idx[i], ());
|
||||
}
|
||||
}
|
||||
if petgraph::algo::toposort(&graph, None).is_err() {
|
||||
bail!("dag spec {:?} contains a dependency cycle", spec.template);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
823
hive-c0re/src/job_queue/tests.rs
Normal file
823
hive-c0re/src/job_queue/tests.rs
Normal file
|
|
@ -0,0 +1,823 @@
|
|||
//! Queue-core unit tests: dedup, cycle rejection, resource
|
||||
//! serialization (build slots / per-agent leases), lease-exempt
|
||||
//! overlap, FIFO fairness, cancel semantics, `AfterAny` failure
|
||||
//! routing, fan-out, and history retention. All synchronous — the
|
||||
//! scheduler's async loop is a thin claim/complete pump over the same
|
||||
//! methods exercised here.
|
||||
|
||||
use super::model::{Dep, DepWhen, NodeKind, NodeSpec};
|
||||
use super::*;
|
||||
|
||||
fn submit(q: &JobQueue, spec: DagSpec) -> u64 {
|
||||
q.submit(spec).expect("valid spec")
|
||||
}
|
||||
|
||||
fn rebuild(agent: &str, reason: &str) -> DagSpec {
|
||||
templates::rebuild(agent, Source::Manual, reason.to_owned(), None, true)
|
||||
}
|
||||
|
||||
/// Claim helper asserting exactly one node comes back.
|
||||
fn claim_one(q: &JobQueue) -> Claim {
|
||||
let mut claims = q.claim_ready();
|
||||
assert_eq!(
|
||||
claims.len(),
|
||||
1,
|
||||
"expected exactly one claim, got {claims:?}"
|
||||
);
|
||||
claims.pop().expect("one claim")
|
||||
}
|
||||
|
||||
fn state_of(q: &JobQueue, dag_id: u64) -> State {
|
||||
q.snapshot()
|
||||
.iter()
|
||||
.find(|d| d.id == dag_id)
|
||||
.expect("dag present")
|
||||
.state
|
||||
}
|
||||
|
||||
// ---- submit / dedup ----
|
||||
|
||||
#[test]
|
||||
fn submit_assigns_distinct_ids() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "first"));
|
||||
let b = submit(&q, rebuild("agent-b", "second"));
|
||||
assert_ne!(a, b);
|
||||
assert_eq!(q.snapshot().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedup_pending_same_template_and_agent() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "first"));
|
||||
let b = submit(&q, rebuild("agent-a", "auto sweep"));
|
||||
assert_eq!(a, b, "dedup should return existing id");
|
||||
let snap = q.snapshot();
|
||||
assert_eq!(snap.len(), 1);
|
||||
assert!(snap[0].reason.contains("first"));
|
||||
assert!(snap[0].reason.contains("auto sweep"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedup_does_not_apply_across_templates_or_agents() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "r"));
|
||||
let b = submit(&q, rebuild("agent-b", "r"));
|
||||
let c = submit(
|
||||
&q,
|
||||
templates::restart("agent-a", Source::Manual, "r".to_owned()),
|
||||
);
|
||||
assert_ne!(a, b);
|
||||
assert_ne!(a, c);
|
||||
assert_eq!(q.snapshot().len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedup_skips_running_dags() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "first"));
|
||||
let claim = claim_one(&q); // Prebuild running
|
||||
assert_eq!(claim.dag_id, a);
|
||||
// While the original runs, re-submit is legitimate new work.
|
||||
let again = submit(&q, rebuild("agent-a", "config bumped during build"));
|
||||
assert_ne!(a, again);
|
||||
assert_eq!(q.snapshot().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_update_dedup_matches_inputs() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(
|
||||
&q,
|
||||
templates::meta_update(
|
||||
vec!["nixpkgs".to_owned()],
|
||||
Source::Manual,
|
||||
"first".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
let b = submit(
|
||||
&q,
|
||||
templates::meta_update(
|
||||
vec!["nixpkgs".to_owned()],
|
||||
Source::Manual,
|
||||
"duplicate click".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
assert_eq!(a, b, "identical-inputs meta-updates should dedup");
|
||||
let c = submit(
|
||||
&q,
|
||||
templates::meta_update(
|
||||
vec!["agent-bitburner/bitburner-agent".to_owned()],
|
||||
Source::Manual,
|
||||
"bump agent".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
assert_ne!(a, c, "different-inputs meta-updates must NOT dedup");
|
||||
assert_eq!(q.snapshot().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn approval_dags_dedup_only_on_matching_id() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(
|
||||
&q,
|
||||
templates::approval_deploy("agent-a", 1, "approval #1".to_owned()),
|
||||
);
|
||||
let b = submit(
|
||||
&q,
|
||||
templates::approval_deploy("agent-a", 2, "approval #2".to_owned()),
|
||||
);
|
||||
assert_ne!(a, b, "distinct approvals must not collapse");
|
||||
// Rapid double-click on the same approval IS a single op.
|
||||
let c = submit(
|
||||
&q,
|
||||
templates::approval_deploy("agent-a", 1, "approval #1 (dup)".to_owned()),
|
||||
);
|
||||
assert_eq!(a, c);
|
||||
assert_eq!(q.snapshot().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn perm_change_dedup_respects_perm_type() {
|
||||
let q = JobQueue::new(1);
|
||||
let groups = templates::perm_change(
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"groups".to_owned(),
|
||||
PermPayload::ToolGroups { groups: vec![] },
|
||||
);
|
||||
let caps = templates::perm_change(
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"caps".to_owned(),
|
||||
PermPayload::Capabilities { caps: vec![] },
|
||||
);
|
||||
let a = submit(&q, groups.clone());
|
||||
let b = submit(&q, caps);
|
||||
assert_ne!(a, b, "tool-groups vs capabilities must not collapse");
|
||||
let c = submit(&q, groups);
|
||||
assert_eq!(a, c, "same perm type dedups");
|
||||
}
|
||||
|
||||
/// A `MetaUpdate` cascade `Rebuild` (with `parent_id = Some(meta_id)`)
|
||||
/// must NOT dedup into a queued `Rebuild` with a different
|
||||
/// `parent_id` (e.g. from a startup sweep) — without the guard the
|
||||
/// cascade child would be swallowed and the agent never rebuilt
|
||||
/// against the post-bump meta.
|
||||
#[test]
|
||||
fn dedup_respects_parent_id() {
|
||||
let q = JobQueue::new(1);
|
||||
let sweep = submit(&q, templates::startup_sweep("boot".to_owned(), vec![]));
|
||||
let sweep_child = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"alice",
|
||||
Source::StartupSweep,
|
||||
"startup sweep".to_owned(),
|
||||
Some(sweep),
|
||||
true,
|
||||
),
|
||||
);
|
||||
let meta = submit(
|
||||
&q,
|
||||
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
|
||||
);
|
||||
let cascade_child = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"alice",
|
||||
Source::MetaUpdate,
|
||||
"meta-update cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
);
|
||||
assert_ne!(sweep_child, cascade_child);
|
||||
let rebuilds = q
|
||||
.snapshot()
|
||||
.iter()
|
||||
.filter(|d| d.kind == Template::Rebuild && d.agent == "alice")
|
||||
.count();
|
||||
assert_eq!(rebuilds, 2, "both rebuilds must be present");
|
||||
}
|
||||
|
||||
// ---- cycle rejection ----
|
||||
|
||||
#[test]
|
||||
fn cyclic_dag_is_rejected_at_submit() {
|
||||
let q = JobQueue::new(1);
|
||||
let mut spec = rebuild("agent-a", "cyclic");
|
||||
// 0 → 1 → 0 cycle.
|
||||
spec.nodes = vec![
|
||||
NodeSpec {
|
||||
kind: NodeKind::StopForUpdate,
|
||||
deps: vec![Dep {
|
||||
on: 1,
|
||||
when: DepWhen::AfterOk,
|
||||
}],
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: vec![Dep {
|
||||
on: 0,
|
||||
when: DepWhen::AfterOk,
|
||||
}],
|
||||
},
|
||||
];
|
||||
assert!(q.submit(spec).is_err(), "cyclic spec must be refused");
|
||||
assert!(q.snapshot().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dep_is_rejected_at_submit() {
|
||||
let q = JobQueue::new(1);
|
||||
let mut spec = rebuild("agent-a", "bad dep");
|
||||
spec.nodes = vec![NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: vec![Dep {
|
||||
on: 9,
|
||||
when: DepWhen::AfterOk,
|
||||
}],
|
||||
}];
|
||||
assert!(q.submit(spec).is_err());
|
||||
}
|
||||
|
||||
// ---- dependency order within a DAG ----
|
||||
|
||||
#[test]
|
||||
fn rebuild_chain_claims_in_dep_order() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
for expected in ["prebuild", "stop_for_update", "swap", "reconcile"] {
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.dag_id, id);
|
||||
assert_eq!(c.kind.as_str(), expected);
|
||||
assert!(
|
||||
q.claim_ready().is_empty(),
|
||||
"chain must serialize: nothing ready while {expected} runs"
|
||||
);
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
assert_eq!(state_of(&q, id), State::Done);
|
||||
}
|
||||
|
||||
// ---- build slots ----
|
||||
|
||||
#[test]
|
||||
fn build_slot_serializes_nix_heavy_nodes() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "r"));
|
||||
let b = submit(&q, rebuild("agent-b", "r"));
|
||||
let first = claim_one(&q); // a's Prebuild takes the only slot
|
||||
assert_eq!(first.dag_id, a);
|
||||
assert_eq!(first.kind.as_str(), "prebuild");
|
||||
q.complete_node(a, first.node_id, Ok(()));
|
||||
// With the slot free again, FIFO gives... a's StopForUpdate is
|
||||
// slot-free (lease) and b's Prebuild takes the slot — both run.
|
||||
let claims = q.claim_ready();
|
||||
let kinds: Vec<(u64, &str)> = claims.iter().map(|c| (c.dag_id, c.kind.as_str())).collect();
|
||||
assert!(kinds.contains(&(a, "stop_for_update")));
|
||||
assert!(kinds.contains(&(b, "prebuild")));
|
||||
assert_eq!(claims.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_build_slots_run_two_prebuilds() {
|
||||
let q = JobQueue::new(2);
|
||||
submit(&q, rebuild("agent-a", "r"));
|
||||
submit(&q, rebuild("agent-b", "r"));
|
||||
let claims = q.claim_ready();
|
||||
assert_eq!(claims.len(), 2, "two slots → two concurrent prebuilds");
|
||||
assert!(claims.iter().all(|c| c.kind.as_str() == "prebuild"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fifo_fairness_for_the_slot() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "r"));
|
||||
let b = submit(&q, rebuild("agent-b", "r"));
|
||||
let c = submit(&q, rebuild("agent-c", "r"));
|
||||
let first = claim_one(&q);
|
||||
assert_eq!(first.dag_id, a, "submit order wins the slot");
|
||||
q.complete_node(a, first.node_id, Ok(()));
|
||||
let next: Vec<u64> = q.claim_ready().iter().map(|cl| cl.dag_id).collect();
|
||||
assert!(next.contains(&b), "b's prebuild before c's");
|
||||
assert!(!next.contains(&c));
|
||||
}
|
||||
|
||||
// ---- per-agent lease ----
|
||||
|
||||
#[test]
|
||||
fn lease_serializes_two_lifecycle_dags_for_same_agent() {
|
||||
let q = JobQueue::new(4);
|
||||
let restart = submit(
|
||||
&q,
|
||||
templates::restart("agent-a", Source::Manual, "restart".to_owned()),
|
||||
);
|
||||
let stop = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
Template::Stop,
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"stop".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
// Restart's StopForUpdate acquires the lease; stop's Reconcile
|
||||
// must wait even though slots are free.
|
||||
let first = claim_one(&q);
|
||||
assert_eq!(first.dag_id, restart);
|
||||
assert!(first.lease_acquired);
|
||||
q.complete_node(restart, first.node_id, Ok(()));
|
||||
// Same DAG keeps the lease for its Reconcile.
|
||||
let second = claim_one(&q);
|
||||
assert_eq!(second.dag_id, restart);
|
||||
assert!(!second.lease_acquired, "lease already held by this DAG");
|
||||
q.complete_node(restart, second.node_id, Ok(()));
|
||||
// Restart terminal → lease released → stop's Reconcile runs.
|
||||
let third = claim_one(&q);
|
||||
assert_eq!(third.dag_id, stop);
|
||||
q.complete_node(stop, third.node_id, Ok(()));
|
||||
assert_eq!(state_of(&q, restart), State::Done);
|
||||
assert_eq!(state_of(&q, stop), State::Done);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lease_exempt_prebuild_overlaps_other_dag_on_same_agent() {
|
||||
let q = JobQueue::new(2);
|
||||
submit(&q, rebuild("agent-a", "rebuild"));
|
||||
let stop = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
Template::Stop,
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"stop".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
// Prebuild is lease-exempt: the stop's Reconcile takes the lease
|
||||
// and runs concurrently with the rebuild's out-of-band nix build.
|
||||
let claims = q.claim_ready();
|
||||
let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect();
|
||||
assert!(kinds.contains(&"prebuild"));
|
||||
assert!(kinds.contains(&"reconcile"));
|
||||
// But the rebuild's StopForUpdate must then wait for the stop DAG
|
||||
// to finish (lease).
|
||||
let prebuild = claims
|
||||
.iter()
|
||||
.find(|c| c.kind.as_str() == "prebuild")
|
||||
.expect("prebuild claim")
|
||||
.clone();
|
||||
q.complete_node(prebuild.dag_id, prebuild.node_id, Ok(()));
|
||||
assert!(
|
||||
q.claim_ready().is_empty(),
|
||||
"StopForUpdate blocked while stop DAG holds the lease"
|
||||
);
|
||||
let reconcile = claims
|
||||
.iter()
|
||||
.find(|c| c.kind.as_str() == "reconcile")
|
||||
.expect("reconcile claim")
|
||||
.clone();
|
||||
q.complete_node(stop, reconcile.node_id, Ok(()));
|
||||
let next = claim_one(&q);
|
||||
assert_eq!(next.kind.as_str(), "stop_for_update");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agents_do_not_contend_on_each_others_leases() {
|
||||
let q = JobQueue::new(4);
|
||||
submit(
|
||||
&q,
|
||||
templates::restart("agent-a", Source::Manual, "r".to_owned()),
|
||||
);
|
||||
submit(
|
||||
&q,
|
||||
templates::restart("agent-b", Source::Manual, "r".to_owned()),
|
||||
);
|
||||
let claims = q.claim_ready();
|
||||
assert_eq!(claims.len(), 2, "different agents run concurrently");
|
||||
}
|
||||
|
||||
// ---- failure: cancel-downstream + AfterAny ----
|
||||
|
||||
#[test]
|
||||
fn failed_node_cancels_downstream_but_afterany_reconcile_runs() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
let prebuild = claim_one(&q);
|
||||
q.complete_node(id, prebuild.node_id, Err("nix build exploded".to_owned()));
|
||||
// StopForUpdate + Swap are cancelled (AfterOk on a failed chain);
|
||||
// the AfterAny Reconcile still runs once Swap is terminal.
|
||||
let reconcile = claim_one(&q);
|
||||
assert_eq!(reconcile.kind.as_str(), "reconcile");
|
||||
q.complete_node(id, reconcile.node_id, Ok(()));
|
||||
let snap = q.snapshot();
|
||||
let dag = snap.iter().find(|d| d.id == id).expect("dag");
|
||||
assert_eq!(dag.state, State::Failed, "roll-up failed");
|
||||
let by_kind = |k: &str| {
|
||||
dag.nodes
|
||||
.iter()
|
||||
.find(|n| n.kind == k)
|
||||
.expect("node present")
|
||||
.state
|
||||
};
|
||||
assert_eq!(by_kind("prebuild"), State::Failed);
|
||||
assert_eq!(by_kind("stop_for_update"), State::Cancelled);
|
||||
assert_eq!(by_kind("swap"), State::Cancelled);
|
||||
assert_eq!(by_kind("reconcile"), State::Done);
|
||||
assert_eq!(
|
||||
dag.nodes
|
||||
.iter()
|
||||
.find(|n| n.kind == "prebuild")
|
||||
.and_then(|n| n.error.as_deref()),
|
||||
Some("nix build exploded")
|
||||
);
|
||||
}
|
||||
|
||||
/// The swap-failure recovery: `Swap` fails → the `AfterAny` edge still
|
||||
/// runs `Reconcile`, which brings a wanted-up agent back on its old
|
||||
/// config.
|
||||
#[test]
|
||||
fn swap_failure_still_runs_reconcile() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
for _ in 0..2 {
|
||||
let c = claim_one(&q);
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
let swap = claim_one(&q);
|
||||
assert_eq!(swap.kind.as_str(), "swap");
|
||||
q.complete_node(id, swap.node_id, Err("update failed".to_owned()));
|
||||
let reconcile = claim_one(&q);
|
||||
assert_eq!(reconcile.kind.as_str(), "reconcile");
|
||||
q.complete_node(id, reconcile.node_id, Ok(()));
|
||||
assert_eq!(state_of(&q, id), State::Failed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_reconcile_marks_dag_failed() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
Template::Start,
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"start".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
let c = claim_one(&q);
|
||||
q.complete_node(id, c.node_id, Err("start failed".to_owned()));
|
||||
assert_eq!(state_of(&q, id), State::Failed);
|
||||
}
|
||||
|
||||
// ---- cancel ----
|
||||
|
||||
#[test]
|
||||
fn cancel_clears_queued_dag() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
assert!(q.cancel(id));
|
||||
assert_eq!(state_of(&q, id), State::Cancelled);
|
||||
assert!(q.claim_ready().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_refuses_running_dag() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
let _ = claim_one(&q);
|
||||
assert!(!q.cancel(id));
|
||||
assert_eq!(state_of(&q, id), State::Running);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_children_marks_queued_children_only() {
|
||||
let q = JobQueue::new(1);
|
||||
let meta = submit(
|
||||
&q,
|
||||
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
|
||||
);
|
||||
// Parent's MetaLock is running while children exist.
|
||||
let lock = claim_one(&q);
|
||||
assert_eq!(lock.dag_id, meta);
|
||||
let child_a = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"agent-a",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
);
|
||||
let child_b = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"agent-b",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
);
|
||||
let unrelated = submit(&q, rebuild("agent-c", "operator queued"));
|
||||
// MetaLock holds the single build slot, so both children (and the
|
||||
// unrelated rebuild) are still fully queued here.
|
||||
let cancelled = q.cancel_children(meta);
|
||||
assert_eq!(cancelled, 2);
|
||||
assert_eq!(state_of(&q, child_a), State::Cancelled);
|
||||
assert_eq!(state_of(&q, child_b), State::Cancelled);
|
||||
assert_eq!(state_of(&q, unrelated), State::Queued);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_children_skips_running_child() {
|
||||
let q = JobQueue::new(2);
|
||||
let meta = submit(
|
||||
&q,
|
||||
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
|
||||
);
|
||||
let lock = claim_one(&q);
|
||||
let running_child = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"agent-a",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
);
|
||||
let queued_child = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"agent-b",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
);
|
||||
// Second slot lets running_child's prebuild start.
|
||||
let child_claim = claim_one(&q);
|
||||
assert_eq!(child_claim.dag_id, running_child);
|
||||
let n = q.cancel_children(meta);
|
||||
assert_eq!(n, 1);
|
||||
assert_eq!(state_of(&q, running_child), State::Running);
|
||||
assert_eq!(state_of(&q, queued_child), State::Cancelled);
|
||||
q.complete_node(meta, lock.node_id, Ok(()));
|
||||
}
|
||||
|
||||
// ---- fan-out ----
|
||||
|
||||
#[test]
|
||||
fn append_children_sets_parent_and_dedups() {
|
||||
let q = JobQueue::new(1);
|
||||
let meta = submit(
|
||||
&q,
|
||||
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
|
||||
);
|
||||
let specs = vec![
|
||||
templates::rebuild(
|
||||
"alice",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
templates::rebuild(
|
||||
"bob",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
// Duplicate — must coalesce into the first alice child.
|
||||
templates::rebuild(
|
||||
"alice",
|
||||
Source::MetaUpdate,
|
||||
"cascade again".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
];
|
||||
let ids = q.append_children(specs);
|
||||
assert_eq!(ids.len(), 3);
|
||||
assert_eq!(ids[0], ids[2], "duplicate child dedups");
|
||||
let snap = q.snapshot();
|
||||
let children: Vec<_> = snap.iter().filter(|d| d.parent_id == Some(meta)).collect();
|
||||
assert_eq!(children.len(), 2);
|
||||
}
|
||||
|
||||
// ---- terminal reporting + lease release ----
|
||||
|
||||
#[test]
|
||||
fn terminal_dag_reported_exactly_once_and_lease_released() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::restart("agent-a", Source::Manual, "r".to_owned()),
|
||||
);
|
||||
let stop = claim_one(&q);
|
||||
let r1 = q.complete_node(id, stop.node_id, Ok(()));
|
||||
assert!(r1.terminal.is_empty(), "dag not terminal yet");
|
||||
let rec = claim_one(&q);
|
||||
let r2 = q.complete_node(id, rec.node_id, Ok(()));
|
||||
assert_eq!(r2.terminal.len(), 1);
|
||||
assert_eq!(r2.terminal[0].dag_id, id);
|
||||
assert_eq!(r2.terminal[0].state, State::Done);
|
||||
// Lease released: a new DAG for the agent can claim immediately.
|
||||
let next = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
Template::Stop,
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"stop".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.dag_id, next);
|
||||
assert!(c.lease_acquired);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancelled_dag_reports_terminal() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
assert!(q.cancel(id));
|
||||
// The cancel path settles internally; a subsequent completion
|
||||
// report must not re-report it. Verify via a second dag's cycle.
|
||||
let other = submit(&q, rebuild("agent-b", "r"));
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.dag_id, other);
|
||||
let report = q.complete_node(other, c.node_id, Err("boom".to_owned()));
|
||||
// agent-b's dag isn't terminal (reconcile still pending) and
|
||||
// agent-a's was already reported by cancel → nothing here.
|
||||
assert!(report.terminal.iter().all(|t| t.dag_id != id));
|
||||
}
|
||||
|
||||
// ---- steps, build logs, history ----
|
||||
|
||||
#[test]
|
||||
fn set_step_only_on_running_and_signals_change() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
assert!(!q.set_step(id, 0, "too early"), "queued node refuses step");
|
||||
let c = claim_one(&q);
|
||||
assert!(q.set_step(id, c.node_id, "nix build"));
|
||||
assert!(
|
||||
!q.set_step(id, c.node_id, "nix build"),
|
||||
"same label → false"
|
||||
);
|
||||
assert!(q.set_step(id, c.node_id, "next phase"));
|
||||
assert!(q.set_step_running(id, "via running lookup"));
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
let snap = q.snapshot();
|
||||
let node = &snap.iter().find(|d| d.id == id).expect("dag").nodes[0];
|
||||
assert_eq!(node.step, None, "step cleared on completion");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_build_log_id_links_running_node() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
assert!(!q.set_build_log_id(id, 0, 41), "queued node refuses log id");
|
||||
let c = claim_one(&q);
|
||||
assert!(q.set_build_log_id(id, c.node_id, 42));
|
||||
assert!(q.set_build_log_id_running(id, 43));
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
let snap = q.snapshot();
|
||||
let node = &snap.iter().find(|d| d.id == id).expect("dag").nodes[0];
|
||||
assert_eq!(node.build_log_id, Some(43), "log id survives completion");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn history_evicts_old_terminals_per_template() {
|
||||
let q = JobQueue::new(1);
|
||||
for i in 0..8 {
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
Template::Start,
|
||||
&format!("agent-{i}"),
|
||||
Source::Manual,
|
||||
"start".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
let c = claim_one(&q);
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
assert_eq!(q.snapshot().len(), 5, "per-template history cap");
|
||||
assert_eq!(q.live_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_is_truncated() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
let c = claim_one(&q);
|
||||
q.complete_node(id, c.node_id, Err("x".repeat(5000)));
|
||||
let snap = q.snapshot();
|
||||
let err = snap.iter().find(|d| d.id == id).expect("dag").nodes[0]
|
||||
.error
|
||||
.clone()
|
||||
.expect("error stored");
|
||||
assert!(err.chars().count() <= 2001, "truncated + ellipsis");
|
||||
assert!(err.ends_with('…'));
|
||||
}
|
||||
|
||||
// ---- template shapes ----
|
||||
|
||||
#[test]
|
||||
fn graceful_stop_shape_signal_drain_reconcile() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::graceful_stop("agent-a", Source::Manual, "graceful".to_owned()),
|
||||
);
|
||||
for expected in ["signal", "drain", "reconcile"] {
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.kind.as_str(), expected);
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
assert_eq!(state_of(&q, id), State::Done);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn graceful_signal_and_drain_hold_no_build_slot() {
|
||||
// A whole-hive graceful stop overlaps every drain even at
|
||||
// buildSlots = 1 while a rebuild hogs the slot.
|
||||
let q = JobQueue::new(1);
|
||||
submit(&q, rebuild("builder", "slot hog"));
|
||||
submit(
|
||||
&q,
|
||||
templates::graceful_stop("agent-a", Source::Manual, "g".to_owned()),
|
||||
);
|
||||
submit(
|
||||
&q,
|
||||
templates::graceful_stop("agent-b", Source::Manual, "g".to_owned()),
|
||||
);
|
||||
let claims = q.claim_ready();
|
||||
let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect();
|
||||
assert_eq!(
|
||||
kinds,
|
||||
vec!["prebuild", "signal", "signal"],
|
||||
"both agents' signals fire while the slot is held"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spawn_shape_create_dropin_reconcile() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::spawn("newbie", 7, "approval #7 spawn".to_owned()),
|
||||
);
|
||||
for expected in ["create", "write_dropin", "reconcile"] {
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.kind.as_str(), expected);
|
||||
assert_eq!(c.approval_id, Some(7));
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
let report_terminal = state_of(&q, id);
|
||||
assert_eq!(report_terminal, State::Done);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn perm_change_shape_prefixes_rebuild_chain() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::perm_change(
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"perm".to_owned(),
|
||||
PermPayload::Combined {
|
||||
groups: Some(vec![]),
|
||||
caps: None,
|
||||
},
|
||||
),
|
||||
);
|
||||
for expected in [
|
||||
"write_perm_file",
|
||||
"prebuild",
|
||||
"stop_for_update",
|
||||
"swap",
|
||||
"reconcile",
|
||||
] {
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.kind.as_str(), expected);
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
assert_eq!(state_of(&q, id), State::Done);
|
||||
}
|
||||
Loading…
Reference in a new issue