diff --git a/hive-c0re/src/dashboard/build_logs.rs b/hive-c0re/src/dashboard/build_logs.rs index 763ee9fb..f4835878 100644 --- a/hive-c0re/src/dashboard/build_logs.rs +++ b/hive-c0re/src/dashboard/build_logs.rs @@ -156,7 +156,7 @@ pub(super) async fn get_build_log_for_node( State(state): State, AxumPath(node_id): AxumPath, ) -> Response { - match state.coord.build_logs.id_for_node(node_id) { + match state.coord.job_queue.build_log_id_of(node_id) { Some(log_id) => get_build_log_full(State(state), AxumPath(log_id)).await, None => ( StatusCode::NOT_FOUND, @@ -183,7 +183,7 @@ pub(super) async fn get_build_log_raw_for_node( State(state): State, AxumPath(node_id): AxumPath, ) -> Response { - match state.coord.build_logs.id_for_node(node_id) { + match state.coord.job_queue.build_log_id_of(node_id) { Some(log_id) => get_build_log_raw(State(state), AxumPath(log_id)).await, None => ( StatusCode::NOT_FOUND, diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index d1449bdd..95486951 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -10,9 +10,11 @@ use std::sync::Arc; use anyhow::{Context as _, Result}; -use hive_jobq::{NodeId, TerminalState}; +use super::{Claim, Declare}; +use hive_jobq::TerminalState; use super::model::NodeKind; +use super::resource::Resource; use crate::coordinator::Coordinator; use crate::power::{ReconcileAction, reconcile_action}; @@ -24,104 +26,109 @@ use crate::power::{ReconcileAction, reconcile_action}; /// N × this timeout. pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(3); +/// Extra signal an executor hands back to the scheduler alongside +/// success. +#[derive(Default)] +pub struct NodeOutput { + /// Whole per-agent *subgraphs* to append into *this same* DAG at + /// runtime — the single in-DAG-growth channel. Each [`Job`] is one + /// independent subgraph, declared but not yet inserted: an executor cannot + /// reach the queue, so it hands the declaration back and the scheduler + /// inserts it via [`super::JobQueue::append_subgraph`] under its own lock, + /// rooted on the emitting node. Used both for the multi-node case + /// (`MetaLock` growing one rebuild subgraph per agent — the startup + /// sweep's stale agents, the meta-update cascade's affected agents) and + /// the single-node case (a `Reconcile` planner emitting its mechanical + /// `Start` / `Stop` as a one-node subgraph). The scheduler applies these + /// *before* the emitting node's completion so the DAG never rolls terminal + /// with the appended work still pending — keeping the lease-window + /// transient held across the sub-step. + pub append_subgraph: Vec, +} + +impl std::fmt::Debug for NodeOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // The subgraphs are closures — how many were emitted is the only thing + // there is to say about them before the queue runs them. + f.debug_struct("NodeOutput") + .field("append_subgraph", &self.append_subgraph.len()) + .finish() + } +} + +/// Build-log sink for one claimed node. +struct Ctx<'a> { + coord: &'a Arc, + dag_id: u64, + node_id: super::NodeId, +} + +impl Ctx<'_> { + fn build_log(&self, log_id: i64) { + if self + .coord + .job_queue + .set_build_log_id(self.dag_id, self.node_id, log_id) + { + self.coord.emit_rebuild_queue_snapshot(); + } + } +} + /// Run one claimed node to completion. Called from a task the /// scheduler spawns per claim; the `Result` (stringified) becomes the /// node's terminal state. -/// -/// `job` is the node's own growth channel: an executor that decides more work -/// is needed declares it here, and the scheduler inserts it under this node -/// when the node completes. Most executors never touch it. Nothing is inserted -/// while the node runs — the builder is local state, so this stays outside the -/// queue's lock for the whole (often multi-minute) execution. -/// -/// ⚠️ Taken **by value and handed back**, not by reference. A `JobBuilder` is -/// `RefCell`-backed: owned it is `Send`, but `&JobBuilder` is not (a shared ref -/// is `Send` only if the referent is `Sync`, and `RefCell` never is). A `&Job` -/// parameter would be live across every `.await` in this fn and make the whole -/// future non-`Send`, which the scheduler's `tokio::spawn` rejects. So the -/// growth executors below return *what to grow* and the declaration happens -/// here, synchronously, between awaits. -/// -/// The node is identified by its own id + payload rather than by a `Claim` -/// side-struct: `kind` already carries the agent, and the DAG id is a -/// derived read (`JobQueue::dag_of`) the three arms that need it take -/// themselves. Nothing here needs a claim to exist as a type. -pub(super) async fn run_node( - coord: &Arc, - job: super::Job, - id: NodeId, - kind: &NodeKind, -) -> (super::Job, Result<()>) { - // The agent this node targets rides the payload — empty for the agentless - // container kinds (`MetaLock`, `Dag`), which never read it. - let agent = kind.agent(); - // Every arm is `Result<()>`; the three that grow work declare into `job` - // *synchronously*, after their own awaits have finished. Borrowing `&job` - // inside an `.await` would make this future non-`Send` (see above), so the - // growth executors return what to grow rather than taking the builder. - let result = match kind { - NodeKind::MetaSync { relock, .. } => run_meta_sync(coord, agent, *relock).await, - NodeKind::Prebuild { .. } => run_prebuild(agent, id).await, - NodeKind::Swap { .. } => run_swap(coord, agent, id).await, - NodeKind::PostSwap { .. } => run_post_swap(coord, agent).await, - NodeKind::Provision { .. } => run_provision(coord, agent).await, - NodeKind::Create { .. } => run_create(agent).await, +pub(super) async fn run_node(coord: &Arc, claim: &Claim) -> Result { + let ctx = Ctx { + coord, + dag_id: claim.dag_id, + node_id: claim.node_id, + }; + match &claim.kind { + NodeKind::MetaSync { relock, .. } => run_meta_sync(coord, claim, *relock).await, + NodeKind::Prebuild { .. } => run_prebuild(claim, &ctx).await, + NodeKind::Swap { .. } => run_swap(coord, claim, &ctx).await, + NodeKind::PostSwap { .. } => run_post_swap(coord, claim).await, + NodeKind::Provision { .. } => run_provision(coord, claim).await, + NodeKind::Create { .. } => run_create(claim).await, NodeKind::MetaLock { sweep, fanout, inputs, - } => run_meta_lock(coord, *sweep, fanout.clone(), inputs) - .await - .map(|(agents, opts)| super::templates::grown_rebuilds(&job, &agents, opts)), - NodeKind::Reconcile { .. } => run_reconcile(coord, agent).await.map(|sub| { - if let Some(kind) = sub { - super::templates::fanned_out_mechanical(&job, kind); - } - }), - NodeKind::Start { .. } => run_start(coord, agent).await, - NodeKind::Stop { .. } => run_stop(coord, agent).await, - NodeKind::StopForUpdate { .. } => run_stop_for_update(coord, agent).await, - NodeKind::Signal { .. } => { - run_signal(coord, agent); - Ok(()) - } - NodeKind::Drain { .. } => run_drain(coord, agent).await, - NodeKind::WriteDropin { .. } => run_write_dropin(coord, agent).await, - // The payload rides the node and is destructured here, so the executor - // takes it directly instead of re-matching the kind behind a `bail!` - // that could never fire. - NodeKind::WritePermFile { payload, .. } => run_write_perm_file(coord, agent, payload).await, - NodeKind::Reparent { moves } => run_reparent(coord, moves).await, + } => run_meta_lock(coord, *sweep, fanout.clone(), inputs).await, + NodeKind::Reconcile { .. } => run_reconcile(coord, claim).await, + NodeKind::Start { .. } => run_start(coord, claim).await, + NodeKind::Stop { .. } => run_stop(coord, claim).await, + NodeKind::StopForUpdate { .. } => run_stop_for_update(coord, claim).await, + NodeKind::Signal { .. } => Ok(run_signal(coord, claim)), + NodeKind::Drain { .. } => run_drain(coord, claim).await, + NodeKind::WriteDropin { .. } => run_write_dropin(coord, claim).await, + NodeKind::WritePermFile { .. } => run_write_perm_file(coord, claim).await, + NodeKind::Reparent { .. } => run_reparent(coord, claim).await, NodeKind::MergeVerify { approval_id, .. } => run_merge_verify(coord, *approval_id).await, NodeKind::DeployApply { approval_id, .. } => { - run_deploy_apply(coord, *approval_id).await.map(|()| { - super::templates::deploy_rebuild_nodes(&job, agent, *approval_id); - }) + run_deploy_apply(coord, claim, *approval_id).await } NodeKind::FinalizeDeploy { approval_id, .. } => { run_finalize_deploy(coord, *approval_id).await } NodeKind::DeployTail { approval_id, .. } => { - run_deploy_tail(coord, coord.job_queue.dag_of(id), agent, *approval_id).await + run_deploy_tail(coord, claim, *approval_id).await } NodeKind::ResolveApproval { approval_id, outcome, - } => run_resolve_approval(coord, coord.job_queue.dag_of(id), *approval_id, *outcome).await, - NodeKind::EmitRebuilt { ok, .. } => { - run_emit_rebuilt(coord, agent, coord.job_queue.dag_of(id), *ok); - Ok(()) - } - NodeKind::SetWanted { up, .. } => run_set_wanted(coord, agent, *up), + } => run_resolve_approval(coord, claim, *approval_id, *outcome).await, + NodeKind::EmitRebuilt { ok, .. } => Ok(run_emit_rebuilt(coord, claim, *ok)), + NodeKind::SetWanted { up, .. } => run_set_wanted(coord, claim, *up), // The two nodes that carry no work of their own; completing either // lets it reach `Finishing` so the nodes under it start. // - `Dag`: pure grouping container. The DAG's terminal side effect, if // any, is its own tail node in the graph. // - `DeployWindow`: pure resource holder — the meta window, agent lease // and build slot it declares stay held until its subtree settles. - NodeKind::Dag { .. } | NodeKind::DeployWindow { .. } => Ok(()), - }; - (job, result) + NodeKind::Dag { .. } | NodeKind::DeployWindow { .. } => Ok(NodeOutput::default()), + } } /// Resolve the DAG's approval row the way this node's own `outcome` says. @@ -133,30 +140,31 @@ pub(super) async fn run_node( /// since the work already happened and failing the tail would only misreport it. async fn run_resolve_approval( coord: &Arc, - dag_id: Option, + claim: &Claim, approval_id: i64, outcome: TerminalState, -) -> Result<()> { +) -> Result { let reason = (outcome == TerminalState::Failed) - .then(|| dag_id.and_then(|dag| coord.job_queue.first_error(dag))) + .then(|| coord.job_queue.first_error(claim.dag_id)) .flatten(); crate::actions::resolve_approval_dag(coord, approval_id, outcome, reason.as_deref()).await; - Ok(()) + Ok(NodeOutput::default()) } /// Emit this agent's `Rebuilt` manager event. `ok` is not computed — it is which /// of the tail pair the graph let run. The failure note comes from the DAG's /// first failing node, since the branch knows *that* it failed but not *why*. -fn run_emit_rebuilt(coord: &Arc, agent: &str, dag_id: Option, ok: bool) { +fn run_emit_rebuilt(coord: &Arc, claim: &Claim, ok: bool) -> NodeOutput { coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { - agent: agent.to_owned(), + agent: claim.agent.clone(), ok, note: (!ok) - .then(|| dag_id.and_then(|dag| coord.job_queue.first_error(dag))) + .then(|| coord.job_queue.first_error(claim.dag_id)) .flatten(), sha: None, tag: None, }); + NodeOutput::default() } /// Write the agent's durable power intent — the DAG-node form of the old @@ -167,7 +175,7 @@ fn run_emit_rebuilt(coord: &Arc, agent: &str, dag_id: Option, /// warn-and-continue write, a failed write fails the node (cancel-downstream /// cancels the `Reconcile`) rather than letting it converge to a stale /// intent — that atomicity is the point of moving it into the DAG. -fn run_set_wanted(coord: &Arc, agent: &str, up: bool) -> Result<()> { +fn run_set_wanted(coord: &Arc, claim: &Claim, up: bool) -> Result { let wanted = if up { crate::power::Wanted::Up } else { @@ -175,9 +183,9 @@ fn run_set_wanted(coord: &Arc, agent: &str, up: bool) -> Result<()> }; coord .power - .set(agent, wanted) - .with_context(|| format!("set wanted={} for agent {agent}", wanted.as_str()))?; - Ok(()) + .set(&claim.agent, wanted) + .with_context(|| format!("set wanted={} for agent {}", wanted.as_str(), claim.agent))?; + Ok(NodeOutput::default()) } /// The rebuild's meta preamble: runtime-dir prep, an idempotent meta @@ -189,7 +197,12 @@ fn run_set_wanted(coord: &Arc, agent: &str, up: bool) -> Result<()> /// Deliberately a separate node from the [`run_prebuild`] it feeds: that /// build takes minutes and only *reads* the store, so keeping the global /// window off it is what lets rebuilds of different agents overlap. -async fn run_meta_sync(coord: &Arc, name: &str, relock: bool) -> Result<()> { +async fn run_meta_sync( + coord: &Arc, + claim: &Claim, + relock: bool, +) -> Result { + let name = &claim.agent; // Runs while the agent is still up — the runtime dir and MCP listener // already exist. Use the pure path accessor; no need to re-register the // listener (event-driven: registered at start/create). @@ -206,7 +219,7 @@ async fn run_meta_sync(coord: &Arc, name: &str, relock: bool) -> Re if relock { crate::meta::lock_update_for_rebuild(name).await?; } - Ok(()) + Ok(NodeOutput::default()) } /// Out-of-band toplevel build while the container keeps serving: warm @@ -216,16 +229,18 @@ async fn run_meta_sync(coord: &Arc, name: &str, relock: bool) -> Re /// container is already down: its only purpose is to shrink the swap's /// downtime window, so a stopped agent (no uptime to preserve) doesn't /// pay the double eval — `Swap` builds inline instead. -async fn run_prebuild(name: &str, id: NodeId) -> Result<()> { +async fn run_prebuild(claim: &Claim, ctx: &Ctx<'_>) -> Result { + let name = &claim.agent; // Warm the toplevel build only when the container is up — the whole // point of prebuild is to shrink the swap's downtime window. A // stopped agent has no uptime to preserve, so skip the (expensive) // eval and let the downstream `Swap` build inline. if crate::lifecycle::is_running(name).await { let flake_ref = format!("{}#{name}", crate::paths::meta_root().display()); - crate::lifecycle::prebuild_toplevel(name, &flake_ref, Some(id.get())).await?; + crate::lifecycle::prebuild_toplevel(name, &flake_ref, &|log_id| ctx.build_log(log_id)) + .await?; } - Ok(()) + Ok(NodeOutput::default()) } /// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb), @@ -233,13 +248,17 @@ async fn run_prebuild(name: &str, id: NodeId) -> Result<()> { /// (rev marker, `Rebuilt` event, forge/matrix sync, kick, rescan). /// The recovery-start on failure is NOT here — the DAG's tail /// `Reconcile` runs after this node terminal ok *or* fail. -async fn run_swap(coord: &Arc, name: &str, id: NodeId) -> Result<()> { +async fn run_swap(coord: &Arc, claim: &Claim, ctx: &Ctx<'_>) -> Result { + let name = &claim.agent; // Swap runs on an already-existing (stopped) container — runtime dir // and listener were created earlier. Pure path accessor suffices. let agent_dir = crate::paths::agent_runtime_dir(name); let hive = coord.hive_env(); let paths = Coordinator::agent_paths(name, agent_dir); - let result = crate::lifecycle::swap_update(name, &hive, &paths, Some(id.get())).await; + let result = crate::lifecycle::swap_update(name, &hive, &paths, &|log_id| { + ctx.build_log(log_id); + }) + .await; // On success the Ok-only bookkeeping tail (rev marker, forge/matrix // sync, kick, rescan, snapshot) runs in the sibling `PostSwap` node, // which deps `AfterOk(Swap)`. On failure `PostSwap` is cancel-cascaded @@ -250,7 +269,7 @@ async fn run_swap(coord: &Arc, name: &str, id: NodeId) -> Result<() if result.is_err() { coord.rescan_containers_and_emit().await; } - result + result.map(|()| NodeOutput::default()) } /// The post-`Swap` bookkeeping tail, split into its own node for dashboard @@ -258,7 +277,8 @@ async fn run_swap(coord: &Arc, name: &str, id: NodeId) -> Result<() /// means the profile swap succeeded. Store/forge/matrix work only — no nix /// build (build-slot-exempt); the agent lease taken at `Swap` is still held /// (the whole chain up to `Reconcile` is one agent's subgraph). -async fn run_post_swap(coord: &Arc, name: &str) -> Result<()> { +async fn run_post_swap(coord: &Arc, claim: &Claim) -> Result { + let name = &claim.agent; if let Some(rev) = crate::auto_update::current_flake_rev(&coord.hyperhive_flake) && let Err(e) = std::fs::write(crate::paths::applied_rev_marker(name), rev) { @@ -278,19 +298,20 @@ async fn run_post_swap(coord: &Arc, name: &str) -> Result<()> { coord.kick_agent(name, "container rebuilt"); coord.rescan_containers_and_emit().await; crate::dashboard::emit_meta_inputs_snapshot(coord); - Ok(()) + Ok(NodeOutput::default()) } /// First-spawn pre-create provisioning: proposed/applied repos, state /// subvolume, and the meta `sync_agents` registration. Runs under the /// deploy window (it declares `Resource::MetaWindow`) so its commit can't /// land inside another node's staged deploy window. -async fn run_provision(coord: &Arc, name: &str) -> Result<()> { +async fn run_provision(coord: &Arc, claim: &Claim) -> Result { + let name = &claim.agent; let agent_dir = crate::paths::agent_runtime_dir(name); let hive = coord.hive_env(); let paths = Coordinator::agent_paths(name, agent_dir); crate::lifecycle::provision_container(name, &hive, &paths).await?; - Ok(()) + Ok(NodeOutput::default()) } /// `nixos-container create` proper — the upstream `Provision` node @@ -299,24 +320,21 @@ async fn run_provision(coord: &Arc, name: &str) -> Result<()> { /// dir creation and MCP listener registration are deferred to the tail /// `Reconcile` (`converge_start_preamble` + `register_agent`) so this /// node stays purely "create", not "create + start". -async fn run_create(name: &str) -> Result<()> { - crate::lifecycle::create_only(name).await?; - Ok(()) +async fn run_create(claim: &Claim) -> Result { + crate::lifecycle::create_only(&claim.agent).await?; + Ok(NodeOutput::default()) } /// Meta flake lock bump. Boot-sweep flavour is non-fatal (a failed /// bump must not cancel the fan-out rebuilds — they proceed against /// the current lock, exactly like today's sweep); the meta-update /// flavour propagates errors, and a failed bump fans out nothing. -/// Returns the agents whose rebuild subgraphs the caller should grow into this -/// node, and the options to build them with — rather than declaring them here. -/// The declaration has to happen outside any `.await` (see [`run_node`]). async fn run_meta_lock( coord: &Arc, sweep: bool, fanout: Option>, inputs: &[String], -) -> Result<(Vec, super::templates::RebuildOpts)> { +) -> Result { if sweep { if let Err(e) = crate::meta::lock_update_hyperhive().await { tracing::warn!(error = ?e, "startup sweep: meta lock_update_hyperhive failed"); @@ -331,13 +349,25 @@ async fn run_meta_lock( // drain window rather than being cut off. The per-agent drains overlap, // so the sweep's cost ceiling is one `GRACEFUL_STOP_TIMEOUT` in total, // not one per agent. - return Ok(( - fanout.unwrap_or_default(), - super::templates::RebuildOpts { - relock: true, - graceful: true, - }, - )); + let append_subgraph = fanout + .unwrap_or_default() + .iter() + .map(|agent| { + let agent = agent.clone(); + Box::new(move |b: &super::Job| { + super::templates::rebuild_nodes( + b, + &agent, + super::templates::RebuildOpts { + relock: true, + graceful: true, + }, + None, + ); + }) as Declare + }) + .collect(); + return Ok(NodeOutput { append_subgraph }); } let _progress = coord.meta_update_guard(); crate::meta::lock_update(inputs).await?; @@ -353,46 +383,69 @@ async fn run_meta_lock( // cascade children must NOT re-lock, which would revert the bump this // node just committed (the property the old `fanout_specs` meta-update // branch encoded). - Ok(( - cascade, - super::templates::RebuildOpts { - relock: false, - graceful: false, - }, - )) + let append_subgraph = cascade + .iter() + .map(|agent| { + let agent = agent.clone(); + Box::new(move |b: &super::Job| { + super::templates::rebuild_nodes( + b, + &agent, + super::templates::RebuildOpts { + relock: false, + graceful: false, + }, + None, + ); + }) as Declare + }) + .collect(); + Ok(NodeOutput { append_subgraph }) } /// Idempotent power-converge *planner*: compare `wanted` (durable /// intent) against observed state and, when they diverge, fan the /// mechanical `Start` / `Stop` out as a first-class node appended to -/// *this* DAG (a single node declared into `job`, rooted on this node). -/// Does no container work itself — the sub-step becomes visible in the -/// DAG and the lease-window transient (or the sub-step's own node-local -/// guard) rides across it. -/// Returns the mechanical node to fan out (`None` on a noop) rather than -/// declaring it — the declaration has to happen outside any `.await`, see -/// [`run_node`]. `NodeKind` carries the agent it targets, so this node's agent -/// is stamped into the fanned-out kind here. -async fn run_reconcile(coord: &Arc, name: &str) -> Result> { +/// *this* DAG (a single-node `NodeOutput::append_subgraph` rooted on +/// this node). Does no container work itself — the sub-step becomes +/// visible in the DAG and the lease-window transient (or the sub-step's +/// own node-local guard) rides across it. +async fn run_reconcile(coord: &Arc, claim: &Claim) -> Result { + let name = &claim.agent; let running = crate::lifecycle::is_running(name).await; let wanted = coord.power.get_or_seed(name, running)?; - Ok(match reconcile_action(wanted, running) { - ReconcileAction::Start => Some(NodeKind::Start { - agent: name.to_owned(), + // One node targeting this agent, rooted on this reconcile node. `NodeKind` + // carries the agent it targets, so stamp `claim.agent` into the fanned-out + // Start/Stop kind (one in-DAG-growth channel). + let sub = |kind: NodeKind| { + // `Start` / `Stop` declare the lease they run under. This node is their + // parent and holds it, so the declaration is a re-entrant borrow — no + // second unit, no deadlock. It exists so the requirement belongs to the + // node rather than to the fact that a `Reconcile` happens to fan it out. + let lease = Resource::Agent(kind.agent().to_owned()); + vec![Box::new(move |b: &super::Job| { + let _ = b.node(kind).needs(lease); + }) as Declare] + }; + let append_subgraph = match reconcile_action(wanted, running) { + ReconcileAction::Start => sub(NodeKind::Start { + agent: name.clone(), }), - ReconcileAction::Stop => Some(NodeKind::Stop { - agent: name.to_owned(), + ReconcileAction::Stop => sub(NodeKind::Stop { + agent: name.clone(), }), ReconcileAction::Noop => { tracing::debug!(%name, wanted = wanted.as_str(), running, "reconcile: noop"); - None + Vec::new() } - }) + }; + Ok(NodeOutput { append_subgraph }) } /// Mechanical container start — the sub-step a `Reconcile` planner fans /// out when it observes `wanted = Up` and the container down. -async fn run_start(coord: &Arc, name: &str) -> Result<()> { +async fn run_start(coord: &Arc, claim: &Claim) -> Result { + let name = &claim.agent; // No node-local transient guard: the pill is derived from the running node // set, and `Start` reports `Starting` via `NodeKind::transient_kind`. This // used to take one "only when the DAG holds none", which was a second @@ -415,26 +468,28 @@ async fn run_start(coord: &Arc, name: &str) -> Result<()> { coord.register_agent(name)?; coord.kick_agent(name, "container started"); coord.rescan_containers_and_emit().await; - Ok(()) + Ok(NodeOutput::default()) } /// Mechanical container stop — the sub-step a `Reconcile` planner fans /// out when it observes `wanted = Offline` and the container up. -async fn run_stop(coord: &Arc, name: &str) -> Result<()> { +async fn run_stop(coord: &Arc, claim: &Claim) -> Result { + let name = &claim.agent; // See `run_start`: no node-local guard — `Stop` reports `Stopping` from its // own kind now. crate::lifecycle::kill(name).await?; coord.unregister_agent(name); coord.notify_manager(&hive_sh4re::HelperEvent::Killed { - agent: name.to_owned(), + agent: name.clone(), }); coord.rescan_containers_and_emit().await; - Ok(()) + Ok(NodeOutput::default()) } /// Mechanical stop for the profile swap. Never *changes* `wanted`; /// noop when already stopped. -async fn run_stop_for_update(coord: &Arc, name: &str) -> Result<()> { +async fn run_stop_for_update(coord: &Arc, claim: &Claim) -> Result { + let name = &claim.agent; if crate::lifecycle::is_running(name).await { // Seed a missing agent_power row from the PRE-stop observation // — the DAG's tail `Reconcile` observes only the mechanically @@ -446,7 +501,7 @@ async fn run_stop_for_update(coord: &Arc, name: &str) -> Result<()> crate::lifecycle::kill(name).await?; coord.rescan_containers_and_emit().await; } - Ok(()) + Ok(NodeOutput::default()) } /// Set the graceful fence + kick so the harness sees it promptly and @@ -458,18 +513,20 @@ async fn run_stop_for_update(coord: &Arc, name: &str) -> Result<()> /// `GRACEFUL_STOP_TIMEOUT`. Safe because the harness tests the marker at /// the top of its loop — a paused agent has no turn in flight, so there /// is nothing to checkpoint. -fn run_signal(coord: &Arc, name: &str) { - if hive_types::Ident::parse(name).is_ok_and(|a| Coordinator::is_paused(&a)) { - return; +fn run_signal(coord: &Arc, claim: &Claim) -> NodeOutput { + if hive_types::Ident::parse(&claim.agent).is_ok_and(|a| Coordinator::is_paused(&a)) { + return NodeOutput::default(); } - coord.mark_graceful_stop(name); - coord.kick_agent(name, "graceful stop requested"); + coord.mark_graceful_stop(&claim.agent); + coord.kick_agent(&claim.agent, "graceful stop requested"); + NodeOutput::default() } /// Await the harness clearing the fence (`GracefulStopComplete`) or /// the timeout — either way the downstream `Reconcile` proceeds with /// the actual stop. -async fn run_drain(coord: &Arc, name: &str) -> Result<()> { +async fn run_drain(coord: &Arc, claim: &Claim) -> Result { + let name = &claim.agent; let deadline = std::time::Instant::now() + GRACEFUL_STOP_TIMEOUT; while coord.is_graceful_stop_pending(name) { if std::time::Instant::now() >= deadline { @@ -479,11 +536,12 @@ async fn run_drain(coord: &Arc, name: &str) -> Result<()> { tokio::time::sleep(std::time::Duration::from_millis(500)).await; } coord.clear_graceful_stop(name); - Ok(()) + Ok(NodeOutput::default()) } /// `set_nspawn_flags` + `set_resource_limits` + daemon-reload. -async fn run_write_dropin(coord: &Arc, name: &str) -> Result<()> { +async fn run_write_dropin(coord: &Arc, claim: &Claim) -> Result { + let name = &claim.agent; // write_dropins only needs the path value to build AgentPaths; the // dir doesn't need to exist at this point (created by ensure_agent_runtime_dir // on the upstream Prebuild/Start node). @@ -491,18 +549,19 @@ async fn run_write_dropin(coord: &Arc, name: &str) -> Result<()> { let hive = coord.hive_env(); let paths = Coordinator::agent_paths(name, agent_dir); crate::lifecycle::write_dropins(name, &hive, &paths).await?; - Ok(()) + Ok(NodeOutput::default()) } /// Write + commit the perm file(s) (fused under `META_LOCK` so the /// working tree is never left dirty), then emit the P3RM1SS10NS-tab /// snapshots so the dashboard reflects the new assignment. -async fn run_write_perm_file( - coord: &Arc, - name: &str, - payload: &super::model::PermPayload, -) -> Result<()> { +async fn run_write_perm_file(coord: &Arc, claim: &Claim) -> Result { use super::model::PermPayload; + let name = &claim.agent; + // The perm file payload rides the node itself (the only consumer). + let NodeKind::WritePermFile { payload, .. } = &claim.kind else { + anyhow::bail!("run_write_perm_file on a non-WritePermFile node"); + }; // Runs under the deploy window (it declares `Resource::MetaWindow`): a // perm commit landing inside another node's staged prepare→finalize // window would sweep the staged deploy lock into its commit (the @@ -532,7 +591,7 @@ async fn run_write_perm_file( } } } - Ok(()) + Ok(NodeOutput::default()) } /// Apply the node's `(child, new_parent)` moves as one `META_LOCK`-fused @@ -542,10 +601,10 @@ async fn run_write_perm_file( /// the deploy window (it declares `Resource::MetaWindow`), same reasoning as /// `run_write_perm_file`: a topology commit landing inside another node's /// staged deploy window would sweep the staged lock into its commit. -async fn run_reparent( - coord: &Arc, - moves: &[(hive_types::Ident, Option)], -) -> Result<()> { +async fn run_reparent(coord: &Arc, claim: &Claim) -> Result { + let NodeKind::Reparent { moves } = &claim.kind else { + anyhow::bail!("run_reparent on a non-Reparent node"); + }; let refs: Vec<(&str, Option<&str>)> = moves .iter() .map(|(child, parent)| { @@ -559,14 +618,16 @@ async fn run_reparent( .reparent_bulk_with_notify(&refs) .await .map_err(|e| anyhow::anyhow!(e))?; - Ok(()) + Ok(NodeOutput::default()) } /// Deploy phase 1 — drift gate, fetch, eval-verify. Mutates nothing, so a /// failure here cancel-cascades the rest of the subtree with the forge and the /// applied repo exactly as they were. -async fn run_merge_verify(coord: &Arc, approval_id: i64) -> Result<()> { - crate::actions::run_deploy_merge_verify(coord, approval_id).await +async fn run_merge_verify(coord: &Arc, approval_id: i64) -> Result { + crate::actions::run_deploy_merge_verify(coord, approval_id) + .await + .map(|()| NodeOutput::default()) } /// Deploy phase 2 — the irreversible half: ff-merge, then phase 1 of the @@ -578,15 +639,27 @@ async fn run_merge_verify(coord: &Arc, approval_id: i64) -> Result< /// their `MetaSync` declares is re-entered rather than deadlocked against the /// ancestor already holding it. On failure nothing is appended and the tail /// compensates, exactly as before. -async fn run_deploy_apply(coord: &Arc, approval_id: i64) -> Result<()> { - crate::actions::run_deploy_apply(coord, approval_id).await +async fn run_deploy_apply( + coord: &Arc, + claim: &Claim, + approval_id: i64, +) -> Result { + crate::actions::run_deploy_apply(coord, approval_id).await?; + Ok(NodeOutput { + append_subgraph: vec![super::templates::deploy_rebuild_nodes( + claim.kind.agent(), + approval_id, + )], + }) } /// Deploy phase 3 — close the staged-lock window once the appended rebuild has /// come up clean: drop the rollback ref, plant the `deployed/` tag, commit /// the staged lock. -async fn run_finalize_deploy(coord: &Arc, approval_id: i64) -> Result<()> { - crate::actions::run_finalize_deploy(coord, approval_id).await +async fn run_finalize_deploy(coord: &Arc, approval_id: i64) -> Result { + crate::actions::run_finalize_deploy(coord, approval_id) + .await + .map(|()| NodeOutput::default()) } /// Deploy compensation + bookkeeping tail. `AfterAny` the apply node, so it @@ -598,12 +671,12 @@ async fn run_finalize_deploy(coord: &Arc, approval_id: i64) -> Resu /// the approval row is gone (deny race, purge). async fn run_deploy_tail( coord: &Arc, - dag_id: Option, - agent: &str, + claim: &Claim, approval_id: i64, -) -> Result<()> { - crate::actions::run_deploy_tail(coord, dag_id, agent, approval_id).await; - Ok(()) +) -> Result { + crate::actions::run_deploy_tail(coord, Some(claim.dag_id), claim.kind.agent(), approval_id) + .await; + Ok(NodeOutput::default()) } /// Compute which agents a `nix flake update ` on the meta diff --git a/hive-c0re/src/job_queue/mod.rs b/hive-c0re/src/job_queue/mod.rs index 597a564f..3db89559 100644 --- a/hive-c0re/src/job_queue/mod.rs +++ b/hive-c0re/src/job_queue/mod.rs @@ -36,7 +36,8 @@ pub mod templates; #[cfg(test)] mod tests; -use std::sync::{Arc, Mutex}; +use std::collections::HashMap; +use std::sync::Mutex; use chrono::{DateTime, Utc}; use hive_host_sock::jobs::NodeView; @@ -54,6 +55,16 @@ use resource::Resource; /// borrowed one; only `hive_jobq` can make or insert it. pub type Job = hive_jobq::JobBuilder; +/// A job's shape as a **recipe**: given a builder, declare the nodes. +/// +/// What a template returns and what an executor hands back, because neither +/// can build a job itself — `hive_jobq` creates the builder inside its own +/// insertion call and never lets one out. So the transferable thing is the +/// declaring closure, and the queue runs it at the moment it inserts. +/// +/// `Send` because an executor's output crosses the scheduler's task boundary. +pub type Declare = Box; + /// A handle to one node a template declared — where its edges, grouping and /// resources are declared. `Copy`; naming a node as a dependency does not /// consume the ability to name it again. @@ -88,6 +99,27 @@ pub struct RunningTransient { pub since: DateTime, } +/// A node claimed for execution — everything the executor needs, snapshotted at +/// claim time. +#[derive(Debug, Clone)] +pub struct Claim { + pub dag_id: u64, + pub node_id: NodeId, + pub kind: NodeKind, + /// The agent this node targets (its own, not a DAG-level field). Empty for + /// the agentless [`NodeKind::MetaLock`] + [`NodeKind::Dag`] container nodes. + pub agent: String, +} + +/// Per-node runtime metadata the crate graph doesn't carry. Lifecycle +/// (`started_at` / `finished_at` / `error`) lives on the `hive_jobq::Node` +/// itself now, so only the build-log row link remains host-side (the +/// client fetches the log by node id). +#[derive(Debug, Default, Clone)] +struct NodeRuntime { + build_log_id: Option, +} + /// An owned read-view of a DAG container's carried metadata ([`NodeKind::Dag`]). /// Derived on read from the container node — the data has a single home (the /// node payload); this is not a stored side-table. @@ -97,29 +129,24 @@ struct DagMeta { created_at: DateTime, } -/// The crate scheduler, specialised to this host's node + resource types. -/// -/// A **DAG is a single container node** ([`NodeKind::Dag`], `parent = None`) -/// whose subtree is the DAG's work — so the container's `NodeId` is the DAG id, -/// its rolled-up state is the DAG state, and there are no grouping side-tables: -/// membership + meta are graph queries ([`container`] / [`dag_meta`] + the -/// `hive_jobq::Graph` accessors). One shared crate [`Graph`] holds every DAG. -/// -/// There is deliberately **no wrapper struct and no per-node side map**. The -/// last map held the `build_logs` row id; that link now lives on the log row -/// itself (`build_logs.node_id`). With nothing else to guard, the mutex holds -/// the scheduler *directly* — which is what lets `hive_jobq` drive the run loop -/// (it takes `&Arc>>`, a type a host-side wrapper could not -/// satisfy). -type Sched = Scheduler; +/// The mutable queue state behind the mutex: the crate scheduler plus the +/// per-node runtime metadata the graph can't carry. A **DAG is a single +/// container node** ([`NodeKind::Dag`], `parent = None`) whose subtree is the +/// DAG's work — so the container's `NodeId` is the DAG id, its rolled-up state +/// is the DAG state, and there are no grouping side-tables: membership + meta +/// are graph queries ([`QueueInner::container`] / [`QueueInner::dag_meta`] + +/// the `hive_jobq::Graph` accessors). One shared crate [`Graph`] holds every DAG. +struct QueueInner { + sched: Scheduler, + /// Per-node runtime metadata (the build-log id) — mutable after + /// insert, so it can't ride the immutable node payload. + node_rt: HashMap, +} /// The queue. Lives on `Coordinator` (one per hive-c0re process); a single /// scheduler task ([`scheduler::run_worker`]) drives it. pub struct JobQueue { - /// The scheduler, held directly rather than behind a host-side wrapper — - /// `hive_jobq`'s run-loop seam takes `&Arc>>`, so this - /// *is* the type the crate drives. - sched: Arc>, + inner: Mutex, /// Wakes the scheduler when something new arrives or state changed. pub(crate) notify: Notify, } @@ -136,19 +163,8 @@ impl Default for JobQueue { } } -/// A node runner's `Result` as the scheduler's [`Outcome`]. -/// -/// The failure reason + `finished_at` are stamped onto the graph `Node` by the -/// scheduler (the reason rides `Outcome::Failed`); there is no host-side copy, -/// so nothing needs clearing on success. -fn outcome_of(result: Result<(), String>) -> Outcome { - match result { - Ok(()) => Outcome::Done, - Err(e) => Outcome::Failed(truncate_error(&e)), - } -} - -/// Insert a declared `job` into the shared graph, returning the inserted ids. +/// Insert a declared `job` into the shared graph and record its per-node +/// `node_rt`, returning the inserted ids. /// /// A node that declared no parent hangs under `group_parent` — the DAG /// container for a template, the emitting node for a runtime-appended @@ -162,11 +178,12 @@ fn outcome_of(result: Result<(), String>) -> Outcome { /// # Errors /// Propagates a crate graph-insert error (malformed dep/parent / dep-scope). fn insert_group( - inner: &mut Sched, + inner: &mut QueueInner, declare: impl FnOnce(&Job), group_parent: Option, ) -> anyhow::Result<()> { inner + .sched .insert_job(group_parent, |b| { declare(b); // c0re names no handles: a DAG is addressed by its container node, @@ -187,13 +204,16 @@ impl JobQueue { u32::try_from(build_slots.max(1)).unwrap_or(u32::MAX), ); Self { - sched: Arc::new(Mutex::new(Scheduler::new(Graph::new(), table))), + inner: Mutex::new(QueueInner { + sched: Scheduler::new(Graph::new(), table), + node_rt: HashMap::new(), + }), notify: Notify::new(), } } - fn lock(&self) -> std::sync::MutexGuard<'_, Sched> { - self.sched.lock().expect("job_queue mutex poisoned") + fn lock(&self) -> std::sync::MutexGuard<'_, QueueInner> { + self.inner.lock().expect("job_queue mutex poisoned") } /// Submit a DAG: insert a [`NodeKind::Dag`] **container node** carrying the @@ -201,13 +221,7 @@ impl JobQueue { /// roots re-parented to the container). Returns the container's id as the /// DAG id — its rolled-up state is the DAG state. /// - /// The container is an ordinary node: it declares no resources, so the - /// scheduler claims it on the next pass, runs its (empty) logic and parks - /// it in `Finishing`, at which point its children become runnable. Nothing - /// here completes it by hand — a node with no work of its own still goes - /// the way every other node goes. - /// - /// Takes the spec's recipe by generic, not as a boxed closure: a spec + /// Takes the spec's recipe by generic, not as a boxed [`Declare`]: a spec /// travels from the template that built it directly into this call, so /// there is nothing to allocate for. /// @@ -217,6 +231,7 @@ impl JobQueue { pub fn submit(&self, spec: DagSpec) -> anyhow::Result { let mut inner = self.lock(); let container = inner + .sched .append( NodeKind::Dag { source: spec.source, @@ -227,29 +242,100 @@ impl JobQueue { None, ) .map_err(|e| anyhow::anyhow!("job_queue: container insert failed: {e}"))?; + inner.node_rt.insert(container, NodeRuntime::default()); insert_group(&mut inner, spec.declare, Some(container))?; + // Settle the container's own (no-op) logic immediately so it parks in + // `Finishing` and its children become runnable — it never needs claiming + // or executing, and stays out of `claim_ready`. It rolls up terminal when + // its whole subtree settles (that's the DAG-done signal). + inner.sched.complete(container, Outcome::Done); drop(inner); self.notify.notify_one(); Ok(container.get()) } - /// The scheduler itself, for `hive_jobq`'s run-loop seam - /// (`Scheduler::claim_next`), which takes exactly this type. - /// - /// Handing out the `Arc` rather than wrapping each crate call keeps the - /// host from growing a parallel API: the run loop uses `hive_jobq`'s - /// functions directly, and this module stays the thin glue it is being - /// reduced to. - pub(crate) fn sched(&self) -> &Arc> { - &self.sched + /// Append a whole *subgraph* into a live DAG at runtime — the single + /// in-DAG-growth primitive. The subgraph is inserted as a [`insert_group`] + /// rooted under `dep_on` (the emitting node): the subgraph's own root becomes + /// a *child* of `dep_on`, its steps children of that root, and the group's + /// agent lease is hoisted onto that root. Ordering root→`dep_on` is the parent + /// gate — the children run once `dep_on` reaches `Finishing`. Because the + /// emitting node stays `Finishing` until this appended subtree is terminal and + /// the DAG's terminal node deps on the top root, roll-up keeps the DAG from + /// settling early with no explicit wiring. A no-op if the DAG is gone. + pub fn append_subgraph(&self, dag_id: u64, declare: Declare, dep_on: NodeId) { + let mut inner = self.lock(); + if inner.container(dag_id).is_none() { + return; + } + // Insert the subgraph as a group rooted under the emitting node: the + // subgraph's own root becomes a child of `dep_on`, its steps children of + // that root. No terminal-node wiring — roll-up carries terminality: the + // emitter stays `Finishing` until this appended subtree settles, and the + // container node rolls up terminal only once its whole subtree (incl. this + // appended work) has settled, so the DAG hook waits for free. + if let Err(e) = insert_group(&mut inner, declare, Some(dep_on)) { + tracing::error!( + dag = dag_id, + error = %e, + "job_queue: append_subgraph insert failed" + ); + return; + } + drop(inner); + self.notify.notify_one(); } - /// The DAG container id owning `node`, for log lines and the dashboard. - /// Derived from the graph rather than carried alongside the node — the - /// parent axis already knows it. - #[must_use] - pub fn dag_of(&self, node: NodeId) -> Option { - self.lock().graph().root_of(node).map(NodeId::get) + /// Claim every currently-runnable node, acquiring its resources, and mark it + /// `Running`. Delegates readiness + resource acquisition to the crate's + /// settle loop; builds a [`Claim`] per started node from its payload + its + /// DAG container's metadata. The container node itself is claimed like any + /// other (its executor is an instant no-op that lets its subtree start). + pub fn claim_ready(&self) -> Vec { + let mut inner = self.lock(); + let inner = &mut *inner; + let started = inner.sched.settle(); + let mut claims = Vec::with_capacity(started.len()); + for id in started { + let Some(node) = inner.sched.graph().node(id) else { + continue; + }; + let kind = node.payload.clone(); + let agent = node.payload.agent().to_owned(); + let Some(container) = inner.sched.graph().root_of(id) else { + continue; + }; + claims.push(Claim { + dag_id: container.get(), + node_id: id, + kind, + agent, + }); + // `started_at` is stamped on the graph `Node` by the scheduler's + // transition to `Running` — no host-side copy needed. + } + claims + } + + /// Mark a claimed node terminal, recording its outcome + (truncated) error. + /// The crate releases the node's build slot immediately and cascades the + /// `AfterOk` failure cancellation + subtree lease release. + /// + /// Nothing is returned: a DAG's terminal side effects are its own tail nodes + /// ([`NodeKind::ResolveApproval`] / [`NodeKind::EmitRebuilt`]), which the + /// scheduler claims and runs like any other node. + pub fn complete_node(&self, node_id: NodeId, result: Result<(), String>) { + let mut inner = self.lock(); + // The failure reason + `finished_at` are stamped onto the graph `Node` + // by the scheduler (the reason rides `Outcome::Failed`); no host-side + // copy, so there is nothing to clear here. + let outcome = match result { + Ok(()) => Outcome::Done, + Err(e) => Outcome::Failed(truncate_error(&e)), + }; + inner.sched.complete(node_id, outcome); + drop(inner); + self.notify.notify_one(); } /// Cancel a DAG that hasn't started yet: every work node is still `Pending`, @@ -274,10 +360,10 @@ impl JobQueue { /// just that branch. Nothing here knows about DAGs. pub fn cancel(&self, id: u64) -> bool { let mut inner = self.lock(); - let Some(node) = inner.graph().resolve_id(id) else { + let Some(node) = inner.sched.graph().resolve_id(id) else { return false; }; - if !inner.cancel_node(node) { + if !inner.sched.cancel_node(node) { return false; } drop(inner); @@ -285,6 +371,33 @@ impl JobQueue { true } + /// Link a `build_logs` row to a specific `Running` node. + pub fn set_build_log_id(&self, dag_id: u64, node_id: NodeId, log_id: i64) -> bool { + let mut inner = self.lock(); + if inner.sched.graph().root_of(node_id).map(NodeId::get) != Some(dag_id) + || !inner.node_running(node_id) + { + return false; + } + inner.node_rt.entry(node_id).or_default().build_log_id = Some(log_id); + true + } + + /// The `build_logs` row id linked to the wire node id `node_id`, if any — + /// the lookup behind the `GET /api/build-log/` query endpoint (the + /// client fetches a node's captured build output on demand rather than + /// receiving it inline). Takes the raw wire `u64` (the endpoint's path + /// param); `node_rt` is keyed by the opaque `NodeId`, so this scans for the + /// matching id — the map is small (live + recently-terminal nodes). + #[must_use] + pub fn build_log_id_of(&self, node_id: u64) -> Option { + self.lock() + .node_rt + .iter() + .find(|(nid, _)| nid.get() == node_id) + .and_then(|(_, rt)| rt.build_log_id) + } + /// The first failed node's error in `dag_id`, if any has failed yet. /// /// Unlike the roll-up summary this is readable *mid-flight*, which is the @@ -297,8 +410,12 @@ impl JobQueue { #[must_use] pub fn first_error(&self, dag_id: u64) -> Option { let inner = self.lock(); - let container = container(&inner, dag_id)?; - inner.graph().first_error(container).map(ToOwned::to_owned) + let container = inner.container(dag_id)?; + inner + .sched + .graph() + .first_error(container) + .map(ToOwned::to_owned) } /// `(agent, label, takes_container_down)` for the live transient-pill set, @@ -330,6 +447,7 @@ impl JobQueue { pub fn running_transients(&self) -> Vec { let inner = self.lock(); inner + .sched .graph() .nodes() .filter(|n| matches!(n.state, State::Running)) @@ -358,227 +476,204 @@ impl JobQueue { #[must_use] pub fn snapshot(&self) -> Vec { let inner = self.lock(); - let mut ids = visible_dags(&inner); + let mut ids = inner.visible_dags(); ids.sort_unstable_by_key(|c| c.get()); - ids.into_iter() - .filter_map(|c| dag_view(&inner, c)) + ids.into_iter().filter_map(|c| inner.dag_view(c)).collect() + } + + /// Number of live (non-terminal) DAGs — tests + diagnostics. + #[cfg(test)] + #[must_use] + pub fn live_count(&self) -> usize { + let inner = self.lock(); + inner + .containers() + .into_iter() + .filter(|&c| inner.sched.graph().is_settled(c) == Some(false)) + .count() + } +} + +impl QueueInner { + /// Whether `id` is a `Running` node. + fn node_running(&self, id: NodeId) -> bool { + self.sched + .graph() + .node(id) + .is_some_and(|n| n.state == State::Running) + } + + /// The container node of `dag_id` — the `NodeKind::Dag` root whose id equals + /// `dag_id`. `NodeId` is un-fabricable from a raw `u64`, so this is a search. + fn container(&self, dag_id: u64) -> Option { + self.sched.graph().nodes().find_map(|n| { + (n.parent.is_none() + && n.id.get() == dag_id + && matches!(n.payload, NodeKind::Dag { .. })) + .then_some(n.id) + }) + } + + /// The container's carried domain metadata as an owned read-view. The data + /// lives solely in the [`NodeKind::Dag`] payload — this is a derived read, + /// not a stored side-table. + fn dag_meta(&self, container: NodeId) -> Option { + let NodeKind::Dag { + source, + reason, + created_at, + } = &self.sched.graph().node(container)?.payload + else { + return None; + }; + Some(DagMeta { + source: *source, + reason: reason.clone(), + created_at: *created_at, + }) + } + + /// Project a DAG into its wire [`DagView`]: a near-raw view of the + /// container's work nodes, with `Done` nodes excluded. Lifecycle + /// (`state` / `started_at` / `finished_at` / `error`) is read straight + /// off each `hive_jobq::Node`; the client derives the DAG label, roll-up + /// state, and DAG timestamps from the node set. Non-derivable per-node + /// payload (`approval_id`, meta `inputs`) rides the owning node. Returns + /// `None` when every work node is `Done` or `Skipped` — a fully-settled + /// DAG drops out of the snapshot entirely (a `Failed` one lingers until + /// aged out). + fn dag_view(&self, container: NodeId) -> Option { + let meta = self.dag_meta(container)?; + let mut nodes = Vec::new(); + // Whether anything in this DAG still has an outcome worth showing. + // Kept separate from `nodes` being non-empty: skipped nodes ride the + // wire so the dashboard can mark the branches that weren't taken, but + // they must not by themselves hold a finished DAG in the snapshot. + let mut any_unsettled = false; + // DAG-level timestamps are taken over *all* subtree nodes (including the + // `Done` ones excluded from the wire) — the client can't derive them + // from a `Done`-filtered node set, so the host computes them here. + let mut started: Vec> = Vec::new(); + let mut finished: Vec> = Vec::new(); + for node in self.sched.graph().descendants(container) { + let id = node.id; + if let Some(s) = node.started_at { + started.push(s); + } + if let Some(f) = node.finished_at { + finished.push(f); + } + // `Done` nodes drop off the wire — a finished step isn't + // interesting. `Skipped` ones stay: which branch a run *didn't* + // take is the readable half of an outcome-branched DAG. + if matches!(node.state, State::Done) { + continue; + } + any_unsettled |= !matches!(node.state, State::Skipped); + let deps: Vec = node + .deps + .iter() + .filter_map(|d| match d { + Dep::Node { id, .. } => Some(id.get()), + Dep::Resource { .. } => None, + }) + .collect(); + // Non-derivable per-node payload rides the node that owns it. Every + // deploy phase carries the approval id, but only the subtree root + // projects it onto the wire — hanging the approval link off all of + // them would render the same card once per phase. + let approval_id = match &node.payload { + NodeKind::DeployWindow { approval_id, .. } => Some(*approval_id), + _ => None, + }; + let inputs = match &node.payload { + NodeKind::MetaLock { inputs, .. } => inputs.clone(), + _ => Vec::new(), + }; + let build_log_id = self.node_rt.get(&id).and_then(|r| r.build_log_id); + // `node.parent` is the structural jobq parent. Top-level nodes + // have `parent == Some(container)` (direct children of the Dag + // container); those become `parent: None` on the wire since the + // container itself is not part of the work-node payload. Sub-nodes + // carry the id of their containing parent work-node. + let parent = node + .parent + .filter(|&p| p != container) + .map(hive_jobq::NodeId::get); + nodes.push(NodeView { + id: id.get(), + agent: node.payload.agent().to_owned(), + kind: node.payload.as_str().to_owned(), + deps, + state: node.state, + started_at: node.started_at, + finished_at: node.finished_at, + error: node.error.clone(), + approval_id, + inputs, + build_log_id, + parent, + }); + } + if !any_unsettled { + return None; + } + let is_terminal = self.sched.graph().is_settled(container) == Some(true); + Some(DagView { + id: container.get(), + source: meta.source, + reason: meta.reason.clone(), + created_at: meta.created_at, + started_at: started.into_iter().min(), + finished_at: is_terminal.then(|| finished.into_iter().max()).flatten(), + nodes, + }) + } + + /// When a DAG's work node finishes on `finished_at` — the max over its + /// subtree (read off the graph `Node`, as unix seconds), for the history + /// cap ordering. + fn dag_finished_at(&self, container: NodeId) -> i64 { + self.sched + .graph() + .descendants(container) + .filter_map(|n| n.finished_at) + .map(|t| t.timestamp()) + .max() + .unwrap_or(0) + } + + /// Every DAG container node id in the graph. + fn containers(&self) -> Vec { + self.sched + .graph() + .nodes() + .filter(|n| n.parent.is_none() && matches!(n.payload, NodeKind::Dag { .. })) + .map(|n| n.id) .collect() } -} -/// The container node of `dag_id` — the `NodeKind::Dag` root whose id equals -/// `dag_id`. `NodeId` is un-fabricable from a raw `u64`, so this is a search. -fn container(sched: &Sched, dag_id: u64) -> Option { - sched.graph().nodes().find_map(|n| { - (n.parent.is_none() && n.id.get() == dag_id && matches!(n.payload, NodeKind::Dag { .. })) - .then_some(n.id) - }) -} - -/// The container's carried domain metadata as an owned read-view. The data -/// lives solely in the [`NodeKind::Dag`] payload — this is a derived read, -/// not a stored side-table. -fn dag_meta(sched: &Sched, container: NodeId) -> Option { - let NodeKind::Dag { - source, - reason, - created_at, - } = &sched.graph().node(container)?.payload - else { - return None; - }; - Some(DagMeta { - source: *source, - reason: reason.clone(), - created_at: *created_at, - }) -} - -/// Project a DAG into its wire [`DagView`]: a near-raw view of the -/// container's work nodes, with `Done` nodes excluded. Lifecycle -/// (`state` / `started_at` / `finished_at` / `error`) is read straight -/// off each `hive_jobq::Node`; the client derives the DAG label, roll-up -/// state, and DAG timestamps from the node set. Non-derivable per-node -/// payload (`approval_id`, meta `inputs`) rides the owning node. Returns -/// `None` when every work node is `Done` or `Skipped` — a fully-settled -/// DAG drops out of the snapshot entirely (a `Failed` one lingers until -/// aged out). -fn dag_view(sched: &Sched, container: NodeId) -> Option { - let meta = dag_meta(sched, container)?; - let all: Vec<_> = sched.graph().descendants(container).collect(); - // DAG-level timestamps are taken over *all* subtree nodes (including the - // `Done` ones excluded from the wire) — the client can't derive them - // from a `Done`-filtered node set, so the host computes them here. - let mut started: Vec> = Vec::new(); - let mut finished: Vec> = Vec::new(); - for node in &all { - if let Some(s) = node.started_at { - started.push(s); - } - if let Some(f) = node.finished_at { - finished.push(f); + /// The **visible** DAG set for the snapshot: every live (non-terminal) DAG, + /// plus the newest [`MAX_HISTORY_DAGS`] terminal ones. Crate nodes for + /// evicted DAGs linger in the graph (bounded-prune is a Stage-C follow-up); + /// this filter is what bounds what the dashboard sees. + fn visible_dags(&self) -> Vec { + let mut live: Vec = Vec::new(); + let mut terminal: Vec<(NodeId, i64)> = Vec::new(); + for c in self.containers() { + if self.sched.graph().is_settled(c) == Some(true) { + terminal.push((c, self.dag_finished_at(c))); + } else { + live.push(c); + } } + // Newest first, so truncating to the cap keeps the most recent. + terminal.sort_by(|a, b| b.1.cmp(&a.1).then(b.0.get().cmp(&a.0.get()))); + terminal.truncate(MAX_HISTORY_DAGS); + let mut kept = live; + kept.extend(terminal.into_iter().map(|(c, _)| c)); + kept } - // Decide which nodes ride the wire *before* projecting any of them: a - // `NodeView` costs a `build_logs` lookup, so building one for a node - // that's about to be dropped would be a query per finished step. - let shown = shown_on_wire(&all.iter().map(|n| n.state).collect::>())?; - let mut nodes = Vec::new(); - for node in shown.into_iter().map(|i| all[i]) { - let id = node.id; - let deps: Vec = node - .deps - .iter() - .filter_map(|d| match d { - Dep::Node { id, .. } => Some(id.get()), - Dep::Resource { .. } => None, - }) - .collect(); - // Non-derivable per-node payload rides the node that owns it. Every - // deploy phase carries the approval id, but only the subtree root - // projects it onto the wire — hanging the approval link off all of - // them would render the same card once per phase. - let approval_id = match &node.payload { - NodeKind::DeployWindow { approval_id, .. } => Some(*approval_id), - _ => None, - }; - let inputs = match &node.payload { - NodeKind::MetaLock { inputs, .. } => inputs.clone(), - _ => Vec::new(), - }; - // Looked up from the log row itself (`build_logs.node_id`), not a - // host-side map. One indexed query per node in the snapshot; the - // node set is bounded by `MAX_HISTORY_DAGS` and the store is a - // local sqlite file, so this is cheaper than the lock contention - // a second shared map would reintroduce. - let build_log_id = crate::build_logs::global().and_then(|h| h.id_for_node(id.get())); - // `node.parent` is the structural jobq parent. Top-level nodes - // have `parent == Some(container)` (direct children of the Dag - // container); those become `parent: None` on the wire since the - // container itself is not part of the work-node payload. Sub-nodes - // carry the id of their containing parent work-node. - let parent = node - .parent - .filter(|&p| p != container) - .map(hive_jobq::NodeId::get); - nodes.push(NodeView { - id: id.get(), - agent: node.payload.agent().to_owned(), - kind: node.payload.as_str().to_owned(), - deps, - state: node.state, - started_at: node.started_at, - finished_at: node.finished_at, - error: node.error.clone(), - approval_id, - inputs, - build_log_id, - parent, - }); - } - let is_terminal = sched.graph().is_settled(container) == Some(true); - Some(DagView { - id: container.get(), - source: meta.source, - reason: meta.reason.clone(), - created_at: meta.created_at, - started_at: started.into_iter().min(), - finished_at: is_terminal.then(|| finished.into_iter().max()).flatten(), - nodes, - }) -} - -/// Which of a DAG's work nodes ride the wire, by index into `states` — or -/// `None` when the DAG has nothing left worth showing and drops out of the -/// snapshot entirely. -/// -/// Two separate decisions, and conflating them pins every completed deploy in -/// the queue view forever: -/// - **`Done` drops off the wire.** A finished step isn't interesting. -/// `Skipped` stays: which branch a run *didn't* take is the readable half of -/// an outcome-branched DAG. -/// - **`Skipped` alone doesn't hold a DAG in the snapshot.** So "the node list -/// is non-empty" and "there's still something here worth showing" are -/// different questions, and only the second one may drop the DAG. -/// -/// Takes states rather than projected nodes so the caller can skip the work of -/// projecting what it's about to discard, and so this is testable without a -/// graph — the states it keys on are ones only a run can produce. -fn shown_on_wire(states: &[State]) -> Option> { - let worth_showing = states - .iter() - .any(|s| !matches!(s, State::Done | State::Skipped)); - if !worth_showing { - return None; - } - Some( - states - .iter() - .enumerate() - .filter(|(_, s)| !matches!(s, State::Done)) - .map(|(i, _)| i) - .collect(), - ) -} - -/// When a DAG's work node finishes on `finished_at` — the max over its -/// subtree (read off the graph `Node`, as unix seconds), for the history -/// cap ordering. -fn dag_finished_at(sched: &Sched, container: NodeId) -> i64 { - sched - .graph() - .descendants(container) - .filter_map(|n| n.finished_at) - .map(|t| t.timestamp()) - .max() - .unwrap_or(0) -} - -/// Every DAG container node id in the graph. -fn containers(sched: &Sched) -> Vec { - sched - .graph() - .nodes() - .filter(|n| n.parent.is_none() && matches!(n.payload, NodeKind::Dag { .. })) - .map(|n| n.id) - .collect() -} - -/// The **visible** DAG set for the snapshot: every live (non-terminal) DAG, -/// plus the newest [`MAX_HISTORY_DAGS`] terminal ones. Crate nodes for -/// evicted DAGs linger in the graph (bounded-prune is a Stage-C follow-up); -/// this filter is what bounds what the dashboard sees. -fn visible_dags(sched: &Sched) -> Vec { - let mut live: Vec = Vec::new(); - let mut terminal: Vec<(NodeId, i64, u64)> = Vec::new(); - for c in containers(sched) { - if sched.graph().is_settled(c) == Some(true) { - terminal.push((c, dag_finished_at(sched, c), c.get())); - } else { - live.push(c); - } - } - retain_history(live, terminal, MAX_HISTORY_DAGS) -} - -/// [`visible_dags`]'s policy, split from the graph it reads: keep every live -/// DAG, plus the newest `cap` terminal ones. -/// -/// `terminal` rows are `(handle, finished_at, tiebreak)`. The tiebreak orders -/// DAGs that settled inside the same wall-clock second — which is *most* of -/// them under a burst, and all of them in a test, so it is load-bearing rather -/// than a formality. -/// -/// Generic over the handle purely so this is reachable without a graph: a -/// `NodeId` cannot be fabricated, so a test that had to pass real ones could -/// only get them by submitting and running DAGs. -fn retain_history(live: Vec, mut terminal: Vec<(T, i64, u64)>, cap: usize) -> Vec { - // Newest first, so truncating to the cap keeps the most recent. - terminal.sort_by(|a, b| b.1.cmp(&a.1).then(b.2.cmp(&a.2))); - terminal.truncate(cap); - let mut kept = live; - kept.extend(terminal.into_iter().map(|(handle, _, _)| handle)); - kept } /// Truncate a node error to [`MAX_ERROR_LEN`] on a char boundary, appending `…`. diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs index b95f89e8..b6a9fe8d 100644 --- a/hive-c0re/src/job_queue/model.rs +++ b/hive-c0re/src/job_queue/model.rs @@ -407,9 +407,9 @@ impl NodeKind { /// Generic over the recipe rather than boxing it: a spec goes from the template /// that returns it straight to the `submit` that consumes it, so the closure's /// concrete type is known the whole way and needs neither an allocation nor a -/// `Send` bound. Nothing boxes a recipe any more — a running node grows its DAG -/// by declaring straight onto the builder it was handed, so there is no recipe -/// to store and replay across a task boundary. +/// `Send` bound. (The executor's `append_subgraph` is the case that *does* need +/// a boxed [`super::Declare`] — its recipes are collected into a `Vec` and +/// applied later, across a task boundary.) pub struct DagSpec { pub source: Source, /// Free-form "why". diff --git a/hive-c0re/src/job_queue/scheduler.rs b/hive-c0re/src/job_queue/scheduler.rs index 897f41ad..3efdf913 100644 --- a/hive-c0re/src/job_queue/scheduler.rs +++ b/hive-c0re/src/job_queue/scheduler.rs @@ -16,18 +16,21 @@ //! the DAG settles. //! //! In-DAG growth (a `MetaLock` growing rebuild subgraphs, a `Reconcile` fanning -//! its `Start`/`Stop`) is declared onto the builder each node is handed, and -//! inserted as part of completing that node. Completion itself is not this -//! module's job any more: it happens *inside* the future -//! [`hive_jobq::scheduler::Scheduler::claim_next`] hands back, so a node that -//! ran but was never completed is not an expressible state here. +//! its `Start`/`Stop`) flows through `NodeOutput.append_subgraph`, applied +//! before the emitting node completes — see `handle_completion`. use std::collections::HashMap; use std::sync::Arc; -use super::exec; +use super::Claim; +use super::exec::{self, NodeOutput}; use crate::coordinator::Coordinator; +struct NodeDone { + claim: Claim, + result: anyhow::Result, +} + /// Scheduler loop. Spawned once at hive-c0re startup from `main.rs`. /// /// Shutdown semantics: subscribes to `coord.shutdown_rx()`. On a true signal @@ -50,6 +53,7 @@ use crate::coordinator::Coordinator; /// reconverging silently. pub async fn run_worker(coord: Arc) { let mut shutdown = coord.shutdown_rx(); + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); // Last derived pill set we published, keyed by agent (its lease is cap-1, // so one pill each). Purely the previous value of a *derived* quantity — // it exists to spot transitions, since the dashboard wants edges @@ -66,66 +70,24 @@ pub async fn run_worker(coord: Arc) { return; } reconcile_transients(&coord, &mut transients); - // Claim exactly one node and get back the work that runs it. `Some` - // means something started, so there may be more runnable right now — - // loop again immediately. `None` means nothing is runnable and the - // loop parks below. That decision is the whole reason the crate hands - // back a task rather than an id. - let runner = { - // Two handles, deliberately: `sched` is the scheduler the crate - // locks, `node_coord` is what the node's own future captures. One - // binding can't do both — passing `coord.job_queue.sched()` borrows - // `coord` for the whole call while the `move` closure wants to take - // it. - let sched = Arc::clone(coord.job_queue.sched()); - let node_coord = Arc::clone(&coord); - hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, job| { - let coord = node_coord; - async move { - tracing::info!( - dag = coord.job_queue.dag_of(id).unwrap_or_default(), - node = id.get(), - kind = kind.as_str(), - agent = %kind.agent(), - "job_queue: node running" - ); - let (grown, result) = exec::run_node(&coord, job, id, &kind).await; - match &result { - Ok(()) => tracing::info!(node = id.get(), "job_queue: node done"), - Err(e) => tracing::warn!( - node = id.get(), - kind = kind.as_str(), - agent = %kind.agent(), - error = %format!("{e:#}"), - grown_nodes = !grown.is_empty(), - "job_queue: node failed" - ), - } - // Growth on a failed node is dropped by `complete_growing`, - // not here: failure cancel-cascades inside jobq, so that - // rule is the crate's to enforce and this loop does not get - // to forget it. - ( - grown, - super::outcome_of(result.map_err(|e| format!("{e:#}"))), - ) - } - }) - }; - if let Some(runner) = runner { - let done_coord = Arc::clone(&coord); - tokio::spawn(async move { - // Completion happens inside `runner` — it cannot be forgotten - // here, which is why there is no completion channel any more. - let (id, grew) = runner.await; - if let Err(e) = grew { - tracing::warn!(node = id.get(), error = %e, "job_queue: grown job rejected"); - } - done_coord.emit_rebuild_queue_snapshot(); - // Wake the loop: this node's completion may have unblocked - // dependents. Previously the completion channel did this. - done_coord.job_queue.notify.notify_one(); - }); + let claims = coord.job_queue.claim_ready(); + if !claims.is_empty() { + for claim in claims { + tracing::info!( + dag = claim.dag_id, + node = claim.node_id.get(), + kind = claim.kind.as_str(), + agent = %claim.agent, + "job_queue: node running" + ); + let coord = Arc::clone(&coord); + let tx = tx.clone(); + tokio::spawn(async move { + let result = exec::run_node(&coord, &claim).await; + // Send failure = scheduler gone (shutdown); drop. + let _ = tx.send(NodeDone { claim, result }); + }); + } // Newly-started owner nodes now hold their leases — surface the pills. reconcile_transients(&coord, &mut transients); coord.emit_rebuild_queue_snapshot(); @@ -139,11 +101,55 @@ pub async fn run_worker(coord: Arc) { return; } } + Some(done) = rx.recv() => { + handle_completion(&coord, done); + } () = coord.job_queue.notify.notified() => {} } } } +fn handle_completion(coord: &Arc, done: NodeDone) { + let NodeDone { claim, result } = done; + match result { + Ok(output) => { + tracing::info!( + dag = claim.dag_id, + node = claim.node_id.get(), + "job_queue: node done" + ); + // Append any in-DAG subgraphs BEFORE completing this node, so + // completing it doesn't roll the DAG terminal while the appended + // work is still pending. Each subgraph roots on this node + // (`AfterOk`), so it becomes ready the instant this one settles + // `Done` just below — covers both the multi-node case (a `MetaLock` + // growing per-agent rebuild subgraphs) and the single-node case (a + // `Reconcile` planner's `Start` / `Stop`). + for subgraph in output.append_subgraph { + coord + .job_queue + .append_subgraph(claim.dag_id, subgraph, claim.node_id); + } + coord.job_queue.complete_node(claim.node_id, Ok(())); + } + Err(e) => { + let msg = format!("{e:#}"); + tracing::warn!( + dag = claim.dag_id, + node = claim.node_id.get(), + kind = claim.kind.as_str(), + agent = %claim.agent, + error = %msg, + "job_queue: node failed" + ); + coord.job_queue.complete_node(claim.node_id, Err(msg)); + } + } + // The next loop iteration re-reconciles the transient pills against the + // post-completion lease state (a settled subgraph drops its pill). + coord.emit_rebuild_queue_snapshot(); +} + /// Publish the transitions between the previously-derived pill set and the /// current one. `prev` is last loop's derived value, keyed by agent (an agent's /// lease is cap-1, so at most one pill each). diff --git a/hive-c0re/src/job_queue/templates.rs b/hive-c0re/src/job_queue/templates.rs index 198b06bf..8607958c 100644 --- a/hive-c0re/src/job_queue/templates.rs +++ b/hive-c0re/src/job_queue/templates.rs @@ -28,7 +28,7 @@ use hive_jobq::TerminalState; use super::model::{DagSpec, NodeKind, PermPayload, Source}; use super::resource::Resource; -use super::{Handle, Job}; +use super::{Declare, Handle, Job}; /// The `Rebuilt`-reporting tail pair for a rebuild-shaped DAG: the success node /// gated on every group-root in `roots`, and the failure node gated on *its* @@ -80,39 +80,6 @@ fn resolve_approval_tails(b: &Job, approval_id: i64, root: Handle<'_>) { } } -/// Declare one rebuild subgraph per agent onto the builder a running -/// [`NodeKind::MetaLock`] was handed. -/// -/// **Into the emitter's own builder, not as new DAGs.** Growing in-DAG is what -/// roots each subgraph on the `MetaLock`, so the whole sweep (or meta-update -/// cascade) stays one unit of work the operator can watch and cancel, and every -/// rebuild builds against the lock the emitter just bumped. -/// -/// Same reason as [`fanned_out_mechanical`] for living here: this was the -/// second construction site declaring nodes inline in an executor. -pub(crate) fn grown_rebuilds(b: &Job, agents: &[String], opts: RebuildOpts) { - for agent in agents { - rebuild_nodes(b, agent, opts, None); - } -} - -/// Declare the mechanical node a [`NodeKind::Reconcile`] planner fans out -/// (`Start` / `Stop`) onto the builder it was handed while running. -/// -/// `Start` / `Stop` declare the agent lease they run under. Their `Reconcile` -/// parent is holding it already, so the declaration is a **re-entrant borrow** -/// — no second unit, no deadlock. It exists so the requirement belongs to the -/// node rather than to the fact that a `Reconcile` happens to fan it out. -/// -/// Lives here rather than inline in `exec.rs` for the same reason every other -/// declaration does: this is the one construction site that was hiding in an -/// executor, which meant the only test of it had to re-declare the same two -/// calls itself and would have kept passing if the executor changed. -pub(crate) fn fanned_out_mechanical(b: &Job, kind: NodeKind) { - let lease = Resource::Agent(kind.agent().to_owned()); - let _ = b.node(kind).needs(lease); -} - /// Knobs for [`rebuild_nodes`]. A struct rather than two positional `bool`s so /// a call site cannot silently swap them. #[derive(Debug, Clone, Copy)] @@ -268,29 +235,32 @@ pub(crate) fn rebuild_nodes<'a>( /// finalized. `Reconcile` alone would not do, being `AfterAny` — it reaches /// `Done` even after a failed `Swap`. /// -/// Declared into a **running** `DeployApply`'s own builder, not submitted: the -/// roots below become children of that node, which puts them inside the -/// `DeployWindow`'s subtree — so the `MetaWindow` this subgraph's `MetaSync` -/// and `FinalizeDeploy` declare is re-entered from the ancestor already holding -/// it rather than deadlocking against it. -pub(crate) fn deploy_rebuild_nodes(b: &Job, agent: &str, approval_id: i64) { - let roots = rebuild_nodes( - b, - agent, - RebuildOpts { - relock: false, - graceful: false, - }, - None, - ); - let _finalize = b - .node(NodeKind::FinalizeDeploy { - agent: agent.to_owned(), - approval_id, - }) - .needs(Resource::MetaWindow) - .after_ok(roots.prebuild) - .after_ok(roots.reconcile); +/// Appended, not submitted: the roots below become children of the emitting +/// `DeployApply` (see [`super::JobQueue::append_subgraph`]), which puts them +/// inside the `DeployWindow`'s subtree — so the `MetaWindow` this subgraph's +/// `MetaSync` and `FinalizeDeploy` declare is re-entered from the ancestor +/// already holding it rather than deadlocking against it. +pub(crate) fn deploy_rebuild_nodes(agent: &str, approval_id: i64) -> Declare { + let agent = agent.to_owned(); + Box::new(move |b: &Job| { + let roots = rebuild_nodes( + b, + &agent, + RebuildOpts { + relock: false, + graceful: false, + }, + None, + ); + let _finalize = b + .node(NodeKind::FinalizeDeploy { + agent: agent.clone(), + approval_id, + }) + .needs(Resource::MetaWindow) + .after_ok(roots.prebuild) + .after_ok(roots.reconcile); + }) } /// One uniform rebuild shape — no `was_running` branch. `StopForUpdate` @@ -405,6 +375,29 @@ pub fn approval_deploy( } } +/// A single `Reconcile` node that converges observed power state to the +/// persisted intent — `wanted` is untouched (no `SetWanted`), unlike the +/// operator `start`/`stop` templates. Test-only helper now (used to build +/// single-node lifecycle DAGs that exercise per-agent lease serialization +/// in the queue tests); production paths no longer emit a bare reconcile. +#[cfg(test)] +pub fn reconcile_only( + agent: &str, + source: Source, + reason: String, +) -> DagSpec> { + let agent = agent.to_owned(); + DagSpec { + source, + reason, + declare: Box::new(move |b: &Job| { + // Name the lease before the agent string is moved into the kind. + let lease = Resource::Agent(agent.clone()); + let _reconcile = b.node(NodeKind::Reconcile { agent }).needs(lease); + }), + } +} + /// First-deploy spawn (approval-driven): `Provision` (proposed/applied /// repos, state subvolume, meta registration) then `Create` /// (`nixos-container create`), drop-in write, then `Reconcile` starts @@ -487,8 +480,8 @@ pub fn perm_change( } /// Meta-input lock bump. The `MetaLock` executor grows one rebuild subgraph -/// per affected agent into *this same* DAG on completion (declared onto the -/// builder it was handed) — appended *after* the bump lands so their prebuilds +/// per affected agent into *this same* DAG on completion (via +/// `append_subgraph`) — appended *after* the bump lands so their prebuilds /// run against the post-bump lock, and a failed bump appends nothing /// (replacing the old fan-out-child-DAGs dance). /// `transient = Rebuilding` because those appended subgraphs are rebuilds: diff --git a/hive-c0re/src/job_queue/tests.rs b/hive-c0re/src/job_queue/tests.rs index 5ef2a0be..13de8d5d 100644 --- a/hive-c0re/src/job_queue/tests.rs +++ b/hive-c0re/src/job_queue/tests.rs @@ -1,18 +1,10 @@ -//! Queue-core unit tests: what c0re's templates **declare** — node kinds, -//! parent nesting, dep edges with the outcomes that satisfy them, and the -//! resources each construction site states it holds — plus the read layer over -//! that graph (wire projection, history retention, error truncation). -//! -//! **Nothing here runs a node.** Everything a template declares is in the graph -//! the moment `submit` returns, so the assertions read it there. Whether the -//! scheduler then honours those declarations — cascade, roll-up, grant -//! borrow/release, fairness, the `Finishing` gate — is `hive_jobq`'s property -//! and is tested in `hive_jobq`, against its own primitives rather than through -//! this module's templates. -//! -//! That split is why this file can't claim or complete: those are not part of -//! c0re's surface. A test helper that reached for them was reaching across the -//! boundary the two crates exist to draw. +//! Queue-core unit tests: submit / no-dedup, cycle rejection, resource +//! serialization (build slots / per-agent leases), lease-exempt +//! overlap, FIFO fairness, cancel semantics, `AfterAny` failure +//! routing, in-DAG subgraph growth, and history retention. All +//! synchronous — the +//! scheduler's async loop is a thin claim/complete pump over the same +//! methods exercised here. use super::model::NodeKind; use super::*; @@ -21,6 +13,20 @@ fn submit(q: &JobQueue, spec: DagSpec) -> u64 { q.submit(spec).expect("valid spec") } +/// Erase a spec's recipe to the boxed [`Declare`] so specs of *different* +/// shapes can share one type — e.g. a table of `(name, spec)` cases. +/// +/// Production never needs this: each submit path builds one spec and hands it +/// straight to `submit`, so the concrete closure type is known end to end. A +/// test table is the case where several shapes must be one type. +fn erase(spec: DagSpec) -> DagSpec { + DagSpec { + source: spec.source, + reason: spec.reason, + declare: Box::new(spec.declare), + } +} + fn ident(s: &str) -> hive_types::Ident { hive_types::Ident::parse(s).expect("valid test ident") } @@ -53,179 +59,47 @@ fn stop_online( submit::stop_spec(&targets, graceful, Source::Manual, reason.to_owned()) } -// `Claimed` / `ClaimReady` / `CompleteNode` lived here: a claim snapshot type -// and two extension traits that let this module start and finish nodes by -// hand. Nothing in this file drives the scheduler any more, so they are gone -// — which is the point. Production completes a node **inside** the future -// `hive_jobq::scheduler::Scheduler::claim_next` hands back, so "run the node, -// then remember to complete it" is not an expressible sequence there. A test -// helper that re-expressed it was a hole in exactly the seam it was testing. - -/// One node's **declared** shape: what it is, what it hangs under, and what it -/// waits for — all by kind, since ids are not stable across runs. -#[derive(Debug, PartialEq, Eq)] -struct Declared { - kind: &'static str, - /// Parent kind, or `None` when the node hangs directly under the DAG - /// container (i.e. it is a group root). - parent: Option<&'static str>, - /// Kinds this node declared a node-dep on, in declaration order, each with - /// the outcome set that satisfies it. - /// - /// The outcome set is **not** decoration: a template emits its tails as a - /// pair edged on the same upstream nodes, and the *only* thing telling the - /// ok-tail from the fail-tail is which outcomes each accepts. Without it - /// two structurally different nodes read as identical. - after: Vec<(&'static str, String)>, -} - -/// Render a dep's outcome set as the outcomes it actually accepts. -/// -/// ⚠️ Spelled out rather than bucketed into `ok` / `any` / other. The first -/// version of this did bucket, and a template's three `ResolveApproval` tails — -/// which differ ONLY in their accepted outcomes — all rendered as `"other"`. -/// A helper that prints two structurally different nodes identically turns an -/// assertion into a tautology. -fn when_tag(when: hive_jobq::DepWhen) -> String { - [ - (TerminalState::Done, "done"), - (TerminalState::Failed, "failed"), - (TerminalState::Cancelled, "cancelled"), - (TerminalState::Skipped, "skipped"), - ] - .into_iter() - .filter(|(outcome, _)| when.accepts(*outcome)) - .map(|(_, name)| name) - .collect::>() - .join("|") -} - -/// Every work node under `dag`, in insertion order, as its declared shape. -/// -/// **This is what the template tests are actually about.** A template's output -/// is fully determined the moment `submit` returns: the kinds, the parent -/// nesting and the dep edges are all sitting in the graph. Reading them here -/// keeps the assertion on c0re's own product. Whether the scheduler then -/// *honours* those edges — runs a chain serially, holds a grant across a -/// subtree — is `hive_jobq`'s property and is tested in `hive_jobq`. -fn declared_shape(q: &JobQueue, dag: u64) -> Vec { - declared_shape_filtered(q, dag, &|_| true) -} - -fn declared_shape_filtered( - q: &JobQueue, - dag: u64, - keep: &dyn Fn(&NodeKind) -> bool, -) -> Vec { - let sched = q.sched().lock().expect("job_queue mutex poisoned"); - let graph = sched.graph(); - let root = graph.resolve_id(dag).expect("dag id is a real node id"); - let kind_of = |id: NodeId| graph.node(id).map(|n| n.payload.as_str()); - graph - .nodes() - .filter(|n| n.id != root && graph.root_of(n.id) == Some(root) && keep(&n.payload)) - .map(|n| Declared { - kind: n.payload.as_str(), - parent: n.parent.filter(|p| *p != root).and_then(kind_of), - after: n - .deps - .iter() - .filter_map(|dep| match dep { - hive_jobq::Dep::Node { id, when } => kind_of(*id).map(|k| (k, when_tag(*when))), - hive_jobq::Dep::Resource { .. } => None, - }) - .collect(), - }) - .collect() -} - -/// The id of the one node of `kind` under `dag`, for the resource assertions. -/// -/// Panics unless there is exactly one — every caller is about a shape where the -/// kind is unique, so two would mean the assertion had quietly stopped being -/// about the node the test names. -fn node_of(q: &JobQueue, dag: u64, kind: &str) -> hive_jobq::NodeId { - let sched = q.sched().lock().expect("job_queue mutex poisoned"); - let graph = sched.graph(); - let root = graph.resolve_id(dag).expect("dag id is a real node id"); - let mut found: Vec<_> = graph - .nodes() - .filter(|n| n.id != root && graph.root_of(n.id) == Some(root) && n.payload.as_str() == kind) - .map(|n| n.id) - .collect(); +/// Claim helper asserting exactly one node comes back. +fn claim_one(q: &JobQueue) -> Claim { + let mut claims = q.claim_ready(); assert_eq!( - found.len(), + claims.len(), 1, - "expected exactly one {kind} node in the dag" + "expected exactly one claim, got {claims:?}" ); - found.pop().expect("checked above") + claims.pop().expect("one claim") } -/// The payload of the one node of `kind` under `dag`, for assertions about what -/// a node *carries* rather than how it is wired. -fn payload_of(q: &JobQueue, dag: u64, kind: &str) -> NodeKind { - let id = node_of(q, dag, kind); - let sched = q.sched().lock().expect("job_queue mutex poisoned"); - sched.graph().node(id).expect("node exists").payload.clone() +/// Claim an approval DAG's `ResolveApproval` tail and complete it, asserting it +/// is the one built for `expect`. +/// +/// A template emits one tail per outcome and the graph runs exactly one, so the +/// assertion is on *which node was claimed* — that alone says what the approval +/// row is about to be resolved as. Nothing computes it. +fn settle_approval_tail(q: &JobQueue, approval_id: i64, expect: TerminalState) { + let tail = claim_one(q); + assert!( + matches!( + tail.kind, + NodeKind::ResolveApproval { approval_id: got, outcome } + if got == approval_id && outcome == expect + ), + "expected the {expect:?} ResolveApproval tail for #{approval_id}, got {:?}", + tail.kind + ); + q.complete_node(tail.node_id, Ok(())); } -/// Kinds of every node under `dag` still `Pending` — the nodes that could yet -/// run. Stronger than asking the scheduler what is *ready right now*: a node -/// blocked on a dep is not ready but is very much still alive. -fn pending_kinds(q: &JobQueue, dag: u64) -> Vec<&'static str> { - pending_kinds_filtered(q, dag, &|_| true) -} - -/// [`pending_kinds`] restricted to the nodes whose payload names `agent`. -fn pending_kinds_for(q: &JobQueue, dag: u64, agent: &str) -> Vec<&'static str> { - pending_kinds_filtered(q, dag, &|kind: &NodeKind| kind.agent() == agent) -} - -/// The payloads of every node under `dag` still `Pending`, for the cases where -/// *which* of a family of same-kind nodes survived is the assertion — a -/// template emits one tail per outcome and they differ only in what they carry. -fn pending_payloads(q: &JobQueue, dag: u64) -> Vec { - let sched = q.sched().lock().expect("job_queue mutex poisoned"); - let graph = sched.graph(); - let root = graph.resolve_id(dag).expect("dag id is a real node id"); - graph - .nodes() - .filter(|n| n.id != root && graph.root_of(n.id) == Some(root) && n.state == State::Pending) - .map(|n| n.payload.clone()) - .collect() -} - -fn pending_kinds_filtered( - q: &JobQueue, - dag: u64, - keep: &dyn Fn(&NodeKind) -> bool, -) -> Vec<&'static str> { - let sched = q.sched().lock().expect("job_queue mutex poisoned"); - let graph = sched.graph(); - let root = graph.resolve_id(dag).expect("dag id is a real node id"); - graph - .nodes() - .filter(|n| { - n.id != root - && graph.root_of(n.id) == Some(root) - && n.state == State::Pending - && keep(&n.payload) - }) - .map(|n| n.payload.as_str()) - .collect() -} - -/// Shorthand for one expected row, so the tables below read as a shape. -fn row( - kind: &'static str, - parent: Option<&'static str>, - after: &[(&'static str, &str)], -) -> Declared { - Declared { - kind, - parent, - after: after.iter().map(|(k, w)| (*k, (*w).to_owned())).collect(), - } +/// The `EmitRebuilt` counterpart of [`settle_approval_tail`] — claim the tail the +/// graph let run and assert it's the `ok` one expected. +fn settle_rebuild_tail(q: &JobQueue, agent: &str, expect_ok: bool) { + let tail = claim_one(q); + assert!( + matches!(&tail.kind, NodeKind::EmitRebuilt { agent: a, ok } if a == agent && *ok == expect_ok), + "expected the ok={expect_ok} EmitRebuilt tail for {agent}, got {:?}", + tail.kind + ); + q.complete_node(tail.node_id, Ok(())); } /// The resources a node **declared**, read off its graph edges. @@ -237,6 +111,7 @@ fn row( fn declared_resources(q: &JobQueue, node_id: hive_jobq::NodeId) -> Vec { let inner = q.lock(); inner + .sched .graph() .node(node_id) .expect("node exists") @@ -249,44 +124,6 @@ fn declared_resources(q: &JobQueue, node_id: hive_jobq::NodeId) -> Vec .collect() } -/// The resources declared by **every** node of `kind` under `dag`, one row per -/// node, sorted so the rows read as a set rather than an insertion order. -/// -/// The per-agent templates emit several nodes of one kind — one per agent — and -/// what makes them concurrent is that each holds only its *own* agent's lease. -/// That is a statement about the whole family, so it needs all the rows, not -/// [`declared_resources`]'s single node. -fn declared_resources_of_kind(q: &JobQueue, dag: u64, kind: &str) -> Vec> { - let sched = q.sched().lock().expect("job_queue mutex poisoned"); - let graph = sched.graph(); - let root = graph.resolve_id(dag).expect("dag id is a real node id"); - let mut rows: Vec> = graph - .nodes() - .filter(|n| n.id != root && graph.root_of(n.id) == Some(root) && n.payload.as_str() == kind) - .map(|n| { - n.deps - .iter() - .filter_map(|dep| match dep { - hive_jobq::Dep::Resource { name, .. } => Some(name.clone()), - hive_jobq::Dep::Node { .. } => None, - }) - .collect() - }) - .collect(); - rows.sort_by_key(|r| format!("{r:?}")); - rows -} - -/// [`declared_shape`] restricted to the nodes whose payload names `agent`. -/// -/// A hive-wide DAG interleaves one subgraph per agent, and the kinds alone -/// cannot tell them apart — two `set_wanted` rows look identical. Slicing by -/// agent is what makes "the fresh agent goes straight to reconcile while the -/// stale one rebuilds first" expressible as a declared shape. -fn declared_shape_for(q: &JobQueue, dag: u64, agent: &str) -> Vec { - declared_shape_filtered(q, dag, &|kind: &NodeKind| kind.agent() == agent) -} - fn state_of(q: &JobQueue, dag_id: u64) -> State { // A DAG whose nodes have all settled `Done` or `Skipped` drops out of the // snapshot — absence is the completion signal, so map it to `Done`. @@ -336,17 +173,11 @@ fn distinct_submits_never_collapse() { #[test] fn resubmit_while_running_is_new_dag() { - // The "while running" is not load-bearing and used to be staged by claiming - // a node first. `submit` appends a container and inserts the declared - // group; it never consults the state of any existing node, so whether an - // earlier DAG is running cannot change the outcome. What is actually being - // asserted — no dedup, ever — is `identical_resubmit_is_a_distinct_dag`. - // - // Kept as the *named* case because "a config bump mid-build must not be - // swallowed" is the scenario people worry about, and a reader looking for - // it should find it. let q = JobQueue::new(1); let a = submit(&q, rebuild("agent-a", "first")); + let claim = claim_one(&q); // Prebuild running + assert_eq!(claim.dag_id, a); + // While the original runs, re-submit is legitimate new work. let again = submit(&q, rebuild("agent-a", "config bumped during build")); assert_ne!(a, again); assert_eq!(q.snapshot().len(), 2); @@ -372,52 +203,28 @@ fn resubmit_while_running_is_new_dag() { // ---- dependency order within a DAG ---- #[test] -fn rebuild_chain_is_declared_serial() { - // Was `rebuild_chain_claims_in_dep_order`, which drove the whole DAG to - // observe an order that is fully declared the moment `submit` returns. - // - // ⚠️ The old name was also wrong about the mechanism, and reading it rather - // than the graph is how you'd stay wrong: **only half this chain is dep - // edges.** `stop_for_update` and `swap` declare no deps at all — they are - // ordered by *parent nesting* ("a node's sub-nodes run after its own - // logic"). Both axes are asserted below because a template can break either - // one independently. +fn rebuild_chain_claims_in_dep_order() { let q = JobQueue::new(1); let id = submit(&q, rebuild("agent-a", "r")); - assert_eq!( - declared_shape(&q, id), - vec![ - row("meta_sync", None, &[]), - row("prebuild", None, &[("meta_sync", "done")]), - // No dep: ordered by hanging under `prebuild`. - row("stop_for_update", Some("prebuild"), &[]), - row("swap", Some("stop_for_update"), &[]), - row("post_swap", Some("stop_for_update"), &[("swap", "done")]), - row("reconcile", None, &[("prebuild", "done|failed|skipped")]), - // The tail pair. The ok tail needs every root to succeed; the !ok - // tail hangs off the ok tail's *elimination* (`skipped`), which is - // what makes exactly one of them run. - row( - "emit_rebuilt", - None, - &[ - ("meta_sync", "done"), - ("prebuild", "done"), - ("reconcile", "done"), - ], - ), - row( - "emit_rebuilt", - None, - &[ - ("emit_rebuilt", "skipped"), - ("meta_sync", "done|failed|skipped"), - ("prebuild", "done|failed|skipped"), - ("reconcile", "done|failed|skipped"), - ], - ), - ] - ); + for expected in [ + "meta_sync", + "prebuild", + "stop_for_update", + "swap", + "post_swap", + "reconcile", + ] { + let c = claim_one(&q); + assert_eq!(c.dag_id, id); + assert_eq!(c.kind.as_str(), expected); + assert!( + q.claim_ready().is_empty(), + "chain must serialize: nothing ready while {expected} runs" + ); + q.complete_node(c.node_id, Ok(())); + } + settle_rebuild_tail(&q, "agent-a", true); + assert_eq!(state_of(&q, id), State::Done); } /// The boot sweep's graceful shape: the agent gets `Signal` → `Drain` to @@ -447,25 +254,26 @@ fn graceful_rebuild_chain_drains_before_stopping() { }), }, ); - assert_eq!( - declared_shape(&q, id) - .iter() - .map(|d| d.kind) - .collect::>(), - vec![ - "meta_sync", - "prebuild", - // The graceful window goes between the build and the stop: the - // agent gets its turn to finish before the container goes down. - "signal", - "drain", - "stop_for_update", - "swap", - "post_swap", - "reconcile", - ], - "graceful inserts signal + drain ahead of the stop, and nothing else" - ); + for expected in [ + "meta_sync", + "prebuild", + "signal", + "drain", + "stop_for_update", + "swap", + "post_swap", + "reconcile", + ] { + let c = claim_one(&q); + assert_eq!(c.dag_id, id); + assert_eq!(c.kind.as_str(), expected); + assert!( + q.claim_ready().is_empty(), + "chain must serialize: nothing ready while {expected} runs" + ); + q.complete_node(c.node_id, Ok(())); + } + assert_eq!(state_of(&q, id), State::Done); } /// The non-graceful shape is the default everywhere except the boot sweep: @@ -495,11 +303,14 @@ fn non_graceful_rebuild_has_no_signal_or_drain() { }), }, ); + let mut kinds = Vec::new(); + for _ in 0..6 { + let c = claim_one(&q); + kinds.push(c.kind.as_str().to_owned()); + q.complete_node(c.node_id, Ok(())); + } assert_eq!( - declared_shape(&q, id) - .iter() - .map(|d| d.kind) - .collect::>(), + kinds, vec![ "meta_sync", "prebuild", @@ -507,106 +318,222 @@ fn non_graceful_rebuild_has_no_signal_or_drain() { "swap", "post_swap", "reconcile" - ], - "exactly six nodes, and neither of them is signal or drain" + ] ); + // Settled after exactly those six — nothing else was declared. + assert_eq!(state_of(&q, id), State::Done); } -/// Which nodes ride the wire, and whether a DAG is still worth showing. -/// -/// This used to submit a rebuild, claim and complete all seven of its nodes, -/// then assert the DAG was absent from `snapshot()` — the scheduler, the -/// roll-up and the whole projection standing in for one predicate over a list -/// of states. `shown_on_wire` is that predicate, so the cases can be named -/// instead of arranged. +/// A cleanly-finished DAG leaves the snapshot even though its not-taken +/// failure branch is still in the graph as `Skipped`. Skipped nodes ride the +/// wire so the dashboard can mark them, which makes "the node list is empty" +/// and "nothing here is still worth showing" two different questions — only +/// the second one may drop the DAG. Conflating them pins every completed +/// deploy in the queue view forever. #[test] -fn skipped_nodes_ride_the_wire_but_do_not_hold_a_settled_dag_in_the_snapshot() { - // Nothing left worth showing → the DAG drops out of the snapshot, and its - // absence is what signals completion. - assert_eq!(shown_on_wire(&[State::Done, State::Done]), None); - // The case the old test was built around: a clean run whose not-taken - // failure branch is still in the graph as `Skipped`. "The node list is - // non-empty" and "there's something here worth showing" are different - // questions, and conflating them pins every completed deploy in the queue - // view forever. - assert_eq!( - shown_on_wire(&[State::Done, State::Skipped, State::Done]), - None, - "a skipped branch does not keep a finished DAG alive" +fn settled_dag_leaves_the_snapshot_despite_its_skipped_branch() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + for _ in 0..6 { + let c = claim_one(&q); + q.complete_node(c.node_id, Ok(())); + } + settle_rebuild_tail(&q, "agent-a", true); + assert!( + q.snapshot().iter().all(|d| d.id != id), + "a fully settled DAG drops out of the snapshot" ); - // A `Failed` DAG lingers — and takes its skipped branches with it, which is - // how an operator sees which steps the run never reached. - assert_eq!( - shown_on_wire(&[State::Done, State::Failed, State::Skipped, State::Done]), - Some(vec![1, 2]), - "done nodes drop off, the failure and what it ruled out stay" - ); - // Live DAGs keep everything but their finished steps. - assert_eq!( - shown_on_wire(&[State::Done, State::Running, State::Pending]), - Some(vec![1, 2]) - ); - assert_eq!( - shown_on_wire(&[State::Cancelled, State::Skipped]), - Some(vec![0, 1]), - "a cancelled DAG is still worth showing to whoever cancelled it" - ); - // An empty DAG has nothing worth showing either — no special case needed, - // but it is the one input where "any" and "all" disagree, so it is pinned. - assert_eq!(shown_on_wire(&[]), None); } // ---- build slots ---- #[test] -fn rebuild_chain_declares_the_slot_where_the_nix_work_is() { - // Was `fifo_fairness_for_the_slot`, which submitted three rebuilds and - // drove one to completion to watch the freed slot go to the earlier - // waiter. **That fairness guarantee is hive_jobq's**, and it had no test - // there at all — its claim primitive scans nodes in insertion order and - // takes the first satisfiable one, and nothing pinned that. It does now: - // `a_contended_resource_goes_to_the_oldest_waiter`. - // - // What is c0re's is *which* nodes contend for the slot in the first place, - // and that is a declaration. "Uniform hold across the chain" then follows - // from the parent nesting asserted in `rebuild_chain_is_declared_serial`: - // a resource unit is held for the acquirer's whole subtree, so the slot - // `Prebuild` takes covers `StopForUpdate` → `Swap` → `PostSwap` beneath it. +fn build_slot_serializes_nix_heavy_nodes() { let q = JobQueue::new(1); - let id = submit(&q, rebuild("agent-a", "r")); - let res = |kind: &str| declared_resources(&q, node_of(&q, id, kind)); + let a = submit(&q, rebuild("agent-a", "r")); + let b = submit(&q, rebuild("agent-b", "r")); + // The rebuild heads are `MetaSync` (slot-free, but serialized on the + // global meta window), so drive each chain's head out of the way first. + let head_a = claim_one(&q); + assert_eq!(head_a.dag_id, a); + assert_eq!(head_a.kind.as_str(), "meta_sync"); + q.complete_node(head_a.node_id, Ok(())); + // a's Prebuild takes the only slot; b's MetaSync is free to run beside it + // (different resources), but b's Prebuild is not. + let claims = q.claim_ready(); + let mut kinds: Vec<(u64, &str)> = claims.iter().map(|c| (c.dag_id, c.kind.as_str())).collect(); + kinds.sort_unstable(); + assert_eq!(kinds, vec![(a, "prebuild"), (b, "meta_sync")]); + for c in &claims { + q.complete_node(c.node_id, Ok(())); + } + // Uniform hold: agent-a keeps the build slot across its whole build chain + // (Swap re-enters it), so a's StopForUpdate (lease, slot-free) runs but b's + // Prebuild must wait for a's slot-needers (through Swap) to finish. + let kinds: Vec<(u64, &str)> = q + .claim_ready() + .iter() + .map(|c| (c.dag_id, c.kind.as_str())) + .collect(); + assert_eq!(kinds, vec![(a, "stop_for_update")]); + assert!( + !kinds.iter().any(|&(d, _)| d == b), + "b's build waits — slot held across a's chain" + ); +} - let agent = || Resource::Agent("agent-a".to_owned()); +#[test] +fn two_build_slots_run_two_prebuilds() { + let q = JobQueue::new(2); + submit(&q, rebuild("agent-a", "r")); + submit(&q, rebuild("agent-b", "r")); + // Each rebuild's head `MetaSync` holds the cap-1 global meta window, so the + // two heads take turns — exactly the serialization the old runtime + // `meta::exclusive()` mutex imposed inside the prebuild executor. What must + // NOT serialize is the build itself: complete only the meta heads and watch + // both prebuilds end up in flight together, neither of them completed. + let mut prebuilds = Vec::new(); + for _ in 0..3 { + for c in q.claim_ready() { + if c.kind.as_str() == "meta_sync" { + q.complete_node(c.node_id, Ok(())); + } else { + prebuilds.push(c); + } + } + } + assert_eq!(prebuilds.len(), 2, "two slots → two concurrent prebuilds"); + assert!(prebuilds.iter().all(|c| c.kind.as_str() == "prebuild")); +} + +#[test] +fn fifo_fairness_for_the_slot() { + let q = JobQueue::new(1); + let a = submit(&q, rebuild("agent-a", "r")); + let b = submit(&q, rebuild("agent-b", "r")); + let c = submit(&q, rebuild("agent-c", "r")); + let first = claim_one(&q); + assert_eq!(first.dag_id, a, "submit order wins the slot"); + q.complete_node(first.node_id, Ok(())); + // Uniform hold: the slot stays with agent-a until its Swap (the last + // slot-needer) completes. Drive a's chain; the moment its slot frees, + // submit order (b before c) wins it. + let mut freed_to = None; + for _ in 0..6 { + let claims = q.claim_ready(); + if let Some(nb) = claims.iter().find(|cl| cl.dag_id == b || cl.dag_id == c) { + freed_to = Some(nb.dag_id); + break; + } + for cl in claims { + if cl.dag_id == a { + q.complete_node(cl.node_id, Ok(())); + } + } + } assert_eq!( - [ - res("meta_sync"), - res("prebuild"), - res("stop_for_update"), - res("swap"), - res("reconcile"), - ], - [ - // The meta preamble takes the global window and *nothing else* — - // no slot (it does no nix work) and no lease. - vec![Resource::MetaWindow], - // The nix build is the slot-needer, and takes **no lease**. That is - // what lets a prebuild overlap another DAG on the same agent: the - // container is still up and untouched while it builds. - vec![Resource::BuildSlot], - // The lease starts here — the first node that touches the - // container — and not one node earlier. - vec![agent()], - // Swap needs both. It re-enters the slot its `Prebuild` ancestor - // holds rather than acquiring a second unit. - vec![Resource::BuildSlot, agent()], - vec![agent()], - ], - "the slot follows the nix work and the lease follows the container" + freed_to, + Some(b), + "b's prebuild wins the freed slot before c's" ); } // ---- per-agent lease ---- +#[test] +fn lease_serializes_two_lifecycle_dags_for_same_agent() { + let q = JobQueue::new(4); + let restart = submit(&q, restart_online(&["agent-a"], false, "restart")); + let stop = submit( + &q, + templates::reconcile_only("agent-a", Source::Manual, "stop".to_owned()), + ); + // Restart's first node (StopForUpdate) takes the lease; stop's + // Reconcile must wait even though slots are free. + let first = claim_one(&q); + assert_eq!(first.dag_id, restart); + assert_eq!(first.kind.as_str(), "stop_for_update"); + q.complete_node(first.node_id, Ok(())); + // Same DAG keeps the lease through the tail Reconcile (re-entered from the + // dep graph — no fresh acquire), since stop's Reconcile can't re-enter it. + let second = claim_one(&q); + assert_eq!(second.dag_id, restart); + assert_eq!(second.kind.as_str(), "reconcile"); + q.complete_node(second.node_id, Ok(())); + // Restart's work is terminal → its lease releases, so stop's now-unblocked + // Reconcile becomes ready (a power op has no tail node, so nothing of + // restart's remains claimable). + let third = claim_one(&q); + assert_eq!(third.dag_id, stop); + assert_eq!(third.kind.as_str(), "reconcile"); + q.complete_node(third.node_id, Ok(())); + assert_eq!(state_of(&q, restart), State::Done); + assert_eq!(state_of(&q, stop), State::Done); +} + +#[test] +fn lease_exempt_prebuild_overlaps_other_dag_on_same_agent() { + let q = JobQueue::new(2); + submit(&q, rebuild("agent-a", "rebuild")); + submit( + &q, + templates::reconcile_only("agent-a", Source::Manual, "stop".to_owned()), + ); + // Both DAGs' heads are lease-independent of each other: the rebuild's + // MetaSync (meta window) and the stop's Reconcile (agent lease). + let heads = q.claim_ready(); + let head_kinds: Vec<&str> = heads.iter().map(|c| c.kind.as_str()).collect(); + assert!(head_kinds.contains(&"meta_sync")); + assert!(head_kinds.contains(&"reconcile")); + let meta_sync = heads + .iter() + .find(|c| c.kind.as_str() == "meta_sync") + .expect("meta_sync claim") + .clone(); + q.complete_node(meta_sync.node_id, Ok(())); + // Prebuild is lease-exempt: the stop's Reconcile keeps the lease + // and runs concurrently with the rebuild's out-of-band nix build. + let claims = q.claim_ready(); + let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect(); + assert!(kinds.contains(&"prebuild")); + // But the rebuild's StopForUpdate must then wait for the stop DAG + // to finish (lease). + let prebuild = claims + .iter() + .find(|c| c.kind.as_str() == "prebuild") + .expect("prebuild claim") + .clone(); + q.complete_node(prebuild.node_id, Ok(())); + assert!( + q.claim_ready().is_empty(), + "StopForUpdate blocked while stop DAG holds the lease" + ); + let reconcile = heads + .iter() + .find(|c| c.kind.as_str() == "reconcile") + .expect("reconcile claim") + .clone(); + q.complete_node(reconcile.node_id, Ok(())); + // stop's Reconcile done → its lease frees, so rebuild's StopForUpdate + // unblocks. (stop's DAG rolls up terminal; a power op has no tail node, so + // nothing of stop's is left in the claim set.) + let after = q.claim_ready(); + let sfu = after + .iter() + .find(|c| c.kind.as_str() == "stop_for_update") + .expect("rebuild StopForUpdate unblocked once the lease frees"); + assert_eq!(sfu.agent, "agent-a"); +} + +#[test] +fn agents_do_not_contend_on_each_others_leases() { + let q = JobQueue::new(4); + submit(&q, restart_online(&["agent-a"], false, "r")); + submit(&q, restart_online(&["agent-b"], false, "r")); + let claims = q.claim_ready(); + assert_eq!(claims.len(), 2, "different agents run concurrently"); +} + #[test] fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() { let q = JobQueue::new(4); @@ -616,31 +543,69 @@ fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() { ); // A hive-wide restart is ONE DAG, not one-per-agent. assert_eq!(q.snapshot().len(), 1); - // Each agent's subgraph head (StopForUpdate, since both are running) is a - // group root with no deps, so nothing orders them against each other; and - // each declares only its OWN agent's lease, so nothing makes them contend. - // Those two declared facts are what "they run concurrently" *means* here — - // that a scheduler then does run independent, resource-disjoint roots at - // once is hive_jobq's property, tested there. - let heads: Vec<_> = declared_shape(&q, id) - .into_iter() - .filter(|d| d.kind == "stop_for_update") + // Each agent's subgraph head (StopForUpdate, since both are running) is + // a root, so both are claimable at once — each takes its OWN agent's + // lease (no contention across distinct agents), all inside the single DAG. + let claims = q.claim_ready(); + assert!(claims.iter().all(|c| c.dag_id == id)); + let mut heads: Vec<(&str, &str)> = claims + .iter() + .map(|c| (c.agent.as_str(), c.kind.as_str())) .collect(); + heads.sort_unstable(); assert_eq!( heads, vec![ - row("stop_for_update", None, &[]), - row("stop_for_update", None, &[]), + ("agent-a", "stop_for_update"), + ("agent-b", "stop_for_update"), ], - "both per-agent heads are independent group roots" + "both per-agent subgraphs start concurrently, each acquiring its own lease" ); - assert_eq!( - declared_resources_of_kind(&q, id, "stop_for_update"), - vec![ - vec![Resource::Agent("agent-a".to_owned())], - vec![Resource::Agent("agent-b".to_owned())], - ], - "each head declares only its own agent's lease — disjoint, so no contention" +} + +/// A multi-agent DAG frees an agent's lease the moment THAT agent's +/// subgraph is terminal — not when the whole DAG finishes. So a +/// concurrent DAG wanting the finished agent can proceed while the rest +/// of the first DAG runs on. +#[test] +fn multi_agent_lease_frees_per_subgraph_not_whole_dag() { + let q = JobQueue::new(4); + let id = submit(&q, restart_online(&["agent-a", "agent-b"], false, "r")); + + // Drive agent-a's ENTIRE subgraph to Done while leaving agent-b's + // head running (so agent-b keeps holding its lease). + let mut b_in_flight = false; + loop { + let mut progressed = false; + for c in q.claim_ready() { + if c.agent == "agent-a" { + q.complete_node(c.node_id, Ok(())); + progressed = true; + } else { + b_in_flight = true; // leave agent-b's node running + } + } + if !progressed { + break; + } + } + assert!(b_in_flight, "agent-b subgraph should still be in flight"); + // The DAG as a whole is NOT terminal — agent-b runs on. + assert_eq!(state_of(&q, id), State::Running); + + // agent-a's lease is freed early → a concurrent agent-a DAG runs; + // an agent-b DAG still blocks on the lease agent-b's subgraph holds. + submit(&q, restart_online(&["agent-a"], false, "concurrent-a")); + submit(&q, restart_online(&["agent-b"], false, "concurrent-b")); + let claims = q.claim_ready(); + let agents: Vec<&str> = claims.iter().map(|c| c.agent.as_str()).collect(); + assert!( + agents.contains(&"agent-a"), + "agent-a lease freed the moment its subgraph settled" + ); + assert!( + !agents.contains(&"agent-b"), + "agent-b lease still held — its subgraph is still in flight" ); } @@ -653,26 +618,17 @@ fn multi_agent_stop_is_one_dag_with_concurrent_per_agent_subgraphs() { ); // A hive-wide stop is ONE DAG, not one-per-agent. assert_eq!(q.snapshot().len(), 1); - // Same declared story as the restart case above: each agent's subgraph head - // is a group root with no node-deps, holding only its own agent's lease. - // Independent roots on disjoint resources is what "concurrently" means at - // this layer — the running of them is hive_jobq's. - let heads: Vec<_> = declared_shape(&q, id) - .into_iter() - .filter(|d| d.kind == "set_wanted") + let claims = q.claim_ready(); + assert!(claims.iter().all(|c| c.dag_id == id)); + let mut heads: Vec<(&str, &str)> = claims + .iter() + .map(|c| (c.agent.as_str(), c.kind.as_str())) .collect(); + heads.sort_unstable(); assert_eq!( heads, - vec![row("set_wanted", None, &[]), row("set_wanted", None, &[])], - "both per-agent stop subgraph heads are independent group roots" - ); - assert_eq!( - declared_resources_of_kind(&q, id, "set_wanted"), - vec![ - vec![Resource::Agent("agent-a".to_owned())], - vec![Resource::Agent("agent-b".to_owned())], - ], - "each head declares only its own agent's lease" + vec![("agent-a", "set_wanted"), ("agent-b", "set_wanted")], + "both per-agent stop subgraphs start concurrently, each on its own lease" ); } @@ -694,31 +650,32 @@ fn multi_agent_start_one_dag_folds_per_agent_stale_rebuild() { ); // One DAG spanning both agents. assert_eq!(q.snapshot().len(), 1); - // The fold is a *declared* difference, readable the moment submit returns: - // both agents get a `SetWanted(Up)` group root, but the fresh agent's - // subgraph ends at the Reconcile behind it while the stale agent's carries - // the whole rebuild chain in between. - assert_eq!( - declared_shape_for(&q, id, "fresh"), - vec![ - row("set_wanted", None, &[]), - row("reconcile", Some("set_wanted"), &[]), - ], - "a fresh agent is intent + convergence, nothing in between" + // Both subgraph heads (SetWanted(Up)) are roots — claimable at once, + // each acquiring its own agent lease. + let heads = q.claim_ready(); + assert!( + heads + .iter() + .all(|c| c.dag_id == id && c.kind.as_str() == "set_wanted") ); + let mut head_agents: Vec<&str> = heads.iter().map(|c| c.agent.as_str()).collect(); + head_agents.sort_unstable(); + assert_eq!(head_agents, vec!["fresh", "stale"]); + // Complete both heads; the fresh agent then reconciles directly while + // the stale agent's subgraph is the rebuild chain (meta_sync first). + for c in &heads { + q.complete_node(c.node_id, Ok(())); + } + let next = q.claim_ready(); + let mut kinds: Vec<(&str, &str)> = next + .iter() + .map(|c| (c.agent.as_str(), c.kind.as_str())) + .collect(); + kinds.sort_unstable(); assert_eq!( - declared_shape_for(&q, id, "stale"), - vec![ - row("set_wanted", None, &[]), - row("meta_sync", None, &[("set_wanted", "done")]), - row("prebuild", None, &[("meta_sync", "done")]), - row("stop_for_update", Some("prebuild"), &[]), - row("swap", Some("stop_for_update"), &[]), - row("post_swap", Some("stop_for_update"), &[("swap", "done")]), - row("reconcile", None, &[("prebuild", "done|failed|skipped")]), - ], - "a stale agent gets the whole rebuild chain wedged between intent and \ - convergence — same DAG, same head kind, more in the middle" + kinds, + vec![("fresh", "reconcile"), ("stale", "meta_sync")], + "fresh agent starts directly; stale agent rebuilds first, all in one DAG" ); } @@ -773,6 +730,74 @@ fn offline_agents_skip_mechanical_nodes_but_keep_reconcile() { ); } +#[test] +fn a_fanned_out_start_declares_the_lease_and_re_enters_its_reconciles_grant() { + // `Start` / `Stop` / `PostSwap` were lease-exempt *as kinds*, which was only + // safe because every construction site fans them out from inside a + // lease-holding ancestor. Now they declare the lease themselves. + // + // The contract says that costs nothing — a descendant re-enters the + // ancestor's grant instead of taking a fresh unit. That is exactly the sort + // of claim that is true until a node is used from a second site, so it is + // pinned here rather than argued: the fanned-out `Start` must (a) actually + // carry the declaration, (b) still run under its parent's grant, and + // (c) not have consumed a second unit of a cap-1 lease. + let q = JobQueue::new(4); + let id = submit( + &q, + templates::reconcile_only("agent-a", Source::Manual, "converge".to_owned()), + ); + // A competing DAG on the same agent, to prove the lease is genuinely held + // (and held *once*) across the fan-out. + let rival = submit( + &q, + templates::reconcile_only("agent-a", Source::Manual, "rival".to_owned()), + ); + + let reconcile = claim_one(&q); + assert_eq!(reconcile.dag_id, id); + assert_eq!(reconcile.kind.as_str(), "reconcile"); + + // What `run_reconcile` does on observing a down container with wanted=Up. + q.append_subgraph( + id, + Box::new(|b: &Job| { + let kind = NodeKind::Start { + agent: "agent-a".to_owned(), + }; + let lease = Resource::Agent(kind.agent().to_owned()); + let _ = b.node(kind).needs(lease); + }), + reconcile.node_id, + ); + q.complete_node(reconcile.node_id, Ok(())); + + // (a) + (b): the child runs, under the parent that parked in `Finishing`. + let start = claim_one(&q); + assert_eq!(start.kind.as_str(), "start"); + assert_eq!( + declared_resources(&q, start.node_id), + vec![Resource::Agent("agent-a".to_owned())], + "a fanned-out Start declares the lease it runs under" + ); + + // (c): one unit, not two. `claim_one` above already asserted the rival did + // not come back in the same pass; make the reason explicit. + assert!( + q.claim_ready().is_empty(), + "the rival DAG's Reconcile must still be blocked — the appended Start \ + borrowed the grant rather than acquiring a second unit" + ); + + q.complete_node(start.node_id, Ok(())); + // Subtree terminal → the grant releases and the rival finally runs. + let rival_reconcile = claim_one(&q); + assert_eq!(rival_reconcile.dag_id, rival); + q.complete_node(rival_reconcile.node_id, Ok(())); + assert_eq!(state_of(&q, id), State::Done); + assert_eq!(state_of(&q, rival), State::Done); +} + #[test] fn boot_sweep_nodes_declare_their_own_resources() { // Regression, and the reason it needs its own test: `workers::auto_update` @@ -784,7 +809,7 @@ fn boot_sweep_nodes_declare_their_own_resources() { // meta commit inside another node's staged deploy window. Nothing failed to // compile; only an exhaustive caller list would have caught it. let q = JobQueue::new(4); - let id = submit( + let _id = submit( &q, DagSpec { source: Source::AutoUpdate, @@ -800,7 +825,16 @@ fn boot_sweep_nodes_declare_their_own_resources() { }, ); - let mut lock = declared_resources(&q, node_of(&q, id, "meta_lock")); + // Both are independent roots on disjoint resources, so both start at once. + let claims = q.claim_ready(); + let by_kind = |kind: &str| { + claims + .iter() + .find(|c| c.kind.as_str() == kind) + .unwrap_or_else(|| panic!("no {kind} claim in {claims:?}")) + }; + + let mut lock = declared_resources(&q, by_kind("meta_lock").node_id); lock.sort_by_key(|r| format!("{r:?}")); assert_eq!( lock, @@ -809,12 +843,88 @@ fn boot_sweep_nodes_declare_their_own_resources() { ); assert_eq!( - declared_resources(&q, node_of(&q, id, "reconcile")), + declared_resources(&q, by_kind("reconcile").node_id), vec![Resource::Agent("drifted-agent".to_owned())], "a boot Reconcile touches the container, so it holds that agent's lease" ); } +#[test] +fn append_subgraph_roots_on_emitter_and_rebases_local_deps() { + // The startup-sweep mechanism: a `MetaLock` emitter grows one rebuild + // subgraph per stale agent into its OWN DAG. Each subgraph is rooted on + // the emitter and its LOCAL 0-based deps are rebased onto the DAG. + let q = JobQueue::new(4); + let spec = DagSpec { + source: Source::AutoUpdate, + reason: "sweep".to_owned(), + declare: Box::new(|b: &Job| { + let _lock = b.node(NodeKind::MetaLock { + sweep: true, + fanout: None, + inputs: Vec::new(), + }); + }), + }; + let id = submit(&q, spec); + let emitter = claim_one(&q); + assert_eq!(emitter.kind.as_str(), "meta_lock"); + // Two independent per-agent subgraphs — the REAL production shape the + // sweep MetaLock grows: root MetaSync → root Prebuild → Signal → Drain → + // StopForUpdate → Swap → Reconcile, local 0-based deps. `graceful` must + // match the sweep arm of `run_meta_lock` or this stops tracking production. + let subgraph = |agent: &str| -> Declare { + let agent = agent.to_owned(); + Box::new(move |b: &Job| { + templates::rebuild_nodes( + b, + &agent, + templates::RebuildOpts { + relock: true, + graceful: true, + }, + None, + ); + }) + }; + // Must append BEFORE completing the emitter (the documented contract). + q.append_subgraph(id, subgraph("a"), emitter.node_id); + q.append_subgraph(id, subgraph("b"), emitter.node_id); + q.complete_node(emitter.node_id, Ok(())); + // Still ONE DAG; both subgraph roots become ready once the emitter is + // Done (rooted on it), each on its own agent lease. Their `MetaSync` heads + // take turns on the cap-1 global meta window, so drain those first — what + // must be concurrent is the builds. + assert_eq!(q.snapshot().len(), 1); + let mut kinds = drain_meta_syncs(&q); + kinds.sort_unstable(); + assert_eq!( + kinds, + vec![ + ("a".to_owned(), "prebuild".to_owned()), + ("b".to_owned(), "prebuild".to_owned()) + ], + "both rebuild subgraphs root on the emitter and run concurrently in one DAG" + ); +} + +/// Complete every `MetaSync` head the queue offers (they take turns on the +/// cap-1 global meta window) and return whatever else got claimed alongside +/// them, as `(agent, kind)` pairs left in flight. +fn drain_meta_syncs(q: &JobQueue) -> Vec<(String, String)> { + let mut rest = Vec::new(); + for _ in 0..3 { + for c in q.claim_ready() { + if c.kind.as_str() == "meta_sync" { + q.complete_node(c.node_id, Ok(())); + } else { + rest.push((c.agent.clone(), c.kind.as_str().to_owned())); + } + } + } + rest +} + /// Crash-watch suppression for a cascade rebuild, which the deleted half of /// `meta_update_grows_cascade_in_dag` used to assert via `DagSpec::transient`. /// @@ -854,201 +964,193 @@ fn rebuild_chain_nodes_suppress_crash_watch() { ); } +#[test] +fn meta_update_grows_cascade_in_dag() { + // The meta-update `MetaLock` grows one rebuild subgraph per affected + // agent into its OWN DAG (via append_subgraph), not child DAGs. + let spec = templates::meta_update( + vec!["nixpkgs".to_owned()], + Source::Manual, + "bump".to_owned(), + None, + ); + let q = JobQueue::new(4); + let id = submit(&q, spec); + let meta_lock = claim_one(&q); + assert_eq!(meta_lock.kind.as_str(), "meta_lock"); + // Simulate the executor growing the cascade in-DAG (`relock = false` — a + // cascade child must not re-lock and revert the parent's bump). + for agent in ["alice", "bob"] { + let declare: Declare = Box::new(move |b: &Job| { + templates::rebuild_nodes( + b, + agent, + templates::RebuildOpts { + relock: false, + graceful: false, + }, + None, + ); + }); + q.append_subgraph(id, declare, meta_lock.node_id); + } + q.complete_node(meta_lock.node_id, Ok(())); + // Still ONE DAG — no child DAGs — and both cascade rebuild subgraphs root + // on the MetaLock, each on its own agent lease. The per-agent `MetaSync` + // heads serialize on the global meta window (they commit to the meta repo); + // the builds behind them do not. + assert_eq!(q.snapshot().len(), 1); + let mut kinds = drain_meta_syncs(&q); + kinds.sort_unstable(); + assert_eq!( + kinds, + vec![ + ("alice".to_owned(), "prebuild".to_owned()), + ("bob".to_owned(), "prebuild".to_owned()) + ], + "cascade rebuilds grow in the meta-update DAG, concurrent per agent" + ); +} + // ---- failure: cancel-downstream + AfterAny ---- -// -// `failed_node_cancels_downstream_but_afterany_reconcile_runs` lived here. It -// drove a rebuild to a failed `Prebuild` and then asserted three unrelated -// things at once, which is why it needed a running scheduler at all: -// -// 1. the cascade — a failed node cancels its `AfterOk` dependants while the -// `AfterAny` reconcile still runs. That is hive_jobq's rule, and it owns -// the test: `failed_after_ok_dep_cancels_dependents_but_after_any_runs`. -// c0re's declaration of *which* edge is which is asserted in -// `rebuild_chain_is_declared_serial` — the reconcile's accepted outcome -// set is right there in the shape table. -// 2. the wire projection — `Done` off, `Skipped` on. That is -// `shown_on_wire`, tested directly above. -// 3. the roll-up — a DAG with a failed node reads `Failed`, and `Skipped` -// contributes nothing. That is `DagView::rollup_state`, which lives in -// hive-host-sock and is now tested there, next to the invariant. -// -// Reconstructing all three from one arranged run made none of them -// individually legible, and the arrangement was the only reason this module -// needed to claim and complete nodes. + +#[test] +fn failed_node_cancels_downstream_but_afterany_reconcile_runs() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + let meta_sync = claim_one(&q); + assert_eq!(meta_sync.kind.as_str(), "meta_sync"); + q.complete_node(meta_sync.node_id, Ok(())); + let prebuild = claim_one(&q); + assert_eq!(prebuild.kind.as_str(), "prebuild"); + q.complete_node(prebuild.node_id, Err("nix build exploded".to_owned())); + // StopForUpdate + Swap are cancelled (AfterOk on a failed chain); + // the AfterAny Reconcile still runs once Swap is terminal. + let reconcile = claim_one(&q); + assert_eq!(reconcile.kind.as_str(), "reconcile"); + q.complete_node(reconcile.node_id, Ok(())); + let snap = q.snapshot(); + let dag = snap.iter().find(|d| d.id == id).expect("dag"); + assert_eq!(dag.rollup_state(), State::Failed, "roll-up failed"); + let by_kind = |k: &str| { + dag.nodes + .iter() + .find(|n| n.kind == k) + .expect("node present") + .state + }; + assert_eq!(by_kind("prebuild"), State::Failed); + // `StopForUpdate` / `Swap` / `PostSwap` were *ruled out* by the failed + // `Prebuild`. They ride the wire as `Skipped` so an operator can see which + // steps the run never reached, without them reading as failures of their + // own — the roll-up ignores `Skipped` entirely. + for ruled_out in ["stop_for_update", "swap", "post_swap"] { + assert_eq!( + by_kind(ruled_out), + State::Skipped, + "{ruled_out} was ruled out, so it is on the wire as skipped" + ); + } + // The AfterAny reconcile ran (claimed + completed Ok above) → it's `Done`, + // and `Done` nodes are excluded from the wire, so it's absent here. + assert!( + dag.nodes.iter().all(|n| n.kind != "reconcile"), + "the completed (Done) reconcile is filtered off the wire" + ); + assert_eq!( + dag.nodes + .iter() + .find(|n| n.kind == "prebuild") + .and_then(|n| n.error.as_deref()), + Some("nix build exploded") + ); +} + +/// The swap-failure recovery: `Swap` fails → the `AfterOk` `PostSwap` is +/// cancel-cascaded → its terminal state still satisfies `Reconcile`'s +/// `AfterAny(PostSwap)` edge, so recovery-start runs and brings a wanted-up +/// agent back on its old config. +#[test] +fn swap_failure_still_runs_reconcile() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + // meta_sync + prebuild + stop_for_update + for _ in 0..3 { + let c = claim_one(&q); + q.complete_node(c.node_id, Ok(())); + } + let swap = claim_one(&q); + assert_eq!(swap.kind.as_str(), "swap"); + q.complete_node(swap.node_id, Err("update failed".to_owned())); + // PostSwap (AfterOk on the failed Swap) is cancel-cascaded; Reconcile is + // next-claimable via its AfterAny(PostSwap) edge. + let reconcile = claim_one(&q); + assert_eq!(reconcile.kind.as_str(), "reconcile"); + q.complete_node(reconcile.node_id, Ok(())); + let all_dags = q.snapshot(); + let dag = all_dags.iter().find(|d| d.id == id).expect("dag"); + assert_eq!( + dag.nodes + .iter() + .find(|n| n.kind == "post_swap") + .expect("post_swap node") + .state, + State::Skipped, + "PostSwap is ruled out by the failed Swap, and says so on the wire" + ); + assert_eq!( + dag.nodes + .iter() + .find(|n| n.kind == "swap") + .expect("swap node") + .state, + State::Failed, + "and the failure that ruled it out is still on the wire" + ); + assert_eq!(state_of(&q, id), State::Failed); +} /// The swap-success path: `Swap` ok → the `AfterOk` `PostSwap` (bookkeeping /// tail) runs, and only then does `Reconcile` fire — serialized behind /// `PostSwap` (not racing it) because `Reconcile` deps `AfterAny(PostSwap)`. #[test] -fn rebuild_reconcile_waits_for_the_whole_build_subtree() { - // Replaces `swap_ok_runs_post_swap_before_reconcile` and - // `swap_failure_still_runs_reconcile`, which walked the same DAG with the - // swap succeeding in one and failing in the other. - // - // The interesting claim was "Reconcile must wait for PostSwap, not race - // it" — and it does *not* come from an edge between them. `reconcile` deps - // `AfterAny(prebuild)`, while `post_swap` sits inside prebuild's subtree - // (post_swap → stop_for_update → prebuild). A parent is not terminal until - // its subtree is, so prebuild cannot satisfy that edge while post_swap is - // outstanding. **The ordering is the parent chain, not a dependency.** - // - // Both facts are asserted in `rebuild_chain_is_declared_serial`; this test - // states the derived property explicitly because the indirection is the - // easy thing to break — someone flattening the chain would keep every edge - // and still lose the guarantee. +fn swap_ok_runs_post_swap_before_reconcile() { let q = JobQueue::new(1); let id = submit(&q, rebuild("agent-a", "r")); - let shape = declared_shape(&q, id); - let parent_of = |kind: &str| { - shape - .iter() - .find(|d| d.kind == kind) - .unwrap_or_else(|| panic!("{kind} node")) - .parent - }; - assert_eq!(parent_of("post_swap"), Some("stop_for_update")); - assert_eq!(parent_of("stop_for_update"), Some("prebuild")); - assert_eq!( - shape - .iter() - .find(|d| d.kind == "reconcile") - .expect("reconcile node") - .after, - vec![("prebuild", "done|failed|skipped".to_owned())], - "reconcile gates on prebuild's roll-up, which covers the whole build \ - subtree — including post_swap — and runs on failure too" + // meta_sync + prebuild + stop_for_update + for _ in 0..3 { + let c = claim_one(&q); + q.complete_node(c.node_id, Ok(())); + } + let swap = claim_one(&q); + assert_eq!(swap.kind.as_str(), "swap"); + q.complete_node(swap.node_id, Ok(())); + // PostSwap runs next, and nothing else is claimable while it does — the + // tail serializes ahead of Reconcile. + let post_swap = claim_one(&q); + assert_eq!(post_swap.kind.as_str(), "post_swap"); + assert!( + q.claim_ready().is_empty(), + "Reconcile must wait for PostSwap, not race it" ); + q.complete_node(post_swap.node_id, Ok(())); + let reconcile = claim_one(&q); + assert_eq!(reconcile.kind.as_str(), "reconcile"); + q.complete_node(reconcile.node_id, Ok(())); + settle_rebuild_tail(&q, "agent-a", true); + assert_eq!(state_of(&q, id), State::Done); } -// `swap_failure_still_runs_reconcile` lived here. -// -// It asserted that a failed swap leaves `post_swap` `Skipped` and `swap` -// `Failed`, and that reconcile still runs. All three are hive_jobq's cascade -// (`failed_after_ok_dep_cancels_dependents_but_after_any_still_runs`), and the -// "says so on the wire" half turned out to be nothing: `snapshot` fills -// `NodeView { state: node.state, .. }`, a straight copy of the same `State` -// type, so there is no c0re-side mapping to get wrong. -// -// `failed_reconcile_marks_dag_failed` lived here too — a one-node DAG whose -// node fails, asserting the DAG reads `Failed`. That is `failed_child_rolls_ -// parent_up_to_failed` in hive_jobq, restated through a c0re template. - -// `multi_agent_lease_frees_per_subgraph_not_whole_dag` and -// `dag_settles_terminal_and_releases_lease_after_work` lived here. -// -// Both drove a DAG to completion to watch an agent lease free up — one when a -// single agent's subgraph settled inside a still-running multi-agent DAG, the -// other when a whole power op finished. Releasing a grant once its owner's -// subtree is terminal is hive_jobq's (`owner_holds_grant_for_its_whole_subtree`, -// `child_borrows_ancestor_grant_released_when_subtree_done`, -// `leaf_owner_goes_done_directly_and_releases`). -// -// The c0re halves are declared and asserted elsewhere: that each agent's -// subgraph is an independent root holding only its own lease is in -// `multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs`, and -// that a power op emits no tail node is in -// `cancelled_power_op_runs_no_compensating_node`, which checks the DAG has no -// pending nodes left at all. - -/// `Start` / `Stop` were lease-exempt *as kinds*, which was only safe because -/// every construction site fans them out from inside a lease-holding ancestor. -/// They declare the lease themselves now, and this pins that they do. -/// -/// Was `a_fanned_out_start_declares_the_lease_and_re_enters_its_reconciles_ -/// grant`, which submitted a `Reconcile`, claimed it, and then **re-declared -/// the fan-out inline** — *"same two calls the scheduler makes"*. That is a -/// copy of production in a test: had `exec.rs` stopped declaring the lease, it -/// would have kept passing. The declaration now lives in `templates:: -/// fanned_out_mechanical`, so this calls the real thing. -/// -/// The other half of the old test — that a descendant *re-enters* its -/// ancestor's grant rather than taking a second unit of a cap-1 lease — is -/// `hive_jobq`'s, and is tested there by -/// `child_borrows_ancestor_grant_released_when_subtree_done` and -/// `nested_borrowers_never_deadlock`. #[test] -fn a_fanned_out_mechanical_node_declares_its_agent_lease() { - let q = JobQueue::new(4); +fn failed_reconcile_marks_dag_failed() { + let q = JobQueue::new(1); let id = submit( &q, - DagSpec { - source: Source::Manual, - reason: "fan-out".to_owned(), - declare: Box::new(|b: &Job| { - templates::fanned_out_mechanical( - b, - NodeKind::Start { - agent: "agent-a".to_owned(), - }, - ); - }), - }, - ); - assert_eq!(declared_shape(&q, id), vec![row("start", None, &[])]); - assert_eq!( - declared_resources(&q, node_of(&q, id, "start")), - vec![Resource::Agent("agent-a".to_owned())], - "the fanned-out node carries the lease itself, rather than relying on \ - whoever happened to fan it out" - ); -} - -/// A running `MetaLock` grows one rebuild subgraph per agent into **its own -/// DAG**, rooted on itself — not as child DAGs. That is what keeps a boot sweep -/// (or a meta-update cascade) one unit of work, with every rebuild building -/// against the lock the emitter just bumped. -/// -/// Replaces `grown_subgraph_roots_on_emitter_and_rebases_local_deps` and -/// `meta_update_grows_cascade_in_dag`, which differed only in `RebuildOpts` and -/// each minted a builder by hand to simulate the graft. What they were checking -/// is `templates::grown_rebuilds`, so this calls it. -/// -/// That the grafted work lands under the emitter, and that the emitter parks in -/// `Finishing` until it settles, is `hive_jobq`'s -/// (`a_completing_node_grows_the_work_it_declared`). -#[test] -fn a_meta_lock_grows_one_rebuild_subgraph_per_agent() { - let q = JobQueue::new(4); - let agents = vec!["alice".to_owned(), "bob".to_owned()]; - let id = submit( - &q, - DagSpec { - source: Source::AutoUpdate, - reason: "sweep".to_owned(), - declare: Box::new(move |b: &Job| { - templates::grown_rebuilds( - b, - &agents, - templates::RebuildOpts { - relock: true, - graceful: true, - }, - ); - }), - }, - ); - - // One chain per agent, each an independent group root — so the two rebuild - // concurrently, each on its own lease. - let shape = declared_shape(&q, id); - let heads: Vec<_> = shape - .iter() - .filter(|d| d.kind == "meta_sync") - .map(|d| d.parent) - .collect(); - assert_eq!(heads, vec![None, None], "one root chain per agent"); - assert_eq!( - shape.iter().filter(|d| d.kind == "prebuild").count(), - 2, - "both agents get their own build" - ); - // `graceful: true` is the sweep's distinguishing knob — agents mid-turn - // when the host came up get their drain window rather than being cut off. - assert_eq!( - shape.iter().filter(|d| d.kind == "drain").count(), - 2, - "a boot sweep is graceful, so each agent gets a drain" + templates::reconcile_only("agent-a", Source::Manual, "start".to_owned()), ); + let c = claim_one(&q); + q.complete_node(c.node_id, Err("start failed".to_owned())); + assert_eq!(state_of(&q, id), State::Failed); } // ---- cancel ---- @@ -1065,13 +1167,12 @@ fn cancel_clears_queued_dag() { assert_eq!(state_of(&q, id), State::Cancelled, "no stale Queued gap"); // Neither `EmitRebuilt` tail accepts a *dropped* dependency — the ok one is // `AFTER_OK`, the failure one keys on elimination — so both are cancelled - // with the work and **nothing is left that could still run**: no node is - // spared, so a rebuild that never ran emits nothing. + // with the work and **nothing is emitted** for a rebuild that never ran. assert!( - pending_kinds(&q, id).is_empty(), - "a dropped rebuild leaves nothing alive, got {:?}", - pending_kinds(&q, id) + q.claim_ready().is_empty(), + "a dropped rebuild reports nothing" ); + assert_eq!(state_of(&q, id), State::Cancelled); } /// `cancel` takes a **node** id, not a DAG id — so an interior node can be @@ -1097,90 +1198,129 @@ fn cancel_drops_one_agents_branch_leaving_the_rest() { assert!(q.cancel(a_root.id), "an interior/group root cancels alone"); - // agent-a's subgraph is gone; agent-b's is untouched and still alive. + // agent-b's work is untouched and still claimable; agent-a's is not. + let claims = q.claim_ready(); assert!( - pending_kinds_for(&q, id, "agent-a").is_empty(), - "agent-a's branch was dropped whole, got {:?}", - pending_kinds_for(&q, id, "agent-a") - ); - assert!( - !pending_kinds_for(&q, id, "agent-b").is_empty(), - "agent-b's branch survives its sibling's cancel" + !claims.is_empty() && claims.iter().all(|c| c.agent == "agent-b"), + "only agent-b remains runnable, got {:?}", + claims.iter().map(|c| c.agent.as_str()).collect::>() ); } +#[test] +fn cancel_refuses_running_dag() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + let _ = claim_one(&q); + assert!(!q.cancel(id)); + assert_eq!(state_of(&q, id), State::Running); +} + /// A cancelled power op must run **no** compensating node — not even one that /// carries a `SetWanted` head. /// /// Now structural rather than a property of a hook enum: a power op emits no /// tail node at all, so once its work nodes cancel there is simply nothing left /// to claim. `cancel` also refuses unless every work node is still `Pending` -/// (`hive_jobq`'s `cancel_node_refuses_a_group_with_anything_running`), so a -/// `Cancelled` DAG provably never executed a node: its `SetWanted` never ran -/// and the agent's intent still reads whatever +/// (`cancel_refuses_running_dag`), so a `Cancelled` DAG provably never executed +/// a node: its `SetWanted` never ran and the agent's intent still reads whatever /// the operator last set. A "revert" instead writes the agent's *observed* /// state, which for a down-but-`wanted = Up` agent (crashed, or caught /// mid-bounce) flips the intent to `Offline` and leaves it /// deliberately-stopped as far as reconcile and crash-watch are concerned. #[test] fn cancelled_power_op_runs_no_compensating_node() { - /// Submit-cancel-assert for one power op. Taking the already-submitted DAG - /// id is what removes the need to put three differently-typed recipes in - /// one array: each caller submits its own spec, so no closure type has to - /// be erased to a boxed one. - fn assert_cancels_clean(q: &JobQueue, id: u64, writes_intent: bool, case: &str) { - // Read the intent head off the submitted DAG rather than out of the - // spec: a declared job holds its own nodes and inserts them. - let has_intent = declared_shape(q, id).iter().any(|d| d.kind == "set_wanted"); - assert_eq!(has_intent, writes_intent, "{case}: intent head"); - assert!(q.cancel(id), "{case}: cancelled while queued"); - assert_eq!(state_of(q, id), State::Cancelled); - // Nothing is left that *could* run. Asserting on the pending set rather - // than on "what is ready this instant" also covers a node that is alive - // but blocked — which is exactly what a leftover compensating node - // would look like. - assert_eq!( - pending_kinds(q, id), - Vec::<&str>::new(), - "{case}: a power op emits no tail node, so a cancelled one leaves nothing" - ); - } - for graceful in [false, true] { for running in [false, true] { let targets = vec![("agent-a".to_owned(), running)]; - let case = format!("graceful={graceful} running={running}"); - - let q = JobQueue::new(1); - let id = submit( - &q, - submit::restart_spec(&targets, graceful, Source::Manual, "bounce".to_owned()), - ); - assert_cancels_clean(&q, id, false, &format!("restart {case}")); - - let q = JobQueue::new(1); - let id = submit( - &q, - submit::stop_spec(&targets, graceful, Source::Manual, "stop".to_owned()), - ); - assert_cancels_clean(&q, id, true, &format!("stop {case}")); - - let q = JobQueue::new(1); - let id = submit( - &q, - submit::start_spec( - &[("agent-a".to_owned(), running, false)], - Source::Manual, - "start".to_owned(), + // Erased to `DagSpec`: three different recipe types have to + // sit in one array. + let cases = [ + ( + "restart", + false, + erase(submit::restart_spec( + &targets, + graceful, + Source::Manual, + "bounce".to_owned(), + )), ), - ); - assert_cancels_clean(&q, id, true, &format!("start {case}")); + ( + "stop", + true, + erase(submit::stop_spec( + &targets, + graceful, + Source::Manual, + "stop".to_owned(), + )), + ), + ( + "start", + true, + erase(submit::start_spec( + &[("agent-a".to_owned(), running, false)], + Source::Manual, + "start".to_owned(), + )), + ), + ]; + for (name, writes_intent, spec) in cases { + let q = JobQueue::new(1); + let id = submit(&q, spec); + // Read the intent head off the submitted DAG rather than out of + // the spec: a declared job holds its own nodes and inserts them. + assert_eq!( + q.snapshot() + .iter() + .find(|d| d.id == id) + .expect("submitted dag") + .nodes + .iter() + .any(|n| n.kind == "set_wanted"), + writes_intent, + "{name} intent head (graceful={graceful}, running={running})" + ); + assert!(q.cancel(id), "cancelled while queued"); + assert_eq!(state_of(&q, id), State::Cancelled); + assert!( + q.claim_ready().is_empty(), + "cancelled {name} (graceful={graceful}, running={running}) must \ + leave nothing to run — a power op emits no tail node" + ); + } } } } // ---- terminal reporting + lease release ---- +#[test] +fn dag_settles_terminal_and_releases_lease_after_work() { + let q = JobQueue::new(1); + let id = submit(&q, restart_online(&["agent-a"], false, "r")); + // restart = StopForUpdate → Reconcile. + let stop = claim_one(&q); + assert_eq!(stop.kind.as_str(), "stop_for_update"); + q.complete_node(stop.node_id, Ok(())); + let rec = claim_one(&q); + assert_eq!(rec.kind.as_str(), "reconcile"); + // Completing the last work node rolls the container up terminal. A power op + // has no tail node, so nothing is left to claim. + q.complete_node(rec.node_id, Ok(())); + assert!(q.claim_ready().is_empty(), "no tail node to claim"); + assert_eq!(state_of(&q, id), State::Done); + // Lease released when the work chain settled: a new DAG for the agent claims + // immediately. + let next = submit( + &q, + templates::reconcile_only("agent-a", Source::Manual, "stop".to_owned()), + ); + let c = claim_one(&q); + assert_eq!(c.dag_id, next); +} + /// A DAG cancelled while fully queued must still **run its tail**, or a queued /// approval DAG cancelled by the operator would dangle its approval forever. /// @@ -1198,25 +1338,15 @@ fn cancelled_dag_still_runs_its_approval_tail() { ); assert!(q.cancel(id), "fully-queued dag cancels"); // The `Cancelled` tail is the only node whose edge accepts a dropped - // dependency, so it is the only one `cancel` spares — and *which* tail - // survives is the whole assertion: the template emits one per outcome and - // the spared one names how the approval row is about to be resolved. - // Nothing computes it, so reading the survivor is reading the answer. - let spared = pending_payloads(&q, id); - assert!( - matches!( - spared.as_slice(), - [NodeKind::ResolveApproval { - approval_id: 7, - outcome: TerminalState::Cancelled - }] - ), - "only the cancelled-outcome tail is spared, got {spared:?}" - ); + // dependency, so it is the only one `cancel` spares — and claiming it *is* + // the assertion that the approval gets resolved as cancelled. + settle_approval_tail(&q, 7, TerminalState::Cancelled); assert_eq!(state_of(&q, id), State::Cancelled); - // An unrelated DAG landing in the same graph doesn't disturb this one's - // roll-up — the snapshot is per-DAG, not a global state machine. - let _other = submit(&q, rebuild("agent-b", "r")); + // Unrelated later activity doesn't disturb the settled DAG. + let other = submit(&q, rebuild("agent-b", "r")); + let c = claim_one(&q); + assert_eq!(c.dag_id, other); + q.complete_node(c.node_id, Err("boom".to_owned())); assert_eq!(state_of(&q, id), State::Cancelled); } @@ -1235,35 +1365,33 @@ fn deploy_dag_runs_phases_in_order_and_tails_a_failed_apply() { templates::approval_deploy("agent-a", 7, "approval #7".to_owned()), ); + let root = claim_one(&q); + assert!( + matches!(root.kind, NodeKind::DeployWindow { .. }), + "root claims first: it holds the meta window for the whole subtree" + ); + q.complete_node(root.node_id, Ok(())); + + let verify = claim_one(&q); + assert!(matches!(verify.kind, NodeKind::MergeVerify { .. })); + q.complete_node(verify.node_id, Ok(())); + + let apply = claim_one(&q); + assert!(matches!(apply.kind, NodeKind::DeployApply { .. })); + q.complete_node(apply.node_id, Err("nixos-container update blew up".into())); + + let tail = claim_one(&q); + assert!( + matches!(tail.kind, NodeKind::DeployTail { .. }), + "AfterAny tail runs on a failed apply — that's the whole point of it" + ); + q.complete_node(tail.node_id, Ok(())); + + settle_approval_tail(&q, 7, TerminalState::Failed); assert_eq!( - declared_shape(&q, id), - vec![ - // The window is the group root and holds the meta window for the - // whole subtree; the three phases are its sub-nodes. - row("deploy_window", None, &[]), - row("merge_verify", Some("deploy_window"), &[]), - // Apply only on a clean verify — a failed verify cancel-cascades - // it, which is what leaves the forge and the applied repo untouched. - row( - "deploy_apply", - Some("deploy_window"), - &[("merge_verify", "done")] - ), - // The compensation tail accepts every terminal outcome of apply, - // *including `skipped`* — which is the state apply lands in when - // verify failed and it never ran. That one edge is the entire - // "still tails a failed apply / a failed verify" behaviour, and it - // is why two separate DAG-driving tests collapsed into this table. - row( - "deploy_tail", - Some("deploy_window"), - &[("deploy_apply", "done|failed|skipped")] - ), - // One approval tail per outcome, gated on the window's roll-up. - row("resolve_approval", None, &[("deploy_window", "done")]), - row("resolve_approval", None, &[("deploy_window", "failed")]), - row("resolve_approval", None, &[("deploy_window", "cancelled")]), - ] + state_of(&q, id), + State::Failed, + "an Ok tail must not launder a failed deploy into a success" ); } @@ -1278,182 +1406,220 @@ fn deploy_dag_runs_phases_in_order_and_tails_a_failed_apply() { /// a resource its own DAG owns and deadlock — this test is what pins that down. #[test] fn deploy_apply_grows_rebuild_subgraph_and_finalizes_after_it() { - // What `DeployApply` grows is `deploy_rebuild_nodes`' output, and that is a - // pure declaration — so it is declared here directly rather than by running - // a deploy far enough to graft it. **Reproducing the runtime path is not - // needed to test what the runtime path declares.** - // - // The grafting mechanism itself is hive_jobq's and tested there: the work - // lands under the emitter *before* it settles, and the emitter parks in - // `Finishing` so a downstream `AfterAny` gate stays shut while the new - // children run (`a_completing_node_grows_the_work_it_declared`, - // `parent_parks_in_finishing_until_children_roll_up`). let q = JobQueue::new(1); let id = submit( &q, - DagSpec { - source: Source::Manual, - reason: "deploy graft".to_owned(), - declare: Box::new(|b: &Job| templates::deploy_rebuild_nodes(b, "agent-a", 11)), - }, + templates::approval_deploy("agent-a", 11, "approval 11".to_owned()), ); + let root = claim_one(&q); + assert!(matches!(root.kind, NodeKind::DeployWindow { .. })); + q.complete_node(root.node_id, Ok(())); + let verify = claim_one(&q); + q.complete_node(verify.node_id, Ok(())); + + let apply = claim_one(&q); + assert!(matches!(apply.kind, NodeKind::DeployApply { .. })); + // Mirrors the scheduler: the executor's `NodeOutput` subgraphs are grafted + // BEFORE the emitting node is completed. Completing first would settle the + // apply node `Done` with nothing under it, opening the tail's `AfterAny` + // gate immediately and letting the deploy "finish" before it had built. + q.append_subgraph( + id, + templates::deploy_rebuild_nodes("agent-a", 11), + apply.node_id, + ); + q.complete_node(apply.node_id, Ok(())); + + // The grafted chain runs in rebuild order. `claim_one` asserts exactly one + // claimable node at each step, which also proves the `AfterAny` tail stays + // shut: `DeployApply` is `Finishing` (not terminal) while its new children + // run, and `Finishing` satisfies neither dep kind. + for expected in [ + "meta_sync", + "prebuild", + "stop_for_update", + "swap", + "post_swap", + "reconcile", + ] { + let c = claim_one(&q); + assert_eq!(c.kind.as_str(), expected, "grafted phase order"); + q.complete_node(c.node_id, Ok(())); + } + + let finalize = claim_one(&q); + assert!( + matches!(finalize.kind, NodeKind::FinalizeDeploy { .. }), + "the deploy tag is planted only after the rebuild came up clean" + ); + q.complete_node(finalize.node_id, Ok(())); + + let tail = claim_one(&q); + assert!(matches!(tail.kind, NodeKind::DeployTail { .. })); + q.complete_node(tail.node_id, Ok(())); + + settle_approval_tail(&q, 11, TerminalState::Done); + assert_eq!(state_of(&q, id), State::Done); +} + +/// A failure *inside* the grafted rebuild is the failure mode the subgraph +/// growth introduces: the deploy is already merged and the container half-swapped. +/// `FinalizeDeploy` must be cancel-cascaded (its `AfterOk` gate never opens) so +/// no `deployed/` tag is planted, while the tail still runs to compensate. +/// `Reconcile` is deliberately still reached — it boots the container back up. +#[test] +fn deploy_dag_skips_finalize_but_still_tails_a_failed_graft() { + let q = JobQueue::new(1); + let id = submit( + &q, + templates::approval_deploy("agent-a", 13, "approval 13".to_owned()), + ); + + let root = claim_one(&q); + q.complete_node(root.node_id, Ok(())); + let verify = claim_one(&q); + q.complete_node(verify.node_id, Ok(())); + let apply = claim_one(&q); + q.append_subgraph( + id, + templates::deploy_rebuild_nodes("agent-a", 13), + apply.node_id, + ); + q.complete_node(apply.node_id, Ok(())); + + for expected in ["meta_sync", "prebuild", "stop_for_update"] { + let c = claim_one(&q); + assert_eq!(c.kind.as_str(), expected); + q.complete_node(c.node_id, Ok(())); + } + let swap = claim_one(&q); + assert_eq!(swap.kind.as_str(), "swap"); + q.complete_node(swap.node_id, Err("profile swap failed".into())); + + // `Reconcile` hangs off `Prebuild` with `AfterAny`, so a failed swap still + // reaches it — bringing the container back up is exactly what it's for. + let reconcile = claim_one(&q); + assert_eq!(reconcile.kind.as_str(), "reconcile"); + q.complete_node(reconcile.node_id, Ok(())); + + let tail = claim_one(&q); + assert!( + matches!(tail.kind, NodeKind::DeployTail { .. }), + "finalize is cancel-cascaded, so the tail is the next claimable node" + ); + q.complete_node(tail.node_id, Ok(())); + + settle_approval_tail(&q, 13, TerminalState::Failed); + assert_eq!(state_of(&q, id), State::Failed); assert_eq!( - declared_shape(&q, id), - vec![ - row("meta_sync", None, &[]), - row("prebuild", None, &[("meta_sync", "done")]), - row("stop_for_update", Some("prebuild"), &[]), - row("swap", Some("stop_for_update"), &[]), - row("post_swap", Some("stop_for_update"), &[("swap", "done")]), - row("reconcile", None, &[("prebuild", "done|failed|skipped")]), - // The deploy tag is planted only after the rebuild came up clean: - // `AfterOk` on **both** roots, so either one failing skips it. That - // pair of edges is the whole "skips finalize on a failed graft" - // behaviour — no run needed to see it. - row( - "finalize_deploy", - None, - &[("prebuild", "done"), ("reconcile", "done")] - ), - ] + q.first_error(id).as_deref(), + Some("profile swap failed"), + "the tail annotates failed/ with this — and it is also what + `exec::failure_reason` falls back to, since the tail's own dep is a + group root that rolled up Failed and so carries no error itself" ); } -// `deploy_dag_skips_finalize_but_still_tails_a_failed_graft` lived here. -// -// A failure *inside* the grafted rebuild is the failure mode subgraph growth -// introduces: the deploy is already merged and the container half-swapped, so -// `FinalizeDeploy` must be cancel-cascaded (no `deployed/` tag planted) -// while the tail still runs to compensate, and `Reconcile` is deliberately -// still reached — it boots the container back up. -// -// Every declared half of that is asserted by -// `deploy_apply_grows_rebuild_subgraph_and_finalizes_after_it`, which reads -// `deploy_rebuild_nodes`' shape directly: -// -// - "a failed swap still reaches Reconcile" is the `reconcile` row's -// `AfterAny` edge on `prebuild` (`done|failed|skipped`); -// - "finalize is cancel-cascaded" is `finalize_deploy`'s `AfterOk` pair — -// either root failing skips it; -// - "the tail still runs" is `deploy_tail`'s own `done|failed|skipped` edge -// on apply, asserted in the `approval_deploy` table above. -// -// The runtime halves are hive_jobq's: cascade on failure, roll-up, and -// `first_error` digging past a group root that rolled up `Failed` while -// carrying no error of its own (`first_error_skips_a_rolled_up_failure_ -// carrying_no_error`). That last one is why the DAG reports "profile swap -// failed" rather than nothing — the mechanism, not this shape. +/// A pre-merge rejection (drift gate, eval failure) cancel-cascades the +/// irreversible half via its `AfterOk` edge, but the tail is still reached — +/// it owns the forge mirror, not just compensation. +#[test] +fn deploy_dag_skips_apply_but_still_runs_tail_when_verify_fails() { + let q = JobQueue::new(1); + let id = submit( + &q, + templates::approval_deploy("agent-a", 9, "approval #9".to_owned()), + ); -// `deploy_dag_skips_apply_but_still_runs_tail_when_verify_fails` lived here. -// -// A pre-merge rejection (drift gate, eval failure) cancel-cascades the -// irreversible half via its `AfterOk` edge, while the tail is still reached — -// it owns the forge mirror, not just compensation. That test and -// `deploy_dag_runs_phases_in_order_and_tails_a_failed_apply` differed only in -// *where* they injected the failure, and each drove the whole DAG to watch the -// tail run anyway. -// -// Both outcomes follow from one declared edge, which the surviving test now -// asserts directly: the tail accepts `done|failed|skipped` on apply, and -// `skipped` is exactly the state apply lands in when verify failed and it never -// ran. The runtime halves are hive_jobq's and tested there — -// `failed_after_ok_dep_cancels_dependents_but_after_any_still_runs` and -// `failed_child_rolls_parent_up_to_failed` (an Ok tail cannot launder a failed -// deploy into a success). + let root = claim_one(&q); + q.complete_node(root.node_id, Ok(())); + let verify = claim_one(&q); + q.complete_node(verify.node_id, Err("PR head drifted since review".into())); -// ---- history ---- -// -// The node → build-log link is no longer queue state: the log row carries -// `node_id` and the lookup lives in `stores::build_logs` (see -// `node_link_survives_completion_and_newest_wins` there). Nothing in the queue -// needs testing for it any more, which is the point of that move. + let tail = claim_one(&q); + assert!( + matches!(tail.kind, NodeKind::DeployTail { .. }), + "apply is cancel-cascaded, so the tail is the next claimable node" + ); + q.complete_node(tail.node_id, Ok(())); + + settle_approval_tail(&q, 9, TerminalState::Failed); + assert_eq!(state_of(&q, id), State::Failed); +} + +// ---- build logs, history ---- + +#[test] +fn set_build_log_id_links_running_node() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + let c = claim_one(&q); + assert!(q.set_build_log_id(id, c.node_id, 42)); + q.complete_node(c.node_id, Ok(())); + assert!( + !q.set_build_log_id(id, c.node_id, 99), + "node no longer running → refused" + ); + // The log id is fetched by node id (the `GET /api/build-log/` lookup), + // not carried on the wire — it survives completion in the node runtime. + assert_eq!( + q.build_log_id_of(c.node_id.get()), + Some(42), + "log id survives completion" + ); +} /// History retention is a **flat** newest-first cap over all terminal DAGs /// (`MAX_HISTORY_DAGS`), not a per-template bucket behind a grace window. /// The dashboard renders one recent-builds list, so one number bounds it — /// and with no bucketing there's nothing for a burst of same-shaped DAGs to /// evict early, which is what the grace window used to paper over. -/// -/// `retain_history` is a sort-and-truncate over `(handle, finished_at, -/// tiebreak)`. This used to submit `MAX_HISTORY_DAGS + 8` DAGs, claim and fail -/// each one's node, then read the ids back out of a snapshot — the scheduler, -/// the roll-up and the wire projection all in the path of a policy that reads -/// none of them. -/// -/// Calling it directly also reaches the half the round-trip never could: every -/// DAG in that loop settled inside the same wall-clock second, so `finished_at` -/// tied on all of them and **only** the tiebreak was ever exercised. Eviction -/// by time — the actual policy — went untested. #[test] -fn history_retains_live_dags_and_the_newest_terminals() { - const CAP: usize = 3; - // Live DAGs are kept whole, regardless of the cap. - assert_eq!( - retain_history(vec!["live-a", "live-b"], vec![], CAP), - vec!["live-a", "live-b"], - ); - // Terminals: newest `finished_at` first, oldest evicted past the cap. - assert_eq!( - retain_history( - vec![], - vec![ - ("oldest", 100, 1), - ("newest", 400, 2), - ("middle", 200, 3), - ("later", 300, 4), - ], - CAP, - ), - vec!["newest", "later", "middle"], - "the oldest terminal falls off" - ); - // Same second → the tiebreak decides, descending, so the DAG inserted - // last wins. This is the *common* case: a burst settles together. - assert_eq!( - retain_history( - vec![], - vec![("a", 100, 1), ("b", 100, 2), ("c", 100, 3), ("d", 100, 4)], - CAP, - ), - vec!["d", "c", "b"], - ); - // A live DAG never competes with history for the cap. - assert_eq!( - retain_history( - vec!["live"], - vec![("a", 100, 1), ("b", 200, 2), ("c", 300, 3), ("d", 400, 4)], - CAP, - ), - vec!["live", "d", "c", "b"], - ); +fn history_evicts_oldest_terminals_past_flat_cap() { + const OVERFLOW: usize = 8; + let q = JobQueue::new(1); + let mut ids = Vec::new(); + for i in 0..(MAX_HISTORY_DAGS + OVERFLOW) { + let id = submit( + &q, + templates::reconcile_only(&format!("agent-{i}"), Source::Manual, "start".to_owned()), + ); + let c = claim_one(&q); + // Fail the single work node so the DAG *lingers*: a fully-`Done` DAG + // drops off the wire entirely, but a `Failed` one is retained (+ + // history-capped) so the operator can still triage it. Completing the + // node rolls the container up terminal. + q.complete_node(c.node_id, Err("boom".to_owned())); + ids.push(id); + } + let kept: std::collections::HashSet = q.snapshot().iter().map(|d| d.id).collect(); + assert_eq!(kept.len(), MAX_HISTORY_DAGS, "flat history cap"); + // Newest-first: the oldest `OVERFLOW` fall off, everything after survives. + // These DAGs settle within the same wall-clock second, so this also pins + // the `NodeId`-descending tiebreak that orders them when `finished_at` ties. + for old in &ids[..OVERFLOW] { + assert!(!kept.contains(old), "oldest terminal {old} evicted"); + } + for recent in &ids[OVERFLOW..] { + assert!(kept.contains(recent), "recent terminal {recent} retained"); + } + assert_eq!(q.live_count(), 0); } #[test] -fn error_truncation_cuts_on_a_char_boundary() { - // `truncate_error` is a pure `&str -> String`. This used to submit a DAG, - // claim its head, fail it with a long string and read the error back out of - // a snapshot — four moving parts to observe one transform, and the DAG - // round-trip is covered by its own tests either way. - // - // Testing it directly also reaches the case the round-trip never did: the - // cap is a **byte** length, so a multibyte char straddling it would panic - // the slice. That boundary scan is the only non-obvious line in the fn and - // it had no coverage at all. - assert_eq!( - truncate_error("short"), - "short", - "under the cap is untouched" - ); - - let ascii = truncate_error(&"x".repeat(5000)); - assert!(ascii.ends_with('…')); - assert!(ascii.len() <= MAX_ERROR_LEN + '…'.len_utf8()); - - // 'é' is 2 bytes, so the 2000-byte cap lands mid-char. - let multibyte = truncate_error(&"é".repeat(5000)); - assert!(multibyte.ends_with('…')); - assert!(multibyte.len() <= MAX_ERROR_LEN + '…'.len_utf8()); +fn error_is_truncated() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + let c = claim_one(&q); + q.complete_node(c.node_id, Err("x".repeat(5000))); + let snap = q.snapshot(); + let err = snap.iter().find(|d| d.id == id).expect("dag").nodes[0] + .error + .clone() + .expect("error stored"); + assert!(err.chars().count() <= 2001, "truncated + ellipsis"); + assert!(err.ends_with('…')); } // ---- template shapes ---- @@ -1462,29 +1628,41 @@ fn error_truncation_cuts_on_a_char_boundary() { fn graceful_stop_shape_signal_drain_reconcile() { let q = JobQueue::new(1); let id = submit(&q, stop_online(&["agent-a"], true, "graceful")); + for expected in ["set_wanted", "signal", "drain", "reconcile"] { + let c = claim_one(&q); + assert_eq!(c.kind.as_str(), expected); + q.complete_node(c.node_id, Ok(())); + } + assert_eq!(state_of(&q, id), State::Done); +} + +#[test] +fn graceful_signal_and_drain_hold_no_build_slot() { + // A whole-hive graceful stop overlaps every drain even at + // buildSlots = 1 while a rebuild hogs the slot. + let q = JobQueue::new(1); + submit(&q, rebuild("builder", "slot hog")); + submit(&q, stop_online(&["agent-a"], true, "g")); + submit(&q, stop_online(&["agent-b"], true, "g")); + // All three DAG heads are build-slot-exempt, so they run at once. + let heads = q.claim_ready(); + let kinds: Vec<&str> = heads.iter().map(|c| c.kind.as_str()).collect(); + assert_eq!(kinds, vec!["meta_sync", "set_wanted", "set_wanted"]); + for c in &heads { + q.complete_node(c.node_id, Ok(())); + } + // Now the rebuild's Prebuild holds the single slot — and both graceful + // stops still proceed to their Signal beside it. + let kinds: Vec<&str> = q + .claim_ready() + .iter() + .map(|c| c.kind.as_str()) + .collect::>(); assert_eq!( - declared_shape(&q, id), - vec![ - // The whole stop hangs under `set_wanted`: the durable intent is - // written first, and the mechanical steps are its sub-nodes. - row("set_wanted", None, &[]), - row("signal", Some("set_wanted"), &[]), - row("drain", Some("set_wanted"), &[("signal", "done")]), - row("reconcile", Some("set_wanted"), &[("drain", "done")]), - ] - ); - // Neither half of the graceful window takes a build slot. That is what lets - // a whole-hive graceful stop overlap every agent's drain even at - // `buildSlots = 1` while a rebuild hogs the slot — the cost ceiling is one - // `GRACEFUL_STOP_TIMEOUT` in total, not one per agent. - let agent = Resource::Agent("agent-a".to_owned()); - assert_eq!( - [ - declared_resources(&q, node_of(&q, id, "signal")), - declared_resources(&q, node_of(&q, id, "drain")), - ], - [vec![agent.clone()], vec![agent]], - "signal and drain hold the lease but never a build slot" + kinds, + vec!["prebuild", "signal", "signal"], + "both agents' graceful-stop signals (build-slot-exempt) run while the \ + rebuild holds the slot" ); } @@ -1495,23 +1673,13 @@ fn spawn_shape_provision_create_dropin_reconcile() { &q, templates::spawn("newbie", 7, "approval #7 spawn".to_owned()), ); - assert_eq!( - declared_shape(&q, id), - vec![ - row("provision", None, &[]), - row("create", Some("provision"), &[]), - row("write_dropin", Some("create"), &[]), - row("reconcile", Some("create"), &[("write_dropin", "done")]), - // One tail per outcome, each edged to accept only that one — so - // *which* tail the graph lets run already is the answer, and - // nothing branches at runtime. The three differ **only** in their - // accepted outcome, which is why `declared_shape` spells the - // outcome set out instead of bucketing it. - row("resolve_approval", None, &[("provision", "done")]), - row("resolve_approval", None, &[("provision", "failed")]), - row("resolve_approval", None, &[("provision", "cancelled")]), - ] - ); + for expected in ["provision", "create", "write_dropin", "reconcile"] { + let c = claim_one(&q); + assert_eq!(c.kind.as_str(), expected); + q.complete_node(c.node_id, Ok(())); + } + settle_approval_tail(&q, 7, TerminalState::Done); + assert_eq!(state_of(&q, id), State::Done); } #[test] @@ -1529,25 +1697,21 @@ fn perm_change_shape_prefixes_rebuild_chain() { }, ), ); - assert_eq!( - declared_shape(&q, id) - .iter() - .map(|d| d.kind) - .collect::>(), - vec![ - "write_perm_file", - "meta_sync", - "prebuild", - "stop_for_update", - "swap", - "post_swap", - "reconcile", - // the ok / !ok tail pair - "emit_rebuilt", - "emit_rebuilt", - ], - "the perm write prefixes an otherwise ordinary rebuild chain" - ); + for expected in [ + "write_perm_file", + "meta_sync", + "prebuild", + "stop_for_update", + "swap", + "post_swap", + "reconcile", + ] { + let c = claim_one(&q); + assert_eq!(c.kind.as_str(), expected); + q.complete_node(c.node_id, Ok(())); + } + settle_rebuild_tail(&q, "agent-a", true); + assert_eq!(state_of(&q, id), State::Done); } #[test] @@ -1565,18 +1729,17 @@ fn reparent_shape_is_a_lone_agentless_meta_window_node() { "set-parent".to_owned(), ), ); + let c = claim_one(&q); + assert_eq!(c.kind.as_str(), "reparent"); + assert_eq!(c.agent, "", "Reparent is agentless — no per-agent lease"); assert_eq!( - declared_shape(&q, id), - vec![row("reparent", None, &[])], - "one node, no rebuild subgraph" - ); - let node = node_of(&q, id, "reparent"); - assert_eq!( - declared_resources(&q, node), + declared_resources(&q, c.node_id), vec![Resource::MetaWindow], "a topology commit must declare the same MetaWindow as WritePermFile, \ and nothing else — no lease (agentless), no build slot (no nix work)" ); + q.complete_node(c.node_id, Ok(())); + assert_eq!(state_of(&q, id), State::Done); } #[test] @@ -1590,13 +1753,12 @@ fn reparent_bulk_shape_carries_every_move_on_one_node() { &q, templates::reparent(moves.clone(), Source::Manual, "set-parent-bulk".to_owned()), ); - assert_eq!( - declared_shape(&q, id), - vec![row("reparent", None, &[])], - "one node for the whole request, not one per move" - ); - let NodeKind::Reparent { moves: got } = payload_of(&q, id, "reparent") else { - panic!("expected a Reparent node"); + let c = claim_one(&q); + assert_eq!(c.kind.as_str(), "reparent"); + let NodeKind::Reparent { moves: got } = &c.kind else { + panic!("expected a Reparent node, got {:?}", c.kind); }; - assert_eq!(got, moves, "every move rides the single node"); + assert_eq!(got, &moves); + q.complete_node(c.node_id, Ok(())); + assert_eq!(state_of(&q, id), State::Done); } diff --git a/hive-c0re/src/lifecycle/mod.rs b/hive-c0re/src/lifecycle/mod.rs index a69b71a9..6f816f02 100644 --- a/hive-c0re/src/lifecycle/mod.rs +++ b/hive-c0re/src/lifecycle/mod.rs @@ -274,10 +274,10 @@ pub async fn swap_update( name: &str, hive: &HiveEnv, paths: &AgentPaths, - node_id: Option, + on_build_log_id: &(dyn Fn(i64) + Send + Sync), ) -> Result<()> { write_dropins(name, hive, paths).await?; - priv_run_inner("update", name, node_id).await + priv_run_inner("update", name, Some(on_build_log_id)).await } /// Build the `AgentSpec` list for the meta flake from `nixos-container @@ -582,10 +582,14 @@ pub async fn destroy(name: &str) -> Result<()> { /// the prebuild happens before stop, and `docs/coordinator.md::Prebuild /// attr path` for why the explicit nixosConfigurations attr is required. /// -/// `node_id` is the queue node this build belongs to, when there is one — -/// it is stored on the `build_logs` row so the dashboard can find the log -/// from the node. Pass `None` for builds that run outside the queue. -pub async fn prebuild_toplevel(name: &str, flake_ref: &str, node_id: Option) -> Result<()> { +/// `on_build_log_id` fires with the `build_logs` row id as soon as the +/// row opens, so queue-side callers can link their node to the live +/// stream. Pass `&|_| ()` when not needed. +pub async fn prebuild_toplevel( + name: &str, + flake_ref: &str, + on_build_log_id: &(dyn Fn(i64) + Send + Sync), +) -> Result<()> { use tokio::io::{AsyncBufReadExt, BufReader}; // Split `#` so we can re-emit with the explicit // `nixosConfigurations.` segment. The flake_ref shape is @@ -620,12 +624,15 @@ pub async fn prebuild_toplevel(name: &str, flake_ref: &str, node_id: Option // into the row; `finish` lands the terminal status before we bail. let logs = crate::build_logs::global(); let log_id = logs.as_ref().and_then(|h| { - h.start(name, "prebuild", &cmdline, node_id) + h.start(name, "prebuild", &cmdline) .map_err(|e| { tracing::warn!(error = ?e, "build_logs: start failed (prebuild log dropped)"); }) .ok() }); + if let Some(id) = log_id { + on_build_log_id(id); + } let mut child = Command::new("nix") .args(&args) @@ -777,25 +784,37 @@ async fn priv_run(kind: &str, name: &str) -> Result<()> { priv_run_inner(kind, name, None).await } -/// Like `priv_run` but stamps `node_id` onto the build-log row it opens, so -/// the dashboard can find the log from the queue node (and link to -/// `/api/build-logs/id/{id}/stream`). +/// Like `priv_run` but calls `on_log_id(log_id)` immediately after the +/// build-log row is opened — before the actual container op starts. +/// This lets callers surface the row id for live streaming (e.g. the +/// rebuild-queue worker sets `build_log_id` on the queue entry so the +/// dashboard can link to `/api/build-logs/id/{id}/stream`). /// -/// This used to be a `Fn(i64)` callback that handed the row id *back* to the -/// queue, which then held it in a side map. The row carries the link itself -/// now, so the id only ever travels one way. -async fn priv_run_inner(kind: &str, name: &str, node_id: Option) -> Result<()> { +/// The callback fires only when a build-log row is successfully opened +/// (i.e. the global `BuildLogs` handle is installed AND `h.start()` +/// succeeds). No-op when `on_log_id` is `None` — that's the path for +/// all callers that don't need the id. +async fn priv_run_inner( + kind: &str, + name: &str, + on_log_id: Option<&(dyn Fn(i64) + Send + Sync)>, +) -> Result<()> { let container = container_name(name); let cmdline = format!("nixos-container {kind} {container}"); let logs = crate::build_logs::global(); let log_id = logs.as_ref().and_then(|h| { - h.start(name, kind, &cmdline, node_id) + h.start(name, kind, &cmdline) .map_err(|e| { tracing::warn!(error = ?e, "build_logs: start failed (priv_run log dropped)"); }) .ok() }); + // Notify the caller as soon as the log row exists so it can surface + // the id for live streaming before the container op even starts. + if let (Some(id), Some(cb)) = (log_id, on_log_id) { + cb(id); + } // For long-running ops use the streaming protocol so build_logs // receives lines in real time rather than as a batch at completion. diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index 6da608f3..8c4100ba 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -1590,12 +1590,7 @@ async fn nix_logged(dir: &Path, args: &[&str], agent: &str, kind: &str) -> Resul let cmdline = format!("nix {}", nix_argv(args).join(" ")); let logs = crate::build_logs::global(); let log_id = logs.as_ref().and_then(|h| { - // No node id: `nix_logged`'s two callers are meta-flake operations - // reached from outside the queue as well as from inside it, and the - // agent+kind+time listing is how they're surfaced today. Linking them - // to a node would mean threading the id through `meta`'s public API - // for no current reader — worth doing when something wants it. - h.start(agent, kind, &cmdline, None) + h.start(agent, kind, &cmdline) .map_err(|e| { tracing::warn!(error = ?e, %kind, "build_logs: start failed (meta log dropped)"); }) diff --git a/hive-c0re/src/stores/build_logs.rs b/hive-c0re/src/stores/build_logs.rs index 67b28c3e..24b302bd 100644 --- a/hive-c0re/src/stores/build_logs.rs +++ b/hive-c0re/src/stores/build_logs.rs @@ -13,8 +13,6 @@ use serde::Serialize; use tokio::sync::broadcast; use utoipa::ToSchema; -use crate::db::Migration; - /// Process-singleton handle, set once at coordinator startup. Lets /// the `lifecycle` module's `run` / `prebuild_toplevel` access the /// writer without threading an `Arc` through every @@ -67,27 +65,6 @@ CREATE INDEX IF NOT EXISTS idx_build_logs_status_finished WHERE finished_at IS NOT NULL; "; -/// Ordered schema migrations tracked in `schema_versions` (key `"build_logs"`). -/// -/// v1 makes the log row carry its node, replacing the host-side -/// `NodeId -> build_log_id` map the job queue used to hold. The link has a -/// single home again, and the direction is the one the type system allows: -/// a `hive_jobq` node payload is immutable after insert, but the log row is -/// written when the build starts and can name the node it belongs to. -/// -/// Legacy rows keep `node_id IS NULL` — they predate the column and no node -/// still exists to link them to, so the dashboard's by-node lookup simply -/// misses them (the by-agent listing, which is how they're reached, is -/// unaffected). -const MIGRATIONS: &[Migration] = &[Migration { - sql: "BEGIN; - ALTER TABLE build_logs ADD COLUMN node_id INTEGER; - CREATE INDEX IF NOT EXISTS idx_build_logs_node - ON build_logs (node_id) WHERE node_id IS NOT NULL; - COMMIT;", - adds_column: Some(("build_logs", "node_id")), -}]; - /// Status of a finished build attempt. Stored as the literal string in /// the `status` column; `NULL` while the attempt is still in progress. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] @@ -175,7 +152,6 @@ impl BuildLogs { let conn = crate::db::open(&path, "build_logs")?; conn.execute_batch(SCHEMA) .context("apply build_logs schema")?; - crate::db::apply_versioned_migrations(&conn, "build_logs", MIGRATIONS)?; let (notify_tx, _) = broadcast::channel(NOTIFY_CAP); Ok(Self { conn: Mutex::new(conn), @@ -193,49 +169,17 @@ impl BuildLogs { /// Open a row for a new build attempt. Returns the assigned id /// — the caller threads it through `append_stdout` / `append_stderr` /// while the child runs and into `finish` once it exits. - /// - /// `node_id` is the queue node this build belongs to, when there is one. - /// It is `None` for builds that run outside the job queue; those are - /// reachable by agent + time, just not by node. - pub fn start( - &self, - agent: &str, - kind: &str, - cmdline: &str, - node_id: Option, - ) -> Result { + pub fn start(&self, agent: &str, kind: &str, cmdline: &str) -> Result { let now = Utc::now().timestamp(); - let node_id = node_id.and_then(|n| i64::try_from(n).ok()); let conn = self.conn.lock().unwrap(); conn.execute( - "INSERT INTO build_logs (agent, kind, cmdline, started_at, node_id) \ - VALUES (?1, ?2, ?3, ?4, ?5)", - params![agent, kind, cmdline, now, node_id], + "INSERT INTO build_logs (agent, kind, cmdline, started_at) VALUES (?1, ?2, ?3, ?4)", + params![agent, kind, cmdline, now], ) .context("insert build_logs row")?; Ok(conn.last_insert_rowid()) } - /// The most recent build-log row for `node_id`, if any. Replaces the job - /// queue's in-memory `NodeId -> build_log_id` side map: the link lives in - /// the row itself now, so it survives a restart and needs no lock held - /// alongside the scheduler's. - /// - /// `MAX(id)` rather than a uniqueness assumption — a node that is retried - /// opens a second row, and the newest is the one the panel should show. - #[must_use] - pub fn id_for_node(&self, node_id: u64) -> Option { - let node_id = i64::try_from(node_id).ok()?; - let conn = self.conn.lock().unwrap(); - conn.query_row( - "SELECT MAX(id) FROM build_logs WHERE node_id = ?1", - params![node_id], - |row| row.get::<_, Option>(0), - ) - .ok() - .flatten() - } - /// Append a single stdout line. Best-effort: errors are logged /// but never returned to the caller, so a transient sqlite blip /// never tears down a rebuild's stdout pump. @@ -509,7 +453,7 @@ mod tests { fn start_appends_finish_flow() { let (_d, db) = tmpdb(); let id = db - .start("alice", "prebuild", "nix build foo", None) + .start("alice", "prebuild", "nix build foo") .expect("start"); db.append_stdout(id, "building '/nix/store/abc.drv'"); db.append_stderr(id, "error: line 12"); @@ -538,9 +482,9 @@ mod tests { // assert id-ordering (autoincrement) is the tiebreaker — list // sorts by started_at DESC but the ORDER BY still produces the // last-inserted row first when timestamps match. - let id_a1 = db.start("alice", "run", "cmd one", None).expect("start"); - let _id_b = db.start("bob", "run", "cmd two", None).expect("start"); - let id_a2 = db.start("alice", "run", "cmd three", None).expect("start"); + let id_a1 = db.start("alice", "run", "cmd one").expect("start"); + let _id_b = db.start("bob", "run", "cmd two").expect("start"); + let id_a2 = db.start("alice", "run", "cmd three").expect("start"); db.finish(id_a1, BuildStatus::Ok); let alice_rows = db.list_recent_for_agent("alice", 10).expect("list"); @@ -571,12 +515,10 @@ mod tests { #[test] fn vacuum_drops_old_finished_only_per_status() { let (_d, db) = tmpdb(); - let id_fresh_fail = db.start("alice", "run", "fresh fail", None).expect("start"); - let id_old_fail = db.start("alice", "run", "old fail", None).expect("start"); - let id_old_ok = db.start("alice", "run", "old ok", None).expect("start"); - let id_running = db - .start("alice", "run", "still running", None) - .expect("start"); + let id_fresh_fail = db.start("alice", "run", "fresh fail").expect("start"); + let id_old_fail = db.start("alice", "run", "old fail").expect("start"); + let id_old_ok = db.start("alice", "run", "old ok").expect("start"); + let id_running = db.start("alice", "run", "still running").expect("start"); db.finish(id_fresh_fail, BuildStatus::Fail); db.finish(id_old_fail, BuildStatus::Fail); db.finish(id_old_ok, BuildStatus::Ok); @@ -608,31 +550,6 @@ mod tests { assert!(db.get_full(id_running).unwrap().is_some()); } - #[test] - fn node_link_survives_completion_and_newest_wins() { - // The queue used to hold this link in an in-memory side map, which - // meant it died with the process and needed the queue lock to read. - // On the row it outlives both the node's completion and a restart. - let (_d, db) = tmpdb(); - let first = db.start("alice", "swap", "cmd", Some(7)).expect("start"); - db.finish(first, BuildStatus::Fail); - assert_eq!( - db.id_for_node(7), - Some(first), - "link survives the build finishing" - ); - - // A retried node opens a second row; the panel wants the current - // attempt, not the first one. - let retry = db.start("alice", "swap", "cmd", Some(7)).expect("start"); - assert_eq!(db.id_for_node(7), Some(retry), "newest attempt wins"); - - // Builds that run outside the queue carry no node and are found by - // agent + time instead — they must not collide with node lookups. - db.start("alice", "run", "no node", None).expect("start"); - assert_eq!(db.id_for_node(999), None, "unknown node → no row"); - } - #[test] fn append_after_finish_still_appends() { // Defensive: if a child's stdout pump fires one last line @@ -640,7 +557,7 @@ mod tests { // append should land on the row (status already set, but the // log stays consistent with what happened). let (_d, db) = tmpdb(); - let id = db.start("alice", "run", "cmd", None).expect("start"); + let id = db.start("alice", "run", "cmd").expect("start"); db.finish(id, BuildStatus::Ok); db.append_stdout(id, "post-finish trailing line"); let full = db.get_full(id).expect("get").expect("Some"); diff --git a/hive-c0re/src/workers/auto_update.rs b/hive-c0re/src/workers/auto_update.rs index 1e0a38e6..572b455b 100644 --- a/hive-c0re/src/workers/auto_update.rs +++ b/hive-c0re/src/workers/auto_update.rs @@ -391,7 +391,8 @@ fn submit_boot_tree( n_skipped, ); - let declare = move |b: &crate::job_queue::Job| boot_nodes(b, any_stale, fanout, drifted); + let declare: crate::job_queue::Declare = + Box::new(move |b| boot_nodes(b, any_stale, fanout, drifted)); let spec = DagSpec { // The sweep's own rebuild subgraphs emit their `Rebuilt` events as they diff --git a/hive-host-sock/src/jobs.rs b/hive-host-sock/src/jobs.rs index 2f567bfd..feba036f 100644 --- a/hive-host-sock/src/jobs.rs +++ b/hive-host-sock/src/jobs.rs @@ -206,106 +206,3 @@ impl DagView { } } } - -#[cfg(test)] -mod tests { - use chrono::Utc; - - use super::{DagView, NodeView, Source, State}; - - /// A node set carrying nothing but the states — the only input - /// `rollup_state` reads. - fn dag(states: &[State]) -> DagView { - DagView { - id: 1, - source: Source::Manual, - reason: "test".to_owned(), - created_at: Utc::now(), - started_at: None, - finished_at: None, - nodes: states - .iter() - .enumerate() - .map(|(i, &state)| NodeView { - id: i as u64, - agent: "a".to_owned(), - kind: "reconcile".to_owned(), - deps: Vec::new(), - state, - started_at: None, - finished_at: None, - error: None, - approval_id: None, - inputs: Vec::new(), - build_log_id: None, - parent: None, - }) - .collect(), - } - } - - #[test] - fn a_failure_outranks_everything_and_skipped_counts_for_nothing() { - // The case this replaces used to be arranged in hive-c0re by running a - // rebuild until its Prebuild failed. Only the states ever mattered. - assert_eq!( - dag(&[State::Done, State::Failed, State::Skipped]).rollup_state(), - State::Failed - ); - // A failure wins even against a node still going — the DAG's verdict - // is already decided. - assert_eq!( - dag(&[State::Running, State::Failed]).rollup_state(), - State::Failed - ); - // Skipped is an expected part of a healthy run: an outcome-branched - // DAG always leaves one branch untaken, so counting it would make - // every successful DAG roll up non-Done. - assert_eq!( - dag(&[State::Done, State::Skipped]).rollup_state(), - State::Done - ); - } - - #[test] - fn cancelled_outranks_running_and_pending() { - // A cancelled DAG still has its weak-edged tail node to run, so - // Pending-then-Running would flicker back at the operator who just - // cancelled it and read as "the cancel didn't take". - assert_eq!( - dag(&[State::Cancelled, State::Pending]).rollup_state(), - State::Cancelled - ); - assert_eq!( - dag(&[State::Cancelled, State::Running]).rollup_state(), - State::Cancelled - ); - } - - #[test] - fn finishing_still_counts_as_running() { - // The node's own work is done but its sub-nodes are still going, so - // the DAG is in flight. A parent parked in Finishing is the normal - // shape of a subtree mid-run, not an edge case. - assert_eq!( - dag(&[State::Finishing, State::Pending]).rollup_state(), - State::Running - ); - assert_eq!( - dag(&[State::Running, State::Pending]).rollup_state(), - State::Running - ); - assert_eq!( - dag(&[State::Done, State::Pending]).rollup_state(), - State::Pending - ); - } - - #[test] - fn an_empty_node_set_reads_done() { - // Every node Done means every node is filtered off the wire, so this - // is what a finished DAG actually looks like to a consumer that has - // one in hand at all. - assert_eq!(dag(&[]).rollup_state(), State::Done); - } -} diff --git a/hive-jobq/src/builder.rs b/hive-jobq/src/builder.rs index a2e207e9..fa1e9807 100644 --- a/hive-jobq/src/builder.rs +++ b/hive-jobq/src/builder.rs @@ -8,10 +8,8 @@ //! **An insertion API, not a spec factory.** A builder is only ever handed to a //! closure by the single insertion entry point //! ([`crate::scheduler::Scheduler::insert_job`]), which inserts the declared -//! nodes and returns the ids the job asked for. It cannot be constructed or -//! inserted from outside this crate — `new()` and `insert_with` are both -//! `pub(crate)`, and there is deliberately no `Default` impl, since a trait impl -//! on a `pub` type is public regardless. There is no intermediate +//! nodes and returns the ids the job asked for. It cannot be constructed, held +//! or inserted from outside this crate, and there is no intermediate //! node-description type to keep in sync with [`crate::Graph::insert`]'s signature — //! so a job has no representation that can be passed around instead of being //! inserted. @@ -244,6 +242,16 @@ pub struct JobBuilder { nodes: RefCell>>, } +// Hand-written rather than derived: `#[derive(Default)]` would demand +// `N: Default, R: Default`, which has nothing to do with an empty builder. +impl Default for JobBuilder { + fn default() -> Self { + Self { + nodes: RefCell::new(Vec::new()), + } + } +} + impl JobBuilder { /// A fresh, empty builder. /// @@ -253,17 +261,8 @@ impl JobBuilder { /// nodes and returns the ids. Nothing job-shaped is constructible or /// carryable outside this crate — otherwise it is a spec factory again, /// just with a builder's name on it. - /// - /// Deliberately **not** a `Default` impl. A trait impl on a `pub` type is - /// public no matter how private its inherent constructors are, so - /// `JobBuilder::default()` would hand every downstream crate the builder - /// this fn is `pub(crate)` to withhold. The body is what `#[derive(Default)]` - /// could not be anyway — deriving would demand `N: Default, R: Default`, - /// which has nothing to do with an empty builder. pub(crate) fn new() -> Self { - Self { - nodes: RefCell::new(Vec::new()), - } + Self::default() } /// Whether nothing has been declared yet — for a caller deciding whether an diff --git a/hive-jobq/src/scheduler.rs b/hive-jobq/src/scheduler.rs index 1ab6015d..337c6b2d 100644 --- a/hive-jobq/src/scheduler.rs +++ b/hive-jobq/src/scheduler.rs @@ -1,14 +1,13 @@ //! The settle loop — drives a [`Graph`] to completion over a resource pool the //! scheduler owns directly. //! -//! [`Scheduler::claim_next`] claims one currently-runnable pending node (its +//! [`Scheduler::settle`] claims every currently-runnable pending node (its //! [`Dep::Node`] edges satisfied *and* all its [`Dep::Resource`] units acquired -//! atomically), marks it `Running`, records the units it holds, and hands back -//! a future that executes the node **and completes it**, so "forgot to finish -//! the node" is not expressible. One at a time is the primitive on purpose: it -//! lets the caller choose between claiming again and backing off, which a batch -//! return can't express. A running node may grow more work by declaring into -//! the builder it was handed. Concurrency is emergent from resource capacity. +//! atomically), marks it `Running`, records the units it holds, and returns the +//! newly-started ids for the caller's runner to execute. The runner reports each +//! node's result back with [`Scheduler::complete`]; a running node may grow more +//! work first via [`Scheduler::append`]. Concurrency is emergent from resource +//! capacity — there is no separate active-node cap. //! //! Single-threaded by design: the scheduler is the only driver, holds the //! [`ResourceTable`] as a plain owned field, mutating it through `&mut self` — @@ -30,9 +29,7 @@ //! is a deferred optimization — unsafe under dynamically-appended subnodes.) use std::collections::HashMap; -use std::future::Future; use std::hash::Hash; -use std::sync::{Arc, Mutex}; use crate::builder::{BuildError, JobBuilder, NodeGuid}; use crate::resources::ResourceTable; @@ -89,8 +86,8 @@ impl Scheduler { } /// Append a node under `parent` — e.g. a running node growing more work into - /// its own subtree. Delegates to [`Graph::insert`]; claim again afterwards - /// to start it once it is runnable. + /// its own subtree. Delegates to [`Graph::insert`]; call [`Scheduler::settle`] + /// afterwards to start it once it is runnable. /// /// # Errors /// Propagates [`GraphError`] for a dangling dependency or parent id. @@ -117,7 +114,8 @@ impl Scheduler { /// [`Graph::insert_unchecked`]: [`crate::builder::check_job_shape`] has /// already decided every rejection the graph could raise, so re-validating /// per node could only report a problem *after* the earlier nodes were - /// inserted. Claim again afterwards to start whatever became runnable. + /// inserted. Call [`Scheduler::settle`] afterwards to start whatever became + /// runnable. /// /// **Atomic in the job's own shape.** A forward edge, a forward parent, or /// a request for a handle this job never declared is rejected *before* the @@ -140,80 +138,27 @@ impl Scheduler { }) } - /// Claim **one** currently-runnable pending node and start it: node-deps + /// Claim every currently-runnable pending node and start it: node-deps /// satisfied and all resource-deps acquired atomically (all-or-nothing). - /// The node is marked `Running`, its acquired units recorded, and its id - /// returned for the caller to execute. `None` means nothing is runnable - /// right now — which is a different statement from "nothing is pending". - /// - /// **Private**: [`Self::claim_next`] is the only way out of this crate. - /// Claiming without the future that completes the node is the sequence the - /// seam exists to make inexpressible, so the primitive stays in here. + /// Each claimed node is marked `Running`, its acquired units recorded, and + /// its id returned for the runner to execute. A single pass suffices — a + /// node started here is `Running`, not terminal, so it cannot satisfy another + /// node's dependency in the same pass; it only consumes resources. #[must_use] - fn claim_one(&mut self) -> Option { + pub fn settle(&mut self) -> Vec { let pending: Vec = self .graph .nodes() .filter(|n| n.state == State::Pending) .map(|n| n.id) .collect(); - pending - .into_iter() - .find(|&id| self.node_deps_satisfied(id) && self.try_start(id)) - } - - /// Claim one runnable node and return **the work that runs it**, or `None` - /// when nothing is runnable right now. - /// - /// This is the seam: the caller supplies how to execute a node and spawns - /// the returned future, but never touches claiming or completion. The - /// future runs the node **and completes it**, so "forgot to finish the - /// node" is not expressible — completion is inside the thing you spawn. - /// - /// The `Option` is answered *synchronously*, before anything is awaited, so - /// the caller can decide "claim again immediately" vs "back off" without - /// waiting on the node it just started. - /// - /// ## Locking - /// The lock is taken twice, briefly, and **never held across the await**: - /// once here to claim, once inside the future to complete. That is what - /// keeps the returned future `Send` — a guard alive across an await point - /// would poison it — and it is why the node itself runs unlocked, for - /// however many minutes it needs. - /// - /// ## Why the payload is cloned - /// `run` gets an owned `N` rather than a borrow: a `&N` parameter is live - /// for the whole future, which both borrows the graph across the await and - /// makes the future non-`Send`. - /// - /// The output carries the insert result rather than swallowing it — this - /// crate has no logger, so a malformed grown job is reported to the caller, - /// who is the one that can log it. The node is completed either way: its - /// own work already happened. - pub fn claim_next( - sched: &Arc>, - run: F, - ) -> Option)> + use> - where - N: Clone, - F: FnOnce(NodeId, N, JobBuilder) -> Fut, - Fut: Future, Outcome)>, - { - let (id, payload) = { - let mut guard = sched.lock().expect("jobq scheduler mutex poisoned"); - let id = guard.claim_one()?; - let payload = guard.graph.node(id)?.payload.clone(); - (id, payload) - }; - let sched = Arc::clone(sched); - Some(async move { - let (grown, outcome) = run(id, payload, JobBuilder::new()).await; - let grew = sched - .lock() - .expect("jobq scheduler mutex poisoned") - .complete_growing(id, outcome, grown); - (id, grew) - }) + let mut started = Vec::new(); + for id in pending { + if self.node_deps_satisfied(id) && self.try_start(id) { + started.push(id); + } + } + started } /// Try to start node `id`. For each resource it needs, decide per the parent @@ -303,13 +248,9 @@ impl Scheduler { /// (every child `Done`) or [`State::Failed`] (any child `Failed`/`Cancelled`). /// On failure it is `Failed` at once and its pending sub-nodes are cancelled /// (gated on a `Finishing` the parent never reached). Terminality then - /// propagates up the parent chain. Claim again afterwards to start - /// newly-unblocked work. - /// - /// `pub(crate)`: completion is reachable only from inside the future - /// [`Self::claim_next`] hands back, so it is not expressible without the - /// claim it answers. - pub(crate) fn complete(&mut self, id: NodeId, outcome: Outcome) { + /// propagates up the parent chain. Call [`Scheduler::settle`] again afterwards + /// to start newly-unblocked work. + pub fn complete(&mut self, id: NodeId, outcome: Outcome) { match outcome { Outcome::Failed(error) => { // Record the reason before the terminal transition so it's set @@ -324,64 +265,6 @@ impl Scheduler { self.release_ready(); } - /// [`Scheduler::complete`], plus whatever the node declared into the builder - /// it was handed while running. - /// - /// `grown`'s nodes are inserted **under `id`** and *before* the completion, - /// so the node cannot roll terminal with its own appended work still - /// pending — the same ordering the caller previously had to arrange by - /// hand. A job that declares nothing costs nothing: the insert is skipped - /// outright, which is the overwhelmingly common case (most nodes grow no - /// work at all). - /// - /// **A failed node grows nothing**, whatever it declared. Failure - /// cancel-cascades to every pending child of `id`, so work inserted here - /// would be `Skipped` by the very next statement — the insert is not wrong, - /// it is provably pointless. This lives here rather than in the caller - /// because it is a consequence of *this crate's* cascade rule; a host that - /// had to remember it could forget it. - /// - /// # Errors - /// [`BuildError`] if `grown` is malformed — **and the node is still - /// completed**. Its own work already happened; refusing to complete it - /// would misreport that, and leaving it `Running` forever would wedge the - /// DAG. So the error is returned for the caller to log, not used to abort - /// the completion. This crate has no logger of its own; the caller does. - pub(crate) fn complete_growing( - &mut self, - id: NodeId, - outcome: Outcome, - grown: JobBuilder, - ) -> Result<(), BuildError> { - // A node that is no longer in the graph grows nothing. The insert below - // is *unchecked* — rooting on a departed parent would plant a dangling - // `parent` edge rather than being rejected. The host used to carry this - // guard itself, as a lookup before a separate append call; it belongs - // here, where the graph is and where it cannot be skipped. - // - // ⚠️ Deliberately untested, and untestable today: nothing removes a node - // from the graph yet (eviction only stops *retaining* a DAG; its nodes - // linger), and `NodeId` cannot be fabricated, so a test would have to - // fake the very condition it checks. This guard is defensive against the - // bounded prune that does not exist yet — when that lands, it needs a - // test, and this comment is the reminder. - let grew = if grown.is_empty() - || matches!(outcome, Outcome::Failed(_)) - || self.graph.node(id).is_none() - { - Ok(()) - } else { - let graph = &mut self.graph; - grown - .insert_with(Some(id), &[], |payload, deps, parent| { - graph.insert_unchecked(payload, deps, parent) - }) - .map(|_ids| ()) - }; - self.complete(id, outcome); - grew - } - /// Whether every direct child of `id` is terminal. fn all_children_terminal(&self, id: NodeId) -> bool { self.graph @@ -608,7 +491,7 @@ impl Scheduler { /// done" signal) supplies parent→child ordering; `Dep::Node` edges (which the /// graph restricts to the same parent group) supply sibling ordering. /// `Dep::Resource` edges are handled by the atomic acquire in - /// [`Scheduler::try_start`], not here. + /// [`Scheduler::settle`], not here. fn node_deps_satisfied(&self, id: NodeId) -> bool { let Some(node) = self.graph.node(id) else { return false; @@ -650,25 +533,6 @@ mod tests { name.to_owned() } - /// Claim every currently-runnable node, as arrangement for the assertions - /// below. Equivalent to calling [`Scheduler::claim_one`] until it yields - /// `None`: a node started by an earlier iteration is `Running`, not - /// terminal, so it cannot satisfy another node's dependency here — it only - /// consumes resources. - /// - /// **Was `Scheduler::settle`, a public method.** It was a `claim_one` loop - /// returning a `Vec`, and production never wanted the batch: the run loop - /// takes one node at a time through [`Scheduler::claim_next`] so it can - /// choose between claiming again and backing off, which a batch return - /// can't express. The only callers were tests, so it lives with them. - fn settle(s: &mut Scheduler) -> Vec { - let mut started = Vec::new(); - while let Some(id) = s.claim_one() { - started.push(id); - } - started - } - /// A graph + a resource table with `build-slot` set to `slots`. fn scheduler_with_slots(slots: u32) -> Scheduler<&'static str, String> { let mut table = ResourceTable::new(); @@ -695,97 +559,13 @@ mod tests { s.resources.available(&res(name)) } - /// Children of `id`, by payload, in insertion order. - fn children_of(s: &Scheduler<&'static str, String>, id: NodeId) -> Vec<&'static str> { - s.graph() - .nodes() - .filter(|n| n.parent == Some(id)) - .map(|n| n.payload) - .collect() - } - - /// A node completing `Done` gets the work it declared while running, - /// inserted **under itself** — so the DAG cannot roll terminal with the - /// appended work still pending. - #[test] - fn a_completing_node_grows_the_work_it_declared() { - let mut s = scheduler_with_slots(1); - let n = s.append("emitter", vec![], None).expect("insert"); - assert_eq!(settle(&mut s), vec![n]); - - let grown = JobBuilder::new(); - grown.node("child-a"); - grown.node("child-b"); - s.complete_growing(n, Outcome::Done, grown) - .expect("well-formed growth"); - - assert_eq!(children_of(&s, n), vec!["child-a", "child-b"]); - // The emitter parks in `Finishing` rather than going terminal: its own - // appended work is still pending under it. That ordering is the whole - // point of growing *as part of* the completion. - assert_eq!(s.graph().node(n).unwrap().state, State::Finishing); - } - - /// A **failed** node grows nothing, whatever it declared. - /// - /// The companion to the test above, and the reason this rule lives in the - /// crate rather than in a caller: failure cancel-cascades to every pending - /// child of the completing node, so anything inserted here would be - /// `Skipped` by the very next statement. Enforcing it host-side means every - /// host has to remember it; enforcing it here means none can forget. - #[test] - fn a_failed_node_grows_nothing() { - let mut s = scheduler_with_slots(1); - let n = s.append("emitter", vec![], None).expect("insert"); - assert_eq!(settle(&mut s), vec![n]); - - let grown = JobBuilder::new(); - grown.node("never-runs"); - s.complete_growing(n, Outcome::Failed("boom".to_owned()), grown) - .expect("growth is dropped, not rejected"); - - assert!( - children_of(&s, n).is_empty(), - "a failed node must not append work, got {:?}", - children_of(&s, n) - ); - assert_eq!(s.graph().node(n).unwrap().state, State::Failed); - } - - /// A contended resource goes to the oldest waiter. - /// - /// [`Scheduler::claim_one`] scans [`Graph::nodes`] — insertion order — and - /// takes the first node whose deps are satisfied and whose resources it can - /// acquire. That *is* the fairness guarantee: there is no queue, no - /// priority, just the scan order. - /// - /// Load-bearing for any host that submits work over time, because without - /// it a steady arrival rate could starve the earliest waiter indefinitely. - /// It was previously only covered downstream, by a host test driving its own - /// templates — which meant the property this crate provides was asserted - /// everywhere except in this crate. - #[test] - fn a_contended_resource_goes_to_the_oldest_waiter() { - let mut s = scheduler_with_slots(1); - let a = s.append("a", res_dep("build-slot"), None).expect("a"); - let b = s.append("b", res_dep("build-slot"), None).expect("b"); - let c = s.append("c", res_dep("build-slot"), None).expect("c"); - - assert_eq!(settle(&mut s), vec![a], "cap 1: only the first can start"); - s.complete(a, Outcome::Done); - // b and c are both satisfiable now; b was inserted first. - assert_eq!(settle(&mut s), vec![b], "the freed unit goes to b, not c"); - s.complete(b, Outcome::Done); - assert_eq!(settle(&mut s), vec![c]); - } - #[test] fn leaf_owner_goes_done_directly_and_releases() { let mut s = scheduler_with_slots(1); let n = s .append("build", res_dep("build-slot"), None) .expect("insert"); - assert_eq!(settle(&mut s), vec![n]); + assert_eq!(s.settle(), vec![n]); assert_eq!(s.graph().node(n).unwrap().state, State::Running); assert_eq!(avail(&s, "build-slot"), 0); // No children → completing it goes straight to Done (skips Finishing). @@ -805,7 +585,7 @@ mod tests { assert!(s.graph().node(ok).unwrap().started_at.is_none()); assert!(s.graph().node(ok).unwrap().finished_at.is_none()); - let started = settle(&mut s); + let started = s.settle(); assert!(started.contains(&ok) && started.contains(&bad)); // Running → started_at stamped, finished_at still none. assert!(s.graph().node(ok).unwrap().started_at.is_some()); @@ -841,11 +621,11 @@ mod tests { let b = s.append("b", res_dep("build-slot"), None).expect("b"); let c = s.append("c", res_dep("build-slot"), None).expect("c"); // cap 2 → a + b start, c blocks on the exhausted slot. - assert_eq!(settle(&mut s), vec![a, b]); + assert_eq!(s.settle(), vec![a, b]); assert_eq!(s.graph().node(c).unwrap().state, State::Pending); // a finishes → its slot frees → c can now start. s.complete(a, Outcome::Done); - assert_eq!(settle(&mut s), vec![c]); + assert_eq!(s.settle(), vec![c]); assert_eq!(s.graph().node(c).unwrap().state, State::Running); } @@ -857,16 +637,16 @@ mod tests { let root = s.append("root", vec![], None).expect("root"); let c1 = s.append("c1", vec![], Some(root)).expect("c1"); let c2 = s.append("c2", vec![], Some(root)).expect("c2"); - assert_eq!(settle(&mut s), vec![root]); + assert_eq!(s.settle(), vec![root]); // Children can't start yet — parent still Running (logic not done). - assert!(settle(&mut s).is_empty(), "children gated on parent logic"); + assert!(s.settle().is_empty(), "children gated on parent logic"); s.complete(root, Outcome::Done); assert_eq!( s.graph().node(root).unwrap().state, State::Finishing, "logic done, children pending → Finishing" ); - let mut started = settle(&mut s); + let mut started = s.settle(); started.sort(); let mut expected = vec![c1, c2]; expected.sort(); @@ -890,9 +670,9 @@ mod tests { let mut s: Scheduler<&str, String> = Scheduler::new(Graph::new(), ResourceTable::new()); let root = s.append("root", vec![], None).expect("root"); let child = s.append("child", vec![], Some(root)).expect("child"); - assert_eq!(settle(&mut s), vec![root]); + assert_eq!(s.settle(), vec![root]); s.complete(root, Outcome::Done); - assert_eq!(settle(&mut s), vec![child]); + assert_eq!(s.settle(), vec![child]); s.complete(child, Outcome::Failed(String::new())); assert_eq!( s.graph().node(root).unwrap().state, @@ -910,10 +690,10 @@ mod tests { let r = s.append("R", res_dep("build-slot"), None).expect("R"); let c1 = s.append("c1", res_dep("build-slot"), Some(r)).expect("c1"); let c2 = s.append("c2", vec![after_ok(c1)], Some(r)).expect("c2"); - assert_eq!(settle(&mut s), vec![r]); + assert_eq!(s.settle(), vec![r]); s.complete(r, Outcome::Done); // → Finishing (children pending) assert_eq!(avail(&s, "build-slot"), 0, "held: subtree not terminal"); - assert_eq!(settle(&mut s), vec![c1], "c1 borrows R's slot"); + assert_eq!(s.settle(), vec![c1], "c1 borrows R's slot"); assert_eq!(avail(&s, "build-slot"), 0, "borrow reuses R's unit"); s.complete(c1, Outcome::Done); assert_eq!( @@ -921,7 +701,7 @@ mod tests { 0, "still held: c2 pending in subtree" ); - assert_eq!(settle(&mut s), vec![c2]); + assert_eq!(s.settle(), vec![c2]); s.complete(c2, Outcome::Done); assert_eq!( avail(&s, "build-slot"), @@ -939,14 +719,14 @@ mod tests { let owner = s .append("owner", res_dep("agent/foo"), None) .expect("owner"); - assert_eq!(settle(&mut s), vec![owner]); + assert_eq!(s.settle(), vec![owner]); assert_eq!(avail(&s, "agent/foo"), 0); let child = s .append("child", res_dep("agent/foo"), Some(owner)) .expect("child"); s.complete(owner, Outcome::Done); // → Finishing assert_eq!(avail(&s, "agent/foo"), 0, "held while a borrower pends"); - assert_eq!(settle(&mut s), vec![child]); + assert_eq!(s.settle(), vec![child]); assert_eq!(avail(&s, "agent/foo"), 0, "borrow reuses the one unit"); s.complete(child, Outcome::Done); assert_eq!(avail(&s, "agent/foo"), 1); @@ -969,13 +749,13 @@ mod tests { let great = s .append("great", res_dep("agent/foo"), Some(grand)) .expect("great"); - assert_eq!(settle(&mut s), vec![r]); + assert_eq!(s.settle(), vec![r]); s.complete(r, Outcome::Done); - assert_eq!(settle(&mut s), vec![child], "child borrows R's grant"); + assert_eq!(s.settle(), vec![child], "child borrows R's grant"); s.complete(child, Outcome::Done); - assert_eq!(settle(&mut s), vec![grand], "grand covered, no deadlock"); + assert_eq!(s.settle(), vec![grand], "grand covered, no deadlock"); s.complete(grand, Outcome::Done); - assert_eq!(settle(&mut s), vec![great], "great covered too"); + assert_eq!(s.settle(), vec![great], "great covered too"); assert_eq!(avail(&s, "agent/foo"), 0, "held across the whole nest"); s.complete(great, Outcome::Done); assert_eq!(s.graph().node(r).unwrap().state, State::Done, "R rolled up"); @@ -989,14 +769,10 @@ mod tests { let mut s: Scheduler<&str, String> = Scheduler::new(Graph::new(), ResourceTable::new()); let a = s.append("a", res_dep("agent/foo"), None).expect("a"); let b = s.append("b", res_dep("agent/foo"), None).expect("b"); - assert_eq!( - settle(&mut s), - vec![a], - "only a acquires; b can't borrow it" - ); + assert_eq!(s.settle(), vec![a], "only a acquires; b can't borrow it"); assert_eq!(s.graph().node(b).unwrap().state, State::Pending); s.complete(a, Outcome::Done); - assert_eq!(settle(&mut s), vec![b]); + assert_eq!(s.settle(), vec![b]); assert_eq!(s.graph().node(b).unwrap().state, State::Running); } @@ -1009,7 +785,7 @@ mod tests { let owner = s .append("owner", res_dep("agent/foo"), None) .expect("owner"); - assert_eq!(settle(&mut s), vec![owner]); + assert_eq!(s.settle(), vec![owner]); let c1 = s .append("c1", res_dep("agent/foo"), Some(owner)) .expect("c1"); @@ -1017,10 +793,10 @@ mod tests { .append("c2", res_dep("agent/foo"), Some(owner)) .expect("c2"); s.complete(owner, Outcome::Done); // → Finishing - assert_eq!(settle(&mut s), vec![c1], "c1 borrows; c2 can't (cap 1)"); + assert_eq!(s.settle(), vec![c1], "c1 borrows; c2 can't (cap 1)"); assert_eq!(s.graph().node(c2).unwrap().state, State::Pending); s.complete(c1, Outcome::Done); - assert_eq!(settle(&mut s), vec![c2], "borrow returned → c2 borrows"); + assert_eq!(s.settle(), vec![c2], "borrow returned → c2 borrows"); assert_eq!(avail(&s, "agent/foo"), 0, "still just the owner's unit"); } @@ -1032,7 +808,7 @@ mod tests { let owner = s .append("owner", res_dep("build-slot"), None) .expect("owner"); - assert_eq!(settle(&mut s), vec![owner]); + assert_eq!(s.settle(), vec![owner]); assert_eq!(avail(&s, "build-slot"), 1, "owner took one of two"); let c1 = s .append("c1", res_dep("build-slot"), Some(owner)) @@ -1041,7 +817,7 @@ mod tests { .append("c2", res_dep("build-slot"), Some(owner)) .expect("c2"); s.complete(owner, Outcome::Done); // → Finishing - let mut started = settle(&mut s); + let mut started = s.settle(); started.sort(); let mut expected = vec![c1, c2]; expected.sort(); @@ -1071,11 +847,11 @@ mod tests { None, ) .expect("weak"); - assert_eq!(settle(&mut s), vec![root]); + assert_eq!(s.settle(), vec![root]); s.complete(root, Outcome::Failed(String::new())); assert_eq!(s.graph().node(strong1).unwrap().state, State::Skipped); assert_eq!(s.graph().node(strong2).unwrap().state, State::Skipped); - assert_eq!(settle(&mut s), vec![weak]); + assert_eq!(s.settle(), vec![weak]); } /// The direction only a *set* edge can express: a branch that runs solely on @@ -1095,7 +871,7 @@ mod tests { None, ) .expect("compensate"); - assert_eq!(settle(&mut s), vec![root]); + assert_eq!(s.settle(), vec![root]); s.complete(root, Outcome::Done); assert_eq!( s.graph().node(on_fail).unwrap().state, @@ -1103,7 +879,7 @@ mod tests { "a Failed-only branch is unsatisfiable once its dep succeeds — and it is \ `Skipped`, not `Cancelled`, so the parent roll-up ignores it" ); - assert!(settle(&mut s).is_empty(), "and nothing is left runnable"); + assert!(s.settle().is_empty(), "and nothing is left runnable"); } /// The mirror: the same branch is exactly what *does* run on failure, while @@ -1125,10 +901,10 @@ mod tests { None, ) .expect("on_fail"); - assert_eq!(settle(&mut s), vec![root]); + assert_eq!(s.settle(), vec![root]); s.complete(root, Outcome::Failed("boom".to_owned())); assert_eq!(s.graph().node(on_ok).unwrap().state, State::Skipped); - assert_eq!(settle(&mut s), vec![on_fail]); + assert_eq!(s.settle(), vec![on_fail]); } /// A weak edge accepts a dependency that was *ruled out*, so a tail still @@ -1149,11 +925,11 @@ mod tests { None, ) .expect("tail"); - assert_eq!(settle(&mut s), vec![root]); + assert_eq!(s.settle(), vec![root]); s.complete(root, Outcome::Failed("boom".to_owned())); assert_eq!(s.graph().node(mid).unwrap().state, State::Skipped); assert_eq!( - settle(&mut s), + s.settle(), vec![tail], "the tail runs off a cancelled dependency" ); @@ -1190,26 +966,22 @@ mod tests { // Everything succeeds: the ok branch runs, the failure branch is ruled out. let (mut s, a, b, on_ok, on_fail) = build(); - assert_eq!( - settle(&mut s), - vec![a, b], - "both roots start; neither tail can" - ); + assert_eq!(s.settle(), vec![a, b], "both roots start; neither tail can"); s.complete(a, Outcome::Done); s.complete(b, Outcome::Done); - assert_eq!(settle(&mut s), vec![on_ok]); + assert_eq!(s.settle(), vec![on_ok]); s.complete(on_ok, Outcome::Done); assert_eq!(s.graph().node(on_fail).unwrap().state, State::Skipped); - assert!(settle(&mut s).is_empty()); + assert!(s.settle().is_empty()); // One of them fails: the ok branch is ruled out, which is precisely the // signal the failure branch waits on. let (mut s, a, b, on_ok, on_fail) = build(); - assert_eq!(settle(&mut s), vec![a, b]); + assert_eq!(s.settle(), vec![a, b]); s.complete(a, Outcome::Failed("boom".to_owned())); s.complete(b, Outcome::Done); assert_eq!(s.graph().node(on_ok).unwrap().state, State::Skipped); - assert_eq!(settle(&mut s), vec![on_fail]); + assert_eq!(s.settle(), vec![on_fail]); } #[test] @@ -1220,7 +992,7 @@ mod tests { let root = s.append("root", vec![], None).expect("root"); let child = s.append("child", vec![], Some(root)).expect("child"); let grandchild = s.append("gc", vec![], Some(child)).expect("gc"); - assert_eq!(settle(&mut s), vec![root]); + assert_eq!(s.settle(), vec![root]); s.complete(root, Outcome::Failed(String::new())); assert_eq!(s.graph().node(child).unwrap().state, State::Skipped); assert_eq!(s.graph().node(grandchild).unwrap().state, State::Skipped); @@ -1236,7 +1008,7 @@ mod tests { assert_eq!(s.graph().node(a).unwrap().state, State::Cancelled); assert_eq!(s.graph().node(b).unwrap().state, State::Skipped); let c = s.append("c", vec![], None).expect("c"); - assert_eq!(settle(&mut s), vec![c]); + assert_eq!(s.settle(), vec![c]); assert!(!s.cancel_node(c)); assert_eq!(s.graph().node(c).unwrap().state, State::Running); } @@ -1252,7 +1024,7 @@ mod tests { let b = s.append("b", vec![after_ok(a)], Some(root)).expect("b"); // The root runs first and parks in `Finishing` while its children are // outstanding — the state a group root is actually in when cancelled. - assert_eq!(settle(&mut s), vec![root]); + assert_eq!(s.settle(), vec![root]); s.complete(root, Outcome::Done); assert_eq!(s.graph().node(root).unwrap().state, State::Finishing); @@ -1273,9 +1045,9 @@ mod tests { let root = s.append("root", vec![], None).expect("root"); let a = s.append("a", vec![], Some(root)).expect("a"); let b = s.append("b", vec![after_ok(a)], Some(root)).expect("b"); - assert_eq!(settle(&mut s), vec![root]); + assert_eq!(s.settle(), vec![root]); s.complete(root, Outcome::Done); - assert_eq!(settle(&mut s), vec![a], "a is claimed and running"); + assert_eq!(s.settle(), vec![a], "a is claimed and running"); assert!(!s.cancel_node(root), "refused while a runs"); assert_eq!(s.graph().node(a).unwrap().state, State::Running); @@ -1304,7 +1076,7 @@ mod tests { Some(root), ) .expect("tail"); - assert_eq!(settle(&mut s), vec![root]); + assert_eq!(s.settle(), vec![root]); s.complete(root, Outcome::Done); assert!(s.cancel_node(root)); @@ -1314,7 +1086,7 @@ mod tests { State::Pending, "spared, and now runnable since its dep is Cancelled" ); - assert_eq!(settle(&mut s), vec![tail], "the tail still gets to report"); + assert_eq!(s.settle(), vec![tail], "the tail still gets to report"); } #[test] @@ -1322,7 +1094,7 @@ mod tests { let mut s = scheduler_with_slots(1); let g = s.append("g", res_dep("agent/foo"), None).expect("g"); let b = s.append("b", res_dep("build-slot"), None).expect("b"); - assert_eq!(settle(&mut s).len(), 2); + assert_eq!(s.settle().len(), 2); let state = s.resource_state(); assert!(state.contains(&(res("agent/foo"), g))); assert!(state.contains(&(res("build-slot"), b)));