diff --git a/hive-c0re/src/dashboard/build_logs.rs b/hive-c0re/src/dashboard/build_logs.rs index f4835878..763ee9fb 100644 --- a/hive-c0re/src/dashboard/build_logs.rs +++ b/hive-c0re/src/dashboard/build_logs.rs @@ -156,7 +156,7 @@ pub(super) async fn get_build_log_for_node( State(state): State, AxumPath(node_id): AxumPath, ) -> Response { - match state.coord.job_queue.build_log_id_of(node_id) { + match state.coord.build_logs.id_for_node(node_id) { Some(log_id) => get_build_log_full(State(state), AxumPath(log_id)).await, None => ( StatusCode::NOT_FOUND, @@ -183,7 +183,7 @@ pub(super) async fn get_build_log_raw_for_node( State(state): State, AxumPath(node_id): AxumPath, ) -> Response { - match state.coord.job_queue.build_log_id_of(node_id) { + match state.coord.build_logs.id_for_node(node_id) { Some(log_id) => get_build_log_raw(State(state), AxumPath(log_id)).await, None => ( StatusCode::NOT_FOUND, diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index 95486951..d1449bdd 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -10,11 +10,9 @@ use std::sync::Arc; use anyhow::{Context as _, Result}; -use super::{Claim, Declare}; -use hive_jobq::TerminalState; +use hive_jobq::{NodeId, TerminalState}; use super::model::NodeKind; -use super::resource::Resource; use crate::coordinator::Coordinator; use crate::power::{ReconcileAction, reconcile_action}; @@ -26,109 +24,104 @@ use crate::power::{ReconcileAction, reconcile_action}; /// N × this timeout. pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(3); -/// Extra signal an executor hands back to the scheduler alongside -/// success. -#[derive(Default)] -pub struct NodeOutput { - /// Whole per-agent *subgraphs* to append into *this same* DAG at - /// runtime — the single in-DAG-growth channel. Each [`Job`] is one - /// independent subgraph, declared but not yet inserted: an executor cannot - /// reach the queue, so it hands the declaration back and the scheduler - /// inserts it via [`super::JobQueue::append_subgraph`] under its own lock, - /// rooted on the emitting node. Used both for the multi-node case - /// (`MetaLock` growing one rebuild subgraph per agent — the startup - /// sweep's stale agents, the meta-update cascade's affected agents) and - /// the single-node case (a `Reconcile` planner emitting its mechanical - /// `Start` / `Stop` as a one-node subgraph). The scheduler applies these - /// *before* the emitting node's completion so the DAG never rolls terminal - /// with the appended work still pending — keeping the lease-window - /// transient held across the sub-step. - pub append_subgraph: Vec, -} - -impl std::fmt::Debug for NodeOutput { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // The subgraphs are closures — how many were emitted is the only thing - // there is to say about them before the queue runs them. - f.debug_struct("NodeOutput") - .field("append_subgraph", &self.append_subgraph.len()) - .finish() - } -} - -/// Build-log sink for one claimed node. -struct Ctx<'a> { - coord: &'a Arc, - dag_id: u64, - node_id: super::NodeId, -} - -impl Ctx<'_> { - fn build_log(&self, log_id: i64) { - if self - .coord - .job_queue - .set_build_log_id(self.dag_id, self.node_id, log_id) - { - self.coord.emit_rebuild_queue_snapshot(); - } - } -} - /// Run one claimed node to completion. Called from a task the /// scheduler spawns per claim; the `Result` (stringified) becomes the /// node's terminal state. -pub(super) async fn run_node(coord: &Arc, claim: &Claim) -> Result { - let ctx = Ctx { - coord, - dag_id: claim.dag_id, - node_id: claim.node_id, - }; - match &claim.kind { - NodeKind::MetaSync { relock, .. } => run_meta_sync(coord, claim, *relock).await, - NodeKind::Prebuild { .. } => run_prebuild(claim, &ctx).await, - NodeKind::Swap { .. } => run_swap(coord, claim, &ctx).await, - NodeKind::PostSwap { .. } => run_post_swap(coord, claim).await, - NodeKind::Provision { .. } => run_provision(coord, claim).await, - NodeKind::Create { .. } => run_create(claim).await, +/// +/// `job` is the node's own growth channel: an executor that decides more work +/// is needed declares it here, and the scheduler inserts it under this node +/// when the node completes. Most executors never touch it. Nothing is inserted +/// while the node runs — the builder is local state, so this stays outside the +/// queue's lock for the whole (often multi-minute) execution. +/// +/// ⚠️ Taken **by value and handed back**, not by reference. A `JobBuilder` is +/// `RefCell`-backed: owned it is `Send`, but `&JobBuilder` is not (a shared ref +/// is `Send` only if the referent is `Sync`, and `RefCell` never is). A `&Job` +/// parameter would be live across every `.await` in this fn and make the whole +/// future non-`Send`, which the scheduler's `tokio::spawn` rejects. So the +/// growth executors below return *what to grow* and the declaration happens +/// here, synchronously, between awaits. +/// +/// The node is identified by its own id + payload rather than by a `Claim` +/// side-struct: `kind` already carries the agent, and the DAG id is a +/// derived read (`JobQueue::dag_of`) the three arms that need it take +/// themselves. Nothing here needs a claim to exist as a type. +pub(super) async fn run_node( + coord: &Arc, + job: super::Job, + id: NodeId, + kind: &NodeKind, +) -> (super::Job, Result<()>) { + // The agent this node targets rides the payload — empty for the agentless + // container kinds (`MetaLock`, `Dag`), which never read it. + let agent = kind.agent(); + // Every arm is `Result<()>`; the three that grow work declare into `job` + // *synchronously*, after their own awaits have finished. Borrowing `&job` + // inside an `.await` would make this future non-`Send` (see above), so the + // growth executors return what to grow rather than taking the builder. + let result = match kind { + NodeKind::MetaSync { relock, .. } => run_meta_sync(coord, agent, *relock).await, + NodeKind::Prebuild { .. } => run_prebuild(agent, id).await, + NodeKind::Swap { .. } => run_swap(coord, agent, id).await, + NodeKind::PostSwap { .. } => run_post_swap(coord, agent).await, + NodeKind::Provision { .. } => run_provision(coord, agent).await, + NodeKind::Create { .. } => run_create(agent).await, NodeKind::MetaLock { sweep, fanout, inputs, - } => run_meta_lock(coord, *sweep, fanout.clone(), inputs).await, - NodeKind::Reconcile { .. } => run_reconcile(coord, claim).await, - NodeKind::Start { .. } => run_start(coord, claim).await, - NodeKind::Stop { .. } => run_stop(coord, claim).await, - NodeKind::StopForUpdate { .. } => run_stop_for_update(coord, claim).await, - NodeKind::Signal { .. } => Ok(run_signal(coord, claim)), - NodeKind::Drain { .. } => run_drain(coord, claim).await, - NodeKind::WriteDropin { .. } => run_write_dropin(coord, claim).await, - NodeKind::WritePermFile { .. } => run_write_perm_file(coord, claim).await, - NodeKind::Reparent { .. } => run_reparent(coord, claim).await, + } => run_meta_lock(coord, *sweep, fanout.clone(), inputs) + .await + .map(|(agents, opts)| super::templates::grown_rebuilds(&job, &agents, opts)), + NodeKind::Reconcile { .. } => run_reconcile(coord, agent).await.map(|sub| { + if let Some(kind) = sub { + super::templates::fanned_out_mechanical(&job, kind); + } + }), + NodeKind::Start { .. } => run_start(coord, agent).await, + NodeKind::Stop { .. } => run_stop(coord, agent).await, + NodeKind::StopForUpdate { .. } => run_stop_for_update(coord, agent).await, + NodeKind::Signal { .. } => { + run_signal(coord, agent); + Ok(()) + } + NodeKind::Drain { .. } => run_drain(coord, agent).await, + NodeKind::WriteDropin { .. } => run_write_dropin(coord, agent).await, + // The payload rides the node and is destructured here, so the executor + // takes it directly instead of re-matching the kind behind a `bail!` + // that could never fire. + NodeKind::WritePermFile { payload, .. } => run_write_perm_file(coord, agent, payload).await, + NodeKind::Reparent { moves } => run_reparent(coord, moves).await, NodeKind::MergeVerify { approval_id, .. } => run_merge_verify(coord, *approval_id).await, NodeKind::DeployApply { approval_id, .. } => { - run_deploy_apply(coord, claim, *approval_id).await + run_deploy_apply(coord, *approval_id).await.map(|()| { + super::templates::deploy_rebuild_nodes(&job, agent, *approval_id); + }) } NodeKind::FinalizeDeploy { approval_id, .. } => { run_finalize_deploy(coord, *approval_id).await } NodeKind::DeployTail { approval_id, .. } => { - run_deploy_tail(coord, claim, *approval_id).await + run_deploy_tail(coord, coord.job_queue.dag_of(id), agent, *approval_id).await } NodeKind::ResolveApproval { approval_id, outcome, - } => run_resolve_approval(coord, claim, *approval_id, *outcome).await, - NodeKind::EmitRebuilt { ok, .. } => Ok(run_emit_rebuilt(coord, claim, *ok)), - NodeKind::SetWanted { up, .. } => run_set_wanted(coord, claim, *up), + } => run_resolve_approval(coord, coord.job_queue.dag_of(id), *approval_id, *outcome).await, + NodeKind::EmitRebuilt { ok, .. } => { + run_emit_rebuilt(coord, agent, coord.job_queue.dag_of(id), *ok); + Ok(()) + } + NodeKind::SetWanted { up, .. } => run_set_wanted(coord, agent, *up), // The two nodes that carry no work of their own; completing either // lets it reach `Finishing` so the nodes under it start. // - `Dag`: pure grouping container. The DAG's terminal side effect, if // any, is its own tail node in the graph. // - `DeployWindow`: pure resource holder — the meta window, agent lease // and build slot it declares stay held until its subtree settles. - NodeKind::Dag { .. } | NodeKind::DeployWindow { .. } => Ok(NodeOutput::default()), - } + NodeKind::Dag { .. } | NodeKind::DeployWindow { .. } => Ok(()), + }; + (job, result) } /// Resolve the DAG's approval row the way this node's own `outcome` says. @@ -140,31 +133,30 @@ pub(super) async fn run_node(coord: &Arc, claim: &Claim) -> Result< /// since the work already happened and failing the tail would only misreport it. async fn run_resolve_approval( coord: &Arc, - claim: &Claim, + dag_id: Option, approval_id: i64, outcome: TerminalState, -) -> Result { +) -> Result<()> { let reason = (outcome == TerminalState::Failed) - .then(|| coord.job_queue.first_error(claim.dag_id)) + .then(|| dag_id.and_then(|dag| coord.job_queue.first_error(dag))) .flatten(); crate::actions::resolve_approval_dag(coord, approval_id, outcome, reason.as_deref()).await; - Ok(NodeOutput::default()) + Ok(()) } /// Emit this agent's `Rebuilt` manager event. `ok` is not computed — it is which /// of the tail pair the graph let run. The failure note comes from the DAG's /// first failing node, since the branch knows *that* it failed but not *why*. -fn run_emit_rebuilt(coord: &Arc, claim: &Claim, ok: bool) -> NodeOutput { +fn run_emit_rebuilt(coord: &Arc, agent: &str, dag_id: Option, ok: bool) { coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { - agent: claim.agent.clone(), + agent: agent.to_owned(), ok, note: (!ok) - .then(|| coord.job_queue.first_error(claim.dag_id)) + .then(|| dag_id.and_then(|dag| coord.job_queue.first_error(dag))) .flatten(), sha: None, tag: None, }); - NodeOutput::default() } /// Write the agent's durable power intent — the DAG-node form of the old @@ -175,7 +167,7 @@ fn run_emit_rebuilt(coord: &Arc, claim: &Claim, ok: bool) -> NodeOu /// warn-and-continue write, a failed write fails the node (cancel-downstream /// cancels the `Reconcile`) rather than letting it converge to a stale /// intent — that atomicity is the point of moving it into the DAG. -fn run_set_wanted(coord: &Arc, claim: &Claim, up: bool) -> Result { +fn run_set_wanted(coord: &Arc, agent: &str, up: bool) -> Result<()> { let wanted = if up { crate::power::Wanted::Up } else { @@ -183,9 +175,9 @@ fn run_set_wanted(coord: &Arc, claim: &Claim, up: bool) -> Result, claim: &Claim, up: bool) -> Result, - claim: &Claim, - relock: bool, -) -> Result { - let name = &claim.agent; +async fn run_meta_sync(coord: &Arc, name: &str, relock: bool) -> Result<()> { // Runs while the agent is still up — the runtime dir and MCP listener // already exist. Use the pure path accessor; no need to re-register the // listener (event-driven: registered at start/create). @@ -219,7 +206,7 @@ async fn run_meta_sync( if relock { crate::meta::lock_update_for_rebuild(name).await?; } - Ok(NodeOutput::default()) + Ok(()) } /// Out-of-band toplevel build while the container keeps serving: warm @@ -229,18 +216,16 @@ async fn run_meta_sync( /// container is already down: its only purpose is to shrink the swap's /// downtime window, so a stopped agent (no uptime to preserve) doesn't /// pay the double eval — `Swap` builds inline instead. -async fn run_prebuild(claim: &Claim, ctx: &Ctx<'_>) -> Result { - let name = &claim.agent; +async fn run_prebuild(name: &str, id: NodeId) -> Result<()> { // Warm the toplevel build only when the container is up — the whole // point of prebuild is to shrink the swap's downtime window. A // stopped agent has no uptime to preserve, so skip the (expensive) // eval and let the downstream `Swap` build inline. if crate::lifecycle::is_running(name).await { let flake_ref = format!("{}#{name}", crate::paths::meta_root().display()); - crate::lifecycle::prebuild_toplevel(name, &flake_ref, &|log_id| ctx.build_log(log_id)) - .await?; + crate::lifecycle::prebuild_toplevel(name, &flake_ref, Some(id.get())).await?; } - Ok(NodeOutput::default()) + Ok(()) } /// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb), @@ -248,17 +233,13 @@ async fn run_prebuild(claim: &Claim, ctx: &Ctx<'_>) -> Result { /// (rev marker, `Rebuilt` event, forge/matrix sync, kick, rescan). /// The recovery-start on failure is NOT here — the DAG's tail /// `Reconcile` runs after this node terminal ok *or* fail. -async fn run_swap(coord: &Arc, claim: &Claim, ctx: &Ctx<'_>) -> Result { - let name = &claim.agent; +async fn run_swap(coord: &Arc, name: &str, id: NodeId) -> Result<()> { // Swap runs on an already-existing (stopped) container — runtime dir // and listener were created earlier. Pure path accessor suffices. let agent_dir = crate::paths::agent_runtime_dir(name); let hive = coord.hive_env(); let paths = Coordinator::agent_paths(name, agent_dir); - let result = crate::lifecycle::swap_update(name, &hive, &paths, &|log_id| { - ctx.build_log(log_id); - }) - .await; + let result = crate::lifecycle::swap_update(name, &hive, &paths, Some(id.get())).await; // On success the Ok-only bookkeeping tail (rev marker, forge/matrix // sync, kick, rescan, snapshot) runs in the sibling `PostSwap` node, // which deps `AfterOk(Swap)`. On failure `PostSwap` is cancel-cascaded @@ -269,7 +250,7 @@ async fn run_swap(coord: &Arc, claim: &Claim, ctx: &Ctx<'_>) -> Res if result.is_err() { coord.rescan_containers_and_emit().await; } - result.map(|()| NodeOutput::default()) + result } /// The post-`Swap` bookkeeping tail, split into its own node for dashboard @@ -277,8 +258,7 @@ async fn run_swap(coord: &Arc, claim: &Claim, ctx: &Ctx<'_>) -> Res /// means the profile swap succeeded. Store/forge/matrix work only — no nix /// build (build-slot-exempt); the agent lease taken at `Swap` is still held /// (the whole chain up to `Reconcile` is one agent's subgraph). -async fn run_post_swap(coord: &Arc, claim: &Claim) -> Result { - let name = &claim.agent; +async fn run_post_swap(coord: &Arc, name: &str) -> Result<()> { if let Some(rev) = crate::auto_update::current_flake_rev(&coord.hyperhive_flake) && let Err(e) = std::fs::write(crate::paths::applied_rev_marker(name), rev) { @@ -298,20 +278,19 @@ async fn run_post_swap(coord: &Arc, claim: &Claim) -> Result, claim: &Claim) -> Result { - let name = &claim.agent; +async fn run_provision(coord: &Arc, name: &str) -> Result<()> { let agent_dir = crate::paths::agent_runtime_dir(name); let hive = coord.hive_env(); let paths = Coordinator::agent_paths(name, agent_dir); crate::lifecycle::provision_container(name, &hive, &paths).await?; - Ok(NodeOutput::default()) + Ok(()) } /// `nixos-container create` proper — the upstream `Provision` node @@ -320,21 +299,24 @@ async fn run_provision(coord: &Arc, claim: &Claim) -> Result Result { - crate::lifecycle::create_only(&claim.agent).await?; - Ok(NodeOutput::default()) +async fn run_create(name: &str) -> Result<()> { + crate::lifecycle::create_only(name).await?; + Ok(()) } /// Meta flake lock bump. Boot-sweep flavour is non-fatal (a failed /// bump must not cancel the fan-out rebuilds — they proceed against /// the current lock, exactly like today's sweep); the meta-update /// flavour propagates errors, and a failed bump fans out nothing. +/// Returns the agents whose rebuild subgraphs the caller should grow into this +/// node, and the options to build them with — rather than declaring them here. +/// The declaration has to happen outside any `.await` (see [`run_node`]). async fn run_meta_lock( coord: &Arc, sweep: bool, fanout: Option>, inputs: &[String], -) -> Result { +) -> Result<(Vec, super::templates::RebuildOpts)> { if sweep { if let Err(e) = crate::meta::lock_update_hyperhive().await { tracing::warn!(error = ?e, "startup sweep: meta lock_update_hyperhive failed"); @@ -349,25 +331,13 @@ async fn run_meta_lock( // drain window rather than being cut off. The per-agent drains overlap, // so the sweep's cost ceiling is one `GRACEFUL_STOP_TIMEOUT` in total, // not one per agent. - let append_subgraph = fanout - .unwrap_or_default() - .iter() - .map(|agent| { - let agent = agent.clone(); - Box::new(move |b: &super::Job| { - super::templates::rebuild_nodes( - b, - &agent, - super::templates::RebuildOpts { - relock: true, - graceful: true, - }, - None, - ); - }) as Declare - }) - .collect(); - return Ok(NodeOutput { append_subgraph }); + return Ok(( + fanout.unwrap_or_default(), + super::templates::RebuildOpts { + relock: true, + graceful: true, + }, + )); } let _progress = coord.meta_update_guard(); crate::meta::lock_update(inputs).await?; @@ -383,69 +353,46 @@ async fn run_meta_lock( // cascade children must NOT re-lock, which would revert the bump this // node just committed (the property the old `fanout_specs` meta-update // branch encoded). - let append_subgraph = cascade - .iter() - .map(|agent| { - let agent = agent.clone(); - Box::new(move |b: &super::Job| { - super::templates::rebuild_nodes( - b, - &agent, - super::templates::RebuildOpts { - relock: false, - graceful: false, - }, - None, - ); - }) as Declare - }) - .collect(); - Ok(NodeOutput { append_subgraph }) + Ok(( + cascade, + super::templates::RebuildOpts { + relock: false, + graceful: false, + }, + )) } /// Idempotent power-converge *planner*: compare `wanted` (durable /// intent) against observed state and, when they diverge, fan the /// mechanical `Start` / `Stop` out as a first-class node appended to -/// *this* DAG (a single-node `NodeOutput::append_subgraph` rooted on -/// this node). Does no container work itself — the sub-step becomes -/// visible in the DAG and the lease-window transient (or the sub-step's -/// own node-local guard) rides across it. -async fn run_reconcile(coord: &Arc, claim: &Claim) -> Result { - let name = &claim.agent; +/// *this* DAG (a single node declared into `job`, rooted on this node). +/// Does no container work itself — the sub-step becomes visible in the +/// DAG and the lease-window transient (or the sub-step's own node-local +/// guard) rides across it. +/// Returns the mechanical node to fan out (`None` on a noop) rather than +/// declaring it — the declaration has to happen outside any `.await`, see +/// [`run_node`]. `NodeKind` carries the agent it targets, so this node's agent +/// is stamped into the fanned-out kind here. +async fn run_reconcile(coord: &Arc, name: &str) -> Result> { let running = crate::lifecycle::is_running(name).await; let wanted = coord.power.get_or_seed(name, running)?; - // One node targeting this agent, rooted on this reconcile node. `NodeKind` - // carries the agent it targets, so stamp `claim.agent` into the fanned-out - // Start/Stop kind (one in-DAG-growth channel). - let sub = |kind: NodeKind| { - // `Start` / `Stop` declare the lease they run under. This node is their - // parent and holds it, so the declaration is a re-entrant borrow — no - // second unit, no deadlock. It exists so the requirement belongs to the - // node rather than to the fact that a `Reconcile` happens to fan it out. - let lease = Resource::Agent(kind.agent().to_owned()); - vec![Box::new(move |b: &super::Job| { - let _ = b.node(kind).needs(lease); - }) as Declare] - }; - let append_subgraph = match reconcile_action(wanted, running) { - ReconcileAction::Start => sub(NodeKind::Start { - agent: name.clone(), + Ok(match reconcile_action(wanted, running) { + ReconcileAction::Start => Some(NodeKind::Start { + agent: name.to_owned(), }), - ReconcileAction::Stop => sub(NodeKind::Stop { - agent: name.clone(), + ReconcileAction::Stop => Some(NodeKind::Stop { + agent: name.to_owned(), }), ReconcileAction::Noop => { tracing::debug!(%name, wanted = wanted.as_str(), running, "reconcile: noop"); - Vec::new() + None } - }; - Ok(NodeOutput { append_subgraph }) + }) } /// Mechanical container start — the sub-step a `Reconcile` planner fans /// out when it observes `wanted = Up` and the container down. -async fn run_start(coord: &Arc, claim: &Claim) -> Result { - let name = &claim.agent; +async fn run_start(coord: &Arc, name: &str) -> Result<()> { // No node-local transient guard: the pill is derived from the running node // set, and `Start` reports `Starting` via `NodeKind::transient_kind`. This // used to take one "only when the DAG holds none", which was a second @@ -468,28 +415,26 @@ async fn run_start(coord: &Arc, claim: &Claim) -> Result, claim: &Claim) -> Result { - let name = &claim.agent; +async fn run_stop(coord: &Arc, name: &str) -> Result<()> { // See `run_start`: no node-local guard — `Stop` reports `Stopping` from its // own kind now. crate::lifecycle::kill(name).await?; coord.unregister_agent(name); coord.notify_manager(&hive_sh4re::HelperEvent::Killed { - agent: name.clone(), + agent: name.to_owned(), }); coord.rescan_containers_and_emit().await; - Ok(NodeOutput::default()) + Ok(()) } /// Mechanical stop for the profile swap. Never *changes* `wanted`; /// noop when already stopped. -async fn run_stop_for_update(coord: &Arc, claim: &Claim) -> Result { - let name = &claim.agent; +async fn run_stop_for_update(coord: &Arc, name: &str) -> Result<()> { if crate::lifecycle::is_running(name).await { // Seed a missing agent_power row from the PRE-stop observation // — the DAG's tail `Reconcile` observes only the mechanically @@ -501,7 +446,7 @@ async fn run_stop_for_update(coord: &Arc, claim: &Claim) -> Result< crate::lifecycle::kill(name).await?; coord.rescan_containers_and_emit().await; } - Ok(NodeOutput::default()) + Ok(()) } /// Set the graceful fence + kick so the harness sees it promptly and @@ -513,20 +458,18 @@ async fn run_stop_for_update(coord: &Arc, claim: &Claim) -> Result< /// `GRACEFUL_STOP_TIMEOUT`. Safe because the harness tests the marker at /// the top of its loop — a paused agent has no turn in flight, so there /// is nothing to checkpoint. -fn run_signal(coord: &Arc, claim: &Claim) -> NodeOutput { - if hive_types::Ident::parse(&claim.agent).is_ok_and(|a| Coordinator::is_paused(&a)) { - return NodeOutput::default(); +fn run_signal(coord: &Arc, name: &str) { + if hive_types::Ident::parse(name).is_ok_and(|a| Coordinator::is_paused(&a)) { + return; } - coord.mark_graceful_stop(&claim.agent); - coord.kick_agent(&claim.agent, "graceful stop requested"); - NodeOutput::default() + coord.mark_graceful_stop(name); + coord.kick_agent(name, "graceful stop requested"); } /// Await the harness clearing the fence (`GracefulStopComplete`) or /// the timeout — either way the downstream `Reconcile` proceeds with /// the actual stop. -async fn run_drain(coord: &Arc, claim: &Claim) -> Result { - let name = &claim.agent; +async fn run_drain(coord: &Arc, name: &str) -> Result<()> { let deadline = std::time::Instant::now() + GRACEFUL_STOP_TIMEOUT; while coord.is_graceful_stop_pending(name) { if std::time::Instant::now() >= deadline { @@ -536,12 +479,11 @@ async fn run_drain(coord: &Arc, claim: &Claim) -> Result, claim: &Claim) -> Result { - let name = &claim.agent; +async fn run_write_dropin(coord: &Arc, name: &str) -> Result<()> { // write_dropins only needs the path value to build AgentPaths; the // dir doesn't need to exist at this point (created by ensure_agent_runtime_dir // on the upstream Prebuild/Start node). @@ -549,19 +491,18 @@ async fn run_write_dropin(coord: &Arc, claim: &Claim) -> Result, claim: &Claim) -> Result { +async fn run_write_perm_file( + coord: &Arc, + name: &str, + payload: &super::model::PermPayload, +) -> Result<()> { use super::model::PermPayload; - let name = &claim.agent; - // The perm file payload rides the node itself (the only consumer). - let NodeKind::WritePermFile { payload, .. } = &claim.kind else { - anyhow::bail!("run_write_perm_file on a non-WritePermFile node"); - }; // Runs under the deploy window (it declares `Resource::MetaWindow`): a // perm commit landing inside another node's staged prepare→finalize // window would sweep the staged deploy lock into its commit (the @@ -591,7 +532,7 @@ async fn run_write_perm_file(coord: &Arc, claim: &Claim) -> Result< } } } - Ok(NodeOutput::default()) + Ok(()) } /// Apply the node's `(child, new_parent)` moves as one `META_LOCK`-fused @@ -601,10 +542,10 @@ async fn run_write_perm_file(coord: &Arc, claim: &Claim) -> Result< /// the deploy window (it declares `Resource::MetaWindow`), same reasoning as /// `run_write_perm_file`: a topology commit landing inside another node's /// staged deploy window would sweep the staged lock into its commit. -async fn run_reparent(coord: &Arc, claim: &Claim) -> Result { - let NodeKind::Reparent { moves } = &claim.kind else { - anyhow::bail!("run_reparent on a non-Reparent node"); - }; +async fn run_reparent( + coord: &Arc, + moves: &[(hive_types::Ident, Option)], +) -> Result<()> { let refs: Vec<(&str, Option<&str>)> = moves .iter() .map(|(child, parent)| { @@ -618,16 +559,14 @@ async fn run_reparent(coord: &Arc, claim: &Claim) -> Result, approval_id: i64) -> Result { - crate::actions::run_deploy_merge_verify(coord, approval_id) - .await - .map(|()| NodeOutput::default()) +async fn run_merge_verify(coord: &Arc, approval_id: i64) -> Result<()> { + crate::actions::run_deploy_merge_verify(coord, approval_id).await } /// Deploy phase 2 — the irreversible half: ff-merge, then phase 1 of the @@ -639,27 +578,15 @@ async fn run_merge_verify(coord: &Arc, approval_id: i64) -> Result< /// their `MetaSync` declares is re-entered rather than deadlocked against the /// ancestor already holding it. On failure nothing is appended and the tail /// compensates, exactly as before. -async fn run_deploy_apply( - coord: &Arc, - claim: &Claim, - approval_id: i64, -) -> Result { - crate::actions::run_deploy_apply(coord, approval_id).await?; - Ok(NodeOutput { - append_subgraph: vec![super::templates::deploy_rebuild_nodes( - claim.kind.agent(), - approval_id, - )], - }) +async fn run_deploy_apply(coord: &Arc, approval_id: i64) -> Result<()> { + crate::actions::run_deploy_apply(coord, approval_id).await } /// Deploy phase 3 — close the staged-lock window once the appended rebuild has /// come up clean: drop the rollback ref, plant the `deployed/` tag, commit /// the staged lock. -async fn run_finalize_deploy(coord: &Arc, approval_id: i64) -> Result { - crate::actions::run_finalize_deploy(coord, approval_id) - .await - .map(|()| NodeOutput::default()) +async fn run_finalize_deploy(coord: &Arc, approval_id: i64) -> Result<()> { + crate::actions::run_finalize_deploy(coord, approval_id).await } /// Deploy compensation + bookkeeping tail. `AfterAny` the apply node, so it @@ -671,12 +598,12 @@ async fn run_finalize_deploy(coord: &Arc, approval_id: i64) -> Resu /// the approval row is gone (deny race, purge). async fn run_deploy_tail( coord: &Arc, - claim: &Claim, + dag_id: Option, + agent: &str, approval_id: i64, -) -> Result { - crate::actions::run_deploy_tail(coord, Some(claim.dag_id), claim.kind.agent(), approval_id) - .await; - Ok(NodeOutput::default()) +) -> Result<()> { + crate::actions::run_deploy_tail(coord, dag_id, agent, approval_id).await; + Ok(()) } /// Compute which agents a `nix flake update ` on the meta diff --git a/hive-c0re/src/job_queue/mod.rs b/hive-c0re/src/job_queue/mod.rs index 3db89559..597a564f 100644 --- a/hive-c0re/src/job_queue/mod.rs +++ b/hive-c0re/src/job_queue/mod.rs @@ -36,8 +36,7 @@ pub mod templates; #[cfg(test)] mod tests; -use std::collections::HashMap; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use chrono::{DateTime, Utc}; use hive_host_sock::jobs::NodeView; @@ -55,16 +54,6 @@ use resource::Resource; /// borrowed one; only `hive_jobq` can make or insert it. pub type Job = hive_jobq::JobBuilder; -/// A job's shape as a **recipe**: given a builder, declare the nodes. -/// -/// What a template returns and what an executor hands back, because neither -/// can build a job itself — `hive_jobq` creates the builder inside its own -/// insertion call and never lets one out. So the transferable thing is the -/// declaring closure, and the queue runs it at the moment it inserts. -/// -/// `Send` because an executor's output crosses the scheduler's task boundary. -pub type Declare = Box; - /// A handle to one node a template declared — where its edges, grouping and /// resources are declared. `Copy`; naming a node as a dependency does not /// consume the ability to name it again. @@ -99,27 +88,6 @@ pub struct RunningTransient { pub since: DateTime, } -/// A node claimed for execution — everything the executor needs, snapshotted at -/// claim time. -#[derive(Debug, Clone)] -pub struct Claim { - pub dag_id: u64, - pub node_id: NodeId, - pub kind: NodeKind, - /// The agent this node targets (its own, not a DAG-level field). Empty for - /// the agentless [`NodeKind::MetaLock`] + [`NodeKind::Dag`] container nodes. - pub agent: String, -} - -/// Per-node runtime metadata the crate graph doesn't carry. Lifecycle -/// (`started_at` / `finished_at` / `error`) lives on the `hive_jobq::Node` -/// itself now, so only the build-log row link remains host-side (the -/// client fetches the log by node id). -#[derive(Debug, Default, Clone)] -struct NodeRuntime { - build_log_id: Option, -} - /// An owned read-view of a DAG container's carried metadata ([`NodeKind::Dag`]). /// Derived on read from the container node — the data has a single home (the /// node payload); this is not a stored side-table. @@ -129,24 +97,29 @@ struct DagMeta { created_at: DateTime, } -/// The mutable queue state behind the mutex: the crate scheduler plus the -/// per-node runtime metadata the graph can't carry. A **DAG is a single -/// container node** ([`NodeKind::Dag`], `parent = None`) whose subtree is the -/// DAG's work — so the container's `NodeId` is the DAG id, its rolled-up state -/// is the DAG state, and there are no grouping side-tables: membership + meta -/// are graph queries ([`QueueInner::container`] / [`QueueInner::dag_meta`] + -/// the `hive_jobq::Graph` accessors). One shared crate [`Graph`] holds every DAG. -struct QueueInner { - sched: Scheduler, - /// Per-node runtime metadata (the build-log id) — mutable after - /// insert, so it can't ride the immutable node payload. - node_rt: HashMap, -} +/// The crate scheduler, specialised to this host's node + resource types. +/// +/// A **DAG is a single container node** ([`NodeKind::Dag`], `parent = None`) +/// whose subtree is the DAG's work — so the container's `NodeId` is the DAG id, +/// its rolled-up state is the DAG state, and there are no grouping side-tables: +/// membership + meta are graph queries ([`container`] / [`dag_meta`] + the +/// `hive_jobq::Graph` accessors). One shared crate [`Graph`] holds every DAG. +/// +/// There is deliberately **no wrapper struct and no per-node side map**. The +/// last map held the `build_logs` row id; that link now lives on the log row +/// itself (`build_logs.node_id`). With nothing else to guard, the mutex holds +/// the scheduler *directly* — which is what lets `hive_jobq` drive the run loop +/// (it takes `&Arc>>`, a type a host-side wrapper could not +/// satisfy). +type Sched = Scheduler; /// The queue. Lives on `Coordinator` (one per hive-c0re process); a single /// scheduler task ([`scheduler::run_worker`]) drives it. pub struct JobQueue { - inner: Mutex, + /// The scheduler, held directly rather than behind a host-side wrapper — + /// `hive_jobq`'s run-loop seam takes `&Arc>>`, so this + /// *is* the type the crate drives. + sched: Arc>, /// Wakes the scheduler when something new arrives or state changed. pub(crate) notify: Notify, } @@ -163,8 +136,19 @@ impl Default for JobQueue { } } -/// Insert a declared `job` into the shared graph and record its per-node -/// `node_rt`, returning the inserted ids. +/// A node runner's `Result` as the scheduler's [`Outcome`]. +/// +/// The failure reason + `finished_at` are stamped onto the graph `Node` by the +/// scheduler (the reason rides `Outcome::Failed`); there is no host-side copy, +/// so nothing needs clearing on success. +fn outcome_of(result: Result<(), String>) -> Outcome { + match result { + Ok(()) => Outcome::Done, + Err(e) => Outcome::Failed(truncate_error(&e)), + } +} + +/// Insert a declared `job` into the shared graph, returning the inserted ids. /// /// A node that declared no parent hangs under `group_parent` — the DAG /// container for a template, the emitting node for a runtime-appended @@ -178,12 +162,11 @@ impl Default for JobQueue { /// # Errors /// Propagates a crate graph-insert error (malformed dep/parent / dep-scope). fn insert_group( - inner: &mut QueueInner, + inner: &mut Sched, declare: impl FnOnce(&Job), group_parent: Option, ) -> anyhow::Result<()> { inner - .sched .insert_job(group_parent, |b| { declare(b); // c0re names no handles: a DAG is addressed by its container node, @@ -204,16 +187,13 @@ impl JobQueue { u32::try_from(build_slots.max(1)).unwrap_or(u32::MAX), ); Self { - inner: Mutex::new(QueueInner { - sched: Scheduler::new(Graph::new(), table), - node_rt: HashMap::new(), - }), + sched: Arc::new(Mutex::new(Scheduler::new(Graph::new(), table))), notify: Notify::new(), } } - fn lock(&self) -> std::sync::MutexGuard<'_, QueueInner> { - self.inner.lock().expect("job_queue mutex poisoned") + fn lock(&self) -> std::sync::MutexGuard<'_, Sched> { + self.sched.lock().expect("job_queue mutex poisoned") } /// Submit a DAG: insert a [`NodeKind::Dag`] **container node** carrying the @@ -221,7 +201,13 @@ impl JobQueue { /// roots re-parented to the container). Returns the container's id as the /// DAG id — its rolled-up state is the DAG state. /// - /// Takes the spec's recipe by generic, not as a boxed [`Declare`]: a spec + /// The container is an ordinary node: it declares no resources, so the + /// scheduler claims it on the next pass, runs its (empty) logic and parks + /// it in `Finishing`, at which point its children become runnable. Nothing + /// here completes it by hand — a node with no work of its own still goes + /// the way every other node goes. + /// + /// Takes the spec's recipe by generic, not as a boxed closure: a spec /// travels from the template that built it directly into this call, so /// there is nothing to allocate for. /// @@ -231,7 +217,6 @@ impl JobQueue { pub fn submit(&self, spec: DagSpec) -> anyhow::Result { let mut inner = self.lock(); let container = inner - .sched .append( NodeKind::Dag { source: spec.source, @@ -242,100 +227,29 @@ impl JobQueue { None, ) .map_err(|e| anyhow::anyhow!("job_queue: container insert failed: {e}"))?; - inner.node_rt.insert(container, NodeRuntime::default()); insert_group(&mut inner, spec.declare, Some(container))?; - // Settle the container's own (no-op) logic immediately so it parks in - // `Finishing` and its children become runnable — it never needs claiming - // or executing, and stays out of `claim_ready`. It rolls up terminal when - // its whole subtree settles (that's the DAG-done signal). - inner.sched.complete(container, Outcome::Done); drop(inner); self.notify.notify_one(); Ok(container.get()) } - /// Append a whole *subgraph* into a live DAG at runtime — the single - /// in-DAG-growth primitive. The subgraph is inserted as a [`insert_group`] - /// rooted under `dep_on` (the emitting node): the subgraph's own root becomes - /// a *child* of `dep_on`, its steps children of that root, and the group's - /// agent lease is hoisted onto that root. Ordering root→`dep_on` is the parent - /// gate — the children run once `dep_on` reaches `Finishing`. Because the - /// emitting node stays `Finishing` until this appended subtree is terminal and - /// the DAG's terminal node deps on the top root, roll-up keeps the DAG from - /// settling early with no explicit wiring. A no-op if the DAG is gone. - pub fn append_subgraph(&self, dag_id: u64, declare: Declare, dep_on: NodeId) { - let mut inner = self.lock(); - if inner.container(dag_id).is_none() { - return; - } - // Insert the subgraph as a group rooted under the emitting node: the - // subgraph's own root becomes a child of `dep_on`, its steps children of - // that root. No terminal-node wiring — roll-up carries terminality: the - // emitter stays `Finishing` until this appended subtree settles, and the - // container node rolls up terminal only once its whole subtree (incl. this - // appended work) has settled, so the DAG hook waits for free. - if let Err(e) = insert_group(&mut inner, declare, Some(dep_on)) { - tracing::error!( - dag = dag_id, - error = %e, - "job_queue: append_subgraph insert failed" - ); - return; - } - drop(inner); - self.notify.notify_one(); - } - - /// Claim every currently-runnable node, acquiring its resources, and mark it - /// `Running`. Delegates readiness + resource acquisition to the crate's - /// settle loop; builds a [`Claim`] per started node from its payload + its - /// DAG container's metadata. The container node itself is claimed like any - /// other (its executor is an instant no-op that lets its subtree start). - pub fn claim_ready(&self) -> Vec { - let mut inner = self.lock(); - let inner = &mut *inner; - let started = inner.sched.settle(); - let mut claims = Vec::with_capacity(started.len()); - for id in started { - let Some(node) = inner.sched.graph().node(id) else { - continue; - }; - let kind = node.payload.clone(); - let agent = node.payload.agent().to_owned(); - let Some(container) = inner.sched.graph().root_of(id) else { - continue; - }; - claims.push(Claim { - dag_id: container.get(), - node_id: id, - kind, - agent, - }); - // `started_at` is stamped on the graph `Node` by the scheduler's - // transition to `Running` — no host-side copy needed. - } - claims - } - - /// Mark a claimed node terminal, recording its outcome + (truncated) error. - /// The crate releases the node's build slot immediately and cascades the - /// `AfterOk` failure cancellation + subtree lease release. + /// The scheduler itself, for `hive_jobq`'s run-loop seam + /// (`Scheduler::claim_next`), which takes exactly this type. /// - /// Nothing is returned: a DAG's terminal side effects are its own tail nodes - /// ([`NodeKind::ResolveApproval`] / [`NodeKind::EmitRebuilt`]), which the - /// scheduler claims and runs like any other node. - pub fn complete_node(&self, node_id: NodeId, result: Result<(), String>) { - let mut inner = self.lock(); - // The failure reason + `finished_at` are stamped onto the graph `Node` - // by the scheduler (the reason rides `Outcome::Failed`); no host-side - // copy, so there is nothing to clear here. - let outcome = match result { - Ok(()) => Outcome::Done, - Err(e) => Outcome::Failed(truncate_error(&e)), - }; - inner.sched.complete(node_id, outcome); - drop(inner); - self.notify.notify_one(); + /// Handing out the `Arc` rather than wrapping each crate call keeps the + /// host from growing a parallel API: the run loop uses `hive_jobq`'s + /// functions directly, and this module stays the thin glue it is being + /// reduced to. + pub(crate) fn sched(&self) -> &Arc> { + &self.sched + } + + /// The DAG container id owning `node`, for log lines and the dashboard. + /// Derived from the graph rather than carried alongside the node — the + /// parent axis already knows it. + #[must_use] + pub fn dag_of(&self, node: NodeId) -> Option { + self.lock().graph().root_of(node).map(NodeId::get) } /// Cancel a DAG that hasn't started yet: every work node is still `Pending`, @@ -360,10 +274,10 @@ impl JobQueue { /// just that branch. Nothing here knows about DAGs. pub fn cancel(&self, id: u64) -> bool { let mut inner = self.lock(); - let Some(node) = inner.sched.graph().resolve_id(id) else { + let Some(node) = inner.graph().resolve_id(id) else { return false; }; - if !inner.sched.cancel_node(node) { + if !inner.cancel_node(node) { return false; } drop(inner); @@ -371,33 +285,6 @@ impl JobQueue { true } - /// Link a `build_logs` row to a specific `Running` node. - pub fn set_build_log_id(&self, dag_id: u64, node_id: NodeId, log_id: i64) -> bool { - let mut inner = self.lock(); - if inner.sched.graph().root_of(node_id).map(NodeId::get) != Some(dag_id) - || !inner.node_running(node_id) - { - return false; - } - inner.node_rt.entry(node_id).or_default().build_log_id = Some(log_id); - true - } - - /// The `build_logs` row id linked to the wire node id `node_id`, if any — - /// the lookup behind the `GET /api/build-log/` query endpoint (the - /// client fetches a node's captured build output on demand rather than - /// receiving it inline). Takes the raw wire `u64` (the endpoint's path - /// param); `node_rt` is keyed by the opaque `NodeId`, so this scans for the - /// matching id — the map is small (live + recently-terminal nodes). - #[must_use] - pub fn build_log_id_of(&self, node_id: u64) -> Option { - self.lock() - .node_rt - .iter() - .find(|(nid, _)| nid.get() == node_id) - .and_then(|(_, rt)| rt.build_log_id) - } - /// The first failed node's error in `dag_id`, if any has failed yet. /// /// Unlike the roll-up summary this is readable *mid-flight*, which is the @@ -410,12 +297,8 @@ impl JobQueue { #[must_use] pub fn first_error(&self, dag_id: u64) -> Option { let inner = self.lock(); - let container = inner.container(dag_id)?; - inner - .sched - .graph() - .first_error(container) - .map(ToOwned::to_owned) + let container = container(&inner, dag_id)?; + inner.graph().first_error(container).map(ToOwned::to_owned) } /// `(agent, label, takes_container_down)` for the live transient-pill set, @@ -447,7 +330,6 @@ impl JobQueue { pub fn running_transients(&self) -> Vec { let inner = self.lock(); inner - .sched .graph() .nodes() .filter(|n| matches!(n.state, State::Running)) @@ -476,204 +358,227 @@ impl JobQueue { #[must_use] pub fn snapshot(&self) -> Vec { let inner = self.lock(); - let mut ids = inner.visible_dags(); + let mut ids = visible_dags(&inner); ids.sort_unstable_by_key(|c| c.get()); - ids.into_iter().filter_map(|c| inner.dag_view(c)).collect() - } - - /// Number of live (non-terminal) DAGs — tests + diagnostics. - #[cfg(test)] - #[must_use] - pub fn live_count(&self) -> usize { - let inner = self.lock(); - inner - .containers() - .into_iter() - .filter(|&c| inner.sched.graph().is_settled(c) == Some(false)) - .count() + ids.into_iter() + .filter_map(|c| dag_view(&inner, c)) + .collect() } } -impl QueueInner { - /// Whether `id` is a `Running` node. - fn node_running(&self, id: NodeId) -> bool { - self.sched - .graph() - .node(id) - .is_some_and(|n| n.state == State::Running) - } - - /// The container node of `dag_id` — the `NodeKind::Dag` root whose id equals - /// `dag_id`. `NodeId` is un-fabricable from a raw `u64`, so this is a search. - fn container(&self, dag_id: u64) -> Option { - self.sched.graph().nodes().find_map(|n| { - (n.parent.is_none() - && n.id.get() == dag_id - && matches!(n.payload, NodeKind::Dag { .. })) +/// The container node of `dag_id` — the `NodeKind::Dag` root whose id equals +/// `dag_id`. `NodeId` is un-fabricable from a raw `u64`, so this is a search. +fn container(sched: &Sched, dag_id: u64) -> Option { + sched.graph().nodes().find_map(|n| { + (n.parent.is_none() && n.id.get() == dag_id && matches!(n.payload, NodeKind::Dag { .. })) .then_some(n.id) - }) - } + }) +} - /// The container's carried domain metadata as an owned read-view. The data - /// lives solely in the [`NodeKind::Dag`] payload — this is a derived read, - /// not a stored side-table. - fn dag_meta(&self, container: NodeId) -> Option { - let NodeKind::Dag { - source, - reason, - created_at, - } = &self.sched.graph().node(container)?.payload - else { - return None; +/// The container's carried domain metadata as an owned read-view. The data +/// lives solely in the [`NodeKind::Dag`] payload — this is a derived read, +/// not a stored side-table. +fn dag_meta(sched: &Sched, container: NodeId) -> Option { + let NodeKind::Dag { + source, + reason, + created_at, + } = &sched.graph().node(container)?.payload + else { + return None; + }; + Some(DagMeta { + source: *source, + reason: reason.clone(), + created_at: *created_at, + }) +} + +/// Project a DAG into its wire [`DagView`]: a near-raw view of the +/// container's work nodes, with `Done` nodes excluded. Lifecycle +/// (`state` / `started_at` / `finished_at` / `error`) is read straight +/// off each `hive_jobq::Node`; the client derives the DAG label, roll-up +/// state, and DAG timestamps from the node set. Non-derivable per-node +/// payload (`approval_id`, meta `inputs`) rides the owning node. Returns +/// `None` when every work node is `Done` or `Skipped` — a fully-settled +/// DAG drops out of the snapshot entirely (a `Failed` one lingers until +/// aged out). +fn dag_view(sched: &Sched, container: NodeId) -> Option { + let meta = dag_meta(sched, container)?; + let all: Vec<_> = sched.graph().descendants(container).collect(); + // DAG-level timestamps are taken over *all* subtree nodes (including the + // `Done` ones excluded from the wire) — the client can't derive them + // from a `Done`-filtered node set, so the host computes them here. + let mut started: Vec> = Vec::new(); + let mut finished: Vec> = Vec::new(); + for node in &all { + if let Some(s) = node.started_at { + started.push(s); + } + if let Some(f) = node.finished_at { + finished.push(f); + } + } + // Decide which nodes ride the wire *before* projecting any of them: a + // `NodeView` costs a `build_logs` lookup, so building one for a node + // that's about to be dropped would be a query per finished step. + let shown = shown_on_wire(&all.iter().map(|n| n.state).collect::>())?; + let mut nodes = Vec::new(); + for node in shown.into_iter().map(|i| all[i]) { + let id = node.id; + let deps: Vec = node + .deps + .iter() + .filter_map(|d| match d { + Dep::Node { id, .. } => Some(id.get()), + Dep::Resource { .. } => None, + }) + .collect(); + // Non-derivable per-node payload rides the node that owns it. Every + // deploy phase carries the approval id, but only the subtree root + // projects it onto the wire — hanging the approval link off all of + // them would render the same card once per phase. + let approval_id = match &node.payload { + NodeKind::DeployWindow { approval_id, .. } => Some(*approval_id), + _ => None, }; - Some(DagMeta { - source: *source, - reason: reason.clone(), - created_at: *created_at, - }) + let inputs = match &node.payload { + NodeKind::MetaLock { inputs, .. } => inputs.clone(), + _ => Vec::new(), + }; + // Looked up from the log row itself (`build_logs.node_id`), not a + // host-side map. One indexed query per node in the snapshot; the + // node set is bounded by `MAX_HISTORY_DAGS` and the store is a + // local sqlite file, so this is cheaper than the lock contention + // a second shared map would reintroduce. + let build_log_id = crate::build_logs::global().and_then(|h| h.id_for_node(id.get())); + // `node.parent` is the structural jobq parent. Top-level nodes + // have `parent == Some(container)` (direct children of the Dag + // container); those become `parent: None` on the wire since the + // container itself is not part of the work-node payload. Sub-nodes + // carry the id of their containing parent work-node. + let parent = node + .parent + .filter(|&p| p != container) + .map(hive_jobq::NodeId::get); + nodes.push(NodeView { + id: id.get(), + agent: node.payload.agent().to_owned(), + kind: node.payload.as_str().to_owned(), + deps, + state: node.state, + started_at: node.started_at, + finished_at: node.finished_at, + error: node.error.clone(), + approval_id, + inputs, + build_log_id, + parent, + }); } + let is_terminal = sched.graph().is_settled(container) == Some(true); + Some(DagView { + id: container.get(), + source: meta.source, + reason: meta.reason.clone(), + created_at: meta.created_at, + started_at: started.into_iter().min(), + finished_at: is_terminal.then(|| finished.into_iter().max()).flatten(), + nodes, + }) +} - /// Project a DAG into its wire [`DagView`]: a near-raw view of the - /// container's work nodes, with `Done` nodes excluded. Lifecycle - /// (`state` / `started_at` / `finished_at` / `error`) is read straight - /// off each `hive_jobq::Node`; the client derives the DAG label, roll-up - /// state, and DAG timestamps from the node set. Non-derivable per-node - /// payload (`approval_id`, meta `inputs`) rides the owning node. Returns - /// `None` when every work node is `Done` or `Skipped` — a fully-settled - /// DAG drops out of the snapshot entirely (a `Failed` one lingers until - /// aged out). - fn dag_view(&self, container: NodeId) -> Option { - let meta = self.dag_meta(container)?; - let mut nodes = Vec::new(); - // Whether anything in this DAG still has an outcome worth showing. - // Kept separate from `nodes` being non-empty: skipped nodes ride the - // wire so the dashboard can mark the branches that weren't taken, but - // they must not by themselves hold a finished DAG in the snapshot. - let mut any_unsettled = false; - // DAG-level timestamps are taken over *all* subtree nodes (including the - // `Done` ones excluded from the wire) — the client can't derive them - // from a `Done`-filtered node set, so the host computes them here. - let mut started: Vec> = Vec::new(); - let mut finished: Vec> = Vec::new(); - for node in self.sched.graph().descendants(container) { - let id = node.id; - if let Some(s) = node.started_at { - started.push(s); - } - if let Some(f) = node.finished_at { - finished.push(f); - } - // `Done` nodes drop off the wire — a finished step isn't - // interesting. `Skipped` ones stay: which branch a run *didn't* - // take is the readable half of an outcome-branched DAG. - if matches!(node.state, State::Done) { - continue; - } - any_unsettled |= !matches!(node.state, State::Skipped); - let deps: Vec = node - .deps - .iter() - .filter_map(|d| match d { - Dep::Node { id, .. } => Some(id.get()), - Dep::Resource { .. } => None, - }) - .collect(); - // Non-derivable per-node payload rides the node that owns it. Every - // deploy phase carries the approval id, but only the subtree root - // projects it onto the wire — hanging the approval link off all of - // them would render the same card once per phase. - let approval_id = match &node.payload { - NodeKind::DeployWindow { approval_id, .. } => Some(*approval_id), - _ => None, - }; - let inputs = match &node.payload { - NodeKind::MetaLock { inputs, .. } => inputs.clone(), - _ => Vec::new(), - }; - let build_log_id = self.node_rt.get(&id).and_then(|r| r.build_log_id); - // `node.parent` is the structural jobq parent. Top-level nodes - // have `parent == Some(container)` (direct children of the Dag - // container); those become `parent: None` on the wire since the - // container itself is not part of the work-node payload. Sub-nodes - // carry the id of their containing parent work-node. - let parent = node - .parent - .filter(|&p| p != container) - .map(hive_jobq::NodeId::get); - nodes.push(NodeView { - id: id.get(), - agent: node.payload.agent().to_owned(), - kind: node.payload.as_str().to_owned(), - deps, - state: node.state, - started_at: node.started_at, - finished_at: node.finished_at, - error: node.error.clone(), - approval_id, - inputs, - build_log_id, - parent, - }); +/// Which of a DAG's work nodes ride the wire, by index into `states` — or +/// `None` when the DAG has nothing left worth showing and drops out of the +/// snapshot entirely. +/// +/// Two separate decisions, and conflating them pins every completed deploy in +/// the queue view forever: +/// - **`Done` drops off the wire.** A finished step isn't interesting. +/// `Skipped` stays: which branch a run *didn't* take is the readable half of +/// an outcome-branched DAG. +/// - **`Skipped` alone doesn't hold a DAG in the snapshot.** So "the node list +/// is non-empty" and "there's still something here worth showing" are +/// different questions, and only the second one may drop the DAG. +/// +/// Takes states rather than projected nodes so the caller can skip the work of +/// projecting what it's about to discard, and so this is testable without a +/// graph — the states it keys on are ones only a run can produce. +fn shown_on_wire(states: &[State]) -> Option> { + let worth_showing = states + .iter() + .any(|s| !matches!(s, State::Done | State::Skipped)); + if !worth_showing { + return None; + } + Some( + states + .iter() + .enumerate() + .filter(|(_, s)| !matches!(s, State::Done)) + .map(|(i, _)| i) + .collect(), + ) +} + +/// When a DAG's work node finishes on `finished_at` — the max over its +/// subtree (read off the graph `Node`, as unix seconds), for the history +/// cap ordering. +fn dag_finished_at(sched: &Sched, container: NodeId) -> i64 { + sched + .graph() + .descendants(container) + .filter_map(|n| n.finished_at) + .map(|t| t.timestamp()) + .max() + .unwrap_or(0) +} + +/// Every DAG container node id in the graph. +fn containers(sched: &Sched) -> Vec { + sched + .graph() + .nodes() + .filter(|n| n.parent.is_none() && matches!(n.payload, NodeKind::Dag { .. })) + .map(|n| n.id) + .collect() +} + +/// The **visible** DAG set for the snapshot: every live (non-terminal) DAG, +/// plus the newest [`MAX_HISTORY_DAGS`] terminal ones. Crate nodes for +/// evicted DAGs linger in the graph (bounded-prune is a Stage-C follow-up); +/// this filter is what bounds what the dashboard sees. +fn visible_dags(sched: &Sched) -> Vec { + let mut live: Vec = Vec::new(); + let mut terminal: Vec<(NodeId, i64, u64)> = Vec::new(); + for c in containers(sched) { + if sched.graph().is_settled(c) == Some(true) { + terminal.push((c, dag_finished_at(sched, c), c.get())); + } else { + live.push(c); } - if !any_unsettled { - return None; - } - let is_terminal = self.sched.graph().is_settled(container) == Some(true); - Some(DagView { - id: container.get(), - source: meta.source, - reason: meta.reason.clone(), - created_at: meta.created_at, - started_at: started.into_iter().min(), - finished_at: is_terminal.then(|| finished.into_iter().max()).flatten(), - nodes, - }) } + retain_history(live, terminal, MAX_HISTORY_DAGS) +} - /// When a DAG's work node finishes on `finished_at` — the max over its - /// subtree (read off the graph `Node`, as unix seconds), for the history - /// cap ordering. - fn dag_finished_at(&self, container: NodeId) -> i64 { - self.sched - .graph() - .descendants(container) - .filter_map(|n| n.finished_at) - .map(|t| t.timestamp()) - .max() - .unwrap_or(0) - } - - /// Every DAG container node id in the graph. - fn containers(&self) -> Vec { - self.sched - .graph() - .nodes() - .filter(|n| n.parent.is_none() && matches!(n.payload, NodeKind::Dag { .. })) - .map(|n| n.id) - .collect() - } - - /// The **visible** DAG set for the snapshot: every live (non-terminal) DAG, - /// plus the newest [`MAX_HISTORY_DAGS`] terminal ones. Crate nodes for - /// evicted DAGs linger in the graph (bounded-prune is a Stage-C follow-up); - /// this filter is what bounds what the dashboard sees. - fn visible_dags(&self) -> Vec { - let mut live: Vec = Vec::new(); - let mut terminal: Vec<(NodeId, i64)> = Vec::new(); - for c in self.containers() { - if self.sched.graph().is_settled(c) == Some(true) { - terminal.push((c, self.dag_finished_at(c))); - } else { - live.push(c); - } - } - // Newest first, so truncating to the cap keeps the most recent. - terminal.sort_by(|a, b| b.1.cmp(&a.1).then(b.0.get().cmp(&a.0.get()))); - terminal.truncate(MAX_HISTORY_DAGS); - let mut kept = live; - kept.extend(terminal.into_iter().map(|(c, _)| c)); - kept - } +/// [`visible_dags`]'s policy, split from the graph it reads: keep every live +/// DAG, plus the newest `cap` terminal ones. +/// +/// `terminal` rows are `(handle, finished_at, tiebreak)`. The tiebreak orders +/// DAGs that settled inside the same wall-clock second — which is *most* of +/// them under a burst, and all of them in a test, so it is load-bearing rather +/// than a formality. +/// +/// Generic over the handle purely so this is reachable without a graph: a +/// `NodeId` cannot be fabricated, so a test that had to pass real ones could +/// only get them by submitting and running DAGs. +fn retain_history(live: Vec, mut terminal: Vec<(T, i64, u64)>, cap: usize) -> Vec { + // Newest first, so truncating to the cap keeps the most recent. + terminal.sort_by(|a, b| b.1.cmp(&a.1).then(b.2.cmp(&a.2))); + terminal.truncate(cap); + let mut kept = live; + kept.extend(terminal.into_iter().map(|(handle, _, _)| handle)); + kept } /// Truncate a node error to [`MAX_ERROR_LEN`] on a char boundary, appending `…`. diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs index b6a9fe8d..b95f89e8 100644 --- a/hive-c0re/src/job_queue/model.rs +++ b/hive-c0re/src/job_queue/model.rs @@ -407,9 +407,9 @@ impl NodeKind { /// Generic over the recipe rather than boxing it: a spec goes from the template /// that returns it straight to the `submit` that consumes it, so the closure's /// concrete type is known the whole way and needs neither an allocation nor a -/// `Send` bound. (The executor's `append_subgraph` is the case that *does* need -/// a boxed [`super::Declare`] — its recipes are collected into a `Vec` and -/// applied later, across a task boundary.) +/// `Send` bound. Nothing boxes a recipe any more — a running node grows its DAG +/// by declaring straight onto the builder it was handed, so there is no recipe +/// to store and replay across a task boundary. pub struct DagSpec { pub source: Source, /// Free-form "why". diff --git a/hive-c0re/src/job_queue/scheduler.rs b/hive-c0re/src/job_queue/scheduler.rs index 3efdf913..897f41ad 100644 --- a/hive-c0re/src/job_queue/scheduler.rs +++ b/hive-c0re/src/job_queue/scheduler.rs @@ -16,21 +16,18 @@ //! the DAG settles. //! //! In-DAG growth (a `MetaLock` growing rebuild subgraphs, a `Reconcile` fanning -//! its `Start`/`Stop`) flows through `NodeOutput.append_subgraph`, applied -//! before the emitting node completes — see `handle_completion`. +//! its `Start`/`Stop`) is declared onto the builder each node is handed, and +//! inserted as part of completing that node. Completion itself is not this +//! module's job any more: it happens *inside* the future +//! [`hive_jobq::scheduler::Scheduler::claim_next`] hands back, so a node that +//! ran but was never completed is not an expressible state here. use std::collections::HashMap; use std::sync::Arc; -use super::Claim; -use super::exec::{self, NodeOutput}; +use super::exec; use crate::coordinator::Coordinator; -struct NodeDone { - claim: Claim, - result: anyhow::Result, -} - /// Scheduler loop. Spawned once at hive-c0re startup from `main.rs`. /// /// Shutdown semantics: subscribes to `coord.shutdown_rx()`. On a true signal @@ -53,7 +50,6 @@ struct NodeDone { /// reconverging silently. pub async fn run_worker(coord: Arc) { let mut shutdown = coord.shutdown_rx(); - let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); // Last derived pill set we published, keyed by agent (its lease is cap-1, // so one pill each). Purely the previous value of a *derived* quantity — // it exists to spot transitions, since the dashboard wants edges @@ -70,24 +66,66 @@ pub async fn run_worker(coord: Arc) { return; } reconcile_transients(&coord, &mut transients); - let claims = coord.job_queue.claim_ready(); - if !claims.is_empty() { - for claim in claims { - tracing::info!( - dag = claim.dag_id, - node = claim.node_id.get(), - kind = claim.kind.as_str(), - agent = %claim.agent, - "job_queue: node running" - ); - let coord = Arc::clone(&coord); - let tx = tx.clone(); - tokio::spawn(async move { - let result = exec::run_node(&coord, &claim).await; - // Send failure = scheduler gone (shutdown); drop. - let _ = tx.send(NodeDone { claim, result }); - }); - } + // Claim exactly one node and get back the work that runs it. `Some` + // means something started, so there may be more runnable right now — + // loop again immediately. `None` means nothing is runnable and the + // loop parks below. That decision is the whole reason the crate hands + // back a task rather than an id. + let runner = { + // Two handles, deliberately: `sched` is the scheduler the crate + // locks, `node_coord` is what the node's own future captures. One + // binding can't do both — passing `coord.job_queue.sched()` borrows + // `coord` for the whole call while the `move` closure wants to take + // it. + let sched = Arc::clone(coord.job_queue.sched()); + let node_coord = Arc::clone(&coord); + hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, job| { + let coord = node_coord; + async move { + tracing::info!( + dag = coord.job_queue.dag_of(id).unwrap_or_default(), + node = id.get(), + kind = kind.as_str(), + agent = %kind.agent(), + "job_queue: node running" + ); + let (grown, result) = exec::run_node(&coord, job, id, &kind).await; + match &result { + Ok(()) => tracing::info!(node = id.get(), "job_queue: node done"), + Err(e) => tracing::warn!( + node = id.get(), + kind = kind.as_str(), + agent = %kind.agent(), + error = %format!("{e:#}"), + grown_nodes = !grown.is_empty(), + "job_queue: node failed" + ), + } + // Growth on a failed node is dropped by `complete_growing`, + // not here: failure cancel-cascades inside jobq, so that + // rule is the crate's to enforce and this loop does not get + // to forget it. + ( + grown, + super::outcome_of(result.map_err(|e| format!("{e:#}"))), + ) + } + }) + }; + if let Some(runner) = runner { + let done_coord = Arc::clone(&coord); + tokio::spawn(async move { + // Completion happens inside `runner` — it cannot be forgotten + // here, which is why there is no completion channel any more. + let (id, grew) = runner.await; + if let Err(e) = grew { + tracing::warn!(node = id.get(), error = %e, "job_queue: grown job rejected"); + } + done_coord.emit_rebuild_queue_snapshot(); + // Wake the loop: this node's completion may have unblocked + // dependents. Previously the completion channel did this. + done_coord.job_queue.notify.notify_one(); + }); // Newly-started owner nodes now hold their leases — surface the pills. reconcile_transients(&coord, &mut transients); coord.emit_rebuild_queue_snapshot(); @@ -101,55 +139,11 @@ pub async fn run_worker(coord: Arc) { return; } } - Some(done) = rx.recv() => { - handle_completion(&coord, done); - } () = coord.job_queue.notify.notified() => {} } } } -fn handle_completion(coord: &Arc, done: NodeDone) { - let NodeDone { claim, result } = done; - match result { - Ok(output) => { - tracing::info!( - dag = claim.dag_id, - node = claim.node_id.get(), - "job_queue: node done" - ); - // Append any in-DAG subgraphs BEFORE completing this node, so - // completing it doesn't roll the DAG terminal while the appended - // work is still pending. Each subgraph roots on this node - // (`AfterOk`), so it becomes ready the instant this one settles - // `Done` just below — covers both the multi-node case (a `MetaLock` - // growing per-agent rebuild subgraphs) and the single-node case (a - // `Reconcile` planner's `Start` / `Stop`). - for subgraph in output.append_subgraph { - coord - .job_queue - .append_subgraph(claim.dag_id, subgraph, claim.node_id); - } - coord.job_queue.complete_node(claim.node_id, Ok(())); - } - Err(e) => { - let msg = format!("{e:#}"); - tracing::warn!( - dag = claim.dag_id, - node = claim.node_id.get(), - kind = claim.kind.as_str(), - agent = %claim.agent, - error = %msg, - "job_queue: node failed" - ); - coord.job_queue.complete_node(claim.node_id, Err(msg)); - } - } - // The next loop iteration re-reconciles the transient pills against the - // post-completion lease state (a settled subgraph drops its pill). - coord.emit_rebuild_queue_snapshot(); -} - /// Publish the transitions between the previously-derived pill set and the /// current one. `prev` is last loop's derived value, keyed by agent (an agent's /// lease is cap-1, so at most one pill each). diff --git a/hive-c0re/src/job_queue/templates.rs b/hive-c0re/src/job_queue/templates.rs index 8607958c..198b06bf 100644 --- a/hive-c0re/src/job_queue/templates.rs +++ b/hive-c0re/src/job_queue/templates.rs @@ -28,7 +28,7 @@ use hive_jobq::TerminalState; use super::model::{DagSpec, NodeKind, PermPayload, Source}; use super::resource::Resource; -use super::{Declare, Handle, Job}; +use super::{Handle, Job}; /// The `Rebuilt`-reporting tail pair for a rebuild-shaped DAG: the success node /// gated on every group-root in `roots`, and the failure node gated on *its* @@ -80,6 +80,39 @@ fn resolve_approval_tails(b: &Job, approval_id: i64, root: Handle<'_>) { } } +/// Declare one rebuild subgraph per agent onto the builder a running +/// [`NodeKind::MetaLock`] was handed. +/// +/// **Into the emitter's own builder, not as new DAGs.** Growing in-DAG is what +/// roots each subgraph on the `MetaLock`, so the whole sweep (or meta-update +/// cascade) stays one unit of work the operator can watch and cancel, and every +/// rebuild builds against the lock the emitter just bumped. +/// +/// Same reason as [`fanned_out_mechanical`] for living here: this was the +/// second construction site declaring nodes inline in an executor. +pub(crate) fn grown_rebuilds(b: &Job, agents: &[String], opts: RebuildOpts) { + for agent in agents { + rebuild_nodes(b, agent, opts, None); + } +} + +/// Declare the mechanical node a [`NodeKind::Reconcile`] planner fans out +/// (`Start` / `Stop`) onto the builder it was handed while running. +/// +/// `Start` / `Stop` declare the agent lease they run under. Their `Reconcile` +/// parent is holding it already, so the declaration is a **re-entrant borrow** +/// — no second unit, no deadlock. It exists so the requirement belongs to the +/// node rather than to the fact that a `Reconcile` happens to fan it out. +/// +/// Lives here rather than inline in `exec.rs` for the same reason every other +/// declaration does: this is the one construction site that was hiding in an +/// executor, which meant the only test of it had to re-declare the same two +/// calls itself and would have kept passing if the executor changed. +pub(crate) fn fanned_out_mechanical(b: &Job, kind: NodeKind) { + let lease = Resource::Agent(kind.agent().to_owned()); + let _ = b.node(kind).needs(lease); +} + /// Knobs for [`rebuild_nodes`]. A struct rather than two positional `bool`s so /// a call site cannot silently swap them. #[derive(Debug, Clone, Copy)] @@ -235,32 +268,29 @@ pub(crate) fn rebuild_nodes<'a>( /// finalized. `Reconcile` alone would not do, being `AfterAny` — it reaches /// `Done` even after a failed `Swap`. /// -/// Appended, not submitted: the roots below become children of the emitting -/// `DeployApply` (see [`super::JobQueue::append_subgraph`]), which puts them -/// inside the `DeployWindow`'s subtree — so the `MetaWindow` this subgraph's -/// `MetaSync` and `FinalizeDeploy` declare is re-entered from the ancestor -/// already holding it rather than deadlocking against it. -pub(crate) fn deploy_rebuild_nodes(agent: &str, approval_id: i64) -> Declare { - let agent = agent.to_owned(); - Box::new(move |b: &Job| { - let roots = rebuild_nodes( - b, - &agent, - RebuildOpts { - relock: false, - graceful: false, - }, - None, - ); - let _finalize = b - .node(NodeKind::FinalizeDeploy { - agent: agent.clone(), - approval_id, - }) - .needs(Resource::MetaWindow) - .after_ok(roots.prebuild) - .after_ok(roots.reconcile); - }) +/// Declared into a **running** `DeployApply`'s own builder, not submitted: the +/// roots below become children of that node, which puts them inside the +/// `DeployWindow`'s subtree — so the `MetaWindow` this subgraph's `MetaSync` +/// and `FinalizeDeploy` declare is re-entered from the ancestor already holding +/// it rather than deadlocking against it. +pub(crate) fn deploy_rebuild_nodes(b: &Job, agent: &str, approval_id: i64) { + let roots = rebuild_nodes( + b, + agent, + RebuildOpts { + relock: false, + graceful: false, + }, + None, + ); + let _finalize = b + .node(NodeKind::FinalizeDeploy { + agent: agent.to_owned(), + approval_id, + }) + .needs(Resource::MetaWindow) + .after_ok(roots.prebuild) + .after_ok(roots.reconcile); } /// One uniform rebuild shape — no `was_running` branch. `StopForUpdate` @@ -375,29 +405,6 @@ pub fn approval_deploy( } } -/// A single `Reconcile` node that converges observed power state to the -/// persisted intent — `wanted` is untouched (no `SetWanted`), unlike the -/// operator `start`/`stop` templates. Test-only helper now (used to build -/// single-node lifecycle DAGs that exercise per-agent lease serialization -/// in the queue tests); production paths no longer emit a bare reconcile. -#[cfg(test)] -pub fn reconcile_only( - agent: &str, - source: Source, - reason: String, -) -> DagSpec> { - let agent = agent.to_owned(); - DagSpec { - source, - reason, - declare: Box::new(move |b: &Job| { - // Name the lease before the agent string is moved into the kind. - let lease = Resource::Agent(agent.clone()); - let _reconcile = b.node(NodeKind::Reconcile { agent }).needs(lease); - }), - } -} - /// First-deploy spawn (approval-driven): `Provision` (proposed/applied /// repos, state subvolume, meta registration) then `Create` /// (`nixos-container create`), drop-in write, then `Reconcile` starts @@ -480,8 +487,8 @@ pub fn perm_change( } /// Meta-input lock bump. The `MetaLock` executor grows one rebuild subgraph -/// per affected agent into *this same* DAG on completion (via -/// `append_subgraph`) — appended *after* the bump lands so their prebuilds +/// per affected agent into *this same* DAG on completion (declared onto the +/// builder it was handed) — appended *after* the bump lands so their prebuilds /// run against the post-bump lock, and a failed bump appends nothing /// (replacing the old fan-out-child-DAGs dance). /// `transient = Rebuilding` because those appended subgraphs are rebuilds: diff --git a/hive-c0re/src/job_queue/tests.rs b/hive-c0re/src/job_queue/tests.rs index 13de8d5d..5ef2a0be 100644 --- a/hive-c0re/src/job_queue/tests.rs +++ b/hive-c0re/src/job_queue/tests.rs @@ -1,10 +1,18 @@ -//! Queue-core unit tests: submit / no-dedup, cycle rejection, resource -//! serialization (build slots / per-agent leases), lease-exempt -//! overlap, FIFO fairness, cancel semantics, `AfterAny` failure -//! routing, in-DAG subgraph growth, and history retention. All -//! synchronous — the -//! scheduler's async loop is a thin claim/complete pump over the same -//! methods exercised here. +//! Queue-core unit tests: what c0re's templates **declare** — node kinds, +//! parent nesting, dep edges with the outcomes that satisfy them, and the +//! resources each construction site states it holds — plus the read layer over +//! that graph (wire projection, history retention, error truncation). +//! +//! **Nothing here runs a node.** Everything a template declares is in the graph +//! the moment `submit` returns, so the assertions read it there. Whether the +//! scheduler then honours those declarations — cascade, roll-up, grant +//! borrow/release, fairness, the `Finishing` gate — is `hive_jobq`'s property +//! and is tested in `hive_jobq`, against its own primitives rather than through +//! this module's templates. +//! +//! That split is why this file can't claim or complete: those are not part of +//! c0re's surface. A test helper that reached for them was reaching across the +//! boundary the two crates exist to draw. use super::model::NodeKind; use super::*; @@ -13,20 +21,6 @@ fn submit(q: &JobQueue, spec: DagSpec) -> u64 { q.submit(spec).expect("valid spec") } -/// Erase a spec's recipe to the boxed [`Declare`] so specs of *different* -/// shapes can share one type — e.g. a table of `(name, spec)` cases. -/// -/// Production never needs this: each submit path builds one spec and hands it -/// straight to `submit`, so the concrete closure type is known end to end. A -/// test table is the case where several shapes must be one type. -fn erase(spec: DagSpec) -> DagSpec { - DagSpec { - source: spec.source, - reason: spec.reason, - declare: Box::new(spec.declare), - } -} - fn ident(s: &str) -> hive_types::Ident { hive_types::Ident::parse(s).expect("valid test ident") } @@ -59,47 +53,179 @@ fn stop_online( submit::stop_spec(&targets, graceful, Source::Manual, reason.to_owned()) } -/// Claim helper asserting exactly one node comes back. -fn claim_one(q: &JobQueue) -> Claim { - let mut claims = q.claim_ready(); - assert_eq!( - claims.len(), - 1, - "expected exactly one claim, got {claims:?}" - ); - claims.pop().expect("one claim") +// `Claimed` / `ClaimReady` / `CompleteNode` lived here: a claim snapshot type +// and two extension traits that let this module start and finish nodes by +// hand. Nothing in this file drives the scheduler any more, so they are gone +// — which is the point. Production completes a node **inside** the future +// `hive_jobq::scheduler::Scheduler::claim_next` hands back, so "run the node, +// then remember to complete it" is not an expressible sequence there. A test +// helper that re-expressed it was a hole in exactly the seam it was testing. + +/// One node's **declared** shape: what it is, what it hangs under, and what it +/// waits for — all by kind, since ids are not stable across runs. +#[derive(Debug, PartialEq, Eq)] +struct Declared { + kind: &'static str, + /// Parent kind, or `None` when the node hangs directly under the DAG + /// container (i.e. it is a group root). + parent: Option<&'static str>, + /// Kinds this node declared a node-dep on, in declaration order, each with + /// the outcome set that satisfies it. + /// + /// The outcome set is **not** decoration: a template emits its tails as a + /// pair edged on the same upstream nodes, and the *only* thing telling the + /// ok-tail from the fail-tail is which outcomes each accepts. Without it + /// two structurally different nodes read as identical. + after: Vec<(&'static str, String)>, } -/// Claim an approval DAG's `ResolveApproval` tail and complete it, asserting it -/// is the one built for `expect`. +/// Render a dep's outcome set as the outcomes it actually accepts. /// -/// A template emits one tail per outcome and the graph runs exactly one, so the -/// assertion is on *which node was claimed* — that alone says what the approval -/// row is about to be resolved as. Nothing computes it. -fn settle_approval_tail(q: &JobQueue, approval_id: i64, expect: TerminalState) { - let tail = claim_one(q); - assert!( - matches!( - tail.kind, - NodeKind::ResolveApproval { approval_id: got, outcome } - if got == approval_id && outcome == expect - ), - "expected the {expect:?} ResolveApproval tail for #{approval_id}, got {:?}", - tail.kind - ); - q.complete_node(tail.node_id, Ok(())); +/// ⚠️ Spelled out rather than bucketed into `ok` / `any` / other. The first +/// version of this did bucket, and a template's three `ResolveApproval` tails — +/// which differ ONLY in their accepted outcomes — all rendered as `"other"`. +/// A helper that prints two structurally different nodes identically turns an +/// assertion into a tautology. +fn when_tag(when: hive_jobq::DepWhen) -> String { + [ + (TerminalState::Done, "done"), + (TerminalState::Failed, "failed"), + (TerminalState::Cancelled, "cancelled"), + (TerminalState::Skipped, "skipped"), + ] + .into_iter() + .filter(|(outcome, _)| when.accepts(*outcome)) + .map(|(_, name)| name) + .collect::>() + .join("|") } -/// The `EmitRebuilt` counterpart of [`settle_approval_tail`] — claim the tail the -/// graph let run and assert it's the `ok` one expected. -fn settle_rebuild_tail(q: &JobQueue, agent: &str, expect_ok: bool) { - let tail = claim_one(q); - assert!( - matches!(&tail.kind, NodeKind::EmitRebuilt { agent: a, ok } if a == agent && *ok == expect_ok), - "expected the ok={expect_ok} EmitRebuilt tail for {agent}, got {:?}", - tail.kind +/// Every work node under `dag`, in insertion order, as its declared shape. +/// +/// **This is what the template tests are actually about.** A template's output +/// is fully determined the moment `submit` returns: the kinds, the parent +/// nesting and the dep edges are all sitting in the graph. Reading them here +/// keeps the assertion on c0re's own product. Whether the scheduler then +/// *honours* those edges — runs a chain serially, holds a grant across a +/// subtree — is `hive_jobq`'s property and is tested in `hive_jobq`. +fn declared_shape(q: &JobQueue, dag: u64) -> Vec { + declared_shape_filtered(q, dag, &|_| true) +} + +fn declared_shape_filtered( + q: &JobQueue, + dag: u64, + keep: &dyn Fn(&NodeKind) -> bool, +) -> Vec { + let sched = q.sched().lock().expect("job_queue mutex poisoned"); + let graph = sched.graph(); + let root = graph.resolve_id(dag).expect("dag id is a real node id"); + let kind_of = |id: NodeId| graph.node(id).map(|n| n.payload.as_str()); + graph + .nodes() + .filter(|n| n.id != root && graph.root_of(n.id) == Some(root) && keep(&n.payload)) + .map(|n| Declared { + kind: n.payload.as_str(), + parent: n.parent.filter(|p| *p != root).and_then(kind_of), + after: n + .deps + .iter() + .filter_map(|dep| match dep { + hive_jobq::Dep::Node { id, when } => kind_of(*id).map(|k| (k, when_tag(*when))), + hive_jobq::Dep::Resource { .. } => None, + }) + .collect(), + }) + .collect() +} + +/// The id of the one node of `kind` under `dag`, for the resource assertions. +/// +/// Panics unless there is exactly one — every caller is about a shape where the +/// kind is unique, so two would mean the assertion had quietly stopped being +/// about the node the test names. +fn node_of(q: &JobQueue, dag: u64, kind: &str) -> hive_jobq::NodeId { + let sched = q.sched().lock().expect("job_queue mutex poisoned"); + let graph = sched.graph(); + let root = graph.resolve_id(dag).expect("dag id is a real node id"); + let mut found: Vec<_> = graph + .nodes() + .filter(|n| n.id != root && graph.root_of(n.id) == Some(root) && n.payload.as_str() == kind) + .map(|n| n.id) + .collect(); + assert_eq!( + found.len(), + 1, + "expected exactly one {kind} node in the dag" ); - q.complete_node(tail.node_id, Ok(())); + found.pop().expect("checked above") +} + +/// The payload of the one node of `kind` under `dag`, for assertions about what +/// a node *carries* rather than how it is wired. +fn payload_of(q: &JobQueue, dag: u64, kind: &str) -> NodeKind { + let id = node_of(q, dag, kind); + let sched = q.sched().lock().expect("job_queue mutex poisoned"); + sched.graph().node(id).expect("node exists").payload.clone() +} + +/// Kinds of every node under `dag` still `Pending` — the nodes that could yet +/// run. Stronger than asking the scheduler what is *ready right now*: a node +/// blocked on a dep is not ready but is very much still alive. +fn pending_kinds(q: &JobQueue, dag: u64) -> Vec<&'static str> { + pending_kinds_filtered(q, dag, &|_| true) +} + +/// [`pending_kinds`] restricted to the nodes whose payload names `agent`. +fn pending_kinds_for(q: &JobQueue, dag: u64, agent: &str) -> Vec<&'static str> { + pending_kinds_filtered(q, dag, &|kind: &NodeKind| kind.agent() == agent) +} + +/// The payloads of every node under `dag` still `Pending`, for the cases where +/// *which* of a family of same-kind nodes survived is the assertion — a +/// template emits one tail per outcome and they differ only in what they carry. +fn pending_payloads(q: &JobQueue, dag: u64) -> Vec { + let sched = q.sched().lock().expect("job_queue mutex poisoned"); + let graph = sched.graph(); + let root = graph.resolve_id(dag).expect("dag id is a real node id"); + graph + .nodes() + .filter(|n| n.id != root && graph.root_of(n.id) == Some(root) && n.state == State::Pending) + .map(|n| n.payload.clone()) + .collect() +} + +fn pending_kinds_filtered( + q: &JobQueue, + dag: u64, + keep: &dyn Fn(&NodeKind) -> bool, +) -> Vec<&'static str> { + let sched = q.sched().lock().expect("job_queue mutex poisoned"); + let graph = sched.graph(); + let root = graph.resolve_id(dag).expect("dag id is a real node id"); + graph + .nodes() + .filter(|n| { + n.id != root + && graph.root_of(n.id) == Some(root) + && n.state == State::Pending + && keep(&n.payload) + }) + .map(|n| n.payload.as_str()) + .collect() +} + +/// Shorthand for one expected row, so the tables below read as a shape. +fn row( + kind: &'static str, + parent: Option<&'static str>, + after: &[(&'static str, &str)], +) -> Declared { + Declared { + kind, + parent, + after: after.iter().map(|(k, w)| (*k, (*w).to_owned())).collect(), + } } /// The resources a node **declared**, read off its graph edges. @@ -111,7 +237,6 @@ fn settle_rebuild_tail(q: &JobQueue, agent: &str, expect_ok: bool) { fn declared_resources(q: &JobQueue, node_id: hive_jobq::NodeId) -> Vec { let inner = q.lock(); inner - .sched .graph() .node(node_id) .expect("node exists") @@ -124,6 +249,44 @@ fn declared_resources(q: &JobQueue, node_id: hive_jobq::NodeId) -> Vec .collect() } +/// The resources declared by **every** node of `kind` under `dag`, one row per +/// node, sorted so the rows read as a set rather than an insertion order. +/// +/// The per-agent templates emit several nodes of one kind — one per agent — and +/// what makes them concurrent is that each holds only its *own* agent's lease. +/// That is a statement about the whole family, so it needs all the rows, not +/// [`declared_resources`]'s single node. +fn declared_resources_of_kind(q: &JobQueue, dag: u64, kind: &str) -> Vec> { + let sched = q.sched().lock().expect("job_queue mutex poisoned"); + let graph = sched.graph(); + let root = graph.resolve_id(dag).expect("dag id is a real node id"); + let mut rows: Vec> = graph + .nodes() + .filter(|n| n.id != root && graph.root_of(n.id) == Some(root) && n.payload.as_str() == kind) + .map(|n| { + n.deps + .iter() + .filter_map(|dep| match dep { + hive_jobq::Dep::Resource { name, .. } => Some(name.clone()), + hive_jobq::Dep::Node { .. } => None, + }) + .collect() + }) + .collect(); + rows.sort_by_key(|r| format!("{r:?}")); + rows +} + +/// [`declared_shape`] restricted to the nodes whose payload names `agent`. +/// +/// A hive-wide DAG interleaves one subgraph per agent, and the kinds alone +/// cannot tell them apart — two `set_wanted` rows look identical. Slicing by +/// agent is what makes "the fresh agent goes straight to reconcile while the +/// stale one rebuilds first" expressible as a declared shape. +fn declared_shape_for(q: &JobQueue, dag: u64, agent: &str) -> Vec { + declared_shape_filtered(q, dag, &|kind: &NodeKind| kind.agent() == agent) +} + fn state_of(q: &JobQueue, dag_id: u64) -> State { // A DAG whose nodes have all settled `Done` or `Skipped` drops out of the // snapshot — absence is the completion signal, so map it to `Done`. @@ -173,11 +336,17 @@ fn distinct_submits_never_collapse() { #[test] fn resubmit_while_running_is_new_dag() { + // The "while running" is not load-bearing and used to be staged by claiming + // a node first. `submit` appends a container and inserts the declared + // group; it never consults the state of any existing node, so whether an + // earlier DAG is running cannot change the outcome. What is actually being + // asserted — no dedup, ever — is `identical_resubmit_is_a_distinct_dag`. + // + // Kept as the *named* case because "a config bump mid-build must not be + // swallowed" is the scenario people worry about, and a reader looking for + // it should find it. let q = JobQueue::new(1); let a = submit(&q, rebuild("agent-a", "first")); - let claim = claim_one(&q); // Prebuild running - assert_eq!(claim.dag_id, a); - // While the original runs, re-submit is legitimate new work. let again = submit(&q, rebuild("agent-a", "config bumped during build")); assert_ne!(a, again); assert_eq!(q.snapshot().len(), 2); @@ -203,28 +372,52 @@ fn resubmit_while_running_is_new_dag() { // ---- dependency order within a DAG ---- #[test] -fn rebuild_chain_claims_in_dep_order() { +fn rebuild_chain_is_declared_serial() { + // Was `rebuild_chain_claims_in_dep_order`, which drove the whole DAG to + // observe an order that is fully declared the moment `submit` returns. + // + // ⚠️ The old name was also wrong about the mechanism, and reading it rather + // than the graph is how you'd stay wrong: **only half this chain is dep + // edges.** `stop_for_update` and `swap` declare no deps at all — they are + // ordered by *parent nesting* ("a node's sub-nodes run after its own + // logic"). Both axes are asserted below because a template can break either + // one independently. let q = JobQueue::new(1); let id = submit(&q, rebuild("agent-a", "r")); - for expected in [ - "meta_sync", - "prebuild", - "stop_for_update", - "swap", - "post_swap", - "reconcile", - ] { - let c = claim_one(&q); - assert_eq!(c.dag_id, id); - assert_eq!(c.kind.as_str(), expected); - assert!( - q.claim_ready().is_empty(), - "chain must serialize: nothing ready while {expected} runs" - ); - q.complete_node(c.node_id, Ok(())); - } - settle_rebuild_tail(&q, "agent-a", true); - assert_eq!(state_of(&q, id), State::Done); + assert_eq!( + declared_shape(&q, id), + vec![ + row("meta_sync", None, &[]), + row("prebuild", None, &[("meta_sync", "done")]), + // No dep: ordered by hanging under `prebuild`. + row("stop_for_update", Some("prebuild"), &[]), + row("swap", Some("stop_for_update"), &[]), + row("post_swap", Some("stop_for_update"), &[("swap", "done")]), + row("reconcile", None, &[("prebuild", "done|failed|skipped")]), + // The tail pair. The ok tail needs every root to succeed; the !ok + // tail hangs off the ok tail's *elimination* (`skipped`), which is + // what makes exactly one of them run. + row( + "emit_rebuilt", + None, + &[ + ("meta_sync", "done"), + ("prebuild", "done"), + ("reconcile", "done"), + ], + ), + row( + "emit_rebuilt", + None, + &[ + ("emit_rebuilt", "skipped"), + ("meta_sync", "done|failed|skipped"), + ("prebuild", "done|failed|skipped"), + ("reconcile", "done|failed|skipped"), + ], + ), + ] + ); } /// The boot sweep's graceful shape: the agent gets `Signal` → `Drain` to @@ -254,26 +447,25 @@ fn graceful_rebuild_chain_drains_before_stopping() { }), }, ); - for expected in [ - "meta_sync", - "prebuild", - "signal", - "drain", - "stop_for_update", - "swap", - "post_swap", - "reconcile", - ] { - let c = claim_one(&q); - assert_eq!(c.dag_id, id); - assert_eq!(c.kind.as_str(), expected); - assert!( - q.claim_ready().is_empty(), - "chain must serialize: nothing ready while {expected} runs" - ); - q.complete_node(c.node_id, Ok(())); - } - assert_eq!(state_of(&q, id), State::Done); + assert_eq!( + declared_shape(&q, id) + .iter() + .map(|d| d.kind) + .collect::>(), + vec![ + "meta_sync", + "prebuild", + // The graceful window goes between the build and the stop: the + // agent gets its turn to finish before the container goes down. + "signal", + "drain", + "stop_for_update", + "swap", + "post_swap", + "reconcile", + ], + "graceful inserts signal + drain ahead of the stop, and nothing else" + ); } /// The non-graceful shape is the default everywhere except the boot sweep: @@ -303,14 +495,11 @@ fn non_graceful_rebuild_has_no_signal_or_drain() { }), }, ); - let mut kinds = Vec::new(); - for _ in 0..6 { - let c = claim_one(&q); - kinds.push(c.kind.as_str().to_owned()); - q.complete_node(c.node_id, Ok(())); - } assert_eq!( - kinds, + declared_shape(&q, id) + .iter() + .map(|d| d.kind) + .collect::>(), vec![ "meta_sync", "prebuild", @@ -318,222 +507,106 @@ fn non_graceful_rebuild_has_no_signal_or_drain() { "swap", "post_swap", "reconcile" - ] + ], + "exactly six nodes, and neither of them is signal or drain" ); - // Settled after exactly those six — nothing else was declared. - assert_eq!(state_of(&q, id), State::Done); } -/// A cleanly-finished DAG leaves the snapshot even though its not-taken -/// failure branch is still in the graph as `Skipped`. Skipped nodes ride the -/// wire so the dashboard can mark them, which makes "the node list is empty" -/// and "nothing here is still worth showing" two different questions — only -/// the second one may drop the DAG. Conflating them pins every completed -/// deploy in the queue view forever. +/// Which nodes ride the wire, and whether a DAG is still worth showing. +/// +/// This used to submit a rebuild, claim and complete all seven of its nodes, +/// then assert the DAG was absent from `snapshot()` — the scheduler, the +/// roll-up and the whole projection standing in for one predicate over a list +/// of states. `shown_on_wire` is that predicate, so the cases can be named +/// instead of arranged. #[test] -fn settled_dag_leaves_the_snapshot_despite_its_skipped_branch() { - let q = JobQueue::new(1); - let id = submit(&q, rebuild("agent-a", "r")); - for _ in 0..6 { - let c = claim_one(&q); - q.complete_node(c.node_id, Ok(())); - } - settle_rebuild_tail(&q, "agent-a", true); - assert!( - q.snapshot().iter().all(|d| d.id != id), - "a fully settled DAG drops out of the snapshot" +fn skipped_nodes_ride_the_wire_but_do_not_hold_a_settled_dag_in_the_snapshot() { + // Nothing left worth showing → the DAG drops out of the snapshot, and its + // absence is what signals completion. + assert_eq!(shown_on_wire(&[State::Done, State::Done]), None); + // The case the old test was built around: a clean run whose not-taken + // failure branch is still in the graph as `Skipped`. "The node list is + // non-empty" and "there's something here worth showing" are different + // questions, and conflating them pins every completed deploy in the queue + // view forever. + assert_eq!( + shown_on_wire(&[State::Done, State::Skipped, State::Done]), + None, + "a skipped branch does not keep a finished DAG alive" ); + // A `Failed` DAG lingers — and takes its skipped branches with it, which is + // how an operator sees which steps the run never reached. + assert_eq!( + shown_on_wire(&[State::Done, State::Failed, State::Skipped, State::Done]), + Some(vec![1, 2]), + "done nodes drop off, the failure and what it ruled out stay" + ); + // Live DAGs keep everything but their finished steps. + assert_eq!( + shown_on_wire(&[State::Done, State::Running, State::Pending]), + Some(vec![1, 2]) + ); + assert_eq!( + shown_on_wire(&[State::Cancelled, State::Skipped]), + Some(vec![0, 1]), + "a cancelled DAG is still worth showing to whoever cancelled it" + ); + // An empty DAG has nothing worth showing either — no special case needed, + // but it is the one input where "any" and "all" disagree, so it is pinned. + assert_eq!(shown_on_wire(&[]), None); } // ---- build slots ---- #[test] -fn build_slot_serializes_nix_heavy_nodes() { +fn rebuild_chain_declares_the_slot_where_the_nix_work_is() { + // Was `fifo_fairness_for_the_slot`, which submitted three rebuilds and + // drove one to completion to watch the freed slot go to the earlier + // waiter. **That fairness guarantee is hive_jobq's**, and it had no test + // there at all — its claim primitive scans nodes in insertion order and + // takes the first satisfiable one, and nothing pinned that. It does now: + // `a_contended_resource_goes_to_the_oldest_waiter`. + // + // What is c0re's is *which* nodes contend for the slot in the first place, + // and that is a declaration. "Uniform hold across the chain" then follows + // from the parent nesting asserted in `rebuild_chain_is_declared_serial`: + // a resource unit is held for the acquirer's whole subtree, so the slot + // `Prebuild` takes covers `StopForUpdate` → `Swap` → `PostSwap` beneath it. let q = JobQueue::new(1); - let a = submit(&q, rebuild("agent-a", "r")); - let b = submit(&q, rebuild("agent-b", "r")); - // The rebuild heads are `MetaSync` (slot-free, but serialized on the - // global meta window), so drive each chain's head out of the way first. - let head_a = claim_one(&q); - assert_eq!(head_a.dag_id, a); - assert_eq!(head_a.kind.as_str(), "meta_sync"); - q.complete_node(head_a.node_id, Ok(())); - // a's Prebuild takes the only slot; b's MetaSync is free to run beside it - // (different resources), but b's Prebuild is not. - let claims = q.claim_ready(); - let mut kinds: Vec<(u64, &str)> = claims.iter().map(|c| (c.dag_id, c.kind.as_str())).collect(); - kinds.sort_unstable(); - assert_eq!(kinds, vec![(a, "prebuild"), (b, "meta_sync")]); - for c in &claims { - q.complete_node(c.node_id, Ok(())); - } - // Uniform hold: agent-a keeps the build slot across its whole build chain - // (Swap re-enters it), so a's StopForUpdate (lease, slot-free) runs but b's - // Prebuild must wait for a's slot-needers (through Swap) to finish. - let kinds: Vec<(u64, &str)> = q - .claim_ready() - .iter() - .map(|c| (c.dag_id, c.kind.as_str())) - .collect(); - assert_eq!(kinds, vec![(a, "stop_for_update")]); - assert!( - !kinds.iter().any(|&(d, _)| d == b), - "b's build waits — slot held across a's chain" - ); -} + let id = submit(&q, rebuild("agent-a", "r")); + let res = |kind: &str| declared_resources(&q, node_of(&q, id, kind)); -#[test] -fn two_build_slots_run_two_prebuilds() { - let q = JobQueue::new(2); - submit(&q, rebuild("agent-a", "r")); - submit(&q, rebuild("agent-b", "r")); - // Each rebuild's head `MetaSync` holds the cap-1 global meta window, so the - // two heads take turns — exactly the serialization the old runtime - // `meta::exclusive()` mutex imposed inside the prebuild executor. What must - // NOT serialize is the build itself: complete only the meta heads and watch - // both prebuilds end up in flight together, neither of them completed. - let mut prebuilds = Vec::new(); - for _ in 0..3 { - for c in q.claim_ready() { - if c.kind.as_str() == "meta_sync" { - q.complete_node(c.node_id, Ok(())); - } else { - prebuilds.push(c); - } - } - } - assert_eq!(prebuilds.len(), 2, "two slots → two concurrent prebuilds"); - assert!(prebuilds.iter().all(|c| c.kind.as_str() == "prebuild")); -} - -#[test] -fn fifo_fairness_for_the_slot() { - let q = JobQueue::new(1); - let a = submit(&q, rebuild("agent-a", "r")); - let b = submit(&q, rebuild("agent-b", "r")); - let c = submit(&q, rebuild("agent-c", "r")); - let first = claim_one(&q); - assert_eq!(first.dag_id, a, "submit order wins the slot"); - q.complete_node(first.node_id, Ok(())); - // Uniform hold: the slot stays with agent-a until its Swap (the last - // slot-needer) completes. Drive a's chain; the moment its slot frees, - // submit order (b before c) wins it. - let mut freed_to = None; - for _ in 0..6 { - let claims = q.claim_ready(); - if let Some(nb) = claims.iter().find(|cl| cl.dag_id == b || cl.dag_id == c) { - freed_to = Some(nb.dag_id); - break; - } - for cl in claims { - if cl.dag_id == a { - q.complete_node(cl.node_id, Ok(())); - } - } - } + let agent = || Resource::Agent("agent-a".to_owned()); assert_eq!( - freed_to, - Some(b), - "b's prebuild wins the freed slot before c's" + [ + res("meta_sync"), + res("prebuild"), + res("stop_for_update"), + res("swap"), + res("reconcile"), + ], + [ + // The meta preamble takes the global window and *nothing else* — + // no slot (it does no nix work) and no lease. + vec![Resource::MetaWindow], + // The nix build is the slot-needer, and takes **no lease**. That is + // what lets a prebuild overlap another DAG on the same agent: the + // container is still up and untouched while it builds. + vec![Resource::BuildSlot], + // The lease starts here — the first node that touches the + // container — and not one node earlier. + vec![agent()], + // Swap needs both. It re-enters the slot its `Prebuild` ancestor + // holds rather than acquiring a second unit. + vec![Resource::BuildSlot, agent()], + vec![agent()], + ], + "the slot follows the nix work and the lease follows the container" ); } // ---- per-agent lease ---- -#[test] -fn lease_serializes_two_lifecycle_dags_for_same_agent() { - let q = JobQueue::new(4); - let restart = submit(&q, restart_online(&["agent-a"], false, "restart")); - let stop = submit( - &q, - templates::reconcile_only("agent-a", Source::Manual, "stop".to_owned()), - ); - // Restart's first node (StopForUpdate) takes the lease; stop's - // Reconcile must wait even though slots are free. - let first = claim_one(&q); - assert_eq!(first.dag_id, restart); - assert_eq!(first.kind.as_str(), "stop_for_update"); - q.complete_node(first.node_id, Ok(())); - // Same DAG keeps the lease through the tail Reconcile (re-entered from the - // dep graph — no fresh acquire), since stop's Reconcile can't re-enter it. - let second = claim_one(&q); - assert_eq!(second.dag_id, restart); - assert_eq!(second.kind.as_str(), "reconcile"); - q.complete_node(second.node_id, Ok(())); - // Restart's work is terminal → its lease releases, so stop's now-unblocked - // Reconcile becomes ready (a power op has no tail node, so nothing of - // restart's remains claimable). - let third = claim_one(&q); - assert_eq!(third.dag_id, stop); - assert_eq!(third.kind.as_str(), "reconcile"); - q.complete_node(third.node_id, Ok(())); - assert_eq!(state_of(&q, restart), State::Done); - assert_eq!(state_of(&q, stop), State::Done); -} - -#[test] -fn lease_exempt_prebuild_overlaps_other_dag_on_same_agent() { - let q = JobQueue::new(2); - submit(&q, rebuild("agent-a", "rebuild")); - submit( - &q, - templates::reconcile_only("agent-a", Source::Manual, "stop".to_owned()), - ); - // Both DAGs' heads are lease-independent of each other: the rebuild's - // MetaSync (meta window) and the stop's Reconcile (agent lease). - let heads = q.claim_ready(); - let head_kinds: Vec<&str> = heads.iter().map(|c| c.kind.as_str()).collect(); - assert!(head_kinds.contains(&"meta_sync")); - assert!(head_kinds.contains(&"reconcile")); - let meta_sync = heads - .iter() - .find(|c| c.kind.as_str() == "meta_sync") - .expect("meta_sync claim") - .clone(); - q.complete_node(meta_sync.node_id, Ok(())); - // Prebuild is lease-exempt: the stop's Reconcile keeps the lease - // and runs concurrently with the rebuild's out-of-band nix build. - let claims = q.claim_ready(); - let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect(); - assert!(kinds.contains(&"prebuild")); - // But the rebuild's StopForUpdate must then wait for the stop DAG - // to finish (lease). - let prebuild = claims - .iter() - .find(|c| c.kind.as_str() == "prebuild") - .expect("prebuild claim") - .clone(); - q.complete_node(prebuild.node_id, Ok(())); - assert!( - q.claim_ready().is_empty(), - "StopForUpdate blocked while stop DAG holds the lease" - ); - let reconcile = heads - .iter() - .find(|c| c.kind.as_str() == "reconcile") - .expect("reconcile claim") - .clone(); - q.complete_node(reconcile.node_id, Ok(())); - // stop's Reconcile done → its lease frees, so rebuild's StopForUpdate - // unblocks. (stop's DAG rolls up terminal; a power op has no tail node, so - // nothing of stop's is left in the claim set.) - let after = q.claim_ready(); - let sfu = after - .iter() - .find(|c| c.kind.as_str() == "stop_for_update") - .expect("rebuild StopForUpdate unblocked once the lease frees"); - assert_eq!(sfu.agent, "agent-a"); -} - -#[test] -fn agents_do_not_contend_on_each_others_leases() { - let q = JobQueue::new(4); - submit(&q, restart_online(&["agent-a"], false, "r")); - submit(&q, restart_online(&["agent-b"], false, "r")); - let claims = q.claim_ready(); - assert_eq!(claims.len(), 2, "different agents run concurrently"); -} - #[test] fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() { let q = JobQueue::new(4); @@ -543,69 +616,31 @@ fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() { ); // A hive-wide restart is ONE DAG, not one-per-agent. assert_eq!(q.snapshot().len(), 1); - // Each agent's subgraph head (StopForUpdate, since both are running) is - // a root, so both are claimable at once — each takes its OWN agent's - // lease (no contention across distinct agents), all inside the single DAG. - let claims = q.claim_ready(); - assert!(claims.iter().all(|c| c.dag_id == id)); - let mut heads: Vec<(&str, &str)> = claims - .iter() - .map(|c| (c.agent.as_str(), c.kind.as_str())) + // Each agent's subgraph head (StopForUpdate, since both are running) is a + // group root with no deps, so nothing orders them against each other; and + // each declares only its OWN agent's lease, so nothing makes them contend. + // Those two declared facts are what "they run concurrently" *means* here — + // that a scheduler then does run independent, resource-disjoint roots at + // once is hive_jobq's property, tested there. + let heads: Vec<_> = declared_shape(&q, id) + .into_iter() + .filter(|d| d.kind == "stop_for_update") .collect(); - heads.sort_unstable(); assert_eq!( heads, vec![ - ("agent-a", "stop_for_update"), - ("agent-b", "stop_for_update"), + row("stop_for_update", None, &[]), + row("stop_for_update", None, &[]), ], - "both per-agent subgraphs start concurrently, each acquiring its own lease" + "both per-agent heads are independent group roots" ); -} - -/// A multi-agent DAG frees an agent's lease the moment THAT agent's -/// subgraph is terminal — not when the whole DAG finishes. So a -/// concurrent DAG wanting the finished agent can proceed while the rest -/// of the first DAG runs on. -#[test] -fn multi_agent_lease_frees_per_subgraph_not_whole_dag() { - let q = JobQueue::new(4); - let id = submit(&q, restart_online(&["agent-a", "agent-b"], false, "r")); - - // Drive agent-a's ENTIRE subgraph to Done while leaving agent-b's - // head running (so agent-b keeps holding its lease). - let mut b_in_flight = false; - loop { - let mut progressed = false; - for c in q.claim_ready() { - if c.agent == "agent-a" { - q.complete_node(c.node_id, Ok(())); - progressed = true; - } else { - b_in_flight = true; // leave agent-b's node running - } - } - if !progressed { - break; - } - } - assert!(b_in_flight, "agent-b subgraph should still be in flight"); - // The DAG as a whole is NOT terminal — agent-b runs on. - assert_eq!(state_of(&q, id), State::Running); - - // agent-a's lease is freed early → a concurrent agent-a DAG runs; - // an agent-b DAG still blocks on the lease agent-b's subgraph holds. - submit(&q, restart_online(&["agent-a"], false, "concurrent-a")); - submit(&q, restart_online(&["agent-b"], false, "concurrent-b")); - let claims = q.claim_ready(); - let agents: Vec<&str> = claims.iter().map(|c| c.agent.as_str()).collect(); - assert!( - agents.contains(&"agent-a"), - "agent-a lease freed the moment its subgraph settled" - ); - assert!( - !agents.contains(&"agent-b"), - "agent-b lease still held — its subgraph is still in flight" + assert_eq!( + declared_resources_of_kind(&q, id, "stop_for_update"), + vec![ + vec![Resource::Agent("agent-a".to_owned())], + vec![Resource::Agent("agent-b".to_owned())], + ], + "each head declares only its own agent's lease — disjoint, so no contention" ); } @@ -618,17 +653,26 @@ fn multi_agent_stop_is_one_dag_with_concurrent_per_agent_subgraphs() { ); // A hive-wide stop is ONE DAG, not one-per-agent. assert_eq!(q.snapshot().len(), 1); - let claims = q.claim_ready(); - assert!(claims.iter().all(|c| c.dag_id == id)); - let mut heads: Vec<(&str, &str)> = claims - .iter() - .map(|c| (c.agent.as_str(), c.kind.as_str())) + // Same declared story as the restart case above: each agent's subgraph head + // is a group root with no node-deps, holding only its own agent's lease. + // Independent roots on disjoint resources is what "concurrently" means at + // this layer — the running of them is hive_jobq's. + let heads: Vec<_> = declared_shape(&q, id) + .into_iter() + .filter(|d| d.kind == "set_wanted") .collect(); - heads.sort_unstable(); assert_eq!( heads, - vec![("agent-a", "set_wanted"), ("agent-b", "set_wanted")], - "both per-agent stop subgraphs start concurrently, each on its own lease" + vec![row("set_wanted", None, &[]), row("set_wanted", None, &[])], + "both per-agent stop subgraph heads are independent group roots" + ); + assert_eq!( + declared_resources_of_kind(&q, id, "set_wanted"), + vec![ + vec![Resource::Agent("agent-a".to_owned())], + vec![Resource::Agent("agent-b".to_owned())], + ], + "each head declares only its own agent's lease" ); } @@ -650,32 +694,31 @@ fn multi_agent_start_one_dag_folds_per_agent_stale_rebuild() { ); // One DAG spanning both agents. assert_eq!(q.snapshot().len(), 1); - // Both subgraph heads (SetWanted(Up)) are roots — claimable at once, - // each acquiring its own agent lease. - let heads = q.claim_ready(); - assert!( - heads - .iter() - .all(|c| c.dag_id == id && c.kind.as_str() == "set_wanted") - ); - let mut head_agents: Vec<&str> = heads.iter().map(|c| c.agent.as_str()).collect(); - head_agents.sort_unstable(); - assert_eq!(head_agents, vec!["fresh", "stale"]); - // Complete both heads; the fresh agent then reconciles directly while - // the stale agent's subgraph is the rebuild chain (meta_sync first). - for c in &heads { - q.complete_node(c.node_id, Ok(())); - } - let next = q.claim_ready(); - let mut kinds: Vec<(&str, &str)> = next - .iter() - .map(|c| (c.agent.as_str(), c.kind.as_str())) - .collect(); - kinds.sort_unstable(); + // The fold is a *declared* difference, readable the moment submit returns: + // both agents get a `SetWanted(Up)` group root, but the fresh agent's + // subgraph ends at the Reconcile behind it while the stale agent's carries + // the whole rebuild chain in between. assert_eq!( - kinds, - vec![("fresh", "reconcile"), ("stale", "meta_sync")], - "fresh agent starts directly; stale agent rebuilds first, all in one DAG" + declared_shape_for(&q, id, "fresh"), + vec![ + row("set_wanted", None, &[]), + row("reconcile", Some("set_wanted"), &[]), + ], + "a fresh agent is intent + convergence, nothing in between" + ); + assert_eq!( + declared_shape_for(&q, id, "stale"), + vec![ + row("set_wanted", None, &[]), + row("meta_sync", None, &[("set_wanted", "done")]), + row("prebuild", None, &[("meta_sync", "done")]), + row("stop_for_update", Some("prebuild"), &[]), + row("swap", Some("stop_for_update"), &[]), + row("post_swap", Some("stop_for_update"), &[("swap", "done")]), + row("reconcile", None, &[("prebuild", "done|failed|skipped")]), + ], + "a stale agent gets the whole rebuild chain wedged between intent and \ + convergence — same DAG, same head kind, more in the middle" ); } @@ -730,74 +773,6 @@ fn offline_agents_skip_mechanical_nodes_but_keep_reconcile() { ); } -#[test] -fn a_fanned_out_start_declares_the_lease_and_re_enters_its_reconciles_grant() { - // `Start` / `Stop` / `PostSwap` were lease-exempt *as kinds*, which was only - // safe because every construction site fans them out from inside a - // lease-holding ancestor. Now they declare the lease themselves. - // - // The contract says that costs nothing — a descendant re-enters the - // ancestor's grant instead of taking a fresh unit. That is exactly the sort - // of claim that is true until a node is used from a second site, so it is - // pinned here rather than argued: the fanned-out `Start` must (a) actually - // carry the declaration, (b) still run under its parent's grant, and - // (c) not have consumed a second unit of a cap-1 lease. - let q = JobQueue::new(4); - let id = submit( - &q, - templates::reconcile_only("agent-a", Source::Manual, "converge".to_owned()), - ); - // A competing DAG on the same agent, to prove the lease is genuinely held - // (and held *once*) across the fan-out. - let rival = submit( - &q, - templates::reconcile_only("agent-a", Source::Manual, "rival".to_owned()), - ); - - let reconcile = claim_one(&q); - assert_eq!(reconcile.dag_id, id); - assert_eq!(reconcile.kind.as_str(), "reconcile"); - - // What `run_reconcile` does on observing a down container with wanted=Up. - q.append_subgraph( - id, - Box::new(|b: &Job| { - let kind = NodeKind::Start { - agent: "agent-a".to_owned(), - }; - let lease = Resource::Agent(kind.agent().to_owned()); - let _ = b.node(kind).needs(lease); - }), - reconcile.node_id, - ); - q.complete_node(reconcile.node_id, Ok(())); - - // (a) + (b): the child runs, under the parent that parked in `Finishing`. - let start = claim_one(&q); - assert_eq!(start.kind.as_str(), "start"); - assert_eq!( - declared_resources(&q, start.node_id), - vec![Resource::Agent("agent-a".to_owned())], - "a fanned-out Start declares the lease it runs under" - ); - - // (c): one unit, not two. `claim_one` above already asserted the rival did - // not come back in the same pass; make the reason explicit. - assert!( - q.claim_ready().is_empty(), - "the rival DAG's Reconcile must still be blocked — the appended Start \ - borrowed the grant rather than acquiring a second unit" - ); - - q.complete_node(start.node_id, Ok(())); - // Subtree terminal → the grant releases and the rival finally runs. - let rival_reconcile = claim_one(&q); - assert_eq!(rival_reconcile.dag_id, rival); - q.complete_node(rival_reconcile.node_id, Ok(())); - assert_eq!(state_of(&q, id), State::Done); - assert_eq!(state_of(&q, rival), State::Done); -} - #[test] fn boot_sweep_nodes_declare_their_own_resources() { // Regression, and the reason it needs its own test: `workers::auto_update` @@ -809,7 +784,7 @@ fn boot_sweep_nodes_declare_their_own_resources() { // meta commit inside another node's staged deploy window. Nothing failed to // compile; only an exhaustive caller list would have caught it. let q = JobQueue::new(4); - let _id = submit( + let id = submit( &q, DagSpec { source: Source::AutoUpdate, @@ -825,16 +800,7 @@ fn boot_sweep_nodes_declare_their_own_resources() { }, ); - // Both are independent roots on disjoint resources, so both start at once. - let claims = q.claim_ready(); - let by_kind = |kind: &str| { - claims - .iter() - .find(|c| c.kind.as_str() == kind) - .unwrap_or_else(|| panic!("no {kind} claim in {claims:?}")) - }; - - let mut lock = declared_resources(&q, by_kind("meta_lock").node_id); + let mut lock = declared_resources(&q, node_of(&q, id, "meta_lock")); lock.sort_by_key(|r| format!("{r:?}")); assert_eq!( lock, @@ -843,88 +809,12 @@ fn boot_sweep_nodes_declare_their_own_resources() { ); assert_eq!( - declared_resources(&q, by_kind("reconcile").node_id), + declared_resources(&q, node_of(&q, id, "reconcile")), vec![Resource::Agent("drifted-agent".to_owned())], "a boot Reconcile touches the container, so it holds that agent's lease" ); } -#[test] -fn append_subgraph_roots_on_emitter_and_rebases_local_deps() { - // The startup-sweep mechanism: a `MetaLock` emitter grows one rebuild - // subgraph per stale agent into its OWN DAG. Each subgraph is rooted on - // the emitter and its LOCAL 0-based deps are rebased onto the DAG. - let q = JobQueue::new(4); - let spec = DagSpec { - source: Source::AutoUpdate, - reason: "sweep".to_owned(), - declare: Box::new(|b: &Job| { - let _lock = b.node(NodeKind::MetaLock { - sweep: true, - fanout: None, - inputs: Vec::new(), - }); - }), - }; - let id = submit(&q, spec); - let emitter = claim_one(&q); - assert_eq!(emitter.kind.as_str(), "meta_lock"); - // Two independent per-agent subgraphs — the REAL production shape the - // sweep MetaLock grows: root MetaSync → root Prebuild → Signal → Drain → - // StopForUpdate → Swap → Reconcile, local 0-based deps. `graceful` must - // match the sweep arm of `run_meta_lock` or this stops tracking production. - let subgraph = |agent: &str| -> Declare { - let agent = agent.to_owned(); - Box::new(move |b: &Job| { - templates::rebuild_nodes( - b, - &agent, - templates::RebuildOpts { - relock: true, - graceful: true, - }, - None, - ); - }) - }; - // Must append BEFORE completing the emitter (the documented contract). - q.append_subgraph(id, subgraph("a"), emitter.node_id); - q.append_subgraph(id, subgraph("b"), emitter.node_id); - q.complete_node(emitter.node_id, Ok(())); - // Still ONE DAG; both subgraph roots become ready once the emitter is - // Done (rooted on it), each on its own agent lease. Their `MetaSync` heads - // take turns on the cap-1 global meta window, so drain those first — what - // must be concurrent is the builds. - assert_eq!(q.snapshot().len(), 1); - let mut kinds = drain_meta_syncs(&q); - kinds.sort_unstable(); - assert_eq!( - kinds, - vec![ - ("a".to_owned(), "prebuild".to_owned()), - ("b".to_owned(), "prebuild".to_owned()) - ], - "both rebuild subgraphs root on the emitter and run concurrently in one DAG" - ); -} - -/// Complete every `MetaSync` head the queue offers (they take turns on the -/// cap-1 global meta window) and return whatever else got claimed alongside -/// them, as `(agent, kind)` pairs left in flight. -fn drain_meta_syncs(q: &JobQueue) -> Vec<(String, String)> { - let mut rest = Vec::new(); - for _ in 0..3 { - for c in q.claim_ready() { - if c.kind.as_str() == "meta_sync" { - q.complete_node(c.node_id, Ok(())); - } else { - rest.push((c.agent.clone(), c.kind.as_str().to_owned())); - } - } - } - rest -} - /// Crash-watch suppression for a cascade rebuild, which the deleted half of /// `meta_update_grows_cascade_in_dag` used to assert via `DagSpec::transient`. /// @@ -964,193 +854,201 @@ fn rebuild_chain_nodes_suppress_crash_watch() { ); } -#[test] -fn meta_update_grows_cascade_in_dag() { - // The meta-update `MetaLock` grows one rebuild subgraph per affected - // agent into its OWN DAG (via append_subgraph), not child DAGs. - let spec = templates::meta_update( - vec!["nixpkgs".to_owned()], - Source::Manual, - "bump".to_owned(), - None, - ); - let q = JobQueue::new(4); - let id = submit(&q, spec); - let meta_lock = claim_one(&q); - assert_eq!(meta_lock.kind.as_str(), "meta_lock"); - // Simulate the executor growing the cascade in-DAG (`relock = false` — a - // cascade child must not re-lock and revert the parent's bump). - for agent in ["alice", "bob"] { - let declare: Declare = Box::new(move |b: &Job| { - templates::rebuild_nodes( - b, - agent, - templates::RebuildOpts { - relock: false, - graceful: false, - }, - None, - ); - }); - q.append_subgraph(id, declare, meta_lock.node_id); - } - q.complete_node(meta_lock.node_id, Ok(())); - // Still ONE DAG — no child DAGs — and both cascade rebuild subgraphs root - // on the MetaLock, each on its own agent lease. The per-agent `MetaSync` - // heads serialize on the global meta window (they commit to the meta repo); - // the builds behind them do not. - assert_eq!(q.snapshot().len(), 1); - let mut kinds = drain_meta_syncs(&q); - kinds.sort_unstable(); - assert_eq!( - kinds, - vec![ - ("alice".to_owned(), "prebuild".to_owned()), - ("bob".to_owned(), "prebuild".to_owned()) - ], - "cascade rebuilds grow in the meta-update DAG, concurrent per agent" - ); -} - // ---- failure: cancel-downstream + AfterAny ---- - -#[test] -fn failed_node_cancels_downstream_but_afterany_reconcile_runs() { - let q = JobQueue::new(1); - let id = submit(&q, rebuild("agent-a", "r")); - let meta_sync = claim_one(&q); - assert_eq!(meta_sync.kind.as_str(), "meta_sync"); - q.complete_node(meta_sync.node_id, Ok(())); - let prebuild = claim_one(&q); - assert_eq!(prebuild.kind.as_str(), "prebuild"); - q.complete_node(prebuild.node_id, Err("nix build exploded".to_owned())); - // StopForUpdate + Swap are cancelled (AfterOk on a failed chain); - // the AfterAny Reconcile still runs once Swap is terminal. - let reconcile = claim_one(&q); - assert_eq!(reconcile.kind.as_str(), "reconcile"); - q.complete_node(reconcile.node_id, Ok(())); - let snap = q.snapshot(); - let dag = snap.iter().find(|d| d.id == id).expect("dag"); - assert_eq!(dag.rollup_state(), State::Failed, "roll-up failed"); - let by_kind = |k: &str| { - dag.nodes - .iter() - .find(|n| n.kind == k) - .expect("node present") - .state - }; - assert_eq!(by_kind("prebuild"), State::Failed); - // `StopForUpdate` / `Swap` / `PostSwap` were *ruled out* by the failed - // `Prebuild`. They ride the wire as `Skipped` so an operator can see which - // steps the run never reached, without them reading as failures of their - // own — the roll-up ignores `Skipped` entirely. - for ruled_out in ["stop_for_update", "swap", "post_swap"] { - assert_eq!( - by_kind(ruled_out), - State::Skipped, - "{ruled_out} was ruled out, so it is on the wire as skipped" - ); - } - // The AfterAny reconcile ran (claimed + completed Ok above) → it's `Done`, - // and `Done` nodes are excluded from the wire, so it's absent here. - assert!( - dag.nodes.iter().all(|n| n.kind != "reconcile"), - "the completed (Done) reconcile is filtered off the wire" - ); - assert_eq!( - dag.nodes - .iter() - .find(|n| n.kind == "prebuild") - .and_then(|n| n.error.as_deref()), - Some("nix build exploded") - ); -} - -/// The swap-failure recovery: `Swap` fails → the `AfterOk` `PostSwap` is -/// cancel-cascaded → its terminal state still satisfies `Reconcile`'s -/// `AfterAny(PostSwap)` edge, so recovery-start runs and brings a wanted-up -/// agent back on its old config. -#[test] -fn swap_failure_still_runs_reconcile() { - let q = JobQueue::new(1); - let id = submit(&q, rebuild("agent-a", "r")); - // meta_sync + prebuild + stop_for_update - for _ in 0..3 { - let c = claim_one(&q); - q.complete_node(c.node_id, Ok(())); - } - let swap = claim_one(&q); - assert_eq!(swap.kind.as_str(), "swap"); - q.complete_node(swap.node_id, Err("update failed".to_owned())); - // PostSwap (AfterOk on the failed Swap) is cancel-cascaded; Reconcile is - // next-claimable via its AfterAny(PostSwap) edge. - let reconcile = claim_one(&q); - assert_eq!(reconcile.kind.as_str(), "reconcile"); - q.complete_node(reconcile.node_id, Ok(())); - let all_dags = q.snapshot(); - let dag = all_dags.iter().find(|d| d.id == id).expect("dag"); - assert_eq!( - dag.nodes - .iter() - .find(|n| n.kind == "post_swap") - .expect("post_swap node") - .state, - State::Skipped, - "PostSwap is ruled out by the failed Swap, and says so on the wire" - ); - assert_eq!( - dag.nodes - .iter() - .find(|n| n.kind == "swap") - .expect("swap node") - .state, - State::Failed, - "and the failure that ruled it out is still on the wire" - ); - assert_eq!(state_of(&q, id), State::Failed); -} +// +// `failed_node_cancels_downstream_but_afterany_reconcile_runs` lived here. It +// drove a rebuild to a failed `Prebuild` and then asserted three unrelated +// things at once, which is why it needed a running scheduler at all: +// +// 1. the cascade — a failed node cancels its `AfterOk` dependants while the +// `AfterAny` reconcile still runs. That is hive_jobq's rule, and it owns +// the test: `failed_after_ok_dep_cancels_dependents_but_after_any_runs`. +// c0re's declaration of *which* edge is which is asserted in +// `rebuild_chain_is_declared_serial` — the reconcile's accepted outcome +// set is right there in the shape table. +// 2. the wire projection — `Done` off, `Skipped` on. That is +// `shown_on_wire`, tested directly above. +// 3. the roll-up — a DAG with a failed node reads `Failed`, and `Skipped` +// contributes nothing. That is `DagView::rollup_state`, which lives in +// hive-host-sock and is now tested there, next to the invariant. +// +// Reconstructing all three from one arranged run made none of them +// individually legible, and the arrangement was the only reason this module +// needed to claim and complete nodes. /// The swap-success path: `Swap` ok → the `AfterOk` `PostSwap` (bookkeeping /// tail) runs, and only then does `Reconcile` fire — serialized behind /// `PostSwap` (not racing it) because `Reconcile` deps `AfterAny(PostSwap)`. #[test] -fn swap_ok_runs_post_swap_before_reconcile() { +fn rebuild_reconcile_waits_for_the_whole_build_subtree() { + // Replaces `swap_ok_runs_post_swap_before_reconcile` and + // `swap_failure_still_runs_reconcile`, which walked the same DAG with the + // swap succeeding in one and failing in the other. + // + // The interesting claim was "Reconcile must wait for PostSwap, not race + // it" — and it does *not* come from an edge between them. `reconcile` deps + // `AfterAny(prebuild)`, while `post_swap` sits inside prebuild's subtree + // (post_swap → stop_for_update → prebuild). A parent is not terminal until + // its subtree is, so prebuild cannot satisfy that edge while post_swap is + // outstanding. **The ordering is the parent chain, not a dependency.** + // + // Both facts are asserted in `rebuild_chain_is_declared_serial`; this test + // states the derived property explicitly because the indirection is the + // easy thing to break — someone flattening the chain would keep every edge + // and still lose the guarantee. let q = JobQueue::new(1); let id = submit(&q, rebuild("agent-a", "r")); - // meta_sync + prebuild + stop_for_update - for _ in 0..3 { - let c = claim_one(&q); - q.complete_node(c.node_id, Ok(())); - } - let swap = claim_one(&q); - assert_eq!(swap.kind.as_str(), "swap"); - q.complete_node(swap.node_id, Ok(())); - // PostSwap runs next, and nothing else is claimable while it does — the - // tail serializes ahead of Reconcile. - let post_swap = claim_one(&q); - assert_eq!(post_swap.kind.as_str(), "post_swap"); - assert!( - q.claim_ready().is_empty(), - "Reconcile must wait for PostSwap, not race it" + let shape = declared_shape(&q, id); + let parent_of = |kind: &str| { + shape + .iter() + .find(|d| d.kind == kind) + .unwrap_or_else(|| panic!("{kind} node")) + .parent + }; + assert_eq!(parent_of("post_swap"), Some("stop_for_update")); + assert_eq!(parent_of("stop_for_update"), Some("prebuild")); + assert_eq!( + shape + .iter() + .find(|d| d.kind == "reconcile") + .expect("reconcile node") + .after, + vec![("prebuild", "done|failed|skipped".to_owned())], + "reconcile gates on prebuild's roll-up, which covers the whole build \ + subtree — including post_swap — and runs on failure too" ); - q.complete_node(post_swap.node_id, Ok(())); - let reconcile = claim_one(&q); - assert_eq!(reconcile.kind.as_str(), "reconcile"); - q.complete_node(reconcile.node_id, Ok(())); - settle_rebuild_tail(&q, "agent-a", true); - assert_eq!(state_of(&q, id), State::Done); } +// `swap_failure_still_runs_reconcile` lived here. +// +// It asserted that a failed swap leaves `post_swap` `Skipped` and `swap` +// `Failed`, and that reconcile still runs. All three are hive_jobq's cascade +// (`failed_after_ok_dep_cancels_dependents_but_after_any_still_runs`), and the +// "says so on the wire" half turned out to be nothing: `snapshot` fills +// `NodeView { state: node.state, .. }`, a straight copy of the same `State` +// type, so there is no c0re-side mapping to get wrong. +// +// `failed_reconcile_marks_dag_failed` lived here too — a one-node DAG whose +// node fails, asserting the DAG reads `Failed`. That is `failed_child_rolls_ +// parent_up_to_failed` in hive_jobq, restated through a c0re template. + +// `multi_agent_lease_frees_per_subgraph_not_whole_dag` and +// `dag_settles_terminal_and_releases_lease_after_work` lived here. +// +// Both drove a DAG to completion to watch an agent lease free up — one when a +// single agent's subgraph settled inside a still-running multi-agent DAG, the +// other when a whole power op finished. Releasing a grant once its owner's +// subtree is terminal is hive_jobq's (`owner_holds_grant_for_its_whole_subtree`, +// `child_borrows_ancestor_grant_released_when_subtree_done`, +// `leaf_owner_goes_done_directly_and_releases`). +// +// The c0re halves are declared and asserted elsewhere: that each agent's +// subgraph is an independent root holding only its own lease is in +// `multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs`, and +// that a power op emits no tail node is in +// `cancelled_power_op_runs_no_compensating_node`, which checks the DAG has no +// pending nodes left at all. + +/// `Start` / `Stop` were lease-exempt *as kinds*, which was only safe because +/// every construction site fans them out from inside a lease-holding ancestor. +/// They declare the lease themselves now, and this pins that they do. +/// +/// Was `a_fanned_out_start_declares_the_lease_and_re_enters_its_reconciles_ +/// grant`, which submitted a `Reconcile`, claimed it, and then **re-declared +/// the fan-out inline** — *"same two calls the scheduler makes"*. That is a +/// copy of production in a test: had `exec.rs` stopped declaring the lease, it +/// would have kept passing. The declaration now lives in `templates:: +/// fanned_out_mechanical`, so this calls the real thing. +/// +/// The other half of the old test — that a descendant *re-enters* its +/// ancestor's grant rather than taking a second unit of a cap-1 lease — is +/// `hive_jobq`'s, and is tested there by +/// `child_borrows_ancestor_grant_released_when_subtree_done` and +/// `nested_borrowers_never_deadlock`. #[test] -fn failed_reconcile_marks_dag_failed() { - let q = JobQueue::new(1); +fn a_fanned_out_mechanical_node_declares_its_agent_lease() { + let q = JobQueue::new(4); let id = submit( &q, - templates::reconcile_only("agent-a", Source::Manual, "start".to_owned()), + DagSpec { + source: Source::Manual, + reason: "fan-out".to_owned(), + declare: Box::new(|b: &Job| { + templates::fanned_out_mechanical( + b, + NodeKind::Start { + agent: "agent-a".to_owned(), + }, + ); + }), + }, + ); + assert_eq!(declared_shape(&q, id), vec![row("start", None, &[])]); + assert_eq!( + declared_resources(&q, node_of(&q, id, "start")), + vec![Resource::Agent("agent-a".to_owned())], + "the fanned-out node carries the lease itself, rather than relying on \ + whoever happened to fan it out" + ); +} + +/// A running `MetaLock` grows one rebuild subgraph per agent into **its own +/// DAG**, rooted on itself — not as child DAGs. That is what keeps a boot sweep +/// (or a meta-update cascade) one unit of work, with every rebuild building +/// against the lock the emitter just bumped. +/// +/// Replaces `grown_subgraph_roots_on_emitter_and_rebases_local_deps` and +/// `meta_update_grows_cascade_in_dag`, which differed only in `RebuildOpts` and +/// each minted a builder by hand to simulate the graft. What they were checking +/// is `templates::grown_rebuilds`, so this calls it. +/// +/// That the grafted work lands under the emitter, and that the emitter parks in +/// `Finishing` until it settles, is `hive_jobq`'s +/// (`a_completing_node_grows_the_work_it_declared`). +#[test] +fn a_meta_lock_grows_one_rebuild_subgraph_per_agent() { + let q = JobQueue::new(4); + let agents = vec!["alice".to_owned(), "bob".to_owned()]; + let id = submit( + &q, + DagSpec { + source: Source::AutoUpdate, + reason: "sweep".to_owned(), + declare: Box::new(move |b: &Job| { + templates::grown_rebuilds( + b, + &agents, + templates::RebuildOpts { + relock: true, + graceful: true, + }, + ); + }), + }, + ); + + // One chain per agent, each an independent group root — so the two rebuild + // concurrently, each on its own lease. + let shape = declared_shape(&q, id); + let heads: Vec<_> = shape + .iter() + .filter(|d| d.kind == "meta_sync") + .map(|d| d.parent) + .collect(); + assert_eq!(heads, vec![None, None], "one root chain per agent"); + assert_eq!( + shape.iter().filter(|d| d.kind == "prebuild").count(), + 2, + "both agents get their own build" + ); + // `graceful: true` is the sweep's distinguishing knob — agents mid-turn + // when the host came up get their drain window rather than being cut off. + assert_eq!( + shape.iter().filter(|d| d.kind == "drain").count(), + 2, + "a boot sweep is graceful, so each agent gets a drain" ); - let c = claim_one(&q); - q.complete_node(c.node_id, Err("start failed".to_owned())); - assert_eq!(state_of(&q, id), State::Failed); } // ---- cancel ---- @@ -1167,12 +1065,13 @@ fn cancel_clears_queued_dag() { assert_eq!(state_of(&q, id), State::Cancelled, "no stale Queued gap"); // Neither `EmitRebuilt` tail accepts a *dropped* dependency — the ok one is // `AFTER_OK`, the failure one keys on elimination — so both are cancelled - // with the work and **nothing is emitted** for a rebuild that never ran. + // with the work and **nothing is left that could still run**: no node is + // spared, so a rebuild that never ran emits nothing. assert!( - q.claim_ready().is_empty(), - "a dropped rebuild reports nothing" + pending_kinds(&q, id).is_empty(), + "a dropped rebuild leaves nothing alive, got {:?}", + pending_kinds(&q, id) ); - assert_eq!(state_of(&q, id), State::Cancelled); } /// `cancel` takes a **node** id, not a DAG id — so an interior node can be @@ -1198,22 +1097,16 @@ fn cancel_drops_one_agents_branch_leaving_the_rest() { assert!(q.cancel(a_root.id), "an interior/group root cancels alone"); - // agent-b's work is untouched and still claimable; agent-a's is not. - let claims = q.claim_ready(); + // agent-a's subgraph is gone; agent-b's is untouched and still alive. assert!( - !claims.is_empty() && claims.iter().all(|c| c.agent == "agent-b"), - "only agent-b remains runnable, got {:?}", - claims.iter().map(|c| c.agent.as_str()).collect::>() + pending_kinds_for(&q, id, "agent-a").is_empty(), + "agent-a's branch was dropped whole, got {:?}", + pending_kinds_for(&q, id, "agent-a") + ); + assert!( + !pending_kinds_for(&q, id, "agent-b").is_empty(), + "agent-b's branch survives its sibling's cancel" ); -} - -#[test] -fn cancel_refuses_running_dag() { - let q = JobQueue::new(1); - let id = submit(&q, rebuild("agent-a", "r")); - let _ = claim_one(&q); - assert!(!q.cancel(id)); - assert_eq!(state_of(&q, id), State::Running); } /// A cancelled power op must run **no** compensating node — not even one that @@ -1222,105 +1115,72 @@ fn cancel_refuses_running_dag() { /// Now structural rather than a property of a hook enum: a power op emits no /// tail node at all, so once its work nodes cancel there is simply nothing left /// to claim. `cancel` also refuses unless every work node is still `Pending` -/// (`cancel_refuses_running_dag`), so a `Cancelled` DAG provably never executed -/// a node: its `SetWanted` never ran and the agent's intent still reads whatever +/// (`hive_jobq`'s `cancel_node_refuses_a_group_with_anything_running`), so a +/// `Cancelled` DAG provably never executed a node: its `SetWanted` never ran +/// and the agent's intent still reads whatever /// the operator last set. A "revert" instead writes the agent's *observed* /// state, which for a down-but-`wanted = Up` agent (crashed, or caught /// mid-bounce) flips the intent to `Offline` and leaves it /// deliberately-stopped as far as reconcile and crash-watch are concerned. #[test] fn cancelled_power_op_runs_no_compensating_node() { + /// Submit-cancel-assert for one power op. Taking the already-submitted DAG + /// id is what removes the need to put three differently-typed recipes in + /// one array: each caller submits its own spec, so no closure type has to + /// be erased to a boxed one. + fn assert_cancels_clean(q: &JobQueue, id: u64, writes_intent: bool, case: &str) { + // Read the intent head off the submitted DAG rather than out of the + // spec: a declared job holds its own nodes and inserts them. + let has_intent = declared_shape(q, id).iter().any(|d| d.kind == "set_wanted"); + assert_eq!(has_intent, writes_intent, "{case}: intent head"); + assert!(q.cancel(id), "{case}: cancelled while queued"); + assert_eq!(state_of(q, id), State::Cancelled); + // Nothing is left that *could* run. Asserting on the pending set rather + // than on "what is ready this instant" also covers a node that is alive + // but blocked — which is exactly what a leftover compensating node + // would look like. + assert_eq!( + pending_kinds(q, id), + Vec::<&str>::new(), + "{case}: a power op emits no tail node, so a cancelled one leaves nothing" + ); + } + for graceful in [false, true] { for running in [false, true] { let targets = vec![("agent-a".to_owned(), running)]; - // Erased to `DagSpec`: three different recipe types have to - // sit in one array. - let cases = [ - ( - "restart", - false, - erase(submit::restart_spec( - &targets, - graceful, - Source::Manual, - "bounce".to_owned(), - )), + let case = format!("graceful={graceful} running={running}"); + + let q = JobQueue::new(1); + let id = submit( + &q, + submit::restart_spec(&targets, graceful, Source::Manual, "bounce".to_owned()), + ); + assert_cancels_clean(&q, id, false, &format!("restart {case}")); + + let q = JobQueue::new(1); + let id = submit( + &q, + submit::stop_spec(&targets, graceful, Source::Manual, "stop".to_owned()), + ); + assert_cancels_clean(&q, id, true, &format!("stop {case}")); + + let q = JobQueue::new(1); + let id = submit( + &q, + submit::start_spec( + &[("agent-a".to_owned(), running, false)], + Source::Manual, + "start".to_owned(), ), - ( - "stop", - true, - erase(submit::stop_spec( - &targets, - graceful, - Source::Manual, - "stop".to_owned(), - )), - ), - ( - "start", - true, - erase(submit::start_spec( - &[("agent-a".to_owned(), running, false)], - Source::Manual, - "start".to_owned(), - )), - ), - ]; - for (name, writes_intent, spec) in cases { - let q = JobQueue::new(1); - let id = submit(&q, spec); - // Read the intent head off the submitted DAG rather than out of - // the spec: a declared job holds its own nodes and inserts them. - assert_eq!( - q.snapshot() - .iter() - .find(|d| d.id == id) - .expect("submitted dag") - .nodes - .iter() - .any(|n| n.kind == "set_wanted"), - writes_intent, - "{name} intent head (graceful={graceful}, running={running})" - ); - assert!(q.cancel(id), "cancelled while queued"); - assert_eq!(state_of(&q, id), State::Cancelled); - assert!( - q.claim_ready().is_empty(), - "cancelled {name} (graceful={graceful}, running={running}) must \ - leave nothing to run — a power op emits no tail node" - ); - } + ); + assert_cancels_clean(&q, id, true, &format!("start {case}")); } } } // ---- terminal reporting + lease release ---- -#[test] -fn dag_settles_terminal_and_releases_lease_after_work() { - let q = JobQueue::new(1); - let id = submit(&q, restart_online(&["agent-a"], false, "r")); - // restart = StopForUpdate → Reconcile. - let stop = claim_one(&q); - assert_eq!(stop.kind.as_str(), "stop_for_update"); - q.complete_node(stop.node_id, Ok(())); - let rec = claim_one(&q); - assert_eq!(rec.kind.as_str(), "reconcile"); - // Completing the last work node rolls the container up terminal. A power op - // has no tail node, so nothing is left to claim. - q.complete_node(rec.node_id, Ok(())); - assert!(q.claim_ready().is_empty(), "no tail node to claim"); - assert_eq!(state_of(&q, id), State::Done); - // Lease released when the work chain settled: a new DAG for the agent claims - // immediately. - let next = submit( - &q, - templates::reconcile_only("agent-a", Source::Manual, "stop".to_owned()), - ); - let c = claim_one(&q); - assert_eq!(c.dag_id, next); -} - /// A DAG cancelled while fully queued must still **run its tail**, or a queued /// approval DAG cancelled by the operator would dangle its approval forever. /// @@ -1338,15 +1198,25 @@ fn cancelled_dag_still_runs_its_approval_tail() { ); assert!(q.cancel(id), "fully-queued dag cancels"); // The `Cancelled` tail is the only node whose edge accepts a dropped - // dependency, so it is the only one `cancel` spares — and claiming it *is* - // the assertion that the approval gets resolved as cancelled. - settle_approval_tail(&q, 7, TerminalState::Cancelled); + // dependency, so it is the only one `cancel` spares — and *which* tail + // survives is the whole assertion: the template emits one per outcome and + // the spared one names how the approval row is about to be resolved. + // Nothing computes it, so reading the survivor is reading the answer. + let spared = pending_payloads(&q, id); + assert!( + matches!( + spared.as_slice(), + [NodeKind::ResolveApproval { + approval_id: 7, + outcome: TerminalState::Cancelled + }] + ), + "only the cancelled-outcome tail is spared, got {spared:?}" + ); assert_eq!(state_of(&q, id), State::Cancelled); - // Unrelated later activity doesn't disturb the settled DAG. - let other = submit(&q, rebuild("agent-b", "r")); - let c = claim_one(&q); - assert_eq!(c.dag_id, other); - q.complete_node(c.node_id, Err("boom".to_owned())); + // An unrelated DAG landing in the same graph doesn't disturb this one's + // roll-up — the snapshot is per-DAG, not a global state machine. + let _other = submit(&q, rebuild("agent-b", "r")); assert_eq!(state_of(&q, id), State::Cancelled); } @@ -1365,33 +1235,35 @@ fn deploy_dag_runs_phases_in_order_and_tails_a_failed_apply() { templates::approval_deploy("agent-a", 7, "approval #7".to_owned()), ); - let root = claim_one(&q); - assert!( - matches!(root.kind, NodeKind::DeployWindow { .. }), - "root claims first: it holds the meta window for the whole subtree" - ); - q.complete_node(root.node_id, Ok(())); - - let verify = claim_one(&q); - assert!(matches!(verify.kind, NodeKind::MergeVerify { .. })); - q.complete_node(verify.node_id, Ok(())); - - let apply = claim_one(&q); - assert!(matches!(apply.kind, NodeKind::DeployApply { .. })); - q.complete_node(apply.node_id, Err("nixos-container update blew up".into())); - - let tail = claim_one(&q); - assert!( - matches!(tail.kind, NodeKind::DeployTail { .. }), - "AfterAny tail runs on a failed apply — that's the whole point of it" - ); - q.complete_node(tail.node_id, Ok(())); - - settle_approval_tail(&q, 7, TerminalState::Failed); assert_eq!( - state_of(&q, id), - State::Failed, - "an Ok tail must not launder a failed deploy into a success" + declared_shape(&q, id), + vec![ + // The window is the group root and holds the meta window for the + // whole subtree; the three phases are its sub-nodes. + row("deploy_window", None, &[]), + row("merge_verify", Some("deploy_window"), &[]), + // Apply only on a clean verify — a failed verify cancel-cascades + // it, which is what leaves the forge and the applied repo untouched. + row( + "deploy_apply", + Some("deploy_window"), + &[("merge_verify", "done")] + ), + // The compensation tail accepts every terminal outcome of apply, + // *including `skipped`* — which is the state apply lands in when + // verify failed and it never ran. That one edge is the entire + // "still tails a failed apply / a failed verify" behaviour, and it + // is why two separate DAG-driving tests collapsed into this table. + row( + "deploy_tail", + Some("deploy_window"), + &[("deploy_apply", "done|failed|skipped")] + ), + // One approval tail per outcome, gated on the window's roll-up. + row("resolve_approval", None, &[("deploy_window", "done")]), + row("resolve_approval", None, &[("deploy_window", "failed")]), + row("resolve_approval", None, &[("deploy_window", "cancelled")]), + ] ); } @@ -1406,220 +1278,182 @@ fn deploy_dag_runs_phases_in_order_and_tails_a_failed_apply() { /// a resource its own DAG owns and deadlock — this test is what pins that down. #[test] fn deploy_apply_grows_rebuild_subgraph_and_finalizes_after_it() { + // What `DeployApply` grows is `deploy_rebuild_nodes`' output, and that is a + // pure declaration — so it is declared here directly rather than by running + // a deploy far enough to graft it. **Reproducing the runtime path is not + // needed to test what the runtime path declares.** + // + // The grafting mechanism itself is hive_jobq's and tested there: the work + // lands under the emitter *before* it settles, and the emitter parks in + // `Finishing` so a downstream `AfterAny` gate stays shut while the new + // children run (`a_completing_node_grows_the_work_it_declared`, + // `parent_parks_in_finishing_until_children_roll_up`). let q = JobQueue::new(1); let id = submit( &q, - templates::approval_deploy("agent-a", 11, "approval 11".to_owned()), + DagSpec { + source: Source::Manual, + reason: "deploy graft".to_owned(), + declare: Box::new(|b: &Job| templates::deploy_rebuild_nodes(b, "agent-a", 11)), + }, ); - let root = claim_one(&q); - assert!(matches!(root.kind, NodeKind::DeployWindow { .. })); - q.complete_node(root.node_id, Ok(())); - let verify = claim_one(&q); - q.complete_node(verify.node_id, Ok(())); - - let apply = claim_one(&q); - assert!(matches!(apply.kind, NodeKind::DeployApply { .. })); - // Mirrors the scheduler: the executor's `NodeOutput` subgraphs are grafted - // BEFORE the emitting node is completed. Completing first would settle the - // apply node `Done` with nothing under it, opening the tail's `AfterAny` - // gate immediately and letting the deploy "finish" before it had built. - q.append_subgraph( - id, - templates::deploy_rebuild_nodes("agent-a", 11), - apply.node_id, - ); - q.complete_node(apply.node_id, Ok(())); - - // The grafted chain runs in rebuild order. `claim_one` asserts exactly one - // claimable node at each step, which also proves the `AfterAny` tail stays - // shut: `DeployApply` is `Finishing` (not terminal) while its new children - // run, and `Finishing` satisfies neither dep kind. - for expected in [ - "meta_sync", - "prebuild", - "stop_for_update", - "swap", - "post_swap", - "reconcile", - ] { - let c = claim_one(&q); - assert_eq!(c.kind.as_str(), expected, "grafted phase order"); - q.complete_node(c.node_id, Ok(())); - } - - let finalize = claim_one(&q); - assert!( - matches!(finalize.kind, NodeKind::FinalizeDeploy { .. }), - "the deploy tag is planted only after the rebuild came up clean" - ); - q.complete_node(finalize.node_id, Ok(())); - - let tail = claim_one(&q); - assert!(matches!(tail.kind, NodeKind::DeployTail { .. })); - q.complete_node(tail.node_id, Ok(())); - - settle_approval_tail(&q, 11, TerminalState::Done); - assert_eq!(state_of(&q, id), State::Done); -} - -/// A failure *inside* the grafted rebuild is the failure mode the subgraph -/// growth introduces: the deploy is already merged and the container half-swapped. -/// `FinalizeDeploy` must be cancel-cascaded (its `AfterOk` gate never opens) so -/// no `deployed/` tag is planted, while the tail still runs to compensate. -/// `Reconcile` is deliberately still reached — it boots the container back up. -#[test] -fn deploy_dag_skips_finalize_but_still_tails_a_failed_graft() { - let q = JobQueue::new(1); - let id = submit( - &q, - templates::approval_deploy("agent-a", 13, "approval 13".to_owned()), - ); - - let root = claim_one(&q); - q.complete_node(root.node_id, Ok(())); - let verify = claim_one(&q); - q.complete_node(verify.node_id, Ok(())); - let apply = claim_one(&q); - q.append_subgraph( - id, - templates::deploy_rebuild_nodes("agent-a", 13), - apply.node_id, - ); - q.complete_node(apply.node_id, Ok(())); - - for expected in ["meta_sync", "prebuild", "stop_for_update"] { - let c = claim_one(&q); - assert_eq!(c.kind.as_str(), expected); - q.complete_node(c.node_id, Ok(())); - } - let swap = claim_one(&q); - assert_eq!(swap.kind.as_str(), "swap"); - q.complete_node(swap.node_id, Err("profile swap failed".into())); - - // `Reconcile` hangs off `Prebuild` with `AfterAny`, so a failed swap still - // reaches it — bringing the container back up is exactly what it's for. - let reconcile = claim_one(&q); - assert_eq!(reconcile.kind.as_str(), "reconcile"); - q.complete_node(reconcile.node_id, Ok(())); - - let tail = claim_one(&q); - assert!( - matches!(tail.kind, NodeKind::DeployTail { .. }), - "finalize is cancel-cascaded, so the tail is the next claimable node" - ); - q.complete_node(tail.node_id, Ok(())); - - settle_approval_tail(&q, 13, TerminalState::Failed); - assert_eq!(state_of(&q, id), State::Failed); assert_eq!( - q.first_error(id).as_deref(), - Some("profile swap failed"), - "the tail annotates failed/ with this — and it is also what - `exec::failure_reason` falls back to, since the tail's own dep is a - group root that rolled up Failed and so carries no error itself" + declared_shape(&q, id), + vec![ + row("meta_sync", None, &[]), + row("prebuild", None, &[("meta_sync", "done")]), + row("stop_for_update", Some("prebuild"), &[]), + row("swap", Some("stop_for_update"), &[]), + row("post_swap", Some("stop_for_update"), &[("swap", "done")]), + row("reconcile", None, &[("prebuild", "done|failed|skipped")]), + // The deploy tag is planted only after the rebuild came up clean: + // `AfterOk` on **both** roots, so either one failing skips it. That + // pair of edges is the whole "skips finalize on a failed graft" + // behaviour — no run needed to see it. + row( + "finalize_deploy", + None, + &[("prebuild", "done"), ("reconcile", "done")] + ), + ] ); } -/// A pre-merge rejection (drift gate, eval failure) cancel-cascades the -/// irreversible half via its `AfterOk` edge, but the tail is still reached — -/// it owns the forge mirror, not just compensation. -#[test] -fn deploy_dag_skips_apply_but_still_runs_tail_when_verify_fails() { - let q = JobQueue::new(1); - let id = submit( - &q, - templates::approval_deploy("agent-a", 9, "approval #9".to_owned()), - ); +// `deploy_dag_skips_finalize_but_still_tails_a_failed_graft` lived here. +// +// A failure *inside* the grafted rebuild is the failure mode subgraph growth +// introduces: the deploy is already merged and the container half-swapped, so +// `FinalizeDeploy` must be cancel-cascaded (no `deployed/` tag planted) +// while the tail still runs to compensate, and `Reconcile` is deliberately +// still reached — it boots the container back up. +// +// Every declared half of that is asserted by +// `deploy_apply_grows_rebuild_subgraph_and_finalizes_after_it`, which reads +// `deploy_rebuild_nodes`' shape directly: +// +// - "a failed swap still reaches Reconcile" is the `reconcile` row's +// `AfterAny` edge on `prebuild` (`done|failed|skipped`); +// - "finalize is cancel-cascaded" is `finalize_deploy`'s `AfterOk` pair — +// either root failing skips it; +// - "the tail still runs" is `deploy_tail`'s own `done|failed|skipped` edge +// on apply, asserted in the `approval_deploy` table above. +// +// The runtime halves are hive_jobq's: cascade on failure, roll-up, and +// `first_error` digging past a group root that rolled up `Failed` while +// carrying no error of its own (`first_error_skips_a_rolled_up_failure_ +// carrying_no_error`). That last one is why the DAG reports "profile swap +// failed" rather than nothing — the mechanism, not this shape. - let root = claim_one(&q); - q.complete_node(root.node_id, Ok(())); - let verify = claim_one(&q); - q.complete_node(verify.node_id, Err("PR head drifted since review".into())); +// `deploy_dag_skips_apply_but_still_runs_tail_when_verify_fails` lived here. +// +// A pre-merge rejection (drift gate, eval failure) cancel-cascades the +// irreversible half via its `AfterOk` edge, while the tail is still reached — +// it owns the forge mirror, not just compensation. That test and +// `deploy_dag_runs_phases_in_order_and_tails_a_failed_apply` differed only in +// *where* they injected the failure, and each drove the whole DAG to watch the +// tail run anyway. +// +// Both outcomes follow from one declared edge, which the surviving test now +// asserts directly: the tail accepts `done|failed|skipped` on apply, and +// `skipped` is exactly the state apply lands in when verify failed and it never +// ran. The runtime halves are hive_jobq's and tested there — +// `failed_after_ok_dep_cancels_dependents_but_after_any_still_runs` and +// `failed_child_rolls_parent_up_to_failed` (an Ok tail cannot launder a failed +// deploy into a success). - let tail = claim_one(&q); - assert!( - matches!(tail.kind, NodeKind::DeployTail { .. }), - "apply is cancel-cascaded, so the tail is the next claimable node" - ); - q.complete_node(tail.node_id, Ok(())); - - settle_approval_tail(&q, 9, TerminalState::Failed); - assert_eq!(state_of(&q, id), State::Failed); -} - -// ---- build logs, history ---- - -#[test] -fn set_build_log_id_links_running_node() { - let q = JobQueue::new(1); - let id = submit(&q, rebuild("agent-a", "r")); - let c = claim_one(&q); - assert!(q.set_build_log_id(id, c.node_id, 42)); - q.complete_node(c.node_id, Ok(())); - assert!( - !q.set_build_log_id(id, c.node_id, 99), - "node no longer running → refused" - ); - // The log id is fetched by node id (the `GET /api/build-log/` lookup), - // not carried on the wire — it survives completion in the node runtime. - assert_eq!( - q.build_log_id_of(c.node_id.get()), - Some(42), - "log id survives completion" - ); -} +// ---- history ---- +// +// The node → build-log link is no longer queue state: the log row carries +// `node_id` and the lookup lives in `stores::build_logs` (see +// `node_link_survives_completion_and_newest_wins` there). Nothing in the queue +// needs testing for it any more, which is the point of that move. /// History retention is a **flat** newest-first cap over all terminal DAGs /// (`MAX_HISTORY_DAGS`), not a per-template bucket behind a grace window. /// The dashboard renders one recent-builds list, so one number bounds it — /// and with no bucketing there's nothing for a burst of same-shaped DAGs to /// evict early, which is what the grace window used to paper over. +/// +/// `retain_history` is a sort-and-truncate over `(handle, finished_at, +/// tiebreak)`. This used to submit `MAX_HISTORY_DAGS + 8` DAGs, claim and fail +/// each one's node, then read the ids back out of a snapshot — the scheduler, +/// the roll-up and the wire projection all in the path of a policy that reads +/// none of them. +/// +/// Calling it directly also reaches the half the round-trip never could: every +/// DAG in that loop settled inside the same wall-clock second, so `finished_at` +/// tied on all of them and **only** the tiebreak was ever exercised. Eviction +/// by time — the actual policy — went untested. #[test] -fn history_evicts_oldest_terminals_past_flat_cap() { - const OVERFLOW: usize = 8; - let q = JobQueue::new(1); - let mut ids = Vec::new(); - for i in 0..(MAX_HISTORY_DAGS + OVERFLOW) { - let id = submit( - &q, - templates::reconcile_only(&format!("agent-{i}"), Source::Manual, "start".to_owned()), - ); - let c = claim_one(&q); - // Fail the single work node so the DAG *lingers*: a fully-`Done` DAG - // drops off the wire entirely, but a `Failed` one is retained (+ - // history-capped) so the operator can still triage it. Completing the - // node rolls the container up terminal. - q.complete_node(c.node_id, Err("boom".to_owned())); - ids.push(id); - } - let kept: std::collections::HashSet = q.snapshot().iter().map(|d| d.id).collect(); - assert_eq!(kept.len(), MAX_HISTORY_DAGS, "flat history cap"); - // Newest-first: the oldest `OVERFLOW` fall off, everything after survives. - // These DAGs settle within the same wall-clock second, so this also pins - // the `NodeId`-descending tiebreak that orders them when `finished_at` ties. - for old in &ids[..OVERFLOW] { - assert!(!kept.contains(old), "oldest terminal {old} evicted"); - } - for recent in &ids[OVERFLOW..] { - assert!(kept.contains(recent), "recent terminal {recent} retained"); - } - assert_eq!(q.live_count(), 0); +fn history_retains_live_dags_and_the_newest_terminals() { + const CAP: usize = 3; + // Live DAGs are kept whole, regardless of the cap. + assert_eq!( + retain_history(vec!["live-a", "live-b"], vec![], CAP), + vec!["live-a", "live-b"], + ); + // Terminals: newest `finished_at` first, oldest evicted past the cap. + assert_eq!( + retain_history( + vec![], + vec![ + ("oldest", 100, 1), + ("newest", 400, 2), + ("middle", 200, 3), + ("later", 300, 4), + ], + CAP, + ), + vec!["newest", "later", "middle"], + "the oldest terminal falls off" + ); + // Same second → the tiebreak decides, descending, so the DAG inserted + // last wins. This is the *common* case: a burst settles together. + assert_eq!( + retain_history( + vec![], + vec![("a", 100, 1), ("b", 100, 2), ("c", 100, 3), ("d", 100, 4)], + CAP, + ), + vec!["d", "c", "b"], + ); + // A live DAG never competes with history for the cap. + assert_eq!( + retain_history( + vec!["live"], + vec![("a", 100, 1), ("b", 200, 2), ("c", 300, 3), ("d", 400, 4)], + CAP, + ), + vec!["live", "d", "c", "b"], + ); } #[test] -fn error_is_truncated() { - let q = JobQueue::new(1); - let id = submit(&q, rebuild("agent-a", "r")); - let c = claim_one(&q); - q.complete_node(c.node_id, Err("x".repeat(5000))); - let snap = q.snapshot(); - let err = snap.iter().find(|d| d.id == id).expect("dag").nodes[0] - .error - .clone() - .expect("error stored"); - assert!(err.chars().count() <= 2001, "truncated + ellipsis"); - assert!(err.ends_with('…')); +fn error_truncation_cuts_on_a_char_boundary() { + // `truncate_error` is a pure `&str -> String`. This used to submit a DAG, + // claim its head, fail it with a long string and read the error back out of + // a snapshot — four moving parts to observe one transform, and the DAG + // round-trip is covered by its own tests either way. + // + // Testing it directly also reaches the case the round-trip never did: the + // cap is a **byte** length, so a multibyte char straddling it would panic + // the slice. That boundary scan is the only non-obvious line in the fn and + // it had no coverage at all. + assert_eq!( + truncate_error("short"), + "short", + "under the cap is untouched" + ); + + let ascii = truncate_error(&"x".repeat(5000)); + assert!(ascii.ends_with('…')); + assert!(ascii.len() <= MAX_ERROR_LEN + '…'.len_utf8()); + + // 'é' is 2 bytes, so the 2000-byte cap lands mid-char. + let multibyte = truncate_error(&"é".repeat(5000)); + assert!(multibyte.ends_with('…')); + assert!(multibyte.len() <= MAX_ERROR_LEN + '…'.len_utf8()); } // ---- template shapes ---- @@ -1628,41 +1462,29 @@ fn error_is_truncated() { fn graceful_stop_shape_signal_drain_reconcile() { let q = JobQueue::new(1); let id = submit(&q, stop_online(&["agent-a"], true, "graceful")); - for expected in ["set_wanted", "signal", "drain", "reconcile"] { - let c = claim_one(&q); - assert_eq!(c.kind.as_str(), expected); - q.complete_node(c.node_id, Ok(())); - } - assert_eq!(state_of(&q, id), State::Done); -} - -#[test] -fn graceful_signal_and_drain_hold_no_build_slot() { - // A whole-hive graceful stop overlaps every drain even at - // buildSlots = 1 while a rebuild hogs the slot. - let q = JobQueue::new(1); - submit(&q, rebuild("builder", "slot hog")); - submit(&q, stop_online(&["agent-a"], true, "g")); - submit(&q, stop_online(&["agent-b"], true, "g")); - // All three DAG heads are build-slot-exempt, so they run at once. - let heads = q.claim_ready(); - let kinds: Vec<&str> = heads.iter().map(|c| c.kind.as_str()).collect(); - assert_eq!(kinds, vec!["meta_sync", "set_wanted", "set_wanted"]); - for c in &heads { - q.complete_node(c.node_id, Ok(())); - } - // Now the rebuild's Prebuild holds the single slot — and both graceful - // stops still proceed to their Signal beside it. - let kinds: Vec<&str> = q - .claim_ready() - .iter() - .map(|c| c.kind.as_str()) - .collect::>(); assert_eq!( - kinds, - vec!["prebuild", "signal", "signal"], - "both agents' graceful-stop signals (build-slot-exempt) run while the \ - rebuild holds the slot" + declared_shape(&q, id), + vec![ + // The whole stop hangs under `set_wanted`: the durable intent is + // written first, and the mechanical steps are its sub-nodes. + row("set_wanted", None, &[]), + row("signal", Some("set_wanted"), &[]), + row("drain", Some("set_wanted"), &[("signal", "done")]), + row("reconcile", Some("set_wanted"), &[("drain", "done")]), + ] + ); + // Neither half of the graceful window takes a build slot. That is what lets + // a whole-hive graceful stop overlap every agent's drain even at + // `buildSlots = 1` while a rebuild hogs the slot — the cost ceiling is one + // `GRACEFUL_STOP_TIMEOUT` in total, not one per agent. + let agent = Resource::Agent("agent-a".to_owned()); + assert_eq!( + [ + declared_resources(&q, node_of(&q, id, "signal")), + declared_resources(&q, node_of(&q, id, "drain")), + ], + [vec![agent.clone()], vec![agent]], + "signal and drain hold the lease but never a build slot" ); } @@ -1673,13 +1495,23 @@ fn spawn_shape_provision_create_dropin_reconcile() { &q, templates::spawn("newbie", 7, "approval #7 spawn".to_owned()), ); - for expected in ["provision", "create", "write_dropin", "reconcile"] { - let c = claim_one(&q); - assert_eq!(c.kind.as_str(), expected); - q.complete_node(c.node_id, Ok(())); - } - settle_approval_tail(&q, 7, TerminalState::Done); - assert_eq!(state_of(&q, id), State::Done); + assert_eq!( + declared_shape(&q, id), + vec![ + row("provision", None, &[]), + row("create", Some("provision"), &[]), + row("write_dropin", Some("create"), &[]), + row("reconcile", Some("create"), &[("write_dropin", "done")]), + // One tail per outcome, each edged to accept only that one — so + // *which* tail the graph lets run already is the answer, and + // nothing branches at runtime. The three differ **only** in their + // accepted outcome, which is why `declared_shape` spells the + // outcome set out instead of bucketing it. + row("resolve_approval", None, &[("provision", "done")]), + row("resolve_approval", None, &[("provision", "failed")]), + row("resolve_approval", None, &[("provision", "cancelled")]), + ] + ); } #[test] @@ -1697,21 +1529,25 @@ fn perm_change_shape_prefixes_rebuild_chain() { }, ), ); - for expected in [ - "write_perm_file", - "meta_sync", - "prebuild", - "stop_for_update", - "swap", - "post_swap", - "reconcile", - ] { - let c = claim_one(&q); - assert_eq!(c.kind.as_str(), expected); - q.complete_node(c.node_id, Ok(())); - } - settle_rebuild_tail(&q, "agent-a", true); - assert_eq!(state_of(&q, id), State::Done); + assert_eq!( + declared_shape(&q, id) + .iter() + .map(|d| d.kind) + .collect::>(), + vec![ + "write_perm_file", + "meta_sync", + "prebuild", + "stop_for_update", + "swap", + "post_swap", + "reconcile", + // the ok / !ok tail pair + "emit_rebuilt", + "emit_rebuilt", + ], + "the perm write prefixes an otherwise ordinary rebuild chain" + ); } #[test] @@ -1729,17 +1565,18 @@ fn reparent_shape_is_a_lone_agentless_meta_window_node() { "set-parent".to_owned(), ), ); - let c = claim_one(&q); - assert_eq!(c.kind.as_str(), "reparent"); - assert_eq!(c.agent, "", "Reparent is agentless — no per-agent lease"); assert_eq!( - declared_resources(&q, c.node_id), + declared_shape(&q, id), + vec![row("reparent", None, &[])], + "one node, no rebuild subgraph" + ); + let node = node_of(&q, id, "reparent"); + assert_eq!( + declared_resources(&q, node), vec![Resource::MetaWindow], "a topology commit must declare the same MetaWindow as WritePermFile, \ and nothing else — no lease (agentless), no build slot (no nix work)" ); - q.complete_node(c.node_id, Ok(())); - assert_eq!(state_of(&q, id), State::Done); } #[test] @@ -1753,12 +1590,13 @@ fn reparent_bulk_shape_carries_every_move_on_one_node() { &q, templates::reparent(moves.clone(), Source::Manual, "set-parent-bulk".to_owned()), ); - let c = claim_one(&q); - assert_eq!(c.kind.as_str(), "reparent"); - let NodeKind::Reparent { moves: got } = &c.kind else { - panic!("expected a Reparent node, got {:?}", c.kind); + assert_eq!( + declared_shape(&q, id), + vec![row("reparent", None, &[])], + "one node for the whole request, not one per move" + ); + let NodeKind::Reparent { moves: got } = payload_of(&q, id, "reparent") else { + panic!("expected a Reparent node"); }; - assert_eq!(got, &moves); - q.complete_node(c.node_id, Ok(())); - assert_eq!(state_of(&q, id), State::Done); + assert_eq!(got, moves, "every move rides the single node"); } diff --git a/hive-c0re/src/lifecycle/mod.rs b/hive-c0re/src/lifecycle/mod.rs index 6f816f02..a69b71a9 100644 --- a/hive-c0re/src/lifecycle/mod.rs +++ b/hive-c0re/src/lifecycle/mod.rs @@ -274,10 +274,10 @@ pub async fn swap_update( name: &str, hive: &HiveEnv, paths: &AgentPaths, - on_build_log_id: &(dyn Fn(i64) + Send + Sync), + node_id: Option, ) -> Result<()> { write_dropins(name, hive, paths).await?; - priv_run_inner("update", name, Some(on_build_log_id)).await + priv_run_inner("update", name, node_id).await } /// Build the `AgentSpec` list for the meta flake from `nixos-container @@ -582,14 +582,10 @@ pub async fn destroy(name: &str) -> Result<()> { /// the prebuild happens before stop, and `docs/coordinator.md::Prebuild /// attr path` for why the explicit nixosConfigurations attr is required. /// -/// `on_build_log_id` fires with the `build_logs` row id as soon as the -/// row opens, so queue-side callers can link their node to the live -/// stream. Pass `&|_| ()` when not needed. -pub async fn prebuild_toplevel( - name: &str, - flake_ref: &str, - on_build_log_id: &(dyn Fn(i64) + Send + Sync), -) -> Result<()> { +/// `node_id` is the queue node this build belongs to, when there is one — +/// it is stored on the `build_logs` row so the dashboard can find the log +/// from the node. Pass `None` for builds that run outside the queue. +pub async fn prebuild_toplevel(name: &str, flake_ref: &str, node_id: Option) -> Result<()> { use tokio::io::{AsyncBufReadExt, BufReader}; // Split `#` so we can re-emit with the explicit // `nixosConfigurations.` segment. The flake_ref shape is @@ -624,15 +620,12 @@ pub async fn prebuild_toplevel( // into the row; `finish` lands the terminal status before we bail. let logs = crate::build_logs::global(); let log_id = logs.as_ref().and_then(|h| { - h.start(name, "prebuild", &cmdline) + h.start(name, "prebuild", &cmdline, node_id) .map_err(|e| { tracing::warn!(error = ?e, "build_logs: start failed (prebuild log dropped)"); }) .ok() }); - if let Some(id) = log_id { - on_build_log_id(id); - } let mut child = Command::new("nix") .args(&args) @@ -784,37 +777,25 @@ async fn priv_run(kind: &str, name: &str) -> Result<()> { priv_run_inner(kind, name, None).await } -/// Like `priv_run` but calls `on_log_id(log_id)` immediately after the -/// build-log row is opened — before the actual container op starts. -/// This lets callers surface the row id for live streaming (e.g. the -/// rebuild-queue worker sets `build_log_id` on the queue entry so the -/// dashboard can link to `/api/build-logs/id/{id}/stream`). +/// Like `priv_run` but stamps `node_id` onto the build-log row it opens, so +/// the dashboard can find the log from the queue node (and link to +/// `/api/build-logs/id/{id}/stream`). /// -/// The callback fires only when a build-log row is successfully opened -/// (i.e. the global `BuildLogs` handle is installed AND `h.start()` -/// succeeds). No-op when `on_log_id` is `None` — that's the path for -/// all callers that don't need the id. -async fn priv_run_inner( - kind: &str, - name: &str, - on_log_id: Option<&(dyn Fn(i64) + Send + Sync)>, -) -> Result<()> { +/// This used to be a `Fn(i64)` callback that handed the row id *back* to the +/// queue, which then held it in a side map. The row carries the link itself +/// now, so the id only ever travels one way. +async fn priv_run_inner(kind: &str, name: &str, node_id: Option) -> Result<()> { let container = container_name(name); let cmdline = format!("nixos-container {kind} {container}"); let logs = crate::build_logs::global(); let log_id = logs.as_ref().and_then(|h| { - h.start(name, kind, &cmdline) + h.start(name, kind, &cmdline, node_id) .map_err(|e| { tracing::warn!(error = ?e, "build_logs: start failed (priv_run log dropped)"); }) .ok() }); - // Notify the caller as soon as the log row exists so it can surface - // the id for live streaming before the container op even starts. - if let (Some(id), Some(cb)) = (log_id, on_log_id) { - cb(id); - } // For long-running ops use the streaming protocol so build_logs // receives lines in real time rather than as a batch at completion. diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index 8c4100ba..6da608f3 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -1590,7 +1590,12 @@ async fn nix_logged(dir: &Path, args: &[&str], agent: &str, kind: &str) -> Resul let cmdline = format!("nix {}", nix_argv(args).join(" ")); let logs = crate::build_logs::global(); let log_id = logs.as_ref().and_then(|h| { - h.start(agent, kind, &cmdline) + // No node id: `nix_logged`'s two callers are meta-flake operations + // reached from outside the queue as well as from inside it, and the + // agent+kind+time listing is how they're surfaced today. Linking them + // to a node would mean threading the id through `meta`'s public API + // for no current reader — worth doing when something wants it. + h.start(agent, kind, &cmdline, None) .map_err(|e| { tracing::warn!(error = ?e, %kind, "build_logs: start failed (meta log dropped)"); }) diff --git a/hive-c0re/src/stores/build_logs.rs b/hive-c0re/src/stores/build_logs.rs index 24b302bd..67b28c3e 100644 --- a/hive-c0re/src/stores/build_logs.rs +++ b/hive-c0re/src/stores/build_logs.rs @@ -13,6 +13,8 @@ use serde::Serialize; use tokio::sync::broadcast; use utoipa::ToSchema; +use crate::db::Migration; + /// Process-singleton handle, set once at coordinator startup. Lets /// the `lifecycle` module's `run` / `prebuild_toplevel` access the /// writer without threading an `Arc` through every @@ -65,6 +67,27 @@ CREATE INDEX IF NOT EXISTS idx_build_logs_status_finished WHERE finished_at IS NOT NULL; "; +/// Ordered schema migrations tracked in `schema_versions` (key `"build_logs"`). +/// +/// v1 makes the log row carry its node, replacing the host-side +/// `NodeId -> build_log_id` map the job queue used to hold. The link has a +/// single home again, and the direction is the one the type system allows: +/// a `hive_jobq` node payload is immutable after insert, but the log row is +/// written when the build starts and can name the node it belongs to. +/// +/// Legacy rows keep `node_id IS NULL` — they predate the column and no node +/// still exists to link them to, so the dashboard's by-node lookup simply +/// misses them (the by-agent listing, which is how they're reached, is +/// unaffected). +const MIGRATIONS: &[Migration] = &[Migration { + sql: "BEGIN; + ALTER TABLE build_logs ADD COLUMN node_id INTEGER; + CREATE INDEX IF NOT EXISTS idx_build_logs_node + ON build_logs (node_id) WHERE node_id IS NOT NULL; + COMMIT;", + adds_column: Some(("build_logs", "node_id")), +}]; + /// Status of a finished build attempt. Stored as the literal string in /// the `status` column; `NULL` while the attempt is still in progress. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] @@ -152,6 +175,7 @@ impl BuildLogs { let conn = crate::db::open(&path, "build_logs")?; conn.execute_batch(SCHEMA) .context("apply build_logs schema")?; + crate::db::apply_versioned_migrations(&conn, "build_logs", MIGRATIONS)?; let (notify_tx, _) = broadcast::channel(NOTIFY_CAP); Ok(Self { conn: Mutex::new(conn), @@ -169,17 +193,49 @@ impl BuildLogs { /// Open a row for a new build attempt. Returns the assigned id /// — the caller threads it through `append_stdout` / `append_stderr` /// while the child runs and into `finish` once it exits. - pub fn start(&self, agent: &str, kind: &str, cmdline: &str) -> Result { + /// + /// `node_id` is the queue node this build belongs to, when there is one. + /// It is `None` for builds that run outside the job queue; those are + /// reachable by agent + time, just not by node. + pub fn start( + &self, + agent: &str, + kind: &str, + cmdline: &str, + node_id: Option, + ) -> Result { let now = Utc::now().timestamp(); + let node_id = node_id.and_then(|n| i64::try_from(n).ok()); let conn = self.conn.lock().unwrap(); conn.execute( - "INSERT INTO build_logs (agent, kind, cmdline, started_at) VALUES (?1, ?2, ?3, ?4)", - params![agent, kind, cmdline, now], + "INSERT INTO build_logs (agent, kind, cmdline, started_at, node_id) \ + VALUES (?1, ?2, ?3, ?4, ?5)", + params![agent, kind, cmdline, now, node_id], ) .context("insert build_logs row")?; Ok(conn.last_insert_rowid()) } + /// The most recent build-log row for `node_id`, if any. Replaces the job + /// queue's in-memory `NodeId -> build_log_id` side map: the link lives in + /// the row itself now, so it survives a restart and needs no lock held + /// alongside the scheduler's. + /// + /// `MAX(id)` rather than a uniqueness assumption — a node that is retried + /// opens a second row, and the newest is the one the panel should show. + #[must_use] + pub fn id_for_node(&self, node_id: u64) -> Option { + let node_id = i64::try_from(node_id).ok()?; + let conn = self.conn.lock().unwrap(); + conn.query_row( + "SELECT MAX(id) FROM build_logs WHERE node_id = ?1", + params![node_id], + |row| row.get::<_, Option>(0), + ) + .ok() + .flatten() + } + /// Append a single stdout line. Best-effort: errors are logged /// but never returned to the caller, so a transient sqlite blip /// never tears down a rebuild's stdout pump. @@ -453,7 +509,7 @@ mod tests { fn start_appends_finish_flow() { let (_d, db) = tmpdb(); let id = db - .start("alice", "prebuild", "nix build foo") + .start("alice", "prebuild", "nix build foo", None) .expect("start"); db.append_stdout(id, "building '/nix/store/abc.drv'"); db.append_stderr(id, "error: line 12"); @@ -482,9 +538,9 @@ mod tests { // assert id-ordering (autoincrement) is the tiebreaker — list // sorts by started_at DESC but the ORDER BY still produces the // last-inserted row first when timestamps match. - let id_a1 = db.start("alice", "run", "cmd one").expect("start"); - let _id_b = db.start("bob", "run", "cmd two").expect("start"); - let id_a2 = db.start("alice", "run", "cmd three").expect("start"); + let id_a1 = db.start("alice", "run", "cmd one", None).expect("start"); + let _id_b = db.start("bob", "run", "cmd two", None).expect("start"); + let id_a2 = db.start("alice", "run", "cmd three", None).expect("start"); db.finish(id_a1, BuildStatus::Ok); let alice_rows = db.list_recent_for_agent("alice", 10).expect("list"); @@ -515,10 +571,12 @@ mod tests { #[test] fn vacuum_drops_old_finished_only_per_status() { let (_d, db) = tmpdb(); - let id_fresh_fail = db.start("alice", "run", "fresh fail").expect("start"); - let id_old_fail = db.start("alice", "run", "old fail").expect("start"); - let id_old_ok = db.start("alice", "run", "old ok").expect("start"); - let id_running = db.start("alice", "run", "still running").expect("start"); + let id_fresh_fail = db.start("alice", "run", "fresh fail", None).expect("start"); + let id_old_fail = db.start("alice", "run", "old fail", None).expect("start"); + let id_old_ok = db.start("alice", "run", "old ok", None).expect("start"); + let id_running = db + .start("alice", "run", "still running", None) + .expect("start"); db.finish(id_fresh_fail, BuildStatus::Fail); db.finish(id_old_fail, BuildStatus::Fail); db.finish(id_old_ok, BuildStatus::Ok); @@ -550,6 +608,31 @@ mod tests { assert!(db.get_full(id_running).unwrap().is_some()); } + #[test] + fn node_link_survives_completion_and_newest_wins() { + // The queue used to hold this link in an in-memory side map, which + // meant it died with the process and needed the queue lock to read. + // On the row it outlives both the node's completion and a restart. + let (_d, db) = tmpdb(); + let first = db.start("alice", "swap", "cmd", Some(7)).expect("start"); + db.finish(first, BuildStatus::Fail); + assert_eq!( + db.id_for_node(7), + Some(first), + "link survives the build finishing" + ); + + // A retried node opens a second row; the panel wants the current + // attempt, not the first one. + let retry = db.start("alice", "swap", "cmd", Some(7)).expect("start"); + assert_eq!(db.id_for_node(7), Some(retry), "newest attempt wins"); + + // Builds that run outside the queue carry no node and are found by + // agent + time instead — they must not collide with node lookups. + db.start("alice", "run", "no node", None).expect("start"); + assert_eq!(db.id_for_node(999), None, "unknown node → no row"); + } + #[test] fn append_after_finish_still_appends() { // Defensive: if a child's stdout pump fires one last line @@ -557,7 +640,7 @@ mod tests { // append should land on the row (status already set, but the // log stays consistent with what happened). let (_d, db) = tmpdb(); - let id = db.start("alice", "run", "cmd").expect("start"); + let id = db.start("alice", "run", "cmd", None).expect("start"); db.finish(id, BuildStatus::Ok); db.append_stdout(id, "post-finish trailing line"); let full = db.get_full(id).expect("get").expect("Some"); diff --git a/hive-c0re/src/workers/auto_update.rs b/hive-c0re/src/workers/auto_update.rs index 572b455b..1e0a38e6 100644 --- a/hive-c0re/src/workers/auto_update.rs +++ b/hive-c0re/src/workers/auto_update.rs @@ -391,8 +391,7 @@ fn submit_boot_tree( n_skipped, ); - let declare: crate::job_queue::Declare = - Box::new(move |b| boot_nodes(b, any_stale, fanout, drifted)); + let declare = move |b: &crate::job_queue::Job| boot_nodes(b, any_stale, fanout, drifted); let spec = DagSpec { // The sweep's own rebuild subgraphs emit their `Rebuilt` events as they diff --git a/hive-host-sock/src/jobs.rs b/hive-host-sock/src/jobs.rs index feba036f..2f567bfd 100644 --- a/hive-host-sock/src/jobs.rs +++ b/hive-host-sock/src/jobs.rs @@ -206,3 +206,106 @@ impl DagView { } } } + +#[cfg(test)] +mod tests { + use chrono::Utc; + + use super::{DagView, NodeView, Source, State}; + + /// A node set carrying nothing but the states — the only input + /// `rollup_state` reads. + fn dag(states: &[State]) -> DagView { + DagView { + id: 1, + source: Source::Manual, + reason: "test".to_owned(), + created_at: Utc::now(), + started_at: None, + finished_at: None, + nodes: states + .iter() + .enumerate() + .map(|(i, &state)| NodeView { + id: i as u64, + agent: "a".to_owned(), + kind: "reconcile".to_owned(), + deps: Vec::new(), + state, + started_at: None, + finished_at: None, + error: None, + approval_id: None, + inputs: Vec::new(), + build_log_id: None, + parent: None, + }) + .collect(), + } + } + + #[test] + fn a_failure_outranks_everything_and_skipped_counts_for_nothing() { + // The case this replaces used to be arranged in hive-c0re by running a + // rebuild until its Prebuild failed. Only the states ever mattered. + assert_eq!( + dag(&[State::Done, State::Failed, State::Skipped]).rollup_state(), + State::Failed + ); + // A failure wins even against a node still going — the DAG's verdict + // is already decided. + assert_eq!( + dag(&[State::Running, State::Failed]).rollup_state(), + State::Failed + ); + // Skipped is an expected part of a healthy run: an outcome-branched + // DAG always leaves one branch untaken, so counting it would make + // every successful DAG roll up non-Done. + assert_eq!( + dag(&[State::Done, State::Skipped]).rollup_state(), + State::Done + ); + } + + #[test] + fn cancelled_outranks_running_and_pending() { + // A cancelled DAG still has its weak-edged tail node to run, so + // Pending-then-Running would flicker back at the operator who just + // cancelled it and read as "the cancel didn't take". + assert_eq!( + dag(&[State::Cancelled, State::Pending]).rollup_state(), + State::Cancelled + ); + assert_eq!( + dag(&[State::Cancelled, State::Running]).rollup_state(), + State::Cancelled + ); + } + + #[test] + fn finishing_still_counts_as_running() { + // The node's own work is done but its sub-nodes are still going, so + // the DAG is in flight. A parent parked in Finishing is the normal + // shape of a subtree mid-run, not an edge case. + assert_eq!( + dag(&[State::Finishing, State::Pending]).rollup_state(), + State::Running + ); + assert_eq!( + dag(&[State::Running, State::Pending]).rollup_state(), + State::Running + ); + assert_eq!( + dag(&[State::Done, State::Pending]).rollup_state(), + State::Pending + ); + } + + #[test] + fn an_empty_node_set_reads_done() { + // Every node Done means every node is filtered off the wire, so this + // is what a finished DAG actually looks like to a consumer that has + // one in hand at all. + assert_eq!(dag(&[]).rollup_state(), State::Done); + } +} diff --git a/hive-jobq/src/builder.rs b/hive-jobq/src/builder.rs index fa1e9807..a2e207e9 100644 --- a/hive-jobq/src/builder.rs +++ b/hive-jobq/src/builder.rs @@ -8,8 +8,10 @@ //! **An insertion API, not a spec factory.** A builder is only ever handed to a //! closure by the single insertion entry point //! ([`crate::scheduler::Scheduler::insert_job`]), which inserts the declared -//! nodes and returns the ids the job asked for. It cannot be constructed, held -//! or inserted from outside this crate, and there is no intermediate +//! nodes and returns the ids the job asked for. It cannot be constructed or +//! inserted from outside this crate — `new()` and `insert_with` are both +//! `pub(crate)`, and there is deliberately no `Default` impl, since a trait impl +//! on a `pub` type is public regardless. There is no intermediate //! node-description type to keep in sync with [`crate::Graph::insert`]'s signature — //! so a job has no representation that can be passed around instead of being //! inserted. @@ -242,16 +244,6 @@ pub struct JobBuilder { nodes: RefCell>>, } -// Hand-written rather than derived: `#[derive(Default)]` would demand -// `N: Default, R: Default`, which has nothing to do with an empty builder. -impl Default for JobBuilder { - fn default() -> Self { - Self { - nodes: RefCell::new(Vec::new()), - } - } -} - impl JobBuilder { /// A fresh, empty builder. /// @@ -261,8 +253,17 @@ impl JobBuilder { /// nodes and returns the ids. Nothing job-shaped is constructible or /// carryable outside this crate — otherwise it is a spec factory again, /// just with a builder's name on it. + /// + /// Deliberately **not** a `Default` impl. A trait impl on a `pub` type is + /// public no matter how private its inherent constructors are, so + /// `JobBuilder::default()` would hand every downstream crate the builder + /// this fn is `pub(crate)` to withhold. The body is what `#[derive(Default)]` + /// could not be anyway — deriving would demand `N: Default, R: Default`, + /// which has nothing to do with an empty builder. pub(crate) fn new() -> Self { - Self::default() + Self { + nodes: RefCell::new(Vec::new()), + } } /// Whether nothing has been declared yet — for a caller deciding whether an diff --git a/hive-jobq/src/scheduler.rs b/hive-jobq/src/scheduler.rs index 337c6b2d..1ab6015d 100644 --- a/hive-jobq/src/scheduler.rs +++ b/hive-jobq/src/scheduler.rs @@ -1,13 +1,14 @@ //! The settle loop — drives a [`Graph`] to completion over a resource pool the //! scheduler owns directly. //! -//! [`Scheduler::settle`] claims every currently-runnable pending node (its +//! [`Scheduler::claim_next`] claims one currently-runnable pending node (its //! [`Dep::Node`] edges satisfied *and* all its [`Dep::Resource`] units acquired -//! atomically), marks it `Running`, records the units it holds, and returns the -//! newly-started ids for the caller's runner to execute. The runner reports each -//! node's result back with [`Scheduler::complete`]; a running node may grow more -//! work first via [`Scheduler::append`]. Concurrency is emergent from resource -//! capacity — there is no separate active-node cap. +//! atomically), marks it `Running`, records the units it holds, and hands back +//! a future that executes the node **and completes it**, so "forgot to finish +//! the node" is not expressible. One at a time is the primitive on purpose: it +//! lets the caller choose between claiming again and backing off, which a batch +//! return can't express. A running node may grow more work by declaring into +//! the builder it was handed. Concurrency is emergent from resource capacity. //! //! Single-threaded by design: the scheduler is the only driver, holds the //! [`ResourceTable`] as a plain owned field, mutating it through `&mut self` — @@ -29,7 +30,9 @@ //! is a deferred optimization — unsafe under dynamically-appended subnodes.) use std::collections::HashMap; +use std::future::Future; use std::hash::Hash; +use std::sync::{Arc, Mutex}; use crate::builder::{BuildError, JobBuilder, NodeGuid}; use crate::resources::ResourceTable; @@ -86,8 +89,8 @@ impl Scheduler { } /// Append a node under `parent` — e.g. a running node growing more work into - /// its own subtree. Delegates to [`Graph::insert`]; call [`Scheduler::settle`] - /// afterwards to start it once it is runnable. + /// its own subtree. Delegates to [`Graph::insert`]; claim again afterwards + /// to start it once it is runnable. /// /// # Errors /// Propagates [`GraphError`] for a dangling dependency or parent id. @@ -114,8 +117,7 @@ impl Scheduler { /// [`Graph::insert_unchecked`]: [`crate::builder::check_job_shape`] has /// already decided every rejection the graph could raise, so re-validating /// per node could only report a problem *after* the earlier nodes were - /// inserted. Call [`Scheduler::settle`] afterwards to start whatever became - /// runnable. + /// inserted. Claim again afterwards to start whatever became runnable. /// /// **Atomic in the job's own shape.** A forward edge, a forward parent, or /// a request for a handle this job never declared is rejected *before* the @@ -138,27 +140,80 @@ impl Scheduler { }) } - /// Claim every currently-runnable pending node and start it: node-deps + /// Claim **one** currently-runnable pending node and start it: node-deps /// satisfied and all resource-deps acquired atomically (all-or-nothing). - /// Each claimed node is marked `Running`, its acquired units recorded, and - /// its id returned for the runner to execute. A single pass suffices — a - /// node started here is `Running`, not terminal, so it cannot satisfy another - /// node's dependency in the same pass; it only consumes resources. + /// The node is marked `Running`, its acquired units recorded, and its id + /// returned for the caller to execute. `None` means nothing is runnable + /// right now — which is a different statement from "nothing is pending". + /// + /// **Private**: [`Self::claim_next`] is the only way out of this crate. + /// Claiming without the future that completes the node is the sequence the + /// seam exists to make inexpressible, so the primitive stays in here. #[must_use] - pub fn settle(&mut self) -> Vec { + fn claim_one(&mut self) -> Option { let pending: Vec = self .graph .nodes() .filter(|n| n.state == State::Pending) .map(|n| n.id) .collect(); - let mut started = Vec::new(); - for id in pending { - if self.node_deps_satisfied(id) && self.try_start(id) { - started.push(id); - } - } - started + pending + .into_iter() + .find(|&id| self.node_deps_satisfied(id) && self.try_start(id)) + } + + /// Claim one runnable node and return **the work that runs it**, or `None` + /// when nothing is runnable right now. + /// + /// This is the seam: the caller supplies how to execute a node and spawns + /// the returned future, but never touches claiming or completion. The + /// future runs the node **and completes it**, so "forgot to finish the + /// node" is not expressible — completion is inside the thing you spawn. + /// + /// The `Option` is answered *synchronously*, before anything is awaited, so + /// the caller can decide "claim again immediately" vs "back off" without + /// waiting on the node it just started. + /// + /// ## Locking + /// The lock is taken twice, briefly, and **never held across the await**: + /// once here to claim, once inside the future to complete. That is what + /// keeps the returned future `Send` — a guard alive across an await point + /// would poison it — and it is why the node itself runs unlocked, for + /// however many minutes it needs. + /// + /// ## Why the payload is cloned + /// `run` gets an owned `N` rather than a borrow: a `&N` parameter is live + /// for the whole future, which both borrows the graph across the await and + /// makes the future non-`Send`. + /// + /// The output carries the insert result rather than swallowing it — this + /// crate has no logger, so a malformed grown job is reported to the caller, + /// who is the one that can log it. The node is completed either way: its + /// own work already happened. + pub fn claim_next( + sched: &Arc>, + run: F, + ) -> Option)> + use> + where + N: Clone, + F: FnOnce(NodeId, N, JobBuilder) -> Fut, + Fut: Future, Outcome)>, + { + let (id, payload) = { + let mut guard = sched.lock().expect("jobq scheduler mutex poisoned"); + let id = guard.claim_one()?; + let payload = guard.graph.node(id)?.payload.clone(); + (id, payload) + }; + let sched = Arc::clone(sched); + Some(async move { + let (grown, outcome) = run(id, payload, JobBuilder::new()).await; + let grew = sched + .lock() + .expect("jobq scheduler mutex poisoned") + .complete_growing(id, outcome, grown); + (id, grew) + }) } /// Try to start node `id`. For each resource it needs, decide per the parent @@ -248,9 +303,13 @@ impl Scheduler { /// (every child `Done`) or [`State::Failed`] (any child `Failed`/`Cancelled`). /// On failure it is `Failed` at once and its pending sub-nodes are cancelled /// (gated on a `Finishing` the parent never reached). Terminality then - /// propagates up the parent chain. Call [`Scheduler::settle`] again afterwards - /// to start newly-unblocked work. - pub fn complete(&mut self, id: NodeId, outcome: Outcome) { + /// propagates up the parent chain. Claim again afterwards to start + /// newly-unblocked work. + /// + /// `pub(crate)`: completion is reachable only from inside the future + /// [`Self::claim_next`] hands back, so it is not expressible without the + /// claim it answers. + pub(crate) fn complete(&mut self, id: NodeId, outcome: Outcome) { match outcome { Outcome::Failed(error) => { // Record the reason before the terminal transition so it's set @@ -265,6 +324,64 @@ impl Scheduler { self.release_ready(); } + /// [`Scheduler::complete`], plus whatever the node declared into the builder + /// it was handed while running. + /// + /// `grown`'s nodes are inserted **under `id`** and *before* the completion, + /// so the node cannot roll terminal with its own appended work still + /// pending — the same ordering the caller previously had to arrange by + /// hand. A job that declares nothing costs nothing: the insert is skipped + /// outright, which is the overwhelmingly common case (most nodes grow no + /// work at all). + /// + /// **A failed node grows nothing**, whatever it declared. Failure + /// cancel-cascades to every pending child of `id`, so work inserted here + /// would be `Skipped` by the very next statement — the insert is not wrong, + /// it is provably pointless. This lives here rather than in the caller + /// because it is a consequence of *this crate's* cascade rule; a host that + /// had to remember it could forget it. + /// + /// # Errors + /// [`BuildError`] if `grown` is malformed — **and the node is still + /// completed**. Its own work already happened; refusing to complete it + /// would misreport that, and leaving it `Running` forever would wedge the + /// DAG. So the error is returned for the caller to log, not used to abort + /// the completion. This crate has no logger of its own; the caller does. + pub(crate) fn complete_growing( + &mut self, + id: NodeId, + outcome: Outcome, + grown: JobBuilder, + ) -> Result<(), BuildError> { + // A node that is no longer in the graph grows nothing. The insert below + // is *unchecked* — rooting on a departed parent would plant a dangling + // `parent` edge rather than being rejected. The host used to carry this + // guard itself, as a lookup before a separate append call; it belongs + // here, where the graph is and where it cannot be skipped. + // + // ⚠️ Deliberately untested, and untestable today: nothing removes a node + // from the graph yet (eviction only stops *retaining* a DAG; its nodes + // linger), and `NodeId` cannot be fabricated, so a test would have to + // fake the very condition it checks. This guard is defensive against the + // bounded prune that does not exist yet — when that lands, it needs a + // test, and this comment is the reminder. + let grew = if grown.is_empty() + || matches!(outcome, Outcome::Failed(_)) + || self.graph.node(id).is_none() + { + Ok(()) + } else { + let graph = &mut self.graph; + grown + .insert_with(Some(id), &[], |payload, deps, parent| { + graph.insert_unchecked(payload, deps, parent) + }) + .map(|_ids| ()) + }; + self.complete(id, outcome); + grew + } + /// Whether every direct child of `id` is terminal. fn all_children_terminal(&self, id: NodeId) -> bool { self.graph @@ -491,7 +608,7 @@ impl Scheduler { /// done" signal) supplies parent→child ordering; `Dep::Node` edges (which the /// graph restricts to the same parent group) supply sibling ordering. /// `Dep::Resource` edges are handled by the atomic acquire in - /// [`Scheduler::settle`], not here. + /// [`Scheduler::try_start`], not here. fn node_deps_satisfied(&self, id: NodeId) -> bool { let Some(node) = self.graph.node(id) else { return false; @@ -533,6 +650,25 @@ mod tests { name.to_owned() } + /// Claim every currently-runnable node, as arrangement for the assertions + /// below. Equivalent to calling [`Scheduler::claim_one`] until it yields + /// `None`: a node started by an earlier iteration is `Running`, not + /// terminal, so it cannot satisfy another node's dependency here — it only + /// consumes resources. + /// + /// **Was `Scheduler::settle`, a public method.** It was a `claim_one` loop + /// returning a `Vec`, and production never wanted the batch: the run loop + /// takes one node at a time through [`Scheduler::claim_next`] so it can + /// choose between claiming again and backing off, which a batch return + /// can't express. The only callers were tests, so it lives with them. + fn settle(s: &mut Scheduler) -> Vec { + let mut started = Vec::new(); + while let Some(id) = s.claim_one() { + started.push(id); + } + started + } + /// A graph + a resource table with `build-slot` set to `slots`. fn scheduler_with_slots(slots: u32) -> Scheduler<&'static str, String> { let mut table = ResourceTable::new(); @@ -559,13 +695,97 @@ mod tests { s.resources.available(&res(name)) } + /// Children of `id`, by payload, in insertion order. + fn children_of(s: &Scheduler<&'static str, String>, id: NodeId) -> Vec<&'static str> { + s.graph() + .nodes() + .filter(|n| n.parent == Some(id)) + .map(|n| n.payload) + .collect() + } + + /// A node completing `Done` gets the work it declared while running, + /// inserted **under itself** — so the DAG cannot roll terminal with the + /// appended work still pending. + #[test] + fn a_completing_node_grows_the_work_it_declared() { + let mut s = scheduler_with_slots(1); + let n = s.append("emitter", vec![], None).expect("insert"); + assert_eq!(settle(&mut s), vec![n]); + + let grown = JobBuilder::new(); + grown.node("child-a"); + grown.node("child-b"); + s.complete_growing(n, Outcome::Done, grown) + .expect("well-formed growth"); + + assert_eq!(children_of(&s, n), vec!["child-a", "child-b"]); + // The emitter parks in `Finishing` rather than going terminal: its own + // appended work is still pending under it. That ordering is the whole + // point of growing *as part of* the completion. + assert_eq!(s.graph().node(n).unwrap().state, State::Finishing); + } + + /// A **failed** node grows nothing, whatever it declared. + /// + /// The companion to the test above, and the reason this rule lives in the + /// crate rather than in a caller: failure cancel-cascades to every pending + /// child of the completing node, so anything inserted here would be + /// `Skipped` by the very next statement. Enforcing it host-side means every + /// host has to remember it; enforcing it here means none can forget. + #[test] + fn a_failed_node_grows_nothing() { + let mut s = scheduler_with_slots(1); + let n = s.append("emitter", vec![], None).expect("insert"); + assert_eq!(settle(&mut s), vec![n]); + + let grown = JobBuilder::new(); + grown.node("never-runs"); + s.complete_growing(n, Outcome::Failed("boom".to_owned()), grown) + .expect("growth is dropped, not rejected"); + + assert!( + children_of(&s, n).is_empty(), + "a failed node must not append work, got {:?}", + children_of(&s, n) + ); + assert_eq!(s.graph().node(n).unwrap().state, State::Failed); + } + + /// A contended resource goes to the oldest waiter. + /// + /// [`Scheduler::claim_one`] scans [`Graph::nodes`] — insertion order — and + /// takes the first node whose deps are satisfied and whose resources it can + /// acquire. That *is* the fairness guarantee: there is no queue, no + /// priority, just the scan order. + /// + /// Load-bearing for any host that submits work over time, because without + /// it a steady arrival rate could starve the earliest waiter indefinitely. + /// It was previously only covered downstream, by a host test driving its own + /// templates — which meant the property this crate provides was asserted + /// everywhere except in this crate. + #[test] + fn a_contended_resource_goes_to_the_oldest_waiter() { + let mut s = scheduler_with_slots(1); + let a = s.append("a", res_dep("build-slot"), None).expect("a"); + let b = s.append("b", res_dep("build-slot"), None).expect("b"); + let c = s.append("c", res_dep("build-slot"), None).expect("c"); + + assert_eq!(settle(&mut s), vec![a], "cap 1: only the first can start"); + s.complete(a, Outcome::Done); + // b and c are both satisfiable now; b was inserted first. + assert_eq!(settle(&mut s), vec![b], "the freed unit goes to b, not c"); + s.complete(b, Outcome::Done); + assert_eq!(settle(&mut s), vec![c]); + } + #[test] fn leaf_owner_goes_done_directly_and_releases() { let mut s = scheduler_with_slots(1); let n = s .append("build", res_dep("build-slot"), None) .expect("insert"); - assert_eq!(s.settle(), vec![n]); + assert_eq!(settle(&mut s), vec![n]); assert_eq!(s.graph().node(n).unwrap().state, State::Running); assert_eq!(avail(&s, "build-slot"), 0); // No children → completing it goes straight to Done (skips Finishing). @@ -585,7 +805,7 @@ mod tests { assert!(s.graph().node(ok).unwrap().started_at.is_none()); assert!(s.graph().node(ok).unwrap().finished_at.is_none()); - let started = s.settle(); + let started = settle(&mut s); assert!(started.contains(&ok) && started.contains(&bad)); // Running → started_at stamped, finished_at still none. assert!(s.graph().node(ok).unwrap().started_at.is_some()); @@ -621,11 +841,11 @@ mod tests { let b = s.append("b", res_dep("build-slot"), None).expect("b"); let c = s.append("c", res_dep("build-slot"), None).expect("c"); // cap 2 → a + b start, c blocks on the exhausted slot. - assert_eq!(s.settle(), vec![a, b]); + assert_eq!(settle(&mut s), vec![a, b]); assert_eq!(s.graph().node(c).unwrap().state, State::Pending); // a finishes → its slot frees → c can now start. s.complete(a, Outcome::Done); - assert_eq!(s.settle(), vec![c]); + assert_eq!(settle(&mut s), vec![c]); assert_eq!(s.graph().node(c).unwrap().state, State::Running); } @@ -637,16 +857,16 @@ mod tests { let root = s.append("root", vec![], None).expect("root"); let c1 = s.append("c1", vec![], Some(root)).expect("c1"); let c2 = s.append("c2", vec![], Some(root)).expect("c2"); - assert_eq!(s.settle(), vec![root]); + assert_eq!(settle(&mut s), vec![root]); // Children can't start yet — parent still Running (logic not done). - assert!(s.settle().is_empty(), "children gated on parent logic"); + assert!(settle(&mut s).is_empty(), "children gated on parent logic"); s.complete(root, Outcome::Done); assert_eq!( s.graph().node(root).unwrap().state, State::Finishing, "logic done, children pending → Finishing" ); - let mut started = s.settle(); + let mut started = settle(&mut s); started.sort(); let mut expected = vec![c1, c2]; expected.sort(); @@ -670,9 +890,9 @@ mod tests { let mut s: Scheduler<&str, String> = Scheduler::new(Graph::new(), ResourceTable::new()); let root = s.append("root", vec![], None).expect("root"); let child = s.append("child", vec![], Some(root)).expect("child"); - assert_eq!(s.settle(), vec![root]); + assert_eq!(settle(&mut s), vec![root]); s.complete(root, Outcome::Done); - assert_eq!(s.settle(), vec![child]); + assert_eq!(settle(&mut s), vec![child]); s.complete(child, Outcome::Failed(String::new())); assert_eq!( s.graph().node(root).unwrap().state, @@ -690,10 +910,10 @@ mod tests { let r = s.append("R", res_dep("build-slot"), None).expect("R"); let c1 = s.append("c1", res_dep("build-slot"), Some(r)).expect("c1"); let c2 = s.append("c2", vec![after_ok(c1)], Some(r)).expect("c2"); - assert_eq!(s.settle(), vec![r]); + assert_eq!(settle(&mut s), vec![r]); s.complete(r, Outcome::Done); // → Finishing (children pending) assert_eq!(avail(&s, "build-slot"), 0, "held: subtree not terminal"); - assert_eq!(s.settle(), vec![c1], "c1 borrows R's slot"); + assert_eq!(settle(&mut s), vec![c1], "c1 borrows R's slot"); assert_eq!(avail(&s, "build-slot"), 0, "borrow reuses R's unit"); s.complete(c1, Outcome::Done); assert_eq!( @@ -701,7 +921,7 @@ mod tests { 0, "still held: c2 pending in subtree" ); - assert_eq!(s.settle(), vec![c2]); + assert_eq!(settle(&mut s), vec![c2]); s.complete(c2, Outcome::Done); assert_eq!( avail(&s, "build-slot"), @@ -719,14 +939,14 @@ mod tests { let owner = s .append("owner", res_dep("agent/foo"), None) .expect("owner"); - assert_eq!(s.settle(), vec![owner]); + assert_eq!(settle(&mut s), vec![owner]); assert_eq!(avail(&s, "agent/foo"), 0); let child = s .append("child", res_dep("agent/foo"), Some(owner)) .expect("child"); s.complete(owner, Outcome::Done); // → Finishing assert_eq!(avail(&s, "agent/foo"), 0, "held while a borrower pends"); - assert_eq!(s.settle(), vec![child]); + assert_eq!(settle(&mut s), vec![child]); assert_eq!(avail(&s, "agent/foo"), 0, "borrow reuses the one unit"); s.complete(child, Outcome::Done); assert_eq!(avail(&s, "agent/foo"), 1); @@ -749,13 +969,13 @@ mod tests { let great = s .append("great", res_dep("agent/foo"), Some(grand)) .expect("great"); - assert_eq!(s.settle(), vec![r]); + assert_eq!(settle(&mut s), vec![r]); s.complete(r, Outcome::Done); - assert_eq!(s.settle(), vec![child], "child borrows R's grant"); + assert_eq!(settle(&mut s), vec![child], "child borrows R's grant"); s.complete(child, Outcome::Done); - assert_eq!(s.settle(), vec![grand], "grand covered, no deadlock"); + assert_eq!(settle(&mut s), vec![grand], "grand covered, no deadlock"); s.complete(grand, Outcome::Done); - assert_eq!(s.settle(), vec![great], "great covered too"); + assert_eq!(settle(&mut s), vec![great], "great covered too"); assert_eq!(avail(&s, "agent/foo"), 0, "held across the whole nest"); s.complete(great, Outcome::Done); assert_eq!(s.graph().node(r).unwrap().state, State::Done, "R rolled up"); @@ -769,10 +989,14 @@ mod tests { let mut s: Scheduler<&str, String> = Scheduler::new(Graph::new(), ResourceTable::new()); let a = s.append("a", res_dep("agent/foo"), None).expect("a"); let b = s.append("b", res_dep("agent/foo"), None).expect("b"); - assert_eq!(s.settle(), vec![a], "only a acquires; b can't borrow it"); + assert_eq!( + settle(&mut s), + vec![a], + "only a acquires; b can't borrow it" + ); assert_eq!(s.graph().node(b).unwrap().state, State::Pending); s.complete(a, Outcome::Done); - assert_eq!(s.settle(), vec![b]); + assert_eq!(settle(&mut s), vec![b]); assert_eq!(s.graph().node(b).unwrap().state, State::Running); } @@ -785,7 +1009,7 @@ mod tests { let owner = s .append("owner", res_dep("agent/foo"), None) .expect("owner"); - assert_eq!(s.settle(), vec![owner]); + assert_eq!(settle(&mut s), vec![owner]); let c1 = s .append("c1", res_dep("agent/foo"), Some(owner)) .expect("c1"); @@ -793,10 +1017,10 @@ mod tests { .append("c2", res_dep("agent/foo"), Some(owner)) .expect("c2"); s.complete(owner, Outcome::Done); // → Finishing - assert_eq!(s.settle(), vec![c1], "c1 borrows; c2 can't (cap 1)"); + assert_eq!(settle(&mut s), vec![c1], "c1 borrows; c2 can't (cap 1)"); assert_eq!(s.graph().node(c2).unwrap().state, State::Pending); s.complete(c1, Outcome::Done); - assert_eq!(s.settle(), vec![c2], "borrow returned → c2 borrows"); + assert_eq!(settle(&mut s), vec![c2], "borrow returned → c2 borrows"); assert_eq!(avail(&s, "agent/foo"), 0, "still just the owner's unit"); } @@ -808,7 +1032,7 @@ mod tests { let owner = s .append("owner", res_dep("build-slot"), None) .expect("owner"); - assert_eq!(s.settle(), vec![owner]); + assert_eq!(settle(&mut s), vec![owner]); assert_eq!(avail(&s, "build-slot"), 1, "owner took one of two"); let c1 = s .append("c1", res_dep("build-slot"), Some(owner)) @@ -817,7 +1041,7 @@ mod tests { .append("c2", res_dep("build-slot"), Some(owner)) .expect("c2"); s.complete(owner, Outcome::Done); // → Finishing - let mut started = s.settle(); + let mut started = settle(&mut s); started.sort(); let mut expected = vec![c1, c2]; expected.sort(); @@ -847,11 +1071,11 @@ mod tests { None, ) .expect("weak"); - assert_eq!(s.settle(), vec![root]); + assert_eq!(settle(&mut s), vec![root]); s.complete(root, Outcome::Failed(String::new())); assert_eq!(s.graph().node(strong1).unwrap().state, State::Skipped); assert_eq!(s.graph().node(strong2).unwrap().state, State::Skipped); - assert_eq!(s.settle(), vec![weak]); + assert_eq!(settle(&mut s), vec![weak]); } /// The direction only a *set* edge can express: a branch that runs solely on @@ -871,7 +1095,7 @@ mod tests { None, ) .expect("compensate"); - assert_eq!(s.settle(), vec![root]); + assert_eq!(settle(&mut s), vec![root]); s.complete(root, Outcome::Done); assert_eq!( s.graph().node(on_fail).unwrap().state, @@ -879,7 +1103,7 @@ mod tests { "a Failed-only branch is unsatisfiable once its dep succeeds — and it is \ `Skipped`, not `Cancelled`, so the parent roll-up ignores it" ); - assert!(s.settle().is_empty(), "and nothing is left runnable"); + assert!(settle(&mut s).is_empty(), "and nothing is left runnable"); } /// The mirror: the same branch is exactly what *does* run on failure, while @@ -901,10 +1125,10 @@ mod tests { None, ) .expect("on_fail"); - assert_eq!(s.settle(), vec![root]); + assert_eq!(settle(&mut s), vec![root]); s.complete(root, Outcome::Failed("boom".to_owned())); assert_eq!(s.graph().node(on_ok).unwrap().state, State::Skipped); - assert_eq!(s.settle(), vec![on_fail]); + assert_eq!(settle(&mut s), vec![on_fail]); } /// A weak edge accepts a dependency that was *ruled out*, so a tail still @@ -925,11 +1149,11 @@ mod tests { None, ) .expect("tail"); - assert_eq!(s.settle(), vec![root]); + assert_eq!(settle(&mut s), vec![root]); s.complete(root, Outcome::Failed("boom".to_owned())); assert_eq!(s.graph().node(mid).unwrap().state, State::Skipped); assert_eq!( - s.settle(), + settle(&mut s), vec![tail], "the tail runs off a cancelled dependency" ); @@ -966,22 +1190,26 @@ mod tests { // Everything succeeds: the ok branch runs, the failure branch is ruled out. let (mut s, a, b, on_ok, on_fail) = build(); - assert_eq!(s.settle(), vec![a, b], "both roots start; neither tail can"); + assert_eq!( + settle(&mut s), + vec![a, b], + "both roots start; neither tail can" + ); s.complete(a, Outcome::Done); s.complete(b, Outcome::Done); - assert_eq!(s.settle(), vec![on_ok]); + assert_eq!(settle(&mut s), vec![on_ok]); s.complete(on_ok, Outcome::Done); assert_eq!(s.graph().node(on_fail).unwrap().state, State::Skipped); - assert!(s.settle().is_empty()); + assert!(settle(&mut s).is_empty()); // One of them fails: the ok branch is ruled out, which is precisely the // signal the failure branch waits on. let (mut s, a, b, on_ok, on_fail) = build(); - assert_eq!(s.settle(), vec![a, b]); + assert_eq!(settle(&mut s), vec![a, b]); s.complete(a, Outcome::Failed("boom".to_owned())); s.complete(b, Outcome::Done); assert_eq!(s.graph().node(on_ok).unwrap().state, State::Skipped); - assert_eq!(s.settle(), vec![on_fail]); + assert_eq!(settle(&mut s), vec![on_fail]); } #[test] @@ -992,7 +1220,7 @@ mod tests { let root = s.append("root", vec![], None).expect("root"); let child = s.append("child", vec![], Some(root)).expect("child"); let grandchild = s.append("gc", vec![], Some(child)).expect("gc"); - assert_eq!(s.settle(), vec![root]); + assert_eq!(settle(&mut s), vec![root]); s.complete(root, Outcome::Failed(String::new())); assert_eq!(s.graph().node(child).unwrap().state, State::Skipped); assert_eq!(s.graph().node(grandchild).unwrap().state, State::Skipped); @@ -1008,7 +1236,7 @@ mod tests { assert_eq!(s.graph().node(a).unwrap().state, State::Cancelled); assert_eq!(s.graph().node(b).unwrap().state, State::Skipped); let c = s.append("c", vec![], None).expect("c"); - assert_eq!(s.settle(), vec![c]); + assert_eq!(settle(&mut s), vec![c]); assert!(!s.cancel_node(c)); assert_eq!(s.graph().node(c).unwrap().state, State::Running); } @@ -1024,7 +1252,7 @@ mod tests { let b = s.append("b", vec![after_ok(a)], Some(root)).expect("b"); // The root runs first and parks in `Finishing` while its children are // outstanding — the state a group root is actually in when cancelled. - assert_eq!(s.settle(), vec![root]); + assert_eq!(settle(&mut s), vec![root]); s.complete(root, Outcome::Done); assert_eq!(s.graph().node(root).unwrap().state, State::Finishing); @@ -1045,9 +1273,9 @@ mod tests { let root = s.append("root", vec![], None).expect("root"); let a = s.append("a", vec![], Some(root)).expect("a"); let b = s.append("b", vec![after_ok(a)], Some(root)).expect("b"); - assert_eq!(s.settle(), vec![root]); + assert_eq!(settle(&mut s), vec![root]); s.complete(root, Outcome::Done); - assert_eq!(s.settle(), vec![a], "a is claimed and running"); + assert_eq!(settle(&mut s), vec![a], "a is claimed and running"); assert!(!s.cancel_node(root), "refused while a runs"); assert_eq!(s.graph().node(a).unwrap().state, State::Running); @@ -1076,7 +1304,7 @@ mod tests { Some(root), ) .expect("tail"); - assert_eq!(s.settle(), vec![root]); + assert_eq!(settle(&mut s), vec![root]); s.complete(root, Outcome::Done); assert!(s.cancel_node(root)); @@ -1086,7 +1314,7 @@ mod tests { State::Pending, "spared, and now runnable since its dep is Cancelled" ); - assert_eq!(s.settle(), vec![tail], "the tail still gets to report"); + assert_eq!(settle(&mut s), vec![tail], "the tail still gets to report"); } #[test] @@ -1094,7 +1322,7 @@ mod tests { let mut s = scheduler_with_slots(1); let g = s.append("g", res_dep("agent/foo"), None).expect("g"); let b = s.append("b", res_dep("build-slot"), None).expect("b"); - assert_eq!(s.settle().len(), 2); + assert_eq!(settle(&mut s).len(), 2); let state = s.resource_state(); assert!(state.contains(&(res("agent/foo"), g))); assert!(state.contains(&(res("build-slot"), b)));