From a5c321a1a00963e40d51fb22732023906a922d83 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 20 Jul 2026 21:46:08 +0200 Subject: [PATCH 1/9] feat(#2591): port hive-c0re job_queue onto the hive-jobq crate Replace the in-tree scheduler with the domain-agnostic hive-jobq crate (merged in #2615): parent-axis grouping + borrow/subtree-reservation resource model + roll-up completion (State::Finishing). Host adaptation: - NodeSpec gains an explicit `parent` axis; templates declare grouping + sibling ordering directly (deps order execution, parent groups a subtree whose resource the descendants borrow). - Rebuild is a nested two-root subtree: Prebuild (root, owns the build slot for the whole subtree, lease-exempt) -> StopForUpdate (child, owns the agent lease) -> Swap/PostSwap (children, borrow both); Reconcile is a separate top-level root (AfterAny Prebuild) so it survives the cancel- cascade of any failed step (recovery-start invariant) and converges to the persisted `wanted` on a fresh lease. This is the multi-root correction to the single-root-chain sketch: node0=root broke lease- exemption (hoisting the lease onto Prebuild) and recovery-reconcile (root failure cancels all children). - Spawn / perm-change / power-ops (stop/start/restart) group-rooted the same way; per-agent power-op subgraphs stay independent roots so a multi-agent DAG runs them concurrently, each on its own lease. - insert_group honours the explicit parent axis (no lease hoisting); the DAG terminal node deps AfterAny on every group root and runs once the whole op rolls up. Drop the old Graph::add_dep terminal wiring. 36/36 job_queue tests, full hive-c0re suite green, clippy --all-targets. --- Cargo.lock | 1 + Cargo.toml | 1 + hive-c0re/Cargo.toml | 1 + hive-c0re/src/job_queue/exec.rs | 134 +-- hive-c0re/src/job_queue/mod.rs | 1214 +++++++++++++++----------- hive-c0re/src/job_queue/model.rs | 195 +---- hive-c0re/src/job_queue/resource.rs | 60 ++ hive-c0re/src/job_queue/scheduler.rs | 119 ++- hive-c0re/src/job_queue/submit.rs | 58 +- hive-c0re/src/job_queue/templates.rs | 75 +- hive-c0re/src/job_queue/tests.rs | 173 ++-- hive-c0re/src/workers/auto_update.rs | 2 + hive-sh4re/src/jobs.rs | 7 +- hivectl/src/dag_progress.rs | 2 +- 14 files changed, 1130 insertions(+), 912 deletions(-) create mode 100644 hive-c0re/src/job_queue/resource.rs diff --git a/Cargo.lock b/Cargo.lock index 8ede80f9..f7bf45d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1620,6 +1620,7 @@ dependencies = [ "forgejo-api", "hive-core-agent-sock", "hive-host-sock", + "hive-jobq", "hive-priv-sock", "hive-sh4re", "hive-types", diff --git a/Cargo.toml b/Cargo.toml index 5ff5da9c..b4e4848f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,7 @@ clap_complete = "4" indicatif = "0.18" hive-sh4re = { path = "hive-sh4re" } hive-agent-sock = { path = "hive-agent-sock" } +hive-jobq = { path = "hive-jobq" } hive-core-agent-sock = { path = "hive-core-agent-sock" } hive-claude = { path = "hive-claude" } hive-host-sock = { path = "hive-host-sock" } diff --git a/hive-c0re/Cargo.toml b/hive-c0re/Cargo.toml index 8809dc9a..9078843f 100644 --- a/hive-c0re/Cargo.toml +++ b/hive-c0re/Cargo.toml @@ -33,6 +33,7 @@ indicatif.workspace = true hive-core-agent-sock.workspace = true hive-sh4re.workspace = true hive-host-sock.workspace = true +hive-jobq.workspace = true hive-priv-sock.workspace = true hive-types.workspace = true libc.workspace = true diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index 66e5b53c..219ea1b2 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -10,8 +10,8 @@ use std::sync::Arc; use anyhow::{Context as _, Result}; -use super::model::{NodeKind, NodeSpec, State, Template}; -use super::{Claim, TerminalDag}; +use super::Claim; +use super::model::{NodeKind, NodeSpec, State}; use crate::coordinator::Coordinator; use crate::power::{ReconcileAction, reconcile_action}; @@ -100,9 +100,74 @@ pub(super) async fn run_node(coord: &Arc, claim: &Claim) -> Result< NodeKind::WritePermFile => run_write_perm_file(coord, claim, &ctx).await, NodeKind::ApprovalDeploy => run_approval_deploy(coord, claim).await, NodeKind::SetWanted { up } => run_set_wanted(coord, claim, *up), + NodeKind::ResolveApproval => run_resolve_approval(coord, claim).await, + NodeKind::EmitRebuilt => Ok(run_emit_rebuilt(coord, claim)), + NodeKind::RevertIntent => run_revert_intent(coord, claim).await, } } +/// Terminal hook (approval DAGs — spawn / opaque deploy): resolve the DAG's +/// approval row from its rolled-up outcome. Its own graph node, weak-dep on the +/// DAG tails, so it runs once everything has settled (any outcome, incl. a +/// cancel before starting — the fallback that resolves a queued-then-cancelled +/// approval whose node never ran). Always succeeds — a hook failure is logged +/// inside, not surfaced as a node failure. +async fn run_resolve_approval(coord: &Arc, claim: &Claim) -> Result { + if let Some(terminal) = coord.job_queue.terminal_summary(claim.dag_id) { + crate::actions::resolve_approval_dag(coord, &terminal).await; + } + Ok(NodeOutput::default()) +} + +/// Terminal hook (rebuild / perm-change DAGs): emit one `Rebuilt` manager event +/// per targeted agent — `ok` on `Done`, `!ok` on `Failed`, none on cancel. +fn run_emit_rebuilt(coord: &Arc, claim: &Claim) -> NodeOutput { + if let Some(terminal) = coord.job_queue.terminal_summary(claim.dag_id) { + for agent in &terminal.agents { + match terminal.state { + State::Done => coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { + agent: agent.clone(), + ok: true, + note: None, + sha: None, + tag: None, + }), + State::Failed => coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { + agent: agent.clone(), + ok: false, + note: terminal.error.clone(), + sha: None, + tag: None, + }), + _ => {} + } + } + } + NodeOutput::default() +} + +/// Terminal hook (power-op DAGs): on a *cancelled* DAG, revert each targeted +/// agent's `wanted` intent to its observed state — the operator's cancel means +/// "don't do it", so the intent snaps back instead of the flip executing as a +/// surprise side effect of some later reconcile. Noop on any non-cancelled +/// outcome. Always succeeds — a revert failure is logged, not surfaced. +async fn run_revert_intent(coord: &Arc, claim: &Claim) -> Result { + if let Some(terminal) = coord.job_queue.terminal_summary(claim.dag_id) + && terminal.state == State::Cancelled + { + for agent in &terminal.agents { + let running = crate::lifecycle::is_running(agent).await; + if let Err(e) = coord + .power + .set(agent, crate::power::Wanted::from_running(running)) + { + tracing::warn!(%agent, error = ?e, "agent_power: cancel revert failed"); + } + } + } + Ok(NodeOutput::default()) +} + /// Write the agent's durable power intent — the DAG-node form of the old /// pre-submit `set_wanted` side effect. Store-only (no container touch), so /// build-slot-exempt; but it takes the agent's lifecycle lease (see @@ -531,71 +596,6 @@ async fn run_approval_deploy(coord: &Arc, claim: &Claim) -> Result< .map(|()| NodeOutput::default()) } -/// Terminal-roll-up hook, fired exactly once per DAG (node completion -/// and cancel paths alike — the queue buffers roll-ups and the -/// scheduler drains them). Three concerns: -/// - approval DAGs resolve their approval row (except the opaque -/// deploy pipeline, which resolves inside its node — unless it was -/// cancelled while still queued and the node never ran); -/// - non-approval rebuild-shaped DAGs emit exactly one `Rebuilt` -/// manager event: ok on `Done`, !ok on `Failed`, none on cancel; -/// - a cancelled power-op DAG reverts the `wanted` intent its submit -/// wrote: the operator's cancel means "don't do it", so intent -/// snaps back to the observed state instead of the flip executing -/// as a surprise side effect of some later reconcile. -pub(super) async fn on_dag_terminal(coord: &Arc, terminal: &TerminalDag) { - if terminal.state == State::Cancelled - && matches!( - terminal.template, - Template::Start - | Template::Stop - | Template::GracefulStop - | Template::Restart - | Template::GracefulRestart - ) - { - // Revert each targeted agent's power intent to its observed state — - // the operator's cancel means "don't do it". Single-agent power-op - // DAGs have one agent here. - for agent in &terminal.agents { - let running = crate::lifecycle::is_running(agent).await; - if let Err(e) = coord - .power - .set(agent, crate::power::Wanted::from_running(running)) - { - tracing::warn!(%agent, error = ?e, "agent_power: cancel revert failed"); - } - } - } - if terminal.approval_id.is_some() { - crate::actions::resolve_approval_dag(coord, terminal).await; - return; - } - if matches!(terminal.template, Template::Rebuild | Template::PermChange) { - // Rebuild / PermChange are single-agent; emit one `Rebuilt` per - // targeted agent (exactly one today). - for agent in &terminal.agents { - match terminal.state { - State::Done => coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { - agent: agent.clone(), - ok: true, - note: None, - sha: None, - tag: None, - }), - State::Failed => coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { - agent: agent.clone(), - ok: false, - note: terminal.error.clone(), - sha: None, - tag: None, - }), - _ => {} - } - } - } -} - /// Compute which agents a `nix flake update ` on the meta /// flake affects — the fan-out set for `MetaUpdate` DAGs. Empty /// `inputs` or any input under `hyperhive` → every container; diff --git a/hive-c0re/src/job_queue/mod.rs b/hive-c0re/src/job_queue/mod.rs index c7362cea..9a23eb80 100644 --- a/hive-c0re/src/job_queue/mod.rs +++ b/hive-c0re/src/job_queue/mod.rs @@ -1,91 +1,95 @@ -//! Generic job-DAG queue + desired-state reconciliation — replaces the -//! old flat `rebuild_queue`. Jobs are nodes in per-request DAGs (see -//! [`templates`]); the special cases (graceful-stop watcher thread, -//! deferred-start follow-up, meta-update cascade) collapse into DAG -//! *shapes* over a shared set of primitive nodes ([`model::NodeKind`]). +//! Generic job-DAG queue + desired-state reconciliation — the host-side +//! wrapper over the domain-agnostic [`hive_jobq`] scheduler. Jobs are nodes in +//! per-request DAGs (see [`templates`]); the special cases (graceful-stop +//! watcher, deferred-start follow-up, meta-update cascade) collapse into DAG +//! *shapes* over the shared node primitives ([`model::NodeKind`]). //! -//! Concurrency is gated by two resource classes: -//! 1. **Build slots** — N permits (`services.hyperhive.c0re.buildSlots`, -//! default 1) held by nix-heavy nodes for the node's duration. -//! 2. **Per-agent lifecycle lease** — keyed on the *node's* agent and -//! globally exclusive per agent across all DAGs: acquired before a -//! container-affecting node runs, held (by the owning DAG) until no -//! live node of that DAG still targets the agent, so two DAGs never -//! interleave container ops on the same agent. A DAG spanning multiple -//! agents holds one lease per agent it touches. +//! [`hive_jobq`] owns the graph, the two-class resource pool, and the settle +//! loop; this module maps hive-c0re's concepts onto it: +//! - `NodeKind` + agent → the crate payload [`resource::JobPayload`]; +//! - the two resource classes → [`resource::Resource`] +//! ([`resource::Resource::BuildSlot`] node-held, [`resource::Resource::Agent`] +//! subtree-held), derived per node by [`resource::JobPayload::resource_deps`]; +//! - a host **DAG id** groups a set of crate nodes; a node declares only its +//! `deps` (chain edges + resource deps), and the crate scheduler infers lease +//! re-entrancy from the [`Dep::Node`] graph — a node needing an agent lease a +//! node it depends on already holds re-enters it, with no parent annotation; +//! - per-DAG terminal work is a *focused* graph node per concern — `ResolveApproval` +//! (approval DAGs), `EmitRebuilt` (rebuild / perm-change), `RevertIntent` +//! (cancelled power-ops) — weak-depending on the DAG's tail nodes (extended onto +//! any runtime-appended subgraph's tail), so it runs once everything settles. +//! Hook-less DAGs (meta-update / boot / reconcile) get none. No drained event +//! stream, and no one node branching on DAG metadata. //! -//! The meta *repo* is serialized by `meta::META_LOCK` inside the -//! executors themselves. Per-agent power *intent* (`wanted`) lives in -//! the durable [`crate::power`] store; the DAGs are the reconcile -//! mechanism. Design + rationale: `docs/coordinator.md::Job queue`. +//! The queue is runtime-only (no persistence): an empty graph on boot; desired +//! state is re-derived by the reconcile sweep. A single scheduler task +//! ([`scheduler::run_worker`]) drives it; concurrency comes from the build-slot +//! capacity, not multiple workers. Design: `docs/coordinator.md::Job queue`. pub mod exec; pub mod model; +pub mod resource; pub mod scheduler; pub mod submit; pub mod templates; #[cfg(test)] mod tests; -use std::collections::{HashMap, VecDeque}; +use std::collections::HashMap; use std::sync::Mutex; +use hive_jobq::resources::ResourceTable; +use hive_jobq::scheduler::{Outcome, Scheduler}; +use hive_jobq::{Dep, DepWhen as JobDepWhen, Graph, NodeId, State as JobState}; +use hive_sh4re::jobs::NodeView; use hive_sh4re::wire_time::now_unix; use tokio::sync::Notify; +use crate::coordinator::TransientKind; pub use model::{ - Dag, DagSpec, DagView, DepWhen, Node, NodeId, NodeKind, NodeSpec, PermPayload, Source, State, - Template, + DagSpec, DagView, DepWhen, NodeKind, NodeSpec, PermPayload, Source, State, Template, }; +use resource::{JobPayload, Resource}; -/// How many terminal DAGs (`Done` / `Failed` / `Cancelled`) to retain -/// per template in the snapshot, matching the old per-kind history cap. +/// How many terminal DAGs (`Done` / `Failed` / `Cancelled`) to retain per +/// template in the snapshot, matching the old per-kind history cap. const MAX_HISTORY_PER_TEMPLATE: usize = 5; -/// Terminal DAGs younger than this are exempt from the per-template -/// history cap. A broad `hivectl stop`/`start` submits many -/// same-template DAGs that can all settle within one poll interval — -/// without the grace, the cap would evict some before the ~1s -/// `QueueDag` poller ever observes their terminal state, silently -/// swallowing failures. +/// Terminal DAGs younger than this are exempt from the per-template history +/// cap, so a burst of same-template DAGs that settle within one `QueueDag` +/// poll interval isn't evicted before the poller observes their terminal state. const HISTORY_GRACE_SECS: i64 = 300; /// Cap on stored node error strings. const MAX_ERROR_LEN: usize = 2_000; -/// A node claimed for execution — everything the executor needs, -/// snapshotted at claim time. +/// A node claimed for execution — everything the executor needs, snapshotted at +/// claim time. #[derive(Debug, Clone)] pub struct Claim { pub dag_id: u64, pub node_id: NodeId, pub kind: NodeKind, - /// The agent this node targets (its own, not a DAG-level field). The - /// executor operates on this agent's container; the lease is keyed on - /// it. + /// The agent this node targets (its own, not a DAG-level field). Empty for + /// the internal [`NodeKind::Finalize`] node. pub agent: String, pub template: Template, pub approval_id: Option, pub inputs: Vec, pub perm_payload: Option, - /// True when claiming this node newly acquired its agent's lease — - /// the scheduler creates the per-`(dag, agent)` transient guard on - /// this edge. - pub lease_acquired: bool, - /// Transient pill kind for the lease window (from the spec). - pub transient: Option, + /// Transient pill kind for the lease window (from the spec). Whether the + /// pill is currently shown is derived from live lease ownership + /// ([`JobQueue::held_transients`]), not a per-claim edge. + pub transient: Option, } -/// Summary of a DAG that just reached its terminal roll-up state — -/// input to the approval-resolution hook and the lease/transient -/// release. +/// Summary of a DAG's terminal roll-up — the input to the terminal node's +/// executor (approval resolution, `Rebuilt` emission, cancelled-power-op intent +/// revert). Computed on demand from live graph state, not drained. #[derive(Debug, Clone)] pub struct TerminalDag { - pub dag_id: u64, pub template: Template, - /// Distinct agents this DAG's nodes targeted (one for a single-agent - /// DAG). The cancel-revert hook walks these to snap each agent's - /// power intent back on a cancelled power-op DAG. + /// Distinct agents this DAG's nodes targeted (one for a single-agent DAG). pub agents: Vec, pub approval_id: Option, pub state: State, @@ -93,559 +97,735 @@ pub struct TerminalDag { pub error: Option, } -/// A single per-agent lease release: agent `agent`'s subgraph within DAG -/// `dag_id` just reached terminal, so the scheduler drops that agent's -/// `(dag_id, agent)` transient guard — ahead of (or coinciding with) the -/// whole-DAG [`TerminalDag`]. The lease itself is freed inside `settle`; -/// this only carries the transient-drop signal out to the scheduler. +/// Per-DAG metadata that isn't a node — the group's display + hook inputs. #[derive(Debug, Clone)] -pub struct AgentRelease { - pub dag_id: u64, - pub agent: String, +struct GroupMeta { + template: Template, + source: Source, + reason: String, + approval_id: Option, + inputs: Vec, + perm_payload: Option, + transient: Option, + created_at: i64, } -#[derive(Debug, Default)] -struct Inner { - dags: VecDeque, - next_id: u64, - build_slots: usize, - slots_used: usize, - /// agent → dag id currently holding that agent's lifecycle lease. - leases: HashMap, - /// Terminal roll-ups not yet consumed by the scheduler - /// ([`JobQueue::drain_terminal`]). Fed by every path that settles - /// state — node completion AND the cancel surfaces — so the - /// terminal hooks (approval resolution, intent revert, transient - /// release) fire exactly once per DAG no matter how it ended. - pending_terminal: Vec, - /// Per-agent lease releases not yet consumed by the scheduler - /// ([`JobQueue::drain_agent_releases`]). Fed by `settle` the moment - /// an agent's subgraph within a DAG goes terminal — earlier than the - /// whole-DAG `pending_terminal` for a multi-agent DAG. Drives the - /// per-agent transient-guard drop. - pending_agent_release: Vec, +/// Per-node runtime metadata the crate graph doesn't carry (kind + agent live +/// in the node payload; state lives in the node). +#[derive(Debug, Default, Clone)] +struct NodeRuntime { + step: Option, + build_log_id: Option, + started_at: Option, + finished_at: Option, + error: Option, } -/// The queue. Lives on `Coordinator` (one per hive-c0re process); a -/// single scheduler task ([`scheduler::run_worker`]) drives it — -/// concurrency comes from the build-slot count, not multiple workers. -#[derive(Debug)] +/// The mutable queue state behind the mutex: the crate scheduler plus the +/// host-side grouping side-tables (`dag_id` ↔ nodes, per-DAG meta, per-node +/// runtime metadata). One shared crate [`Graph`] holds every DAG's nodes. +struct QueueInner { + sched: Scheduler, + next_dag: u64, + /// Per-DAG metadata, keyed by DAG id. + dag_meta: HashMap, + /// Work nodes of each DAG, in insert order (drives rollup + the view). + /// Excludes the internal terminal node. + dag_nodes: HashMap>, + /// The terminal-hook node id of each DAG that has one (approval-resolve / + /// rebuilt-emit / intent-revert). Hook-less DAGs are absent from the map. + terminal_node: HashMap, + /// Reverse lookup: crate node id → its owning DAG id. + node_dag: HashMap, + /// Per-node runtime metadata. + node_rt: HashMap, +} + +/// The queue. Lives on `Coordinator` (one per hive-c0re process); a single +/// scheduler task ([`scheduler::run_worker`]) drives it. pub struct JobQueue { - inner: Mutex, + inner: Mutex, /// Wakes the scheduler when something new arrives or state changed. pub(crate) notify: Notify, } +impl std::fmt::Debug for JobQueue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("JobQueue").finish_non_exhaustive() + } +} + impl Default for JobQueue { fn default() -> Self { Self::new(1) } } +/// Map a spec dependency edge kind onto the crate's. +fn to_crate_when(when: DepWhen) -> JobDepWhen { + match when { + DepWhen::AfterOk => JobDepWhen::AfterOk, + DepWhen::AfterAny => JobDepWhen::AfterAny, + } +} + +/// The terminal-hook node kind a DAG needs, from its template + approval id — +/// or `None` for a DAG with no terminal side effect (meta-update, boot, bare +/// reconcile). Each concern is its own focused node rather than one node that +/// branches on metadata: an approval DAG resolves its approval, a rebuild / +/// perm-change emits `Rebuilt`, a power-op reverts its `wanted` intent on cancel. +fn terminal_kind(template: Template, approval_id: Option) -> Option { + if approval_id.is_some() { + return Some(NodeKind::ResolveApproval); + } + match template { + Template::Rebuild | Template::PermChange => Some(NodeKind::EmitRebuilt), + Template::Start + | Template::Stop + | Template::GracefulStop + | Template::Restart + | Template::GracefulRestart => Some(NodeKind::RevertIntent), + _ => None, + } +} + +/// Map a crate node state onto the wire state (`Pending` ↔ `Queued`; +/// `Finishing` — own logic done, sub-nodes still running — reads as `Running`). +fn to_wire_state(state: JobState) -> State { + match state { + JobState::Pending => State::Queued, + JobState::Running | JobState::Finishing => State::Running, + JobState::Done => State::Done, + JobState::Failed => State::Failed, + JobState::Cancelled => State::Cancelled, + } +} + +/// Insert `nodes` into the shared graph, honouring the spec's explicit **parent +/// axis**: a node with `parent = None` is a top-level group root (re-parented to +/// `group_parent`, which is `None` for `submit` and the emitting node for +/// `append_subgraph`); a node with `parent = Some(idx)` becomes a child of the +/// already-inserted node at spec index `idx`. `deps` are translated to crate +/// `Dep::Node` edges verbatim — templates declare the parent axis + sibling +/// ordering directly, so there is no dep-on-root to drop and no lease to hoist: +/// each node declares its own `Dep::Resource`, and the crate's borrow model +/// keeps a resource continuous across a subtree (a root owns it, descendants +/// borrow it). Independent group roots (multiple `parent = None` nodes) carry no +/// cross-links, so a multi-agent DAG's per-agent subgraphs run concurrently, each +/// on its own lease. Records per-node bookkeeping (`node_dag`, `node_rt`); the +/// caller owns `dag_nodes`. Returns the inserted ids (index-aligned with `nodes`) +/// and the group roots (the `parent = None` nodes). A node's `parent` / dep +/// targets must precede it in `nodes` (submit-time `validate` enforces density + +/// acyclicity). +/// +/// # Errors +/// Propagates a crate graph-insert error (malformed dep/parent / dep-scope). +fn insert_group( + inner: &mut QueueInner, + dag_id: u64, + nodes: &[NodeSpec], + group_parent: Option, +) -> anyhow::Result<(Vec, Vec)> { + let mut ids: Vec = Vec::with_capacity(nodes.len()); + let mut roots: Vec = Vec::new(); + for ns in nodes { + let payload = JobPayload { + kind: ns.kind.clone(), + agent: ns.agent.clone(), + }; + let mut deps = payload.resource_deps(); + for d in &ns.deps { + deps.push(Dep::Node { + id: ids[dep_index(d.on)], + when: to_crate_when(d.when), + }); + } + let parent = match ns.parent { + Some(idx) => Some(ids[dep_index(idx)]), + None => group_parent, + }; + let id = inner + .sched + .append(payload, deps, parent) + .map_err(|e| anyhow::anyhow!("job_queue: graph insert failed: {e}"))?; + if ns.parent.is_none() { + roots.push(id); + } + ids.push(id); + inner.node_dag.insert(id, dag_id); + inner.node_rt.insert(id, NodeRuntime::default()); + } + Ok((ids, roots)) +} + impl JobQueue { + #[must_use] pub fn new(build_slots: usize) -> Self { + let mut table = ResourceTable::new(); + table.set_capacity( + Resource::BuildSlot, + u32::try_from(build_slots.max(1)).unwrap_or(u32::MAX), + ); Self { - inner: Mutex::new(Inner { - build_slots: build_slots.max(1), - ..Inner::default() + inner: Mutex::new(QueueInner { + sched: Scheduler::new(Graph::new(), table), + next_dag: 0, + dag_meta: HashMap::new(), + dag_nodes: HashMap::new(), + terminal_node: HashMap::new(), + node_dag: HashMap::new(), + node_rt: HashMap::new(), }), notify: Notify::new(), } } - /// Submit a DAG. Validates the spec (cycle rejection) and returns the - /// newly-allocated DAG id. - /// - /// Submit-time dedup was removed with the agent-per-node refactor - /// (a multi-agent DAG has no single agent to key a dedup on) — every - /// submit now enqueues a fresh DAG. Whether any dedup needs - /// reintroducing (and in what form) is tracked as a follow-up; see the - /// dedup re-evaluation issue. - pub fn submit(&self, spec: DagSpec) -> anyhow::Result { - templates::validate(&spec)?; - let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); - let id = Self::push_dag(&mut inner, spec); - drop(inner); - self.notify.notify_one(); - Ok(id) + fn lock(&self) -> std::sync::MutexGuard<'_, QueueInner> { + self.inner.lock().expect("job_queue mutex poisoned") } - /// Append a whole *subgraph* into a live (non-terminal) DAG at runtime — - /// the single in-DAG-growth primitive. Each [`NodeSpec`] carries its own - /// `agent` and subgraph-relative `deps` (indices into `nodes`); this - /// rebases those onto the DAG's node-id space (`id == index`) and attaches - /// every subgraph *root* — a node with no internal deps — to `dep_on` with - /// an `AfterOk` edge. Used both for multi-node growth (the `MetaLock` - /// growing per-agent rebuild subgraphs into the same boot / meta-update - /// DAG instead of fanning out child DAGs) and the single-node case (a - /// `Reconcile` planner's `Start` / `Stop` as a one-node subgraph). Must be - /// called *before* the emitting node's [`Self::complete_node`] so the DAG - /// can't roll terminal with the appended work still pending. Returns the - /// new node ids; empty if the DAG is gone or `nodes` is empty. - pub fn append_subgraph( - &self, - dag_id: u64, - nodes: Vec, - dep_on: NodeId, - ) -> Vec { + /// Submit a DAG. Validates the spec (cycle rejection), inserts its nodes + /// into the shared graph (chain deps + resource edges), appends the per-DAG + /// terminal node (weak-depending on the DAG's tail nodes), and returns the + /// new DAG id. + /// + /// # Errors + /// Propagates the spec-validation error (empty / cyclic) or a graph-insert + /// error (a spec whose dependencies aren't dependency-topological). + pub fn submit(&self, spec: DagSpec) -> anyhow::Result { + templates::validate(&spec)?; + let mut inner = self.lock(); + inner.next_dag += 1; + let dag_id = inner.next_dag; + + let (work, roots) = insert_group(&mut inner, dag_id, &spec.nodes, None)?; + + // Append the DAG's terminal-hook node — but only if it needs one. It + // weak-depends (`AfterAny`) on every group **root**, each of which the + // crate rolls up terminal only once its whole subtree — the entire op, + // including any runtime-appended subgraphs (children of nodes inside the + // group) — has settled. So the hook runs exactly when the DAG is done, + // with no `add_dep` wiring. Hook-less DAGs (meta-update, boot, reconcile) + // get none. + if let Some(kind) = terminal_kind(spec.template, spec.approval_id) { + let term = inner + .sched + .append( + JobPayload { + kind, + agent: String::new(), + }, + roots + .iter() + .map(|&id| Dep::Node { + id, + when: JobDepWhen::AfterAny, + }) + .collect(), + None, + ) + .map_err(|e| anyhow::anyhow!("job_queue: terminal node insert failed: {e}"))?; + inner.terminal_node.insert(dag_id, term); + inner.node_dag.insert(term, dag_id); + inner.node_rt.insert(term, NodeRuntime::default()); + } + + inner.dag_nodes.insert(dag_id, work); + inner.dag_meta.insert( + dag_id, + GroupMeta { + template: spec.template, + source: spec.source, + reason: spec.reason, + approval_id: spec.approval_id, + inputs: spec.inputs, + perm_payload: spec.perm_payload, + transient: spec.transient, + created_at: now_unix(), + }, + ); + drop(inner); + self.notify.notify_one(); + Ok(dag_id) + } + + /// Append a whole *subgraph* into a live DAG at runtime — the single + /// in-DAG-growth primitive. The subgraph is inserted as a [`insert_group`] + /// rooted under `dep_on` (the emitting node): the subgraph's own root becomes + /// a *child* of `dep_on`, its steps children of that root, and the group's + /// agent lease is hoisted onto that root. Ordering root→`dep_on` is the parent + /// gate — the children run once `dep_on` reaches `Finishing`. Because the + /// emitting node stays `Finishing` until this appended subtree is terminal and + /// the DAG's terminal node deps on the top root, roll-up keeps the DAG from + /// settling early with no explicit wiring. Returns the new node ids; empty if + /// the DAG is gone or `nodes` is empty. + pub fn append_subgraph(&self, dag_id: u64, nodes: &[NodeSpec], dep_on: NodeId) -> Vec { if nodes.is_empty() { return Vec::new(); } - let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); - let Some(dag) = inner.dags.iter_mut().find(|d| d.id == dag_id) else { + let mut inner = self.lock(); + if !inner.dag_meta.contains_key(&dag_id) { return Vec::new(); + } + // Insert the subgraph as a group rooted under the emitting node: the + // subgraph's own root becomes a child of `dep_on`, its steps children of + // that root. No terminal-node wiring — roll-up carries terminality: the + // emitter stays `Finishing` until this appended subtree settles, and the + // DAG's terminal node deps on the top root, so the hook waits for free. + let ids = match insert_group(&mut inner, dag_id, nodes, Some(dep_on)) { + Ok((ids, _roots)) => ids, + Err(e) => { + tracing::error!( + dag = dag_id, + error = %e, + "job_queue: append_subgraph insert failed" + ); + return Vec::new(); + } }; - let base: NodeId = u32::try_from(dag.nodes.len()).unwrap_or(u32::MAX); - let mut new_ids = Vec::with_capacity(nodes.len()); - for (i, spec) in nodes.into_iter().enumerate() { - let new_id: NodeId = base + u32::try_from(i).unwrap_or(u32::MAX); - // Subgraph roots (no internal deps) hang off the emitting node; - // internal deps rebase from subgraph-relative onto the DAG id - // space (both start at `base`). - let deps = if spec.deps.is_empty() { - vec![model::Dep { - on: dep_on, - when: DepWhen::AfterOk, - }] - } else { - spec.deps - .into_iter() - .map(|d| model::Dep { - on: base + d.on, - when: d.when, - }) - .collect() - }; - dag.nodes.push(Node { - id: new_id, - agent: spec.agent, - kind: spec.kind, - deps, - state: State::Queued, - step: None, - build_log_id: None, - started_at: None, - finished_at: None, - error: None, - }); - new_ids.push(new_id); + if let Some(list) = inner.dag_nodes.get_mut(&dag_id) { + list.extend(ids.iter().copied()); } drop(inner); self.notify.notify_one(); - new_ids + ids } - fn push_dag(inner: &mut Inner, spec: DagSpec) -> u64 { - inner.next_id += 1; - let id = inner.next_id; - let nodes = spec - .nodes - .into_iter() - .enumerate() - .map(|(i, n)| Node { - id: u32::try_from(i).unwrap_or(u32::MAX), - agent: n.agent, - kind: n.kind, - deps: n.deps, - state: State::Queued, - step: None, - build_log_id: None, - started_at: None, - finished_at: None, - error: None, - }) - .collect(); - inner.dags.push_back(Dag { - id, - template: spec.template, - source: spec.source, - reason: spec.reason, - approval_id: spec.approval_id, - inputs: spec.inputs, - perm_payload: spec.perm_payload, - transient: spec.transient, - created_at: now_unix(), - nodes, - terminal_reported: false, - }); - id - } - - /// Claim every currently-ready node, acquiring resources, and mark - /// them `Running`. A node is ready when it's `Queued`, every dep is - /// satisfied (`AfterOk`: dep `Done`; `AfterAny`: dep terminal), and - /// its resources are free (build slot; agent lease free or already - /// held by this DAG). Iteration is in DAG-submit order, so - /// simultaneously-ready nodes compete FIFO — bulk operations drain - /// predictably. + /// Claim every currently-runnable node, acquiring its resources, and mark it + /// `Running`. Delegates readiness + resource acquisition to the crate's + /// settle loop; builds a [`Claim`] per started node from its payload + the + /// DAG's metadata. The internal terminal node is claimed like any other + /// (its executor runs the terminal hooks). pub fn claim_ready(&self) -> Vec { - let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); - Self::propagate_cancellations(&mut inner); - let mut claims = Vec::new(); + let mut inner = self.lock(); let inner = &mut *inner; - for di in 0..inner.dags.len() { - // Split-borrow dance: deps are checked against the same - // DAG's other nodes, so snapshot the states first. - let dag = &inner.dags[di]; - let dag_id = dag.id; - let ready_ids: Vec = dag - .nodes - .iter() - .filter(|n| n.state == State::Queued && Self::deps_satisfied(dag, n)) - .map(|n| n.id) - .collect(); - for node_id in ready_ids { - let dag = &inner.dags[di]; - let node = dag.node(node_id).expect("node id from same dag"); - let needs_slot = node.kind.needs_build_slot(); - let needs_lease = node.kind.needs_lease(); - // The lifecycle lease is keyed on the *node's* agent, held - // by this DAG (dag_id) — still globally exclusive per agent - // across all DAGs. A multi-agent DAG acquires one lease per - // agent it touches; each is released in `settle` once no - // live node of this DAG still targets that agent. - let node_agent = node.agent.clone(); - if needs_slot && inner.slots_used >= inner.build_slots { - continue; - } - let mut lease_acquired = false; - if needs_lease { - match inner.leases.get(node_agent.as_str()) { - Some(&holder) if holder != dag_id => continue, - Some(_) => {} - None => { - inner.leases.insert(node_agent.clone(), dag_id); - lease_acquired = true; - } - } - } - if needs_slot { - inner.slots_used += 1; - } - let dag = &mut inner.dags[di]; - let claim = Claim { - dag_id, - node_id, - kind: dag.node(node_id).expect("node").kind.clone(), - agent: node_agent, - template: dag.template, - approval_id: dag.approval_id, - inputs: dag.inputs.clone(), - perm_payload: dag.perm_payload.clone(), - lease_acquired, - transient: dag.transient, - }; - let node = dag.node_mut(node_id).expect("node"); - node.state = State::Running; - node.started_at = Some(now_unix()); - claims.push(claim); + let started = inner.sched.settle(); + let now = now_unix(); + let mut claims = Vec::with_capacity(started.len()); + for id in started { + let Some(node) = inner.sched.graph().node(id) else { + continue; + }; + let kind = node.payload.kind.clone(); + let agent = node.payload.agent.clone(); + let Some(&dag_id) = inner.node_dag.get(&id) else { + continue; + }; + let Some(meta) = inner.dag_meta.get(&dag_id) else { + continue; + }; + claims.push(Claim { + dag_id, + node_id: id, + kind, + agent, + template: meta.template, + approval_id: meta.approval_id, + inputs: meta.inputs.clone(), + perm_payload: meta.perm_payload.clone(), + transient: meta.transient, + }); + if let Some(rt) = inner.node_rt.get_mut(&id) { + rt.started_at = Some(now); } } claims } - fn deps_satisfied(dag: &Dag, node: &Node) -> bool { - node.deps.iter().all(|dep| { - dag.node(dep.on).is_some_and(|d| match dep.when { - DepWhen::AfterOk => d.state == State::Done, - DepWhen::AfterAny => d.state.is_terminal(), - }) - }) - } - - /// Cancel-downstream: a `Queued` node with an `AfterOk` dep that - /// `Failed` / `Cancelled` becomes `Cancelled` itself. Loops to a - /// fixpoint so the cancellation cascades through chains. - fn propagate_cancellations(inner: &mut Inner) { - for dag in &mut inner.dags { - loop { - let doomed: Vec = dag - .nodes - .iter() - .filter(|n| { - n.state == State::Queued - && n.deps.iter().any(|dep| { - dep.when == DepWhen::AfterOk - && dag.node(dep.on).is_some_and(|d| { - matches!(d.state, State::Failed | State::Cancelled) - }) - }) - }) - .map(|n| n.id) - .collect(); - if doomed.is_empty() { - break; - } - let now = now_unix(); - for id in doomed { - if let Some(n) = dag.node_mut(id) { - n.state = State::Cancelled; - n.finished_at = Some(now); - } - } - } - } - } - - /// Mark a claimed node terminal, release its build slot, cascade - /// cancellations, and settle terminal DAGs (lease release + history - /// trim; the terminal roll-up lands in the [`Self::drain_terminal`] - /// buffer). `error` is stored (truncated) when `result` is `Err`. - pub fn complete_node(&self, dag_id: u64, node_id: NodeId, result: Result<(), String>) { - let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); - if let Some(dag) = inner.dags.iter_mut().find(|d| d.id == dag_id) - && let Some(node) = dag.node_mut(node_id) - && node.state == State::Running - { - let needs_slot = node.kind.needs_build_slot(); - node.finished_at = Some(now_unix()); - node.step = None; - match result { - Ok(()) => node.state = State::Done, - Err(e) => { - node.state = State::Failed; - let mut msg = e; - if msg.len() > MAX_ERROR_LEN { - msg.truncate( - (0..=MAX_ERROR_LEN) - .rev() - .find(|i| msg.is_char_boundary(*i)) - .unwrap_or(0), - ); - msg.push('…'); - } - node.error = Some(msg); - } - } - if needs_slot { - inner.slots_used = inner.slots_used.saturating_sub(1); - } - } - Self::settle(&mut inner); - drop(inner); - self.notify.notify_one(); - } - - /// Take the terminal roll-ups accumulated since the last drain. - /// The scheduler calls this after every wakeup and runs the - /// terminal hooks on each entry. - pub fn drain_terminal(&self) -> Vec { - let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); - std::mem::take(&mut inner.pending_terminal) - } - - /// Take the per-agent lease releases accumulated since the last - /// drain. The scheduler calls this every wakeup and drops the - /// matching `(dag_id, agent)` transient guard for each — freeing an - /// agent's dashboard pill the moment its subgraph settles, ahead of - /// the whole-DAG terminal for a multi-agent DAG. - pub fn drain_agent_releases(&self) -> Vec { - let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); - std::mem::take(&mut inner.pending_agent_release) - } - - /// Propagate cancellations, release each agent's lease the moment its - /// own subgraph settles (not at whole-DAG terminal), buffer each - /// terminal roll-up exactly once (the `terminal_reported` flag) for - /// [`Self::drain_terminal`], and trim history. - fn settle(inner: &mut Inner) { - Self::propagate_cancellations(inner); - - // Per-agent early lease release. A lease gates an agent's - // container globally, so it must be held for exactly as long as - // that agent's work in the DAG is in flight — no longer. The - // moment no live node of a DAG still targets an agent, free that - // agent's lease so a concurrent DAG wanting the same agent can - // proceed, even while the rest of this DAG runs on. For a - // single-agent DAG the agent's subgraph goes terminal exactly - // when the whole DAG does, so this reduces to the old behaviour. - // Runs over ALL dags, gated on this dag actually holding the - // lease, so it fires exactly once per (dag, agent). - let mut released: Vec = Vec::new(); - for dag in &inner.dags { - for agent in dag.agents() { - if inner.leases.get(agent.as_str()) == Some(&dag.id) - && dag.agent_subgraph_terminal(&agent) - { - released.push(AgentRelease { - dag_id: dag.id, - agent, - }); - } - } - } - for rel in &released { - inner.leases.remove(&rel.agent); - } - inner.pending_agent_release.extend(released); - - // Whole-DAG terminal roll-up (approval resolution, intent revert, - // `Rebuilt` events) — still fires once per DAG. Leases are already - // freed by the per-agent pass above by the time we get here. - let mut reports: Vec = Vec::new(); - for dag in &mut inner.dags { - if !dag.is_terminal() || dag.terminal_reported { - continue; - } - dag.terminal_reported = true; - reports.push(TerminalDag { - dag_id: dag.id, - template: dag.template, - agents: dag.agents(), - approval_id: dag.approval_id, - state: dag.rollup(), - error: dag.first_error().map(str::to_owned), - }); - } - inner.pending_terminal.append(&mut reports); - Self::trim_history(inner, now_unix() - HISTORY_GRACE_SECS); - } - - /// Cancel a DAG that hasn't started yet (roll-up `Queued`): every - /// node flips to `Cancelled`. No-op (returns `false`) once any node - /// is running or terminal — an in-flight nix build isn't - /// interruptible, matching the old queue's rule. - pub fn cancel(&self, dag_id: u64) -> bool { - let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); - let Some(dag) = inner.dags.iter_mut().find(|d| d.id == dag_id) else { - return false; - }; - if dag.rollup() != State::Queued { - return false; - } + /// Mark a claimed node terminal, recording its outcome + (truncated) error. + /// The crate releases the node's build slot immediately and cascades the + /// `AfterOk` failure cancellation + subtree lease release; terminal hooks + /// run later as the terminal node (no drain). + pub fn complete_node(&self, _dag_id: u64, node_id: NodeId, result: Result<(), String>) { + let mut inner = self.lock(); let now = now_unix(); - for n in &mut dag.nodes { - n.state = State::Cancelled; - n.finished_at = Some(now); + let (error, outcome) = match result { + Ok(()) => (None, Outcome::Done), + Err(e) => (Some(truncate_error(&e)), Outcome::Failed), + }; + if let Some(rt) = inner.node_rt.get_mut(&node_id) { + rt.finished_at = Some(now); + rt.step = None; + if let Some(e) = error { + rt.error = Some(e); + } } - // Settle buffers the terminal roll-up; the notify wakes the - // scheduler, which drains it and fires the terminal hooks - // (approval resolution, power-intent revert). - Self::settle(&mut inner); + inner.sched.complete(node_id, outcome); + inner.trim_history(now - HISTORY_GRACE_SECS); drop(inner); self.notify.notify_one(); + } + + /// Cancel a DAG that hasn't started yet: every work node is still `Queued`, + /// so each is cancelled. No-op (`false`) once any node is running or + /// terminal — an in-flight nix build isn't interruptible. The terminal node + /// (a weak edge) still runs afterwards, so the cancel's terminal hooks + /// (approval resolution, power-intent revert) fire. + pub fn cancel(&self, dag_id: u64) -> bool { + let mut inner = self.lock(); + let inner = &mut *inner; + let Some(nodes) = inner.dag_nodes.get(&dag_id) else { + return false; + }; + let all_pending = nodes.iter().all(|&id| { + inner + .sched + .graph() + .node(id) + .is_some_and(|n| n.state == JobState::Pending) + }); + if !all_pending { + return false; + } + let ids: Vec = nodes.clone(); + for id in ids { + inner.sched.cancel_node(id); + } + self.notify.notify_one(); true } - /// Set the step label on a `Running` node. Returns `true` when the - /// label actually changed (callers emit a snapshot only then). + /// Set the step label on a `Running` node. Returns `true` when it changed. pub fn set_step(&self, dag_id: u64, node_id: NodeId, step: &str) -> bool { - let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); - let Some(node) = inner - .dags - .iter_mut() - .find(|d| d.id == dag_id) - .and_then(|d| d.node_mut(node_id)) - else { - return false; - }; - if node.state != State::Running || node.step.as_deref() == Some(step) { + let mut inner = self.lock(); + if inner.node_dag.get(&node_id) != Some(&dag_id) || !inner.node_running(node_id) { return false; } - node.step = Some(step.to_owned()); + let rt = inner.node_rt.entry(node_id).or_default(); + if rt.step.as_deref() == Some(step) { + return false; + } + rt.step = Some(step.to_owned()); true } - /// Set the step label on the DAG's currently-running node — - /// compatibility surface for the opaque approval pipeline, whose - /// callbacks only know the DAG id. Single-node approval DAGs make - /// this exact. + /// Set the step label on the DAG's currently-running node — the DAG-id-only + /// compatibility surface for the opaque approval pipeline. pub fn set_step_running(&self, dag_id: u64, step: &str) -> bool { - let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); - let Some(node) = inner - .dags - .iter_mut() - .find(|d| d.id == dag_id) - .and_then(|d| d.nodes.iter_mut().find(|n| n.state == State::Running)) - else { + let mut inner = self.lock(); + let Some(node_id) = inner.running_node_of(dag_id) else { return false; }; - if node.step.as_deref() == Some(step) { + let rt = inner.node_rt.entry(node_id).or_default(); + if rt.step.as_deref() == Some(step) { return false; } - node.step = Some(step.to_owned()); + rt.step = Some(step.to_owned()); true } /// Link a `build_logs` row to a specific `Running` node. pub fn set_build_log_id(&self, dag_id: u64, node_id: NodeId, log_id: i64) -> bool { - let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); - let Some(node) = inner - .dags - .iter_mut() - .find(|d| d.id == dag_id) - .and_then(|d| d.node_mut(node_id)) - else { - return false; - }; - if node.state != State::Running { + let mut inner = self.lock(); + if inner.node_dag.get(&node_id) != Some(&dag_id) || !inner.node_running(node_id) { return false; } - node.build_log_id = Some(log_id); + inner.node_rt.entry(node_id).or_default().build_log_id = Some(log_id); true } - /// Link a `build_logs` row to the DAG's currently-running node — - /// DAG-id-only compatibility surface (approval pipeline callbacks). + /// Link a `build_logs` row to the DAG's currently-running node — DAG-id-only + /// compatibility surface (approval pipeline callbacks). pub fn set_build_log_id_running(&self, dag_id: u64, log_id: i64) -> bool { - let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); - let Some(node) = inner - .dags - .iter_mut() - .find(|d| d.id == dag_id) - .and_then(|d| d.nodes.iter_mut().find(|n| n.state == State::Running)) - else { + let mut inner = self.lock(); + let Some(node_id) = inner.running_node_of(dag_id) else { return false; }; - node.build_log_id = Some(log_id); + inner.node_rt.entry(node_id).or_default().build_log_id = Some(log_id); true } - /// Snapshot every DAG for `/api/state` + `RebuildQueueChanged`. - pub fn snapshot(&self) -> Vec { - let inner = self.inner.lock().expect("job_queue mutex poisoned"); - inner.dags.iter().map(Dag::view).collect() + /// A DAG's terminal roll-up summary, computed on demand — a terminal hook + /// node's executor calls this to run its side effect. `None` if the DAG + /// is unknown (already history-trimmed). + #[must_use] + pub fn terminal_summary(&self, dag_id: u64) -> Option { + let inner = self.lock(); + let meta = inner.dag_meta.get(&dag_id)?; + Some(TerminalDag { + template: meta.template, + agents: inner.dag_agents(dag_id), + approval_id: meta.approval_id, + state: inner.dag_rollup(dag_id), + error: inner.dag_first_error(dag_id), + }) } - /// Number of live (non-terminal) DAGs — used by tests and - /// diagnostics. - #[cfg(test)] - pub fn live_count(&self) -> usize { - let inner = self.inner.lock().expect("job_queue mutex poisoned"); - inner.dags.iter().filter(|d| !d.is_terminal()).count() - } - - /// Keep only the newest `MAX_HISTORY_PER_TEMPLATE` terminal DAGs - /// per template. Never evicted: live DAGs; and terminal DAGs that - /// finished after `grace_cutoff` (see [`HISTORY_GRACE_SECS`]). - fn trim_history(inner: &mut Inner, grace_cutoff: i64) { - let mut counts: HashMap = HashMap::new(); - let kept: Vec = inner - .dags - .iter() - .rev() - .filter(|d| { - if !d.is_terminal() { - return true; - } - let finished = d.nodes.iter().filter_map(|n| n.finished_at).max(); - if finished.is_none_or(|t| t > grace_cutoff) { - return true; - } - let n = counts.entry(d.template).or_insert(0); - *n += 1; - *n <= MAX_HISTORY_PER_TEMPLATE + /// The `(dag_id, agent, kind)` triples for every per-agent lease currently + /// held by a DAG that carries a transient pill — the live transient-pill + /// set, a pull query over crate resource ownership (replaces the old + /// lease-release event stream). A DAG with no transient kind is omitted. + #[must_use] + pub fn held_transients(&self) -> Vec<(u64, String, TransientKind)> { + let inner = self.lock(); + inner + .sched + .resource_state() + .into_iter() + .filter_map(|(res, holder)| { + let Resource::Agent(agent) = res else { + return None; + }; + let dag = *inner.node_dag.get(&holder)?; + let kind = inner.dag_meta.get(&dag)?.transient?; + Some((dag, agent, kind)) }) - .cloned() - .collect(); - inner.dags = kept.into_iter().rev().collect(); + .collect() } - /// Test hook: trim with the grace window disabled, so eviction - /// behavior is assertable without aging real timestamps. + /// Snapshot every live + retained DAG for `/api/state` + `RebuildQueueChanged`. + #[must_use] + pub fn snapshot(&self) -> Vec { + let inner = self.lock(); + let mut ids: Vec = inner.dag_meta.keys().copied().collect(); + ids.sort_unstable(); + ids.into_iter().filter_map(|d| inner.dag_view(d)).collect() + } + + /// Number of live (non-terminal) DAGs — tests + diagnostics. + #[cfg(test)] + #[must_use] + pub fn live_count(&self) -> usize { + let inner = self.lock(); + inner + .dag_meta + .keys() + .filter(|&&d| !inner.dag_is_terminal(d)) + .count() + } + + /// Test hook: trim history with the grace window disabled. #[cfg(test)] pub(crate) fn trim_ignoring_grace(&self) { - let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); - Self::trim_history(&mut inner, i64::MAX); + let mut inner = self.lock(); + inner.trim_history(i64::MAX); } } + +impl QueueInner { + /// Whether `id` is a `Running` node. + fn node_running(&self, id: NodeId) -> bool { + self.sched + .graph() + .node(id) + .is_some_and(|n| n.state == JobState::Running) + } + + /// The DAG's currently-running work node, if any (the opaque approval + /// pipeline's single-node DAGs make this exact). + fn running_node_of(&self, dag_id: u64) -> Option { + self.dag_nodes + .get(&dag_id)? + .iter() + .copied() + .find(|&id| self.node_running(id)) + } + + /// Roll-up state over a DAG's work nodes: `Failed` if any failed; else + /// `Running` if any running; else `Queued` if any queued; else `Cancelled` + /// if any cancelled; else `Done`. + fn dag_rollup(&self, dag_id: u64) -> State { + let Some(nodes) = self.dag_nodes.get(&dag_id) else { + return State::Done; + }; + let mut any_running = false; + let mut any_queued = false; + let mut any_cancelled = false; + for &id in nodes { + match self.sched.graph().node(id).map(|n| n.state) { + Some(JobState::Failed) => return State::Failed, + Some(JobState::Running | JobState::Finishing) => any_running = true, + Some(JobState::Pending) => any_queued = true, + Some(JobState::Cancelled) => any_cancelled = true, + Some(JobState::Done) | None => {} + } + } + if any_running { + State::Running + } else if any_queued { + State::Queued + } else if any_cancelled { + State::Cancelled + } else { + State::Done + } + } + + /// True when every work node of the DAG is terminal. + fn dag_is_terminal(&self, dag_id: u64) -> bool { + self.dag_nodes.get(&dag_id).is_some_and(|nodes| { + nodes.iter().all(|&id| { + self.sched + .graph() + .node(id) + .is_some_and(|n| n.state.is_terminal()) + }) + }) + } + + /// Distinct agents a DAG's work nodes target, in first-seen order. + fn dag_agents(&self, dag_id: u64) -> Vec { + let mut seen: Vec = Vec::new(); + if let Some(nodes) = self.dag_nodes.get(&dag_id) { + for &id in nodes { + if let Some(n) = self.sched.graph().node(id) { + let agent = &n.payload.agent; + if !agent.is_empty() && !seen.iter().any(|s| s == agent) { + seen.push(agent.clone()); + } + } + } + } + seen + } + + /// First failed work node's stored error, for the roll-up `error` field. + fn dag_first_error(&self, dag_id: u64) -> Option { + let nodes = self.dag_nodes.get(&dag_id)?; + for &id in nodes { + if self + .sched + .graph() + .node(id) + .is_some_and(|n| n.state == JobState::Failed) + && let Some(e) = self.node_rt.get(&id).and_then(|r| r.error.clone()) + { + return Some(e); + } + } + None + } + + /// Rebuild the wire [`DagView`] for a DAG from its metadata + work nodes + + /// per-node runtime. The internal terminal node is excluded. + fn dag_view(&self, dag_id: u64) -> Option { + let meta = self.dag_meta.get(&dag_id)?; + let node_ids = self.dag_nodes.get(&dag_id)?; + let mut nodes = Vec::with_capacity(node_ids.len()); + let mut started: Vec = Vec::new(); + let mut finished: Vec = Vec::new(); + for &id in node_ids { + let Some(node) = self.sched.graph().node(id) else { + continue; + }; + let rt = self.node_rt.get(&id); + let deps: Vec = node + .deps + .iter() + .filter_map(|d| match d { + Dep::Node { id, .. } => Some(id.get()), + Dep::Resource { .. } => None, + }) + .collect(); + if let Some(s) = rt.and_then(|r| r.started_at) { + started.push(s); + } + if let Some(fin) = rt.and_then(|r| r.finished_at) { + finished.push(fin); + } + nodes.push(NodeView { + id: id.get(), + agent: node.payload.agent.clone(), + kind: node.payload.kind.as_str().to_owned(), + deps, + state: to_wire_state(node.state), + step: rt.and_then(|r| r.step.clone()), + build_log_id: rt.and_then(|r| r.build_log_id), + started_at: rt.and_then(|r| r.started_at), + finished_at: rt.and_then(|r| r.finished_at), + error: rt.and_then(|r| r.error.clone()), + }); + } + let is_terminal = self.dag_is_terminal(dag_id); + Some(DagView { + id: dag_id, + kind: meta.template, + state: self.dag_rollup(dag_id), + source: meta.source, + reason: meta.reason.clone(), + enqueued_at: meta.created_at, + started_at: started.into_iter().min(), + finished_at: if is_terminal { + finished.into_iter().max() + } else { + None + }, + inputs: meta.inputs.clone(), + approval_id: meta.approval_id, + perm_payload: meta.perm_payload.clone(), + nodes, + }) + } + + /// Keep only the newest [`MAX_HISTORY_PER_TEMPLATE`] terminal DAGs per + /// template, evicting older ones' side-tables (their now-terminal crate + /// nodes linger harmlessly in the graph — a bounded-prune primitive is a + /// tracked follow-up). Terminal DAGs finished after `grace_cutoff` are + /// exempt (never counted). + fn trim_history(&mut self, grace_cutoff: i64) { + let mut terminal: Vec<(u64, Template, i64)> = self + .dag_meta + .keys() + .copied() + .filter(|&d| self.dag_is_terminal(d)) + .map(|d| { + let finished = self + .dag_nodes + .get(&d) + .into_iter() + .flatten() + .filter_map(|id| self.node_rt.get(id).and_then(|r| r.finished_at)) + .max() + .unwrap_or(0); + (d, self.dag_meta[&d].template, finished) + }) + .collect(); + // Newest first, so the cap keeps the most recent per template. + terminal.sort_by(|a, b| b.2.cmp(&a.2).then(b.0.cmp(&a.0))); + let mut counts: HashMap = HashMap::new(); + let mut evict: Vec = Vec::new(); + for (d, template, finished) in terminal { + if finished > grace_cutoff { + continue; + } + let c = counts.entry(template).or_insert(0); + *c += 1; + if *c > MAX_HISTORY_PER_TEMPLATE { + evict.push(d); + } + } + for d in evict { + if let Some(nodes) = self.dag_nodes.remove(&d) { + for id in nodes { + self.node_dag.remove(&id); + self.node_rt.remove(&id); + } + } + if let Some(term) = self.terminal_node.remove(&d) { + self.node_dag.remove(&term); + self.node_rt.remove(&term); + } + self.dag_meta.remove(&d); + } + } +} + +/// A spec dependency index (`Dep.on`, a wire `u64`) as a `usize` for indexing +/// into the node/id vectors. `templates::validate` guarantees it's in range. +fn dep_index(on: u64) -> usize { + usize::try_from(on).unwrap_or(usize::MAX) +} + +/// Truncate a node error to [`MAX_ERROR_LEN`] on a char boundary, appending `…`. +fn truncate_error(e: &str) -> String { + if e.len() <= MAX_ERROR_LEN { + return e.to_owned(); + } + let cut = (0..=MAX_ERROR_LEN) + .rev() + .find(|i| e.is_char_boundary(*i)) + .unwrap_or(0); + let mut msg = e[..cut].to_owned(); + msg.push('…'); + msg +} diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs index a20a1ed8..619572b9 100644 --- a/hive-c0re/src/job_queue/model.rs +++ b/hive-c0re/src/job_queue/model.rs @@ -11,7 +11,7 @@ //! DAG can span agents). See `docs/coordinator.md::Job queue` for the //! full design. -pub use hive_sh4re::jobs::{DagView, NodeId, NodeView, PermPayload, Source, State, Template}; +pub use hive_sh4re::jobs::{DagView, NodeId, PermPayload, Source, State, Template}; use serde::Serialize; /// When a dependency edge is considered satisfied. @@ -139,6 +139,21 @@ pub enum NodeKind { /// `Prebuild`, but that's a no-op there — the agent is down, so prebuild /// is skipped.) SetWanted { up: bool }, + /// Per-DAG terminal hook (approval-driven DAGs — spawn / opaque deploy): + /// resolve the DAG's approval row from the rolled-up outcome. Appended once + /// with a weak (`AfterAny`) edge on the DAG's tails, so it runs exactly when + /// the DAG has settled (any outcome, including a cancel before starting). + /// Build-slot- and lease-exempt; always runs (weak edge ⇒ never cascaded). + ResolveApproval, + /// Per-DAG terminal hook (rebuild / perm-change DAGs): emit one `Rebuilt` + /// manager event per targeted agent — `ok` on `Done`, `!ok` on `Failed`, + /// none on cancel. Appended weak-dep on the tails; slot/lease-exempt. + EmitRebuilt, + /// Per-DAG terminal hook (power-op DAGs): on a *cancelled* DAG, revert each + /// agent's `wanted` intent to its observed state — the operator's cancel + /// means "don't do it". Noop on any non-cancelled outcome. Appended weak-dep + /// on the tails; slot/lease-exempt. + RevertIntent, } impl NodeKind { @@ -161,6 +176,9 @@ impl NodeKind { NodeKind::WritePermFile => "write_perm_file", NodeKind::ApprovalDeploy => "approval_deploy", NodeKind::SetWanted { .. } => "set_wanted", + NodeKind::ResolveApproval => "resolve_approval", + NodeKind::EmitRebuilt => "emit_rebuilt", + NodeKind::RevertIntent => "revert_intent", } } @@ -201,39 +219,23 @@ impl NodeKind { } } -/// One schedulable unit inside a DAG. -#[derive(Debug, Clone)] -pub struct Node { - pub id: NodeId, - /// The agent this node's work targets. Per-node so a single DAG can - /// span agents (e.g. a hive-wide restart); the lifecycle lease is - /// acquired against *this* agent (still globally exclusive per agent - /// across all DAGs). `"hyperhive"` for meta-level nodes. - pub agent: String, - pub kind: NodeKind, - pub deps: Vec, - pub state: State, - /// Live sub-label while `Running` (kept for parity with the old - /// per-entry `step`). - pub step: Option, - /// Row id of the `build_logs` entry this node opened (`Prebuild` / - /// `Swap` / `ApprovalDeploy`), for the dashboard's live-stream link. - pub build_log_id: Option, - pub started_at: Option, - pub finished_at: Option, - /// Populated when `state == Failed` (truncated by the queue). - pub error: Option, -} - /// Submit-time spec for one node. #[derive(Debug, Clone)] pub struct NodeSpec { - /// The agent this node targets (see [`Node::agent`]). Built by the - /// `templates.rs` `node` helper, which stamps the template's agent - /// onto every node. + /// The agent this node's work targets. Built by the `templates.rs` `node` + /// helper, which stamps the template's agent onto every node. pub agent: String, pub kind: NodeKind, pub deps: Vec, + /// The **structural parent** axis — the spec-local index of this node's + /// group parent, or `None` for a top-level (group-root) node. Independent + /// of `deps`: `deps` order execution, `parent` groups nodes into a subtree + /// whose resource the whole subtree borrows (the agent lease is owned by a + /// group root and re-entered by its descendants for continuity). A child + /// runs once its parent reaches `Finishing` (the parent gate), so a child + /// never `deps` on its own parent (that would deadlock — dep-scope + /// validation rejects it). + pub parent: Option, } /// Submit-time spec for a whole DAG. Built by `templates.rs`; validated @@ -258,140 +260,3 @@ pub struct DagSpec { pub transient: Option, pub nodes: Vec, } - -/// A live DAG in the queue. No DAG-level `agent`: agent is per-[`Node`], -/// so a DAG can span agents. Per-agent leasing is derived from the -/// nodes' agents. -#[derive(Debug, Clone)] -pub struct Dag { - pub id: u64, - pub template: Template, - pub source: Source, - pub reason: String, - pub approval_id: Option, - pub inputs: Vec, - pub perm_payload: Option, - pub transient: Option, - pub created_at: i64, - pub nodes: Vec, - /// Terminal roll-up already reported to the scheduler's hooks - /// (approval resolution, transient release). Internal bookkeeping, - /// never serialized. - pub terminal_reported: bool, -} - -impl Dag { - /// Roll-up state: `Failed` if any node failed; else `Running` if - /// any running; else `Queued` if any queued; else `Cancelled` if - /// any cancelled; else `Done`. - pub fn rollup(&self) -> State { - let mut any_cancelled = false; - let mut any_queued = false; - let mut any_running = false; - for n in &self.nodes { - match n.state { - State::Failed => return State::Failed, - State::Running => any_running = true, - State::Queued => any_queued = true, - State::Cancelled => any_cancelled = true, - State::Done => {} - } - } - if any_running { - State::Running - } else if any_queued { - State::Queued - } else if any_cancelled { - State::Cancelled - } else { - State::Done - } - } - - /// True when every node is terminal. - pub fn is_terminal(&self) -> bool { - self.nodes.iter().all(|n| n.state.is_terminal()) - } - - /// True when no live (non-terminal) node of this DAG still targets - /// `agent` — i.e. that agent's subgraph within the DAG has settled. - /// Used to release an agent's lifecycle lease the moment its own - /// work is done, rather than waiting for the whole DAG to terminate. - /// Vacuously true for an agent the DAG has no node for; callers gate - /// on actually holding that agent's lease first. - pub fn agent_subgraph_terminal(&self, agent: &str) -> bool { - self.nodes - .iter() - .filter(|n| n.agent == agent) - .all(|n| n.state.is_terminal()) - } - - /// First failed node's error, for the roll-up `error` field. - pub fn first_error(&self) -> Option<&str> { - self.nodes - .iter() - .find(|n| n.state == State::Failed) - .and_then(|n| n.error.as_deref()) - } - - pub fn node(&self, id: NodeId) -> Option<&Node> { - self.nodes.iter().find(|n| n.id == id) - } - - pub fn node_mut(&mut self, id: NodeId) -> Option<&mut Node> { - self.nodes.iter_mut().find(|n| n.id == id) - } - - /// Distinct agents this DAG's nodes target, in first-seen order. - /// Used for per-agent lease release and the terminal cancel-revert — - /// a single-agent DAG yields one, a multi-agent DAG yields several. - pub fn agents(&self) -> Vec { - let mut seen: Vec = Vec::new(); - for n in &self.nodes { - if !seen.iter().any(|a| a == &n.agent) { - seen.push(n.agent.clone()); - } - } - seen - } -} - -impl Dag { - pub fn view(&self) -> DagView { - let started_at = self.nodes.iter().filter_map(|n| n.started_at).min(); - let finished_at = if self.is_terminal() { - self.nodes.iter().filter_map(|n| n.finished_at).max() - } else { - None - }; - DagView { - id: self.id, - kind: self.template, - state: self.rollup(), - source: self.source, - reason: self.reason.clone(), - enqueued_at: self.created_at, - started_at, - finished_at, - inputs: self.inputs.clone(), - approval_id: self.approval_id, - perm_payload: self.perm_payload.clone(), - nodes: self - .nodes - .iter() - .map(|n| NodeView { - id: n.id, - agent: n.agent.clone(), - kind: n.kind.as_str().to_owned(), - deps: n.deps.iter().map(|d| d.on).collect(), - state: n.state, - step: n.step.clone(), - build_log_id: n.build_log_id, - started_at: n.started_at, - finished_at: n.finished_at, - error: n.error.clone(), - }) - .collect(), - } - } -} diff --git a/hive-c0re/src/job_queue/resource.rs b/hive-c0re/src/job_queue/resource.rs new file mode 100644 index 00000000..9358af5f --- /dev/null +++ b/hive-c0re/src/job_queue/resource.rs @@ -0,0 +1,60 @@ +//! The concrete resource + payload types the rebuild queue schedules over — +//! the bridge from hive-c0re's [`NodeKind`] onto the domain-agnostic +//! `hive-jobq` crate. `hive-jobq` is generic over a resource type +//! `R: Clone + Eq + Hash` and a node payload `N`; here `R` is [`Resource`] and +//! `N` is [`JobPayload`]. + +use hive_jobq::Dep; + +use super::model::NodeKind; + +/// The two resource classes the queue gates concurrency on, as the crate's +/// generic resource type `R`. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Resource { + /// One of the `buildSlots` permits, held by a nix-heavy node for its + /// duration. Capacity is `services.hyperhive.c0re.buildSlots` (default 1), + /// set on the [`hive_jobq::resources::ResourceTable`] at construction. + BuildSlot, + /// The per-agent lifecycle lease — globally exclusive per agent across all + /// DAGs (unconfigured, so the crate's default capacity 1 applies). Held by + /// a DAG's first container-affecting node for that agent and re-entered by + /// the rest of that agent's subtree via the crate's recursive lock, so two + /// DAGs never interleave container ops on one agent. + Agent(String), +} + +/// A schedulable node's payload — the crate's generic `N`. Carries the +/// primitive operation and the agent it targets. The agent is per-node (a DAG +/// spans agents), and the lease [`Resource::Agent`] is keyed on it. +#[derive(Debug, Clone)] +pub struct JobPayload { + pub kind: NodeKind, + pub agent: String, +} + +impl JobPayload { + /// The [`Dep::Resource`] edges this node must acquire to run, derived from + /// its kind + agent: a build slot for nix-heavy kinds + /// ([`NodeKind::needs_build_slot`]) and the agent lease for + /// container-affecting kinds ([`NodeKind::needs_lease`]). Lease-exempt + /// container ops (`Start` / `Stop`, fanned out by a lease-holding + /// `Reconcile`) hold no lease of their own — they re-enter the ancestor's + /// `Agent` lock through the crate's recursive re-entrancy. + pub fn resource_deps(&self) -> Vec> { + let mut deps = Vec::new(); + if self.kind.needs_build_slot() { + deps.push(Dep::Resource { + name: Resource::BuildSlot, + count: 1, + }); + } + if self.kind.needs_lease() { + deps.push(Dep::Resource { + name: Resource::Agent(self.agent.clone()), + count: 1, + }); + } + deps + } +} diff --git a/hive-c0re/src/job_queue/scheduler.rs b/hive-c0re/src/job_queue/scheduler.rs index 3e9959d5..1b9a843d 100644 --- a/hive-c0re/src/job_queue/scheduler.rs +++ b/hive-c0re/src/job_queue/scheduler.rs @@ -1,18 +1,25 @@ -//! The single scheduler task that drives all DAGs: claim every ready -//! node (as many as the build slots / leases allow), spawn one -//! executor task per claim, and on any completion re-evaluate. -//! Concurrency comes from the build-slot count, not multiple workers. +//! The single scheduler task that drives all DAGs: claim every ready node (as +//! many as the build slots / leases allow), spawn one executor task per claim, +//! and on any completion re-evaluate. Concurrency comes from the build-slot +//! count, not multiple workers. //! -//! Also owns the per-DAG transient guard (dashboard pill + crash-watch -//! suppression) that the sync queue core can't hold itself — created when a -//! DAG acquires its agent lease, dropped when the DAG settles terminal. +//! Owns the per-DAG transient guard (dashboard pill + crash-watch suppression) +//! that the sync queue core can't hold itself. The guard set is *reconciled* +//! from live lease ownership ([`super::JobQueue::held_transients`]) each loop: +//! a `(dag, agent)` pill exists for exactly as long as that agent's lease is +//! held, so it appears when the agent's owner node starts and disappears when +//! its subgraph settles — one pill per agent a DAG touches. //! -//! In-DAG growth (a `MetaLock` growing rebuild subgraphs after the lock -//! bump, a `Reconcile` fanning its `Start`/`Stop`) flows through -//! `NodeOutput.append_subgraph`, applied before the emitting node -//! completes — see `handle_completion`. +//! Per-DAG terminal work (approval resolution, `Rebuilt`, cancelled-power-op +//! intent revert) is not drained here: it runs as the DAG's focused terminal +//! node (`ResolveApproval` / `EmitRebuilt` / `RevertIntent`), dispatched through +//! `exec::run_node` like any other node once the DAG settles. +//! +//! In-DAG growth (a `MetaLock` growing rebuild subgraphs, a `Reconcile` fanning +//! its `Start`/`Stop`) flows through `NodeOutput.append_subgraph`, applied +//! before the emitting node completes — see `handle_completion`. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use super::Claim; @@ -26,38 +33,24 @@ struct NodeDone { /// Scheduler loop. Spawned once at hive-c0re startup from `main.rs`. /// -/// Shutdown semantics: subscribes to `coord.shutdown_rx()`. On a true -/// signal the loop exits immediately; already-running node tasks ride -/// the runtime down with the process, and pending `Queued` DAGs are -/// dropped — desired state is re-derived on next boot (boot sweep + -/// reconcile), so the in-memory queue is deliberately not durable. +/// Shutdown semantics: subscribes to `coord.shutdown_rx()`. On a true signal +/// the loop exits immediately; already-running node tasks ride the runtime down +/// with the process, and pending `Queued` DAGs are dropped — desired state is +/// re-derived on next boot (boot sweep + reconcile), so the in-memory queue is +/// deliberately not durable. pub async fn run_worker(coord: Arc) { let mut shutdown = coord.shutdown_rx(); let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); - // (DAG id, agent) → transient guard held for that agent's lease - // window. Keyed per-agent so a multi-agent DAG shows one transient - // pill per agent it touches. + // (DAG id, agent) → transient guard held for that agent's lease window. let mut transients: HashMap<(u64, String), crate::coordinator::TransientGuard> = HashMap::new(); loop { - // Terminal roll-ups can appear without a node completion — - // the cancel surfaces settle DAGs directly and wake this loop - // via notify — so drain on every iteration, not just inside - // handle_completion. - process_terminals(&coord, &mut transients).await; + reconcile_transients(&coord, &mut transients); let claims = coord.job_queue.claim_ready(); if !claims.is_empty() { for claim in claims { - if claim.lease_acquired - && let Some(kind) = claim.transient - { - transients.insert( - (claim.dag_id, claim.agent.clone()), - coord.transient_guard(&claim.agent, kind), - ); - } tracing::info!( dag = claim.dag_id, - node = claim.node_id, + node = claim.node_id.get(), kind = claim.kind.as_str(), agent = %claim.agent, template = claim.template.as_str(), @@ -71,6 +64,8 @@ pub async fn run_worker(coord: Arc) { let _ = tx.send(NodeDone { claim, result }); }); } + // Newly-started owner nodes now hold their leases — surface the pills. + reconcile_transients(&coord, &mut transients); coord.emit_rebuild_queue_snapshot(); continue; } @@ -83,35 +78,30 @@ pub async fn run_worker(coord: Arc) { } } Some(done) = rx.recv() => { - handle_completion(&coord, &mut transients, done).await; + handle_completion(&coord, done); } () = coord.job_queue.notify.notified() => {} } } } -async fn handle_completion( - coord: &Arc, - transients: &mut HashMap<(u64, String), crate::coordinator::TransientGuard>, - done: NodeDone, -) { +fn handle_completion(coord: &Arc, done: NodeDone) { let NodeDone { claim, result } = done; match result { Ok(output) => { tracing::info!( dag = claim.dag_id, - node = claim.node_id, + node = claim.node_id.get(), "job_queue: node done" ); // Append any in-DAG subgraphs BEFORE completing this node, so // completing it doesn't roll the DAG terminal while the appended - // work is still pending — that keeps the lease-window transient - // held across it. Each subgraph is independent, rooted on this - // node (`AfterOk`), so it becomes ready the instant this one - // settles `Done` just below. Covers both the multi-node case (a - // `MetaLock` growing per-agent rebuild subgraphs) and the - // single-node case (a `Reconcile` planner's `Start` / `Stop`). - for subgraph in output.append_subgraph { + // work is still pending. Each subgraph roots on this node + // (`AfterOk`), so it becomes ready the instant this one settles + // `Done` just below — covers both the multi-node case (a `MetaLock` + // growing per-agent rebuild subgraphs) and the single-node case (a + // `Reconcile` planner's `Start` / `Stop`). + for subgraph in &output.append_subgraph { coord .job_queue .append_subgraph(claim.dag_id, subgraph, claim.node_id); @@ -124,7 +114,7 @@ async fn handle_completion( let msg = format!("{e:#}"); tracing::warn!( dag = claim.dag_id, - node = claim.node_id, + node = claim.node_id.get(), kind = claim.kind.as_str(), agent = %claim.agent, error = %msg, @@ -135,30 +125,23 @@ async fn handle_completion( .complete_node(claim.dag_id, claim.node_id, Err(msg)); } } - process_terminals(coord, transients).await; + // The next loop iteration re-reconciles the transient pills against the + // post-completion lease state (a settled subgraph drops its pill). coord.emit_rebuild_queue_snapshot(); } -/// Drain per-agent lease releases and buffered terminal roll-ups. -/// -/// Per-agent first: an agent's subgraph within a DAG went terminal (its -/// lease was freed in `settle`), so drop that agent's `(dag, agent)` -/// transient pill now — ahead of whole-DAG terminal for a multi-agent -/// DAG. Then the whole-DAG terminals: drop any remaining transient the -/// DAG still held and run the terminal hook (approval resolution, -/// `Rebuilt` events, cancelled-power-op intent revert). -async fn process_terminals( +/// Reconcile the transient-guard set against live lease ownership: drop pills +/// whose lease is no longer held, create one for each newly-held `(dag, agent)`. +fn reconcile_transients( coord: &Arc, transients: &mut HashMap<(u64, String), crate::coordinator::TransientGuard>, ) { - for rel in coord.job_queue.drain_agent_releases() { - transients.remove(&(rel.dag_id, rel.agent)); - } - for terminal in coord.job_queue.drain_terminal() { - // Drop any per-agent transient guard the DAG still held (the - // per-agent pass above already dropped the ones whose subgraphs - // settled early). - transients.retain(|(dag_id, _), _| *dag_id != terminal.dag_id); - exec::on_dag_terminal(coord, &terminal).await; + let held = coord.job_queue.held_transients(); + let keys: HashSet<(u64, String)> = held.iter().map(|(d, a, _)| (*d, a.clone())).collect(); + transients.retain(|k, _| keys.contains(k)); + for (dag_id, agent, kind) in held { + transients + .entry((dag_id, agent.clone())) + .or_insert_with(|| coord.transient_guard(&agent, kind)); } } diff --git a/hive-c0re/src/job_queue/submit.rs b/hive-c0re/src/job_queue/submit.rs index 805e4d78..1a741813 100644 --- a/hive-c0re/src/job_queue/submit.rs +++ b/hive-c0re/src/job_queue/submit.rs @@ -25,7 +25,7 @@ use std::sync::Arc; use super::model::{DagSpec, Dep, NodeKind, NodeSpec, Template}; -use super::templates::{after_ok, node, rebuild_nodes}; +use super::templates::{after_ok, child, node, rebuild_nodes}; use super::{Source, templates}; use crate::coordinator::{Coordinator, TransientKind}; use crate::lifecycle; @@ -60,13 +60,16 @@ pub fn rebuild(coord: &Arc, agent: &str, source: Source, reason: St /// stays even for a down agent so a race-up between the state read and exec /// is still stopped in-DAG. fn stop_chain(agent: &str, graceful: bool, running: bool) -> Vec { + // `SetWanted` is the group root and owns the agent lease; the mechanical + // steps are its children (borrow the lease, run once it reaches `Finishing`, + // dep-ordered among themselves). let mut n = vec![node(agent, NodeKind::SetWanted { up: false }, Vec::new())]; if graceful && running { - n.push(node(agent, NodeKind::Signal, after_ok(0))); - n.push(node(agent, NodeKind::Drain, after_ok(1))); - n.push(node(agent, NodeKind::Reconcile, after_ok(2))); + n.push(child(0, agent, NodeKind::Signal, Vec::new())); + n.push(child(0, agent, NodeKind::Drain, after_ok(1))); + n.push(child(0, agent, NodeKind::Reconcile, after_ok(2))); } else { - n.push(node(agent, NodeKind::Reconcile, after_ok(0))); + n.push(child(0, agent, NodeKind::Reconcile, Vec::new())); } n } @@ -78,11 +81,12 @@ fn stop_chain(agent: &str, graceful: bool, running: bool) -> Vec { fn start_chain(agent: &str, running: bool, stale: bool) -> Vec { let mut n = vec![node(agent, NodeKind::SetWanted { up: true }, Vec::new())]; if !running && stale { - // Rebuild subgraph rooted at the SetWanted head (base = 1, so - // `Prebuild` deps `after_ok(0)` = the head). + // Rebuild subtree after the SetWanted head (base = 1, so the rebuild's + // `Prebuild` root deps `after_ok(0)` = the head). `Prebuild` + + // `Reconcile` are their own group roots (top-level, per `rebuild_nodes`). n.extend(rebuild_nodes(agent, true, 1)); } else { - n.push(node(agent, NodeKind::Reconcile, after_ok(0))); + n.push(child(0, agent, NodeKind::Reconcile, Vec::new())); } n } @@ -101,18 +105,30 @@ fn restart_chain(agent: &str, graceful: bool, running: bool) -> Vec { // Nothing to bounce — a lone Reconcile converges to intent. return vec![node(agent, NodeKind::Reconcile, Vec::new())]; } - // Running: mechanical stop then Reconcile. The first stop node is the - // subgraph root (no SetWanted head) and acquires the agent lease. - let mut n = Vec::new(); - if graceful { - n.push(node(agent, NodeKind::Signal, Vec::new())); - n.push(node(agent, NodeKind::Drain, after_ok(0))); - n.push(node(agent, NodeKind::StopForUpdate, after_ok(1))); + // Running: mechanical stop then Reconcile. The first stop node is the group + // ROOT (no SetWanted head) and owns the agent lease; the rest are its + // children (borrow the lease, dep-ordered), so the bounce holds one + // continuous lease and `Reconcile` cancel-cascades if a stop step fails. + let mut n = vec![if graceful { + node(agent, NodeKind::Signal, Vec::new()) } else { - n.push(node(agent, NodeKind::StopForUpdate, Vec::new())); + node(agent, NodeKind::StopForUpdate, Vec::new()) + }]; + if graceful { + n.push(child(0, agent, NodeKind::Drain, Vec::new())); + n.push(child(0, agent, NodeKind::StopForUpdate, after_ok(1))); } - let stop_idx = u32::try_from(n.len() - 1).unwrap_or(0); - n.push(node(agent, NodeKind::Reconcile, after_ok(stop_idx))); + // `Reconcile` gates on the last mechanical step. When the only step is the + // root itself (non-graceful, `StopForUpdate` == index 0), the parent gate + // already orders `Reconcile` after it — a child must NOT dep on its own + // parent (dep-scope). So the sibling dep is added only for a graceful + // bounce, where the last step is a sibling child. + let deps = if n.len() > 1 { + after_ok(u64::try_from(n.len() - 1).unwrap_or(0)) + } else { + Vec::new() + }; + n.push(child(0, agent, NodeKind::Reconcile, deps)); n } @@ -124,7 +140,7 @@ fn restart_chain(agent: &str, graceful: bool, running: bool) -> Vec { fn concat_subgraphs(chains: Vec>) -> Vec { let mut out: Vec = Vec::new(); for chain in chains { - let base = u32::try_from(out.len()).unwrap_or(u32::MAX); + let base = u64::try_from(out.len()).unwrap_or(u64::MAX); for spec in chain { let deps = spec .deps @@ -138,6 +154,10 @@ fn concat_subgraphs(chains: Vec>) -> Vec { agent: spec.agent, kind: spec.kind, deps, + // Rebase the structural parent by the same offset (a subgraph + // root keeps `parent = None`, so the per-agent groups stay + // independent + concurrent). + parent: spec.parent.map(|p| base + p), }); } } diff --git a/hive-c0re/src/job_queue/templates.rs b/hive-c0re/src/job_queue/templates.rs index 898f7b96..460655c8 100644 --- a/hive-c0re/src/job_queue/templates.rs +++ b/hive-c0re/src/job_queue/templates.rs @@ -35,34 +35,58 @@ use crate::coordinator::TransientKind; /// After-ok edge on the previous node — the common chain link. Shared with /// the async power-op builders in `submit.rs` (which assemble per-agent /// chains dynamically from live container state). -pub(crate) fn after_ok(on: u32) -> Vec { +pub(crate) fn after_ok(on: u64) -> Vec { vec![Dep { on, when: DepWhen::AfterOk, }] } -/// Build one node targeting `agent`. The single place a node's agent is -/// stamped. Shared with `submit.rs`'s dynamic power-op builders. +/// Build one **top-level (group-root)** node targeting `agent` — `parent = +/// None`. The single place a node's agent is stamped. Shared with `submit.rs`'s +/// dynamic power-op builders. A root owns whatever resource it declares for its +/// whole subtree; its descendants borrow it (agent-lease / build-slot +/// continuity). Ordering vs other nodes is `deps`; grouping is `parent`. pub(crate) fn node(agent: &str, kind: NodeKind, deps: Vec) -> NodeSpec { NodeSpec { agent: agent.to_owned(), kind, deps, + parent: None, } } -/// The rebuild node chain. `PostSwap` carries the swap's Ok-only -/// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan) and deps -/// `Swap` with `AfterOk`. `Reconcile` then deps on `PostSwap` with -/// `AfterAny`: it must run even when the swap failed, so a previously-up -/// agent comes back on its old config (today's recovery-start). On swap -/// failure the `AfterOk` `PostSwap` is cancel-cascaded to a terminal state, -/// which still satisfies `Reconcile`'s `AfterAny` edge — the only `AfterAny` -/// edge in v1. Pointing `Reconcile` at `PostSwap` (not `Swap`) also -/// serializes the tail ahead of the reconcile, so there's no double -/// rescan/kick race. -pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u32) -> Vec { +/// Build a **child** node whose structural parent is spec-index `parent`. The +/// child runs once its parent reaches `Finishing` (the parent gate), so it must +/// NOT `deps` on `parent` (dep-scope validation rejects a dep on one's own +/// parent). `deps` here order the child against its *siblings* only. +pub(crate) fn child(parent: u64, agent: &str, kind: NodeKind, deps: Vec) -> NodeSpec { + NodeSpec { + agent: agent.to_owned(), + kind, + deps, + parent: Some(parent), + } +} + +/// The rebuild node subtree (nested, two group roots). `base` is the spec index +/// of the first node (`Prebuild`). Structure: +/// - `Prebuild` (base+0, **root**): owns the build slot for the whole subtree. +/// Lease-exempt — the nix build overlaps other DAGs on the same agent. +/// - `StopForUpdate` (base+1, child of `Prebuild`): owns the agent lease. Runs +/// once `Prebuild` reaches `Finishing` (parent gate). +/// - `Swap` (base+2, child of `StopForUpdate`): borrows the agent lease from its +/// parent and the build slot from grand-ancestor `Prebuild` — both continuous. +/// - `PostSwap` (base+3, child of `StopForUpdate`): the swap's Ok-only +/// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan), `AfterOk` +/// its sibling `Swap`. +/// - `Reconcile` (base+4, **root**): `AfterAny` `Prebuild`, which rolls up +/// terminal only once its whole mechanical subtree (SFU→Swap→PostSwap) has +/// settled — so `Reconcile` runs after the swap regardless of outcome, and as +/// a top-level root it survives the cancel-cascade of a failed `Prebuild` +/// (recovery-start invariant). It takes a fresh lease; the tiny gap is +/// harmless — `Reconcile` converges to the persisted `wanted` idempotently. +pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u64) -> Vec { vec![ node( agent, @@ -73,14 +97,14 @@ pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u32) -> Vec DagSpec { DagSpec { template: Template::Spawn, @@ -161,9 +190,9 @@ pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec { transient: Some(TransientKind::Spawning), nodes: vec![ node(agent, NodeKind::Provision, Vec::new()), - node(agent, NodeKind::Create, after_ok(0)), - node(agent, NodeKind::WriteDropin, after_ok(1)), - node(agent, NodeKind::Reconcile, after_ok(2)), + child(0, agent, NodeKind::Create, Vec::new()), + child(1, agent, NodeKind::WriteDropin, Vec::new()), + child(1, agent, NodeKind::Reconcile, after_ok(2)), ], } } @@ -241,7 +270,7 @@ pub fn validate(spec: &DagSpec) -> Result<()> { .collect(); for (i, node) in spec.nodes.iter().enumerate() { for dep in &node.deps { - let Some(&dep_idx) = idx.get(dep.on as usize) else { + let Some(&dep_idx) = usize::try_from(dep.on).ok().and_then(|i| idx.get(i)) else { bail!( "dag spec {:?} node {i} depends on unknown node {}", spec.template, diff --git a/hive-c0re/src/job_queue/tests.rs b/hive-c0re/src/job_queue/tests.rs index eae41d91..33608bae 100644 --- a/hive-c0re/src/job_queue/tests.rs +++ b/hive-c0re/src/job_queue/tests.rs @@ -115,6 +115,7 @@ fn cyclic_dag_is_rejected_at_submit() { on: 1, when: DepWhen::AfterOk, }], + parent: None, }, NodeSpec { agent: "agent-a".to_owned(), @@ -123,6 +124,7 @@ fn cyclic_dag_is_rejected_at_submit() { on: 0, when: DepWhen::AfterOk, }], + parent: None, }, ]; assert!(q.submit(spec).is_err(), "cyclic spec must be refused"); @@ -140,6 +142,7 @@ fn unknown_dep_is_rejected_at_submit() { on: 9, when: DepWhen::AfterOk, }], + parent: None, }]; assert!(q.submit(spec).is_err()); } @@ -180,13 +183,16 @@ fn build_slot_serializes_nix_heavy_nodes() { assert_eq!(first.dag_id, a); assert_eq!(first.kind.as_str(), "prebuild"); q.complete_node(a, first.node_id, Ok(())); - // With the slot free again, FIFO gives... a's StopForUpdate is - // slot-free (lease) and b's Prebuild takes the slot — both run. + // Uniform hold: agent-a keeps the build slot across its whole build chain + // (Swap re-enters it), so a's StopForUpdate (lease, slot-free) runs but b's + // Prebuild must wait for a's slot-needers (through Swap) to finish. let claims = q.claim_ready(); let kinds: Vec<(u64, &str)> = claims.iter().map(|c| (c.dag_id, c.kind.as_str())).collect(); - assert!(kinds.contains(&(a, "stop_for_update"))); - assert!(kinds.contains(&(b, "prebuild"))); - assert_eq!(claims.len(), 2); + assert_eq!(kinds, vec![(a, "stop_for_update")]); + assert!( + !kinds.iter().any(|&(d, _)| d == b), + "b's build waits — slot held across a's chain" + ); } #[test] @@ -208,9 +214,27 @@ fn fifo_fairness_for_the_slot() { let first = claim_one(&q); assert_eq!(first.dag_id, a, "submit order wins the slot"); q.complete_node(a, first.node_id, Ok(())); - let next: Vec = q.claim_ready().iter().map(|cl| cl.dag_id).collect(); - assert!(next.contains(&b), "b's prebuild before c's"); - assert!(!next.contains(&c)); + // Uniform hold: the slot stays with agent-a until its Swap (the last + // slot-needer) completes. Drive a's chain; the moment its slot frees, + // submit order (b before c) wins it. + let mut freed_to = None; + for _ in 0..6 { + let claims = q.claim_ready(); + if let Some(nb) = claims.iter().find(|cl| cl.dag_id == b || cl.dag_id == c) { + freed_to = Some(nb.dag_id); + break; + } + for cl in claims { + if cl.dag_id == a { + q.complete_node(a, cl.node_id, Ok(())); + } + } + } + assert_eq!( + freed_to, + Some(b), + "b's prebuild wins the freed slot before c's" + ); } // ---- per-agent lease ---- @@ -234,18 +258,31 @@ fn lease_serializes_two_lifecycle_dags_for_same_agent() { let first = claim_one(&q); assert_eq!(first.dag_id, restart); assert_eq!(first.kind.as_str(), "stop_for_update"); - assert!(first.lease_acquired); q.complete_node(restart, first.node_id, Ok(())); - // Same DAG keeps the lease through the tail Reconcile. + // Same DAG keeps the lease through the tail Reconcile (re-entered from the + // dep graph — no fresh acquire), since stop's Reconcile can't re-enter it. let second = claim_one(&q); assert_eq!(second.dag_id, restart); assert_eq!(second.kind.as_str(), "reconcile"); - assert!(!second.lease_acquired, "lease already held by this DAG"); q.complete_node(restart, second.node_id, Ok(())); - // Restart terminal → lease released → stop's Reconcile runs. - let third = claim_one(&q); - assert_eq!(third.dag_id, stop); + // Restart's work is terminal → its lease releases. Its terminal node and + // stop's now-unblocked Reconcile both become ready in the same pass. + let ready = q.claim_ready(); + let restart_fin = ready + .iter() + .find(|c| c.dag_id == restart && c.kind.as_str() == "revert_intent") + .expect("restart finalize ready"); + q.complete_node(restart, restart_fin.node_id, Ok(())); + let third = ready + .iter() + .find(|c| c.dag_id == stop) + .expect("stop reconcile ready once the lease is freed"); + assert_eq!(third.kind.as_str(), "reconcile"); q.complete_node(stop, third.node_id, Ok(())); + // stop's terminal node (revert-intent) then runs. + let stop_fin = claim_one(&q); + assert_eq!(stop_fin.kind.as_str(), "revert_intent"); + q.complete_node(stop, stop_fin.node_id, Ok(())); assert_eq!(state_of(&q, restart), State::Done); assert_eq!(state_of(&q, stop), State::Done); } @@ -288,8 +325,20 @@ fn lease_exempt_prebuild_overlaps_other_dag_on_same_agent() { .expect("reconcile claim") .clone(); q.complete_node(stop, reconcile.node_id, Ok(())); - let next = claim_one(&q); - assert_eq!(next.kind.as_str(), "stop_for_update"); + // stop's Reconcile done → its lease frees (rebuild's StopForUpdate unblocks) + // and its terminal node becomes ready; both surface in the same pass. + let after = q.claim_ready(); + assert!( + after + .iter() + .any(|c| c.dag_id == stop && c.kind.as_str() == "revert_intent"), + "stop DAG's finalize runs once its work settles" + ); + let sfu = after + .iter() + .find(|c| c.kind.as_str() == "stop_for_update") + .expect("rebuild StopForUpdate unblocked once the lease frees"); + assert_eq!(sfu.agent, "agent-a"); } #[test] @@ -315,16 +364,16 @@ fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() { // lease (no contention across distinct agents), all inside the single DAG. let claims = q.claim_ready(); assert!(claims.iter().all(|c| c.dag_id == id)); - let mut heads: Vec<(&str, &str, bool)> = claims + let mut heads: Vec<(&str, &str)> = claims .iter() - .map(|c| (c.agent.as_str(), c.kind.as_str(), c.lease_acquired)) + .map(|c| (c.agent.as_str(), c.kind.as_str())) .collect(); heads.sort_unstable(); assert_eq!( heads, vec![ - ("agent-a", "stop_for_update", true), - ("agent-b", "stop_for_update", true), + ("agent-a", "stop_for_update"), + ("agent-b", "stop_for_update"), ], "both per-agent subgraphs start concurrently, each acquiring its own lease" ); @@ -387,17 +436,14 @@ fn multi_agent_stop_is_one_dag_with_concurrent_per_agent_subgraphs() { assert_eq!(q.snapshot().len(), 1); let claims = q.claim_ready(); assert!(claims.iter().all(|c| c.dag_id == id)); - let mut heads: Vec<(&str, &str, bool)> = claims + let mut heads: Vec<(&str, &str)> = claims .iter() - .map(|c| (c.agent.as_str(), c.kind.as_str(), c.lease_acquired)) + .map(|c| (c.agent.as_str(), c.kind.as_str())) .collect(); heads.sort_unstable(); assert_eq!( heads, - vec![ - ("agent-a", "set_wanted", true), - ("agent-b", "set_wanted", true), - ], + vec![("agent-a", "set_wanted"), ("agent-b", "set_wanted")], "both per-agent stop subgraphs start concurrently, each on its own lease" ); } @@ -521,6 +567,7 @@ fn append_subgraph_roots_on_emitter_and_rebases_local_deps() { fanout: None, }, deps: Vec::new(), + parent: None, }], }; let id = submit(&q, spec); @@ -532,8 +579,8 @@ fn append_subgraph_roots_on_emitter_and_rebases_local_deps() { // tracks any drift in that builder's root-first (`base = 0`) shape. let subgraph = |agent: &str| templates::rebuild_nodes(agent, true, 0); // Must append BEFORE completing the emitter (the documented contract). - q.append_subgraph(id, subgraph("a"), emitter.node_id); - q.append_subgraph(id, subgraph("b"), emitter.node_id); + q.append_subgraph(id, &subgraph("a"), emitter.node_id); + q.append_subgraph(id, &subgraph("b"), emitter.node_id); q.complete_node(id, emitter.node_id, Ok(())); // Still ONE DAG; both subgraph roots become ready once the emitter is // Done (rooted on it), each on its own agent lease. @@ -580,7 +627,7 @@ fn meta_update_carries_rebuilding_transient_and_grows_cascade_in_dag() { for agent in ["alice", "bob"] { q.append_subgraph( id, - templates::rebuild_nodes(agent, false, 0), + &templates::rebuild_nodes(agent, false, 0), meta_lock.node_id, ); } @@ -728,6 +775,11 @@ fn cancel_clears_queued_dag() { let id = submit(&q, rebuild("agent-a", "r")); assert!(q.cancel(id)); assert_eq!(state_of(&q, id), State::Cancelled); + // The terminal node's weak edges are satisfied by the cancelled (terminal) + // work nodes, so it still runs its hooks — it's the one thing left claimable. + let fin = claim_one(&q); + assert_eq!(fin.kind.as_str(), "emit_rebuilt"); + q.complete_node(id, fin.node_id, Ok(())); assert!(q.claim_ready().is_empty()); } @@ -743,22 +795,25 @@ fn cancel_refuses_running_dag() { // ---- terminal reporting + lease release ---- #[test] -fn terminal_dag_reported_exactly_once_and_lease_released() { +fn terminal_node_runs_after_work_settles_and_lease_released() { let q = JobQueue::new(1); let id = submit(&q, restart_online(&["agent-a"], false, "r")); - // restart = StopForUpdate → Reconcile; not terminal until the last - // node completes. + // restart = StopForUpdate → Reconcile; the terminal node (which weak-deps on + // the chain tail) isn't runnable until the whole chain is terminal. let stop = claim_one(&q); q.complete_node(id, stop.node_id, Ok(())); - assert!(q.drain_terminal().is_empty(), "dag not terminal yet"); let rec = claim_one(&q); + assert_eq!(rec.kind.as_str(), "reconcile"); q.complete_node(id, rec.node_id, Ok(())); - let reports = q.drain_terminal(); - assert_eq!(reports.len(), 1); - assert_eq!(reports[0].dag_id, id); - assert_eq!(reports[0].state, State::Done); - assert!(q.drain_terminal().is_empty(), "reported exactly once"); - // Lease released: a new DAG for the agent can claim immediately. + // Work settled → the terminal node is now the runnable one; it carries the + // terminal roll-up the hook consumes via `terminal_summary`. + let fin = claim_one(&q); + assert_eq!(fin.kind.as_str(), "revert_intent"); + let summary = q.terminal_summary(id).expect("terminal summary"); + assert_eq!(summary.state, State::Done); + q.complete_node(id, fin.node_id, Ok(())); + // Lease released (freed when the work chain settled, ahead of finalize): a + // new DAG for the agent claims immediately. let next = submit( &q, templates::reconcile_only( @@ -771,31 +826,38 @@ fn terminal_dag_reported_exactly_once_and_lease_released() { ); let c = claim_one(&q); assert_eq!(c.dag_id, next); - assert!(c.lease_acquired); } /// A DAG cancelled while fully queued must still surface a terminal /// roll-up for the scheduler's hooks — otherwise a queued approval /// DAG cancelled by the operator would dangle its approval forever. #[test] -fn cancelled_dag_reports_terminal_once() { +fn cancelled_dag_finalizes_with_terminal_rollup() { let q = JobQueue::new(1); let id = submit( &q, templates::approval_deploy("agent-a", 7, "approval #7".to_owned()), ); assert!(q.cancel(id)); - let reports = q.drain_terminal(); - assert_eq!(reports.len(), 1); - assert_eq!(reports[0].dag_id, id); - assert_eq!(reports[0].state, State::Cancelled); - assert_eq!(reports[0].approval_id, Some(7)); - // Never re-reported by later activity. + // The terminal node's weak edges still fire on a fully-cancelled DAG, so its + // hook (approval resolution) runs — surfaced here as a claimable resolve- + // approval node whose `terminal_summary` is Cancelled + carries the approval id. + let fin = claim_one(&q); + assert_eq!(fin.kind.as_str(), "resolve_approval"); + let summary = q.terminal_summary(id).expect("terminal summary"); + assert_eq!(summary.state, State::Cancelled); + assert_eq!(summary.approval_id, Some(7)); + q.complete_node(id, fin.node_id, Ok(())); + // The cancelled DAG's summary stays available (until history-trimmed) and + // unrelated later activity doesn't disturb it. let other = submit(&q, rebuild("agent-b", "r")); let c = claim_one(&q); assert_eq!(c.dag_id, other); q.complete_node(other, c.node_id, Err("boom".to_owned())); - assert!(q.drain_terminal().iter().all(|t| t.dag_id != id)); + assert_eq!( + q.terminal_summary(id).map(|t| t.state), + Some(State::Cancelled) + ); } // ---- steps, build logs, history ---- @@ -804,7 +866,10 @@ fn cancelled_dag_reports_terminal_once() { fn set_step_only_on_running_and_signals_change() { let q = JobQueue::new(1); let id = submit(&q, rebuild("agent-a", "r")); - assert!(!q.set_step(id, 0, "too early"), "queued node refuses step"); + assert!( + !q.set_step_running(id, "too early"), + "no running node yet → refused" + ); let c = claim_one(&q); assert!(q.set_step(id, c.node_id, "nix build")); assert!( @@ -823,7 +888,10 @@ fn set_step_only_on_running_and_signals_change() { fn set_build_log_id_links_running_node() { let q = JobQueue::new(1); let id = submit(&q, rebuild("agent-a", "r")); - assert!(!q.set_build_log_id(id, 0, 41), "queued node refuses log id"); + assert!( + !q.set_build_log_id_running(id, 41), + "no running node yet → refused" + ); let c = claim_one(&q); assert!(q.set_build_log_id(id, c.node_id, 42)); assert!(q.set_build_log_id_running(id, 43)); @@ -849,6 +917,11 @@ fn history_evicts_old_terminals_per_template() { ); let c = claim_one(&q); q.complete_node(id, c.node_id, Ok(())); + // Drain the DAG's terminal node too, so the next iteration's claim sees + // only its own work (the terminal node is excluded from the view + rollup). + let fin = claim_one(&q); + assert_eq!(fin.kind.as_str(), "revert_intent"); + q.complete_node(id, fin.node_id, Ok(())); } // Fresh terminals are inside the grace window: nothing evicts yet, // so a ~1s QueueDag poller can still observe every terminal state diff --git a/hive-c0re/src/workers/auto_update.rs b/hive-c0re/src/workers/auto_update.rs index d115e232..c95ab682 100644 --- a/hive-c0re/src/workers/auto_update.rs +++ b/hive-c0re/src/workers/auto_update.rs @@ -331,6 +331,7 @@ fn submit_boot_tree( fanout: Some(fanout), }, deps: Vec::new(), + parent: None, }); } // One boot Reconcile per drifted agent — independent roots. @@ -339,6 +340,7 @@ fn submit_boot_tree( agent: name, kind: NodeKind::Reconcile, deps: Vec::new(), + parent: None, }); } diff --git a/hive-sh4re/src/jobs.rs b/hive-sh4re/src/jobs.rs index d060a1ba..cb5ef9b3 100644 --- a/hive-sh4re/src/jobs.rs +++ b/hive-sh4re/src/jobs.rs @@ -134,8 +134,11 @@ pub enum PermPayload { }, } -/// Node id, unique within its DAG. -pub type NodeId = u32; +/// Node id. Carries the scheduler crate's globally-monotonic node id +/// (`hive_jobq::NodeId`) verbatim on the wire — unique across all DAGs, not +/// just within one. Consumers treat it opaquely (grouping + dep matching), +/// so the widening from the old dag-local `u32` is transparent. +pub type NodeId = u64; /// One node of a queued DAG, as serialized. Step labels, build-log /// links, errors, and timestamps are per-node; the DAG-level `state` diff --git a/hivectl/src/dag_progress.rs b/hivectl/src/dag_progress.rs index 8fb6d76a..74c77d2d 100644 --- a/hivectl/src/dag_progress.rs +++ b/hivectl/src/dag_progress.rs @@ -325,7 +325,7 @@ mod tests { use super::render_dag_line; - fn node(id: u32, agent: &str, kind: &str, state: State, step: Option<&str>) -> NodeView { + fn node(id: u64, agent: &str, kind: &str, state: State, step: Option<&str>) -> NodeView { NodeView { id, agent: agent.to_owned(), From 456847eaa1d9802ea749a7fe36253aa2fadf2c32 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 20 Jul 2026 21:58:42 +0200 Subject: [PATCH 2/9] fix(#2591): validate() rejects out-of-bounds/forward parent index (argus review) --- hive-c0re/src/job_queue/templates.rs | 17 ++++++++++++++--- hive-c0re/src/job_queue/tests.rs | 15 +++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/hive-c0re/src/job_queue/templates.rs b/hive-c0re/src/job_queue/templates.rs index 460655c8..61852e1f 100644 --- a/hive-c0re/src/job_queue/templates.rs +++ b/hive-c0re/src/job_queue/templates.rs @@ -256,9 +256,9 @@ pub fn meta_update( // per-agent child DAGs. /// Validate a spec before it enters the queue: node ids are dense -/// (index = id), deps reference existing nodes, and the dep graph is -/// acyclic (petgraph `toposort`). Rejecting cycles here fixes the old -/// queue's documented "circular dep silently deadlocks forever" caveat. +/// (index = id), deps + parents reference existing *earlier* nodes, and the +/// dep graph is acyclic (petgraph `toposort`). Rejecting cycles here fixes the +/// old queue's documented "circular dep silently deadlocks forever" caveat. pub fn validate(spec: &DagSpec) -> Result<()> { if spec.nodes.is_empty() { bail!("dag spec {:?} has no nodes", spec.template); @@ -269,6 +269,17 @@ pub fn validate(spec: &DagSpec) -> Result<()> { .map(|i| graph.add_node(u32::try_from(i).unwrap_or(u32::MAX))) .collect(); for (i, node) in spec.nodes.iter().enumerate() { + // A `parent` must index an earlier node — `insert_group` resolves it to + // an already-inserted `NodeId`, so a forward/out-of-bounds parent would + // otherwise panic there. + if let Some(p) = node.parent + && usize::try_from(p).is_ok_and(|p| p >= i) + { + bail!( + "dag spec {:?} node {i} has invalid parent {p} (must be an earlier node)", + spec.template + ); + } for dep in &node.deps { let Some(&dep_idx) = usize::try_from(dep.on).ok().and_then(|i| idx.get(i)) else { bail!( diff --git a/hive-c0re/src/job_queue/tests.rs b/hive-c0re/src/job_queue/tests.rs index 33608bae..2d184803 100644 --- a/hive-c0re/src/job_queue/tests.rs +++ b/hive-c0re/src/job_queue/tests.rs @@ -147,6 +147,21 @@ fn unknown_dep_is_rejected_at_submit() { assert!(q.submit(spec).is_err()); } +#[test] +fn invalid_parent_is_rejected_at_submit() { + let q = JobQueue::new(1); + let mut spec = rebuild("agent-a", "bad parent"); + // A forward/out-of-bounds parent index must be refused at validate, not + // panic in `insert_group`. + spec.nodes = vec![NodeSpec { + agent: "agent-a".to_owned(), + kind: NodeKind::Reconcile, + deps: Vec::new(), + parent: Some(3), + }]; + assert!(q.submit(spec).is_err()); +} + // ---- dependency order within a DAG ---- #[test] From 50172d1716abf23f70cea0e0138ab16d7ca2a2b2 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 20 Jul 2026 22:14:07 +0200 Subject: [PATCH 3/9] refactor(#2591): derive PartialEq/Eq/Serialize on TransientKind --- hive-c0re/src/coordinator.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hive-c0re/src/coordinator.rs b/hive-c0re/src/coordinator.rs index 9d40377d..e5f8b3b5 100644 --- a/hive-c0re/src/coordinator.rs +++ b/hive-c0re/src/coordinator.rs @@ -369,7 +369,8 @@ impl Drop for MetaUpdateGuard { } } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] pub enum TransientKind { /// `lifecycle::spawn` is running (nixos-container create + update + start). Spawning, From 8834161fb93c19f0608c0051714604f924a80fae Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 20 Jul 2026 22:16:27 +0200 Subject: [PATCH 4/9] feat(#2591): add NodeKind::Dag DAG-container variant --- hive-c0re/src/job_queue/exec.rs | 4 ++++ hive-c0re/src/job_queue/model.rs | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index 219ea1b2..52eba21b 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -103,6 +103,10 @@ pub(super) async fn run_node(coord: &Arc, claim: &Claim) -> Result< NodeKind::ResolveApproval => run_resolve_approval(coord, claim).await, NodeKind::EmitRebuilt => Ok(run_emit_rebuilt(coord, claim)), NodeKind::RevertIntent => run_revert_intent(coord, claim).await, + // Pure grouping container — no work; completing it lets it reach + // `Finishing` so its child template nodes start. The DAG's terminal + // hook fires when the container itself rolls up terminal. + NodeKind::Dag { .. } => Ok(NodeOutput::default()), } } diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs index 619572b9..aa85eaf9 100644 --- a/hive-c0re/src/job_queue/model.rs +++ b/hive-c0re/src/job_queue/model.rs @@ -14,6 +14,8 @@ pub use hive_sh4re::jobs::{DagView, NodeId, PermPayload, Source, State, Template}; use serde::Serialize; +use crate::coordinator::TransientKind; + /// When a dependency edge is considered satisfied. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] @@ -154,6 +156,24 @@ pub enum NodeKind { /// means "don't do it". Noop on any non-cancelled outcome. Appended weak-dep /// on the tails; slot/lease-exempt. RevertIntent, + /// The **DAG container** node: one per submitted DAG, carrying the group's + /// domain metadata. Every template node hangs *under* it (its subtree), so + /// the container's `NodeId` **is** the DAG id, its rolled-up state **is** the + /// DAG state, and it reaching terminal **is** the completion signal that + /// fires the DAG's inline hook (approval-resolve / rebuilt-emit / + /// intent-revert, dispatched off `template`). Pure grouping — lease- and + /// build-slot-exempt; the executor instant-completes it (`Done`) so it + /// reaches `Finishing` and its children start. + Dag { + template: Template, + source: Source, + reason: String, + transient: Option, + approval_id: Option, + inputs: Vec, + perm_payload: Option, + created_at: i64, + }, } impl NodeKind { @@ -179,6 +199,7 @@ impl NodeKind { NodeKind::ResolveApproval => "resolve_approval", NodeKind::EmitRebuilt => "emit_rebuilt", NodeKind::RevertIntent => "revert_intent", + NodeKind::Dag { .. } => "dag", } } From a78280feedabcdfa7c7c28bda42f1d7f8c352f60 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 20 Jul 2026 22:33:43 +0200 Subject: [PATCH 5/9] =?UTF-8?q?refactor(#2591):=20model=20a=20DAG=20as=20a?= =?UTF-8?q?=20container=20node=20=E2=80=94=20grouping=20side-tables=20beco?= =?UTF-8?q?me=20graph=20walks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hive-c0re/src/job_queue/exec.rs | 103 +++-- hive-c0re/src/job_queue/mod.rs | 548 +++++++++++++++------------ hive-c0re/src/job_queue/model.rs | 18 - hive-c0re/src/job_queue/scheduler.rs | 19 +- 4 files changed, 363 insertions(+), 325 deletions(-) diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index 52eba21b..0c7d9d08 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -100,76 +100,69 @@ pub(super) async fn run_node(coord: &Arc, claim: &Claim) -> Result< NodeKind::WritePermFile => run_write_perm_file(coord, claim, &ctx).await, NodeKind::ApprovalDeploy => run_approval_deploy(coord, claim).await, NodeKind::SetWanted { up } => run_set_wanted(coord, claim, *up), - NodeKind::ResolveApproval => run_resolve_approval(coord, claim).await, - NodeKind::EmitRebuilt => Ok(run_emit_rebuilt(coord, claim)), - NodeKind::RevertIntent => run_revert_intent(coord, claim).await, // Pure grouping container — no work; completing it lets it reach // `Finishing` so its child template nodes start. The DAG's terminal - // hook fires when the container itself rolls up terminal. + // hook fires (inline, via `run_terminal_hook`) when the container itself + // rolls up terminal — not as a scheduled node. NodeKind::Dag { .. } => Ok(NodeOutput::default()), } } -/// Terminal hook (approval DAGs — spawn / opaque deploy): resolve the DAG's -/// approval row from its rolled-up outcome. Its own graph node, weak-dep on the -/// DAG tails, so it runs once everything has settled (any outcome, incl. a -/// cancel before starting — the fallback that resolves a queued-then-cancelled -/// approval whose node never ran). Always succeeds — a hook failure is logged -/// inside, not surfaced as a node failure. -async fn run_resolve_approval(coord: &Arc, claim: &Claim) -> Result { - if let Some(terminal) = coord.job_queue.terminal_summary(claim.dag_id) { - crate::actions::resolve_approval_dag(coord, &terminal).await; +/// Run a settled DAG's inline terminal hook, dispatched off its rolled-up +/// summary — the container-terminal replacement for the old per-DAG hook node. +/// Always best-effort: a hook failure is logged inside, never surfaced. +pub(super) async fn run_terminal_hook(coord: &Arc, terminal: &super::TerminalDag) { + match super::terminal_hook(terminal.template, terminal.approval_id) { + Some(super::HookKind::ResolveApproval) => { + crate::actions::resolve_approval_dag(coord, terminal).await; + } + Some(super::HookKind::EmitRebuilt) => emit_rebuilt(coord, terminal), + Some(super::HookKind::RevertIntent) => revert_intent(coord, terminal).await, + None => {} } - Ok(NodeOutput::default()) } -/// Terminal hook (rebuild / perm-change DAGs): emit one `Rebuilt` manager event -/// per targeted agent — `ok` on `Done`, `!ok` on `Failed`, none on cancel. -fn run_emit_rebuilt(coord: &Arc, claim: &Claim) -> NodeOutput { - if let Some(terminal) = coord.job_queue.terminal_summary(claim.dag_id) { - for agent in &terminal.agents { - match terminal.state { - State::Done => coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { - agent: agent.clone(), - ok: true, - note: None, - sha: None, - tag: None, - }), - State::Failed => coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { - agent: agent.clone(), - ok: false, - note: terminal.error.clone(), - sha: None, - tag: None, - }), - _ => {} - } +/// Rebuild / perm-change hook: emit one `Rebuilt` manager event per targeted +/// agent — `ok` on `Done`, `!ok` on `Failed`, none on cancel. +fn emit_rebuilt(coord: &Arc, terminal: &super::TerminalDag) { + for agent in &terminal.agents { + match terminal.state { + State::Done => coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { + agent: agent.clone(), + ok: true, + note: None, + sha: None, + tag: None, + }), + State::Failed => coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { + agent: agent.clone(), + ok: false, + note: terminal.error.clone(), + sha: None, + tag: None, + }), + _ => {} } } - NodeOutput::default() } -/// Terminal hook (power-op DAGs): on a *cancelled* DAG, revert each targeted -/// agent's `wanted` intent to its observed state — the operator's cancel means -/// "don't do it", so the intent snaps back instead of the flip executing as a -/// surprise side effect of some later reconcile. Noop on any non-cancelled -/// outcome. Always succeeds — a revert failure is logged, not surfaced. -async fn run_revert_intent(coord: &Arc, claim: &Claim) -> Result { - if let Some(terminal) = coord.job_queue.terminal_summary(claim.dag_id) - && terminal.state == State::Cancelled - { - for agent in &terminal.agents { - let running = crate::lifecycle::is_running(agent).await; - if let Err(e) = coord - .power - .set(agent, crate::power::Wanted::from_running(running)) - { - tracing::warn!(%agent, error = ?e, "agent_power: cancel revert failed"); - } +/// Power-op hook: on a *cancelled* DAG, revert each targeted agent's `wanted` +/// intent to its observed state — the operator's cancel means "don't do it", so +/// the intent snaps back instead of the flip executing as a surprise side effect +/// of some later reconcile. Noop on any non-cancelled outcome. +async fn revert_intent(coord: &Arc, terminal: &super::TerminalDag) { + if terminal.state != State::Cancelled { + return; + } + for agent in &terminal.agents { + let running = crate::lifecycle::is_running(agent).await; + if let Err(e) = coord + .power + .set(agent, crate::power::Wanted::from_running(running)) + { + tracing::warn!(%agent, error = ?e, "agent_power: cancel revert failed"); } } - Ok(NodeOutput::default()) } /// Write the agent's durable power intent — the DAG-node form of the old diff --git a/hive-c0re/src/job_queue/mod.rs b/hive-c0re/src/job_queue/mod.rs index 9a23eb80..1d15acdd 100644 --- a/hive-c0re/src/job_queue/mod.rs +++ b/hive-c0re/src/job_queue/mod.rs @@ -97,19 +97,6 @@ pub struct TerminalDag { pub error: Option, } -/// Per-DAG metadata that isn't a node — the group's display + hook inputs. -#[derive(Debug, Clone)] -struct GroupMeta { - template: Template, - source: Source, - reason: String, - approval_id: Option, - inputs: Vec, - perm_payload: Option, - transient: Option, - created_at: i64, -} - /// Per-node runtime metadata the crate graph doesn't carry (kind + agent live /// in the node payload; state lives in the node). #[derive(Debug, Default, Clone)] @@ -121,23 +108,31 @@ struct NodeRuntime { error: Option, } +/// An owned read-view of a DAG container's carried metadata ([`NodeKind::Dag`]). +/// Derived on read from the container node — the data has a single home (the +/// node payload); this is not a stored side-table. +struct DagMeta { + template: Template, + source: Source, + reason: String, + transient: Option, + approval_id: Option, + inputs: Vec, + perm_payload: Option, + created_at: i64, +} + /// The mutable queue state behind the mutex: the crate scheduler plus the -/// host-side grouping side-tables (`dag_id` ↔ nodes, per-DAG meta, per-node -/// runtime metadata). One shared crate [`Graph`] holds every DAG's nodes. +/// per-node runtime metadata the graph can't carry. A **DAG is a single +/// container node** ([`NodeKind::Dag`], `parent = None`) whose subtree is the +/// DAG's work — so the container's `NodeId` is the DAG id, its rolled-up state +/// is the DAG state, and there are no grouping side-tables: membership + meta +/// are graph queries ([`QueueInner::container`] / [`QueueInner::subtree`] / +/// [`QueueInner::dag_meta`]). One shared crate [`Graph`] holds every DAG. struct QueueInner { sched: Scheduler, - next_dag: u64, - /// Per-DAG metadata, keyed by DAG id. - dag_meta: HashMap, - /// Work nodes of each DAG, in insert order (drives rollup + the view). - /// Excludes the internal terminal node. - dag_nodes: HashMap>, - /// The terminal-hook node id of each DAG that has one (approval-resolve / - /// rebuilt-emit / intent-revert). Hook-less DAGs are absent from the map. - terminal_node: HashMap, - /// Reverse lookup: crate node id → its owning DAG id. - node_dag: HashMap, - /// Per-node runtime metadata. + /// Per-node runtime metadata (build-log id, step, timestamps, error) — + /// mutable after insert, so it can't ride the immutable node payload. node_rt: HashMap, } @@ -169,22 +164,32 @@ fn to_crate_when(when: DepWhen) -> JobDepWhen { } } -/// The terminal-hook node kind a DAG needs, from its template + approval id — -/// or `None` for a DAG with no terminal side effect (meta-update, boot, bare -/// reconcile). Each concern is its own focused node rather than one node that -/// branches on metadata: an approval DAG resolves its approval, a rebuild / -/// perm-change emits `Rebuilt`, a power-op reverts its `wanted` intent on cancel. -fn terminal_kind(template: Template, approval_id: Option) -> Option { +/// The inline terminal-hook a settled DAG fires — dispatched off its container's +/// template + approval id when the container rolls up terminal (no hook node). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HookKind { + /// Approval-driven DAG (spawn / opaque deploy): resolve the approval row. + ResolveApproval, + /// Rebuild / perm-change: emit one `Rebuilt` manager event per agent. + EmitRebuilt, + /// Power-op: on a *cancelled* DAG, revert each agent's `wanted` intent. + RevertIntent, +} + +/// The terminal hook a DAG needs, from its template + approval id — or `None` +/// for a DAG with no terminal side effect (meta-update, boot, bare reconcile). +#[must_use] +pub fn terminal_hook(template: Template, approval_id: Option) -> Option { if approval_id.is_some() { - return Some(NodeKind::ResolveApproval); + return Some(HookKind::ResolveApproval); } match template { - Template::Rebuild | Template::PermChange => Some(NodeKind::EmitRebuilt), + Template::Rebuild | Template::PermChange => Some(HookKind::EmitRebuilt), Template::Start | Template::Stop | Template::GracefulStop | Template::Restart - | Template::GracefulRestart => Some(NodeKind::RevertIntent), + | Template::GracefulRestart => Some(HookKind::RevertIntent), _ => None, } } @@ -212,22 +217,20 @@ fn to_wire_state(state: JobState) -> State { /// keeps a resource continuous across a subtree (a root owns it, descendants /// borrow it). Independent group roots (multiple `parent = None` nodes) carry no /// cross-links, so a multi-agent DAG's per-agent subgraphs run concurrently, each -/// on its own lease. Records per-node bookkeeping (`node_dag`, `node_rt`); the -/// caller owns `dag_nodes`. Returns the inserted ids (index-aligned with `nodes`) -/// and the group roots (the `parent = None` nodes). A node's `parent` / dep -/// targets must precede it in `nodes` (submit-time `validate` enforces density + -/// acyclicity). +/// on its own lease. Records per-node `node_rt`. Returns the inserted ids +/// (index-aligned with `nodes`). A node with `parent = None` is re-parented to +/// `group_parent` (the DAG container for a template, or the emitting node for a +/// runtime-appended subgraph); a node's `parent` / dep targets must precede it +/// in `nodes` (submit-time `validate` enforces density + acyclicity). /// /// # Errors /// Propagates a crate graph-insert error (malformed dep/parent / dep-scope). fn insert_group( inner: &mut QueueInner, - dag_id: u64, nodes: &[NodeSpec], group_parent: Option, -) -> anyhow::Result<(Vec, Vec)> { +) -> anyhow::Result> { let mut ids: Vec = Vec::with_capacity(nodes.len()); - let mut roots: Vec = Vec::new(); for ns in nodes { let payload = JobPayload { kind: ns.kind.clone(), @@ -248,14 +251,10 @@ fn insert_group( .sched .append(payload, deps, parent) .map_err(|e| anyhow::anyhow!("job_queue: graph insert failed: {e}"))?; - if ns.parent.is_none() { - roots.push(id); - } ids.push(id); - inner.node_dag.insert(id, dag_id); inner.node_rt.insert(id, NodeRuntime::default()); } - Ok((ids, roots)) + Ok(ids) } impl JobQueue { @@ -269,11 +268,6 @@ impl JobQueue { Self { inner: Mutex::new(QueueInner { sched: Scheduler::new(Graph::new(), table), - next_dag: 0, - dag_meta: HashMap::new(), - dag_nodes: HashMap::new(), - terminal_node: HashMap::new(), - node_dag: HashMap::new(), node_rt: HashMap::new(), }), notify: Notify::new(), @@ -284,69 +278,43 @@ impl JobQueue { self.inner.lock().expect("job_queue mutex poisoned") } - /// Submit a DAG. Validates the spec (cycle rejection), inserts its nodes - /// into the shared graph (chain deps + resource edges), appends the per-DAG - /// terminal node (weak-depending on the DAG's tail nodes), and returns the - /// new DAG id. + /// Submit a DAG. Validates the spec, inserts a [`NodeKind::Dag`] **container + /// node** carrying the group's metadata, then inserts the template's nodes as + /// its subtree (their roots re-parented to the container). Returns the + /// container's id as the DAG id — its rolled-up state is the DAG state and it + /// reaching terminal fires the DAG's inline hook. /// /// # Errors - /// Propagates the spec-validation error (empty / cyclic) or a graph-insert - /// error (a spec whose dependencies aren't dependency-topological). + /// Propagates the spec-validation error (empty / cyclic / bad parent) or a + /// graph-insert error (dependencies that aren't dependency-topological). pub fn submit(&self, spec: DagSpec) -> anyhow::Result { templates::validate(&spec)?; let mut inner = self.lock(); - inner.next_dag += 1; - let dag_id = inner.next_dag; - - let (work, roots) = insert_group(&mut inner, dag_id, &spec.nodes, None)?; - - // Append the DAG's terminal-hook node — but only if it needs one. It - // weak-depends (`AfterAny`) on every group **root**, each of which the - // crate rolls up terminal only once its whole subtree — the entire op, - // including any runtime-appended subgraphs (children of nodes inside the - // group) — has settled. So the hook runs exactly when the DAG is done, - // with no `add_dep` wiring. Hook-less DAGs (meta-update, boot, reconcile) - // get none. - if let Some(kind) = terminal_kind(spec.template, spec.approval_id) { - let term = inner - .sched - .append( - JobPayload { - kind, - agent: String::new(), + let container = inner + .sched + .append( + JobPayload { + kind: NodeKind::Dag { + template: spec.template, + source: spec.source, + reason: spec.reason, + transient: spec.transient, + approval_id: spec.approval_id, + inputs: spec.inputs, + perm_payload: spec.perm_payload, + created_at: now_unix(), }, - roots - .iter() - .map(|&id| Dep::Node { - id, - when: JobDepWhen::AfterAny, - }) - .collect(), - None, - ) - .map_err(|e| anyhow::anyhow!("job_queue: terminal node insert failed: {e}"))?; - inner.terminal_node.insert(dag_id, term); - inner.node_dag.insert(term, dag_id); - inner.node_rt.insert(term, NodeRuntime::default()); - } - - inner.dag_nodes.insert(dag_id, work); - inner.dag_meta.insert( - dag_id, - GroupMeta { - template: spec.template, - source: spec.source, - reason: spec.reason, - approval_id: spec.approval_id, - inputs: spec.inputs, - perm_payload: spec.perm_payload, - transient: spec.transient, - created_at: now_unix(), - }, - ); + agent: String::new(), + }, + Vec::new(), + None, + ) + .map_err(|e| anyhow::anyhow!("job_queue: container insert failed: {e}"))?; + inner.node_rt.insert(container, NodeRuntime::default()); + insert_group(&mut inner, &spec.nodes, Some(container))?; drop(inner); self.notify.notify_one(); - Ok(dag_id) + Ok(container.get()) } /// Append a whole *subgraph* into a live DAG at runtime — the single @@ -364,16 +332,17 @@ impl JobQueue { return Vec::new(); } let mut inner = self.lock(); - if !inner.dag_meta.contains_key(&dag_id) { + if inner.container(dag_id).is_none() { return Vec::new(); } // Insert the subgraph as a group rooted under the emitting node: the // subgraph's own root becomes a child of `dep_on`, its steps children of // that root. No terminal-node wiring — roll-up carries terminality: the // emitter stays `Finishing` until this appended subtree settles, and the - // DAG's terminal node deps on the top root, so the hook waits for free. - let ids = match insert_group(&mut inner, dag_id, nodes, Some(dep_on)) { - Ok((ids, _roots)) => ids, + // container node rolls up terminal only once its whole subtree (incl. this + // appended work) has settled, so the DAG hook waits for free. + let ids = match insert_group(&mut inner, nodes, Some(dep_on)) { + Ok(ids) => ids, Err(e) => { tracing::error!( dag = dag_id, @@ -383,9 +352,6 @@ impl JobQueue { return Vec::new(); } }; - if let Some(list) = inner.dag_nodes.get_mut(&dag_id) { - list.extend(ids.iter().copied()); - } drop(inner); self.notify.notify_one(); ids @@ -393,9 +359,9 @@ impl JobQueue { /// Claim every currently-runnable node, acquiring its resources, and mark it /// `Running`. Delegates readiness + resource acquisition to the crate's - /// settle loop; builds a [`Claim`] per started node from its payload + the - /// DAG's metadata. The internal terminal node is claimed like any other - /// (its executor runs the terminal hooks). + /// settle loop; builds a [`Claim`] per started node from its payload + its + /// DAG container's metadata. The container node itself is claimed like any + /// other (its executor is an instant no-op that lets its subtree start). pub fn claim_ready(&self) -> Vec { let mut inner = self.lock(); let inner = &mut *inner; @@ -408,21 +374,21 @@ impl JobQueue { }; let kind = node.payload.kind.clone(); let agent = node.payload.agent.clone(); - let Some(&dag_id) = inner.node_dag.get(&id) else { + let Some(container) = inner.dag_of(id) else { continue; }; - let Some(meta) = inner.dag_meta.get(&dag_id) else { + let Some(meta) = inner.dag_meta(container) else { continue; }; claims.push(Claim { - dag_id, + dag_id: container.get(), node_id: id, kind, agent, template: meta.template, approval_id: meta.approval_id, - inputs: meta.inputs.clone(), - perm_payload: meta.perm_payload.clone(), + inputs: meta.inputs, + perm_payload: meta.perm_payload, transient: meta.transient, }); if let Some(rt) = inner.node_rt.get_mut(&id) { @@ -434,9 +400,15 @@ impl JobQueue { /// Mark a claimed node terminal, recording its outcome + (truncated) error. /// The crate releases the node's build slot immediately and cascades the - /// `AfterOk` failure cancellation + subtree lease release; terminal hooks - /// run later as the terminal node (no drain). - pub fn complete_node(&self, _dag_id: u64, node_id: NodeId, result: Result<(), String>) { + /// `AfterOk` failure cancellation + subtree lease release. Returns the DAG's + /// terminal summary **iff** this completion rolled its container terminal — + /// the scheduler runs the DAG's inline hook off it. + pub fn complete_node( + &self, + _dag_id: u64, + node_id: NodeId, + result: Result<(), String>, + ) -> Option { let mut inner = self.lock(); let now = now_unix(); let (error, outcome) = match result { @@ -450,24 +422,33 @@ impl JobQueue { rt.error = Some(e); } } + let container = inner.dag_of(node_id); inner.sched.complete(node_id, outcome); - inner.trim_history(now - HISTORY_GRACE_SECS); + // If this completion rolled the DAG's container up to a terminal state, + // hand its summary back so the scheduler fires the inline hook once. + let terminal = container + .filter(|&c| c != node_id && inner.dag_is_terminal(c)) + .and_then(|c| inner.terminal_dag(c)); drop(inner); self.notify.notify_one(); + terminal } - /// Cancel a DAG that hasn't started yet: every work node is still `Queued`, - /// so each is cancelled. No-op (`false`) once any node is running or - /// terminal — an in-flight nix build isn't interruptible. The terminal node - /// (a weak edge) still runs afterwards, so the cancel's terminal hooks - /// (approval resolution, power-intent revert) fire. + /// Cancel a DAG that hasn't started yet: the container + every work node is + /// still `Pending`, so each is cancelled. No-op (`false`) once any node is + /// running or terminal — an in-flight nix build isn't interruptible. The + /// cancel rolls the container up to `Cancelled`, so its inline hook (approval + /// resolution, power-intent revert) still fires. pub fn cancel(&self, dag_id: u64) -> bool { let mut inner = self.lock(); let inner = &mut *inner; - let Some(nodes) = inner.dag_nodes.get(&dag_id) else { + let Some(container) = inner.container(dag_id) else { return false; }; - let all_pending = nodes.iter().all(|&id| { + let ids: Vec = std::iter::once(container) + .chain(inner.subtree(container)) + .collect(); + let all_pending = ids.iter().all(|&id| { inner .sched .graph() @@ -477,7 +458,6 @@ impl JobQueue { if !all_pending { return false; } - let ids: Vec = nodes.clone(); for id in ids { inner.sched.cancel_node(id); } @@ -488,7 +468,7 @@ impl JobQueue { /// Set the step label on a `Running` node. Returns `true` when it changed. pub fn set_step(&self, dag_id: u64, node_id: NodeId, step: &str) -> bool { let mut inner = self.lock(); - if inner.node_dag.get(&node_id) != Some(&dag_id) || !inner.node_running(node_id) { + if inner.dag_of(node_id).map(NodeId::get) != Some(dag_id) || !inner.node_running(node_id) { return false; } let rt = inner.node_rt.entry(node_id).or_default(); @@ -517,7 +497,7 @@ impl JobQueue { /// Link a `build_logs` row to a specific `Running` node. pub fn set_build_log_id(&self, dag_id: u64, node_id: NodeId, log_id: i64) -> bool { let mut inner = self.lock(); - if inner.node_dag.get(&node_id) != Some(&dag_id) || !inner.node_running(node_id) { + if inner.dag_of(node_id).map(NodeId::get) != Some(dag_id) || !inner.node_running(node_id) { return false; } inner.node_rt.entry(node_id).or_default().build_log_id = Some(log_id); @@ -535,20 +515,15 @@ impl JobQueue { true } - /// A DAG's terminal roll-up summary, computed on demand — a terminal hook - /// node's executor calls this to run its side effect. `None` if the DAG - /// is unknown (already history-trimmed). + /// A DAG's terminal roll-up summary, computed on demand from its container. + /// `None` if the DAG id is unknown. Test-only — production reads the summary + /// `complete_node` returns when the container rolls up terminal. + #[cfg(test)] #[must_use] - pub fn terminal_summary(&self, dag_id: u64) -> Option { + pub(crate) fn terminal_summary(&self, dag_id: u64) -> Option { let inner = self.lock(); - let meta = inner.dag_meta.get(&dag_id)?; - Some(TerminalDag { - template: meta.template, - agents: inner.dag_agents(dag_id), - approval_id: meta.approval_id, - state: inner.dag_rollup(dag_id), - error: inner.dag_first_error(dag_id), - }) + let container = inner.container(dag_id)?; + inner.terminal_dag(container) } /// The `(dag_id, agent, kind)` triples for every per-agent lease currently @@ -566,9 +541,9 @@ impl JobQueue { let Resource::Agent(agent) = res else { return None; }; - let dag = *inner.node_dag.get(&holder)?; - let kind = inner.dag_meta.get(&dag)?.transient?; - Some((dag, agent, kind)) + let container = inner.dag_of(holder)?; + let kind = inner.dag_meta(container)?.transient?; + Some((container.get(), agent, kind)) }) .collect() } @@ -576,10 +551,16 @@ impl JobQueue { /// Snapshot every live + retained DAG for `/api/state` + `RebuildQueueChanged`. #[must_use] pub fn snapshot(&self) -> Vec { + self.snapshot_capped(now_unix() - HISTORY_GRACE_SECS) + } + + /// Snapshot the visible DAG set (live + newest-per-template terminal, terminal + /// ones after `grace_cutoff` always kept), sorted by container id. + fn snapshot_capped(&self, grace_cutoff: i64) -> Vec { let inner = self.lock(); - let mut ids: Vec = inner.dag_meta.keys().copied().collect(); - ids.sort_unstable(); - ids.into_iter().filter_map(|d| inner.dag_view(d)).collect() + let mut ids = inner.visible_dags(grace_cutoff); + ids.sort_unstable_by_key(|c| c.get()); + ids.into_iter().filter_map(|c| inner.dag_view(c)).collect() } /// Number of live (non-terminal) DAGs — tests + diagnostics. @@ -588,17 +569,18 @@ impl JobQueue { pub fn live_count(&self) -> usize { let inner = self.lock(); inner - .dag_meta - .keys() - .filter(|&&d| !inner.dag_is_terminal(d)) + .containers() + .into_iter() + .filter(|&c| !inner.dag_is_terminal(c)) .count() } - /// Test hook: trim history with the grace window disabled. + /// Test hook: snapshot with the history grace window disabled, so the + /// per-template cap applies to just-finished terminal DAGs too. #[cfg(test)] - pub(crate) fn trim_ignoring_grace(&self) { - let mut inner = self.lock(); - inner.trim_history(i64::MAX); + #[must_use] + pub(crate) fn snapshot_no_grace(&self) -> Vec { + self.snapshot_capped(i64::MAX) } } @@ -611,27 +593,88 @@ impl QueueInner { .is_some_and(|n| n.state == JobState::Running) } + /// The container node of `dag_id` — the `NodeKind::Dag` root whose id equals + /// `dag_id`. `NodeId` is un-fabricable from a raw `u64`, so this is a search. + fn container(&self, dag_id: u64) -> Option { + self.sched.graph().nodes().find_map(|n| { + (n.parent.is_none() + && n.id.get() == dag_id + && matches!(n.payload.kind, NodeKind::Dag { .. })) + .then_some(n.id) + }) + } + + /// The DAG container a node belongs to — walk its parent chain to the root + /// (`parent == None`), which is the container. Returns `id` itself for a + /// container node. + fn dag_of(&self, id: NodeId) -> Option { + let mut cur = id; + loop { + match self.sched.graph().node(cur)?.parent { + Some(p) => cur = p, + None => return Some(cur), + } + } + } + + /// The DAG's work nodes — its `container`'s subtree, excluding the container. + fn subtree(&self, container: NodeId) -> Vec { + self.sched + .graph() + .nodes() + .filter(|n| n.id != container && self.dag_of(n.id) == Some(container)) + .map(|n| n.id) + .collect() + } + + /// The container's carried domain metadata as an owned read-view. The data + /// lives solely in the [`NodeKind::Dag`] payload — this is a derived read, + /// not a stored side-table. + fn dag_meta(&self, container: NodeId) -> Option { + let NodeKind::Dag { + template, + source, + reason, + transient, + approval_id, + inputs, + perm_payload, + created_at, + } = &self.sched.graph().node(container)?.payload.kind + else { + return None; + }; + Some(DagMeta { + template: *template, + source: *source, + reason: reason.clone(), + transient: *transient, + approval_id: *approval_id, + inputs: inputs.clone(), + perm_payload: perm_payload.clone(), + created_at: *created_at, + }) + } + /// The DAG's currently-running work node, if any (the opaque approval /// pipeline's single-node DAGs make this exact). fn running_node_of(&self, dag_id: u64) -> Option { - self.dag_nodes - .get(&dag_id)? - .iter() - .copied() + let container = self.container(dag_id)?; + self.subtree(container) + .into_iter() .find(|&id| self.node_running(id)) } /// Roll-up state over a DAG's work nodes: `Failed` if any failed; else /// `Running` if any running; else `Queued` if any queued; else `Cancelled` - /// if any cancelled; else `Done`. - fn dag_rollup(&self, dag_id: u64) -> State { - let Some(nodes) = self.dag_nodes.get(&dag_id) else { - return State::Done; - }; + /// if any cancelled; else `Done`. (Kept eager over the subtree — a failed + /// child shows `Failed` immediately, before the container finishes rolling + /// up — matching the pre-container behaviour.) + fn dag_rollup(&self, container: NodeId) -> State { let mut any_running = false; let mut any_queued = false; let mut any_cancelled = false; - for &id in nodes { + for id in self.subtree(container) { match self.sched.graph().node(id).map(|n| n.state) { Some(JobState::Failed) => return State::Failed, Some(JobState::Running | JobState::Finishing) => any_running = true, @@ -651,28 +694,23 @@ impl QueueInner { } } - /// True when every work node of the DAG is terminal. - fn dag_is_terminal(&self, dag_id: u64) -> bool { - self.dag_nodes.get(&dag_id).is_some_and(|nodes| { - nodes.iter().all(|&id| { - self.sched - .graph() - .node(id) - .is_some_and(|n| n.state.is_terminal()) - }) - }) + /// True when the DAG has settled — its container has rolled up terminal + /// (equivalent to every work node being terminal). + fn dag_is_terminal(&self, container: NodeId) -> bool { + self.sched + .graph() + .node(container) + .is_some_and(|n| n.state.is_terminal()) } /// Distinct agents a DAG's work nodes target, in first-seen order. - fn dag_agents(&self, dag_id: u64) -> Vec { + fn dag_agents(&self, container: NodeId) -> Vec { let mut seen: Vec = Vec::new(); - if let Some(nodes) = self.dag_nodes.get(&dag_id) { - for &id in nodes { - if let Some(n) = self.sched.graph().node(id) { - let agent = &n.payload.agent; - if !agent.is_empty() && !seen.iter().any(|s| s == agent) { - seen.push(agent.clone()); - } + for id in self.subtree(container) { + if let Some(n) = self.sched.graph().node(id) { + let agent = &n.payload.agent; + if !agent.is_empty() && !seen.iter().any(|s| s == agent) { + seen.push(agent.clone()); } } } @@ -680,9 +718,8 @@ impl QueueInner { } /// First failed work node's stored error, for the roll-up `error` field. - fn dag_first_error(&self, dag_id: u64) -> Option { - let nodes = self.dag_nodes.get(&dag_id)?; - for &id in nodes { + fn dag_first_error(&self, container: NodeId) -> Option { + for id in self.subtree(container) { if self .sched .graph() @@ -696,15 +733,27 @@ impl QueueInner { None } - /// Rebuild the wire [`DagView`] for a DAG from its metadata + work nodes + - /// per-node runtime. The internal terminal node is excluded. - fn dag_view(&self, dag_id: u64) -> Option { - let meta = self.dag_meta.get(&dag_id)?; - let node_ids = self.dag_nodes.get(&dag_id)?; + /// A DAG's terminal roll-up summary — the input to its inline hook. + fn terminal_dag(&self, container: NodeId) -> Option { + let meta = self.dag_meta(container)?; + Some(TerminalDag { + template: meta.template, + agents: self.dag_agents(container), + approval_id: meta.approval_id, + state: self.dag_rollup(container), + error: self.dag_first_error(container), + }) + } + + /// Rebuild the wire [`DagView`] for a DAG from its container metadata + work + /// nodes + per-node runtime. + fn dag_view(&self, container: NodeId) -> Option { + let meta = self.dag_meta(container)?; + let node_ids = self.subtree(container); let mut nodes = Vec::with_capacity(node_ids.len()); let mut started: Vec = Vec::new(); let mut finished: Vec = Vec::new(); - for &id in node_ids { + for &id in &node_ids { let Some(node) = self.sched.graph().node(id) else { continue; }; @@ -736,11 +785,11 @@ impl QueueInner { error: rt.and_then(|r| r.error.clone()), }); } - let is_terminal = self.dag_is_terminal(dag_id); + let is_terminal = self.dag_is_terminal(container); Some(DagView { - id: dag_id, + id: container.get(), kind: meta.template, - state: self.dag_rollup(dag_id), + state: self.dag_rollup(container), source: meta.source, reason: meta.reason.clone(), enqueued_at: meta.created_at, @@ -757,56 +806,55 @@ impl QueueInner { }) } - /// Keep only the newest [`MAX_HISTORY_PER_TEMPLATE`] terminal DAGs per - /// template, evicting older ones' side-tables (their now-terminal crate - /// nodes linger harmlessly in the graph — a bounded-prune primitive is a - /// tracked follow-up). Terminal DAGs finished after `grace_cutoff` are - /// exempt (never counted). - fn trim_history(&mut self, grace_cutoff: i64) { - let mut terminal: Vec<(u64, Template, i64)> = self - .dag_meta - .keys() - .copied() - .filter(|&d| self.dag_is_terminal(d)) - .map(|d| { - let finished = self - .dag_nodes - .get(&d) - .into_iter() - .flatten() - .filter_map(|id| self.node_rt.get(id).and_then(|r| r.finished_at)) - .max() - .unwrap_or(0); - (d, self.dag_meta[&d].template, finished) - }) - .collect(); - // Newest first, so the cap keeps the most recent per template. - terminal.sort_by(|a, b| b.2.cmp(&a.2).then(b.0.cmp(&a.0))); - let mut counts: HashMap = HashMap::new(); - let mut evict: Vec = Vec::new(); - for (d, template, finished) in terminal { - if finished > grace_cutoff { - continue; - } - let c = counts.entry(template).or_insert(0); - *c += 1; - if *c > MAX_HISTORY_PER_TEMPLATE { - evict.push(d); - } - } - for d in evict { - if let Some(nodes) = self.dag_nodes.remove(&d) { - for id in nodes { - self.node_dag.remove(&id); - self.node_rt.remove(&id); + /// When a DAG's work node finishes on `finished_at` — the max over its + /// subtree, for the history cap ordering. + fn dag_finished_at(&self, container: NodeId) -> i64 { + self.subtree(container) + .iter() + .filter_map(|id| self.node_rt.get(id).and_then(|r| r.finished_at)) + .max() + .unwrap_or(0) + } + + /// Every DAG container node id in the graph. + fn containers(&self) -> Vec { + self.sched + .graph() + .nodes() + .filter(|n| n.parent.is_none() && matches!(n.payload.kind, NodeKind::Dag { .. })) + .map(|n| n.id) + .collect() + } + + /// The **visible** DAG set for the snapshot: every live (non-terminal) DAG, + /// plus the newest [`MAX_HISTORY_PER_TEMPLATE`] terminal DAGs per template + /// (terminal DAGs finished after `grace_cutoff` are always kept). Crate nodes + /// for evicted DAGs linger in the graph (bounded-prune is a Stage-C + /// follow-up); this filter is what bounds what the dashboard sees. + fn visible_dags(&self, grace_cutoff: i64) -> Vec { + let mut live: Vec = Vec::new(); + let mut terminal: Vec<(NodeId, Template, i64)> = Vec::new(); + for c in self.containers() { + if self.dag_is_terminal(c) { + if let Some(meta) = self.dag_meta(c) { + terminal.push((c, meta.template, self.dag_finished_at(c))); } + } else { + live.push(c); } - if let Some(term) = self.terminal_node.remove(&d) { - self.node_dag.remove(&term); - self.node_rt.remove(&term); - } - self.dag_meta.remove(&d); } + // Newest first so the per-template cap keeps the most recent. + terminal.sort_by(|a, b| b.2.cmp(&a.2).then(b.0.get().cmp(&a.0.get()))); + let mut counts: HashMap = HashMap::new(); + let mut kept = live; + for (c, template, finished) in terminal { + let n = counts.entry(template).or_insert(0); + *n += 1; + if *n <= MAX_HISTORY_PER_TEMPLATE || finished > grace_cutoff { + kept.push(c); + } + } + kept } } diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs index aa85eaf9..7365ce89 100644 --- a/hive-c0re/src/job_queue/model.rs +++ b/hive-c0re/src/job_queue/model.rs @@ -141,21 +141,6 @@ pub enum NodeKind { /// `Prebuild`, but that's a no-op there — the agent is down, so prebuild /// is skipped.) SetWanted { up: bool }, - /// Per-DAG terminal hook (approval-driven DAGs — spawn / opaque deploy): - /// resolve the DAG's approval row from the rolled-up outcome. Appended once - /// with a weak (`AfterAny`) edge on the DAG's tails, so it runs exactly when - /// the DAG has settled (any outcome, including a cancel before starting). - /// Build-slot- and lease-exempt; always runs (weak edge ⇒ never cascaded). - ResolveApproval, - /// Per-DAG terminal hook (rebuild / perm-change DAGs): emit one `Rebuilt` - /// manager event per targeted agent — `ok` on `Done`, `!ok` on `Failed`, - /// none on cancel. Appended weak-dep on the tails; slot/lease-exempt. - EmitRebuilt, - /// Per-DAG terminal hook (power-op DAGs): on a *cancelled* DAG, revert each - /// agent's `wanted` intent to its observed state — the operator's cancel - /// means "don't do it". Noop on any non-cancelled outcome. Appended weak-dep - /// on the tails; slot/lease-exempt. - RevertIntent, /// The **DAG container** node: one per submitted DAG, carrying the group's /// domain metadata. Every template node hangs *under* it (its subtree), so /// the container's `NodeId` **is** the DAG id, its rolled-up state **is** the @@ -196,9 +181,6 @@ impl NodeKind { NodeKind::WritePermFile => "write_perm_file", NodeKind::ApprovalDeploy => "approval_deploy", NodeKind::SetWanted { .. } => "set_wanted", - NodeKind::ResolveApproval => "resolve_approval", - NodeKind::EmitRebuilt => "emit_rebuilt", - NodeKind::RevertIntent => "revert_intent", NodeKind::Dag { .. } => "dag", } } diff --git a/hive-c0re/src/job_queue/scheduler.rs b/hive-c0re/src/job_queue/scheduler.rs index 1b9a843d..1c2022f9 100644 --- a/hive-c0re/src/job_queue/scheduler.rs +++ b/hive-c0re/src/job_queue/scheduler.rs @@ -106,9 +106,10 @@ fn handle_completion(coord: &Arc, done: NodeDone) { .job_queue .append_subgraph(claim.dag_id, subgraph, claim.node_id); } - coord + let terminal = coord .job_queue .complete_node(claim.dag_id, claim.node_id, Ok(())); + fire_terminal_hook(coord, terminal); } Err(e) => { let msg = format!("{e:#}"); @@ -120,9 +121,10 @@ fn handle_completion(coord: &Arc, done: NodeDone) { error = %msg, "job_queue: node failed" ); - coord + let terminal = coord .job_queue .complete_node(claim.dag_id, claim.node_id, Err(msg)); + fire_terminal_hook(coord, terminal); } } // The next loop iteration re-reconciles the transient pills against the @@ -130,6 +132,19 @@ fn handle_completion(coord: &Arc, done: NodeDone) { coord.emit_rebuild_queue_snapshot(); } +/// Fire a settled DAG's inline terminal hook (approval-resolve / rebuilt-emit / +/// intent-revert) off the container-terminal summary `complete_node` returned — +/// spawned so the async hook doesn't block the scheduler loop. +fn fire_terminal_hook(coord: &Arc, terminal: Option) { + let Some(terminal) = terminal else { + return; + }; + let coord = Arc::clone(coord); + tokio::spawn(async move { + exec::run_terminal_hook(&coord, &terminal).await; + }); +} + /// Reconcile the transient-guard set against live lease ownership: drop pills /// whose lease is no longer held, create one for each newly-held `(dag, agent)`. fn reconcile_transients( From 2294cd4516fdc44af4217c4eb04d24c1955677a1 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 20 Jul 2026 22:44:19 +0200 Subject: [PATCH 6/9] feat(#2591): auto-complete the DAG container + run terminal hooks inline --- hive-c0re/src/dashboard/schedules.rs | 6 +- hive-c0re/src/job_queue/exec.rs | 2 +- hive-c0re/src/job_queue/mod.rs | 40 +++++++------ hive-c0re/src/job_queue/tests.rs | 89 ++++++++++------------------ 4 files changed, 59 insertions(+), 78 deletions(-) diff --git a/hive-c0re/src/dashboard/schedules.rs b/hive-c0re/src/dashboard/schedules.rs index 8ad220d9..6b990f93 100644 --- a/hive-c0re/src/dashboard/schedules.rs +++ b/hive-c0re/src/dashboard/schedules.rs @@ -127,8 +127,10 @@ pub(super) async fn post_rebuild_queue_cancel( State(state): State, AxumPath(id): AxumPath, ) -> Response { - let cancelled = state.coord.job_queue.cancel(id); - if cancelled { + if let Some(terminal) = state.coord.job_queue.cancel(id) { + // Fire the DAG's inline terminal hook (power-op intent revert / approval + // resolution) off the cancel roll-up, then surface the flip live. + crate::job_queue::exec::run_terminal_hook(&state.coord, &terminal).await; state.coord.emit_rebuild_queue_snapshot(); axum::Json(serde_json::json!({"cancelled": true})).into_response() } else { diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index 0c7d9d08..83bb8014 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -111,7 +111,7 @@ pub(super) async fn run_node(coord: &Arc, claim: &Claim) -> Result< /// Run a settled DAG's inline terminal hook, dispatched off its rolled-up /// summary — the container-terminal replacement for the old per-DAG hook node. /// Always best-effort: a hook failure is logged inside, never surfaced. -pub(super) async fn run_terminal_hook(coord: &Arc, terminal: &super::TerminalDag) { +pub(crate) async fn run_terminal_hook(coord: &Arc, terminal: &super::TerminalDag) { match super::terminal_hook(terminal.template, terminal.approval_id) { Some(super::HookKind::ResolveApproval) => { crate::actions::resolve_approval_dag(coord, terminal).await; diff --git a/hive-c0re/src/job_queue/mod.rs b/hive-c0re/src/job_queue/mod.rs index 1d15acdd..b6abc7f6 100644 --- a/hive-c0re/src/job_queue/mod.rs +++ b/hive-c0re/src/job_queue/mod.rs @@ -312,6 +312,11 @@ impl JobQueue { .map_err(|e| anyhow::anyhow!("job_queue: container insert failed: {e}"))?; inner.node_rt.insert(container, NodeRuntime::default()); insert_group(&mut inner, &spec.nodes, Some(container))?; + // Settle the container's own (no-op) logic immediately so it parks in + // `Finishing` and its children become runnable — it never needs claiming + // or executing, and stays out of `claim_ready`. It rolls up terminal when + // its whole subtree settles (that's the DAG-done signal). + inner.sched.complete(container, Outcome::Done); drop(inner); self.notify.notify_one(); Ok(container.get()) @@ -434,21 +439,17 @@ impl JobQueue { terminal } - /// Cancel a DAG that hasn't started yet: the container + every work node is - /// still `Pending`, so each is cancelled. No-op (`false`) once any node is - /// running or terminal — an in-flight nix build isn't interruptible. The - /// cancel rolls the container up to `Cancelled`, so its inline hook (approval - /// resolution, power-intent revert) still fires. - pub fn cancel(&self, dag_id: u64) -> bool { + /// Cancel a DAG that hasn't started yet: every work node is still `Pending`, + /// so each is cancelled. `None` once any work node is running or terminal — + /// an in-flight nix build isn't interruptible. Otherwise the container is + /// rolled up so the DAG settles (wire state `Cancelled`) and its terminal + /// summary is returned — the caller fires the inline hook (power-intent + /// revert / approval resolution) off it. + pub fn cancel(&self, dag_id: u64) -> Option { let mut inner = self.lock(); - let inner = &mut *inner; - let Some(container) = inner.container(dag_id) else { - return false; - }; - let ids: Vec = std::iter::once(container) - .chain(inner.subtree(container)) - .collect(); - let all_pending = ids.iter().all(|&id| { + let container = inner.container(dag_id)?; + let work = inner.subtree(container); + let all_pending = work.iter().all(|&id| { inner .sched .graph() @@ -456,13 +457,18 @@ impl JobQueue { .is_some_and(|n| n.state == JobState::Pending) }); if !all_pending { - return false; + return None; } - for id in ids { + for id in work { inner.sched.cancel_node(id); } + // Roll the container up so the DAG reaches a terminal state (all children + // now `Cancelled`); `dag_rollup` reports `Cancelled` to the wire. + inner.sched.complete(container, Outcome::Done); + let terminal = inner.terminal_dag(container); + drop(inner); self.notify.notify_one(); - true + terminal } /// Set the step label on a `Running` node. Returns `true` when it changed. diff --git a/hive-c0re/src/job_queue/tests.rs b/hive-c0re/src/job_queue/tests.rs index 2d184803..0063b7fc 100644 --- a/hive-c0re/src/job_queue/tests.rs +++ b/hive-c0re/src/job_queue/tests.rs @@ -280,24 +280,13 @@ fn lease_serializes_two_lifecycle_dags_for_same_agent() { assert_eq!(second.dag_id, restart); assert_eq!(second.kind.as_str(), "reconcile"); q.complete_node(restart, second.node_id, Ok(())); - // Restart's work is terminal → its lease releases. Its terminal node and - // stop's now-unblocked Reconcile both become ready in the same pass. - let ready = q.claim_ready(); - let restart_fin = ready - .iter() - .find(|c| c.dag_id == restart && c.kind.as_str() == "revert_intent") - .expect("restart finalize ready"); - q.complete_node(restart, restart_fin.node_id, Ok(())); - let third = ready - .iter() - .find(|c| c.dag_id == stop) - .expect("stop reconcile ready once the lease is freed"); + // Restart's work is terminal → its lease releases, so stop's now-unblocked + // Reconcile becomes ready (restart's inline hook fired off the returned + // summary — no terminal-hook node). + let third = claim_one(&q); + assert_eq!(third.dag_id, stop); assert_eq!(third.kind.as_str(), "reconcile"); q.complete_node(stop, third.node_id, Ok(())); - // stop's terminal node (revert-intent) then runs. - let stop_fin = claim_one(&q); - assert_eq!(stop_fin.kind.as_str(), "revert_intent"); - q.complete_node(stop, stop_fin.node_id, Ok(())); assert_eq!(state_of(&q, restart), State::Done); assert_eq!(state_of(&q, stop), State::Done); } @@ -340,15 +329,10 @@ fn lease_exempt_prebuild_overlaps_other_dag_on_same_agent() { .expect("reconcile claim") .clone(); q.complete_node(stop, reconcile.node_id, Ok(())); - // stop's Reconcile done → its lease frees (rebuild's StopForUpdate unblocks) - // and its terminal node becomes ready; both surface in the same pass. + // stop's Reconcile done → its lease frees, so rebuild's StopForUpdate + // unblocks. (stop's DAG rolls up terminal; its inline hook fires off the + // returned summary — no terminal-hook node in the claim set.) let after = q.claim_ready(); - assert!( - after - .iter() - .any(|c| c.dag_id == stop && c.kind.as_str() == "revert_intent"), - "stop DAG's finalize runs once its work settles" - ); let sfu = after .iter() .find(|c| c.kind.as_str() == "stop_for_update") @@ -788,13 +772,11 @@ fn failed_reconcile_marks_dag_failed() { fn cancel_clears_queued_dag() { let q = JobQueue::new(1); let id = submit(&q, rebuild("agent-a", "r")); - assert!(q.cancel(id)); + // Cancel returns the terminal summary (state `Cancelled`) — the inline hook + // fires off it at the caller; there's no terminal-hook node to claim. + let terminal = q.cancel(id).expect("cancelled"); + assert_eq!(terminal.state, State::Cancelled); assert_eq!(state_of(&q, id), State::Cancelled); - // The terminal node's weak edges are satisfied by the cancelled (terminal) - // work nodes, so it still runs its hooks — it's the one thing left claimable. - let fin = claim_one(&q); - assert_eq!(fin.kind.as_str(), "emit_rebuilt"); - q.complete_node(id, fin.node_id, Ok(())); assert!(q.claim_ready().is_empty()); } @@ -803,32 +785,32 @@ fn cancel_refuses_running_dag() { let q = JobQueue::new(1); let id = submit(&q, rebuild("agent-a", "r")); let _ = claim_one(&q); - assert!(!q.cancel(id)); + assert!(q.cancel(id).is_none()); assert_eq!(state_of(&q, id), State::Running); } // ---- terminal reporting + lease release ---- #[test] -fn terminal_node_runs_after_work_settles_and_lease_released() { +fn dag_settles_terminal_and_releases_lease_after_work() { let q = JobQueue::new(1); let id = submit(&q, restart_online(&["agent-a"], false, "r")); - // restart = StopForUpdate → Reconcile; the terminal node (which weak-deps on - // the chain tail) isn't runnable until the whole chain is terminal. + // restart = StopForUpdate → Reconcile. let stop = claim_one(&q); + assert_eq!(stop.kind.as_str(), "stop_for_update"); q.complete_node(id, stop.node_id, Ok(())); let rec = claim_one(&q); assert_eq!(rec.kind.as_str(), "reconcile"); - q.complete_node(id, rec.node_id, Ok(())); - // Work settled → the terminal node is now the runnable one; it carries the - // terminal roll-up the hook consumes via `terminal_summary`. - let fin = claim_one(&q); - assert_eq!(fin.kind.as_str(), "revert_intent"); - let summary = q.terminal_summary(id).expect("terminal summary"); + // Completing the last work node rolls the container up terminal and returns + // the summary the inline hook consumes — there is no terminal-hook node. + let summary = q + .complete_node(id, rec.node_id, Ok(())) + .expect("terminal summary"); assert_eq!(summary.state, State::Done); - q.complete_node(id, fin.node_id, Ok(())); - // Lease released (freed when the work chain settled, ahead of finalize): a - // new DAG for the agent claims immediately. + assert!(q.claim_ready().is_empty(), "no terminal-hook node to claim"); + assert_eq!(state_of(&q, id), State::Done); + // Lease released when the work chain settled: a new DAG for the agent claims + // immediately. let next = submit( &q, templates::reconcile_only( @@ -853,16 +835,11 @@ fn cancelled_dag_finalizes_with_terminal_rollup() { &q, templates::approval_deploy("agent-a", 7, "approval #7".to_owned()), ); - assert!(q.cancel(id)); - // The terminal node's weak edges still fire on a fully-cancelled DAG, so its - // hook (approval resolution) runs — surfaced here as a claimable resolve- - // approval node whose `terminal_summary` is Cancelled + carries the approval id. - let fin = claim_one(&q); - assert_eq!(fin.kind.as_str(), "resolve_approval"); - let summary = q.terminal_summary(id).expect("terminal summary"); + // Cancel rolls the DAG up terminal and returns its summary — the inline hook + // (approval resolution) runs off it at the caller. Cancelled + approval id 7. + let summary = q.cancel(id).expect("cancelled"); assert_eq!(summary.state, State::Cancelled); assert_eq!(summary.approval_id, Some(7)); - q.complete_node(id, fin.node_id, Ok(())); // The cancelled DAG's summary stays available (until history-trimmed) and // unrelated later activity doesn't disturb it. let other = submit(&q, rebuild("agent-b", "r")); @@ -931,12 +908,9 @@ fn history_evicts_old_terminals_per_template() { ), ); let c = claim_one(&q); + // Completing the single work node rolls the container up terminal (its + // inline hook fires off the returned summary — no terminal-hook node). q.complete_node(id, c.node_id, Ok(())); - // Drain the DAG's terminal node too, so the next iteration's claim sees - // only its own work (the terminal node is excluded from the view + rollup). - let fin = claim_one(&q); - assert_eq!(fin.kind.as_str(), "revert_intent"); - q.complete_node(id, fin.node_id, Ok(())); } // Fresh terminals are inside the grace window: nothing evicts yet, // so a ~1s QueueDag poller can still observe every terminal state @@ -947,8 +921,7 @@ fn history_evicts_old_terminals_per_template() { "grace window protects fresh terminals" ); // Past the grace window the per-template cap applies. - q.trim_ignoring_grace(); - assert_eq!(q.snapshot().len(), 5, "per-template history cap"); + assert_eq!(q.snapshot_no_grace().len(), 5, "per-template history cap"); assert_eq!(q.live_count(), 0); } From be2dfa8cd3f7b0da00da701bd25a9d15c39bf2af Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 20 Jul 2026 23:00:20 +0200 Subject: [PATCH 7/9] =?UTF-8?q?refactor(#2591):=20make=20NodeKind=20the=20?= =?UTF-8?q?queue=20payload=20=E2=80=94=20drop=20JobPayload,=20agent=20into?= =?UTF-8?q?=20variants?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hive-c0re/src/job_queue/exec.rs | 47 ++++++------ hive-c0re/src/job_queue/mod.rs | 80 +++++++++----------- hive-c0re/src/job_queue/model.rs | 109 ++++++++++++++++----------- hive-c0re/src/job_queue/resource.rs | 27 +++---- hive-c0re/src/job_queue/submit.rs | 51 +++++++++---- hive-c0re/src/job_queue/templates.rs | 71 ++++++++++------- hive-c0re/src/job_queue/tests.rs | 21 +++--- hive-c0re/src/workers/auto_update.rs | 4 +- 8 files changed, 233 insertions(+), 177 deletions(-) diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index 83bb8014..86ab6994 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -82,24 +82,24 @@ pub(super) async fn run_node(coord: &Arc, claim: &Claim) -> Result< node_id: claim.node_id, }; match &claim.kind { - NodeKind::Prebuild { relock } => run_prebuild(coord, claim, &ctx, *relock).await, - NodeKind::Swap => run_swap(coord, claim, &ctx).await, - NodeKind::PostSwap => run_post_swap(coord, claim, &ctx).await, - NodeKind::Provision => run_provision(coord, claim, &ctx).await, - NodeKind::Create => run_create(claim, &ctx).await, + NodeKind::Prebuild { relock, .. } => run_prebuild(coord, claim, &ctx, *relock).await, + NodeKind::Swap { .. } => run_swap(coord, claim, &ctx).await, + NodeKind::PostSwap { .. } => run_post_swap(coord, claim, &ctx).await, + NodeKind::Provision { .. } => run_provision(coord, claim, &ctx).await, + NodeKind::Create { .. } => run_create(claim, &ctx).await, NodeKind::MetaLock { sweep, fanout } => { run_meta_lock(coord, claim, &ctx, *sweep, fanout.clone()).await } - NodeKind::Reconcile => run_reconcile(coord, claim).await, - NodeKind::Start => run_start(coord, claim, &ctx).await, - NodeKind::Stop => run_stop(coord, claim, &ctx).await, - NodeKind::StopForUpdate => run_stop_for_update(coord, claim, &ctx).await, - NodeKind::Signal => Ok(run_signal(coord, claim, &ctx)), - NodeKind::Drain => run_drain(coord, claim, &ctx).await, - NodeKind::WriteDropin => run_write_dropin(coord, claim).await, - NodeKind::WritePermFile => run_write_perm_file(coord, claim, &ctx).await, - NodeKind::ApprovalDeploy => run_approval_deploy(coord, claim).await, - NodeKind::SetWanted { up } => run_set_wanted(coord, claim, *up), + NodeKind::Reconcile { .. } => run_reconcile(coord, claim).await, + NodeKind::Start { .. } => run_start(coord, claim, &ctx).await, + NodeKind::Stop { .. } => run_stop(coord, claim, &ctx).await, + NodeKind::StopForUpdate { .. } => run_stop_for_update(coord, claim, &ctx).await, + NodeKind::Signal { .. } => Ok(run_signal(coord, claim, &ctx)), + NodeKind::Drain { .. } => run_drain(coord, claim, &ctx).await, + NodeKind::WriteDropin { .. } => run_write_dropin(coord, claim).await, + NodeKind::WritePermFile { .. } => run_write_perm_file(coord, claim, &ctx).await, + NodeKind::ApprovalDeploy { .. } => run_approval_deploy(coord, claim).await, + NodeKind::SetWanted { up, .. } => run_set_wanted(coord, claim, *up), // Pure grouping container — no work; completing it lets it reach // `Finishing` so its child template nodes start. The DAG's terminal // hook fires (inline, via `run_terminal_hook`) when the container itself @@ -398,14 +398,17 @@ async fn run_reconcile(coord: &Arc, claim: &Claim) -> Result sub(NodeKind::Start), - ReconcileAction::Stop => sub(NodeKind::Stop), + ReconcileAction::Start => sub(NodeKind::Start { + agent: name.clone(), + }), + ReconcileAction::Stop => sub(NodeKind::Stop { + agent: name.clone(), + }), ReconcileAction::Noop => { tracing::debug!(%name, wanted = wanted.as_str(), running, "reconcile: noop"); Vec::new() diff --git a/hive-c0re/src/job_queue/mod.rs b/hive-c0re/src/job_queue/mod.rs index b6abc7f6..bf6158be 100644 --- a/hive-c0re/src/job_queue/mod.rs +++ b/hive-c0re/src/job_queue/mod.rs @@ -4,22 +4,22 @@ //! watcher, deferred-start follow-up, meta-update cascade) collapse into DAG //! *shapes* over the shared node primitives ([`model::NodeKind`]). //! -//! [`hive_jobq`] owns the graph, the two-class resource pool, and the settle -//! loop; this module maps hive-c0re's concepts onto it: -//! - `NodeKind` + agent → the crate payload [`resource::JobPayload`]; -//! - the two resource classes → [`resource::Resource`] -//! ([`resource::Resource::BuildSlot`] node-held, [`resource::Resource::Agent`] -//! subtree-held), derived per node by [`resource::JobPayload::resource_deps`]; -//! - a host **DAG id** groups a set of crate nodes; a node declares only its -//! `deps` (chain edges + resource deps), and the crate scheduler infers lease -//! re-entrancy from the [`Dep::Node`] graph — a node needing an agent lease a -//! node it depends on already holds re-enters it, with no parent annotation; -//! - per-DAG terminal work is a *focused* graph node per concern — `ResolveApproval` -//! (approval DAGs), `EmitRebuilt` (rebuild / perm-change), `RevertIntent` -//! (cancelled power-ops) — weak-depending on the DAG's tail nodes (extended onto -//! any runtime-appended subgraph's tail), so it runs once everything settles. -//! Hook-less DAGs (meta-update / boot / reconcile) get none. No drained event -//! stream, and no one node branching on DAG metadata. +//! [`hive_jobq`] owns the graph, the two-class resource pool, and the roll-up +//! settle loop; this module maps hive-c0re's concepts onto it: +//! - [`model::NodeKind`] **is** the crate payload `N` directly — each variant +//! carries the agent it targets ([`NodeKind::agent`]); the two resource +//! classes are [`resource::Resource`] (`BuildSlot` node-held, `Agent` lease +//! subtree-held), derived per node by [`NodeKind::resource_deps`]; +//! - a **DAG is a single container node** ([`NodeKind::Dag`], `parent = None`) +//! carrying the group's metadata, with the template's nodes hung under it as +//! its subtree (the **parent axis** groups; `deps` order). So the container's +//! `NodeId` is the DAG id, its rolled-up state is the DAG state, and membership +//! is a graph walk — there are no host grouping side-tables. The lease is owned +//! by a subtree root and borrowed by its descendants (continuity); +//! - per-DAG terminal work runs **inline** ([`exec::run_terminal_hook`]) when the +//! container rolls up terminal — dispatched off its template +//! ([`terminal_hook`]): approval-resolve, `Rebuilt`-emit, or power-intent +//! revert. No terminal-hook node, no drained event stream. //! //! The queue is runtime-only (no persistence): an empty graph on boot; desired //! state is re-derived by the reconcile sweep. A single scheduler task @@ -49,7 +49,7 @@ use crate::coordinator::TransientKind; pub use model::{ DagSpec, DagView, DepWhen, NodeKind, NodeSpec, PermPayload, Source, State, Template, }; -use resource::{JobPayload, Resource}; +use resource::Resource; /// How many terminal DAGs (`Done` / `Failed` / `Cancelled`) to retain per /// template in the snapshot, matching the old per-kind history cap. @@ -130,7 +130,7 @@ struct DagMeta { /// are graph queries ([`QueueInner::container`] / [`QueueInner::subtree`] / /// [`QueueInner::dag_meta`]). One shared crate [`Graph`] holds every DAG. struct QueueInner { - sched: Scheduler, + sched: Scheduler, /// Per-node runtime metadata (build-log id, step, timestamps, error) — /// mutable after insert, so it can't ride the immutable node payload. node_rt: HashMap, @@ -232,10 +232,7 @@ fn insert_group( ) -> anyhow::Result> { let mut ids: Vec = Vec::with_capacity(nodes.len()); for ns in nodes { - let payload = JobPayload { - kind: ns.kind.clone(), - agent: ns.agent.clone(), - }; + let payload = ns.kind.clone(); let mut deps = payload.resource_deps(); for d in &ns.deps { deps.push(Dep::Node { @@ -293,18 +290,15 @@ impl JobQueue { let container = inner .sched .append( - JobPayload { - kind: NodeKind::Dag { - template: spec.template, - source: spec.source, - reason: spec.reason, - transient: spec.transient, - approval_id: spec.approval_id, - inputs: spec.inputs, - perm_payload: spec.perm_payload, - created_at: now_unix(), - }, - agent: String::new(), + NodeKind::Dag { + template: spec.template, + source: spec.source, + reason: spec.reason, + transient: spec.transient, + approval_id: spec.approval_id, + inputs: spec.inputs, + perm_payload: spec.perm_payload, + created_at: now_unix(), }, Vec::new(), None, @@ -377,8 +371,8 @@ impl JobQueue { let Some(node) = inner.sched.graph().node(id) else { continue; }; - let kind = node.payload.kind.clone(); - let agent = node.payload.agent.clone(); + let kind = node.payload.clone(); + let agent = node.payload.agent().to_owned(); let Some(container) = inner.dag_of(id) else { continue; }; @@ -605,7 +599,7 @@ impl QueueInner { self.sched.graph().nodes().find_map(|n| { (n.parent.is_none() && n.id.get() == dag_id - && matches!(n.payload.kind, NodeKind::Dag { .. })) + && matches!(n.payload, NodeKind::Dag { .. })) .then_some(n.id) }) } @@ -646,7 +640,7 @@ impl QueueInner { inputs, perm_payload, created_at, - } = &self.sched.graph().node(container)?.payload.kind + } = &self.sched.graph().node(container)?.payload else { return None; }; @@ -714,9 +708,9 @@ impl QueueInner { let mut seen: Vec = Vec::new(); for id in self.subtree(container) { if let Some(n) = self.sched.graph().node(id) { - let agent = &n.payload.agent; + let agent = n.payload.agent(); if !agent.is_empty() && !seen.iter().any(|s| s == agent) { - seen.push(agent.clone()); + seen.push(agent.to_owned()); } } } @@ -780,8 +774,8 @@ impl QueueInner { } nodes.push(NodeView { id: id.get(), - agent: node.payload.agent.clone(), - kind: node.payload.kind.as_str().to_owned(), + agent: node.payload.agent().to_owned(), + kind: node.payload.as_str().to_owned(), deps, state: to_wire_state(node.state), step: rt.and_then(|r| r.step.clone()), @@ -827,7 +821,7 @@ impl QueueInner { self.sched .graph() .nodes() - .filter(|n| n.parent.is_none() && matches!(n.payload.kind, NodeKind::Dag { .. })) + .filter(|n| n.parent.is_none() && matches!(n.payload, NodeKind::Dag { .. })) .map(|n| n.id) .collect() } diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs index 7365ce89..9c4da05c 100644 --- a/hive-c0re/src/job_queue/model.rs +++ b/hive-c0re/src/job_queue/model.rs @@ -56,12 +56,12 @@ pub enum NodeKind { /// skipped when the container is already down — it only exists to /// shrink the swap's downtime, which a stopped agent doesn't need /// (the sync + dir prep still run; `Swap` builds inline). - Prebuild { relock: bool }, + Prebuild { agent: String, relock: bool }, /// `nixos-container update` profile-swap (requires the container /// stopped). Re-applies nspawn flags + resource limits first — /// rebuild is the reconcile verb. The post-rebuild bookkeeping tail /// lives in the sibling `PostSwap` node. - Swap, + Swap { agent: String }, /// The post-`Swap` bookkeeping tail as a first-class node: rev marker, /// forge + matrix sync, manager kick, container rescan, meta-inputs /// snapshot. Split out of `Swap` for dashboard visibility + retry @@ -71,16 +71,16 @@ pub enum NodeKind { /// recovery still runs. Store/forge/matrix work only — no nix build, so /// build-slot-exempt; the agent lease taken at `Swap` is held across the /// whole chain until `Reconcile` settles, so it's not re-declared here. - PostSwap, + PostSwap { agent: String }, /// First-spawn pre-create provisioning: proposed/applied repos, /// state subvolume, and meta registration (`sync_agents`). Runs /// ahead of `Create` so the `nixos-container create --flake /// meta#` ref resolves. Store/meta-only — no container yet — /// so it's lease- and build-slot-exempt like `Prebuild`. - Provision, + Provision { agent: String }, /// First-spawn `nixos-container create` proper. Assumes the /// upstream `Provision` node already registered the agent in meta. - Create, + Create { agent: String }, /// Meta flake lock bump. `sweep = false`: `meta::lock_update` /// (commit fused, under `META_LOCK`) with the DAG's `inputs`; /// `sweep = true`: `meta::lock_update_hyperhive`, *non-fatal* (a @@ -97,36 +97,36 @@ pub enum NodeKind { /// `Offline` & up, else noop). The mechanical work is not done in /// this node — it fans a child [`NodeKind::Start`] / [`NodeKind::Stop`] /// DAG out at runtime so the sub-step is a first-class DAG node. - Reconcile, + Reconcile { agent: String }, /// Mechanical container start: the start preamble (runtime dir + /// drop-ins), `start_with_fallback`, MCP listener registration, and /// the manager kick. Fanned out by a [`NodeKind::Reconcile`] that /// observed `wanted = Up` and the container down. - Start, + Start { agent: String }, /// Mechanical container stop: `nixos-container` kill, MCP listener /// unregister, and the `Killed` manager notify. Fanned out by a /// [`NodeKind::Reconcile`] that observed `wanted = Offline` and up. - Stop, + Stop { agent: String }, /// Mechanical `nixos-container stop` for the profile swap. Never /// touches `wanted`. Noop if already stopped. - StopForUpdate, + StopForUpdate { agent: String }, /// Set the graceful-stop fence + kick the harness so it runs one /// stop-checkpoint turn. - Signal, + Signal { agent: String }, /// Await the harness clearing the fence, bounded by /// `GRACEFUL_STOP_TIMEOUT`. Resolves ok either way — the /// downstream `Reconcile` performs the actual stop. - Drain, + Drain { agent: String }, /// `set_nspawn_flags` + `set_resource_limits` + daemon-reload. - WriteDropin, + WriteDropin { agent: String }, /// Commit `tool-groups.json` / `capabilities.json` per the DAG's /// `perm_payload` (commit fused under `META_LOCK`). - WritePermFile, + WritePermFile { agent: String }, /// Opaque approval deploy pipeline (`MergeConfigPr`): the two-phase /// prepare/finalize/abort meta deploy stays inside `actions.rs` in v1 — /// deliberately not /// modeled as scheduler nodes (see the design doc §9). - ApprovalDeploy, + ApprovalDeploy { agent: String }, /// Write the agent's durable power intent (`wanted = Up` when `up`, else /// `Offline`) as a first-class DAG node, at the head of a power-op /// template so the downstream `Reconcile` reads it. Replaces the old @@ -140,7 +140,7 @@ pub enum NodeKind { /// the DAG. (In `stale_start` the lease is thus held across the head /// `Prebuild`, but that's a no-op there — the agent is down, so prebuild /// is skipped.) - SetWanted { up: bool }, + SetWanted { agent: String, up: bool }, /// The **DAG container** node: one per submitted DAG, carrying the group's /// domain metadata. Every template node hangs *under* it (its subtree), so /// the container's `NodeId` **is** the DAG id, its rolled-up state **is** the @@ -166,35 +166,60 @@ impl NodeKind { pub fn as_str(&self) -> &'static str { match self { NodeKind::Prebuild { .. } => "prebuild", - NodeKind::Swap => "swap", - NodeKind::PostSwap => "post_swap", - NodeKind::Provision => "provision", - NodeKind::Create => "create", + NodeKind::Swap { .. } => "swap", + NodeKind::PostSwap { .. } => "post_swap", + NodeKind::Provision { .. } => "provision", + NodeKind::Create { .. } => "create", NodeKind::MetaLock { .. } => "meta_lock", - NodeKind::Reconcile => "reconcile", - NodeKind::Start => "start", - NodeKind::Stop => "stop", - NodeKind::StopForUpdate => "stop_for_update", - NodeKind::Signal => "signal", - NodeKind::Drain => "drain", - NodeKind::WriteDropin => "write_dropin", - NodeKind::WritePermFile => "write_perm_file", - NodeKind::ApprovalDeploy => "approval_deploy", + NodeKind::Reconcile { .. } => "reconcile", + NodeKind::Start { .. } => "start", + NodeKind::Stop { .. } => "stop", + NodeKind::StopForUpdate { .. } => "stop_for_update", + NodeKind::Signal { .. } => "signal", + NodeKind::Drain { .. } => "drain", + NodeKind::WriteDropin { .. } => "write_dropin", + NodeKind::WritePermFile { .. } => "write_perm_file", + NodeKind::ApprovalDeploy { .. } => "approval_deploy", NodeKind::SetWanted { .. } => "set_wanted", NodeKind::Dag { .. } => "dag", } } + /// The agent this node targets, or `""` for agentless kinds + /// ([`NodeKind::MetaLock`] on the `hyperhive` pseudo-agent, and the + /// [`NodeKind::Dag`] container). + #[must_use] + pub fn agent(&self) -> &str { + match self { + NodeKind::Prebuild { agent, .. } + | NodeKind::Swap { agent } + | NodeKind::PostSwap { agent } + | NodeKind::Provision { agent } + | NodeKind::Create { agent } + | NodeKind::Reconcile { agent } + | NodeKind::Start { agent } + | NodeKind::Stop { agent } + | NodeKind::StopForUpdate { agent } + | NodeKind::Signal { agent } + | NodeKind::Drain { agent } + | NodeKind::WriteDropin { agent } + | NodeKind::WritePermFile { agent } + | NodeKind::ApprovalDeploy { agent } + | NodeKind::SetWanted { agent, .. } => agent, + NodeKind::MetaLock { .. } | NodeKind::Dag { .. } => "", + } + } + /// Nix-heavy kinds hold one of the `buildSlots` semaphore permits /// for the node's duration. pub fn needs_build_slot(&self) -> bool { matches!( self, NodeKind::Prebuild { .. } - | NodeKind::Swap - | NodeKind::Create + | NodeKind::Swap { .. } + | NodeKind::Create { .. } | NodeKind::MetaLock { .. } - | NodeKind::ApprovalDeploy + | NodeKind::ApprovalDeploy { .. } ) } @@ -209,14 +234,14 @@ impl NodeKind { pub fn needs_lease(&self) -> bool { matches!( self, - NodeKind::Swap - | NodeKind::Create - | NodeKind::Reconcile - | NodeKind::StopForUpdate - | NodeKind::Signal - | NodeKind::Drain - | NodeKind::WriteDropin - | NodeKind::ApprovalDeploy + NodeKind::Swap { .. } + | NodeKind::Create { .. } + | NodeKind::Reconcile { .. } + | NodeKind::StopForUpdate { .. } + | NodeKind::Signal { .. } + | NodeKind::Drain { .. } + | NodeKind::WriteDropin { .. } + | NodeKind::ApprovalDeploy { .. } | NodeKind::SetWanted { .. } ) } @@ -225,9 +250,9 @@ impl NodeKind { /// Submit-time spec for one node. #[derive(Debug, Clone)] pub struct NodeSpec { - /// The agent this node's work targets. Built by the `templates.rs` `node` - /// helper, which stamps the template's agent onto every node. - pub agent: String, + /// The node's payload — [`NodeKind`] is the queue's payload type directly, + /// and each variant carries the agent it targets (a DAG can span agents; + /// the queue derives per-agent leasing from [`NodeKind::agent`]). pub kind: NodeKind, pub deps: Vec, /// The **structural parent** axis — the spec-local index of this node's diff --git a/hive-c0re/src/job_queue/resource.rs b/hive-c0re/src/job_queue/resource.rs index 9358af5f..a59a428f 100644 --- a/hive-c0re/src/job_queue/resource.rs +++ b/hive-c0re/src/job_queue/resource.rs @@ -1,8 +1,8 @@ -//! The concrete resource + payload types the rebuild queue schedules over — -//! the bridge from hive-c0re's [`NodeKind`] onto the domain-agnostic -//! `hive-jobq` crate. `hive-jobq` is generic over a resource type -//! `R: Clone + Eq + Hash` and a node payload `N`; here `R` is [`Resource`] and -//! `N` is [`JobPayload`]. +//! The concrete resource type the rebuild queue schedules over — the bridge +//! from hive-c0re's [`NodeKind`] onto the domain-agnostic `hive-jobq` crate. +//! `hive-jobq` is generic over a resource type `R: Clone + Eq + Hash` and a node +//! payload `N`; here `R` is [`Resource`] and `N` is [`NodeKind`] directly (each +//! variant carries the agent it targets). use hive_jobq::Dep; @@ -24,16 +24,7 @@ pub enum Resource { Agent(String), } -/// A schedulable node's payload — the crate's generic `N`. Carries the -/// primitive operation and the agent it targets. The agent is per-node (a DAG -/// spans agents), and the lease [`Resource::Agent`] is keyed on it. -#[derive(Debug, Clone)] -pub struct JobPayload { - pub kind: NodeKind, - pub agent: String, -} - -impl JobPayload { +impl NodeKind { /// The [`Dep::Resource`] edges this node must acquire to run, derived from /// its kind + agent: a build slot for nix-heavy kinds /// ([`NodeKind::needs_build_slot`]) and the agent lease for @@ -43,15 +34,15 @@ impl JobPayload { /// `Agent` lock through the crate's recursive re-entrancy. pub fn resource_deps(&self) -> Vec> { let mut deps = Vec::new(); - if self.kind.needs_build_slot() { + if self.needs_build_slot() { deps.push(Dep::Resource { name: Resource::BuildSlot, count: 1, }); } - if self.kind.needs_lease() { + if self.needs_lease() { deps.push(Dep::Resource { - name: Resource::Agent(self.agent.clone()), + name: Resource::Agent(self.agent().to_owned()), count: 1, }); } diff --git a/hive-c0re/src/job_queue/submit.rs b/hive-c0re/src/job_queue/submit.rs index 1a741813..5c0ddf4b 100644 --- a/hive-c0re/src/job_queue/submit.rs +++ b/hive-c0re/src/job_queue/submit.rs @@ -63,13 +63,20 @@ fn stop_chain(agent: &str, graceful: bool, running: bool) -> Vec { // `SetWanted` is the group root and owns the agent lease; the mechanical // steps are its children (borrow the lease, run once it reaches `Finishing`, // dep-ordered among themselves). - let mut n = vec![node(agent, NodeKind::SetWanted { up: false }, Vec::new())]; + let a = || agent.to_owned(); + let mut n = vec![node( + NodeKind::SetWanted { + agent: a(), + up: false, + }, + Vec::new(), + )]; if graceful && running { - n.push(child(0, agent, NodeKind::Signal, Vec::new())); - n.push(child(0, agent, NodeKind::Drain, after_ok(1))); - n.push(child(0, agent, NodeKind::Reconcile, after_ok(2))); + n.push(child(0, NodeKind::Signal { agent: a() }, Vec::new())); + n.push(child(0, NodeKind::Drain { agent: a() }, after_ok(1))); + n.push(child(0, NodeKind::Reconcile { agent: a() }, after_ok(2))); } else { - n.push(child(0, agent, NodeKind::Reconcile, Vec::new())); + n.push(child(0, NodeKind::Reconcile { agent: a() }, Vec::new())); } n } @@ -79,14 +86,26 @@ fn stop_chain(agent: &str, graceful: bool, running: bool) -> Vec { /// current derivations), otherwise a plain `Reconcile` (which starts a down /// agent and noops an already-running one). fn start_chain(agent: &str, running: bool, stale: bool) -> Vec { - let mut n = vec![node(agent, NodeKind::SetWanted { up: true }, Vec::new())]; + let mut n = vec![node( + NodeKind::SetWanted { + agent: agent.to_owned(), + up: true, + }, + Vec::new(), + )]; if !running && stale { // Rebuild subtree after the SetWanted head (base = 1, so the rebuild's // `Prebuild` root deps `after_ok(0)` = the head). `Prebuild` + // `Reconcile` are their own group roots (top-level, per `rebuild_nodes`). n.extend(rebuild_nodes(agent, true, 1)); } else { - n.push(child(0, agent, NodeKind::Reconcile, Vec::new())); + n.push(child( + 0, + NodeKind::Reconcile { + agent: agent.to_owned(), + }, + Vec::new(), + )); } n } @@ -101,22 +120,27 @@ fn start_chain(agent: &str, running: bool, stale: bool) -> Vec { /// converges to intent — a stopped (`wanted = Off`) agent stays stopped, /// a crashed (`wanted = Up`) agent comes back up. fn restart_chain(agent: &str, graceful: bool, running: bool) -> Vec { + let a = || agent.to_owned(); if !running { // Nothing to bounce — a lone Reconcile converges to intent. - return vec![node(agent, NodeKind::Reconcile, Vec::new())]; + return vec![node(NodeKind::Reconcile { agent: a() }, Vec::new())]; } // Running: mechanical stop then Reconcile. The first stop node is the group // ROOT (no SetWanted head) and owns the agent lease; the rest are its // children (borrow the lease, dep-ordered), so the bounce holds one // continuous lease and `Reconcile` cancel-cascades if a stop step fails. let mut n = vec![if graceful { - node(agent, NodeKind::Signal, Vec::new()) + node(NodeKind::Signal { agent: a() }, Vec::new()) } else { - node(agent, NodeKind::StopForUpdate, Vec::new()) + node(NodeKind::StopForUpdate { agent: a() }, Vec::new()) }]; if graceful { - n.push(child(0, agent, NodeKind::Drain, Vec::new())); - n.push(child(0, agent, NodeKind::StopForUpdate, after_ok(1))); + n.push(child(0, NodeKind::Drain { agent: a() }, Vec::new())); + n.push(child( + 0, + NodeKind::StopForUpdate { agent: a() }, + after_ok(1), + )); } // `Reconcile` gates on the last mechanical step. When the only step is the // root itself (non-graceful, `StopForUpdate` == index 0), the parent gate @@ -128,7 +152,7 @@ fn restart_chain(agent: &str, graceful: bool, running: bool) -> Vec { } else { Vec::new() }; - n.push(child(0, agent, NodeKind::Reconcile, deps)); + n.push(child(0, NodeKind::Reconcile { agent: a() }, deps)); n } @@ -151,7 +175,6 @@ fn concat_subgraphs(chains: Vec>) -> Vec { }) .collect(); out.push(NodeSpec { - agent: spec.agent, kind: spec.kind, deps, // Rebase the structural parent by the same offset (a subgraph diff --git a/hive-c0re/src/job_queue/templates.rs b/hive-c0re/src/job_queue/templates.rs index 61852e1f..5c739962 100644 --- a/hive-c0re/src/job_queue/templates.rs +++ b/hive-c0re/src/job_queue/templates.rs @@ -42,14 +42,14 @@ pub(crate) fn after_ok(on: u64) -> Vec { }] } -/// Build one **top-level (group-root)** node targeting `agent` — `parent = -/// None`. The single place a node's agent is stamped. Shared with `submit.rs`'s -/// dynamic power-op builders. A root owns whatever resource it declares for its -/// whole subtree; its descendants borrow it (agent-lease / build-slot -/// continuity). Ordering vs other nodes is `deps`; grouping is `parent`. -pub(crate) fn node(agent: &str, kind: NodeKind, deps: Vec) -> NodeSpec { +/// Build one **top-level (group-root)** node — `parent = None`. `kind` carries +/// the agent it targets ([`NodeKind`] is the payload directly). Shared with +/// `submit.rs`'s dynamic power-op builders. A root owns whatever resource it +/// declares for its whole subtree; its descendants borrow it (agent-lease / +/// build-slot continuity). Ordering vs other nodes is `deps`; grouping is +/// `parent`. +pub(crate) fn node(kind: NodeKind, deps: Vec) -> NodeSpec { NodeSpec { - agent: agent.to_owned(), kind, deps, parent: None, @@ -60,9 +60,8 @@ pub(crate) fn node(agent: &str, kind: NodeKind, deps: Vec) -> NodeSpec { /// child runs once its parent reaches `Finishing` (the parent gate), so it must /// NOT `deps` on `parent` (dep-scope validation rejects a dep on one's own /// parent). `deps` here order the child against its *siblings* only. -pub(crate) fn child(parent: u64, agent: &str, kind: NodeKind, deps: Vec) -> NodeSpec { +pub(crate) fn child(parent: u64, kind: NodeKind, deps: Vec) -> NodeSpec { NodeSpec { - agent: agent.to_owned(), kind, deps, parent: Some(parent), @@ -87,22 +86,25 @@ pub(crate) fn child(parent: u64, agent: &str, kind: NodeKind, deps: Vec) -> /// (recovery-start invariant). It takes a fresh lease; the tiny gap is /// harmless — `Reconcile` converges to the persisted `wanted` idempotently. pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u64) -> Vec { + let a = || agent.to_owned(); vec![ node( - agent, - NodeKind::Prebuild { relock }, + NodeKind::Prebuild { agent: a(), relock }, if base == 0 { Vec::new() } else { after_ok(base - 1) }, ), - child(base, agent, NodeKind::StopForUpdate, Vec::new()), - child(base + 1, agent, NodeKind::Swap, Vec::new()), - child(base + 1, agent, NodeKind::PostSwap, after_ok(base + 2)), + child(base, NodeKind::StopForUpdate { agent: a() }, Vec::new()), + child(base + 1, NodeKind::Swap { agent: a() }, Vec::new()), + child( + base + 1, + NodeKind::PostSwap { agent: a() }, + after_ok(base + 2), + ), node( - agent, - NodeKind::Reconcile, + NodeKind::Reconcile { agent: a() }, vec![Dep { on: base, when: DepWhen::AfterAny, @@ -141,7 +143,12 @@ pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec inputs: Vec::new(), perm_payload: None, transient: Some(TransientKind::Rebuilding), - nodes: vec![node(agent, NodeKind::ApprovalDeploy, Vec::new())], + nodes: vec![node( + NodeKind::ApprovalDeploy { + agent: agent.to_owned(), + }, + Vec::new(), + )], } } @@ -166,7 +173,12 @@ pub fn reconcile_only( inputs: Vec::new(), perm_payload: None, transient, - nodes: vec![node(agent, NodeKind::Reconcile, Vec::new())], + nodes: vec![node( + NodeKind::Reconcile { + agent: agent.to_owned(), + }, + Vec::new(), + )], } } @@ -188,12 +200,15 @@ pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec { inputs: Vec::new(), perm_payload: None, transient: Some(TransientKind::Spawning), - nodes: vec![ - node(agent, NodeKind::Provision, Vec::new()), - child(0, agent, NodeKind::Create, Vec::new()), - child(1, agent, NodeKind::WriteDropin, Vec::new()), - child(1, agent, NodeKind::Reconcile, after_ok(2)), - ], + nodes: { + let a = || agent.to_owned(); + vec![ + node(NodeKind::Provision { agent: a() }, Vec::new()), + child(0, NodeKind::Create { agent: a() }, Vec::new()), + child(1, NodeKind::WriteDropin { agent: a() }, Vec::new()), + child(1, NodeKind::Reconcile { agent: a() }, after_ok(2)), + ] + }, } } @@ -201,7 +216,12 @@ pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec { /// the updated `HIVE_TOOL_GROUPS` / `HIVE_CAPABILITIES` env var takes /// effect in the container. pub fn perm_change(agent: &str, source: Source, reason: String, payload: PermPayload) -> DagSpec { - let mut nodes = vec![node(agent, NodeKind::WritePermFile, Vec::new())]; + let mut nodes = vec![node( + NodeKind::WritePermFile { + agent: agent.to_owned(), + }, + Vec::new(), + )]; nodes.extend(rebuild_nodes(agent, true, 1)); DagSpec { template: Template::PermChange, @@ -240,7 +260,6 @@ pub fn meta_update( perm_payload: None, transient: Some(TransientKind::Rebuilding), nodes: vec![node( - "hyperhive", NodeKind::MetaLock { sweep: false, fanout: None, diff --git a/hive-c0re/src/job_queue/tests.rs b/hive-c0re/src/job_queue/tests.rs index 0063b7fc..b70fc195 100644 --- a/hive-c0re/src/job_queue/tests.rs +++ b/hive-c0re/src/job_queue/tests.rs @@ -109,8 +109,9 @@ fn cyclic_dag_is_rejected_at_submit() { // 0 → 1 → 0 cycle. spec.nodes = vec![ NodeSpec { - agent: "agent-a".to_owned(), - kind: NodeKind::StopForUpdate, + kind: NodeKind::StopForUpdate { + agent: "agent-a".to_owned(), + }, deps: vec![Dep { on: 1, when: DepWhen::AfterOk, @@ -118,8 +119,9 @@ fn cyclic_dag_is_rejected_at_submit() { parent: None, }, NodeSpec { - agent: "agent-a".to_owned(), - kind: NodeKind::Reconcile, + kind: NodeKind::Reconcile { + agent: "agent-a".to_owned(), + }, deps: vec![Dep { on: 0, when: DepWhen::AfterOk, @@ -136,8 +138,9 @@ fn unknown_dep_is_rejected_at_submit() { let q = JobQueue::new(1); let mut spec = rebuild("agent-a", "bad dep"); spec.nodes = vec![NodeSpec { - agent: "agent-a".to_owned(), - kind: NodeKind::Reconcile, + kind: NodeKind::Reconcile { + agent: "agent-a".to_owned(), + }, deps: vec![Dep { on: 9, when: DepWhen::AfterOk, @@ -154,8 +157,9 @@ fn invalid_parent_is_rejected_at_submit() { // A forward/out-of-bounds parent index must be refused at validate, not // panic in `insert_group`. spec.nodes = vec![NodeSpec { - agent: "agent-a".to_owned(), - kind: NodeKind::Reconcile, + kind: NodeKind::Reconcile { + agent: "agent-a".to_owned(), + }, deps: Vec::new(), parent: Some(3), }]; @@ -560,7 +564,6 @@ fn append_subgraph_roots_on_emitter_and_rebases_local_deps() { perm_payload: None, transient: None, nodes: vec![NodeSpec { - agent: "hyperhive".to_owned(), kind: NodeKind::MetaLock { sweep: true, fanout: None, diff --git a/hive-c0re/src/workers/auto_update.rs b/hive-c0re/src/workers/auto_update.rs index c95ab682..2ed340e1 100644 --- a/hive-c0re/src/workers/auto_update.rs +++ b/hive-c0re/src/workers/auto_update.rs @@ -325,7 +325,6 @@ fn submit_boot_tree( // MetaLock into `run_meta_lock`, which appends the rebuild subgraphs. if any_stale { nodes.push(NodeSpec { - agent: "hyperhive".to_owned(), kind: NodeKind::MetaLock { sweep: true, fanout: Some(fanout), @@ -337,8 +336,7 @@ fn submit_boot_tree( // One boot Reconcile per drifted agent — independent roots. for name in drifted { nodes.push(NodeSpec { - agent: name, - kind: NodeKind::Reconcile, + kind: NodeKind::Reconcile { agent: name }, deps: Vec::new(), parent: None, }); From 600bc051e193c84dbfa4170a7d5c0079a5abe3e2 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 20 Jul 2026 23:21:46 +0200 Subject: [PATCH 8/9] refactor(#2591): move the perm-change payload onto the WritePermFile node --- hive-c0re/src/job_queue/exec.rs | 16 ++++++++-------- hive-c0re/src/job_queue/mod.rs | 7 ------- hive-c0re/src/job_queue/model.rs | 16 ++++++++-------- hive-c0re/src/job_queue/submit.rs | 1 - hive-c0re/src/job_queue/templates.rs | 7 +------ hive-c0re/src/job_queue/tests.rs | 1 - hive-c0re/src/workers/auto_update.rs | 1 - hive-sh4re/src/jobs.rs | 2 -- hivectl/src/dag_progress.rs | 2 -- 9 files changed, 17 insertions(+), 36 deletions(-) diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index 86ab6994..3a232a3b 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -540,26 +540,30 @@ async fn run_write_perm_file( ) -> Result { use super::model::PermPayload; let name = &claim.agent; + // The perm file payload rides the node itself (the only consumer). + let NodeKind::WritePermFile { payload, .. } = &claim.kind else { + anyhow::bail!("run_write_perm_file on a non-WritePermFile node"); + }; ctx.step("writing + committing perm file"); // Deploy-window gate: a perm commit landing inside another node's // staged prepare→finalize window would sweep the staged deploy // lock into its commit (the commits are also path-limited in // meta.rs — belt and braces). let _window = crate::meta::exclusive().await; - match &claim.perm_payload { - Some(PermPayload::ToolGroups { groups }) => { + match payload { + PermPayload::ToolGroups { groups } => { crate::meta::commit_tool_groups(name, groups) .await .with_context(|| format!("commit tool-groups for {name}"))?; coord.emit_tool_groups_snapshot(); } - Some(PermPayload::Capabilities { caps }) => { + PermPayload::Capabilities { caps } => { crate::meta::commit_capabilities(name, caps) .await .with_context(|| format!("commit capabilities for {name}"))?; coord.emit_capabilities_snapshot(); } - Some(PermPayload::Combined { groups, caps }) => { + PermPayload::Combined { groups, caps } => { crate::meta::commit_perms(name, groups.as_deref(), caps.as_deref()) .await .with_context(|| format!("commit perms for {name}"))?; @@ -570,10 +574,6 @@ async fn run_write_perm_file( coord.emit_capabilities_snapshot(); } } - None => anyhow::bail!( - "perm_change dag {} for {name} is missing perm_payload", - claim.dag_id - ), } Ok(NodeOutput::default()) } diff --git a/hive-c0re/src/job_queue/mod.rs b/hive-c0re/src/job_queue/mod.rs index bf6158be..1b9fb040 100644 --- a/hive-c0re/src/job_queue/mod.rs +++ b/hive-c0re/src/job_queue/mod.rs @@ -76,7 +76,6 @@ pub struct Claim { pub template: Template, pub approval_id: Option, pub inputs: Vec, - pub perm_payload: Option, /// Transient pill kind for the lease window (from the spec). Whether the /// pill is currently shown is derived from live lease ownership /// ([`JobQueue::held_transients`]), not a per-claim edge. @@ -118,7 +117,6 @@ struct DagMeta { transient: Option, approval_id: Option, inputs: Vec, - perm_payload: Option, created_at: i64, } @@ -297,7 +295,6 @@ impl JobQueue { transient: spec.transient, approval_id: spec.approval_id, inputs: spec.inputs, - perm_payload: spec.perm_payload, created_at: now_unix(), }, Vec::new(), @@ -387,7 +384,6 @@ impl JobQueue { template: meta.template, approval_id: meta.approval_id, inputs: meta.inputs, - perm_payload: meta.perm_payload, transient: meta.transient, }); if let Some(rt) = inner.node_rt.get_mut(&id) { @@ -638,7 +634,6 @@ impl QueueInner { transient, approval_id, inputs, - perm_payload, created_at, } = &self.sched.graph().node(container)?.payload else { @@ -651,7 +646,6 @@ impl QueueInner { transient: *transient, approval_id: *approval_id, inputs: inputs.clone(), - perm_payload: perm_payload.clone(), created_at: *created_at, }) } @@ -801,7 +795,6 @@ impl QueueInner { }, inputs: meta.inputs.clone(), approval_id: meta.approval_id, - perm_payload: meta.perm_payload.clone(), nodes, }) } diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs index 9c4da05c..bfc66aa0 100644 --- a/hive-c0re/src/job_queue/model.rs +++ b/hive-c0re/src/job_queue/model.rs @@ -119,9 +119,10 @@ pub enum NodeKind { Drain { agent: String }, /// `set_nspawn_flags` + `set_resource_limits` + daemon-reload. WriteDropin { agent: String }, - /// Commit `tool-groups.json` / `capabilities.json` per the DAG's - /// `perm_payload` (commit fused under `META_LOCK`). - WritePermFile { agent: String }, + /// Commit `tool-groups.json` / `capabilities.json` per its `payload` + /// (commit fused under `META_LOCK`). The payload rides this node — the only + /// consumer — rather than the generic DAG container. + WritePermFile { agent: String, payload: PermPayload }, /// Opaque approval deploy pipeline (`MergeConfigPr`): the two-phase /// prepare/finalize/abort meta deploy stays inside `actions.rs` in v1 — /// deliberately not @@ -156,7 +157,6 @@ pub enum NodeKind { transient: Option, approval_id: Option, inputs: Vec, - perm_payload: Option, created_at: i64, }, } @@ -203,7 +203,7 @@ impl NodeKind { | NodeKind::Signal { agent } | NodeKind::Drain { agent } | NodeKind::WriteDropin { agent } - | NodeKind::WritePermFile { agent } + | NodeKind::WritePermFile { agent, .. } | NodeKind::ApprovalDeploy { agent } | NodeKind::SetWanted { agent, .. } => agent, NodeKind::MetaLock { .. } | NodeKind::Dag { .. } => "", @@ -269,7 +269,9 @@ pub struct NodeSpec { /// Submit-time spec for a whole DAG. Built by `templates.rs`; validated /// (cycle rejection) by `JobQueue::submit`. No DAG-level `agent` — every /// node carries its own (a DAG can span agents), and the queue derives -/// per-agent leasing from [`NodeSpec::agent`]. +/// per-agent leasing from [`NodeKind::agent`]. Type-specific payloads +/// (`PermChange`'s file payload) ride the node that consumes them +/// ([`NodeKind::WritePermFile`]), not this generic spec. #[derive(Debug, Clone)] pub struct DagSpec { pub template: Template, @@ -281,8 +283,6 @@ pub struct DagSpec { /// `MetaUpdate`-only: the inputs to bump (also part of the dedup /// key for that template). Display copy lives on the DAG. pub inputs: Vec, - /// `PermChange`-only payload. - pub perm_payload: Option, /// Dashboard transient pill (and crash-watch suppression) held for /// the lease window — from lease acquisition to DAG terminal. pub transient: Option, diff --git a/hive-c0re/src/job_queue/submit.rs b/hive-c0re/src/job_queue/submit.rs index 5c0ddf4b..b4d85415 100644 --- a/hive-c0re/src/job_queue/submit.rs +++ b/hive-c0re/src/job_queue/submit.rs @@ -201,7 +201,6 @@ fn power_dag( reason, approval_id: None, inputs: Vec::new(), - perm_payload: None, transient: Some(transient), nodes, } diff --git a/hive-c0re/src/job_queue/templates.rs b/hive-c0re/src/job_queue/templates.rs index 5c739962..f456da8e 100644 --- a/hive-c0re/src/job_queue/templates.rs +++ b/hive-c0re/src/job_queue/templates.rs @@ -125,7 +125,6 @@ pub fn rebuild(agent: &str, source: Source, reason: String, relock: bool) -> Dag reason, approval_id: None, inputs: Vec::new(), - perm_payload: None, transient: Some(TransientKind::Rebuilding), nodes: rebuild_nodes(agent, relock, 0), } @@ -141,7 +140,6 @@ pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec reason, approval_id: Some(approval_id), inputs: Vec::new(), - perm_payload: None, transient: Some(TransientKind::Rebuilding), nodes: vec![node( NodeKind::ApprovalDeploy { @@ -171,7 +169,6 @@ pub fn reconcile_only( reason, approval_id: None, inputs: Vec::new(), - perm_payload: None, transient, nodes: vec![node( NodeKind::Reconcile { @@ -198,7 +195,6 @@ pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec { reason, approval_id: Some(approval_id), inputs: Vec::new(), - perm_payload: None, transient: Some(TransientKind::Spawning), nodes: { let a = || agent.to_owned(); @@ -219,6 +215,7 @@ pub fn perm_change(agent: &str, source: Source, reason: String, payload: PermPay let mut nodes = vec![node( NodeKind::WritePermFile { agent: agent.to_owned(), + payload, }, Vec::new(), )]; @@ -229,7 +226,6 @@ pub fn perm_change(agent: &str, source: Source, reason: String, payload: PermPay reason, approval_id: None, inputs: Vec::new(), - perm_payload: Some(payload), transient: Some(TransientKind::Rebuilding), nodes, } @@ -257,7 +253,6 @@ pub fn meta_update( reason, approval_id, inputs, - perm_payload: None, transient: Some(TransientKind::Rebuilding), nodes: vec![node( NodeKind::MetaLock { diff --git a/hive-c0re/src/job_queue/tests.rs b/hive-c0re/src/job_queue/tests.rs index b70fc195..b385e420 100644 --- a/hive-c0re/src/job_queue/tests.rs +++ b/hive-c0re/src/job_queue/tests.rs @@ -561,7 +561,6 @@ fn append_subgraph_roots_on_emitter_and_rebases_local_deps() { reason: "sweep".to_owned(), approval_id: None, inputs: Vec::new(), - perm_payload: None, transient: None, nodes: vec![NodeSpec { kind: NodeKind::MetaLock { diff --git a/hive-c0re/src/workers/auto_update.rs b/hive-c0re/src/workers/auto_update.rs index 2ed340e1..8db07795 100644 --- a/hive-c0re/src/workers/auto_update.rs +++ b/hive-c0re/src/workers/auto_update.rs @@ -348,7 +348,6 @@ fn submit_boot_tree( reason, approval_id: None, inputs: Vec::new(), - perm_payload: None, // Rebuilding when the sweep will grow rebuild subgraphs (per-agent // crash-watch suppression during their Swap, applied at claim time); // a reconcile-only boot needs no transient. diff --git a/hive-sh4re/src/jobs.rs b/hive-sh4re/src/jobs.rs index cb5ef9b3..a429b72b 100644 --- a/hive-sh4re/src/jobs.rs +++ b/hive-sh4re/src/jobs.rs @@ -195,7 +195,5 @@ pub struct DagView { pub inputs: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub approval_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub perm_payload: Option, pub nodes: Vec, } diff --git a/hivectl/src/dag_progress.rs b/hivectl/src/dag_progress.rs index 74c77d2d..c726773f 100644 --- a/hivectl/src/dag_progress.rs +++ b/hivectl/src/dag_progress.rs @@ -353,7 +353,6 @@ mod tests { finished_at: None, inputs: vec![], approval_id: None, - perm_payload: None, nodes: vec![ node(0, "alice", "prebuild", State::Done, None), node(1, "alice", "stop_for_update", State::Done, None), @@ -392,7 +391,6 @@ mod tests { finished_at: Some(2), inputs: vec![], approval_id: None, - perm_payload: None, nodes: vec![failed], }; let line = render_dag_line(&dag); From 0ab6b764be417942282c9ddcf4eb2b90caac3b9b Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 20 Jul 2026 23:27:33 +0200 Subject: [PATCH 9/9] docs(#2591): fix stale Claim.agent doc + explain cancel's container roll-up (argus review) --- hive-c0re/src/job_queue/mod.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/hive-c0re/src/job_queue/mod.rs b/hive-c0re/src/job_queue/mod.rs index 1b9fb040..e2068cb7 100644 --- a/hive-c0re/src/job_queue/mod.rs +++ b/hive-c0re/src/job_queue/mod.rs @@ -71,7 +71,7 @@ pub struct Claim { pub node_id: NodeId, pub kind: NodeKind, /// The agent this node targets (its own, not a DAG-level field). Empty for - /// the internal [`NodeKind::Finalize`] node. + /// the agentless [`NodeKind::MetaLock`] + [`NodeKind::Dag`] container nodes. pub agent: String, pub template: Template, pub approval_id: Option, @@ -452,8 +452,12 @@ impl JobQueue { for id in work { inner.sched.cancel_node(id); } - // Roll the container up so the DAG reaches a terminal state (all children - // now `Cancelled`); `dag_rollup` reports `Cancelled` to the wire. + // The container was settled to `Finishing` at submit; completing it again + // now re-runs the roll-up with its children all `Cancelled`, driving it to + // a terminal state synchronously within this lock — so the caller reads + // the terminal summary immediately instead of waiting for the scheduler + // loop to observe the cancellation. `dag_rollup` reports `Cancelled` to + // the wire (a container whose children all cancelled). inner.sched.complete(container, Outcome::Done); let terminal = inner.terminal_dag(container); drop(inner);