jobq: one insertion entry point, and make it atomic

Three findings from the operator's review, all correct.

1. Two insert_job's. Graph::insert_job had no caller outside hive-jobq's
   own tests -- production only ever went through Scheduler::insert_job.
   It existed because the graph-level one got written first. Deleted; the
   tests moved onto a Scheduler, which is where insertion belongs anyway.

2. insert_job was not atomic, and the previous commit made that worse: a
   forward edge or forward parent surfaced mid-loop, leaving the nodes
   before it in the graph, and resolve_wanted ran after every insert, so
   an unknown handle failed once the whole job was already committed.
   The module documented this under "Partial insertion" instead of fixing
   it -- prose describing a hole is not a design.

   All three are decidable from what the builder holds, so
   check_declaration_order now runs before the first insert and the loop
   indexes ids directly. A malformed job leaves the graph untouched.
   What remains mid-insert is the graph's own rejection (out-of-group
   dep, empty DepWhen); closing that needs a dry-run validate on Graph,
   which is a separate change.

3. DagSpec no longer boxes its recipe: it is generic over the closure,
   which travels from the template that built it straight into submit.
   The box bought type inference, and paying for it costs annotations --
   `|b: &Job|` at each declaration site (the field needs an HRTB, and an
   unannotated closure binds one lifetime) and `+ use<>` on each
   returning signature (or the opaque type captures the caller's borrows).
   Erasure is still needed where several recipe shapes share one type:
   the boxed Declare stays for the executor's append_subgraph, and a test
   table uses an erase() helper.

hive-c0re/src/job_queue/mod.rs

@@ -179,7 +179,7 @@ impl Default for JobQueue {
 /// Propagates a crate graph-insert error (malformed dep/parent / dep-scope).
 fn insert_group(
     inner: &mut QueueInner,
-    declare: Declare,
+    declare: impl FnOnce(&Job),
     group_parent: Option<NodeId>,
 ) -> anyhow::Result<()> {
     inner
@@ -216,15 +216,19 @@ impl JobQueue {
         self.inner.lock().expect("job_queue mutex poisoned")
     }
 
-    /// Submit a DAG. Validates the spec, inserts a [`NodeKind::Dag`] **container
-    /// node** carrying the group's metadata, then inserts the template's nodes as
-    /// its subtree (their roots re-parented to the container). Returns the
-    /// container's id as the DAG id — its rolled-up state is the DAG state.
+    /// Submit a DAG: insert a [`NodeKind::Dag`] **container node** carrying the
+    /// group's metadata, then insert the template's nodes as its subtree (their
+    /// roots re-parented to the container). Returns the container's id as the
+    /// DAG id — its rolled-up state is the DAG state.
+    ///
+    /// Takes the spec's recipe by generic, not as a boxed [`Declare`]: a spec
+    /// travels from the template that built it directly into this call, so
+    /// there is nothing to allocate for.
     ///
     /// # Errors
-    /// Propagates the spec-validation error (empty / cyclic / bad parent) or a
-    /// graph-insert error (dependencies that aren't dependency-topological).
-    pub fn submit(&self, spec: DagSpec) -> anyhow::Result<u64> {
+    /// Propagates a graph-insert error (dependencies that aren't
+    /// dependency-topological).
+    pub fn submit<F: FnOnce(&Job)>(&self, spec: DagSpec<F>) -> anyhow::Result<u64> {
         let mut inner = self.lock();
         let container = inner
             .sched

hive-c0re/src/job_queue/model.rs

@@ -475,16 +475,23 @@ impl NodeKind {
 /// moment it inserts. A shape that has been declared is therefore always
 /// insertable — a dangling edge or a cycle cannot be expressed, so there is
 /// nothing left for a submit-time validation pass to reject.
-pub struct DagSpec {
+///
+/// Generic over the recipe rather than boxing it: a spec goes from the template
+/// that returns it straight to the `submit` that consumes it, so the closure's
+/// concrete type is known the whole way and needs neither an allocation nor a
+/// `Send` bound. (The executor's `append_subgraph` is the case that *does* need
+/// a boxed [`super::Declare`] — its recipes are collected into a `Vec` and
+/// applied later, across a task boundary.)
+pub struct DagSpec<F> {
     pub source: Source,
     /// Free-form "why".
     pub reason: String,
     /// Declares the DAG's nodes — their edges, grouping and resources — onto
     /// the builder the queue hands it.
-    pub declare: super::Declare,
+    pub declare: F,
 }
 
-impl std::fmt::Debug for DagSpec {
+impl<F> std::fmt::Debug for DagSpec<F> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         // The recipe is a closure; there is nothing to show of it, and its
         // nodes do not exist until the queue runs it.

hive-c0re/src/job_queue/submit.rs

@@ -26,11 +26,11 @@ use std::sync::Arc;
 
 use super::model::{DagSpec, NodeKind};
 use super::templates::{RebuildOpts, node, rebuild_nodes};
-use super::{Declare, Job, Source, templates};
+use super::{Job, Source, templates};
 use crate::coordinator::Coordinator;
 use crate::lifecycle;
 
-fn submit_and_emit(coord: &Arc<Coordinator>, spec: super::DagSpec) -> u64 {
+fn submit_and_emit<F: FnOnce(&Job)>(coord: &Arc<Coordinator>, spec: super::DagSpec<F>) -> u64 {
     let id = coord
         .job_queue
         .submit(spec)
@@ -169,7 +169,7 @@ fn restart_chain(b: &Job, agent: &str, graceful: bool, running: bool) {
 /// and each keeps its own root, so the per-agent subgraphs are independent and
 /// run concurrently, each on its own lease. Rebasing one subgraph's indices
 /// onto another's used to be a function.
-fn power_dag(source: Source, reason: String, declare: Declare) -> DagSpec {
+fn power_dag<F: FnOnce(&Job)>(source: Source, reason: String, declare: F) -> DagSpec<F> {
     DagSpec {
         source,
         reason,
@@ -188,12 +188,12 @@ pub(crate) fn stop_spec(
     graceful: bool,
     source: Source,
     reason: String,
-) -> DagSpec {
+) -> DagSpec<impl FnOnce(&Job) + use<>> {
     let targets = targets.to_vec();
     power_dag(
         source,
         reason,
-        Box::new(move |b| {
+        Box::new(move |b: &Job| {
             for (agent, running) in targets {
                 stop_chain(b, &agent, graceful, running);
             }
@@ -211,12 +211,12 @@ pub(crate) fn start_spec(
     targets: &[(String, bool, bool)],
     source: Source,
     reason: String,
-) -> DagSpec {
+) -> DagSpec<impl FnOnce(&Job) + use<>> {
     let targets = targets.to_vec();
     power_dag(
         source,
         reason,
-        Box::new(move |b| {
+        Box::new(move |b: &Job| {
             for (agent, running, stale) in targets {
                 start_chain(b, &agent, running, stale);
             }
@@ -230,12 +230,12 @@ pub(crate) fn restart_spec(
     graceful: bool,
     source: Source,
     reason: String,
-) -> DagSpec {
+) -> DagSpec<impl FnOnce(&Job) + use<>> {
     let targets = targets.to_vec();
     power_dag(
         source,
         reason,
-        Box::new(move |b| {
+        Box::new(move |b: &Job| {
             for (agent, running) in targets {
                 restart_chain(b, &agent, graceful, running);
             }

hive-c0re/src/job_queue/templates.rs

@@ -241,7 +241,7 @@ pub(crate) fn rebuild_nodes<'a>(
 /// already holding it rather than deadlocking against it.
 pub(crate) fn deploy_rebuild_nodes(agent: &str, approval_id: i64) -> Declare {
     let agent = agent.to_owned();
-    Box::new(move |b| {
+    Box::new(move |b: &Job| {
         let roots = rebuild_nodes(
             b,
             &agent,
@@ -274,12 +274,17 @@ pub(crate) fn deploy_rebuild_nodes(agent: &str, approval_id: i64) -> Declare {
 /// whole `StopForUpdate`→`Swap`→`PostSwap` subtree, so those three cover every
 /// node. Edging `Reconcile` alone would not do: it is `AfterAny` `Prebuild`, so
 /// it reaches `Done` even after a failed swap and the tail would report success.
-pub fn rebuild(agent: &str, source: Source, reason: String, relock: bool) -> DagSpec {
+pub fn rebuild(
+    agent: &str,
+    source: Source,
+    reason: String,
+    relock: bool,
+) -> DagSpec<impl FnOnce(&Job) + use<>> {
     let agent = agent.to_owned();
     DagSpec {
         source,
         reason,
-        declare: Box::new(move |b| {
+        declare: Box::new(move |b: &Job| {
             let roots = rebuild_nodes(
                 b,
                 &agent,
@@ -319,12 +324,16 @@ pub fn rebuild(agent: &str, source: Source, reason: String, relock: bool) -> Dag
 ///
 /// The window still spans the container build, as it must: `prepare_deploy`
 /// leaves `flake.lock` staged-uncommitted for the build's whole duration.
-pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec {
+pub fn approval_deploy(
+    agent: &str,
+    approval_id: i64,
+    reason: String,
+) -> DagSpec<impl FnOnce(&Job) + use<>> {
     let agent = agent.to_owned();
     DagSpec {
         source: Source::Approval,
         reason,
-        declare: Box::new(move |b| {
+        declare: Box::new(move |b: &Job| {
             let a = || agent.clone();
             let window = node(
                 b,
@@ -371,12 +380,16 @@ pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec
 /// single-node lifecycle DAGs that exercise per-agent lease serialization
 /// in the queue tests); production paths no longer emit a bare reconcile.
 #[cfg(test)]
-pub fn reconcile_only(agent: &str, source: Source, reason: String) -> DagSpec {
+pub fn reconcile_only(
+    agent: &str,
+    source: Source,
+    reason: String,
+) -> DagSpec<impl FnOnce(&Job) + use<>> {
     let agent = agent.to_owned();
     DagSpec {
         source,
         reason,
-        declare: Box::new(move |b| {
+        declare: Box::new(move |b: &Job| {
             let _reconcile = node(b, NodeKind::Reconcile { agent });
         }),
     }
@@ -393,12 +406,12 @@ pub fn reconcile_only(agent: &str, source: Source, reason: String) -> DagSpec {
 /// container was never created). Closed by a `ResolveApproval` tail root edged
 /// `AfterAny` onto `Provision` — the DAG's only other group-root, so its roll-up
 /// already carries the whole cascade.
-pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
+pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec<impl FnOnce(&Job) + use<>> {
     let agent = agent.to_owned();
     DagSpec {
         source: Source::Approval,
         reason,
-        declare: Box::new(move |b| {
+        declare: Box::new(move |b: &Job| {
             let a = || agent.clone();
             let provision = node(b, NodeKind::Provision { agent: a() });
             let create = node(b, NodeKind::Create { agent: a() }).part_of(provision);
@@ -417,12 +430,17 @@ pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
 /// effect in the container. Group-roots are `WritePermFile` plus the rebuild
 /// subgraph's `MetaSync` / `Prebuild` / `Reconcile`, so the `EmitRebuilt` tail
 /// edges all four.
-pub fn perm_change(agent: &str, source: Source, reason: String, payload: PermPayload) -> DagSpec {
+pub fn perm_change(
+    agent: &str,
+    source: Source,
+    reason: String,
+    payload: PermPayload,
+) -> DagSpec<impl FnOnce(&Job) + use<>> {
     let agent = agent.to_owned();
     DagSpec {
         source,
         reason,
-        declare: Box::new(move |b| {
+        declare: Box::new(move |b: &Job| {
             let write = node(
                 b,
                 NodeKind::WritePermFile {
@@ -463,11 +481,11 @@ pub fn meta_update(
     source: Source,
     reason: String,
     approval_id: Option<i64>,
-) -> DagSpec {
+) -> DagSpec<impl FnOnce(&Job) + use<>> {
     DagSpec {
         source,
         reason,
-        declare: Box::new(move |b| {
+        declare: Box::new(move |b: &Job| {
             let lock = node(
                 b,
                 NodeKind::MetaLock {
@@ -501,11 +519,11 @@ pub fn reparent(
     moves: Vec<(hive_types::Ident, Option<hive_types::Ident>)>,
     source: Source,
     reason: String,
-) -> DagSpec {
+) -> DagSpec<impl FnOnce(&Job) + use<>> {
     DagSpec {
         source,
         reason,
-        declare: Box::new(move |b| {
+        declare: Box::new(move |b: &Job| {
             let _reparent = node(b, NodeKind::Reparent { moves });
         }),
     }

hive-c0re/src/job_queue/tests.rs

@@ -9,15 +9,29 @@
 use super::model::NodeKind;
 use super::*;
 
-fn submit(q: &JobQueue, spec: DagSpec) -> u64 {
+fn submit<F: FnOnce(&Job)>(q: &JobQueue, spec: DagSpec<F>) -> u64 {
     q.submit(spec).expect("valid spec")
 }
 
+/// Erase a spec's recipe to the boxed [`Declare`] so specs of *different*
+/// shapes can share one type — e.g. a table of `(name, spec)` cases.
+///
+/// Production never needs this: each submit path builds one spec and hands it
+/// straight to `submit`, so the concrete closure type is known end to end. A
+/// test table is the case where several shapes must be one type.
+fn erase<F: FnOnce(&Job) + Send + 'static>(spec: DagSpec<F>) -> DagSpec<Declare> {
+    DagSpec {
+        source: spec.source,
+        reason: spec.reason,
+        declare: Box::new(spec.declare),
+    }
+}
+
 fn ident(s: &str) -> hive_types::Ident {
     hive_types::Ident::parse(s).expect("valid test ident")
 }
 
-fn rebuild(agent: &str, reason: &str) -> DagSpec {
+fn rebuild(agent: &str, reason: &str) -> DagSpec<impl FnOnce(&Job) + use<>> {
     templates::rebuild(agent, Source::Manual, reason.to_owned(), true)
 }
 
@@ -25,14 +39,22 @@ fn rebuild(agent: &str, reason: &str) -> DagSpec {
 /// shape (`[Signal→Drain→] StopForUpdate → Reconcile`, no `SetWanted` head)
 /// most queue-mechanics tests assume. Mirrors the pre-dynamic
 /// `templates::restart` (which is now the state-aware `submit::restart_spec`).
-fn restart_online(agents: &[&str], graceful: bool, reason: &str) -> DagSpec {
+fn restart_online(
+    agents: &[&str],
+    graceful: bool,
+    reason: &str,
+) -> DagSpec<impl FnOnce(&Job) + use<>> {
     let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
     submit::restart_spec(&targets, graceful, Source::Manual, reason.to_owned())
 }
 
 /// Stop DAG spec with every agent treated as **running** — the online shape
 /// (`SetWanted → [Signal→Drain→](graceful) Reconcile`).
-fn stop_online(agents: &[&str], graceful: bool, reason: &str) -> DagSpec {
+fn stop_online(
+    agents: &[&str],
+    graceful: bool,
+    reason: &str,
+) -> DagSpec<impl FnOnce(&Job) + use<>> {
     let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
     submit::stop_spec(&targets, graceful, Source::Manual, reason.to_owned())
 }
@@ -197,7 +219,7 @@ fn graceful_rebuild_chain_drains_before_stopping() {
         DagSpec {
             source: Source::AutoUpdate,
             reason: "sweep".to_owned(),
-            declare: Box::new(|b| {
+            declare: Box::new(|b: &Job| {
                 templates::rebuild_nodes(
                     b,
                     "agent-a",
@@ -246,7 +268,7 @@ fn non_graceful_rebuild_has_no_signal_or_drain() {
         DagSpec {
             source: Source::Manual,
             reason: "manual".to_owned(),
-            declare: Box::new(|b| {
+            declare: Box::new(|b: &Job| {
                 templates::rebuild_nodes(
                     b,
                     "agent-a",
@@ -695,7 +717,7 @@ fn append_subgraph_roots_on_emitter_and_rebases_local_deps() {
     let spec = DagSpec {
         source: Source::AutoUpdate,
         reason: "sweep".to_owned(),
-        declare: Box::new(|b| {
+        declare: Box::new(|b: &Job| {
             let _lock = templates::node(
                 b,
                 NodeKind::MetaLock {
@@ -715,7 +737,7 @@ fn append_subgraph_roots_on_emitter_and_rebases_local_deps() {
     // match the sweep arm of `run_meta_lock` or this stops tracking production.
     let subgraph = |agent: &str| -> Declare {
         let agent = agent.to_owned();
-        Box::new(move |b| {
+        Box::new(move |b: &Job| {
             templates::rebuild_nodes(
                 b,
                 &agent,
@@ -821,7 +843,7 @@ fn meta_update_grows_cascade_in_dag() {
     // Simulate the executor growing the cascade in-DAG (`relock = false` — a
     // cascade child must not re-lock and revert the parent's bump).
     for agent in ["alice", "bob"] {
-        let declare: Declare = Box::new(move |b| {
+        let declare: Declare = Box::new(move |b: &Job| {
             templates::rebuild_nodes(
                 b,
                 agent,
@@ -1073,25 +1095,37 @@ fn cancelled_power_op_runs_no_compensating_node() {
     for graceful in [false, true] {
         for running in [false, true] {
             let targets = vec![("agent-a".to_owned(), running)];
+            // Erased to `DagSpec<Declare>`: three different recipe types have to
+            // sit in one array.
             let cases = [
                 (
                     "restart",
                     false,
-                    submit::restart_spec(&targets, graceful, Source::Manual, "bounce".to_owned()),
+                    erase(submit::restart_spec(
+                        &targets,
+                        graceful,
+                        Source::Manual,
+                        "bounce".to_owned(),
+                    )),
                 ),
                 (
                     "stop",
                     true,
-                    submit::stop_spec(&targets, graceful, Source::Manual, "stop".to_owned()),
+                    erase(submit::stop_spec(
+                        &targets,
+                        graceful,
+                        Source::Manual,
+                        "stop".to_owned(),
+                    )),
                 ),
                 (
                     "start",
                     true,
-                    submit::start_spec(
+                    erase(submit::start_spec(
                         &[("agent-a".to_owned(), running, false)],
                         Source::Manual,
                         "start".to_owned(),
-                    ),
+                    )),
                 ),
             ];
             for (name, writes_intent, spec) in cases {