feat(hive-c0re): replace rebuild queue with generic job-DAG queue

jobs are now DAGs of primitive nodes (prebuild, stop-for-update, swap,
reconcile, signal, drain, ...) driven by one scheduler with N build
slots + per-agent lifecycle leases. per-agent power intent (wanted
up/offline) is durable in agent_power.sqlite; Reconcile nodes converge
observed state to it. kills the graceful-stop watcher thread, the
deferred-start follow-up, and the cascade pre-enqueue (fan-out on
MetaLock completion instead). tracker: #2166

Cargo.lock

@@ -1029,6 +1029,12 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
 
+[[package]]
+name = "fixedbitset"
+version = "0.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
+
 [[package]]
 name = "flate2"
 version = "1.1.9"
@@ -1381,6 +1387,7 @@ dependencies = [
  "hive-sh4re",
  "libc",
  "listenfd",
+ "petgraph",
  "problem_details",
  "reqwest",
  "rusqlite",
@@ -2554,6 +2561,17 @@ version = "2.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
 
+[[package]]
+name = "petgraph"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455"
+dependencies = [
+ "fixedbitset",
+ "hashbrown 0.15.5",
+ "indexmap",
+]
+
 [[package]]
 name = "phf"
 version = "0.11.3"

Cargo.toml

@@ -68,6 +68,7 @@ reqwest = { version = "0.12", default-features = false, features = [
   "json",
   "rustls-tls",
 ] }
+petgraph = { version = "0.8", default-features = false, features = ["std"] }
 matrix-sdk = { version = "0.14", default-features = false, features = [
   "rustls-tls",
   "sqlite",

hive-c0re/Cargo.toml

@@ -18,6 +18,7 @@ clap-markdown = "0.1"
 hive-sh4re.workspace = true
 libc.workspace = true
 listenfd = "1"
+petgraph.workspace = true
 rusqlite.workspace = true
 serde.workspace = true
 serde_json.workspace = true

hive-c0re/src/actions.rs

@@ -13,19 +13,20 @@ use crate::lifecycle;
 
 /// Approve a pending request. Marks the approval row durably, then
 /// either runs the work inline (`InitConfig`, sub-second git ops) or
-/// enqueues it into `rebuild_queue` so the dashboard POST returns
+/// submits it to the job queue so the dashboard POST returns
 /// immediately while the long-running pipeline runs off-thread
 /// (operator no longer blocks on a 30-90s spinner for `ApplyCommit`).
 ///
 /// Dispatch:
-/// - `ApplyCommit`  → `QueueKind::Rebuild`     (~30-90s wall time)
+/// - `ApplyCommit` / `MergeConfigPr` → a single-node `ApprovalDeploy`
-/// - `UpdateMetaInputs` → `QueueKind::MetaUpdate` (~3-15s)
+///   DAG (the two-phase meta deploy stays opaque in v1; ~30-90s)
-/// - `Spawn`       → `QueueKind::Spawn`        (~30-90s)
+/// - `UpdateMetaInputs` → a `MetaUpdate` DAG (fan-out on completion)
+/// - `Spawn`       → a `Spawn` DAG (`Create → WriteDropin → Reconcile`)
 /// - `InitConfig`  → inline (<1s; queue card would be noise)
 ///
-/// The queue worker re-fetches the approval row on dispatch, runs
+/// `ApprovalDeploy` resolves the approval inside its pipeline; the
-/// the kind-specific pipeline, and fires `ApprovalResolved` /
+/// `MetaUpdate` / `Spawn` DAGs resolve via [`resolve_approval_dag`]
-/// `Spawned` / `Rebuilt` / `ConfigReady` via `finish_approval`.
+/// when their DAG settles terminal.
 pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
     let approval = coord.approvals.mark_approved(id)?;
     tracing::info!(
@@ -56,54 +57,41 @@ pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
         }
         ApprovalKind::UpdateMetaInputs => {
             // Inputs JSON-encoded into commit_ref by the manager's
-            // submit path — surface them on the queue entry so the
+            // submit path — surface them on the DAG so the dashboard
-            // dashboard can show *which* inputs are about to bump.
+            // can show *which* inputs are about to bump. The cascade
+            // rebuilds fan out when the lock bump lands (so they build
+            // against the post-bump lock, and a failed bump fans out
+            // nothing).
             let inputs: Vec<String> =
                 serde_json::from_str(&approval.commit_ref).unwrap_or_default();
-            let parent_id = coord
+            let submitted = coord
-                .rebuild_queue
+                .job_queue
-                .enqueue_full(crate::rebuild_queue::FullEnqueue {
+                .submit(crate::job_queue::templates::meta_update(
-                    kind: crate::rebuild_queue::QueueKind::MetaUpdate,
+                    inputs,
-                    agent: approval.agent.clone(),
+                    crate::job_queue::Source::Approval,
-                    source: crate::rebuild_queue::QueueSource::Approval,
+                    format!("approval #{id} meta input update"),
-                    reason: format!("approval #{id} meta input update"),
+                    Some(id),
-                    parent_id: None,
+                ));
-                    inputs: inputs.clone(),
+            if let Err(e) = submitted {
-                    approval_id: Some(id),
+                return Err(e.context("submit meta-update dag"));
-                    perm_payload: None,
-                    depends_on: Vec::new(),
-                });
-            // Pre-enqueue cascade rebuilds in topological order so
-            // agents depending on updated inputs are rebuilt after the
-            // lock bump, matching the dashboard post_meta_update path.
-            let cascade_agents = crate::rebuild_queue::meta_update_cascade_agents(&inputs).await;
-            let cascade_reason = format!("approval #{id} meta input cascade");
-            for name in cascade_agents {
-                coord.rebuild_queue.enqueue(
-                    crate::rebuild_queue::QueueKind::Rebuild,
-                    name,
-                    crate::rebuild_queue::QueueSource::MetaUpdate,
-                    cascade_reason.clone(),
-                    Some(parent_id),
-                );
             }
             coord.emit_rebuild_queue_snapshot();
             Ok(())
         }
         ApprovalKind::Spawn => {
-            coord
+            // The spawn's tail `Reconcile` starts the container, so the
-                .rebuild_queue
+            // new agent's power intent is `Up` from the outset.
-                .enqueue_full(crate::rebuild_queue::FullEnqueue {
+            if let Err(e) = coord.power.set(&approval.agent, crate::power::Wanted::Up) {
-                    kind: crate::rebuild_queue::QueueKind::Spawn,
+                tracing::warn!(agent = %approval.agent, error = ?e, "agent_power: seed on spawn failed");
-                    agent: approval.agent.clone(),
+            }
-                    source: crate::rebuild_queue::QueueSource::Approval,
+            let submitted = coord.job_queue.submit(crate::job_queue::templates::spawn(
-                    reason: format!("approval #{id} spawn"),
+                &approval.agent,
-                    parent_id: None,
+                id,
-                    inputs: Vec::new(),
+                format!("approval #{id} spawn"),
-                    approval_id: Some(id),
+            ));
-                    perm_payload: None,
+            if let Err(e) = submitted {
-                    depends_on: Vec::new(),
+                return Err(e.context("submit spawn dag"));
-                });
+            }
             coord.emit_rebuild_queue_snapshot();
             Ok(())
         }
@@ -137,29 +125,27 @@ pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
     }
 }
 
-/// Enqueue a `Rebuild` queue entry tied to an approval id. Shared by the
+/// Submit the single-node `ApprovalDeploy` DAG tied to an approval id.
-/// `ApplyCommit` and `MergeConfigPr` dispatch arms — both end in a container
+/// Shared by the `ApplyCommit` and `MergeConfigPr` dispatch arms — both
-/// rebuild routed through the queue, differing only in the queue `reason`.
+/// end in a container rebuild routed through the queue, differing only
-/// The queue worker branches on the approval's kind to pick the right handler.
+/// in the `reason`. The node executor branches on the approval's kind
+/// to pick the right handler.
 fn enqueue_approval_rebuild(
     coord: &Arc<Coordinator>,
     agent: &str,
     approval_id: i64,
     reason: String,
 ) {
-    coord
+    if let Err(e) = coord
-        .rebuild_queue
+        .job_queue
-        .enqueue_full(crate::rebuild_queue::FullEnqueue {
+        .submit(crate::job_queue::templates::approval_deploy(
-            kind: crate::rebuild_queue::QueueKind::Rebuild,
+            agent,
-            agent: agent.to_owned(),
+            approval_id,
-            source: crate::rebuild_queue::QueueSource::Approval,
             reason,
-            parent_id: None,
+        ))
-            inputs: Vec::new(),
+    {
-            approval_id: Some(approval_id),
+        tracing::error!(%agent, approval_id, error = ?e, "submit approval deploy dag failed");
-            perm_payload: None,
+    }
-            depends_on: Vec::new(),
-        });
     coord.emit_rebuild_queue_snapshot();
 }
 
@@ -375,69 +361,60 @@ async fn run_approval_schedule_prompt(
     finish_approval(coord, &approval, result, None, false)
 }
 
-/// Worker entry point for `ApprovalKind::UpdateMetaInputs` queue
+/// Terminal hook for approval-carrying DAGs — the job queue's
-/// entries. Inputs come from the approval row's `commit_ref` field
+/// scheduler calls this exactly once when such a DAG settles terminal.
-/// (JSON-encoded by the manager submit path), not the queue entry's
+/// `MetaUpdate` and `Spawn` approval DAGs resolve here (their work is
-/// `inputs` — the queue copy is for dashboard display only.
+/// ordinary queue nodes); the opaque `ApprovalDeploy` pipeline resolves
-pub async fn run_approval_update_meta_inputs(
+/// *inside* its node, so its DAG is skipped — unless it was cancelled
+/// while still queued, in which case the node never ran and the row
+/// would otherwise dangle forever.
+pub(crate) async fn resolve_approval_dag(
     coord: &Arc<Coordinator>,
-    queue_entry_id: Option<u64>,
+    terminal: &crate::job_queue::TerminalDag,
-    approval_id: i64,
+) {
-) -> Result<()> {
+    use crate::job_queue::{State, Template};
-    let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::UpdateMetaInputs)?;
+    let Some(approval_id) = terminal.approval_id else {
-    let inputs: Vec<String> = serde_json::from_str(&approval.commit_ref).unwrap_or_default();
+        return;
-    coord.set_queue_step(queue_entry_id, "nix flake update");
+    };
-    let result = crate::meta::lock_update(&inputs).await;
+    if terminal.template == Template::Rebuild && terminal.state != State::Cancelled {
-    finish_approval(coord, &approval, result, None, false)
+        return; // ApprovalDeploy resolved inside the node.
-}
+    }
-
+    let approval = match coord.approvals.get(approval_id) {
-/// Worker entry point for `ApprovalKind::Spawn` queue entries.
+        Ok(Some(a)) => a,
-/// Differs from `run_approval_apply_commit` only in routing through
+        Ok(None) => {
-/// `lifecycle::spawn` (the deprecated direct-spawn path). Synchronous
+            tracing::warn!(approval_id, "approval dag terminal: row no longer exists");
-/// in the queue worker — the previous `tokio::spawn` wrapper is gone
+            return;
-/// (the queue worker itself is the async task).
-pub async fn run_approval_spawn(
-    coord: &Arc<Coordinator>,
-    queue_entry_id: Option<u64>,
-    approval_id: i64,
-) -> Result<()> {
-    let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::Spawn)?;
-    let agent_dir = coord.ensure_runtime(&approval.agent)?;
-    let hive = coord.hive_env();
-    let paths = Coordinator::agent_paths(&approval.agent, agent_dir);
-    // Transient guard keeps the per-container "Spawning" pill lit while
-    // the worker is doing the actual nixos-container create. Auto-clears
-    // on the function's scope exit (success or panic).
-    let _guard = coord.transient_guard(&approval.agent, TransientKind::Spawning);
-    coord.set_queue_step(queue_entry_id, "lifecycle::spawn");
-    let result = lifecycle::spawn(&approval.agent, &hive, &paths).await;
-    if result.is_ok() {
-        coord.set_queue_step(queue_entry_id, "forge user");
-        if let Err(e) = crate::forge::ensure_user_for(&approval.agent).await {
-            tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_user after spawn failed");
         }
-        coord.set_queue_step(queue_entry_id, "forge config repo");
+        Err(e) => {
-        if let Err(e) = crate::forge::ensure_config_repo(&approval.agent).await {
+            tracing::warn!(approval_id, error = ?e, "approval dag terminal: row read failed");
-            tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_config_repo after spawn failed");
+            return;
         }
-        coord.set_queue_step(queue_entry_id, "forge push");
+    };
-        if let Err(e) = crate::forge::push_config(&approval.agent).await {
+    let result: Result<()> = match terminal.state {
-            tracing::warn!(agent = %approval.agent, error = ?e, "forge: push_config after spawn failed");
+        State::Done => Ok(()),
-        }
+        State::Cancelled => Err(anyhow::anyhow!("cancelled before completion")),
-        coord.set_queue_step(queue_entry_id, "forge meta access");
+        _ => Err(anyhow::anyhow!(
-        if let Some(core_token) = crate::forge::core_token()
+            "{}",
-            && let Err(e) = crate::forge::meta_read_access(&approval.agent, &core_token).await
+            terminal
-        {
+                .error
-            tracing::warn!(agent = %approval.agent, error = ?e, "forge: meta_read_access after spawn failed");
+                .clone()
-        }
+                .unwrap_or_else(|| "job dag failed".to_owned())
-        if let Err(e) = crate::forge::ensure_meta_remote(&approval.agent).await {
+        )),
-            tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_meta_remote after spawn failed");
+    };
+    if approval.kind == ApprovalKind::Spawn {
+        // Post-spawn forge bookkeeping (user, config repo mirror, meta
+        // access) — warn-only, then the resolution events + a rescan so
+        // the dashboard reflects the post-spawn state either way.
+        if result.is_ok() {
+            forge_after_first_spawn(coord, &approval.agent).await;
+        } else {
+            coord.rescan_containers_and_emit().await;
+            crate::dashboard::emit_tombstones_snapshot(coord).await;
         }
     }
-    let final_result = finish_approval(coord, &approval, result, None, false);
+    if let Err(e) = finish_approval(coord, &approval, result, None, false) {
-    coord.rescan_containers_and_emit().await;
+        tracing::warn!(approval_id, error = ?e, "approval dag resolved with failure");
-    crate::dashboard::emit_tombstones_snapshot(coord).await;
+    }
-    final_result
 }
 
 /// Re-fetch an approval row from sqlite for a queue-worker dispatch.
@@ -867,13 +844,7 @@ async fn deploy_applied_target(
         // part of this entry rather than a deferred fast-lane follow-up.
         false,
         &|step| coord.set_queue_step(queue_entry_id, step),
-        &|log_id| {
+        &|log_id| coord.set_queue_build_log(queue_entry_id, log_id),
-            if let Some(qid) = queue_entry_id
-                && coord.rebuild_queue.set_build_log_id(qid, log_id)
-            {
-                coord.emit_rebuild_queue_snapshot();
-            }
-        },
     )
     .await;
 
@@ -984,6 +955,11 @@ pub async fn destroy(coord: &Arc<Coordinator>, name: &str, purge: bool) -> Resul
             "agent destroyed"
         },
     );
+    // Drop the durable power intent — a future agent of the same name
+    // seeds fresh from its observed state.
+    if let Err(e) = coord.power.remove(name) {
+        tracing::warn!(%name, error = ?e, "agent_power: remove on destroy failed");
+    }
     drop(guard);
     coord.notify_manager(&HelperEvent::Destroyed {
         agent: name.to_owned(),

hive-c0re/src/auto_update.rs

@@ -1,18 +1,25 @@
-//! Startup auto-update: on `hive-c0re serve` boot, rebuild containers that
+//! Boot reconcile: on `hive-c0re serve` boot, (a) run the config path
-//! actually need it. Two skip rules keep boot-time work minimal:
+//! for agents whose per-agent rev marker is stale — a `StartupSweep`
+//! DAG (meta hyperhive lock bump) fanning out `Rebuild` children for
+//! the stale agents whose `wanted` power intent is `Up` — and (b)
+//! converge every other drifted agent to its persisted `wanted` via
+//! `Reconcile` DAGs. Two rules keep boot-time nix work minimal:
 //!
-//! 1. **Stopped containers** are deferred — they will be rebuilt the first
+//! 1. **Stale but wanted-offline agents** get no rebuild — their
-//!    time the operator starts them (see `rebuild_queue::run_start` and
+//!    rebuild happens the first time they're started (the start
-//!    `socket_server::handle_start`).
+//!    submit path upgrades a stale start to rebuild+start). The sweep
-//! 2. **Running containers whose rev marker matches** the current hyperhive
+//!    parent still runs whenever *any* marker is stale so the meta
-//!    flake path are skipped — nothing changed, no nix work to do.
+//!    hyperhive lock is bumped for those later start-upgrades.
+//! 2. **Agents whose rev marker matches** the current hyperhive flake
+//!    path are skipped — nothing changed, no nix work to do.
 //!
-//! See `docs/coordinator.md::Auto-update sweep`.
+//! Booting with no config change performs no meta commit — only
+//! reconciles. See `docs/coordinator.md::Boot reconcile`.
 
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
-use anyhow::{Context, Result};
+use anyhow::Result;
 
 use crate::coordinator::Coordinator;
 use crate::lifecycle::{self, AGENT_PREFIX, MANAGER_NAME};
@@ -71,143 +78,6 @@ pub async fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> boo
     }
 }
 
-/// Rebuild one sub-agent and refresh its marker. Used by both the startup
-/// scanner and the dashboard's manual "update" button so the two paths
-/// can't diverge.
-///
-/// `queue_entry_id` is `Some(id)` when the rebuild was dispatched from
-/// the `rebuild_queue` worker (lets the function annotate its phase via
-/// `coord.set_queue_step`) and `None` when called directly (e.g. the
-/// root-agent migration nudge in `ensure_root_agent`).
-///
-/// `relock` bumps the agent's meta input to `applied/<n>/main` before
-/// the container rebuild. Pass `false` only for meta-update cascade
-/// rebuilds, where re-locking would revert the bump the cascade just
-/// committed (see `lifecycle::rebuild`).
-///
-/// `defer_start_source` is `Some(source)` for queue-dispatched rebuilds:
-/// instead of holding the serialized build lane through the container
-/// boot, the start-after-rebuild is enqueued as a fast-lane `Start`
-/// entry (grouped under this rebuild via `parent_id`, same split as the
-/// graceful-stop follow-up). Pass `None` for direct callers to keep the
-/// start inline.
-///
-/// # Errors
-///
-/// Propagates errors from `coord.ensure_runtime` and `lifecycle::rebuild`.
-pub async fn rebuild_agent(
-    coord: &Arc<Coordinator>,
-    name: &str,
-    current_rev: &str,
-    queue_entry_id: Option<u64>,
-    relock: bool,
-    defer_start_source: Option<crate::rebuild_queue::QueueSource>,
-) -> Result<()> {
-    tracing::info!(%name, rev = %current_rev, "rebuild agent");
-    let agent_dir = coord
-        .ensure_runtime(name)
-        .with_context(|| format!("ensure_runtime {name}"))?;
-    let hive = coord.hive_env();
-    let paths = Coordinator::agent_paths(name, agent_dir);
-    // Suppress crash_watch during the stop+start window inside
-    // lifecycle::rebuild. Dashboard rebuilds already do this via
-    // lifecycle_action; this catches the auto-update scan + any
-    // other direct caller.
-    let guard = coord.transient_guard(name, crate::coordinator::TransientKind::Rebuilding);
-    let result = lifecycle::rebuild(
-        name,
-        &hive,
-        &paths,
-        relock,
-        defer_start_source.is_some(),
-        &|step| coord.set_queue_step(queue_entry_id, step),
-        &|log_id| {
-            if let Some(qid) = queue_entry_id
-                && coord.rebuild_queue.set_build_log_id(qid, log_id)
-            {
-                coord.emit_rebuild_queue_snapshot();
-            }
-        },
-    )
-    .await;
-    drop(guard);
-    match &result {
-        Ok(needs_start) => {
-            if let Err(e) = std::fs::write(rev_marker_path(name), current_rev) {
-                tracing::warn!(%name, error = ?e, "write rev marker failed");
-            }
-            // Deferred start: hand the container boot to the fast lane so
-            // this build-lane entry completes now and the next queued
-            // rebuild's nix build overlaps with the boot. `parent_id`
-            // groups the follow-up under this rebuild on the dashboard —
-            // same split the graceful-stop path uses for its container
-            // stop. A start failure surfaces on the Start entry (with
-            // the cold-start fallback) instead of failing the rebuild.
-            if *needs_start && let Some(source) = defer_start_source {
-                coord
-                    .rebuild_queue
-                    .enqueue_full(crate::rebuild_queue::FullEnqueue {
-                        kind: crate::rebuild_queue::QueueKind::Start,
-                        agent: name.to_owned(),
-                        source,
-                        reason: format!("start after rebuild of {name}"),
-                        parent_id: queue_entry_id,
-                        inputs: Vec::new(),
-                        approval_id: None,
-                        perm_payload: None,
-                        depends_on: Vec::new(),
-                    });
-                coord.emit_rebuild_queue_snapshot();
-            }
-            coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
-                agent: name.to_owned(),
-                ok: true,
-                note: None,
-                sha: None,
-                tag: None,
-            });
-            coord.set_queue_step(queue_entry_id, "forge sync");
-            // Run the full forge sync on every successful rebuild so
-            // the rebuild path is equivalent to the hive-c0re startup
-            // sweep: token, config-repo mirror, meta read access, and
-            // meta remote are all kept in sync. Recovers missing tokens
-            // (e.g. first-spawn seeding failed transiently) without
-            // requiring a full hive-c0re restart.
-            crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await;
-            // Mirror the matrix side of the startup sweep: if hive-matrix
-            // is present, ensure this agent has a registered user + token.
-            // Idempotent (skips if token file already exists). Keeps the
-            // rebuild path equivalent to the startup sweep for newly-spawned
-            // agents that missed ensure_all().
-            crate::matrix::sync_agent_standalone(name).await;
-            // Wake the agent on its next turn so claude sees a
-            // "you were rebuilt — check /state/ for notes, --continue
-            // session intact" hint. Covers dashboard rebuild, admin
-            // CLI rebuild, auto-update startup scan, and the
-            // dashboard's meta-input update path — all of which
-            // route through rebuild_agent.
-            coord.kick_agent(name, "container rebuilt");
-            // Container state (needs_update, deployed_sha) may have
-            // shifted — rescan so dashboards drop the "needs update"
-            // chip without waiting for the next /api/state poll.
-            coord.rescan_containers_and_emit().await;
-            // Lock bump → meta-inputs panel needs to re-render.
-            crate::dashboard::emit_meta_inputs_snapshot(coord);
-        }
-        Err(e) => {
-            coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
-                agent: name.to_owned(),
-                ok: false,
-                note: Some(format!("{e:#}")),
-                sha: None,
-                tag: None,
-            });
-            coord.rescan_containers_and_emit().await;
-        }
-    }
-    result.map(|_| ())
-}
-
 /// Whether this hive is "ruthless" — running with no root/manager agent at
 /// all (no ruth). When true, hive-c0re skips the root-agent create/start
 /// sweep entirely. Controlled by the host option
@@ -246,17 +116,18 @@ pub async fn ensure_root_agent(coord: &Arc<Coordinator>) -> Result<()> {
         // running whatever the host-declarative config was at create
         // time, with a wrong systemd unit and port.
         let applied_flake = Coordinator::agent_applied_dir(MANAGER_NAME).join("flake.nix");
-        if !applied_flake.exists()
+        if !applied_flake.exists() && current_rev.is_some() {
-            && let Some(rev) = current_rev.as_ref()
-        {
             tracing::warn!(
                 "manager container exists but no applied flake — forcing rebuild to migrate"
             );
-            let coord_clone = coord.clone();
+            if let Err(e) = coord.job_queue.submit(crate::job_queue::templates::rebuild(
-            if let Err(e) =
+                MANAGER_NAME,
-                rebuild_agent(&coord_clone, MANAGER_NAME, rev.as_str(), None, true, None).await
+                crate::job_queue::Source::AutoUpdate,
-            {
+                "manager migration: no applied flake".to_owned(),
-                tracing::warn!(error = ?e, "manager migration rebuild failed");
+                None,
+                true,
+            )) {
+                tracing::warn!(error = ?e, "manager migration rebuild submit failed");
             }
         } else {
             tracing::debug!("manager container already present");
@@ -272,6 +143,9 @@ pub async fn ensure_root_agent(coord: &Arc<Coordinator>) -> Result<()> {
         // this function.)
         if !lifecycle::is_running(MANAGER_NAME).await {
             tracing::info!("manager container present but not running — starting");
+            if let Err(e) = coord.power.set(MANAGER_NAME, crate::power::Wanted::Up) {
+                tracing::warn!(error = ?e, "agent_power: set manager wanted=up failed");
+            }
             if let Err(e) = lifecycle::start(MANAGER_NAME).await {
                 tracing::warn!(error = ?e, "manager start failed");
             }
@@ -283,6 +157,9 @@ pub async fn ensure_root_agent(coord: &Arc<Coordinator>) -> Result<()> {
     let hive = coord.hive_env();
     let paths = Coordinator::agent_paths(MANAGER_NAME, runtime);
     lifecycle::spawn(MANAGER_NAME, &hive, &paths).await?;
+    if let Err(e) = coord.power.set(MANAGER_NAME, crate::power::Wanted::Up) {
+        tracing::warn!(error = ?e, "agent_power: set manager wanted=up failed");
+    }
     if let Some(rev) = current_rev {
         let _ = std::fs::write(rev_marker_path(MANAGER_NAME), &rev);
     }
@@ -327,18 +204,17 @@ pub fn topology_sort(
     });
 }
 
-/// Rebuild containers that need it on startup. Skips:
+/// Boot reconcile (see the module doc): classify every agent by rev
-/// - **Stopped containers**: deferred to on-start (`run_start` / `handle_start`
+/// freshness + persisted `wanted` intent, submit one `StartupSweep`
-///   upgrades a plain start to rebuild+start when the rev marker is stale).
+/// DAG (hyperhive lock bump → fan-out rebuilds for stale wanted-up
-/// - **Running containers with a matching rev marker**: no nix work needed.
+/// agents) when anything is stale, and `Reconcile` DAGs for agents
-///
+/// whose observed power state drifted from `wanted`. Returns Ok even
-/// Enqueues a `StartupSweep` parent entry followed by per-agent `Rebuild`
+/// if some submissions failed.
-/// children linked via `parent_id`. Returns Ok even if some rebuilds failed.
 pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
     let containers = match lifecycle::list().await {
         Ok(c) => c,
         Err(e) => {
-            tracing::warn!(error = ?e, "auto-update: nixos-container list failed");
+            tracing::warn!(error = ?e, "boot reconcile: nixos-container list failed");
             return Ok(());
         }
     };
@@ -356,62 +232,88 @@ pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
     let topo = crate::topology::read();
     topology_sort(&mut logical_names, &topo);
 
-    // Pre-classify: decide which agents need a rebuild now vs can be skipped.
+    // Classify. `get_or_seed` doubles as the one-time migration: an
-    let mut to_rebuild: Vec<String> = Vec::new();
+    // agent without an `agent_power` row is seeded from its observed
+    // state (running ⇒ Up), after which the DB is authoritative.
+    let mut any_stale = false;
+    let mut fanout: Vec<String> = Vec::new(); // stale ∧ wanted=Up → sweep rebuild
+    let mut drifted: Vec<String> = Vec::new(); // fresh ∧ wanted≠observed → reconcile
     let mut n_deferred = 0usize;
     let mut n_skipped = 0usize;
     for name in &logical_names {
-        // Idea 2: stopped containers are deferred — rebuild happens the first
+        let running = lifecycle::is_running(name).await;
-        // time the operator starts them.
+        let wanted = match coord.power.get_or_seed(name, running) {
-        if !lifecycle::is_running(name).await {
+            Ok(w) => w,
-            n_deferred += 1;
+            Err(e) => {
-            tracing::debug!(%name, "startup sweep: stopped — deferring rebuild to on-start");
+                tracing::warn!(%name, error = ?e, "agent_power read failed — assuming observed");
-            continue;
+                crate::power::Wanted::from_running(running)
-        }
+            }
-        // Idea 1: running containers with a matching rev marker need no rebuild.
+        };
-        if let Some(ref rev) = current_rev {
+        let fresh = current_rev.as_ref().is_some_and(|rev| {
-            let stored = std::fs::read_to_string(rev_marker_path(name)).ok();
+            std::fs::read_to_string(rev_marker_path(name))
-            if stored.as_deref() == Some(rev.as_str()) {
+                .is_ok_and(|stored| stored == rev.as_str())
-                n_skipped += 1;
+        });
-                tracing::debug!(%name, "startup sweep: rev unchanged — skipping rebuild");
+        if fresh {
+            n_skipped += 1;
+        } else {
+            any_stale = true;
+            if wanted == crate::power::Wanted::Up {
+                // Rebuild against the post-bump lock; the DAG's tail
+                // Reconcile brings the agent (back) up — covering both
+                // the running-stale and stopped-but-wanted-up cases.
+                fanout.push(name.clone());
                 continue;
             }
+            // Stale but wanted offline: no boot-time nix work — the
+            // start submit path upgrades a stale start to a rebuild.
+            n_deferred += 1;
+            tracing::debug!(%name, "boot reconcile: stale but offline — deferring rebuild to on-start");
+        }
+        if crate::power::reconcile_action(wanted, running) != crate::power::ReconcileAction::Noop {
+            drifted.push(name.clone());
         }
-        to_rebuild.push(name.clone());
     }
 
-    // Enqueue the parent sweep entry. The worker processes it trivially
-    // (no-op dispatch); its purpose is to give the dashboard a "why" header.
-    let sweep_id = coord.rebuild_queue.enqueue(
-        crate::rebuild_queue::QueueKind::StartupSweep,
-        "hyperhive".to_owned(),
-        crate::rebuild_queue::QueueSource::AutoUpdate,
-        format!(
-            "startup sweep: {} rebuild(s), {} deferred (stopped), {} skipped (up-to-date)",
-            to_rebuild.len(),
-            n_deferred,
-            n_skipped,
-        ),
-        None,
-    );
-
     tracing::info!(
         total = containers.len(),
-        rebuilds = to_rebuild.len(),
+        rebuilds = fanout.len(),
+        reconciles = drifted.len(),
         deferred = n_deferred,
-        skipped = n_skipped,
+        up_to_date = n_skipped,
-        sweep_id,
+        "boot reconcile"
-        "auto-update: startup sweep"
     );
 
-    for name in to_rebuild {
+    // Sweep parent whenever ANY marker is stale — even when every
-        coord.rebuild_queue.enqueue(
+    // stale agent is wanted-offline: the hyperhive lock bump must land
-            crate::rebuild_queue::QueueKind::Rebuild,
+    // now so their later start-upgrade rebuilds build against it.
-            name,
+    // No stale agents ⇒ no sweep ⇒ no meta commit on a no-change boot.
-            crate::rebuild_queue::QueueSource::StartupSweep,
+    if any_stale {
-            "startup sweep".to_owned(),
+        let reason = format!(
-            Some(sweep_id),
+            "startup sweep: {} rebuild(s), {} deferred (offline), {} up-to-date",
+            fanout.len(),
+            n_deferred,
+            n_skipped,
         );
+        if let Err(e) = coord
+            .job_queue
+            .submit(crate::job_queue::templates::startup_sweep(reason, fanout))
+        {
+            tracing::warn!(error = ?e, "boot reconcile: sweep submit failed");
+        }
+    }
+    for name in drifted {
+        if let Err(e) = coord
+            .job_queue
+            .submit(crate::job_queue::templates::reconcile_only(
+                crate::job_queue::Template::Reconcile,
+                &name,
+                crate::job_queue::Source::AutoUpdate,
+                "boot reconcile".to_owned(),
+                None,
+            ))
+        {
+            tracing::warn!(%name, error = ?e, "boot reconcile: submit failed");
+        }
     }
     coord.emit_rebuild_queue_snapshot();
     Ok(())

hive-c0re/src/coordinator.rs

@@ -173,12 +173,17 @@ pub struct Coordinator {
     /// tokio mutex so the rescan can `await` `lifecycle::list` /
     /// `is_running` without blocking other coordinator paths.
     last_containers: tokio::sync::Mutex<HashMap<String, ContainerView>>,
-    /// Global rebuild queue. Every long-running container/meta op
+    /// Global job-DAG queue. Every container/meta op (rebuild,
-    /// (rebuild, meta-update, first-spawn) goes through this queue so
+    /// meta-update, first-spawn, power changes) is submitted as a DAG
-    /// hive-c0re runs at most one at a time and the dashboard can
+    /// of primitive nodes; a single scheduler drives them with
-    /// render a single ordered view of pending + running work. See
+    /// build-slot + per-agent-lease gating so the dashboard renders one
-    /// `rebuild_queue.rs` for the dedup rules + history retention.
+    /// ordered view of pending + running work. See `job_queue/` for
-    pub rebuild_queue: Arc<crate::rebuild_queue::RebuildQueue>,
+    /// the dedup rules, resource classes, and history retention.
+    pub job_queue: Arc<crate::job_queue::JobQueue>,
+    /// Durable per-agent power intent (`wanted: Up | Offline`) — the
+    /// spec half of desired-state reconciliation; the queue's
+    /// `Reconcile` nodes converge observed state to it.
+    pub power: Arc<crate::power::PowerStore>,
     /// Shutdown signal broadcast to all background tasks. Sending
     /// `true` asks every loop to exit after its current work item.
     /// Use `shutdown_rx()` to subscribe; `request_shutdown()` to fire.
@@ -244,12 +249,27 @@ impl Default for HiveEnv {
 /// instead of every host-level setting as its own JSON-blob argument.
 /// `#[serde(default)]` lets any field be omitted and fall back to its
 /// canonical default.
-#[derive(Clone, Debug, Default, serde::Deserialize)]
+#[derive(Clone, Debug, serde::Deserialize)]
 #[serde(default)]
 pub struct ServeConfig {
     #[serde(flatten)]
     pub env: HiveEnv,
     pub model_prices: crate::hive_stats::PriceTable,
+    /// Number of concurrent nix-heavy job-queue nodes (prebuild /
+    /// profile-swap / create / meta lock). hive-c0re-local like
+    /// `model_prices` — never injected into containers. Set via
+    /// `services.hyperhive.c0re.buildSlots`.
+    pub build_slots: usize,
+}
+
+impl Default for ServeConfig {
+    fn default() -> Self {
+        Self {
+            env: HiveEnv::default(),
+            model_prices: crate::hive_stats::PriceTable::default(),
+            build_slots: 1,
+        }
+    }
 }
 
 #[cfg(test)]
@@ -433,6 +453,7 @@ impl Coordinator {
         db_path: &Path,
         env: HiveEnv,
         model_prices: crate::hive_stats::PriceTable,
+        build_slots: usize,
     ) -> Result<Self> {
         let HiveEnv {
             hyperhive_flake,
@@ -469,6 +490,8 @@ impl Coordinator {
         let audit_log =
             Arc::new(crate::audit_log::AuditLog::open(build_logs_dir).context("open audit_log")?);
         crate::audit_log::install(audit_log.clone());
+        let power =
+            Arc::new(crate::power::PowerStore::open(build_logs_dir).context("open agent_power")?);
         let (dashboard_events, _) = broadcast::channel(DASHBOARD_CHANNEL);
         let (shutdown_tx, _) = watch::channel(false);
         Ok(Self {
@@ -497,7 +520,8 @@ impl Coordinator {
             event_seq: AtomicU64::new(0),
             meta_updates_active: AtomicU64::new(0),
             last_containers: tokio::sync::Mutex::new(HashMap::new()),
-            rebuild_queue: Arc::new(crate::rebuild_queue::RebuildQueue::new()),
+            job_queue: Arc::new(crate::job_queue::JobQueue::new(build_slots)),
+            power,
             shutdown_tx,
         })
     }
@@ -543,7 +567,7 @@ impl Coordinator {
     /// wrappers below) and the worker so every state transition
     /// surfaces on the dashboard without extra plumbing.
     pub fn emit_rebuild_queue_snapshot(self: &Arc<Self>) {
-        let queue = self.rebuild_queue.snapshot();
+        let queue = self.job_queue.snapshot();
         self.emit_dashboard_event(DashboardEvent::RebuildQueueChanged {
             seq: self.next_seq(),
             queue,
@@ -665,15 +689,27 @@ impl Coordinator {
         });
     }
 
-    /// Update the `step` label on a running queue entry and (if it
+    /// Update the `step` label on the currently-running node of DAG
-    /// actually changed) re-emit the queue snapshot so the dashboard
+    /// `id` and (if it actually changed) re-emit the queue snapshot so
-    /// renders the new phase. Returns `true` when the label was new
+    /// the dashboard renders the new phase. DAG-id-only surface for
-    /// and an emit fired, mostly for tracing/logging callers; safe to
+    /// the opaque approval pipeline in `actions.rs`, whose callbacks
-    /// ignore. No-op when `id` is `None` (e.g. callers that aren't
+    /// don't know node ids (its DAGs are single-node, so the lookup is
-    /// running from the queue worker) or when the row isn't `Running`.
+    /// exact); queue executors use the precise per-node sink in
+    /// `job_queue::exec` instead. No-op when `id` is `None` (callers
+    /// not running from the queue) or when nothing is `Running`.
     pub fn set_queue_step(self: &Arc<Self>, id: Option<u64>, step: &str) {
         let Some(id) = id else { return };
-        if self.rebuild_queue.set_step(id, step) {
+        if self.job_queue.set_step_running(id, step) {
+            self.emit_rebuild_queue_snapshot();
+        }
+    }
+
+    /// Link a `build_logs` row to the currently-running node of DAG
+    /// `id` and re-emit the snapshot. Same DAG-id-only compatibility
+    /// surface as [`Self::set_queue_step`].
+    pub fn set_queue_build_log(self: &Arc<Self>, id: Option<u64>, log_id: i64) {
+        let Some(id) = id else { return };
+        if self.job_queue.set_build_log_id_running(id, log_id) {
             self.emit_rebuild_queue_snapshot();
         }
     }

hive-c0re/src/dashboard.rs

@@ -299,12 +299,13 @@ struct StateSnapshot {
     /// disabled "updating…" state; live transitions arrive via the
     /// `MetaUpdateRunning` event.
     meta_update_running: bool,
-    /// Current state of the global rebuild queue — pending + running
+    /// Current state of the global job queue — pending + running DAGs
-    /// long-lived ops (rebuild / meta-update / spawn) plus the most
+    /// (rebuild / meta-update / spawn / power ops) with their per-node
-    /// recent few terminal entries the queue retains for history.
+    /// breakdowns, plus the most recent few terminal DAGs the queue
-    /// Live transitions arrive via the `RebuildQueueChanged` event.
+    /// retains for history. Live transitions arrive via the
-    /// See `rebuild_queue.rs`.
+    /// `RebuildQueueChanged` event. See `job_queue/`. Field name kept
-    rebuild_queue: Vec<crate::rebuild_queue::QueueEntry>,
+    /// from the old flat queue for wire compatibility.
+    rebuild_queue: Vec<crate::job_queue::DagView>,
     /// Whether the hive-forge container is up. When true the dashboard
     /// links each container's config + each approval's commit into the
     /// forge's `agent-configs` repos.
@@ -607,7 +608,7 @@ async fn api_state(headers: HeaderMap, State(state): State<AppState>) -> axum::J
         question_history,
         tombstones,
         port_conflicts,
-        rebuild_queue: state.coord.rebuild_queue.snapshot(),
+        rebuild_queue: state.coord.job_queue.snapshot(),
         forge_present: crate::forge::is_present().await,
         matrix_gui_enabled: std::env::var_os("HIVE_MATRIX_GUI_ENABLED").is_some_and(|v| {
             // Accept any truthy string ("1", "true", "yes") since the
@@ -1602,30 +1603,15 @@ async fn post_meta_update(
         return error_response("meta-update: no inputs selected");
     }
     let inputs_label = inputs.join(", ");
-    let parent_id = state.coord.rebuild_queue.enqueue_with_inputs(
+    // Cascade rebuild children fan out from the MetaLock node when the
-        crate::rebuild_queue::QueueKind::MetaUpdate,
+    // lock bump lands — appended by the scheduler so they build against
-        "hyperhive".to_owned(),
+    // the post-bump lock, and a failed bump simply fans out nothing.
-        crate::rebuild_queue::QueueSource::Manual,
+    crate::job_queue::submit::meta_update(
+        &state.coord,
+        inputs,
+        crate::job_queue::Source::Manual,
         format!("meta-update via dashboard ({inputs_label})"),
-        None,
-        inputs.clone(),
     );
-    // Pre-enqueue cascade rebuilds NOW so they're visible in the queue
-    // alongside the parent. The worker's MetaUpdate arm
-    // no longer enqueues children — it just runs the lock bump and
-    // (on failure) cancels these pre-queued children.
-    let cascade_agents = crate::rebuild_queue::meta_update_cascade_agents(&inputs).await;
-    let cascade_reason = format!("meta-update cascade ({inputs_label})");
-    for name in cascade_agents {
-        state.coord.rebuild_queue.enqueue(
-            crate::rebuild_queue::QueueKind::Rebuild,
-            name,
-            crate::rebuild_queue::QueueSource::MetaUpdate,
-            cascade_reason.clone(),
-            Some(parent_id),
-        );
-    }
-    state.coord.emit_rebuild_queue_snapshot();
     (StatusCode::OK, "ok").into_response()
 }
 

hive-c0re/src/dashboard/lifecycle_ops.rs

@@ -1,10 +1,12 @@
 //! Container lifecycle endpoints for the dashboard.
 //!
 //! Rebuild / restart / start / stop (hard + graceful) / update-all all
-//! enqueue onto the rebuild queue, so each shows a visible queued→running
+//! submit DAGs to the job queue (`job_queue::submit`), so each shows a
-//! transient on the dashboard — a direct sub-second start/stop only flashed
+//! visible queued→running transient on the dashboard — a direct
-//! the badge; destroy delegates to `actions::destroy` (optionally
+//! sub-second start/stop only flashed the badge. Start/stop also
-//! purging).
+//! persist the agent's `wanted` power intent before submitting; the
+//! DAG's `Reconcile` converges to it. Destroy delegates to
+//! `actions::destroy` (optionally purging).
 
 use axum::{
     extract::{Form, Path as AxumPath, Query, State},
@@ -23,6 +25,7 @@ pub(super) struct KillParams {
 }
 
 use super::{AppState, error_response, guard_agent_name, strip_container_prefix};
+use crate::job_queue::{Source, submit};
 use crate::{actions, lifecycle};
 
 pub(super) async fn post_rebuild(
@@ -33,14 +36,12 @@ pub(super) async fn post_rebuild(
     if let Some(reject) = guard_agent_name(&state, &logical).await {
         return reject;
     }
-    state.coord.rebuild_queue.enqueue(
+    submit::rebuild(
-        crate::rebuild_queue::QueueKind::Rebuild,
+        &state.coord,
-        logical,
+        &logical,
-        crate::rebuild_queue::QueueSource::Manual,
+        Source::Manual,
         "manual via dashboard ↻ R3BU1LD button".to_owned(),
-        None,
     );
-    state.coord.emit_rebuild_queue_snapshot();
     (StatusCode::OK, "ok").into_response()
 }
 
@@ -54,19 +55,17 @@ pub(super) async fn post_kill(
         return reject;
     }
     if params.graceful {
-        // Graceful stop: enqueue the quiesce orchestration (signal the harness
+        // Graceful stop: submit the quiesce DAG (signal the harness →
-        // → one stop-checkpoint turn → drain → container stop, with a timeout
+        // one stop-checkpoint turn → drain → container stop, with a
-        // fallback to a hard stop). Serialised through the rebuild queue so it
+        // timeout fallback to a hard stop). The agent's lifecycle
-        // can't race an in-flight rebuild for the same agent, and its per-step
+        // lease keeps it from racing an in-flight rebuild for the same
-        // progress surfaces on the queue snapshot + build log.
+        // agent, and per-node progress surfaces on the queue snapshot.
-        state.coord.rebuild_queue.enqueue(
+        submit::graceful_stop(
-            crate::rebuild_queue::QueueKind::GracefulStop,
+            &state.coord,
-            logical,
+            &logical,
-            crate::rebuild_queue::QueueSource::Manual,
+            Source::Manual,
             "manual via dashboard graceful stop".to_owned(),
-            None,
         );
-        state.coord.emit_rebuild_queue_snapshot();
         return (StatusCode::OK, "ok").into_response();
     }
     // Manager is stoppable from the dashboard like any other
@@ -79,14 +78,12 @@ pub(super) async fn post_kill(
     // `socket_server.rs::ManagerRequest::Kill` stays in place: a
     // manager calling Kill on its own container is self-suicide
     // mid-call, not a legitimate operator action.
-    state.coord.rebuild_queue.enqueue(
+    submit::stop(
-        crate::rebuild_queue::QueueKind::Stop,
+        &state.coord,
-        logical,
+        &logical,
-        crate::rebuild_queue::QueueSource::Manual,
+        Source::Manual,
         "manual via dashboard stop".to_owned(),
-        None,
     );
-    state.coord.emit_rebuild_queue_snapshot();
     (StatusCode::OK, "ok").into_response()
 }
 
@@ -98,14 +95,12 @@ pub(super) async fn post_restart(
     if let Some(reject) = guard_agent_name(&state, &logical).await {
         return reject;
     }
-    state.coord.rebuild_queue.enqueue(
+    submit::restart(
-        crate::rebuild_queue::QueueKind::Restart,
+        &state.coord,
-        logical,
+        &logical,
-        crate::rebuild_queue::QueueSource::Manual,
+        Source::Manual,
         "manual via dashboard ↺ R3START button".to_owned(),
-        None,
     );
-    state.coord.emit_rebuild_queue_snapshot();
     (StatusCode::OK, "ok").into_response()
 }
 
@@ -117,14 +112,12 @@ pub(super) async fn post_start(
     if let Some(reject) = guard_agent_name(&state, &logical).await {
         return reject;
     }
-    state.coord.rebuild_queue.enqueue(
+    submit::start(
-        crate::rebuild_queue::QueueKind::Start,
+        &state.coord,
-        logical,
+        &logical,
-        crate::rebuild_queue::QueueSource::Manual,
+        Source::Manual,
         "manual via dashboard start".to_owned(),
-        None,
     );
-    state.coord.emit_rebuild_queue_snapshot();
     (StatusCode::OK, "ok").into_response()
 }
 
@@ -137,15 +130,13 @@ pub(super) async fn post_update_all(State(state): State<AppState>) -> Response {
         else {
             continue;
         };
-        state.coord.rebuild_queue.enqueue(
+        submit::rebuild(
-            crate::rebuild_queue::QueueKind::Rebuild,
+            &state.coord,
-            logical,
+            &logical,
-            crate::rebuild_queue::QueueSource::Manual,
+            Source::Manual,
             "manual via dashboard 🌀 UPDATE ALL".to_owned(),
-            None,
         );
     }
-    state.coord.emit_rebuild_queue_snapshot();
     (StatusCode::OK, "ok").into_response()
 }
 

hive-c0re/src/dashboard/permissions.rs

@@ -128,18 +128,19 @@ pub(super) async fn post_tool_groups(
         return Err(ProblemDetails::from_status_code(StatusCode::BAD_REQUEST)
             .with_detail(format!("invalid tool-groups for {logical}: {e}")));
     }
-    // Enqueue a PermChange so the JSON file write is serialised through
+    // Submit a PermChange DAG: the JSON file write commits under
-    // the FIFO worker. Prevents concurrent batch-apply actions for
+    // META_LOCK inside the WritePermFile node, so concurrent
-    // different agents from racing on the shared tool-groups.json.
+    // batch-apply actions for different agents never race on the
-    state.coord.rebuild_queue.enqueue_with_perm(
+    // shared tool-groups.json.
-        logical.clone(),
+    crate::job_queue::submit::perm_change(
-        crate::rebuild_queue::QueueSource::Manual,
+        &state.coord,
+        &logical,
+        crate::job_queue::Source::Manual,
         "tool-group change via permissions UI".to_owned(),
-        crate::rebuild_queue::PermPayload::ToolGroups {
+        crate::job_queue::PermPayload::ToolGroups {
             groups: body.groups.clone(),
         },
     );
-    state.coord.emit_rebuild_queue_snapshot();
     tracing::info!(agent = %logical, groups = ?body.groups, "operator: set tool-groups via dashboard");
     Ok((StatusCode::OK, "ok").into_response())
 }
@@ -214,18 +215,19 @@ pub(super) async fn post_capabilities(
                 .with_detail(format!("unknown capability: {cap}")));
         }
     }
-    // Enqueue a PermChange so the JSON file write is serialised through
+    // Submit a PermChange DAG: the JSON file write commits under
-    // the FIFO worker. Prevents concurrent batch-apply actions for
+    // META_LOCK inside the WritePermFile node, so concurrent
-    // different agents from racing on the shared capabilities.json.
+    // batch-apply actions for different agents never race on the
-    state.coord.rebuild_queue.enqueue_with_perm(
+    // shared capabilities.json.
-        logical.clone(),
+    crate::job_queue::submit::perm_change(
-        crate::rebuild_queue::QueueSource::Manual,
+        &state.coord,
+        &logical,
+        crate::job_queue::Source::Manual,
         "capability change via dashboard".to_owned(),
-        crate::rebuild_queue::PermPayload::Capabilities {
+        crate::job_queue::PermPayload::Capabilities {
             caps: body.caps.clone(),
         },
     );
-    state.coord.emit_rebuild_queue_snapshot();
     tracing::info!(agent = %logical, caps = ?body.caps, "operator: set capabilities via dashboard");
     Ok((StatusCode::OK, "ok").into_response())
 }
@@ -298,17 +300,17 @@ pub(super) async fn post_permissions(
             ));
         }
     }
-    // Phase 2 — enqueue one combined PermChange per affected agent.
+    // Phase 2 — submit one combined PermChange DAG per affected agent.
     for (logical, groups, caps) in staged {
-        state.coord.rebuild_queue.enqueue_with_perm(
+        crate::job_queue::submit::perm_change(
-            logical.clone(),
+            &state.coord,
-            crate::rebuild_queue::QueueSource::Manual,
+            &logical,
+            crate::job_queue::Source::Manual,
             "batch permission change via permissions UI".to_owned(),
-            crate::rebuild_queue::PermPayload::Combined { groups, caps },
+            crate::job_queue::PermPayload::Combined { groups, caps },
         );
         tracing::info!(agent = %logical, "operator: batch perm change via dashboard");
     }
-    state.coord.emit_rebuild_queue_snapshot();
     Ok((StatusCode::OK, "ok").into_response())
 }
 

hive-c0re/src/dashboard/schedules.rs

@@ -115,20 +115,19 @@ pub(super) async fn post_schedule_fire_now(
     }
 }
 
-/// `POST /api/rebuild-queue/{id}/cancel` — drop a `Queued` entry
+/// `POST /api/rebuild-queue/{id}/cancel` — drop a still-fully-queued
-/// from the rebuild queue. Refuses `Running` / terminal
+/// DAG from the job queue. Refuses `Running` / terminal DAGs: an
-/// entries: an in-flight rebuild owns the agent's nix store +
+/// in-flight node owns the agent's nix store + nixos-container update
-/// nixos-container update lock and can't be safely interrupted
+/// lock and can't be safely interrupted from the queue side. Always
-/// from the queue side. Always returns 200; the body is
+/// returns 200; the body is `{"cancelled": true}` on a successful
-/// `{"cancelled": true}` on a successful flip from Queued →
+/// flip to Cancelled, `{"cancelled": false}` when the DAG was
-/// Cancelled, `{"cancelled": false}` when the row was Running /
+/// Running / terminal / gone. On success a fresh `RebuildQueueChanged`
-/// terminal / gone. On success a fresh `RebuildQueueChanged`
+/// snapshot fires so the state flip surfaces live.
-/// snapshot fires so the row's state flip surfaces live.
 pub(super) async fn post_rebuild_queue_cancel(
     State(state): State<AppState>,
     AxumPath(id): AxumPath<u64>,
 ) -> Response {
-    let cancelled = state.coord.rebuild_queue.cancel(id);
+    let cancelled = state.coord.job_queue.cancel(id);
     if cancelled {
         state.coord.emit_rebuild_queue_snapshot();
         axum::Json(serde_json::json!({"cancelled": true})).into_response()

hive-c0re/src/dashboard_events.rs

@@ -8,7 +8,7 @@ use serde::Serialize;
 
 use crate::container_view::ContainerView;
 use crate::dashboard::{MetaInputView, TombstoneView};
-use crate::rebuild_queue::QueueEntry;
+use crate::job_queue::DagView;
 use chrono::{DateTime, Utc};
 
 #[derive(Debug, Clone, Serialize)]
@@ -210,7 +210,7 @@ pub enum DashboardEvent {
     /// the add/remove races a per-row event would have, and the
     /// dashboard's grouping (`parent_id`) is most naturally re-derived
     /// from the full list.
-    RebuildQueueChanged { seq: u64, queue: Vec<QueueEntry> },
+    RebuildQueueChanged { seq: u64, queue: Vec<DagView> },
     /// Full snapshot of all scheduled prompts. Emitted after every
     /// operator mutation (new / edit / cancel / fire-now) and after the
     /// worker fires or rearms a row. Same snapshot-shape rationale as

hive-c0re/src/job_queue/exec.rs

@@ -0,0 +1,433 @@
+//! Node executors — one async fn per [`NodeKind`], each a thin wrapper
+//! over existing `lifecycle.rs` / `meta.rs` / `actions.rs` code. Node
+//! executors keep their own internal error handling where it exists
+//! today (cold-start fallback inside `Reconcile`, non-fatal boot-time
+//! lock bump inside the sweep `MetaLock`, warn-only forge sync in the
+//! `Swap` tail); DAG-level failure handling is cancel-downstream in
+//! the queue.
+
+use std::sync::Arc;
+
+use anyhow::{Context as _, Result};
+
+use super::model::{NodeKind, State, Template};
+use super::{Claim, TerminalDag};
+use crate::coordinator::Coordinator;
+use crate::power::{ReconcileAction, reconcile_action};
+
+/// Max time `Drain` waits for the harness to run its stop-checkpoint
+/// turn before falling back to the hard stop. Generous — a checkpoint
+/// turn can take a while — but bounded so a wedged agent never blocks
+/// the stop indefinitely. Drains hold no build slot, so a whole-hive
+/// graceful stop overlaps every agent's drain instead of serialising
+/// N × this timeout.
+pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(3);
+
+/// Extra signal an executor hands back to the scheduler alongside
+/// success.
+#[derive(Debug, Default)]
+pub struct NodeOutput {
+    /// Agents to fan child `Rebuild` DAGs out for (`MetaLock` only).
+    pub fanout: Vec<String>,
+}
+
+/// Step-label + build-log sink for one claimed node.
+struct Ctx<'a> {
+    coord: &'a Arc<Coordinator>,
+    dag_id: u64,
+    node_id: super::NodeId,
+}
+
+impl Ctx<'_> {
+    fn step(&self, step: &str) {
+        if self
+            .coord
+            .job_queue
+            .set_step(self.dag_id, self.node_id, step)
+        {
+            self.coord.emit_rebuild_queue_snapshot();
+        }
+    }
+
+    fn build_log(&self, log_id: i64) {
+        if self
+            .coord
+            .job_queue
+            .set_build_log_id(self.dag_id, self.node_id, log_id)
+        {
+            self.coord.emit_rebuild_queue_snapshot();
+        }
+    }
+}
+
+/// Run one claimed node to completion. Called from a task the
+/// scheduler spawns per claim; the `Result` (stringified) becomes the
+/// node's terminal state.
+pub(super) async fn run_node(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
+    let ctx = Ctx {
+        coord,
+        dag_id: claim.dag_id,
+        node_id: claim.node_id,
+    };
+    match &claim.kind {
+        NodeKind::Prebuild { relock } => run_prebuild(coord, claim, &ctx, *relock).await,
+        NodeKind::Swap => run_swap(coord, claim, &ctx).await,
+        NodeKind::Create => run_create(coord, claim, &ctx).await,
+        NodeKind::MetaLock { sweep, fanout } => {
+            run_meta_lock(coord, claim, &ctx, *sweep, fanout.clone()).await
+        }
+        NodeKind::Reconcile => run_reconcile(coord, claim, &ctx).await,
+        NodeKind::StopForUpdate => run_stop_for_update(coord, claim, &ctx).await,
+        NodeKind::Signal => Ok(run_signal(coord, claim, &ctx)),
+        NodeKind::Drain => run_drain(coord, claim, &ctx).await,
+        NodeKind::WriteDropin => run_write_dropin(coord, claim).await,
+        NodeKind::WritePermFile => run_write_perm_file(coord, claim, &ctx).await,
+        NodeKind::ApprovalDeploy => run_approval_deploy(coord, claim).await,
+    }
+}
+
+/// Out-of-band toplevel build while the container keeps serving: meta
+/// sync + optional per-agent relock, then warm
+/// `system.build.toplevel` so the later `Swap` hits cache and skips
+/// straight to the profile-swap.
+async fn run_prebuild(
+    coord: &Arc<Coordinator>,
+    claim: &Claim,
+    ctx: &Ctx<'_>,
+    relock: bool,
+) -> Result<NodeOutput> {
+    let name = &claim.agent;
+    let agent_dir = coord
+        .ensure_runtime(name)
+        .with_context(|| format!("ensure_runtime {name}"))?;
+    let hive = coord.hive_env();
+    let paths = Coordinator::agent_paths(name, agent_dir);
+    crate::lifecycle::prepare_rebuild_dirs(name, &paths).await?;
+    // Idempotent meta sync so a manual rebuild can also recover from a
+    // divergent meta repo; then bump just this agent's input. `relock =
+    // false` only for meta-update cascade children, where re-locking
+    // would revert the bump the cascade just committed.
+    let agents = crate::lifecycle::agents_for_meta_listing().await?;
+    crate::meta::sync_agents(&hive, &agents).await?;
+    if relock {
+        crate::meta::lock_update_for_rebuild(name).await?;
+    }
+    ctx.step("nix build");
+    let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display());
+    crate::lifecycle::prebuild_toplevel(name, &flake_ref, &|log_id| ctx.build_log(log_id)).await?;
+    Ok(NodeOutput::default())
+}
+
+/// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb),
+/// `nixos-container update`, then the post-rebuild bookkeeping tail
+/// (rev marker, `Rebuilt` event, forge/matrix sync, kick, rescan).
+/// The recovery-start on failure is NOT here — the DAG's tail
+/// `Reconcile` runs after this node terminal ok *or* fail.
+async fn run_swap(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
+    let name = &claim.agent;
+    let agent_dir = coord.ensure_runtime(name)?;
+    let hive = coord.hive_env();
+    let paths = Coordinator::agent_paths(name, agent_dir);
+    let result =
+        crate::lifecycle::swap_update(name, &hive, &paths, &|step| ctx.step(step), &|log_id| {
+            ctx.build_log(log_id)
+        })
+        .await;
+    match &result {
+        Ok(()) => {
+            if let Some(rev) = crate::auto_update::current_flake_rev(&coord.hyperhive_flake)
+                && let Err(e) = std::fs::write(crate::auto_update::rev_marker_path(name), rev)
+            {
+                tracing::warn!(%name, error = ?e, "write rev marker failed");
+            }
+            coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
+                agent: name.clone(),
+                ok: true,
+                note: None,
+                sha: None,
+                tag: None,
+            });
+            ctx.step("forge sync");
+            // Full forge + matrix sync on every successful rebuild so
+            // the rebuild path is equivalent to the startup sweep:
+            // tokens, config-repo mirror, meta access all recover
+            // without a hive-c0re restart.
+            crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await;
+            crate::matrix::sync_agent_standalone(name).await;
+            // Wake the agent on its next turn so claude sees a "you
+            // were rebuilt" hint; rescan so dashboards drop the
+            // "needs update" chip; lock bump → meta-inputs re-render.
+            coord.kick_agent(name, "container rebuilt");
+            coord.rescan_containers_and_emit().await;
+            crate::dashboard::emit_meta_inputs_snapshot(coord);
+        }
+        Err(_) => {
+            // The `Rebuilt { ok: false }` manager event fires once per
+            // DAG from the terminal hook (any node may be the one that
+            // failed); here only refresh the observed state.
+            coord.rescan_containers_and_emit().await;
+        }
+    }
+    result.map(|()| NodeOutput::default())
+}
+
+/// First-spawn provisioning + `nixos-container create` (atomic
+/// build+create — no prebuild needed).
+async fn run_create(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
+    let name = &claim.agent;
+    let agent_dir = coord.ensure_runtime(name)?;
+    let hive = coord.hive_env();
+    let paths = Coordinator::agent_paths(name, agent_dir);
+    ctx.step("nixos-container create");
+    crate::lifecycle::create_container(name, &hive, &paths).await?;
+    Ok(NodeOutput::default())
+}
+
+/// Meta flake lock bump. Boot-sweep flavour is non-fatal (a failed
+/// bump must not cancel the fan-out rebuilds — they proceed against
+/// the current lock, exactly like today's sweep); the meta-update
+/// flavour propagates errors, and a failed bump fans out nothing.
+async fn run_meta_lock(
+    coord: &Arc<Coordinator>,
+    claim: &Claim,
+    ctx: &Ctx<'_>,
+    sweep: bool,
+    fanout: Option<Vec<String>>,
+) -> Result<NodeOutput> {
+    if sweep {
+        ctx.step("nix flake update hyperhive");
+        if let Err(e) = crate::meta::lock_update_hyperhive().await {
+            tracing::warn!(error = ?e, "startup sweep: meta lock_update_hyperhive failed");
+        }
+        return Ok(NodeOutput {
+            fanout: fanout.unwrap_or_default(),
+        });
+    }
+    let _progress = coord.meta_update_guard();
+    ctx.step("nix flake update");
+    crate::meta::lock_update(&claim.inputs).await?;
+    // Lock file changed — meta-inputs panel re-renders.
+    crate::dashboard::emit_meta_inputs_snapshot(coord);
+    let cascade = match fanout {
+        Some(list) => list,
+        None => meta_update_cascade_agents(&claim.inputs).await,
+    };
+    Ok(NodeOutput { fanout: cascade })
+}
+
+/// Idempotent power converge: `wanted` (durable intent) vs observed.
+async fn run_reconcile(
+    coord: &Arc<Coordinator>,
+    claim: &Claim,
+    ctx: &Ctx<'_>,
+) -> Result<NodeOutput> {
+    let name = &claim.agent;
+    let running = crate::lifecycle::is_running(name).await;
+    let wanted = coord.power.get_or_seed(name, running)?;
+    match reconcile_action(wanted, running) {
+        ReconcileAction::Start => {
+            // Node-local transient only when the DAG holds none (the
+            // boot-reconcile template); lease-window guards otherwise
+            // already cover this node.
+            let _guard = claim
+                .transient
+                .is_none()
+                .then(|| coord.transient_guard(name, crate::coordinator::TransientKind::Starting));
+            ctx.step("nixos-container start");
+            crate::lifecycle::start_with_fallback(name).await?;
+            coord.kick_agent(name, "container started");
+            coord.rescan_containers_and_emit().await;
+        }
+        ReconcileAction::Stop => {
+            let _guard = claim
+                .transient
+                .is_none()
+                .then(|| coord.transient_guard(name, crate::coordinator::TransientKind::Stopping));
+            ctx.step("nixos-container stop");
+            crate::lifecycle::kill(name).await?;
+            coord.unregister_agent(name);
+            coord.notify_manager(&hive_sh4re::HelperEvent::Killed {
+                agent: name.clone(),
+            });
+            coord.rescan_containers_and_emit().await;
+        }
+        ReconcileAction::Noop => {
+            tracing::debug!(%name, wanted = wanted.as_str(), running, "reconcile: noop");
+        }
+    }
+    Ok(NodeOutput::default())
+}
+
+/// Mechanical stop for the profile swap. Never touches `wanted`; noop
+/// when already stopped.
+async fn run_stop_for_update(
+    coord: &Arc<Coordinator>,
+    claim: &Claim,
+    ctx: &Ctx<'_>,
+) -> Result<NodeOutput> {
+    let name = &claim.agent;
+    if crate::lifecycle::is_running(name).await {
+        ctx.step("nixos-container stop");
+        crate::lifecycle::kill(name).await?;
+        coord.rescan_containers_and_emit().await;
+    }
+    Ok(NodeOutput::default())
+}
+
+/// Set the graceful fence + kick so the harness sees it promptly and
+/// runs its one stop-checkpoint turn.
+fn run_signal(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> NodeOutput {
+    ctx.step("graceful stop: signalling agent");
+    coord.mark_graceful_stop(&claim.agent);
+    coord.kick_agent(&claim.agent, "graceful stop requested");
+    NodeOutput::default()
+}
+
+/// Await the harness clearing the fence (`GracefulStopComplete`) or
+/// the timeout — either way the downstream `Reconcile` proceeds with
+/// the actual stop.
+async fn run_drain(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
+    let name = &claim.agent;
+    ctx.step("graceful stop: draining");
+    let deadline = std::time::Instant::now() + GRACEFUL_STOP_TIMEOUT;
+    while coord.is_graceful_stop_pending(name) {
+        if std::time::Instant::now() >= deadline {
+            tracing::warn!(agent = %name, "graceful stop: drain timed out — hard-stopping");
+            break;
+        }
+        tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+    }
+    coord.clear_graceful_stop(name);
+    Ok(NodeOutput::default())
+}
+
+/// `set_nspawn_flags` + `set_resource_limits` + daemon-reload.
+async fn run_write_dropin(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
+    let name = &claim.agent;
+    let agent_dir = coord.ensure_runtime(name)?;
+    let hive = coord.hive_env();
+    let paths = Coordinator::agent_paths(name, agent_dir);
+    crate::lifecycle::write_dropins(name, &hive, &paths).await?;
+    Ok(NodeOutput::default())
+}
+
+/// Write + commit the perm file(s) (fused under `META_LOCK` so the
+/// working tree is never left dirty), then emit the P3RM1SS10NS-tab
+/// snapshots so the dashboard reflects the new assignment.
+async fn run_write_perm_file(
+    coord: &Arc<Coordinator>,
+    claim: &Claim,
+    ctx: &Ctx<'_>,
+) -> Result<NodeOutput> {
+    use super::model::PermPayload;
+    let name = &claim.agent;
+    ctx.step("writing + committing perm file");
+    match &claim.perm_payload {
+        Some(PermPayload::ToolGroups { groups }) => {
+            crate::meta::commit_tool_groups(name, groups)
+                .await
+                .with_context(|| format!("commit tool-groups for {name}"))?;
+            coord.emit_tool_groups_snapshot();
+        }
+        Some(PermPayload::Capabilities { caps }) => {
+            crate::meta::commit_capabilities(name, caps)
+                .await
+                .with_context(|| format!("commit capabilities for {name}"))?;
+            coord.emit_capabilities_snapshot();
+        }
+        Some(PermPayload::Combined { groups, caps }) => {
+            crate::meta::commit_perms(name, groups.as_deref(), caps.as_deref())
+                .await
+                .with_context(|| format!("commit perms for {name}"))?;
+            if groups.is_some() {
+                coord.emit_tool_groups_snapshot();
+            }
+            if caps.is_some() {
+                coord.emit_capabilities_snapshot();
+            }
+        }
+        None => anyhow::bail!(
+            "perm_change dag {} for {name} is missing perm_payload",
+            claim.dag_id
+        ),
+    }
+    Ok(NodeOutput::default())
+}
+
+/// Opaque approval deploy pipeline: `ApplyCommit` and `MergeConfigPr`
+/// both end in a container rebuild; branch on the approval row's kind
+/// (the authoritative source). The two-phase prepare/finalize/abort
+/// meta deploy — and the approval resolution — stay inside
+/// `actions.rs` in v1 (design doc §9).
+async fn run_approval_deploy(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
+    let approval_id = claim
+        .approval_id
+        .with_context(|| format!("approval_deploy dag {} has no approval_id", claim.dag_id))?;
+    let kind = coord
+        .approvals
+        .get(approval_id)
+        .ok()
+        .flatten()
+        .map(|a| a.kind);
+    let result = if kind == Some(hive_sh4re::ApprovalKind::MergeConfigPr) {
+        crate::actions::run_approval_merge_config_pr(coord, Some(claim.dag_id), approval_id).await
+    } else {
+        crate::actions::run_approval_apply_commit(coord, Some(claim.dag_id), approval_id).await
+    };
+    result.map(|()| NodeOutput::default())
+}
+
+/// Terminal-roll-up hook, fired exactly once per DAG. Approval DAGs
+/// resolve their approval row (except the opaque deploy pipeline,
+/// which resolves inside its node); non-approval rebuild-shaped DAGs
+/// surface the `Rebuilt { ok: false }` manager event on failure —
+/// success fires from the `Swap` tail, matching today's timing.
+pub(super) async fn on_dag_terminal(coord: &Arc<Coordinator>, terminal: &TerminalDag) {
+    if terminal.approval_id.is_some() {
+        crate::actions::resolve_approval_dag(coord, terminal).await;
+        return;
+    }
+    if matches!(terminal.template, Template::Rebuild | Template::PermChange)
+        && terminal.state == State::Failed
+    {
+        coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
+            agent: terminal.agent.clone(),
+            ok: false,
+            note: terminal.error.clone(),
+            sha: None,
+            tag: None,
+        });
+    }
+}
+
+/// Compute which agents a `nix flake update <inputs>` on the meta
+/// flake affects — the fan-out set for `MetaUpdate` DAGs. Empty
+/// `inputs` or any input under `hyperhive` → every container;
+/// otherwise just the agents named by `agent-<name>` inputs.
+/// Topology-sorted so parents rebuild before their children.
+pub async fn meta_update_cascade_agents(inputs: &[String]) -> Vec<String> {
+    let touched_hyperhive = inputs
+        .iter()
+        .any(|i| i == "hyperhive" || i.starts_with("hyperhive/"));
+    let touched_agents: Vec<String> = inputs
+        .iter()
+        .filter_map(|i| i.strip_prefix("agent-"))
+        .map(|rest| rest.split('/').next().unwrap_or(rest).to_owned())
+        .collect();
+    let mut names = if touched_hyperhive || inputs.is_empty() {
+        crate::lifecycle::list()
+            .await
+            .unwrap_or_default()
+            .into_iter()
+            .filter_map(|c| {
+                c.strip_prefix(crate::lifecycle::AGENT_PREFIX)
+                    .map(str::to_owned)
+            })
+            .collect()
+    } else {
+        touched_agents
+    };
+    let topo = crate::topology::read();
+    crate::auto_update::topology_sort(&mut names, &topo);
+    names
+}

hive-c0re/src/job_queue/mod.rs

@@ -0,0 +1,565 @@
+//! Generic job-DAG queue + desired-state reconciliation — replaces the
+//! old flat `rebuild_queue`. Jobs are nodes in per-request DAGs (see
+//! [`templates`]); the special cases (graceful-stop watcher thread,
+//! deferred-start follow-up, meta-update cascade) collapse into DAG
+//! *shapes* over a shared set of primitive nodes ([`model::NodeKind`]).
+//!
+//! Concurrency is gated by two resource classes:
+//! 1. **Build slots** — N permits (`services.hyperhive.c0re.buildSlots`,
+//!    default 1) held by nix-heavy nodes for the node's duration.
+//! 2. **Per-agent lifecycle lease** — DAG-scoped: acquired before the
+//!    DAG's first container-affecting node runs, held until the DAG is
+//!    terminal, so two lifecycle DAGs for one agent never interleave
+//!    their container ops.
+//!
+//! The meta *repo* is serialized by `meta::META_LOCK` inside the
+//! executors themselves. Per-agent power *intent* (`wanted`) lives in
+//! the durable [`crate::power`] store; the DAGs are the reconcile
+//! mechanism. Design + rationale: `docs/coordinator.md::Job queue`.
+
+pub mod exec;
+pub mod model;
+pub mod scheduler;
+pub mod submit;
+pub mod templates;
+#[cfg(test)]
+mod tests;
+
+use std::collections::{HashMap, VecDeque};
+use std::sync::Mutex;
+
+use tokio::sync::Notify;
+
+pub use model::{
+    Dag, DagSpec, DagView, DepWhen, Node, NodeId, NodeKind, PermPayload, Source, State, Template,
+};
+
+/// How many terminal DAGs (`Done` / `Failed` / `Cancelled`) to retain
+/// per template in the snapshot, matching the old per-kind history cap.
+const MAX_HISTORY_PER_TEMPLATE: usize = 5;
+
+/// Cap on stored node error strings.
+const MAX_ERROR_LEN: usize = 2_000;
+
+/// A node claimed for execution — everything the executor needs,
+/// snapshotted at claim time.
+#[derive(Debug, Clone)]
+pub struct Claim {
+    pub dag_id: u64,
+    pub node_id: NodeId,
+    pub kind: NodeKind,
+    pub agent: String,
+    pub template: Template,
+    pub source: Source,
+    pub approval_id: Option<i64>,
+    pub inputs: Vec<String>,
+    pub perm_payload: Option<PermPayload>,
+    /// True when claiming this node acquired the DAG's agent lease —
+    /// the scheduler creates the DAG-scoped transient guard on this
+    /// edge.
+    pub lease_acquired: bool,
+    /// Transient pill kind for the lease window (from the spec).
+    pub transient: Option<crate::coordinator::TransientKind>,
+}
+
+/// Summary of a DAG that just reached its terminal roll-up state —
+/// input to the approval-resolution hook and the lease/transient
+/// release.
+#[derive(Debug, Clone)]
+pub struct TerminalDag {
+    pub dag_id: u64,
+    pub template: Template,
+    pub agent: String,
+    pub approval_id: Option<i64>,
+    pub state: State,
+    /// First failed node's error when `state == Failed`.
+    pub error: Option<String>,
+}
+
+/// Report from [`JobQueue::complete_node`].
+#[derive(Debug, Default)]
+pub struct CompletionReport {
+    /// DAGs that became terminal as a result of this completion
+    /// (the completed node's own DAG, plus none others — but kept as a
+    /// Vec so cancel paths can reuse the same settle plumbing).
+    pub terminal: Vec<TerminalDag>,
+}
+
+#[derive(Debug, Default)]
+struct Inner {
+    dags: VecDeque<Dag>,
+    next_id: u64,
+    build_slots: usize,
+    slots_used: usize,
+    /// agent → dag id currently holding that agent's lifecycle lease.
+    leases: HashMap<String, u64>,
+}
+
+/// The queue. Lives on `Coordinator` (one per hive-c0re process); a
+/// single scheduler task ([`scheduler::run_worker`]) drives it —
+/// concurrency comes from the build-slot count, not multiple workers.
+#[derive(Debug)]
+pub struct JobQueue {
+    inner: Mutex<Inner>,
+    /// Wakes the scheduler when something new arrives or state changed.
+    pub(crate) notify: Notify,
+}
+
+impl Default for JobQueue {
+    fn default() -> Self {
+        Self::new(1)
+    }
+}
+
+impl JobQueue {
+    pub fn new(build_slots: usize) -> Self {
+        Self {
+            inner: Mutex::new(Inner {
+                build_slots: build_slots.max(1),
+                ..Inner::default()
+            }),
+            notify: Notify::new(),
+        }
+    }
+
+    /// Submit a DAG. Validates the spec (cycle rejection) and dedups
+    /// against non-started DAGs; returns the DAG id (newly-allocated,
+    /// or the existing DAG's id with the new reason appended).
+    ///
+    /// Dedup: a DAG whose roll-up is still `Queued` (no node started)
+    /// with the same `(template, agent, parent_id, approval_id)` — plus
+    /// `inputs` for `MetaUpdate` and the perm-type discriminant for
+    /// `PermChange` — swallows the repeat. `parent_id` is part of the
+    /// key so a meta-update cascade rebuild never collapses into a
+    /// standalone or sweep rebuild. Running / terminal DAGs never
+    /// dedup — operators are free to re-queue.
+    pub fn submit(&self, spec: DagSpec) -> anyhow::Result<u64> {
+        templates::validate(&spec)?;
+        let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
+        if let Some(existing) = Self::dedup_target(&mut inner, &spec) {
+            if !existing.reason.contains(&spec.reason) {
+                use std::fmt::Write as _;
+                let _ = write!(existing.reason, "\nalso requested by: {}", spec.reason);
+            }
+            return Ok(existing.id);
+        }
+        let id = Self::push_dag(&mut inner, spec);
+        drop(inner);
+        self.notify.notify_one();
+        Ok(id)
+    }
+
+    /// Append fan-out children under a parent DAG (meta-update / sweep
+    /// cascade). Applies the same dedup as [`Self::submit`]; returns
+    /// the child ids actually created or coalesced into.
+    pub fn append_children(&self, specs: Vec<DagSpec>) -> Vec<u64> {
+        let mut ids = Vec::with_capacity(specs.len());
+        for spec in specs {
+            match self.submit(spec) {
+                Ok(id) => ids.push(id),
+                Err(e) => tracing::error!(error = ?e, "job_queue: invalid fan-out child spec"),
+            }
+        }
+        ids
+    }
+
+    fn dedup_target<'a>(inner: &'a mut Inner, spec: &DagSpec) -> Option<&'a mut Dag> {
+        inner.dags.iter_mut().find(|d| {
+            d.rollup() == State::Queued
+                && d.template == spec.template
+                && d.agent == spec.agent
+                && d.parent_id == spec.parent_id
+                && d.approval_id == spec.approval_id
+                && (d.template != Template::MetaUpdate || d.inputs == spec.inputs)
+                && PermPayload::same_type(d.perm_payload.as_ref(), spec.perm_payload.as_ref())
+        })
+    }
+
+    fn push_dag(inner: &mut Inner, spec: DagSpec) -> u64 {
+        inner.next_id += 1;
+        let id = inner.next_id;
+        let nodes = spec
+            .nodes
+            .into_iter()
+            .enumerate()
+            .map(|(i, n)| Node {
+                id: u32::try_from(i).unwrap_or(u32::MAX),
+                kind: n.kind,
+                deps: n.deps,
+                state: State::Queued,
+                step: None,
+                build_log_id: None,
+                started_at: None,
+                finished_at: None,
+                error: None,
+            })
+            .collect();
+        inner.dags.push_back(Dag {
+            id,
+            template: spec.template,
+            agent: spec.agent,
+            source: spec.source,
+            reason: spec.reason,
+            parent_id: spec.parent_id,
+            approval_id: spec.approval_id,
+            inputs: spec.inputs,
+            perm_payload: spec.perm_payload,
+            transient: spec.transient,
+            created_at: now_unix(),
+            nodes,
+            terminal_reported: false,
+        });
+        id
+    }
+
+    /// Claim every currently-ready node, acquiring resources, and mark
+    /// them `Running`. A node is ready when it's `Queued`, every dep is
+    /// satisfied (`AfterOk`: dep `Done`; `AfterAny`: dep terminal), and
+    /// its resources are free (build slot; agent lease free or already
+    /// held by this DAG). Iteration is in DAG-submit order, so
+    /// simultaneously-ready nodes compete FIFO — bulk operations drain
+    /// predictably.
+    pub fn claim_ready(&self) -> Vec<Claim> {
+        let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
+        Self::propagate_cancellations(&mut inner);
+        let mut claims = Vec::new();
+        let inner = &mut *inner;
+        for di in 0..inner.dags.len() {
+            // Split-borrow dance: deps are checked against the same
+            // DAG's other nodes, so snapshot the states first.
+            let dag = &inner.dags[di];
+            let dag_id = dag.id;
+            let ready_ids: Vec<NodeId> = dag
+                .nodes
+                .iter()
+                .filter(|n| n.state == State::Queued && Self::deps_satisfied(dag, n))
+                .map(|n| n.id)
+                .collect();
+            for node_id in ready_ids {
+                let dag = &inner.dags[di];
+                let node = dag.node(node_id).expect("node id from same dag");
+                let needs_slot = node.kind.needs_build_slot();
+                if needs_slot && inner.slots_used >= inner.build_slots {
+                    continue;
+                }
+                let mut lease_acquired = false;
+                if node.kind.needs_lease() {
+                    match inner.leases.get(dag.agent.as_str()) {
+                        Some(&holder) if holder != dag_id => continue,
+                        Some(_) => {}
+                        None => {
+                            inner.leases.insert(dag.agent.clone(), dag_id);
+                            lease_acquired = true;
+                        }
+                    }
+                }
+                if needs_slot {
+                    inner.slots_used += 1;
+                }
+                let dag = &mut inner.dags[di];
+                let claim = Claim {
+                    dag_id,
+                    node_id,
+                    kind: dag.node(node_id).expect("node").kind.clone(),
+                    agent: dag.agent.clone(),
+                    template: dag.template,
+                    source: dag.source,
+                    approval_id: dag.approval_id,
+                    inputs: dag.inputs.clone(),
+                    perm_payload: dag.perm_payload.clone(),
+                    lease_acquired,
+                    transient: dag.transient,
+                };
+                let node = dag.node_mut(node_id).expect("node");
+                node.state = State::Running;
+                node.started_at = Some(now_unix());
+                claims.push(claim);
+            }
+        }
+        claims
+    }
+
+    fn deps_satisfied(dag: &Dag, node: &Node) -> bool {
+        node.deps.iter().all(|dep| {
+            dag.node(dep.on).is_some_and(|d| match dep.when {
+                DepWhen::AfterOk => d.state == State::Done,
+                DepWhen::AfterAny => d.state.is_terminal(),
+            })
+        })
+    }
+
+    /// Cancel-downstream: a `Queued` node with an `AfterOk` dep that
+    /// `Failed` / `Cancelled` becomes `Cancelled` itself. Loops to a
+    /// fixpoint so the cancellation cascades through chains.
+    fn propagate_cancellations(inner: &mut Inner) {
+        for dag in &mut inner.dags {
+            loop {
+                let doomed: Vec<NodeId> = dag
+                    .nodes
+                    .iter()
+                    .filter(|n| {
+                        n.state == State::Queued
+                            && n.deps.iter().any(|dep| {
+                                dep.when == DepWhen::AfterOk
+                                    && dag.node(dep.on).is_some_and(|d| {
+                                        matches!(d.state, State::Failed | State::Cancelled)
+                                    })
+                            })
+                    })
+                    .map(|n| n.id)
+                    .collect();
+                if doomed.is_empty() {
+                    break;
+                }
+                let now = now_unix();
+                for id in doomed {
+                    if let Some(n) = dag.node_mut(id) {
+                        n.state = State::Cancelled;
+                        n.finished_at = Some(now);
+                    }
+                }
+            }
+        }
+    }
+
+    /// Mark a claimed node terminal, release its build slot, cascade
+    /// cancellations, and settle terminal DAGs (lease release + history
+    /// trim). `error` is stored (truncated) when `result` is `Err`.
+    pub fn complete_node(
+        &self,
+        dag_id: u64,
+        node_id: NodeId,
+        result: Result<(), String>,
+    ) -> CompletionReport {
+        let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
+        if let Some(dag) = inner.dags.iter_mut().find(|d| d.id == dag_id)
+            && let Some(node) = dag.node_mut(node_id)
+            && node.state == State::Running
+        {
+            let needs_slot = node.kind.needs_build_slot();
+            node.finished_at = Some(now_unix());
+            node.step = None;
+            match result {
+                Ok(()) => node.state = State::Done,
+                Err(e) => {
+                    node.state = State::Failed;
+                    let mut msg = e;
+                    if msg.len() > MAX_ERROR_LEN {
+                        msg.truncate(
+                            (0..=MAX_ERROR_LEN)
+                                .rev()
+                                .find(|i| msg.is_char_boundary(*i))
+                                .unwrap_or(0),
+                        );
+                        msg.push('…');
+                    }
+                    node.error = Some(msg);
+                }
+            }
+            if needs_slot {
+                inner.slots_used = inner.slots_used.saturating_sub(1);
+            }
+        }
+        let report = Self::settle(&mut inner);
+        drop(inner);
+        self.notify.notify_one();
+        report
+    }
+
+    /// Propagate cancellations, release the leases of newly-terminal
+    /// DAGs, and trim history. Each terminal DAG is reported exactly
+    /// once (the `terminal_reported` flag) so the scheduler's hooks —
+    /// approval resolution, transient-guard release — fire once per
+    /// DAG.
+    fn settle(inner: &mut Inner) -> CompletionReport {
+        Self::propagate_cancellations(inner);
+        let mut report = CompletionReport::default();
+        let mut freed: Vec<String> = Vec::new();
+        for dag in &mut inner.dags {
+            if !dag.is_terminal() || dag.terminal_reported {
+                continue;
+            }
+            dag.terminal_reported = true;
+            if inner.leases.get(dag.agent.as_str()) == Some(&dag.id) {
+                freed.push(dag.agent.clone());
+            }
+            report.terminal.push(TerminalDag {
+                dag_id: dag.id,
+                template: dag.template,
+                agent: dag.agent.clone(),
+                approval_id: dag.approval_id,
+                state: dag.rollup(),
+                error: dag.first_error().map(str::to_owned),
+            });
+        }
+        for agent in freed {
+            inner.leases.remove(&agent);
+        }
+        Self::trim_history(inner);
+        report
+    }
+
+    /// Cancel a DAG that hasn't started yet (roll-up `Queued`): every
+    /// node flips to `Cancelled`. No-op (returns `false`) once any node
+    /// is running or terminal — an in-flight nix build isn't
+    /// interruptible, matching the old queue's rule.
+    pub fn cancel(&self, dag_id: u64) -> bool {
+        let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
+        let Some(dag) = inner.dags.iter_mut().find(|d| d.id == dag_id) else {
+            return false;
+        };
+        if dag.rollup() != State::Queued {
+            return false;
+        }
+        let now = now_unix();
+        for n in &mut dag.nodes {
+            n.state = State::Cancelled;
+            n.finished_at = Some(now);
+        }
+        let _ = Self::settle(&mut inner);
+        drop(inner);
+        self.notify.notify_one();
+        true
+    }
+
+    /// Cancel every still-fully-queued child DAG of `parent`. Running
+    /// children are left alone. Returns the count of cancelled DAGs.
+    pub fn cancel_children(&self, parent: u64) -> usize {
+        let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
+        let now = now_unix();
+        let mut count = 0;
+        for dag in &mut inner.dags {
+            if dag.parent_id == Some(parent) && dag.rollup() == State::Queued {
+                for n in &mut dag.nodes {
+                    n.state = State::Cancelled;
+                    n.finished_at = Some(now);
+                }
+                count += 1;
+            }
+        }
+        if count > 0 {
+            let _ = Self::settle(&mut inner);
+            drop(inner);
+            self.notify.notify_one();
+        }
+        count
+    }
+
+    /// Set the step label on a `Running` node. Returns `true` when the
+    /// label actually changed (callers emit a snapshot only then).
+    pub fn set_step(&self, dag_id: u64, node_id: NodeId, step: &str) -> bool {
+        let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
+        let Some(node) = inner
+            .dags
+            .iter_mut()
+            .find(|d| d.id == dag_id)
+            .and_then(|d| d.node_mut(node_id))
+        else {
+            return false;
+        };
+        if node.state != State::Running || node.step.as_deref() == Some(step) {
+            return false;
+        }
+        node.step = Some(step.to_owned());
+        true
+    }
+
+    /// Set the step label on the DAG's currently-running node —
+    /// compatibility surface for the opaque approval pipeline, whose
+    /// callbacks only know the DAG id. Single-node approval DAGs make
+    /// this exact.
+    pub fn set_step_running(&self, dag_id: u64, step: &str) -> bool {
+        let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
+        let Some(node) = inner
+            .dags
+            .iter_mut()
+            .find(|d| d.id == dag_id)
+            .and_then(|d| d.nodes.iter_mut().find(|n| n.state == State::Running))
+        else {
+            return false;
+        };
+        if node.step.as_deref() == Some(step) {
+            return false;
+        }
+        node.step = Some(step.to_owned());
+        true
+    }
+
+    /// Link a `build_logs` row to a specific `Running` node.
+    pub fn set_build_log_id(&self, dag_id: u64, node_id: NodeId, log_id: i64) -> bool {
+        let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
+        let Some(node) = inner
+            .dags
+            .iter_mut()
+            .find(|d| d.id == dag_id)
+            .and_then(|d| d.node_mut(node_id))
+        else {
+            return false;
+        };
+        if node.state != State::Running {
+            return false;
+        }
+        node.build_log_id = Some(log_id);
+        true
+    }
+
+    /// Link a `build_logs` row to the DAG's currently-running node —
+    /// DAG-id-only compatibility surface (approval pipeline callbacks).
+    pub fn set_build_log_id_running(&self, dag_id: u64, log_id: i64) -> bool {
+        let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
+        let Some(node) = inner
+            .dags
+            .iter_mut()
+            .find(|d| d.id == dag_id)
+            .and_then(|d| d.nodes.iter_mut().find(|n| n.state == State::Running))
+        else {
+            return false;
+        };
+        node.build_log_id = Some(log_id);
+        true
+    }
+
+    /// Snapshot every DAG for `/api/state` + `RebuildQueueChanged`.
+    pub fn snapshot(&self) -> Vec<DagView> {
+        let inner = self.inner.lock().expect("job_queue mutex poisoned");
+        inner.dags.iter().map(Dag::view).collect()
+    }
+
+    /// Number of live (non-terminal) DAGs — used by tests and
+    /// diagnostics.
+    #[cfg(test)]
+    pub fn live_count(&self) -> usize {
+        let inner = self.inner.lock().expect("job_queue mutex poisoned");
+        inner.dags.iter().filter(|d| !d.is_terminal()).count()
+    }
+
+    /// Keep only the newest `MAX_HISTORY_PER_TEMPLATE` terminal DAGs
+    /// per template; live DAGs are never evicted.
+    fn trim_history(inner: &mut Inner) {
+        let mut counts: HashMap<Template, usize> = HashMap::new();
+        let kept: Vec<Dag> = inner
+            .dags
+            .iter()
+            .rev()
+            .filter(|d| {
+                if !d.is_terminal() {
+                    return true;
+                }
+                let n = counts.entry(d.template).or_insert(0);
+                *n += 1;
+                *n <= MAX_HISTORY_PER_TEMPLATE
+            })
+            .cloned()
+            .collect();
+        inner.dags = kept.into_iter().rev().collect();
+    }
+}
+
+/// Current unix timestamp in seconds.
+fn now_unix() -> i64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .ok()
+        .and_then(|d| i64::try_from(d.as_secs()).ok())
+        .unwrap_or(0)
+}

hive-c0re/src/job_queue/model.rs

@@ -0,0 +1,520 @@
+//! Data model for the generic job-DAG queue: templates (what a DAG
+//! *means*), node kinds (the primitive operations), dependency edges,
+//! states, and the wire-facing `DagView` / `NodeView` snapshot shapes.
+//!
+//! Two levels: the **DAG** is the unit of dedup / cancel /
+//! approval-resolution and the dashboard group; the **node** is the
+//! unit of scheduling / execution / build-log / step label. See
+//! `docs/coordinator.md::Job queue` for the full design.
+
+use serde::{Deserialize, Serialize};
+
+/// What a DAG *means* — the request-level shape. Wire strings match the
+/// old `QueueKind` values (serialized as the `kind` field on `DagView`)
+/// so the dashboard's glyph map keeps working.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Template {
+    /// Rebuild one agent's container: `Prebuild → StopForUpdate → Swap
+    /// → Reconcile` (the tail `Reconcile` runs after `Swap` terminal,
+    /// ok *or* fail — the recovery-start).
+    Rebuild,
+    /// Bump meta flake locks: one `MetaLock` node; child `Rebuild`
+    /// DAGs fan out on completion for every affected agent.
+    MetaUpdate,
+    /// First-deploy spawn (approval-driven): `Create → WriteDropin →
+    /// Reconcile`.
+    Spawn,
+    /// Reserved for a future destroy integration — kept so the wire
+    /// shape doesn't need to change later.
+    #[allow(dead_code, reason = "wire shape — routed by a future PR")]
+    Destroy,
+    /// Boot-time config sweep: one `MetaLock` (hyperhive input,
+    /// non-fatal) node; stale agents' `Rebuild` DAGs fan out on
+    /// completion.
+    StartupSweep,
+    /// `StopForUpdate → Reconcile` — stop + converge back to `wanted`
+    /// (unchanged), i.e. a restart for a wanted-up agent.
+    Restart,
+    /// `WritePermFile → Prebuild → StopForUpdate → Swap → Reconcile` —
+    /// perm-file commit followed by the rebuild subgraph.
+    PermChange,
+    /// `Signal → Drain → Reconcile` with `wanted` set to `Offline` at
+    /// submit time: quiesce the harness, await the drain (bounded),
+    /// then the tail `Reconcile` performs the actual container stop.
+    GracefulStop,
+    /// Single `Reconcile` with `wanted` set to `Up` at submit time.
+    Start,
+    /// Single `Reconcile` with `wanted` set to `Offline` at submit time.
+    Stop,
+    /// Single `Reconcile` with `wanted` untouched — boot-time converge
+    /// of observed state to the persisted intent.
+    Reconcile,
+}
+
+impl Template {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Template::Rebuild => "rebuild",
+            Template::MetaUpdate => "meta_update",
+            Template::Spawn => "spawn",
+            Template::Destroy => "destroy",
+            Template::StartupSweep => "startup_sweep",
+            Template::Restart => "restart",
+            Template::PermChange => "perm_change",
+            Template::GracefulStop => "graceful_stop",
+            Template::Start => "start",
+            Template::Stop => "stop",
+            Template::Reconcile => "reconcile",
+        }
+    }
+}
+
+/// Where the submit request originated. Same variants + wire strings
+/// as the old `QueueSource` — drives the "why" chip on the dashboard.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Source {
+    /// Operator action (dashboard button, CLI, manager tool).
+    Manual,
+    /// Cascade child of a `MetaUpdate` DAG's fan-out; `parent_id`
+    /// points back at the originating meta-update.
+    MetaUpdate,
+    /// Boot-time submission (sweep parent, boot reconciles).
+    AutoUpdate,
+    /// Cascade child of a `StartupSweep` DAG's fan-out.
+    StartupSweep,
+    /// Crash recovery path (future use).
+    #[allow(dead_code, reason = "wire shape — used by a future feature")]
+    CrashRecover,
+    /// Operator approved a pending `Approval` row; `approval_id` on
+    /// the DAG points back at the source row.
+    Approval,
+}
+
+impl Source {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Source::Manual => "manual",
+            Source::MetaUpdate => "meta_update",
+            Source::AutoUpdate => "auto_update",
+            Source::StartupSweep => "startup_sweep",
+            Source::CrashRecover => "crash_recover",
+            Source::Approval => "approval",
+        }
+    }
+}
+
+/// Lifecycle state of a node — and, rolled up, of a DAG. Same wire
+/// strings as the old `QueueState`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum State {
+    Queued,
+    Running,
+    Done,
+    Failed,
+    Cancelled,
+}
+
+impl State {
+    pub fn is_terminal(self) -> bool {
+        matches!(self, State::Done | State::Failed | State::Cancelled)
+    }
+}
+
+/// Kind-specific payload for `Template::PermChange` DAGs. Carried on
+/// the DAG (not the node) so dedup can compare the perm *type*
+/// discriminant. Identical to the old `rebuild_queue::PermPayload`.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum PermPayload {
+    /// Set the tool groups for one agent (`tool-groups.json`).
+    ToolGroups { groups: Vec<String> },
+    /// Set the capabilities for one agent (`capabilities.json`).
+    Capabilities { caps: Vec<String> },
+    /// Set both perm-types in one entry — the batch
+    /// `POST /api/permissions` path. `None` leaves that file untouched;
+    /// the worker commits whichever are present in one git commit,
+    /// then rebuilds once.
+    Combined {
+        groups: Option<Vec<String>>,
+        caps: Option<Vec<String>>,
+    },
+}
+
+impl PermPayload {
+    /// Dedup compares the perm *type*, not the value — a tool-groups
+    /// change and a capabilities change for the same agent are
+    /// distinct operations that must not collapse.
+    pub fn same_type(a: Option<&PermPayload>, b: Option<&PermPayload>) -> bool {
+        matches!(
+            (a, b),
+            (
+                Some(PermPayload::ToolGroups { .. }),
+                Some(PermPayload::ToolGroups { .. })
+            ) | (
+                Some(PermPayload::Capabilities { .. }),
+                Some(PermPayload::Capabilities { .. })
+            ) | (
+                Some(PermPayload::Combined { .. }),
+                Some(PermPayload::Combined { .. })
+            ) | (None, None)
+        )
+    }
+}
+
+/// Node id, unique within its DAG (dense small ints assigned by the
+/// template builders).
+pub type NodeId = u32;
+
+/// When a dependency edge is considered satisfied.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum DepWhen {
+    /// Dep must reach `Done`. A `Failed` / `Cancelled` dep cancels this
+    /// node (cancel-downstream).
+    AfterOk,
+    /// Dep must merely reach a terminal state (ok *or* fail). Used only
+    /// by `rebuild`'s tail `Reconcile` so the recovery-start runs even
+    /// when `Swap` failed.
+    AfterAny,
+}
+
+/// A dependency edge (intra-DAG only — cross-DAG ordering comes from
+/// the per-agent lease + dedup, never from edges between DAGs).
+#[derive(Debug, Clone, Copy, Serialize)]
+pub struct Dep {
+    pub on: NodeId,
+    pub when: DepWhen,
+}
+
+/// The primitive operations — each kind maps to one executor fn in
+/// `exec.rs`, a thin wrapper over existing `lifecycle.rs` / `meta.rs`
+/// code. Concurrency is gated by two resource classes (see
+/// [`NodeKind::needs_build_slot`] / [`NodeKind::needs_lease`]); the
+/// meta *repo* is serialized by `meta::META_LOCK` inside the wrapped
+/// functions themselves, which is why there is no `GitCommit` node —
+/// a standalone commit node would open a dirty-working-tree window
+/// between nodes that the fused `meta.rs` ops deliberately close.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(tag = "kind", rename_all = "snake_case")]
+pub enum NodeKind {
+    /// Out-of-band toplevel build while the container keeps serving:
+    /// meta `sync_agents`, optional per-agent relock, then
+    /// `lifecycle::prebuild_toplevel`. `relock = false` only for
+    /// meta-update cascade rebuilds (re-locking would revert the bump
+    /// the cascade just committed).
+    Prebuild { relock: bool },
+    /// `nixos-container update` profile-swap (requires the container
+    /// stopped). Re-applies nspawn flags + resource limits first —
+    /// rebuild is the reconcile verb — and carries the post-rebuild
+    /// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan).
+    Swap,
+    /// First-spawn `nixos-container create` plus the pre-create
+    /// provisioning (proposed/applied repos, state subvolume, meta
+    /// registration).
+    Create,
+    /// Meta flake lock bump. `sweep = false`: `meta::lock_update`
+    /// (commit fused, under `META_LOCK`) with the DAG's `inputs`;
+    /// `sweep = true`: `meta::lock_update_hyperhive`, *non-fatal* (a
+    /// failed boot-time bump must not cancel the fan-out rebuilds).
+    /// On success the scheduler appends child `Rebuild` DAGs: the
+    /// precomputed `fanout` list when present (boot sweep), else the
+    /// post-bump affected set (`meta_update_cascade_agents`).
+    MetaLock {
+        sweep: bool,
+        fanout: Option<Vec<String>>,
+    },
+    /// Idempotent power converge: read `wanted` + observed state;
+    /// start if `Up` & down (with cold-start fallback), stop if
+    /// `Offline` & up, else noop.
+    Reconcile,
+    /// Mechanical `nixos-container stop` for the profile swap. Never
+    /// touches `wanted`. Noop if already stopped.
+    StopForUpdate,
+    /// Set the graceful-stop fence + kick the harness so it runs one
+    /// stop-checkpoint turn.
+    Signal,
+    /// Await the harness clearing the fence, bounded by
+    /// `GRACEFUL_STOP_TIMEOUT`. Resolves ok either way — the
+    /// downstream `Reconcile` performs the actual stop.
+    Drain,
+    /// `set_nspawn_flags` + `set_resource_limits` + daemon-reload.
+    WriteDropin,
+    /// Commit `tool-groups.json` / `capabilities.json` per the DAG's
+    /// `perm_payload` (commit fused under `META_LOCK`).
+    WritePermFile,
+    /// Opaque approval deploy pipeline (`ApplyCommit` /
+    /// `MergeConfigPr`): the two-phase prepare/finalize/abort meta
+    /// deploy stays inside `actions.rs` in v1 — deliberately not
+    /// modeled as scheduler nodes (see the design doc §9).
+    ApprovalDeploy,
+}
+
+impl NodeKind {
+    /// Wire string for `NodeView.kind`.
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            NodeKind::Prebuild { .. } => "prebuild",
+            NodeKind::Swap => "swap",
+            NodeKind::Create => "create",
+            NodeKind::MetaLock { .. } => "meta_lock",
+            NodeKind::Reconcile => "reconcile",
+            NodeKind::StopForUpdate => "stop_for_update",
+            NodeKind::Signal => "signal",
+            NodeKind::Drain => "drain",
+            NodeKind::WriteDropin => "write_dropin",
+            NodeKind::WritePermFile => "write_perm_file",
+            NodeKind::ApprovalDeploy => "approval_deploy",
+        }
+    }
+
+    /// Nix-heavy kinds hold one of the `buildSlots` semaphore permits
+    /// for the node's duration.
+    pub fn needs_build_slot(&self) -> bool {
+        matches!(
+            self,
+            NodeKind::Prebuild { .. }
+                | NodeKind::Swap
+                | NodeKind::Create
+                | NodeKind::MetaLock { .. }
+                | NodeKind::ApprovalDeploy
+        )
+    }
+
+    /// Container-affecting kinds require the DAG to hold the agent's
+    /// lifecycle lease (acquired at the first such node, held until the
+    /// DAG is terminal). Lease-exempt kinds (`Prebuild`, `MetaLock`,
+    /// `WritePermFile`) touch the store / meta repo, not the running
+    /// container — which is exactly why a `Prebuild` can overlap
+    /// another DAG's work on the same agent.
+    pub fn needs_lease(&self) -> bool {
+        matches!(
+            self,
+            NodeKind::Swap
+                | NodeKind::Create
+                | NodeKind::Reconcile
+                | NodeKind::StopForUpdate
+                | NodeKind::Signal
+                | NodeKind::Drain
+                | NodeKind::WriteDropin
+                | NodeKind::ApprovalDeploy
+        )
+    }
+}
+
+/// One schedulable unit inside a DAG.
+#[derive(Debug, Clone)]
+pub struct Node {
+    pub id: NodeId,
+    pub kind: NodeKind,
+    pub deps: Vec<Dep>,
+    pub state: State,
+    /// Live sub-label while `Running` (kept for parity with the old
+    /// per-entry `step`).
+    pub step: Option<String>,
+    /// Row id of the `build_logs` entry this node opened (`Prebuild` /
+    /// `Swap` / `ApprovalDeploy`), for the dashboard's live-stream link.
+    pub build_log_id: Option<i64>,
+    pub started_at: Option<i64>,
+    pub finished_at: Option<i64>,
+    /// Populated when `state == Failed` (truncated by the queue).
+    pub error: Option<String>,
+}
+
+/// Submit-time spec for one node.
+#[derive(Debug, Clone)]
+pub struct NodeSpec {
+    pub kind: NodeKind,
+    pub deps: Vec<Dep>,
+}
+
+/// Submit-time spec for a whole DAG. Built by `templates.rs`; validated
+/// (cycle rejection) and dedup'd by `JobQueue::submit`.
+#[derive(Debug, Clone)]
+pub struct DagSpec {
+    pub template: Template,
+    /// Primary target agent, or `"hyperhive"` for meta-level DAGs.
+    pub agent: String,
+    pub source: Source,
+    /// Free-form "why"; dedup appends "also requested by …" lines.
+    pub reason: String,
+    /// Cascade grouping (meta-update / sweep children).
+    pub parent_id: Option<u64>,
+    /// Fires the approval-resolution hook on DAG terminal.
+    pub approval_id: Option<i64>,
+    /// `MetaUpdate`-only: the inputs to bump (also part of the dedup
+    /// key for that template). Display copy lives on the DAG.
+    pub inputs: Vec<String>,
+    /// `PermChange`-only payload.
+    pub perm_payload: Option<PermPayload>,
+    /// Dashboard transient pill (and crash-watch suppression) held for
+    /// the lease window — from lease acquisition to DAG terminal.
+    pub transient: Option<crate::coordinator::TransientKind>,
+    pub nodes: Vec<NodeSpec>,
+}
+
+/// A live DAG in the queue.
+#[derive(Debug, Clone)]
+pub struct Dag {
+    pub id: u64,
+    pub template: Template,
+    pub agent: String,
+    pub source: Source,
+    pub reason: String,
+    pub parent_id: Option<u64>,
+    pub approval_id: Option<i64>,
+    pub inputs: Vec<String>,
+    pub perm_payload: Option<PermPayload>,
+    pub transient: Option<crate::coordinator::TransientKind>,
+    pub created_at: i64,
+    pub nodes: Vec<Node>,
+    /// Terminal roll-up already reported to the scheduler's hooks
+    /// (approval resolution, transient release). Internal bookkeeping,
+    /// never serialized.
+    pub terminal_reported: bool,
+}
+
+impl Dag {
+    /// Roll-up state: `Failed` if any node failed; else `Running` if
+    /// any running; else `Queued` if any queued; else `Cancelled` if
+    /// any cancelled; else `Done`.
+    pub fn rollup(&self) -> State {
+        let mut any_cancelled = false;
+        let mut any_queued = false;
+        let mut any_running = false;
+        for n in &self.nodes {
+            match n.state {
+                State::Failed => return State::Failed,
+                State::Running => any_running = true,
+                State::Queued => any_queued = true,
+                State::Cancelled => any_cancelled = true,
+                State::Done => {}
+            }
+        }
+        if any_running {
+            State::Running
+        } else if any_queued {
+            State::Queued
+        } else if any_cancelled {
+            State::Cancelled
+        } else {
+            State::Done
+        }
+    }
+
+    /// True when every node is terminal.
+    pub fn is_terminal(&self) -> bool {
+        self.nodes.iter().all(|n| n.state.is_terminal())
+    }
+
+    /// First failed node's error, for the roll-up `error` field.
+    pub fn first_error(&self) -> Option<&str> {
+        self.nodes
+            .iter()
+            .find(|n| n.state == State::Failed)
+            .and_then(|n| n.error.as_deref())
+    }
+
+    pub fn node(&self, id: NodeId) -> Option<&Node> {
+        self.nodes.iter().find(|n| n.id == id)
+    }
+
+    pub fn node_mut(&mut self, id: NodeId) -> Option<&mut Node> {
+        self.nodes.iter_mut().find(|n| n.id == id)
+    }
+}
+
+/// Wire shape of one node inside a `DagView`.
+#[derive(Debug, Clone, Serialize)]
+pub struct NodeView {
+    pub id: NodeId,
+    /// Flattened `NodeKind` tag ("prebuild", "swap", …).
+    pub kind: &'static str,
+    /// Ids of the nodes this one waits for.
+    pub deps: Vec<NodeId>,
+    pub state: State,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub step: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub build_log_id: Option<i64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub started_at: Option<i64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub finished_at: Option<i64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub error: Option<String>,
+}
+
+/// Wire shape of a DAG, serialized onto `RebuildQueueChanged` and the
+/// `/api/state` snapshot. DAG-level fields mirror the old `QueueEntry`
+/// names (`kind` = template string, roll-up `state`); everything
+/// per-node — step labels, build-log links, errors, timestamps —
+/// appears exactly once, inside `nodes`.
+#[derive(Debug, Clone, Serialize)]
+pub struct DagView {
+    pub id: u64,
+    pub agent: String,
+    /// Template wire string — same values the old `kind` field used.
+    pub kind: Template,
+    /// Roll-up state (see [`Dag::rollup`]).
+    pub state: State,
+    pub source: Source,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub parent_id: Option<u64>,
+    pub reason: String,
+    pub enqueued_at: i64,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub started_at: Option<i64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub finished_at: Option<i64>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub inputs: Vec<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub approval_id: Option<i64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub perm_payload: Option<PermPayload>,
+    pub nodes: Vec<NodeView>,
+}
+
+impl Dag {
+    pub fn view(&self) -> DagView {
+        let started_at = self.nodes.iter().filter_map(|n| n.started_at).min();
+        let finished_at = if self.is_terminal() {
+            self.nodes.iter().filter_map(|n| n.finished_at).max()
+        } else {
+            None
+        };
+        DagView {
+            id: self.id,
+            agent: self.agent.clone(),
+            kind: self.template,
+            state: self.rollup(),
+            source: self.source,
+            parent_id: self.parent_id,
+            reason: self.reason.clone(),
+            enqueued_at: self.created_at,
+            started_at,
+            finished_at,
+            inputs: self.inputs.clone(),
+            approval_id: self.approval_id,
+            perm_payload: self.perm_payload.clone(),
+            nodes: self
+                .nodes
+                .iter()
+                .map(|n| NodeView {
+                    id: n.id,
+                    kind: n.kind.as_str(),
+                    deps: n.deps.iter().map(|d| d.on).collect(),
+                    state: n.state,
+                    step: n.step.clone(),
+                    build_log_id: n.build_log_id,
+                    started_at: n.started_at,
+                    finished_at: n.finished_at,
+                    error: n.error.clone(),
+                })
+                .collect(),
+        }
+    }
+}

hive-c0re/src/job_queue/scheduler.rs

@@ -0,0 +1,150 @@
+//! The single scheduler task that drives all DAGs: claim every ready
+//! node (as many as the build slots / leases allow), spawn one
+//! executor task per claim, and on any completion re-evaluate.
+//! Concurrency comes from the build-slot count, not multiple workers.
+//!
+//! Also owns the two DAG-lifetime side channels the sync queue core
+//! can't hold itself:
+//! - the per-DAG transient guard (dashboard pill + crash-watch
+//!   suppression), created when a DAG acquires its agent lease and
+//!   dropped when the DAG settles terminal;
+//! - the `MetaLock` fan-out: appending child `Rebuild` DAGs once the
+//!   lock bump lands, so children build against the post-bump lock
+//!   (and a failed bump fans out nothing — replacing the old
+//!   pre-enqueue + cancel-children dance).
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use super::exec::{self, NodeOutput};
+use super::{Claim, Source, Template, templates};
+use crate::coordinator::Coordinator;
+
+struct NodeDone {
+    claim: Claim,
+    result: anyhow::Result<NodeOutput>,
+}
+
+/// Scheduler loop. Spawned once at hive-c0re startup from `main.rs`.
+///
+/// Shutdown semantics: subscribes to `coord.shutdown_rx()`. On a true
+/// signal the loop exits immediately; already-running node tasks ride
+/// the runtime down with the process, and pending `Queued` DAGs are
+/// dropped — desired state is re-derived on next boot (boot sweep +
+/// reconcile), so the in-memory queue is deliberately not durable.
+pub async fn run_worker(coord: Arc<Coordinator>) {
+    let mut shutdown = coord.shutdown_rx();
+    let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<NodeDone>();
+    // DAG id → transient guard held for the lease window.
+    let mut transients: HashMap<u64, crate::coordinator::TransientGuard> = HashMap::new();
+    loop {
+        let claims = coord.job_queue.claim_ready();
+        if !claims.is_empty() {
+            for claim in claims {
+                if claim.lease_acquired
+                    && let Some(kind) = claim.transient
+                {
+                    transients.insert(claim.dag_id, coord.transient_guard(&claim.agent, kind));
+                }
+                tracing::info!(
+                    dag = claim.dag_id,
+                    node = claim.node_id,
+                    kind = claim.kind.as_str(),
+                    agent = %claim.agent,
+                    template = claim.template.as_str(),
+                    "job_queue: node running"
+                );
+                let coord = Arc::clone(&coord);
+                let tx = tx.clone();
+                tokio::spawn(async move {
+                    let result = exec::run_node(&coord, &claim).await;
+                    // Send failure = scheduler gone (shutdown); drop.
+                    let _ = tx.send(NodeDone { claim, result });
+                });
+            }
+            coord.emit_rebuild_queue_snapshot();
+            continue;
+        }
+        tokio::select! {
+            biased;
+            res = shutdown.changed() => {
+                if res.is_err() || *shutdown.borrow() {
+                    tracing::info!("job_queue: scheduler exiting on shutdown");
+                    return;
+                }
+            }
+            Some(done) = rx.recv() => {
+                handle_completion(&coord, &mut transients, done).await;
+            }
+            () = coord.job_queue.notify.notified() => {}
+        }
+    }
+}
+
+async fn handle_completion(
+    coord: &Arc<Coordinator>,
+    transients: &mut HashMap<u64, crate::coordinator::TransientGuard>,
+    done: NodeDone,
+) {
+    let NodeDone { claim, result } = done;
+    let (queue_result, fanout) = match result {
+        Ok(output) => {
+            tracing::info!(
+                dag = claim.dag_id,
+                node = claim.node_id,
+                "job_queue: node done"
+            );
+            (Ok(()), output.fanout)
+        }
+        Err(e) => {
+            let msg = format!("{e:#}");
+            tracing::warn!(
+                dag = claim.dag_id,
+                node = claim.node_id,
+                kind = claim.kind.as_str(),
+                agent = %claim.agent,
+                error = %msg,
+                "job_queue: node failed"
+            );
+            (Err(msg), Vec::new())
+        }
+    };
+    let report = coord
+        .job_queue
+        .complete_node(claim.dag_id, claim.node_id, queue_result);
+    if !fanout.is_empty() {
+        let specs = fanout_specs(&claim, fanout);
+        coord.job_queue.append_children(specs);
+    }
+    for terminal in report.terminal {
+        // Drop the lease-window transient guard, then let the hook
+        // fire approval resolution / failure events.
+        transients.remove(&terminal.dag_id);
+        exec::on_dag_terminal(coord, &terminal).await;
+    }
+    coord.emit_rebuild_queue_snapshot();
+}
+
+/// Child `Rebuild` specs for a completed `MetaLock` fan-out, grouped
+/// under the parent via `parent_id`. Meta-update children skip the
+/// per-agent relock (it would revert the bump the parent just
+/// committed); sweep children relock like a manual rebuild.
+fn fanout_specs(claim: &Claim, agents: Vec<String>) -> Vec<super::DagSpec> {
+    let sweep = claim.template == Template::StartupSweep;
+    let (source, relock) = if sweep {
+        (Source::StartupSweep, true)
+    } else {
+        (Source::MetaUpdate, false)
+    };
+    let reason = if sweep {
+        "startup sweep".to_owned()
+    } else if let Some(approval_id) = claim.approval_id {
+        format!("approval #{approval_id} meta input cascade")
+    } else {
+        "meta-update cascade".to_owned()
+    };
+    agents
+        .into_iter()
+        .map(|agent| templates::rebuild(&agent, source, reason.clone(), Some(claim.dag_id), relock))
+        .collect()
+}

hive-c0re/src/job_queue/submit.rs

@@ -0,0 +1,121 @@
+//! Request-level submit API — the surface the dashboard POST handlers,
+//! the MCP socket handlers, and `hivectl` paths call. Owns the
+//! submit-time side effects the DAG templates deliberately don't:
+//! writing the durable `wanted` power intent (synchronously,
+//! last-writer-wins) before the DAG whose `Reconcile` reads it, and
+//! upgrading a stale start to a full rebuild. Every helper emits a
+//! fresh queue snapshot so the dashboard shows the new DAG immediately.
+
+use std::sync::Arc;
+
+use super::{Source, Template, templates};
+use crate::coordinator::Coordinator;
+use crate::power::Wanted;
+
+fn submit_and_emit(coord: &Arc<Coordinator>, spec: super::DagSpec) -> u64 {
+    let id = coord
+        .job_queue
+        .submit(spec)
+        .expect("template-built dag specs are acyclic");
+    coord.emit_rebuild_queue_snapshot();
+    id
+}
+
+fn set_wanted(coord: &Arc<Coordinator>, agent: &str, wanted: Wanted) {
+    if let Err(e) = coord.power.set(agent, wanted) {
+        tracing::warn!(%agent, wanted = wanted.as_str(), error = ?e, "agent_power: set failed");
+    }
+}
+
+/// Manual/approval-independent rebuild (always relocks the agent's
+/// meta input — cascade children are built by the scheduler's fan-out
+/// instead of this surface).
+pub fn rebuild(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
+    submit_and_emit(coord, templates::rebuild(agent, source, reason, None, true))
+}
+
+/// Restart: mechanical stop + converge back to `wanted` (unchanged).
+pub fn restart(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
+    submit_and_emit(coord, templates::restart(agent, source, reason))
+}
+
+/// Start: persist `wanted = Up`, then reconcile. A stale rev marker
+/// upgrades the start to a full rebuild (whose tail `Reconcile` does
+/// the start) so the container always comes up on current derivations
+/// — the old fast-lane `run_start` upgrade, moved to submit time.
+pub fn start(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
+    set_wanted(coord, agent, Wanted::Up);
+    let stored = std::fs::read_to_string(crate::auto_update::rev_marker_path(agent)).ok();
+    let stale = crate::auto_update::current_flake_rev(&coord.hyperhive_flake)
+        .is_some_and(|rev| stored.as_deref() != Some(rev.as_str()));
+    if stale {
+        tracing::info!(%agent, "start: rev stale — upgrading to rebuild+start");
+        return submit_and_emit(
+            coord,
+            templates::rebuild(
+                agent,
+                source,
+                format!("{reason} (stale — rebuild+start)"),
+                None,
+                true,
+            ),
+        );
+    }
+    submit_and_emit(
+        coord,
+        templates::reconcile_only(
+            Template::Start,
+            agent,
+            source,
+            reason,
+            Some(crate::coordinator::TransientKind::Starting),
+        ),
+    )
+}
+
+/// Hard stop: persist `wanted = Offline`, then reconcile (kill +
+/// unregister + `Killed` event).
+pub fn stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
+    set_wanted(coord, agent, Wanted::Offline);
+    submit_and_emit(
+        coord,
+        templates::reconcile_only(
+            Template::Stop,
+            agent,
+            source,
+            reason,
+            Some(crate::coordinator::TransientKind::Stopping),
+        ),
+    )
+}
+
+/// Graceful stop: persist `wanted = Offline`, then signal → drain →
+/// reconcile (the actual stop).
+pub fn graceful_stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
+    set_wanted(coord, agent, Wanted::Offline);
+    submit_and_emit(coord, templates::graceful_stop(agent, source, reason))
+}
+
+/// Perm change: commit the JSON file(s) then rebuild.
+pub fn perm_change(
+    coord: &Arc<Coordinator>,
+    agent: &str,
+    source: Source,
+    reason: String,
+    payload: super::PermPayload,
+) -> u64 {
+    submit_and_emit(
+        coord,
+        templates::perm_change(agent, source, reason, payload),
+    )
+}
+
+/// Meta-input lock bump; cascade rebuilds fan out on completion.
+pub fn meta_update(
+    coord: &Arc<Coordinator>,
+    inputs: Vec<String>,
+    source: Source,
+    reason: String,
+) -> u64 {
+    submit_and_emit(coord, templates::meta_update(inputs, source, reason, None))
+}

hive-c0re/src/job_queue/templates.rs

@@ -0,0 +1,333 @@
+//! DAG shape builders — every operation as a template over the shared
+//! node primitives — plus submit-time cycle validation (petgraph is
+//! confined to this validation; the runtime store stays the plain
+//! `Vec<Node>` + `deps`).
+//!
+//! ```text
+//! rebuild(a):        Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a)
+//! graceful-stop(a):  [wanted=Offline]  Signal(a) → Drain(a) → Reconcile(a)
+//! restart(a):        StopForUpdate(a) → Reconcile(a)      (wanted unchanged)
+//! start(a):          [wanted=Up]       Reconcile(a)
+//! stop(a):           [wanted=Offline]  Reconcile(a)
+//! spawn(a):          [wanted=Up]  Create(a) → WriteDropin(a) → Reconcile(a)
+//! perm-change(a):    WritePermFile(a) → «rebuild subgraph»
+//! meta-update(inp):  MetaLock(inp) → «fan-out rebuild(a) per affected a»
+//! startup sweep:     MetaLock(hyperhive, non-fatal) → «fan-out rebuild(stale a)»
+//! ```
+
+use anyhow::{Result, bail};
+
+use super::model::{DagSpec, Dep, DepWhen, NodeKind, NodeSpec, PermPayload, Source, Template};
+use crate::coordinator::TransientKind;
+
+/// After-ok edge on the previous node — the common chain link.
+fn after_ok(on: u32) -> Vec<Dep> {
+    vec![Dep {
+        on,
+        when: DepWhen::AfterOk,
+    }]
+}
+
+/// The rebuild node chain. `Reconcile` deps on `Swap` with `AfterAny`:
+/// it must run even when the profile swap failed, so a previously-up
+/// agent comes back on its old config (today's recovery-start). This
+/// is the only `AfterAny` edge in v1.
+fn rebuild_nodes(relock: bool, base: u32) -> Vec<NodeSpec> {
+    vec![
+        NodeSpec {
+            kind: NodeKind::Prebuild { relock },
+            deps: if base == 0 {
+                Vec::new()
+            } else {
+                after_ok(base - 1)
+            },
+        },
+        NodeSpec {
+            kind: NodeKind::StopForUpdate,
+            deps: after_ok(base),
+        },
+        NodeSpec {
+            kind: NodeKind::Swap,
+            deps: after_ok(base + 1),
+        },
+        NodeSpec {
+            kind: NodeKind::Reconcile,
+            deps: vec![Dep {
+                on: base + 2,
+                when: DepWhen::AfterAny,
+            }],
+        },
+    ]
+}
+
+/// One uniform rebuild shape — no `was_running` branch. `StopForUpdate`
+/// noops when already down; the tail `Reconcile` auto-noops the start
+/// when `wanted = Offline` (a rebuild of a deliberately-stopped agent
+/// leaves it stopped). `relock = false` only for meta-update cascade
+/// children.
+pub fn rebuild(
+    agent: &str,
+    source: Source,
+    reason: String,
+    parent_id: Option<u64>,
+    relock: bool,
+) -> DagSpec {
+    DagSpec {
+        template: Template::Rebuild,
+        agent: agent.to_owned(),
+        source,
+        reason,
+        parent_id,
+        approval_id: None,
+        inputs: Vec::new(),
+        perm_payload: None,
+        transient: Some(TransientKind::Rebuilding),
+        nodes: rebuild_nodes(relock, 0),
+    }
+}
+
+/// Approval-driven deploy (`ApplyCommit` / `MergeConfigPr`): the whole
+/// two-phase pipeline stays one opaque node in v1 (design doc §9) —
+/// wire-visible as a `rebuild` card like today.
+pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec {
+    DagSpec {
+        template: Template::Rebuild,
+        agent: agent.to_owned(),
+        source: Source::Approval,
+        reason,
+        parent_id: None,
+        approval_id: Some(approval_id),
+        inputs: Vec::new(),
+        perm_payload: None,
+        transient: Some(TransientKind::Rebuilding),
+        nodes: vec![NodeSpec {
+            kind: NodeKind::ApprovalDeploy,
+            deps: Vec::new(),
+        }],
+    }
+}
+
+/// Graceful stop: cheap `Signal` fires immediately (no build slot), the
+/// `Drain` awaits the harness checkpoint (bounded), and the tail
+/// `Reconcile` performs the actual container stop — the caller sets
+/// `wanted = Offline` at submit time. A whole-hive graceful stop
+/// therefore signals every agent up front and overlaps every drain,
+/// replacing the old detached-watcher thread structurally.
+pub fn graceful_stop(agent: &str, source: Source, reason: String) -> DagSpec {
+    DagSpec {
+        template: Template::GracefulStop,
+        agent: agent.to_owned(),
+        source,
+        reason,
+        parent_id: None,
+        approval_id: None,
+        inputs: Vec::new(),
+        perm_payload: None,
+        transient: Some(TransientKind::Stopping),
+        nodes: vec![
+            NodeSpec {
+                kind: NodeKind::Signal,
+                deps: Vec::new(),
+            },
+            NodeSpec {
+                kind: NodeKind::Drain,
+                deps: after_ok(0),
+            },
+            NodeSpec {
+                kind: NodeKind::Reconcile,
+                deps: after_ok(1),
+            },
+        ],
+    }
+}
+
+/// Restart: mechanical stop, then converge back to `wanted`
+/// (unchanged) — a stop + start for a wanted-up agent.
+pub fn restart(agent: &str, source: Source, reason: String) -> DagSpec {
+    DagSpec {
+        template: Template::Restart,
+        agent: agent.to_owned(),
+        source,
+        reason,
+        parent_id: None,
+        approval_id: None,
+        inputs: Vec::new(),
+        perm_payload: None,
+        transient: Some(TransientKind::Restarting),
+        nodes: vec![
+            NodeSpec {
+                kind: NodeKind::StopForUpdate,
+                deps: Vec::new(),
+            },
+            NodeSpec {
+                kind: NodeKind::Reconcile,
+                deps: after_ok(0),
+            },
+        ],
+    }
+}
+
+/// Single-`Reconcile` DAG: `Start` / `Stop` (caller writes `wanted`
+/// first) and the boot-time `Reconcile` converge (wanted untouched).
+pub fn reconcile_only(
+    template: Template,
+    agent: &str,
+    source: Source,
+    reason: String,
+    transient: Option<TransientKind>,
+) -> DagSpec {
+    DagSpec {
+        template,
+        agent: agent.to_owned(),
+        source,
+        reason,
+        parent_id: None,
+        approval_id: None,
+        inputs: Vec::new(),
+        perm_payload: None,
+        transient,
+        nodes: vec![NodeSpec {
+            kind: NodeKind::Reconcile,
+            deps: Vec::new(),
+        }],
+    }
+}
+
+/// First-deploy spawn (approval-driven): pre-start provisioning +
+/// `nixos-container create`, drop-in write, then `Reconcile` starts the
+/// container (`wanted = Up` written at approve time).
+pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
+    DagSpec {
+        template: Template::Spawn,
+        agent: agent.to_owned(),
+        source: Source::Approval,
+        reason,
+        parent_id: None,
+        approval_id: Some(approval_id),
+        inputs: Vec::new(),
+        perm_payload: None,
+        transient: Some(TransientKind::Spawning),
+        nodes: vec![
+            NodeSpec {
+                kind: NodeKind::Create,
+                deps: Vec::new(),
+            },
+            NodeSpec {
+                kind: NodeKind::WriteDropin,
+                deps: after_ok(0),
+            },
+            NodeSpec {
+                kind: NodeKind::Reconcile,
+                deps: after_ok(1),
+            },
+        ],
+    }
+}
+
+/// Perm change: commit the JSON file(s), then the rebuild subgraph so
+/// the updated `HIVE_TOOL_GROUPS` / `HIVE_CAPABILITIES` env var takes
+/// effect in the container.
+pub fn perm_change(agent: &str, source: Source, reason: String, payload: PermPayload) -> DagSpec {
+    let mut nodes = vec![NodeSpec {
+        kind: NodeKind::WritePermFile,
+        deps: Vec::new(),
+    }];
+    nodes.extend(rebuild_nodes(true, 1));
+    DagSpec {
+        template: Template::PermChange,
+        agent: agent.to_owned(),
+        source,
+        reason,
+        parent_id: None,
+        approval_id: None,
+        inputs: Vec::new(),
+        perm_payload: Some(payload),
+        transient: Some(TransientKind::Rebuilding),
+        nodes,
+    }
+}
+
+/// Meta-input lock bump. Child `Rebuild` DAGs fan out on completion —
+/// appended *after* the bump lands so their prebuilds run against the
+/// post-bump lock (and so a failed bump simply fans out nothing,
+/// replacing the old pre-enqueue + `cancel_children` dance).
+pub fn meta_update(
+    inputs: Vec<String>,
+    source: Source,
+    reason: String,
+    approval_id: Option<i64>,
+) -> DagSpec {
+    DagSpec {
+        template: Template::MetaUpdate,
+        agent: "hyperhive".to_owned(),
+        source,
+        reason,
+        parent_id: None,
+        approval_id,
+        inputs,
+        perm_payload: None,
+        transient: None,
+        nodes: vec![NodeSpec {
+            kind: NodeKind::MetaLock {
+                sweep: false,
+                fanout: None,
+            },
+            deps: Vec::new(),
+        }],
+    }
+}
+
+/// Boot-time sweep parent: bump meta's hyperhive input (non-fatal),
+/// then fan out `Rebuild` children for the precomputed stale agent
+/// list (topology-sorted by the caller).
+pub fn startup_sweep(reason: String, stale_agents: Vec<String>) -> DagSpec {
+    DagSpec {
+        template: Template::StartupSweep,
+        agent: "hyperhive".to_owned(),
+        source: Source::AutoUpdate,
+        reason,
+        parent_id: None,
+        approval_id: None,
+        inputs: Vec::new(),
+        perm_payload: None,
+        transient: None,
+        nodes: vec![NodeSpec {
+            kind: NodeKind::MetaLock {
+                sweep: true,
+                fanout: Some(stale_agents),
+            },
+            deps: Vec::new(),
+        }],
+    }
+}
+
+/// Validate a spec before it enters the queue: node ids are dense
+/// (index = id), deps reference existing nodes, and the dep graph is
+/// acyclic (petgraph `toposort`). Rejecting cycles here fixes the old
+/// queue's documented "circular dep silently deadlocks forever" caveat.
+pub fn validate(spec: &DagSpec) -> Result<()> {
+    if spec.nodes.is_empty() {
+        bail!("dag spec {:?} has no nodes", spec.template);
+    }
+    let n = spec.nodes.len();
+    let mut graph = petgraph::graph::DiGraph::<u32, ()>::new();
+    let idx: Vec<_> = (0..n)
+        .map(|i| graph.add_node(u32::try_from(i).unwrap_or(u32::MAX)))
+        .collect();
+    for (i, node) in spec.nodes.iter().enumerate() {
+        for dep in &node.deps {
+            let Some(&dep_idx) = idx.get(dep.on as usize) else {
+                bail!(
+                    "dag spec {:?} node {i} depends on unknown node {}",
+                    spec.template,
+                    dep.on
+                );
+            };
+            graph.add_edge(dep_idx, idx[i], ());
+        }
+    }
+    if petgraph::algo::toposort(&graph, None).is_err() {
+        bail!("dag spec {:?} contains a dependency cycle", spec.template);
+    }
+    Ok(())
+}

hive-c0re/src/job_queue/tests.rs

@@ -0,0 +1,823 @@
+//! Queue-core unit tests: dedup, cycle rejection, resource
+//! serialization (build slots / per-agent leases), lease-exempt
+//! overlap, FIFO fairness, cancel semantics, `AfterAny` failure
+//! routing, fan-out, and history retention. All synchronous — the
+//! scheduler's async loop is a thin claim/complete pump over the same
+//! methods exercised here.
+
+use super::model::{Dep, DepWhen, NodeKind, NodeSpec};
+use super::*;
+
+fn submit(q: &JobQueue, spec: DagSpec) -> u64 {
+    q.submit(spec).expect("valid spec")
+}
+
+fn rebuild(agent: &str, reason: &str) -> DagSpec {
+    templates::rebuild(agent, Source::Manual, reason.to_owned(), None, true)
+}
+
+/// Claim helper asserting exactly one node comes back.
+fn claim_one(q: &JobQueue) -> Claim {
+    let mut claims = q.claim_ready();
+    assert_eq!(
+        claims.len(),
+        1,
+        "expected exactly one claim, got {claims:?}"
+    );
+    claims.pop().expect("one claim")
+}
+
+fn state_of(q: &JobQueue, dag_id: u64) -> State {
+    q.snapshot()
+        .iter()
+        .find(|d| d.id == dag_id)
+        .expect("dag present")
+        .state
+}
+
+// ---- submit / dedup ----
+
+#[test]
+fn submit_assigns_distinct_ids() {
+    let q = JobQueue::new(1);
+    let a = submit(&q, rebuild("agent-a", "first"));
+    let b = submit(&q, rebuild("agent-b", "second"));
+    assert_ne!(a, b);
+    assert_eq!(q.snapshot().len(), 2);
+}
+
+#[test]
+fn dedup_pending_same_template_and_agent() {
+    let q = JobQueue::new(1);
+    let a = submit(&q, rebuild("agent-a", "first"));
+    let b = submit(&q, rebuild("agent-a", "auto sweep"));
+    assert_eq!(a, b, "dedup should return existing id");
+    let snap = q.snapshot();
+    assert_eq!(snap.len(), 1);
+    assert!(snap[0].reason.contains("first"));
+    assert!(snap[0].reason.contains("auto sweep"));
+}
+
+#[test]
+fn dedup_does_not_apply_across_templates_or_agents() {
+    let q = JobQueue::new(1);
+    let a = submit(&q, rebuild("agent-a", "r"));
+    let b = submit(&q, rebuild("agent-b", "r"));
+    let c = submit(
+        &q,
+        templates::restart("agent-a", Source::Manual, "r".to_owned()),
+    );
+    assert_ne!(a, b);
+    assert_ne!(a, c);
+    assert_eq!(q.snapshot().len(), 3);
+}
+
+#[test]
+fn dedup_skips_running_dags() {
+    let q = JobQueue::new(1);
+    let a = submit(&q, rebuild("agent-a", "first"));
+    let claim = claim_one(&q); // Prebuild running
+    assert_eq!(claim.dag_id, a);
+    // While the original runs, re-submit is legitimate new work.
+    let again = submit(&q, rebuild("agent-a", "config bumped during build"));
+    assert_ne!(a, again);
+    assert_eq!(q.snapshot().len(), 2);
+}
+
+#[test]
+fn meta_update_dedup_matches_inputs() {
+    let q = JobQueue::new(1);
+    let a = submit(
+        &q,
+        templates::meta_update(
+            vec!["nixpkgs".to_owned()],
+            Source::Manual,
+            "first".to_owned(),
+            None,
+        ),
+    );
+    let b = submit(
+        &q,
+        templates::meta_update(
+            vec!["nixpkgs".to_owned()],
+            Source::Manual,
+            "duplicate click".to_owned(),
+            None,
+        ),
+    );
+    assert_eq!(a, b, "identical-inputs meta-updates should dedup");
+    let c = submit(
+        &q,
+        templates::meta_update(
+            vec!["agent-bitburner/bitburner-agent".to_owned()],
+            Source::Manual,
+            "bump agent".to_owned(),
+            None,
+        ),
+    );
+    assert_ne!(a, c, "different-inputs meta-updates must NOT dedup");
+    assert_eq!(q.snapshot().len(), 2);
+}
+
+#[test]
+fn approval_dags_dedup_only_on_matching_id() {
+    let q = JobQueue::new(1);
+    let a = submit(
+        &q,
+        templates::approval_deploy("agent-a", 1, "approval #1".to_owned()),
+    );
+    let b = submit(
+        &q,
+        templates::approval_deploy("agent-a", 2, "approval #2".to_owned()),
+    );
+    assert_ne!(a, b, "distinct approvals must not collapse");
+    // Rapid double-click on the same approval IS a single op.
+    let c = submit(
+        &q,
+        templates::approval_deploy("agent-a", 1, "approval #1 (dup)".to_owned()),
+    );
+    assert_eq!(a, c);
+    assert_eq!(q.snapshot().len(), 2);
+}
+
+#[test]
+fn perm_change_dedup_respects_perm_type() {
+    let q = JobQueue::new(1);
+    let groups = templates::perm_change(
+        "agent-a",
+        Source::Manual,
+        "groups".to_owned(),
+        PermPayload::ToolGroups { groups: vec![] },
+    );
+    let caps = templates::perm_change(
+        "agent-a",
+        Source::Manual,
+        "caps".to_owned(),
+        PermPayload::Capabilities { caps: vec![] },
+    );
+    let a = submit(&q, groups.clone());
+    let b = submit(&q, caps);
+    assert_ne!(a, b, "tool-groups vs capabilities must not collapse");
+    let c = submit(&q, groups);
+    assert_eq!(a, c, "same perm type dedups");
+}
+
+/// A `MetaUpdate` cascade `Rebuild` (with `parent_id = Some(meta_id)`)
+/// must NOT dedup into a queued `Rebuild` with a different
+/// `parent_id` (e.g. from a startup sweep) — without the guard the
+/// cascade child would be swallowed and the agent never rebuilt
+/// against the post-bump meta.
+#[test]
+fn dedup_respects_parent_id() {
+    let q = JobQueue::new(1);
+    let sweep = submit(&q, templates::startup_sweep("boot".to_owned(), vec![]));
+    let sweep_child = submit(
+        &q,
+        templates::rebuild(
+            "alice",
+            Source::StartupSweep,
+            "startup sweep".to_owned(),
+            Some(sweep),
+            true,
+        ),
+    );
+    let meta = submit(
+        &q,
+        templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
+    );
+    let cascade_child = submit(
+        &q,
+        templates::rebuild(
+            "alice",
+            Source::MetaUpdate,
+            "meta-update cascade".to_owned(),
+            Some(meta),
+            false,
+        ),
+    );
+    assert_ne!(sweep_child, cascade_child);
+    let rebuilds = q
+        .snapshot()
+        .iter()
+        .filter(|d| d.kind == Template::Rebuild && d.agent == "alice")
+        .count();
+    assert_eq!(rebuilds, 2, "both rebuilds must be present");
+}
+
+// ---- cycle rejection ----
+
+#[test]
+fn cyclic_dag_is_rejected_at_submit() {
+    let q = JobQueue::new(1);
+    let mut spec = rebuild("agent-a", "cyclic");
+    // 0 → 1 → 0 cycle.
+    spec.nodes = vec![
+        NodeSpec {
+            kind: NodeKind::StopForUpdate,
+            deps: vec![Dep {
+                on: 1,
+                when: DepWhen::AfterOk,
+            }],
+        },
+        NodeSpec {
+            kind: NodeKind::Reconcile,
+            deps: vec![Dep {
+                on: 0,
+                when: DepWhen::AfterOk,
+            }],
+        },
+    ];
+    assert!(q.submit(spec).is_err(), "cyclic spec must be refused");
+    assert!(q.snapshot().is_empty());
+}
+
+#[test]
+fn unknown_dep_is_rejected_at_submit() {
+    let q = JobQueue::new(1);
+    let mut spec = rebuild("agent-a", "bad dep");
+    spec.nodes = vec![NodeSpec {
+        kind: NodeKind::Reconcile,
+        deps: vec![Dep {
+            on: 9,
+            when: DepWhen::AfterOk,
+        }],
+    }];
+    assert!(q.submit(spec).is_err());
+}
+
+// ---- dependency order within a DAG ----
+
+#[test]
+fn rebuild_chain_claims_in_dep_order() {
+    let q = JobQueue::new(1);
+    let id = submit(&q, rebuild("agent-a", "r"));
+    for expected in ["prebuild", "stop_for_update", "swap", "reconcile"] {
+        let c = claim_one(&q);
+        assert_eq!(c.dag_id, id);
+        assert_eq!(c.kind.as_str(), expected);
+        assert!(
+            q.claim_ready().is_empty(),
+            "chain must serialize: nothing ready while {expected} runs"
+        );
+        q.complete_node(id, c.node_id, Ok(()));
+    }
+    assert_eq!(state_of(&q, id), State::Done);
+}
+
+// ---- build slots ----
+
+#[test]
+fn build_slot_serializes_nix_heavy_nodes() {
+    let q = JobQueue::new(1);
+    let a = submit(&q, rebuild("agent-a", "r"));
+    let b = submit(&q, rebuild("agent-b", "r"));
+    let first = claim_one(&q); // a's Prebuild takes the only slot
+    assert_eq!(first.dag_id, a);
+    assert_eq!(first.kind.as_str(), "prebuild");
+    q.complete_node(a, first.node_id, Ok(()));
+    // With the slot free again, FIFO gives... a's StopForUpdate is
+    // slot-free (lease) and b's Prebuild takes the slot — both run.
+    let claims = q.claim_ready();
+    let kinds: Vec<(u64, &str)> = claims.iter().map(|c| (c.dag_id, c.kind.as_str())).collect();
+    assert!(kinds.contains(&(a, "stop_for_update")));
+    assert!(kinds.contains(&(b, "prebuild")));
+    assert_eq!(claims.len(), 2);
+}
+
+#[test]
+fn two_build_slots_run_two_prebuilds() {
+    let q = JobQueue::new(2);
+    submit(&q, rebuild("agent-a", "r"));
+    submit(&q, rebuild("agent-b", "r"));
+    let claims = q.claim_ready();
+    assert_eq!(claims.len(), 2, "two slots → two concurrent prebuilds");
+    assert!(claims.iter().all(|c| c.kind.as_str() == "prebuild"));
+}
+
+#[test]
+fn fifo_fairness_for_the_slot() {
+    let q = JobQueue::new(1);
+    let a = submit(&q, rebuild("agent-a", "r"));
+    let b = submit(&q, rebuild("agent-b", "r"));
+    let c = submit(&q, rebuild("agent-c", "r"));
+    let first = claim_one(&q);
+    assert_eq!(first.dag_id, a, "submit order wins the slot");
+    q.complete_node(a, first.node_id, Ok(()));
+    let next: Vec<u64> = q.claim_ready().iter().map(|cl| cl.dag_id).collect();
+    assert!(next.contains(&b), "b's prebuild before c's");
+    assert!(!next.contains(&c));
+}
+
+// ---- per-agent lease ----
+
+#[test]
+fn lease_serializes_two_lifecycle_dags_for_same_agent() {
+    let q = JobQueue::new(4);
+    let restart = submit(
+        &q,
+        templates::restart("agent-a", Source::Manual, "restart".to_owned()),
+    );
+    let stop = submit(
+        &q,
+        templates::reconcile_only(
+            Template::Stop,
+            "agent-a",
+            Source::Manual,
+            "stop".to_owned(),
+            None,
+        ),
+    );
+    // Restart's StopForUpdate acquires the lease; stop's Reconcile
+    // must wait even though slots are free.
+    let first = claim_one(&q);
+    assert_eq!(first.dag_id, restart);
+    assert!(first.lease_acquired);
+    q.complete_node(restart, first.node_id, Ok(()));
+    // Same DAG keeps the lease for its Reconcile.
+    let second = claim_one(&q);
+    assert_eq!(second.dag_id, restart);
+    assert!(!second.lease_acquired, "lease already held by this DAG");
+    q.complete_node(restart, second.node_id, Ok(()));
+    // Restart terminal → lease released → stop's Reconcile runs.
+    let third = claim_one(&q);
+    assert_eq!(third.dag_id, stop);
+    q.complete_node(stop, third.node_id, Ok(()));
+    assert_eq!(state_of(&q, restart), State::Done);
+    assert_eq!(state_of(&q, stop), State::Done);
+}
+
+#[test]
+fn lease_exempt_prebuild_overlaps_other_dag_on_same_agent() {
+    let q = JobQueue::new(2);
+    submit(&q, rebuild("agent-a", "rebuild"));
+    let stop = submit(
+        &q,
+        templates::reconcile_only(
+            Template::Stop,
+            "agent-a",
+            Source::Manual,
+            "stop".to_owned(),
+            None,
+        ),
+    );
+    // Prebuild is lease-exempt: the stop's Reconcile takes the lease
+    // and runs concurrently with the rebuild's out-of-band nix build.
+    let claims = q.claim_ready();
+    let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect();
+    assert!(kinds.contains(&"prebuild"));
+    assert!(kinds.contains(&"reconcile"));
+    // But the rebuild's StopForUpdate must then wait for the stop DAG
+    // to finish (lease).
+    let prebuild = claims
+        .iter()
+        .find(|c| c.kind.as_str() == "prebuild")
+        .expect("prebuild claim")
+        .clone();
+    q.complete_node(prebuild.dag_id, prebuild.node_id, Ok(()));
+    assert!(
+        q.claim_ready().is_empty(),
+        "StopForUpdate blocked while stop DAG holds the lease"
+    );
+    let reconcile = claims
+        .iter()
+        .find(|c| c.kind.as_str() == "reconcile")
+        .expect("reconcile claim")
+        .clone();
+    q.complete_node(stop, reconcile.node_id, Ok(()));
+    let next = claim_one(&q);
+    assert_eq!(next.kind.as_str(), "stop_for_update");
+}
+
+#[test]
+fn agents_do_not_contend_on_each_others_leases() {
+    let q = JobQueue::new(4);
+    submit(
+        &q,
+        templates::restart("agent-a", Source::Manual, "r".to_owned()),
+    );
+    submit(
+        &q,
+        templates::restart("agent-b", Source::Manual, "r".to_owned()),
+    );
+    let claims = q.claim_ready();
+    assert_eq!(claims.len(), 2, "different agents run concurrently");
+}
+
+// ---- failure: cancel-downstream + AfterAny ----
+
+#[test]
+fn failed_node_cancels_downstream_but_afterany_reconcile_runs() {
+    let q = JobQueue::new(1);
+    let id = submit(&q, rebuild("agent-a", "r"));
+    let prebuild = claim_one(&q);
+    q.complete_node(id, prebuild.node_id, Err("nix build exploded".to_owned()));
+    // StopForUpdate + Swap are cancelled (AfterOk on a failed chain);
+    // the AfterAny Reconcile still runs once Swap is terminal.
+    let reconcile = claim_one(&q);
+    assert_eq!(reconcile.kind.as_str(), "reconcile");
+    q.complete_node(id, reconcile.node_id, Ok(()));
+    let snap = q.snapshot();
+    let dag = snap.iter().find(|d| d.id == id).expect("dag");
+    assert_eq!(dag.state, State::Failed, "roll-up failed");
+    let by_kind = |k: &str| {
+        dag.nodes
+            .iter()
+            .find(|n| n.kind == k)
+            .expect("node present")
+            .state
+    };
+    assert_eq!(by_kind("prebuild"), State::Failed);
+    assert_eq!(by_kind("stop_for_update"), State::Cancelled);
+    assert_eq!(by_kind("swap"), State::Cancelled);
+    assert_eq!(by_kind("reconcile"), State::Done);
+    assert_eq!(
+        dag.nodes
+            .iter()
+            .find(|n| n.kind == "prebuild")
+            .and_then(|n| n.error.as_deref()),
+        Some("nix build exploded")
+    );
+}
+
+/// The swap-failure recovery: `Swap` fails → the `AfterAny` edge still
+/// runs `Reconcile`, which brings a wanted-up agent back on its old
+/// config.
+#[test]
+fn swap_failure_still_runs_reconcile() {
+    let q = JobQueue::new(1);
+    let id = submit(&q, rebuild("agent-a", "r"));
+    for _ in 0..2 {
+        let c = claim_one(&q);
+        q.complete_node(id, c.node_id, Ok(()));
+    }
+    let swap = claim_one(&q);
+    assert_eq!(swap.kind.as_str(), "swap");
+    q.complete_node(id, swap.node_id, Err("update failed".to_owned()));
+    let reconcile = claim_one(&q);
+    assert_eq!(reconcile.kind.as_str(), "reconcile");
+    q.complete_node(id, reconcile.node_id, Ok(()));
+    assert_eq!(state_of(&q, id), State::Failed);
+}
+
+#[test]
+fn failed_reconcile_marks_dag_failed() {
+    let q = JobQueue::new(1);
+    let id = submit(
+        &q,
+        templates::reconcile_only(
+            Template::Start,
+            "agent-a",
+            Source::Manual,
+            "start".to_owned(),
+            None,
+        ),
+    );
+    let c = claim_one(&q);
+    q.complete_node(id, c.node_id, Err("start failed".to_owned()));
+    assert_eq!(state_of(&q, id), State::Failed);
+}
+
+// ---- cancel ----
+
+#[test]
+fn cancel_clears_queued_dag() {
+    let q = JobQueue::new(1);
+    let id = submit(&q, rebuild("agent-a", "r"));
+    assert!(q.cancel(id));
+    assert_eq!(state_of(&q, id), State::Cancelled);
+    assert!(q.claim_ready().is_empty());
+}
+
+#[test]
+fn cancel_refuses_running_dag() {
+    let q = JobQueue::new(1);
+    let id = submit(&q, rebuild("agent-a", "r"));
+    let _ = claim_one(&q);
+    assert!(!q.cancel(id));
+    assert_eq!(state_of(&q, id), State::Running);
+}
+
+#[test]
+fn cancel_children_marks_queued_children_only() {
+    let q = JobQueue::new(1);
+    let meta = submit(
+        &q,
+        templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
+    );
+    // Parent's MetaLock is running while children exist.
+    let lock = claim_one(&q);
+    assert_eq!(lock.dag_id, meta);
+    let child_a = submit(
+        &q,
+        templates::rebuild(
+            "agent-a",
+            Source::MetaUpdate,
+            "cascade".to_owned(),
+            Some(meta),
+            false,
+        ),
+    );
+    let child_b = submit(
+        &q,
+        templates::rebuild(
+            "agent-b",
+            Source::MetaUpdate,
+            "cascade".to_owned(),
+            Some(meta),
+            false,
+        ),
+    );
+    let unrelated = submit(&q, rebuild("agent-c", "operator queued"));
+    // MetaLock holds the single build slot, so both children (and the
+    // unrelated rebuild) are still fully queued here.
+    let cancelled = q.cancel_children(meta);
+    assert_eq!(cancelled, 2);
+    assert_eq!(state_of(&q, child_a), State::Cancelled);
+    assert_eq!(state_of(&q, child_b), State::Cancelled);
+    assert_eq!(state_of(&q, unrelated), State::Queued);
+}
+
+#[test]
+fn cancel_children_skips_running_child() {
+    let q = JobQueue::new(2);
+    let meta = submit(
+        &q,
+        templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
+    );
+    let lock = claim_one(&q);
+    let running_child = submit(
+        &q,
+        templates::rebuild(
+            "agent-a",
+            Source::MetaUpdate,
+            "cascade".to_owned(),
+            Some(meta),
+            false,
+        ),
+    );
+    let queued_child = submit(
+        &q,
+        templates::rebuild(
+            "agent-b",
+            Source::MetaUpdate,
+            "cascade".to_owned(),
+            Some(meta),
+            false,
+        ),
+    );
+    // Second slot lets running_child's prebuild start.
+    let child_claim = claim_one(&q);
+    assert_eq!(child_claim.dag_id, running_child);
+    let n = q.cancel_children(meta);
+    assert_eq!(n, 1);
+    assert_eq!(state_of(&q, running_child), State::Running);
+    assert_eq!(state_of(&q, queued_child), State::Cancelled);
+    q.complete_node(meta, lock.node_id, Ok(()));
+}
+
+// ---- fan-out ----
+
+#[test]
+fn append_children_sets_parent_and_dedups() {
+    let q = JobQueue::new(1);
+    let meta = submit(
+        &q,
+        templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
+    );
+    let specs = vec![
+        templates::rebuild(
+            "alice",
+            Source::MetaUpdate,
+            "cascade".to_owned(),
+            Some(meta),
+            false,
+        ),
+        templates::rebuild(
+            "bob",
+            Source::MetaUpdate,
+            "cascade".to_owned(),
+            Some(meta),
+            false,
+        ),
+        // Duplicate — must coalesce into the first alice child.
+        templates::rebuild(
+            "alice",
+            Source::MetaUpdate,
+            "cascade again".to_owned(),
+            Some(meta),
+            false,
+        ),
+    ];
+    let ids = q.append_children(specs);
+    assert_eq!(ids.len(), 3);
+    assert_eq!(ids[0], ids[2], "duplicate child dedups");
+    let snap = q.snapshot();
+    let children: Vec<_> = snap.iter().filter(|d| d.parent_id == Some(meta)).collect();
+    assert_eq!(children.len(), 2);
+}
+
+// ---- terminal reporting + lease release ----
+
+#[test]
+fn terminal_dag_reported_exactly_once_and_lease_released() {
+    let q = JobQueue::new(1);
+    let id = submit(
+        &q,
+        templates::restart("agent-a", Source::Manual, "r".to_owned()),
+    );
+    let stop = claim_one(&q);
+    let r1 = q.complete_node(id, stop.node_id, Ok(()));
+    assert!(r1.terminal.is_empty(), "dag not terminal yet");
+    let rec = claim_one(&q);
+    let r2 = q.complete_node(id, rec.node_id, Ok(()));
+    assert_eq!(r2.terminal.len(), 1);
+    assert_eq!(r2.terminal[0].dag_id, id);
+    assert_eq!(r2.terminal[0].state, State::Done);
+    // Lease released: a new DAG for the agent can claim immediately.
+    let next = submit(
+        &q,
+        templates::reconcile_only(
+            Template::Stop,
+            "agent-a",
+            Source::Manual,
+            "stop".to_owned(),
+            None,
+        ),
+    );
+    let c = claim_one(&q);
+    assert_eq!(c.dag_id, next);
+    assert!(c.lease_acquired);
+}
+
+#[test]
+fn cancelled_dag_reports_terminal() {
+    let q = JobQueue::new(1);
+    let id = submit(&q, rebuild("agent-a", "r"));
+    assert!(q.cancel(id));
+    // The cancel path settles internally; a subsequent completion
+    // report must not re-report it. Verify via a second dag's cycle.
+    let other = submit(&q, rebuild("agent-b", "r"));
+    let c = claim_one(&q);
+    assert_eq!(c.dag_id, other);
+    let report = q.complete_node(other, c.node_id, Err("boom".to_owned()));
+    // agent-b's dag isn't terminal (reconcile still pending) and
+    // agent-a's was already reported by cancel → nothing here.
+    assert!(report.terminal.iter().all(|t| t.dag_id != id));
+}
+
+// ---- steps, build logs, history ----
+
+#[test]
+fn set_step_only_on_running_and_signals_change() {
+    let q = JobQueue::new(1);
+    let id = submit(&q, rebuild("agent-a", "r"));
+    assert!(!q.set_step(id, 0, "too early"), "queued node refuses step");
+    let c = claim_one(&q);
+    assert!(q.set_step(id, c.node_id, "nix build"));
+    assert!(
+        !q.set_step(id, c.node_id, "nix build"),
+        "same label → false"
+    );
+    assert!(q.set_step(id, c.node_id, "next phase"));
+    assert!(q.set_step_running(id, "via running lookup"));
+    q.complete_node(id, c.node_id, Ok(()));
+    let snap = q.snapshot();
+    let node = &snap.iter().find(|d| d.id == id).expect("dag").nodes[0];
+    assert_eq!(node.step, None, "step cleared on completion");
+}
+
+#[test]
+fn set_build_log_id_links_running_node() {
+    let q = JobQueue::new(1);
+    let id = submit(&q, rebuild("agent-a", "r"));
+    assert!(!q.set_build_log_id(id, 0, 41), "queued node refuses log id");
+    let c = claim_one(&q);
+    assert!(q.set_build_log_id(id, c.node_id, 42));
+    assert!(q.set_build_log_id_running(id, 43));
+    q.complete_node(id, c.node_id, Ok(()));
+    let snap = q.snapshot();
+    let node = &snap.iter().find(|d| d.id == id).expect("dag").nodes[0];
+    assert_eq!(node.build_log_id, Some(43), "log id survives completion");
+}
+
+#[test]
+fn history_evicts_old_terminals_per_template() {
+    let q = JobQueue::new(1);
+    for i in 0..8 {
+        let id = submit(
+            &q,
+            templates::reconcile_only(
+                Template::Start,
+                &format!("agent-{i}"),
+                Source::Manual,
+                "start".to_owned(),
+                None,
+            ),
+        );
+        let c = claim_one(&q);
+        q.complete_node(id, c.node_id, Ok(()));
+    }
+    assert_eq!(q.snapshot().len(), 5, "per-template history cap");
+    assert_eq!(q.live_count(), 0);
+}
+
+#[test]
+fn error_is_truncated() {
+    let q = JobQueue::new(1);
+    let id = submit(&q, rebuild("agent-a", "r"));
+    let c = claim_one(&q);
+    q.complete_node(id, c.node_id, Err("x".repeat(5000)));
+    let snap = q.snapshot();
+    let err = snap.iter().find(|d| d.id == id).expect("dag").nodes[0]
+        .error
+        .clone()
+        .expect("error stored");
+    assert!(err.chars().count() <= 2001, "truncated + ellipsis");
+    assert!(err.ends_with('…'));
+}
+
+// ---- template shapes ----
+
+#[test]
+fn graceful_stop_shape_signal_drain_reconcile() {
+    let q = JobQueue::new(1);
+    let id = submit(
+        &q,
+        templates::graceful_stop("agent-a", Source::Manual, "graceful".to_owned()),
+    );
+    for expected in ["signal", "drain", "reconcile"] {
+        let c = claim_one(&q);
+        assert_eq!(c.kind.as_str(), expected);
+        q.complete_node(id, c.node_id, Ok(()));
+    }
+    assert_eq!(state_of(&q, id), State::Done);
+}
+
+#[test]
+fn graceful_signal_and_drain_hold_no_build_slot() {
+    // A whole-hive graceful stop overlaps every drain even at
+    // buildSlots = 1 while a rebuild hogs the slot.
+    let q = JobQueue::new(1);
+    submit(&q, rebuild("builder", "slot hog"));
+    submit(
+        &q,
+        templates::graceful_stop("agent-a", Source::Manual, "g".to_owned()),
+    );
+    submit(
+        &q,
+        templates::graceful_stop("agent-b", Source::Manual, "g".to_owned()),
+    );
+    let claims = q.claim_ready();
+    let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect();
+    assert_eq!(
+        kinds,
+        vec!["prebuild", "signal", "signal"],
+        "both agents' signals fire while the slot is held"
+    );
+}
+
+#[test]
+fn spawn_shape_create_dropin_reconcile() {
+    let q = JobQueue::new(1);
+    let id = submit(
+        &q,
+        templates::spawn("newbie", 7, "approval #7 spawn".to_owned()),
+    );
+    for expected in ["create", "write_dropin", "reconcile"] {
+        let c = claim_one(&q);
+        assert_eq!(c.kind.as_str(), expected);
+        assert_eq!(c.approval_id, Some(7));
+        q.complete_node(id, c.node_id, Ok(()));
+    }
+    let report_terminal = state_of(&q, id);
+    assert_eq!(report_terminal, State::Done);
+}
+
+#[test]
+fn perm_change_shape_prefixes_rebuild_chain() {
+    let q = JobQueue::new(1);
+    let id = submit(
+        &q,
+        templates::perm_change(
+            "agent-a",
+            Source::Manual,
+            "perm".to_owned(),
+            PermPayload::Combined {
+                groups: Some(vec![]),
+                caps: None,
+            },
+        ),
+    );
+    for expected in [
+        "write_perm_file",
+        "prebuild",
+        "stop_for_update",
+        "swap",
+        "reconcile",
+    ] {
+        let c = claim_one(&q);
+        assert_eq!(c.kind.as_str(), expected);
+        q.complete_node(id, c.node_id, Ok(()));
+    }
+    assert_eq!(state_of(&q, id), State::Done);
+}

hive-c0re/src/lib.rs

@@ -32,6 +32,7 @@ pub mod forge;
 pub mod gateway_nginx;
 pub mod hive_stats;
 pub mod host_stats;
+pub mod job_queue;
 pub mod knowledge;
 pub mod lifecycle;
 pub mod limits;
@@ -41,9 +42,9 @@ pub mod meta;
 pub mod migrate;
 pub mod operator_questions;
 pub mod paths;
+pub mod power;
 pub mod priv_client;
 pub mod questions;
-pub mod rebuild_queue;
 pub mod reminder_scheduler;
 pub mod scheduled_prompts;
 pub mod scheduled_prompts_worker;

hive-c0re/src/lifecycle.rs

@@ -260,6 +260,16 @@ async fn port_collision(self_name: &str) -> Option<String> {
 }
 
 pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
+    create_container(name, hive, paths).await?;
+    write_dropins(name, hive, paths).await?;
+    priv_run("start", name).await
+}
+
+/// First-spawn provisioning + `nixos-container create`, without the
+/// drop-in write or the start — the job queue's `Create` node.
+/// `spawn` composes this with `write_dropins` + start for direct
+/// callers (root-agent bootstrap).
+pub async fn create_container(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
     validate(name)?;
     if let Some(other) = port_collision(name).await {
         bail!(
@@ -277,8 +287,17 @@ pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()>
     // ref resolves.
     let agents = agents_after_spawn(name).await?;
     crate::meta::sync_agents(hive, &agents).await?;
+    priv_run("create", name).await
+}
+
+/// Re-apply the per-container host-side config: nspawn flags (bind
+/// mounts etc.), the systemd resource-limits drop-in, and a daemon
+/// reload so both take effect on the next unit (re)start. Idempotent —
+/// the job queue's `WriteDropin` node, also folded into every `Swap`
+/// (rebuild is the reconcile verb).
+pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
+    validate(name)?;
     let container = container_name(name);
-    priv_run("create", name).await?;
     set_nspawn_flags(
         &container,
         &paths.agent_dir,
@@ -287,8 +306,42 @@ pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()>
     )
     .await?;
     set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?;
-    systemd_daemon_reload().await?;
+    systemd_daemon_reload().await
-    priv_run("start", name).await
+}
+
+/// Rebuild-path preamble shared by the job queue's `Prebuild` node and
+/// `rebuild_no_meta`: fail fast on a port collision, then make sure
+/// the applied repo + state dirs exist. Container untouched.
+pub async fn prepare_rebuild_dirs(name: &str, paths: &AgentPaths) -> Result<()> {
+    validate(name)?;
+    if let Some(other) = port_collision(name).await {
+        bail!(
+            "port {} is already taken by '{other}' — rename one of them and retry",
+            agent_web_port(name)
+        );
+    }
+    setup_applied(&paths.applied_dir, None, name).await?;
+    ensure_agent_state_subvolume(name).await?;
+    ensure_claude_dir(&paths.claude_dir)?;
+    ensure_state_dir(&paths.notes_dir)?;
+    Ok(())
+}
+
+/// Profile-swap for an existing, stopped container: re-apply the
+/// drop-ins, then `nixos-container update`. The job queue's `Swap`
+/// node. Requires the container stopped (the queue's `StopForUpdate`
+/// upstream); does NOT start it — the DAG's tail `Reconcile` owns
+/// bringing the agent back to its wanted power state.
+pub async fn swap_update(
+    name: &str,
+    hive: &HiveEnv,
+    paths: &AgentPaths,
+    on_step: &(dyn Fn(&str) + Send + Sync),
+    on_build_log_id: &(dyn Fn(i64) + Send + Sync),
+) -> Result<()> {
+    write_dropins(name, hive, paths).await?;
+    on_step("nixos-container update");
+    priv_run_inner("update", name, Some(on_build_log_id)).await
 }
 
 /// Build the `AgentSpec` list for the meta flake from `nixos-container
@@ -532,35 +585,16 @@ pub async fn rebuild_no_meta(
     on_step: &(dyn Fn(&str) + Send + Sync),
     on_build_log_id: &(dyn Fn(i64) + Send + Sync),
 ) -> Result<bool> {
-    validate(name)?;
+    prepare_rebuild_dirs(name, paths).await?;
-    if let Some(other) = port_collision(name).await {
-        bail!(
-            "port {} is already taken by '{other}' — rename one of them and retry",
-            agent_web_port(name)
-        );
-    }
-    setup_applied(&paths.applied_dir, None, name).await?;
-    ensure_agent_state_subvolume(name).await?;
-    ensure_claude_dir(&paths.claude_dir)?;
-    ensure_state_dir(&paths.notes_dir)?;
-    let container = container_name(name);
     let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display());
     if container_exists(name).await {
         // Rebuild strategy: stop-before-update + pre-build.
         // See `docs/coordinator.md::Container lifecycle`.
         let was_running = is_running(name).await;
-        set_nspawn_flags(
+        write_dropins(name, hive, paths).await?;
-            &container,
-            &paths.agent_dir,
-            &paths.claude_dir,
-            &paths.notes_dir,
-        )
-        .await?;
-        set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?;
-        systemd_daemon_reload().await?;
         if was_running {
             on_step("nix build");
-            prebuild_toplevel(name, &flake_ref).await?;
+            prebuild_toplevel(name, &flake_ref, &|_| ()).await?;
             on_step("nixos-container stop");
             priv_run("stop", name).await?;
         }
@@ -601,15 +635,7 @@ pub async fn rebuild_no_meta(
         // See `docs/coordinator.md::Spawn path`.
         on_step("nixos-container create");
         priv_run("create", name).await?;
-        set_nspawn_flags(
+        write_dropins(name, hive, paths).await?;
-            &container,
-            &paths.agent_dir,
-            &paths.claude_dir,
-            &paths.notes_dir,
-        )
-        .await?;
-        set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?;
-        systemd_daemon_reload().await?;
         on_step("nixos-container start");
         priv_run("start", name).await?;
         Ok(false)
@@ -622,7 +648,15 @@ pub async fn rebuild_no_meta(
 /// is untouched. See `docs/coordinator.md::Rebuild path` for why
 /// the prebuild happens before stop, and `docs/coordinator.md::Prebuild
 /// attr path` for why the explicit nixosConfigurations attr is required.
-async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> {
+///
+/// `on_build_log_id` fires with the `build_logs` row id as soon as the
+/// row opens, so queue-side callers can link their node to the live
+/// stream. Pass `&|_| ()` when not needed.
+pub async fn prebuild_toplevel(
+    name: &str,
+    flake_ref: &str,
+    on_build_log_id: &(dyn Fn(i64) + Send + Sync),
+) -> Result<()> {
     use tokio::io::{AsyncBufReadExt, BufReader};
     // Split `<root>#<name>` so we can re-emit with the explicit
     // `nixosConfigurations.<name>` segment. The flake_ref shape is
@@ -663,6 +697,9 @@ async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> {
             })
             .ok()
     });
+    if let Some(id) = log_id {
+        on_build_log_id(id);
+    }
 
     let mut child = Command::new("nix")
         .args(&args)

hive-c0re/src/main.rs

@@ -13,8 +13,8 @@ use hive_sh4re::{HostRequest, HostResponse};
 use hive_c0re::coordinator::{Coordinator, HiveEnv, ServeConfig};
 use hive_c0re::{
     agent_sockets, auto_update, broker, client, crash_watch, dashboard, dashboard_events, forge,
-    knowledge, matrix, migrate, rebuild_queue, reminder_scheduler, scheduled_prompts_worker,
+    job_queue, knowledge, matrix, migrate, reminder_scheduler, scheduled_prompts_worker, server,
-    server, socket_server,
+    socket_server,
 };
 
 #[derive(Parser)]
@@ -85,6 +85,11 @@ enum Cmd {
         /// option.
         #[arg(long)]
         model_prices: Option<String>,
+        /// Override: number of concurrent nix-heavy job-queue nodes
+        /// (prebuild / profile-swap / create / meta lock). Set via the
+        /// `services.hyperhive.c0re.buildSlots` NixOS option.
+        #[arg(long)]
+        build_slots: Option<usize>,
     },
     /// Spawn a new agent container directly (`hive-agent-<name>`). Bypasses
     /// the approval queue — use only as an operator on the host. For
@@ -156,6 +161,7 @@ async fn main() -> Result<()> {
             agent_cpu_quota,
             agent_memory_max,
             model_prices,
+            build_slots,
         } => {
             // Base config from the --config file (or the built-in
             // defaults), then apply any per-flag overrides — config
@@ -195,7 +201,10 @@ async fn main() -> Result<()> {
                 sc.model_prices =
                     serde_json::from_str(&v).context("--model-prices: invalid JSON")?;
             }
-            cmd_serve(sc.env, sc.model_prices, db, &cli.socket).await
+            if let Some(v) = build_slots {
+                sc.build_slots = v;
+            }
+            cmd_serve(sc.env, sc.model_prices, sc.build_slots, db, &cli.socket).await
         }
         Cmd::Spawn { name } => {
             render(client::request(&cli.socket, HostRequest::Spawn { name }).await?)
@@ -244,6 +253,7 @@ async fn main() -> Result<()> {
 async fn cmd_serve(
     env: HiveEnv,
     model_prices: hive_c0re::hive_stats::PriceTable,
+    build_slots: usize,
     db: std::path::PathBuf,
     socket: &std::path::Path,
 ) -> Result<()> {
@@ -255,7 +265,7 @@ async fn cmd_serve(
     // `dashboard_port` is consumed into the Coordinator below; capture the
     // Copy value first for the dashboard + knowledge-webhook tasks.
     let dashboard_port = env.dashboard_port;
-    let coord = Arc::new(Coordinator::open(&db, env, model_prices)?);
+    let coord = Arc::new(Coordinator::open(&db, env, model_prices, build_slots)?);
     socket_server::start_manager(coord.clone())?;
     // Idempotent pre-flight: rewrite pre-meta-layout applied
     // repos, ensure proposed repos carry the `applied`
@@ -413,25 +423,17 @@ async fn cmd_serve(
     // and fans the body out to each active target's inbox. See
     // scheduled_prompts_worker.rs.
     scheduled_prompts_worker::spawn(coord.clone());
-    // Rebuild-queue worker: drains the global rebuild/meta-update/
+    // Job-queue scheduler: drives the global DAG queue (rebuild /
-    // spawn queue FIFO so hive-c0re never runs two heavyweight
+    // meta-update / spawn / power ops). Concurrency comes from the
-    // container ops concurrently. Existing rebuild call sites
+    // build-slot count + per-agent leases inside the queue, not from
-    // (auto_update, dashboard, manager, approval handler) enqueue
+    // multiple workers — cheap nodes (graceful signals, drains,
-    // here instead of awaiting `rebuild_agent` inline. See
+    // reconciles) overlap nix-heavy ones structurally. Call sites
-    // `rebuild_queue.rs`.
+    // (auto_update, dashboard, manager, approval handler) submit DAGs
+    // instead of awaiting lifecycle work inline. See `job_queue/`.
     {
         let q_coord = coord.clone();
         tokio::spawn(async move {
-            rebuild_queue::run_worker(q_coord).await;
+            job_queue::scheduler::run_worker(q_coord).await;
-        });
-        // Fast lane: a second serial worker for hard Start/Stop, running
-        // concurrently with the build worker above so a stop/start never
-        // waits behind a slow build for another container. Per-agent
-        // ordering vs that agent's own build is enforced in the queue's
-        // claim logic (a fast op defers behind its agent's running build).
-        let fast_coord = coord.clone();
-        tokio::spawn(async move {
-            rebuild_queue::run_fast_worker(fast_coord).await;
         });
     }
     // Forward every broker event onto the unified dashboard

hive-c0re/src/power.rs

@@ -0,0 +1,203 @@
+//! Durable per-agent power *intent* (`wanted: Up | Offline`) — the
+//! spec half of spec-vs-status desired-state reconciliation.
+//! `container_view` remains the observed *status*; the job queue's
+//! `Reconcile` nodes are the mechanism that converges the two.
+//!
+//! Stored in `/var/lib/hyperhive/db/agent_power.sqlite` (one tiny row
+//! per agent). Intent persists across hive-c0re restarts; in-flight
+//! queue work deliberately does not. Setting `wanted` is never a
+//! queued node: operator/intent actions update the row synchronously
+//! at request time, then submit the DAG whose terminal `Reconcile`
+//! reads the fresh value — rapid toggles are last-writer-wins and the
+//! reconciles converge. Power toggles never commit to the meta repo.
+
+use std::path::Path;
+use std::sync::Mutex;
+
+use anyhow::{Context, Result};
+use rusqlite::{Connection, OptionalExtension, params};
+
+const SCHEMA: &str = "
+CREATE TABLE IF NOT EXISTS agent_power (
+    agent      TEXT PRIMARY KEY,
+    wanted     TEXT NOT NULL,
+    updated_at INTEGER NOT NULL
+);
+";
+
+/// Per-agent power intent.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Wanted {
+    Up,
+    Offline,
+}
+
+impl Wanted {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Wanted::Up => "up",
+            Wanted::Offline => "offline",
+        }
+    }
+
+    fn parse(s: &str) -> Option<Self> {
+        match s {
+            "up" => Some(Wanted::Up),
+            "offline" => Some(Wanted::Offline),
+            _ => None,
+        }
+    }
+
+    /// Seed value from an observed running state (first boot after
+    /// this store lands, or an agent spawned outside the normal path).
+    pub fn from_running(running: bool) -> Self {
+        if running { Wanted::Up } else { Wanted::Offline }
+    }
+}
+
+/// What a `Reconcile` should do given intent + observation. Pure so
+/// the `{Up,Offline} × {up,down}` matrix is unit-testable.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ReconcileAction {
+    Start,
+    Stop,
+    Noop,
+}
+
+#[must_use]
+pub fn reconcile_action(wanted: Wanted, running: bool) -> ReconcileAction {
+    match (wanted, running) {
+        (Wanted::Up, false) => ReconcileAction::Start,
+        (Wanted::Offline, true) => ReconcileAction::Stop,
+        (Wanted::Up, true) | (Wanted::Offline, false) => ReconcileAction::Noop,
+    }
+}
+
+/// Sqlite-backed store. `Arc`-friendly: all methods take `&self`, the
+/// internal `Mutex<Connection>` serializes access.
+pub struct PowerStore {
+    conn: Mutex<Connection>,
+}
+
+impl PowerStore {
+    pub fn open(db_dir: &Path) -> Result<Self> {
+        std::fs::create_dir_all(db_dir)
+            .with_context(|| format!("create agent_power db parent {}", db_dir.display()))?;
+        let path = db_dir.join("agent_power.sqlite");
+        let conn = Connection::open(&path)
+            .with_context(|| format!("open agent_power db {}", path.display()))?;
+        conn.execute_batch(SCHEMA)
+            .context("apply agent_power schema")?;
+        Ok(Self {
+            conn: Mutex::new(conn),
+        })
+    }
+
+    /// In-memory store for tests.
+    #[cfg(test)]
+    pub fn open_in_memory() -> Result<Self> {
+        let conn = Connection::open_in_memory().context("open in-memory agent_power db")?;
+        conn.execute_batch(SCHEMA)
+            .context("apply agent_power schema")?;
+        Ok(Self {
+            conn: Mutex::new(conn),
+        })
+    }
+
+    /// Read an agent's intent. `None` when the agent has no row yet
+    /// (callers seed from observed state via [`Self::get_or_seed`]).
+    pub fn get(&self, agent: &str) -> Result<Option<Wanted>> {
+        let conn = self.conn.lock().expect("agent_power mutex poisoned");
+        let row: Option<String> = conn
+            .query_row(
+                "SELECT wanted FROM agent_power WHERE agent = ?1",
+                params![agent],
+                |r| r.get(0),
+            )
+            .optional()
+            .context("select agent_power")?;
+        Ok(row.and_then(|s| Wanted::parse(&s)))
+    }
+
+    /// Write an agent's intent (last-writer-wins, synchronous at
+    /// request time).
+    pub fn set(&self, agent: &str, wanted: Wanted) -> Result<()> {
+        let conn = self.conn.lock().expect("agent_power mutex poisoned");
+        conn.execute(
+            "INSERT INTO agent_power (agent, wanted, updated_at) VALUES (?1, ?2, ?3)
+             ON CONFLICT(agent) DO UPDATE SET wanted = ?2, updated_at = ?3",
+            params![agent, wanted.as_str(), now_secs()],
+        )
+        .context("upsert agent_power")?;
+        Ok(())
+    }
+
+    /// Read an agent's intent, seeding the row from the observed
+    /// running state when absent — the migration rule for agents that
+    /// predate this store (running ⇒ `Up`, stopped ⇒ `Offline`), after
+    /// which the DB is authoritative.
+    pub fn get_or_seed(&self, agent: &str, running: bool) -> Result<Wanted> {
+        if let Some(w) = self.get(agent)? {
+            return Ok(w);
+        }
+        let seeded = Wanted::from_running(running);
+        self.set(agent, seeded)?;
+        tracing::info!(%agent, wanted = seeded.as_str(), "agent_power: seeded from observed state");
+        Ok(seeded)
+    }
+
+    /// Drop an agent's row (container destroyed).
+    pub fn remove(&self, agent: &str) -> Result<()> {
+        let conn = self.conn.lock().expect("agent_power mutex poisoned");
+        conn.execute("DELETE FROM agent_power WHERE agent = ?1", params![agent])
+            .context("delete agent_power")?;
+        Ok(())
+    }
+}
+
+fn now_secs() -> i64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .ok()
+        .and_then(|d| i64::try_from(d.as_secs()).ok())
+        .unwrap_or(0)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// The full `{Up,Offline} × {up,down}` reconcile matrix:
+    /// start / stop / noop / noop.
+    #[test]
+    fn reconcile_matrix() {
+        assert_eq!(reconcile_action(Wanted::Up, false), ReconcileAction::Start);
+        assert_eq!(
+            reconcile_action(Wanted::Offline, true),
+            ReconcileAction::Stop
+        );
+        assert_eq!(reconcile_action(Wanted::Up, true), ReconcileAction::Noop);
+        assert_eq!(
+            reconcile_action(Wanted::Offline, false),
+            ReconcileAction::Noop
+        );
+    }
+
+    #[test]
+    fn get_set_roundtrip_and_seed() {
+        let store = PowerStore::open_in_memory().expect("open");
+        assert_eq!(store.get("alice").expect("get"), None);
+        // Seed from observed running state, once.
+        assert_eq!(store.get_or_seed("alice", true).expect("seed"), Wanted::Up);
+        // Thereafter the DB is authoritative — observed state no longer
+        // overrides.
+        assert_eq!(
+            store.get_or_seed("alice", false).expect("seeded"),
+            Wanted::Up
+        );
+        store.set("alice", Wanted::Offline).expect("set");
+        assert_eq!(store.get("alice").expect("get"), Some(Wanted::Offline));
+        store.remove("alice").expect("remove");
+        assert_eq!(store.get("alice").expect("get"), None);
+    }
+}

hive-c0re/src/server.rs

@@ -131,7 +131,7 @@ async fn dispatch(req: &HostRequest, coord: Arc<Coordinator>) -> HostResponse {
                     agents.retain(|a| prev.contains(a));
                 }
                 let infra = scoped_infra(scope);
-                handle_start(&agents, &infra).await?
+                handle_start(&coord, &agents, &infra).await?
             }
             HostRequest::Destroy { name, purge } => {
                 actions::destroy(&coord, name, *purge).await?;
@@ -201,6 +201,9 @@ async fn handle_spawn(coord: &Arc<Coordinator>, name: &str) -> Result<HostRespon
     let paths = Coordinator::agent_paths(name, agent_dir);
     match lifecycle::spawn(name, &hive, &paths).await {
         Ok(()) => {
+            if let Err(e) = coord.power.set(name, crate::power::Wanted::Up) {
+                tracing::warn!(%name, error = ?e, "agent_power: set wanted=up failed");
+            }
             coord.notify_manager(&hive_sh4re::HelperEvent::Spawned {
                 agent: name.to_owned(),
                 ok: true,
@@ -224,8 +227,12 @@ async fn handle_spawn(coord: &Arc<Coordinator>, name: &str) -> Result<HostRespon
 }
 
 /// Kill `name`'s container, unregister its socket, notify the manager.
+/// Persists `wanted = Offline` first so reconciles don't undo the kill.
 async fn handle_kill(coord: &Arc<Coordinator>, name: &str) -> Result<HostResponse> {
     tracing::info!(%name, "kill");
+    if let Err(e) = coord.power.set(name, crate::power::Wanted::Offline) {
+        tracing::warn!(%name, error = ?e, "agent_power: set wanted=offline failed");
+    }
     lifecycle::kill(name).await?;
     coord.unregister_agent(name);
     coord.notify_manager(&hive_sh4re::HelperEvent::Killed {
@@ -284,25 +291,29 @@ async fn handle_stop(
     tracing::info!(?agents, ?infra, graceful, "stop");
     let mut ok_items: Vec<String> = Vec::new();
     let mut errors: Vec<String> = Vec::new();
-    let mut enqueued_graceful = false;
 
     for agent in agents {
         if graceful {
-            // Graceful stop: enqueue the quiesce orchestration rather than a
+            // Graceful stop: submit the quiesce DAG rather than a hard
-            // hard kill. Serialised through the rebuild queue so it can't race
+            // kill. The per-agent lease keeps it from racing an
-            // an in-flight rebuild for the same agent, and its per-step
+            // in-flight rebuild for the same agent; the cheap Signal
-            // progress surfaces on the queue snapshot + build log.
+            // nodes all fire immediately so every agent's drain
-            coord.rebuild_queue.enqueue(
+            // overlaps. `submit::graceful_stop` also persists
-                crate::rebuild_queue::QueueKind::GracefulStop,
+            // `wanted = Offline` and emits the queue snapshot.
-                agent.clone(),
+            crate::job_queue::submit::graceful_stop(
-                crate::rebuild_queue::QueueSource::Manual,
+                coord,
+                agent,
+                crate::job_queue::Source::Manual,
                 "manual via hivectl graceful stop".to_owned(),
-                None,
             );
             ok_items.push(agent.clone());
-            enqueued_graceful = true;
             continue;
         }
+        // Persist the intent even if the kill itself fails — otherwise
+        // the next boot reconcile would restart the agent.
+        if let Err(e) = coord.power.set(agent, crate::power::Wanted::Offline) {
+            tracing::warn!(%agent, error = ?e, "agent_power: set wanted=offline failed");
+        }
         match lifecycle::kill(agent).await {
             Ok(()) => ok_items.push(agent.clone()),
             Err(e) => {
@@ -311,9 +322,6 @@ async fn handle_stop(
             }
         }
     }
-    if enqueued_graceful {
-        coord.emit_rebuild_queue_snapshot();
-    }
 
     for &container in infra {
         let name = container.unit_name();
@@ -333,7 +341,11 @@ async fn handle_stop(
 /// inverse of [`handle_stop`]. Infra comes up before agents so the agents
 /// find forge/matrix/gateway ready. Per-target failures aggregated. Callers
 /// resolve the [`LifecycleScope`] to these explicit name lists up front.
-async fn handle_start(agents: &[String], infra: &[InfraContainer]) -> Result<HostResponse> {
+async fn handle_start(
+    coord: &Arc<Coordinator>,
+    agents: &[String],
+    infra: &[InfraContainer],
+) -> Result<HostResponse> {
     tracing::info!(?agents, ?infra, "start");
     let mut ok_items: Vec<String> = Vec::new();
     let mut errors: Vec<String> = Vec::new();
@@ -350,6 +362,11 @@ async fn handle_start(agents: &[String], infra: &[InfraContainer]) -> Result<Hos
     }
 
     for agent in agents {
+        // Persist the intent even if the start itself fails — the next
+        // reconcile (boot or queued) retries toward `Up`.
+        if let Err(e) = coord.power.set(agent, crate::power::Wanted::Up) {
+            tracing::warn!(%agent, error = ?e, "agent_power: set wanted=up failed");
+        }
         match lifecycle::start(agent).await {
             Ok(()) => ok_items.push(agent.clone()),
             Err(e) => {

hive-c0re/src/socket_server.rs

@@ -510,7 +510,7 @@ async fn dispatch(req: &AgentRequest, agent: &str, coord: &Arc<Coordinator>) ->
     match req {
         // Lifecycle + config: caller must be an ancestor of the target
         // (a parent owns its whole subtree; the root covers every agent).
-        AgentRequest::Start { name } => handle_start(coord, agent, name).await,
+        AgentRequest::Start { name } => handle_start(coord, agent, name),
         AgentRequest::Restart { name } => handle_restart(coord, agent, name).await,
         AgentRequest::Kill { name } => handle_kill(coord, agent, name).await,
         AgentRequest::Update { name } => handle_update(coord, agent, name),
@@ -786,39 +786,21 @@ fn handle_reminder_rollup(
 
 /// `Start` — start a container, kicking its next turn. The caller must be an
 /// ancestor of `name` in the topology (the root covers every agent).
-async fn handle_start(coord: &Arc<Coordinator>, agent: &str, name: &str) -> AgentResponse {
+fn handle_start(coord: &Arc<Coordinator>, agent: &str, name: &str) -> AgentResponse {
     if let Some(err) = require_descendant(agent, name, "start") {
         return err;
     }
     tracing::info!(%agent, %name, "start container");
-    // If the hyperhive rev is stale, route through the rebuild queue so the
+    // Persist `wanted = Up` and submit the Start DAG; the submit layer
-    // container runs current nix derivations before it starts. Same logic as
+    // upgrades a stale-rev start to a full rebuild so the container
-    // `run_start`; this covers the MCP `start` tool path.
+    // runs current nix derivations before it starts.
-    let current_rev = crate::auto_update::current_flake_rev(&coord.hyperhive_flake);
+    crate::job_queue::submit::start(
-    if let Some(ref rev) = current_rev {
+        coord,
-        let stored = std::fs::read_to_string(crate::auto_update::rev_marker_path(name)).ok();
+        name,
-        if stored.as_deref() != Some(rev.as_str()) {
+        crate::job_queue::Source::Manual,
-            tracing::info!(%agent, %name, "start: rev stale — enqueuing rebuild");
+        format!("agent `{agent}` start tool"),
-            coord.rebuild_queue.enqueue(
+    );
-                crate::rebuild_queue::QueueKind::Rebuild,
+    AgentResponse::Ok
-                name.to_owned(),
-                crate::rebuild_queue::QueueSource::Manual,
-                format!("start {name}: rev stale — rebuilding first"),
-                None,
-            );
-            coord.emit_rebuild_queue_snapshot();
-            return AgentResponse::Ok;
-        }
-    }
-    match crate::lifecycle::start(name).await {
-        Ok(()) => {
-            coord.kick_agent(name, "container started");
-            AgentResponse::Ok
-        }
-        Err(e) => AgentResponse::Err {
-            message: format!("{e:#}"),
-        },
-    }
 }
 
 /// `Restart` — enqueue a restart for a container. The caller must be an
@@ -838,15 +820,13 @@ async fn handle_restart(coord: &Arc<Coordinator>, agent: &str, name: &str) -> Ag
     if let Some(err) = require_descendant(agent, name, "restart") {
         return err;
     }
-    tracing::info!(%agent, %name, "enqueue restart");
+    tracing::info!(%agent, %name, "submit restart");
-    coord.rebuild_queue.enqueue(
+    crate::job_queue::submit::restart(
-        crate::rebuild_queue::QueueKind::Restart,
+        coord,
-        name.to_owned(),
+        name,
-        crate::rebuild_queue::QueueSource::Manual,
+        crate::job_queue::Source::Manual,
         format!("agent `{agent}` restart tool"),
-        None,
     );
-    coord.emit_rebuild_queue_snapshot();
     AgentResponse::Ok
 }
 
@@ -908,6 +888,11 @@ async fn handle_kill(coord: &Arc<Coordinator>, agent: &str, name: &str) -> Agent
         return err;
     }
     tracing::info!(%agent, %name, "kill container");
+    // Persist the intent even if the kill fails — otherwise the next
+    // reconcile would restart the container.
+    if let Err(e) = coord.power.set(name, crate::power::Wanted::Offline) {
+        tracing::warn!(%name, error = ?e, "agent_power: set wanted=offline failed");
+    }
     let result: anyhow::Result<()> = async {
         crate::lifecycle::kill(name).await?;
         coord.unregister_agent(name);
@@ -933,15 +918,13 @@ fn handle_update(coord: &Arc<Coordinator>, agent: &str, name: &str) -> AgentResp
     if let Some(err) = require_descendant(agent, name, "rebuild") {
         return err;
     }
-    tracing::info!(%agent, %name, "enqueue rebuild");
+    tracing::info!(%agent, %name, "submit rebuild");
-    coord.rebuild_queue.enqueue(
+    crate::job_queue::submit::rebuild(
-        crate::rebuild_queue::QueueKind::Rebuild,
+        coord,
-        name.to_owned(),
+        name,
-        crate::rebuild_queue::QueueSource::Manual,
+        crate::job_queue::Source::Manual,
         format!("agent `{agent}` update tool"),
-        None,
     );
-    coord.emit_rebuild_queue_snapshot();
     AgentResponse::Ok
 }