job_queue: grow the rebuild subgraph from DeployApply (#2664)

The config-PR deploy's apply node still did the whole container rebuild
inline, through the last surviving `lifecycle::rebuild_no_meta` call. It
now merges, opens the two-phase meta deploy, and returns the ordinary
rebuild chain as a subgraph the scheduler grafts into the live DAG under
it. A new `FinalizeDeploy` node, gated on that graft, plants the deploy
tag and commits the staged lock.

Net effect: "did the agent come back up?" is answered by `Reconcile`
succeeding, the same way it is for every other rebuild, instead of by a
fused inline start — and each deploy phase is its own queue node, so the
dashboard shows which one is running.

The grafted nodes root on the apply node, so they land inside
`DeployWindow`'s subtree and re-enter the meta window and build slot it
already holds rather than deadlocking against them. The new happy-path
test runs on a one-slot queue specifically to pin that down.

`FinalizeDeploy`'s two git writes are fatal, deliberately: they are what
tells `DeployTail` a deploy confirmed good, so a node that merely warned
on them could report success while leaving the tail looking at the git
state of a failure — and the tail would then roll a good deploy back.
The trailing `meta::finalize_deploy` stays warn-only, since by then the
container already runs the new config.

The `failed/<id>` annotated tag moves into the tail, which is now the
only place holding a failed deploy. It reads the reason off the DAG via
a new `JobQueue::first_error`, and is gated on `main` having actually
moved — the rollback ref is parked *before* the merge, so its existence
alone does not mean a merge happened, and a pre-merge rejection must not
tag the previous, innocent head.

Removing the last inline rebuild orphaned a chain of now-dead code:
`rebuild_no_meta`, `container_exists`, `Coordinator::set_queue_build_log`
and `JobQueue::set_build_log_id_running`, all deleted here.

hive-c0re/src/job_queue/exec.rs

@@ -31,7 +31,7 @@ pub struct NodeOutput {
     /// runtime — the single in-DAG-growth channel. Each inner
     /// `Vec<NodeSpec>` is one independent subgraph whose `deps` are local
     /// (0-based within that subgraph); the scheduler appends each via
-    /// [`JobQueue::append_subgraph`], which rebases the deps onto the DAG's
+    /// [`super::JobQueue::append_subgraph`], which rebases the deps onto the DAG's
     /// node-id space and roots the subgraph on the emitting node. Used both
     /// for the multi-node case (`MetaLock` growing one rebuild subgraph per
     /// agent — the startup sweep's stale agents, the meta-update cascade's
@@ -102,6 +102,7 @@ pub(super) async fn run_node(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
         NodeKind::DeployWindow { .. } => run_deploy_window(claim),
         NodeKind::MergeVerify { .. } => run_merge_verify(coord, claim).await,
         NodeKind::DeployApply { .. } => run_deploy_apply(coord, claim).await,
+        NodeKind::FinalizeDeploy { .. } => run_finalize_deploy(coord, claim).await,
         NodeKind::DeployTail { .. } => run_deploy_tail(coord, claim).await,
         NodeKind::SetWanted { up, .. } => run_set_wanted(coord, claim, *up),
         // Pure grouping container — no work; completing it lets it reach
@@ -615,10 +616,27 @@ async fn run_merge_verify(coord: &Arc<Coordinator>, claim: &Claim) -> Result<Nod
         .map(|()| NodeOutput::default())
 }
 
-/// Deploy phase 2 — the irreversible half: ff-merge, two-phase meta deploy,
-/// container rebuild, finalize.
+/// Deploy phase 2 — the irreversible half: ff-merge, then phase 1 of the
+/// two-phase meta deploy.
+///
+/// On success it grows the ordinary rebuild subgraph (plus its closing
+/// `FinalizeDeploy`) into this DAG rooted on *this* node — which is what puts
+/// the appended nodes inside the `DeployWindow`'s subtree, so the `MetaWindow`
+/// their `MetaSync` declares is re-entered rather than deadlocked against the
+/// ancestor already holding it. On failure nothing is appended and the tail
+/// compensates, exactly as before.
 async fn run_deploy_apply(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
-    crate::actions::run_deploy_apply(coord, Some(claim.dag_id), deploy_approval_id(claim)?)
+    crate::actions::run_deploy_apply(coord, Some(claim.dag_id), deploy_approval_id(claim)?).await?;
+    Ok(NodeOutput {
+        append_subgraph: vec![super::templates::deploy_rebuild_nodes(claim.kind.agent())],
+    })
+}
+
+/// Deploy phase 3 — close the staged-lock window once the appended rebuild has
+/// come up clean: drop the rollback ref, plant the `deployed/<id>` tag, commit
+/// the staged lock.
+async fn run_finalize_deploy(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
+    crate::actions::run_finalize_deploy(coord, Some(claim.dag_id), deploy_approval_id(claim)?)
         .await
         .map(|()| NodeOutput::default())
 }

hive-c0re/src/job_queue/mod.rs

@@ -500,17 +500,6 @@ impl JobQueue {
         true
     }
 
-    /// Link a `build_logs` row to the DAG's currently-running node — DAG-id-only
-    /// compatibility surface (approval pipeline callbacks).
-    pub fn set_build_log_id_running(&self, dag_id: u64, log_id: i64) -> bool {
-        let mut inner = self.lock();
-        let Some(node_id) = inner.running_node_of(dag_id) else {
-            return false;
-        };
-        inner.node_rt.entry(node_id).or_default().build_log_id = Some(log_id);
-        true
-    }
-
     /// The `build_logs` row id linked to the wire node id `node_id`, if any —
     /// the lookup behind the `GET /api/build-log/<node_id>` query endpoint (the
     /// client fetches a node's captured build output on demand rather than
@@ -526,6 +515,22 @@ impl JobQueue {
             .and_then(|(_, rt)| rt.build_log_id)
     }
 
+    /// The first failed node's error in `dag_id`, if any has failed yet.
+    ///
+    /// Unlike the roll-up summary this is readable *mid-flight*, which is the
+    /// point: a compensation node runs `AfterAny` its subject, so when it asks,
+    /// the DAG is still `Finishing` (the compensation node itself is running)
+    /// while the node it is compensating for has already settled `Failed`. That
+    /// lets the compensation annotate its bookkeeping with the reason the deploy
+    /// failed, instead of having the error handed down from the node that hit
+    /// it. `None` when nothing has failed — the ordinary success path.
+    #[must_use]
+    pub fn first_error(&self, dag_id: u64) -> Option<String> {
+        let inner = self.lock();
+        let container = inner.container(dag_id)?;
+        inner.dag_first_error(container)
+    }
+
     /// A DAG's terminal roll-up summary, computed on demand from its container.
     /// `None` if the DAG id is unknown. Test-only — production reads the summary
     /// `complete_node` returns when the container rolls up terminal.

hive-c0re/src/job_queue/model.rs

@@ -209,16 +209,36 @@ pub enum NodeKind {
     /// untouched, so it is safely retryable and cancel-safe: nothing downstream
     /// has happened yet.
     MergeVerify { agent: String },
-    /// Deploy phase 2 — everything from the irreversible fast-forward onward:
-    /// ff-merge the reviewed head to `main` via the forge API, two-phase meta
-    /// `prepare_deploy`, the container rebuild, then on success the
-    /// `deployed/<id>` tag + `finalize_deploy`.
+    /// Deploy phase 2 — the irreversible fast-forward plus the *opening* half of
+    /// the two-phase meta deploy: park the rollback ref, ff-merge the reviewed
+    /// head to `main` via the forge API, ff `applied/main`, and
+    /// `meta::prepare_deploy` (which stages `flake.lock` uncommitted).
     ///
-    /// Still one node in this increment: splitting the tail into
-    /// `FfMain`/`PrepareDeploy`/rebuild/`FinalizeDeploy` children is the next
-    /// one. What *is* already split out is the compensation path — see
+    /// It does **not** run the container rebuild itself. It grows the ordinary
+    /// rebuild subgraph into this DAG as its own children
+    /// ([`super::templates::deploy_rebuild_nodes`], `relock = false` — the lock
+    /// is already staged), so the multi-minute build renders as the same real
+    /// nodes every other rebuild does instead of one opaque box. Closing the
+    /// staged-lock window is likewise its own node
+    /// ([`NodeKind::FinalizeDeploy`]), and the compensation path is
     /// [`NodeKind::DeployTail`].
     DeployApply { agent: String },
+    /// Deploy phase 3 — close the two-phase meta deploy once the rebuild
+    /// subgraph under [`NodeKind::DeployApply`] has come up clean: drop the
+    /// rollback ref, plant the `deployed/<id>` tag, commit the staged
+    /// `flake.lock` (`meta::finalize_deploy`).
+    ///
+    /// Its two git steps are **fatal**, deliberately. They are the writes that
+    /// tell [`NodeKind::DeployTail`] a deploy confirmed good, so a node that
+    /// merely warned on them could report success while leaving the tail
+    /// looking at the git state of a failure — and the tail would then roll a
+    /// *good* deploy back. Failing loudly keeps the node's outcome and the
+    /// repo's state saying the same thing.
+    ///
+    /// The trailing `meta::finalize_deploy` stays warn-only: by then the
+    /// container already runs the new config, and an uncommitted staged lock is
+    /// something the operator can commit by hand.
+    FinalizeDeploy { agent: String },
     /// Deploy compensation **and bookkeeping** tail — `AfterAny`
     /// [`NodeKind::DeployApply`], so it runs on success, failure, and cancel
     /// alike, in the same spirit as the rebuild template's tail `Reconcile`
@@ -238,7 +258,7 @@ pub enum NodeKind {
     /// parked in the applied repo rather than passed between nodes:
     /// `DeployApply` writes the pre-merge `main` sha to
     /// `refs/hyperhive/rollback/<approval-id>` before the fast-forward and
-    /// deletes it once the deploy has been finalized. So the ref existing *is*
+    /// [`NodeKind::FinalizeDeploy`] deletes it. So the ref existing *is*
     /// the "a merge landed and was not finalized" signal, and its absence makes
     /// this node a no-op. Parking it in git rather than in a node payload also
     /// means it survives a `hive-c0re` restart mid-deploy, which an in-memory
@@ -299,6 +319,7 @@ impl NodeKind {
             NodeKind::DeployWindow { .. } => "deploy_window",
             NodeKind::MergeVerify { .. } => "merge_verify",
             NodeKind::DeployApply { .. } => "deploy_apply",
+            NodeKind::FinalizeDeploy { .. } => "finalize_deploy",
             NodeKind::DeployTail { .. } => "deploy_tail",
             NodeKind::SetWanted { .. } => "set_wanted",
             NodeKind::Dag { .. } => "dag",
@@ -328,6 +349,7 @@ impl NodeKind {
             | NodeKind::DeployWindow { agent }
             | NodeKind::MergeVerify { agent }
             | NodeKind::DeployApply { agent }
+            | NodeKind::FinalizeDeploy { agent }
             | NodeKind::DeployTail { agent }
             | NodeKind::SetWanted { agent, .. } => agent,
             NodeKind::MetaLock { .. } | NodeKind::Dag { .. } => "",
@@ -389,6 +411,13 @@ impl NodeKind {
     /// That is why the meta preamble is its own [`NodeKind::MetaSync`] node,
     /// and why that node is a sibling rather than `Prebuild`'s parent (a
     /// resource held by a parent covers its whole subtree).
+    ///
+    /// Two of these kinds run *inside* a [`NodeKind::DeployWindow`]'s subtree
+    /// (the appended rebuild's `MetaSync`, and [`NodeKind::FinalizeDeploy`]).
+    /// They still declare the window: a descendant re-enters an ancestor's hold
+    /// through the crate's recursive lock, exactly as `Start` / `Stop` re-enter
+    /// a `Reconcile`'s agent lease. Declaring it is what keeps the requirement
+    /// true of the *node* rather than of one particular DAG shape.
     pub fn needs_meta_window(&self) -> bool {
         matches!(
             self,
@@ -397,6 +426,7 @@ impl NodeKind {
                 | NodeKind::MetaLock { .. }
                 | NodeKind::WritePermFile { .. }
                 | NodeKind::DeployWindow { .. }
+                | NodeKind::FinalizeDeploy { .. }
         )
     }
 }

hive-c0re/src/job_queue/templates.rs

@@ -123,6 +123,50 @@ pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u64) -> Vec<NodeSpe
     ]
 }
 
+/// The rebuild subgraph a [`NodeKind::DeployApply`] grows into its own DAG once
+/// the merge has landed and `prepare_deploy` has staged the lock, plus the
+/// [`NodeKind::FinalizeDeploy`] that closes the window behind it.
+///
+/// `relock = false` is the whole reason this composes: `prepare_deploy` already
+/// relocked and staged `flake.lock`, so the appended `MetaSync` must do the dir
+/// prep + `sync_agents` *without* re-locking over it.
+///
+/// `FinalizeDeploy` waits on **two** siblings, which together reproduce the gate
+/// the old fused node had around its inline `rebuild_no_meta` call:
+/// - `AfterOk` `Prebuild` — a parent's state is its roll-up, so this is `Done`
+///   only once `StopForUpdate` → `Swap` → `PostSwap` all are (a failed *or*
+///   cancelled child rolls the parent up `Failed`). That's the old
+///   `build_result`.
+/// - `AfterOk` `Reconcile` — the old call passed `deferred_start = false` on
+///   purpose: the container had to come back up *before* the deploy was
+///   finalized. `Reconcile` alone would not do, being `AfterAny` — it reaches
+///   `Done` even after a failed `Swap`.
+///
+/// Appended, not submitted: the roots below become children of the emitting
+/// `DeployApply` (see [`super::JobQueue::append_subgraph`]), which puts them
+/// inside the `DeployWindow`'s subtree — so the `MetaWindow` this subgraph's
+/// `MetaSync` and `FinalizeDeploy` declare is re-entered from the ancestor
+/// already holding it rather than deadlocking against it.
+pub(crate) fn deploy_rebuild_nodes(agent: &str) -> Vec<NodeSpec> {
+    let mut nodes = rebuild_nodes(agent, false, 0);
+    nodes.push(node(
+        NodeKind::FinalizeDeploy {
+            agent: agent.to_owned(),
+        },
+        vec![
+            Dep {
+                on: 1,
+                when: DepWhen::AfterOk,
+            },
+            Dep {
+                on: 5,
+                when: DepWhen::AfterOk,
+            },
+        ],
+    ));
+    nodes
+}
+
 /// One uniform rebuild shape — no `was_running` branch. `StopForUpdate`
 /// noops when already down; the tail `Reconcile` auto-noops the start
 /// when `wanted = Offline` (a rebuild of a deliberately-stopped agent
@@ -149,7 +193,9 @@ pub fn rebuild(agent: &str, source: Source, reason: String, relock: bool) -> Dag
 ///   nothing, so a failure here cancel-cascades its siblings with the forge and
 ///   the applied repo exactly as they were.
 /// - `DeployApply` (2, child, `AfterOk` `MergeVerify`): the irreversible half —
-///   ff-merge, `prepare_deploy`, rebuild, `finalize_deploy`.
+///   ff-merge + `prepare_deploy`. It doesn't rebuild inline; it grows
+///   [`deploy_rebuild_nodes`] into this DAG as its own children, so the build
+///   and the closing `FinalizeDeploy` are real nodes under the same window.
 /// - `DeployTail` (3, child, `AfterAny` `DeployApply`): the compensation +
 ///   bookkeeping tail — rollback when a merge landed unfinalized, forge tag
 ///   mirror, PR failure comment (see [`NodeKind::DeployTail`]).

hive-c0re/src/job_queue/tests.rs

@@ -976,6 +976,132 @@ fn deploy_dag_runs_phases_in_order_and_tails_a_failed_apply() {
     assert_eq!(summary.approval_id, Some(7));
 }
 
+/// The deploy's happy path: `DeployApply` does not build. It grows the ordinary
+/// rebuild chain into the live DAG under itself, and `FinalizeDeploy` — gated on
+/// that graft finishing — plants the deploy tag last.
+///
+/// The queue is built with **one** build slot on purpose. `DeployWindow` already
+/// holds that slot (and the meta window) for the whole subtree, so the grafted
+/// `Prebuild` can only ever claim by *re-entering* its ancestor's hold. If the
+/// graft were rooted anywhere outside `DeployWindow`'s subtree it would block on
+/// a resource its own DAG owns and deadlock — this test is what pins that down.
+#[test]
+fn deploy_apply_grows_rebuild_subgraph_and_finalizes_after_it() {
+    let q = JobQueue::new(1);
+    let id = submit(
+        &q,
+        templates::approval_deploy("agent-a", 11, "approval 11".to_owned()),
+    );
+
+    let root = claim_one(&q);
+    assert!(matches!(root.kind, NodeKind::DeployWindow { .. }));
+    q.complete_node(id, root.node_id, Ok(()));
+    let verify = claim_one(&q);
+    q.complete_node(id, verify.node_id, Ok(()));
+
+    let apply = claim_one(&q);
+    assert!(matches!(apply.kind, NodeKind::DeployApply { .. }));
+    // Mirrors the scheduler: the executor's `NodeOutput` subgraphs are grafted
+    // BEFORE the emitting node is completed. Completing first would settle the
+    // apply node `Done` with nothing under it, opening the tail's `AfterAny`
+    // gate immediately and letting the deploy "finish" before it had built.
+    let grown = q.append_subgraph(
+        id,
+        &templates::deploy_rebuild_nodes("agent-a"),
+        apply.node_id,
+    );
+    assert!(!grown.is_empty(), "subgraph grafted onto the apply node");
+    q.complete_node(id, apply.node_id, Ok(()));
+
+    // The grafted chain runs in rebuild order. `claim_one` asserts exactly one
+    // claimable node at each step, which also proves the `AfterAny` tail stays
+    // shut: `DeployApply` is `Finishing` (not terminal) while its new children
+    // run, and `Finishing` satisfies neither dep kind.
+    for expected in [
+        "meta_sync",
+        "prebuild",
+        "stop_for_update",
+        "swap",
+        "post_swap",
+        "reconcile",
+    ] {
+        let c = claim_one(&q);
+        assert_eq!(c.kind.as_str(), expected, "grafted phase order");
+        q.complete_node(id, c.node_id, Ok(()));
+    }
+
+    let finalize = claim_one(&q);
+    assert!(
+        matches!(finalize.kind, NodeKind::FinalizeDeploy { .. }),
+        "the deploy tag is planted only after the rebuild came up clean"
+    );
+    q.complete_node(id, finalize.node_id, Ok(()));
+
+    let tail = claim_one(&q);
+    assert!(matches!(tail.kind, NodeKind::DeployTail { .. }));
+    q.complete_node(id, tail.node_id, Ok(()));
+
+    let summary = q.terminal_summary(id).expect("dag terminal");
+    assert_eq!(summary.state, State::Done);
+    assert_eq!(summary.approval_id, Some(11));
+}
+
+/// A failure *inside* the grafted rebuild is the failure mode the subgraph
+/// growth introduces: the deploy is already merged and the container half-swapped.
+/// `FinalizeDeploy` must be cancel-cascaded (its `AfterOk` gate never opens) so
+/// no `deployed/<id>` tag is planted, while the tail still runs to compensate.
+/// `Reconcile` is deliberately still reached — it boots the container back up.
+#[test]
+fn deploy_dag_skips_finalize_but_still_tails_a_failed_graft() {
+    let q = JobQueue::new(1);
+    let id = submit(
+        &q,
+        templates::approval_deploy("agent-a", 13, "approval 13".to_owned()),
+    );
+
+    let root = claim_one(&q);
+    q.complete_node(id, root.node_id, Ok(()));
+    let verify = claim_one(&q);
+    q.complete_node(id, verify.node_id, Ok(()));
+    let apply = claim_one(&q);
+    q.append_subgraph(
+        id,
+        &templates::deploy_rebuild_nodes("agent-a"),
+        apply.node_id,
+    );
+    q.complete_node(id, apply.node_id, Ok(()));
+
+    for expected in ["meta_sync", "prebuild", "stop_for_update"] {
+        let c = claim_one(&q);
+        assert_eq!(c.kind.as_str(), expected);
+        q.complete_node(id, c.node_id, Ok(()));
+    }
+    let swap = claim_one(&q);
+    assert_eq!(swap.kind.as_str(), "swap");
+    q.complete_node(id, swap.node_id, Err("profile swap failed".into()));
+
+    // `Reconcile` hangs off `Prebuild` with `AfterAny`, so a failed swap still
+    // reaches it — bringing the container back up is exactly what it's for.
+    let reconcile = claim_one(&q);
+    assert_eq!(reconcile.kind.as_str(), "reconcile");
+    q.complete_node(id, reconcile.node_id, Ok(()));
+
+    let tail = claim_one(&q);
+    assert!(
+        matches!(tail.kind, NodeKind::DeployTail { .. }),
+        "finalize is cancel-cascaded, so the tail is the next claimable node"
+    );
+    q.complete_node(id, tail.node_id, Ok(()));
+
+    let summary = q.terminal_summary(id).expect("dag terminal");
+    assert_eq!(summary.state, State::Failed);
+    assert_eq!(
+        q.first_error(id).as_deref(),
+        Some("profile swap failed"),
+        "the tail annotates failed/<id> with this"
+    );
+}
+
 /// A pre-merge rejection (drift gate, eval failure) cancel-cascades the
 /// irreversible half via its `AfterOk` edge, but the tail is still reached —
 /// it owns the forge mirror, not just compensation.
@@ -1036,19 +1162,18 @@ fn set_step_only_on_running_and_signals_change() {
 fn set_build_log_id_links_running_node() {
     let q = JobQueue::new(1);
     let id = submit(&q, rebuild("agent-a", "r"));
-    assert!(
-        !q.set_build_log_id_running(id, 41),
-        "no running node yet → refused"
-    );
     let c = claim_one(&q);
     assert!(q.set_build_log_id(id, c.node_id, 42));
-    assert!(q.set_build_log_id_running(id, 43));
     q.complete_node(id, c.node_id, Ok(()));
+    assert!(
+        !q.set_build_log_id(id, c.node_id, 99),
+        "node no longer running → refused"
+    );
     // The log id is fetched by node id (the `GET /api/build-log/<id>` lookup),
     // not carried on the wire — it survives completion in the node runtime.
     assert_eq!(
         q.build_log_id_of(c.node_id.get()),
-        Some(43),
+        Some(42),
         "log id survives completion"
     );
 }