job_queue: grow the rebuild subgraph from DeployApply (#2664)

The config-PR deploy's apply node still did the whole container rebuild
inline, through the last surviving `lifecycle::rebuild_no_meta` call. It
now merges, opens the two-phase meta deploy, and returns the ordinary
rebuild chain as a subgraph the scheduler grafts into the live DAG under
it. A new `FinalizeDeploy` node, gated on that graft, plants the deploy
tag and commits the staged lock.

Net effect: "did the agent come back up?" is answered by `Reconcile`
succeeding, the same way it is for every other rebuild, instead of by a
fused inline start — and each deploy phase is its own queue node, so the
dashboard shows which one is running.

The grafted nodes root on the apply node, so they land inside
`DeployWindow`'s subtree and re-enter the meta window and build slot it
already holds rather than deadlocking against them. The new happy-path
test runs on a one-slot queue specifically to pin that down.

`FinalizeDeploy`'s two git writes are fatal, deliberately: they are what
tells `DeployTail` a deploy confirmed good, so a node that merely warned
on them could report success while leaving the tail looking at the git
state of a failure — and the tail would then roll a good deploy back.
The trailing `meta::finalize_deploy` stays warn-only, since by then the
container already runs the new config.

The `failed/<id>` annotated tag moves into the tail, which is now the
only place holding a failed deploy. It reads the reason off the DAG via
a new `JobQueue::first_error`, and is gated on `main` having actually
moved — the rollback ref is parked *before* the merge, so its existence
alone does not mean a merge happened, and a pre-merge rejection must not
tag the previous, innocent head.

Removing the last inline rebuild orphaned a chain of now-dead code:
`rebuild_no_meta`, `container_exists`, `Coordinator::set_queue_build_log`
and `JobQueue::set_build_log_id_running`, all deleted here.
This commit is contained in:
atlas 2026-07-25 23:24:48 +02:00 committed by mara
commit 3429a8c5a6
10 changed files with 388 additions and 257 deletions

View file

@ -976,6 +976,132 @@ fn deploy_dag_runs_phases_in_order_and_tails_a_failed_apply() {
assert_eq!(summary.approval_id, Some(7));
}
/// The deploy's happy path: `DeployApply` does not build. It grows the ordinary
/// rebuild chain into the live DAG under itself, and `FinalizeDeploy` — gated on
/// that graft finishing — plants the deploy tag last.
///
/// The queue is built with **one** build slot on purpose. `DeployWindow` already
/// holds that slot (and the meta window) for the whole subtree, so the grafted
/// `Prebuild` can only ever claim by *re-entering* its ancestor's hold. If the
/// graft were rooted anywhere outside `DeployWindow`'s subtree it would block on
/// a resource its own DAG owns and deadlock — this test is what pins that down.
#[test]
fn deploy_apply_grows_rebuild_subgraph_and_finalizes_after_it() {
let q = JobQueue::new(1);
let id = submit(
&q,
templates::approval_deploy("agent-a", 11, "approval 11".to_owned()),
);
let root = claim_one(&q);
assert!(matches!(root.kind, NodeKind::DeployWindow { .. }));
q.complete_node(id, root.node_id, Ok(()));
let verify = claim_one(&q);
q.complete_node(id, verify.node_id, Ok(()));
let apply = claim_one(&q);
assert!(matches!(apply.kind, NodeKind::DeployApply { .. }));
// Mirrors the scheduler: the executor's `NodeOutput` subgraphs are grafted
// BEFORE the emitting node is completed. Completing first would settle the
// apply node `Done` with nothing under it, opening the tail's `AfterAny`
// gate immediately and letting the deploy "finish" before it had built.
let grown = q.append_subgraph(
id,
&templates::deploy_rebuild_nodes("agent-a"),
apply.node_id,
);
assert!(!grown.is_empty(), "subgraph grafted onto the apply node");
q.complete_node(id, apply.node_id, Ok(()));
// The grafted chain runs in rebuild order. `claim_one` asserts exactly one
// claimable node at each step, which also proves the `AfterAny` tail stays
// shut: `DeployApply` is `Finishing` (not terminal) while its new children
// run, and `Finishing` satisfies neither dep kind.
for expected in [
"meta_sync",
"prebuild",
"stop_for_update",
"swap",
"post_swap",
"reconcile",
] {
let c = claim_one(&q);
assert_eq!(c.kind.as_str(), expected, "grafted phase order");
q.complete_node(id, c.node_id, Ok(()));
}
let finalize = claim_one(&q);
assert!(
matches!(finalize.kind, NodeKind::FinalizeDeploy { .. }),
"the deploy tag is planted only after the rebuild came up clean"
);
q.complete_node(id, finalize.node_id, Ok(()));
let tail = claim_one(&q);
assert!(matches!(tail.kind, NodeKind::DeployTail { .. }));
q.complete_node(id, tail.node_id, Ok(()));
let summary = q.terminal_summary(id).expect("dag terminal");
assert_eq!(summary.state, State::Done);
assert_eq!(summary.approval_id, Some(11));
}
/// A failure *inside* the grafted rebuild is the failure mode the subgraph
/// growth introduces: the deploy is already merged and the container half-swapped.
/// `FinalizeDeploy` must be cancel-cascaded (its `AfterOk` gate never opens) so
/// no `deployed/<id>` tag is planted, while the tail still runs to compensate.
/// `Reconcile` is deliberately still reached — it boots the container back up.
#[test]
fn deploy_dag_skips_finalize_but_still_tails_a_failed_graft() {
let q = JobQueue::new(1);
let id = submit(
&q,
templates::approval_deploy("agent-a", 13, "approval 13".to_owned()),
);
let root = claim_one(&q);
q.complete_node(id, root.node_id, Ok(()));
let verify = claim_one(&q);
q.complete_node(id, verify.node_id, Ok(()));
let apply = claim_one(&q);
q.append_subgraph(
id,
&templates::deploy_rebuild_nodes("agent-a"),
apply.node_id,
);
q.complete_node(id, apply.node_id, Ok(()));
for expected in ["meta_sync", "prebuild", "stop_for_update"] {
let c = claim_one(&q);
assert_eq!(c.kind.as_str(), expected);
q.complete_node(id, c.node_id, Ok(()));
}
let swap = claim_one(&q);
assert_eq!(swap.kind.as_str(), "swap");
q.complete_node(id, swap.node_id, Err("profile swap failed".into()));
// `Reconcile` hangs off `Prebuild` with `AfterAny`, so a failed swap still
// reaches it — bringing the container back up is exactly what it's for.
let reconcile = claim_one(&q);
assert_eq!(reconcile.kind.as_str(), "reconcile");
q.complete_node(id, reconcile.node_id, Ok(()));
let tail = claim_one(&q);
assert!(
matches!(tail.kind, NodeKind::DeployTail { .. }),
"finalize is cancel-cascaded, so the tail is the next claimable node"
);
q.complete_node(id, tail.node_id, Ok(()));
let summary = q.terminal_summary(id).expect("dag terminal");
assert_eq!(summary.state, State::Failed);
assert_eq!(
q.first_error(id).as_deref(),
Some("profile swap failed"),
"the tail annotates failed/<id> with this"
);
}
/// A pre-merge rejection (drift gate, eval failure) cancel-cascades the
/// irreversible half via its `AfterOk` edge, but the tail is still reached —
/// it owns the forge mirror, not just compensation.
@ -1036,19 +1162,18 @@ fn set_step_only_on_running_and_signals_change() {
fn set_build_log_id_links_running_node() {
let q = JobQueue::new(1);
let id = submit(&q, rebuild("agent-a", "r"));
assert!(
!q.set_build_log_id_running(id, 41),
"no running node yet → refused"
);
let c = claim_one(&q);
assert!(q.set_build_log_id(id, c.node_id, 42));
assert!(q.set_build_log_id_running(id, 43));
q.complete_node(id, c.node_id, Ok(()));
assert!(
!q.set_build_log_id(id, c.node_id, 99),
"node no longer running → refused"
);
// The log id is fetched by node id (the `GET /api/build-log/<id>` lookup),
// not carried on the wire — it survives completion in the node runtime.
assert_eq!(
q.build_log_id_of(c.node_id.get()),
Some(43),
Some(42),
"log id survives completion"
);
}