job_queue: grow the rebuild subgraph from DeployApply (#2664)
The config-PR deploy's apply node still did the whole container rebuild inline, through the last surviving `lifecycle::rebuild_no_meta` call. It now merges, opens the two-phase meta deploy, and returns the ordinary rebuild chain as a subgraph the scheduler grafts into the live DAG under it. A new `FinalizeDeploy` node, gated on that graft, plants the deploy tag and commits the staged lock. Net effect: "did the agent come back up?" is answered by `Reconcile` succeeding, the same way it is for every other rebuild, instead of by a fused inline start — and each deploy phase is its own queue node, so the dashboard shows which one is running. The grafted nodes root on the apply node, so they land inside `DeployWindow`'s subtree and re-enter the meta window and build slot it already holds rather than deadlocking against them. The new happy-path test runs on a one-slot queue specifically to pin that down. `FinalizeDeploy`'s two git writes are fatal, deliberately: they are what tells `DeployTail` a deploy confirmed good, so a node that merely warned on them could report success while leaving the tail looking at the git state of a failure — and the tail would then roll a good deploy back. The trailing `meta::finalize_deploy` stays warn-only, since by then the container already runs the new config. The `failed/<id>` annotated tag moves into the tail, which is now the only place holding a failed deploy. It reads the reason off the DAG via a new `JobQueue::first_error`, and is gated on `main` having actually moved — the rollback ref is parked *before* the merge, so its existence alone does not mean a merge happened, and a pre-merge rejection must not tag the previous, innocent head. Removing the last inline rebuild orphaned a chain of now-dead code: `rebuild_no_meta`, `container_exists`, `Coordinator::set_queue_build_log` and `JobQueue::set_build_log_id_running`, all deleted here.
This commit is contained in:
parent
7b2645078a
commit
3429a8c5a6
10 changed files with 388 additions and 257 deletions
|
|
@ -976,6 +976,132 @@ fn deploy_dag_runs_phases_in_order_and_tails_a_failed_apply() {
|
|||
assert_eq!(summary.approval_id, Some(7));
|
||||
}
|
||||
|
||||
/// The deploy's happy path: `DeployApply` does not build. It grows the ordinary
|
||||
/// rebuild chain into the live DAG under itself, and `FinalizeDeploy` — gated on
|
||||
/// that graft finishing — plants the deploy tag last.
|
||||
///
|
||||
/// The queue is built with **one** build slot on purpose. `DeployWindow` already
|
||||
/// holds that slot (and the meta window) for the whole subtree, so the grafted
|
||||
/// `Prebuild` can only ever claim by *re-entering* its ancestor's hold. If the
|
||||
/// graft were rooted anywhere outside `DeployWindow`'s subtree it would block on
|
||||
/// a resource its own DAG owns and deadlock — this test is what pins that down.
|
||||
#[test]
|
||||
fn deploy_apply_grows_rebuild_subgraph_and_finalizes_after_it() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::approval_deploy("agent-a", 11, "approval 11".to_owned()),
|
||||
);
|
||||
|
||||
let root = claim_one(&q);
|
||||
assert!(matches!(root.kind, NodeKind::DeployWindow { .. }));
|
||||
q.complete_node(id, root.node_id, Ok(()));
|
||||
let verify = claim_one(&q);
|
||||
q.complete_node(id, verify.node_id, Ok(()));
|
||||
|
||||
let apply = claim_one(&q);
|
||||
assert!(matches!(apply.kind, NodeKind::DeployApply { .. }));
|
||||
// Mirrors the scheduler: the executor's `NodeOutput` subgraphs are grafted
|
||||
// BEFORE the emitting node is completed. Completing first would settle the
|
||||
// apply node `Done` with nothing under it, opening the tail's `AfterAny`
|
||||
// gate immediately and letting the deploy "finish" before it had built.
|
||||
let grown = q.append_subgraph(
|
||||
id,
|
||||
&templates::deploy_rebuild_nodes("agent-a"),
|
||||
apply.node_id,
|
||||
);
|
||||
assert!(!grown.is_empty(), "subgraph grafted onto the apply node");
|
||||
q.complete_node(id, apply.node_id, Ok(()));
|
||||
|
||||
// The grafted chain runs in rebuild order. `claim_one` asserts exactly one
|
||||
// claimable node at each step, which also proves the `AfterAny` tail stays
|
||||
// shut: `DeployApply` is `Finishing` (not terminal) while its new children
|
||||
// run, and `Finishing` satisfies neither dep kind.
|
||||
for expected in [
|
||||
"meta_sync",
|
||||
"prebuild",
|
||||
"stop_for_update",
|
||||
"swap",
|
||||
"post_swap",
|
||||
"reconcile",
|
||||
] {
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.kind.as_str(), expected, "grafted phase order");
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
|
||||
let finalize = claim_one(&q);
|
||||
assert!(
|
||||
matches!(finalize.kind, NodeKind::FinalizeDeploy { .. }),
|
||||
"the deploy tag is planted only after the rebuild came up clean"
|
||||
);
|
||||
q.complete_node(id, finalize.node_id, Ok(()));
|
||||
|
||||
let tail = claim_one(&q);
|
||||
assert!(matches!(tail.kind, NodeKind::DeployTail { .. }));
|
||||
q.complete_node(id, tail.node_id, Ok(()));
|
||||
|
||||
let summary = q.terminal_summary(id).expect("dag terminal");
|
||||
assert_eq!(summary.state, State::Done);
|
||||
assert_eq!(summary.approval_id, Some(11));
|
||||
}
|
||||
|
||||
/// A failure *inside* the grafted rebuild is the failure mode the subgraph
|
||||
/// growth introduces: the deploy is already merged and the container half-swapped.
|
||||
/// `FinalizeDeploy` must be cancel-cascaded (its `AfterOk` gate never opens) so
|
||||
/// no `deployed/<id>` tag is planted, while the tail still runs to compensate.
|
||||
/// `Reconcile` is deliberately still reached — it boots the container back up.
|
||||
#[test]
|
||||
fn deploy_dag_skips_finalize_but_still_tails_a_failed_graft() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::approval_deploy("agent-a", 13, "approval 13".to_owned()),
|
||||
);
|
||||
|
||||
let root = claim_one(&q);
|
||||
q.complete_node(id, root.node_id, Ok(()));
|
||||
let verify = claim_one(&q);
|
||||
q.complete_node(id, verify.node_id, Ok(()));
|
||||
let apply = claim_one(&q);
|
||||
q.append_subgraph(
|
||||
id,
|
||||
&templates::deploy_rebuild_nodes("agent-a"),
|
||||
apply.node_id,
|
||||
);
|
||||
q.complete_node(id, apply.node_id, Ok(()));
|
||||
|
||||
for expected in ["meta_sync", "prebuild", "stop_for_update"] {
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.kind.as_str(), expected);
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
let swap = claim_one(&q);
|
||||
assert_eq!(swap.kind.as_str(), "swap");
|
||||
q.complete_node(id, swap.node_id, Err("profile swap failed".into()));
|
||||
|
||||
// `Reconcile` hangs off `Prebuild` with `AfterAny`, so a failed swap still
|
||||
// reaches it — bringing the container back up is exactly what it's for.
|
||||
let reconcile = claim_one(&q);
|
||||
assert_eq!(reconcile.kind.as_str(), "reconcile");
|
||||
q.complete_node(id, reconcile.node_id, Ok(()));
|
||||
|
||||
let tail = claim_one(&q);
|
||||
assert!(
|
||||
matches!(tail.kind, NodeKind::DeployTail { .. }),
|
||||
"finalize is cancel-cascaded, so the tail is the next claimable node"
|
||||
);
|
||||
q.complete_node(id, tail.node_id, Ok(()));
|
||||
|
||||
let summary = q.terminal_summary(id).expect("dag terminal");
|
||||
assert_eq!(summary.state, State::Failed);
|
||||
assert_eq!(
|
||||
q.first_error(id).as_deref(),
|
||||
Some("profile swap failed"),
|
||||
"the tail annotates failed/<id> with this"
|
||||
);
|
||||
}
|
||||
|
||||
/// A pre-merge rejection (drift gate, eval failure) cancel-cascades the
|
||||
/// irreversible half via its `AfterOk` edge, but the tail is still reached —
|
||||
/// it owns the forge mirror, not just compensation.
|
||||
|
|
@ -1036,19 +1162,18 @@ fn set_step_only_on_running_and_signals_change() {
|
|||
fn set_build_log_id_links_running_node() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
assert!(
|
||||
!q.set_build_log_id_running(id, 41),
|
||||
"no running node yet → refused"
|
||||
);
|
||||
let c = claim_one(&q);
|
||||
assert!(q.set_build_log_id(id, c.node_id, 42));
|
||||
assert!(q.set_build_log_id_running(id, 43));
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
assert!(
|
||||
!q.set_build_log_id(id, c.node_id, 99),
|
||||
"node no longer running → refused"
|
||||
);
|
||||
// The log id is fetched by node id (the `GET /api/build-log/<id>` lookup),
|
||||
// not carried on the wire — it survives completion in the node runtime.
|
||||
assert_eq!(
|
||||
q.build_log_id_of(c.node_id.get()),
|
||||
Some(43),
|
||||
Some(42),
|
||||
"log id survives completion"
|
||||
);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue