refactor(job-queue): a job is a recipe, not a value you carry
Follows the jobq change: a builder can no longer be constructed or inserted outside `hive_jobq`, so `DagSpec` cannot hold one. It carries a `Declare` — `Box<dyn FnOnce(&Job) + Send>` — and the queue runs it against a builder jobq owns, at the moment it inserts. `NodeOutput.append_subgraph` becomes `Vec<Declare>` for the same reason, and this is where the shape was always heading: that field's doc already said an executor "cannot reach the queue, so it hands the declaration back", while its type was a `Vec<Job>` the executor had built itself. The rejected `build_nodes -> Vec<NodeSpec>` was the first version of that escape hatch; a recipe is the last one, because there is no job-shaped value to hand over at all. Templates and the power-op assemblers move their owned data into the closure and are otherwise unchanged — `rebuild_nodes`, `node` and the tail helpers already took `&Job` and returned handles, so only each template's outermost frame moved. Two `Debug` impls are hand-written: a closure has nothing to show, and its nodes do not exist until the queue runs it. `NodeOutput` reports how many subgraphs were emitted, `DagSpec` its source and reason. `append_subgraph`'s `is_empty()` early-return is gone — you cannot ask a recipe whether it will declare anything without running it. It now inserts and returns an empty id list if nothing was declared, which takes the queue lock in a case that previously skipped it. The two in-DAG-growth tests build `Declare`s now, so they exercise the shape an executor actually produces rather than one only a test could construct. 45 job-queue tests unchanged and passing.
This commit is contained in:
parent
f7548e4535
commit
9c97365f8f
8 changed files with 336 additions and 284 deletions
|
|
@ -24,7 +24,7 @@
|
|||
use hive_jobq::TerminalState;
|
||||
|
||||
use super::model::{DagSpec, NodeKind, PermPayload, Source};
|
||||
use super::{Handle, Job};
|
||||
use super::{Declare, Handle, Job};
|
||||
|
||||
/// Declare one node carrying `kind`, with the resources that kind needs.
|
||||
///
|
||||
|
|
@ -239,27 +239,28 @@ pub(crate) fn rebuild_nodes<'a>(
|
|||
/// inside the `DeployWindow`'s subtree — so the `MetaWindow` this subgraph's
|
||||
/// `MetaSync` and `FinalizeDeploy` declare is re-entered from the ancestor
|
||||
/// already holding it rather than deadlocking against it.
|
||||
pub(crate) fn deploy_rebuild_nodes(agent: &str, approval_id: i64) -> Job {
|
||||
let b = Job::new();
|
||||
let roots = rebuild_nodes(
|
||||
&b,
|
||||
agent,
|
||||
RebuildOpts {
|
||||
relock: false,
|
||||
graceful: false,
|
||||
},
|
||||
None,
|
||||
);
|
||||
let _finalize = node(
|
||||
&b,
|
||||
NodeKind::FinalizeDeploy {
|
||||
agent: agent.to_owned(),
|
||||
approval_id,
|
||||
},
|
||||
)
|
||||
.after_ok(roots.prebuild)
|
||||
.after_ok(roots.reconcile);
|
||||
b
|
||||
pub(crate) fn deploy_rebuild_nodes(agent: &str, approval_id: i64) -> Declare {
|
||||
let agent = agent.to_owned();
|
||||
Box::new(move |b| {
|
||||
let roots = rebuild_nodes(
|
||||
b,
|
||||
&agent,
|
||||
RebuildOpts {
|
||||
relock: false,
|
||||
graceful: false,
|
||||
},
|
||||
None,
|
||||
);
|
||||
let _finalize = node(
|
||||
b,
|
||||
NodeKind::FinalizeDeploy {
|
||||
agent: agent.clone(),
|
||||
approval_id,
|
||||
},
|
||||
)
|
||||
.after_ok(roots.prebuild)
|
||||
.after_ok(roots.reconcile);
|
||||
})
|
||||
}
|
||||
|
||||
/// One uniform rebuild shape — no `was_running` branch. `StopForUpdate`
|
||||
|
|
@ -274,21 +275,22 @@ pub(crate) fn deploy_rebuild_nodes(agent: &str, approval_id: i64) -> Job {
|
|||
/// node. Edging `Reconcile` alone would not do: it is `AfterAny` `Prebuild`, so
|
||||
/// it reaches `Done` even after a failed swap and the tail would report success.
|
||||
pub fn rebuild(agent: &str, source: Source, reason: String, relock: bool) -> DagSpec {
|
||||
let job = Job::new();
|
||||
let roots = rebuild_nodes(
|
||||
&job,
|
||||
agent,
|
||||
RebuildOpts {
|
||||
relock,
|
||||
graceful: false,
|
||||
},
|
||||
None,
|
||||
);
|
||||
emit_rebuilt_tails(&job, agent, &roots.all());
|
||||
let agent = agent.to_owned();
|
||||
DagSpec {
|
||||
source,
|
||||
reason,
|
||||
job,
|
||||
declare: Box::new(move |b| {
|
||||
let roots = rebuild_nodes(
|
||||
b,
|
||||
&agent,
|
||||
RebuildOpts {
|
||||
relock,
|
||||
graceful: false,
|
||||
},
|
||||
None,
|
||||
);
|
||||
emit_rebuilt_tails(b, &agent, &roots.all());
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -318,48 +320,48 @@ pub fn rebuild(agent: &str, source: Source, reason: String, relock: bool) -> Dag
|
|||
/// The window still spans the container build, as it must: `prepare_deploy`
|
||||
/// leaves `flake.lock` staged-uncommitted for the build's whole duration.
|
||||
pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec {
|
||||
let a = || agent.to_owned();
|
||||
let job = Job::new();
|
||||
|
||||
let window = node(
|
||||
&job,
|
||||
NodeKind::DeployWindow {
|
||||
agent: a(),
|
||||
approval_id,
|
||||
},
|
||||
);
|
||||
let verify = node(
|
||||
&job,
|
||||
NodeKind::MergeVerify {
|
||||
agent: a(),
|
||||
approval_id,
|
||||
},
|
||||
)
|
||||
.part_of(window);
|
||||
let apply = node(
|
||||
&job,
|
||||
NodeKind::DeployApply {
|
||||
agent: a(),
|
||||
approval_id,
|
||||
},
|
||||
)
|
||||
.part_of(window)
|
||||
.after_ok(verify);
|
||||
let _tail = node(
|
||||
&job,
|
||||
NodeKind::DeployTail {
|
||||
agent: a(),
|
||||
approval_id,
|
||||
},
|
||||
)
|
||||
.part_of(window)
|
||||
.after_any(apply);
|
||||
|
||||
resolve_approval_tails(&job, approval_id, window);
|
||||
let agent = agent.to_owned();
|
||||
DagSpec {
|
||||
source: Source::Approval,
|
||||
reason,
|
||||
job,
|
||||
declare: Box::new(move |b| {
|
||||
let a = || agent.clone();
|
||||
let window = node(
|
||||
b,
|
||||
NodeKind::DeployWindow {
|
||||
agent: a(),
|
||||
approval_id,
|
||||
},
|
||||
);
|
||||
let verify = node(
|
||||
b,
|
||||
NodeKind::MergeVerify {
|
||||
agent: a(),
|
||||
approval_id,
|
||||
},
|
||||
)
|
||||
.part_of(window);
|
||||
let apply = node(
|
||||
b,
|
||||
NodeKind::DeployApply {
|
||||
agent: a(),
|
||||
approval_id,
|
||||
},
|
||||
)
|
||||
.part_of(window)
|
||||
.after_ok(verify);
|
||||
let _tail = node(
|
||||
b,
|
||||
NodeKind::DeployTail {
|
||||
agent: a(),
|
||||
approval_id,
|
||||
},
|
||||
)
|
||||
.part_of(window)
|
||||
.after_any(apply);
|
||||
|
||||
resolve_approval_tails(b, approval_id, window);
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -370,17 +372,13 @@ pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec
|
|||
/// in the queue tests); production paths no longer emit a bare reconcile.
|
||||
#[cfg(test)]
|
||||
pub fn reconcile_only(agent: &str, source: Source, reason: String) -> DagSpec {
|
||||
let job = Job::new();
|
||||
let _reconcile = node(
|
||||
&job,
|
||||
NodeKind::Reconcile {
|
||||
agent: agent.to_owned(),
|
||||
},
|
||||
);
|
||||
let agent = agent.to_owned();
|
||||
DagSpec {
|
||||
source,
|
||||
reason,
|
||||
job,
|
||||
declare: Box::new(move |b| {
|
||||
let _reconcile = node(b, NodeKind::Reconcile { agent });
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -396,21 +394,21 @@ pub fn reconcile_only(agent: &str, source: Source, reason: String) -> DagSpec {
|
|||
/// `AfterAny` onto `Provision` — the DAG's only other group-root, so its roll-up
|
||||
/// already carries the whole cascade.
|
||||
pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
|
||||
let a = || agent.to_owned();
|
||||
let job = Job::new();
|
||||
|
||||
let provision = node(&job, NodeKind::Provision { agent: a() });
|
||||
let create = node(&job, NodeKind::Create { agent: a() }).part_of(provision);
|
||||
let dropin = node(&job, NodeKind::WriteDropin { agent: a() }).part_of(create);
|
||||
let _reconcile = node(&job, NodeKind::Reconcile { agent: a() })
|
||||
.part_of(create)
|
||||
.after_ok(dropin);
|
||||
|
||||
resolve_approval_tails(&job, approval_id, provision);
|
||||
let agent = agent.to_owned();
|
||||
DagSpec {
|
||||
source: Source::Approval,
|
||||
reason,
|
||||
job,
|
||||
declare: Box::new(move |b| {
|
||||
let a = || agent.clone();
|
||||
let provision = node(b, NodeKind::Provision { agent: a() });
|
||||
let create = node(b, NodeKind::Create { agent: a() }).part_of(provision);
|
||||
let dropin = node(b, NodeKind::WriteDropin { agent: a() }).part_of(create);
|
||||
let _reconcile = node(b, NodeKind::Reconcile { agent: a() })
|
||||
.part_of(create)
|
||||
.after_ok(dropin);
|
||||
|
||||
resolve_approval_tails(b, approval_id, provision);
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -420,32 +418,33 @@ pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
|
|||
/// subgraph's `MetaSync` / `Prebuild` / `Reconcile`, so the `EmitRebuilt` tail
|
||||
/// edges all four.
|
||||
pub fn perm_change(agent: &str, source: Source, reason: String, payload: PermPayload) -> DagSpec {
|
||||
let job = Job::new();
|
||||
let write = node(
|
||||
&job,
|
||||
NodeKind::WritePermFile {
|
||||
agent: agent.to_owned(),
|
||||
payload,
|
||||
},
|
||||
);
|
||||
let roots = rebuild_nodes(
|
||||
&job,
|
||||
agent,
|
||||
RebuildOpts {
|
||||
relock: true,
|
||||
graceful: false,
|
||||
},
|
||||
Some(write),
|
||||
);
|
||||
emit_rebuilt_tails(
|
||||
&job,
|
||||
agent,
|
||||
&[write, roots.meta_sync, roots.prebuild, roots.reconcile],
|
||||
);
|
||||
let agent = agent.to_owned();
|
||||
DagSpec {
|
||||
source,
|
||||
reason,
|
||||
job,
|
||||
declare: Box::new(move |b| {
|
||||
let write = node(
|
||||
b,
|
||||
NodeKind::WritePermFile {
|
||||
agent: agent.clone(),
|
||||
payload,
|
||||
},
|
||||
);
|
||||
let roots = rebuild_nodes(
|
||||
b,
|
||||
&agent,
|
||||
RebuildOpts {
|
||||
relock: true,
|
||||
graceful: false,
|
||||
},
|
||||
Some(write),
|
||||
);
|
||||
emit_rebuilt_tails(
|
||||
b,
|
||||
&agent,
|
||||
&[write, roots.meta_sync, roots.prebuild, roots.reconcile],
|
||||
);
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -465,26 +464,27 @@ pub fn meta_update(
|
|||
reason: String,
|
||||
approval_id: Option<i64>,
|
||||
) -> DagSpec {
|
||||
let job = Job::new();
|
||||
let lock = node(
|
||||
&job,
|
||||
NodeKind::MetaLock {
|
||||
sweep: false,
|
||||
fanout: None,
|
||||
inputs,
|
||||
},
|
||||
);
|
||||
// The bump itself has no side effect, so an operator-driven one ends at the
|
||||
// `MetaLock`; an approval-driven one still has its row to resolve and gets the
|
||||
// per-outcome tails edged onto that single group-root — whose roll-up covers
|
||||
// the rebuild subgraphs `MetaLock` grows into itself.
|
||||
if let Some(approval_id) = approval_id {
|
||||
resolve_approval_tails(&job, approval_id, lock);
|
||||
}
|
||||
DagSpec {
|
||||
source,
|
||||
reason,
|
||||
job,
|
||||
declare: Box::new(move |b| {
|
||||
let lock = node(
|
||||
b,
|
||||
NodeKind::MetaLock {
|
||||
sweep: false,
|
||||
fanout: None,
|
||||
inputs,
|
||||
},
|
||||
);
|
||||
// The bump itself has no side effect, so an operator-driven one ends
|
||||
// at the `MetaLock`; an approval-driven one still has its row to
|
||||
// resolve and gets the per-outcome tails edged onto that single
|
||||
// group-root — whose roll-up covers the rebuild subgraphs `MetaLock`
|
||||
// grows into itself.
|
||||
if let Some(approval_id) = approval_id {
|
||||
resolve_approval_tails(b, approval_id, lock);
|
||||
}
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -502,12 +502,12 @@ pub fn reparent(
|
|||
source: Source,
|
||||
reason: String,
|
||||
) -> DagSpec {
|
||||
let job = Job::new();
|
||||
let _reparent = node(&job, NodeKind::Reparent { moves });
|
||||
DagSpec {
|
||||
source,
|
||||
reason,
|
||||
job,
|
||||
declare: Box::new(move |b| {
|
||||
let _reparent = node(b, NodeKind::Reparent { moves });
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue