jobq: rename the builder parameters too, not just the alias
Renaming `pub type Job` fixed the definition and left every use site reading `b` and `job` — including `job: super::JobBuilder`, where the parameter still asserted it was a job while its type said otherwise. The propagation is what the issue was about, so the parameters are the half that matters at a call site. Two spots deliberately untouched: `auto_update`'s `sort_by(|a, b| …)` comparator, and the prose that means the job *queue* (main.rs's "Job-queue scheduler", scheduler.rs's "not this module's job any more", the "grown job rejected" log). 315 tests pass unchanged.
This commit is contained in:
parent
379c9bb570
commit
684f6da78e
6 changed files with 203 additions and 180 deletions
|
|
@ -28,7 +28,7 @@ pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from
|
||||||
/// scheduler spawns per claim; the `Result` (stringified) becomes the
|
/// scheduler spawns per claim; the `Result` (stringified) becomes the
|
||||||
/// node's terminal state.
|
/// node's terminal state.
|
||||||
///
|
///
|
||||||
/// `job` is the node's own growth channel: an executor that decides more work
|
/// `builder` is the node's own growth channel: an executor that decides more work
|
||||||
/// is needed declares it here, and the scheduler inserts it under this node
|
/// is needed declares it here, and the scheduler inserts it under this node
|
||||||
/// when the node completes. Most executors never touch it. Nothing is inserted
|
/// when the node completes. Most executors never touch it. Nothing is inserted
|
||||||
/// while the node runs — the builder is local state, so this stays outside the
|
/// while the node runs — the builder is local state, so this stays outside the
|
||||||
|
|
@ -48,15 +48,15 @@ pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from
|
||||||
/// themselves. Nothing here needs a claim to exist as a type.
|
/// themselves. Nothing here needs a claim to exist as a type.
|
||||||
pub(super) async fn run_node(
|
pub(super) async fn run_node(
|
||||||
coord: &Arc<Coordinator>,
|
coord: &Arc<Coordinator>,
|
||||||
job: super::JobBuilder,
|
builder: super::JobBuilder,
|
||||||
id: NodeId,
|
id: NodeId,
|
||||||
kind: &NodeKind,
|
kind: &NodeKind,
|
||||||
) -> (super::JobBuilder, Result<()>) {
|
) -> (super::JobBuilder, Result<()>) {
|
||||||
// The agent this node targets rides the payload — empty for the agentless
|
// The agent this node targets rides the payload — empty for the agentless
|
||||||
// container kinds (`MetaLock`, `Dag`), which never read it.
|
// container kinds (`MetaLock`, `Dag`), which never read it.
|
||||||
let agent = kind.agent();
|
let agent = kind.agent();
|
||||||
// Every arm is `Result<()>`; the three that grow work declare into `job`
|
// Every arm is `Result<()>`; the three that grow work declare into `builder`
|
||||||
// *synchronously*, after their own awaits have finished. Borrowing `&job`
|
// *synchronously*, after their own awaits have finished. Borrowing `&builder`
|
||||||
// inside an `.await` would make this future non-`Send` (see above), so the
|
// inside an `.await` would make this future non-`Send` (see above), so the
|
||||||
// growth executors return what to grow rather than taking the builder.
|
// growth executors return what to grow rather than taking the builder.
|
||||||
let result = match kind {
|
let result = match kind {
|
||||||
|
|
@ -79,14 +79,14 @@ pub(super) async fn run_node(
|
||||||
// than returned, since `run_meta_lock` would only be deriving
|
// than returned, since `run_meta_lock` would only be deriving
|
||||||
// it from the `sweep` this call site already holds.
|
// it from the `sweep` this call site already holds.
|
||||||
if *sweep {
|
if *sweep {
|
||||||
super::templates::grown_graceful_rebuilds(&job, &agents, true);
|
super::templates::grown_graceful_rebuilds(&builder, &agents, true);
|
||||||
} else {
|
} else {
|
||||||
super::templates::grown_rebuilds(&job, &agents, false);
|
super::templates::grown_rebuilds(&builder, &agents, false);
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
NodeKind::Reconcile { .. } => run_reconcile(coord, agent).await.map(|sub| {
|
NodeKind::Reconcile { .. } => run_reconcile(coord, agent).await.map(|sub| {
|
||||||
if let Some(kind) = sub {
|
if let Some(kind) = sub {
|
||||||
super::templates::fanned_out_mechanical(&job, kind);
|
super::templates::fanned_out_mechanical(&builder, kind);
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
NodeKind::Start { .. } => run_start(coord, agent).await,
|
NodeKind::Start { .. } => run_start(coord, agent).await,
|
||||||
|
|
@ -106,7 +106,7 @@ pub(super) async fn run_node(
|
||||||
NodeKind::MergeVerify { approval_id, .. } => run_merge_verify(coord, *approval_id).await,
|
NodeKind::MergeVerify { approval_id, .. } => run_merge_verify(coord, *approval_id).await,
|
||||||
NodeKind::DeployApply { approval_id, .. } => {
|
NodeKind::DeployApply { approval_id, .. } => {
|
||||||
run_deploy_apply(coord, *approval_id).await.map(|()| {
|
run_deploy_apply(coord, *approval_id).await.map(|()| {
|
||||||
super::templates::deploy_rebuild_nodes(&job, agent, *approval_id);
|
super::templates::deploy_rebuild_nodes(&builder, agent, *approval_id);
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
NodeKind::FinalizeDeploy { approval_id, .. } => {
|
NodeKind::FinalizeDeploy { approval_id, .. } => {
|
||||||
|
|
@ -132,7 +132,7 @@ pub(super) async fn run_node(
|
||||||
// and build slot it declares stay held until its subtree settles.
|
// and build slot it declares stay held until its subtree settles.
|
||||||
NodeKind::Dag { .. } | NodeKind::DeployWindow { .. } => Ok(()),
|
NodeKind::Dag { .. } | NodeKind::DeployWindow { .. } => Ok(()),
|
||||||
};
|
};
|
||||||
(job, result)
|
(builder, result)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resolve the DAG's approval row the way this node's own `outcome` says.
|
/// Resolve the DAG's approval row the way this node's own `outcome` says.
|
||||||
|
|
@ -368,7 +368,7 @@ async fn run_meta_lock(
|
||||||
/// Idempotent power-converge *planner*: compare `wanted` (durable
|
/// Idempotent power-converge *planner*: compare `wanted` (durable
|
||||||
/// intent) against observed state and, when they diverge, fan the
|
/// intent) against observed state and, when they diverge, fan the
|
||||||
/// mechanical `Start` / `Stop` out as a first-class node appended to
|
/// mechanical `Start` / `Stop` out as a first-class node appended to
|
||||||
/// *this* DAG (a single node declared into `job`, rooted on this node).
|
/// *this* DAG (a single node declared into `builder`, rooted on this node).
|
||||||
/// Does no container work itself — the sub-step becomes visible in the
|
/// Does no container work itself — the sub-step becomes visible in the
|
||||||
/// DAG and the lease-window transient (or the sub-step's own node-local
|
/// DAG and the lease-window transient (or the sub-step's own node-local
|
||||||
/// guard) rides across it.
|
/// guard) rides across it.
|
||||||
|
|
|
||||||
|
|
@ -89,7 +89,7 @@ pub async fn run_worker(coord: Arc<Coordinator>) {
|
||||||
// it.
|
// it.
|
||||||
let sched = Arc::clone(coord.job_queue.sched());
|
let sched = Arc::clone(coord.job_queue.sched());
|
||||||
let node_coord = Arc::clone(&coord);
|
let node_coord = Arc::clone(&coord);
|
||||||
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, job| {
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
||||||
let coord = node_coord;
|
let coord = node_coord;
|
||||||
async move {
|
async move {
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
|
|
@ -99,7 +99,7 @@ pub async fn run_worker(coord: Arc<Coordinator>) {
|
||||||
agent = %kind.agent(),
|
agent = %kind.agent(),
|
||||||
"job_queue: node running"
|
"job_queue: node running"
|
||||||
);
|
);
|
||||||
let (grown, result) = exec::run_node(&coord, job, id, &kind).await;
|
let (grown, result) = exec::run_node(&coord, builder, id, &kind).await;
|
||||||
match &result {
|
match &result {
|
||||||
Ok(()) => tracing::info!(node = id.get(), "job_queue: node done"),
|
Ok(()) => tracing::info!(node = id.get(), "job_queue: node done"),
|
||||||
Err(e) => tracing::warn!(
|
Err(e) => tracing::warn!(
|
||||||
|
|
|
||||||
|
|
@ -49,8 +49,8 @@ fn submit_and_emit(
|
||||||
/// meta input — the meta-update cascade grows its own rebuild subgraphs
|
/// meta input — the meta-update cascade grows its own rebuild subgraphs
|
||||||
/// in-DAG instead of going through this surface).
|
/// in-DAG instead of going through this surface).
|
||||||
pub fn rebuild(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
pub fn rebuild(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||||
submit_and_emit(coord, source, reason, |b| {
|
submit_and_emit(coord, source, reason, |builder| {
|
||||||
templates::rebuild(b, agent, true);
|
templates::rebuild(builder, agent, true);
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -67,12 +67,12 @@ pub fn rebuild(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: St
|
||||||
/// actually running (nothing to drain on a down container). The `Reconcile`
|
/// actually running (nothing to drain on a down container). The `Reconcile`
|
||||||
/// stays even for a down agent so a race-up between the state read and exec
|
/// stays even for a down agent so a race-up between the state read and exec
|
||||||
/// is still stopped in-DAG.
|
/// is still stopped in-DAG.
|
||||||
fn stop_chain(b: &JobBuilder, agent: &str, graceful: bool, running: bool) {
|
fn stop_chain(builder: &JobBuilder, agent: &str, graceful: bool, running: bool) {
|
||||||
// `SetWanted` is the group root and owns the agent lease; the mechanical
|
// `SetWanted` is the group root and owns the agent lease; the mechanical
|
||||||
// steps are its children (borrow the lease, run once it reaches `Finishing`,
|
// steps are its children (borrow the lease, run once it reaches `Finishing`,
|
||||||
// dep-ordered among themselves).
|
// dep-ordered among themselves).
|
||||||
let a = || agent.to_owned();
|
let a = || agent.to_owned();
|
||||||
let wanted = b
|
let wanted = builder
|
||||||
.node(NodeKind::SetWanted {
|
.node(NodeKind::SetWanted {
|
||||||
agent: a(),
|
agent: a(),
|
||||||
up: false,
|
up: false,
|
||||||
|
|
@ -81,22 +81,22 @@ fn stop_chain(b: &JobBuilder, agent: &str, graceful: bool, running: bool) {
|
||||||
// Declaration order is dependency order: the quiesce steps come first so
|
// Declaration order is dependency order: the quiesce steps come first so
|
||||||
// the `Reconcile` that waits on them can name them.
|
// the `Reconcile` that waits on them can name them.
|
||||||
if graceful && running {
|
if graceful && running {
|
||||||
let signal = b
|
let signal = builder
|
||||||
.node(NodeKind::Signal { agent: a() })
|
.node(NodeKind::Signal { agent: a() })
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.part_of(wanted);
|
.part_of(wanted);
|
||||||
let drain = b
|
let drain = builder
|
||||||
.node(NodeKind::Drain { agent: a() })
|
.node(NodeKind::Drain { agent: a() })
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.part_of(wanted)
|
.part_of(wanted)
|
||||||
.after_ok(signal);
|
.after_ok(signal);
|
||||||
let _ = b
|
let _ = builder
|
||||||
.node(NodeKind::Reconcile { agent: a() })
|
.node(NodeKind::Reconcile { agent: a() })
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.part_of(wanted)
|
.part_of(wanted)
|
||||||
.after_ok(drain);
|
.after_ok(drain);
|
||||||
} else {
|
} else {
|
||||||
let _ = b
|
let _ = builder
|
||||||
.node(NodeKind::Reconcile { agent: a() })
|
.node(NodeKind::Reconcile { agent: a() })
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.part_of(wanted);
|
.part_of(wanted);
|
||||||
|
|
@ -107,8 +107,8 @@ fn stop_chain(b: &JobBuilder, agent: &str, graceful: bool, running: bool) {
|
||||||
/// agent gets the rebuild subgraph (its tail `Reconcile` starts it on
|
/// agent gets the rebuild subgraph (its tail `Reconcile` starts it on
|
||||||
/// current derivations), otherwise a plain `Reconcile` (which starts a down
|
/// current derivations), otherwise a plain `Reconcile` (which starts a down
|
||||||
/// agent and noops an already-running one).
|
/// agent and noops an already-running one).
|
||||||
fn start_chain(b: &JobBuilder, agent: &str, running: bool, stale: bool) {
|
fn start_chain(builder: &JobBuilder, agent: &str, running: bool, stale: bool) {
|
||||||
let wanted = b
|
let wanted = builder
|
||||||
.node(NodeKind::SetWanted {
|
.node(NodeKind::SetWanted {
|
||||||
agent: agent.to_owned(),
|
agent: agent.to_owned(),
|
||||||
up: true,
|
up: true,
|
||||||
|
|
@ -118,9 +118,9 @@ fn start_chain(b: &JobBuilder, agent: &str, running: bool, stale: bool) {
|
||||||
// Rebuild subtree chained behind the `SetWanted` head. `MetaSync`,
|
// Rebuild subtree chained behind the `SetWanted` head. `MetaSync`,
|
||||||
// `Prebuild` + `Reconcile` are their own group roots (top-level, per
|
// `Prebuild` + `Reconcile` are their own group roots (top-level, per
|
||||||
// `rebuild_nodes`).
|
// `rebuild_nodes`).
|
||||||
rebuild_nodes(b, agent, true, Some(wanted));
|
rebuild_nodes(builder, agent, true, Some(wanted));
|
||||||
} else {
|
} else {
|
||||||
let _ = b
|
let _ = builder
|
||||||
.node(NodeKind::Reconcile {
|
.node(NodeKind::Reconcile {
|
||||||
agent: agent.to_owned(),
|
agent: agent.to_owned(),
|
||||||
})
|
})
|
||||||
|
|
@ -138,11 +138,11 @@ fn start_chain(b: &JobBuilder, agent: &str, running: bool, stale: bool) {
|
||||||
/// before `Reconcile`; a down agent gets just `Reconcile`, which
|
/// before `Reconcile`; a down agent gets just `Reconcile`, which
|
||||||
/// converges to intent — a stopped (`wanted = Off`) agent stays stopped,
|
/// converges to intent — a stopped (`wanted = Off`) agent stays stopped,
|
||||||
/// a crashed (`wanted = Up`) agent comes back up.
|
/// a crashed (`wanted = Up`) agent comes back up.
|
||||||
fn restart_chain(b: &JobBuilder, agent: &str, graceful: bool, running: bool) {
|
fn restart_chain(builder: &JobBuilder, agent: &str, graceful: bool, running: bool) {
|
||||||
let a = || agent.to_owned();
|
let a = || agent.to_owned();
|
||||||
if !running {
|
if !running {
|
||||||
// Nothing to bounce — a lone Reconcile converges to intent.
|
// Nothing to bounce — a lone Reconcile converges to intent.
|
||||||
let _ = b
|
let _ = builder
|
||||||
.node(NodeKind::Reconcile { agent: a() })
|
.node(NodeKind::Reconcile { agent: a() })
|
||||||
.needs(Resource::Agent(a()));
|
.needs(Resource::Agent(a()));
|
||||||
return;
|
return;
|
||||||
|
|
@ -156,28 +156,28 @@ fn restart_chain(b: &JobBuilder, agent: &str, graceful: bool, running: bool) {
|
||||||
// that step *is* the root, and the parent gate already orders it — a child
|
// that step *is* the root, and the parent gate already orders it — a child
|
||||||
// must NOT dep on its own parent (dep-scope), so it takes no sibling edge.
|
// must NOT dep on its own parent (dep-scope), so it takes no sibling edge.
|
||||||
if graceful {
|
if graceful {
|
||||||
let signal = b
|
let signal = builder
|
||||||
.node(NodeKind::Signal { agent: a() })
|
.node(NodeKind::Signal { agent: a() })
|
||||||
.needs(Resource::Agent(a()));
|
.needs(Resource::Agent(a()));
|
||||||
let drain = b
|
let drain = builder
|
||||||
.node(NodeKind::Drain { agent: a() })
|
.node(NodeKind::Drain { agent: a() })
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.part_of(signal);
|
.part_of(signal);
|
||||||
let stop = b
|
let stop = builder
|
||||||
.node(NodeKind::StopForUpdate { agent: a() })
|
.node(NodeKind::StopForUpdate { agent: a() })
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.part_of(signal)
|
.part_of(signal)
|
||||||
.after_ok(drain);
|
.after_ok(drain);
|
||||||
let _ = b
|
let _ = builder
|
||||||
.node(NodeKind::Reconcile { agent: a() })
|
.node(NodeKind::Reconcile { agent: a() })
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.part_of(signal)
|
.part_of(signal)
|
||||||
.after_ok(stop);
|
.after_ok(stop);
|
||||||
} else {
|
} else {
|
||||||
let stop = b
|
let stop = builder
|
||||||
.node(NodeKind::StopForUpdate { agent: a() })
|
.node(NodeKind::StopForUpdate { agent: a() })
|
||||||
.needs(Resource::Agent(a()));
|
.needs(Resource::Agent(a()));
|
||||||
let _ = b
|
let _ = builder
|
||||||
.node(NodeKind::Reconcile { agent: a() })
|
.node(NodeKind::Reconcile { agent: a() })
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.part_of(stop);
|
.part_of(stop);
|
||||||
|
|
@ -198,9 +198,9 @@ fn restart_chain(b: &JobBuilder, agent: &str, graceful: bool, running: bool) {
|
||||||
// subgraph's indices onto another's used to be a function.
|
// subgraph's indices onto another's used to be a function.
|
||||||
|
|
||||||
/// Declare the stop DAG from explicit `(agent, running)` targets.
|
/// Declare the stop DAG from explicit `(agent, running)` targets.
|
||||||
pub(crate) fn stop_nodes(b: &JobBuilder, targets: &[(String, bool)], graceful: bool) {
|
pub(crate) fn stop_nodes(builder: &JobBuilder, targets: &[(String, bool)], graceful: bool) {
|
||||||
for (agent, running) in targets {
|
for (agent, running) in targets {
|
||||||
stop_chain(b, agent, graceful, *running);
|
stop_chain(builder, agent, graceful, *running);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -210,16 +210,16 @@ pub(crate) fn stop_nodes(b: &JobBuilder, targets: &[(String, bool)], graceful: b
|
||||||
/// running under its lease, so a down+stale agent that grew a rebuild subgraph
|
/// running under its lease, so a down+stale agent that grew a rebuild subgraph
|
||||||
/// reports `rebuilding` during its swap and `starting` at its reconcile,
|
/// reports `rebuilding` during its swap and `starting` at its reconcile,
|
||||||
/// without the DAG having to guess one label covering every target.
|
/// without the DAG having to guess one label covering every target.
|
||||||
pub(crate) fn start_nodes(b: &JobBuilder, targets: &[(String, bool, bool)]) {
|
pub(crate) fn start_nodes(builder: &JobBuilder, targets: &[(String, bool, bool)]) {
|
||||||
for (agent, running, stale) in targets {
|
for (agent, running, stale) in targets {
|
||||||
start_chain(b, agent, *running, *stale);
|
start_chain(builder, agent, *running, *stale);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Declare the restart DAG from explicit `(agent, running)` targets.
|
/// Declare the restart DAG from explicit `(agent, running)` targets.
|
||||||
pub(crate) fn restart_nodes(b: &JobBuilder, targets: &[(String, bool)], graceful: bool) {
|
pub(crate) fn restart_nodes(builder: &JobBuilder, targets: &[(String, bool)], graceful: bool) {
|
||||||
for (agent, running) in targets {
|
for (agent, running) in targets {
|
||||||
restart_chain(b, agent, graceful, *running);
|
restart_chain(builder, agent, graceful, *running);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -258,8 +258,8 @@ pub async fn restart_many(
|
||||||
for agent in agents {
|
for agent in agents {
|
||||||
targets.push((agent.clone(), lifecycle::is_running(agent).await));
|
targets.push((agent.clone(), lifecycle::is_running(agent).await));
|
||||||
}
|
}
|
||||||
submit_and_emit(coord, source, reason, |b| {
|
submit_and_emit(coord, source, reason, |builder| {
|
||||||
restart_nodes(b, &targets, graceful);
|
restart_nodes(builder, &targets, graceful);
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -293,8 +293,8 @@ pub async fn start_many(
|
||||||
}
|
}
|
||||||
targets.push((agent.clone(), running, stale));
|
targets.push((agent.clone(), running, stale));
|
||||||
}
|
}
|
||||||
submit_and_emit(coord, source, reason, |b| {
|
submit_and_emit(coord, source, reason, |builder| {
|
||||||
start_nodes(b, &targets);
|
start_nodes(builder, &targets);
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -331,8 +331,8 @@ pub async fn stop_many(
|
||||||
for agent in agents {
|
for agent in agents {
|
||||||
targets.push((agent.clone(), lifecycle::is_running(agent).await));
|
targets.push((agent.clone(), lifecycle::is_running(agent).await));
|
||||||
}
|
}
|
||||||
submit_and_emit(coord, source, reason, |b| {
|
submit_and_emit(coord, source, reason, |builder| {
|
||||||
stop_nodes(b, &targets, graceful);
|
stop_nodes(builder, &targets, graceful);
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -344,8 +344,8 @@ pub fn perm_change(
|
||||||
reason: String,
|
reason: String,
|
||||||
payload: super::PermPayload,
|
payload: super::PermPayload,
|
||||||
) -> u64 {
|
) -> u64 {
|
||||||
submit_and_emit(coord, source, reason, |b| {
|
submit_and_emit(coord, source, reason, |builder| {
|
||||||
templates::perm_change(b, agent, payload);
|
templates::perm_change(builder, agent, payload);
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -356,8 +356,8 @@ pub fn meta_update(
|
||||||
source: Source,
|
source: Source,
|
||||||
reason: String,
|
reason: String,
|
||||||
) -> u64 {
|
) -> u64 {
|
||||||
submit_and_emit(coord, source, reason, |b| {
|
submit_and_emit(coord, source, reason, |builder| {
|
||||||
templates::meta_update(b, inputs, None);
|
templates::meta_update(builder, inputs, None);
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -374,7 +374,7 @@ pub fn reparent(
|
||||||
source: Source,
|
source: Source,
|
||||||
reason: String,
|
reason: String,
|
||||||
) -> u64 {
|
) -> u64 {
|
||||||
submit_and_emit(coord, source, reason, |b| {
|
submit_and_emit(coord, source, reason, |builder| {
|
||||||
templates::reparent(b, moves);
|
templates::reparent(builder, moves);
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -36,9 +36,9 @@ use super::{Handle, JobBuilder};
|
||||||
///
|
///
|
||||||
/// Exactly one runs on a DAG that executed, and neither runs on one the operator
|
/// Exactly one runs on a DAG that executed, and neither runs on one the operator
|
||||||
/// dropped — see [`hive_jobq::NodeRef::on_elimination_of`].
|
/// dropped — see [`hive_jobq::NodeRef::on_elimination_of`].
|
||||||
fn emit_rebuilt_tails(b: &JobBuilder, agent: &str, roots: &[Handle<'_>]) {
|
fn emit_rebuilt_tails(builder: &JobBuilder, agent: &str, roots: &[Handle<'_>]) {
|
||||||
let ok = roots.iter().fold(
|
let ok = roots.iter().fold(
|
||||||
b.node(NodeKind::EmitRebuilt {
|
builder.node(NodeKind::EmitRebuilt {
|
||||||
agent: agent.to_owned(),
|
agent: agent.to_owned(),
|
||||||
ok: true,
|
ok: true,
|
||||||
}),
|
}),
|
||||||
|
|
@ -51,11 +51,12 @@ fn emit_rebuilt_tails(b: &JobBuilder, agent: &str, roots: &[Handle<'_>]) {
|
||||||
// `Reconcile` is still bringing the container back up, so reporting straight
|
// `Reconcile` is still bringing the container back up, so reporting straight
|
||||||
// off the elimination would announce the failure mid-recovery.
|
// off the elimination would announce the failure mid-recovery.
|
||||||
let _failed = roots.iter().fold(
|
let _failed = roots.iter().fold(
|
||||||
b.node(NodeKind::EmitRebuilt {
|
builder
|
||||||
agent: agent.to_owned(),
|
.node(NodeKind::EmitRebuilt {
|
||||||
ok: false,
|
agent: agent.to_owned(),
|
||||||
})
|
ok: false,
|
||||||
.on_elimination_of(ok),
|
})
|
||||||
|
.on_elimination_of(ok),
|
||||||
hive_jobq::NodeRef::after_any,
|
hive_jobq::NodeRef::after_any,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -65,13 +66,13 @@ fn emit_rebuilt_tails(b: &JobBuilder, agent: &str, roots: &[Handle<'_>]) {
|
||||||
///
|
///
|
||||||
/// The `Cancelled` node is what keeps a dropped approval DAG from dangling its
|
/// The `Cancelled` node is what keeps a dropped approval DAG from dangling its
|
||||||
/// row forever — its edge is the only one [`super::JobQueue::cancel`] spares.
|
/// row forever — its edge is the only one [`super::JobQueue::cancel`] spares.
|
||||||
fn resolve_approval_tails(b: &JobBuilder, approval_id: i64, root: Handle<'_>) {
|
fn resolve_approval_tails(builder: &JobBuilder, approval_id: i64, root: Handle<'_>) {
|
||||||
for outcome in [
|
for outcome in [
|
||||||
TerminalState::Done,
|
TerminalState::Done,
|
||||||
TerminalState::Failed,
|
TerminalState::Failed,
|
||||||
TerminalState::Cancelled,
|
TerminalState::Cancelled,
|
||||||
] {
|
] {
|
||||||
let _ = b
|
let _ = builder
|
||||||
.node(NodeKind::ResolveApproval {
|
.node(NodeKind::ResolveApproval {
|
||||||
approval_id,
|
approval_id,
|
||||||
outcome,
|
outcome,
|
||||||
|
|
@ -90,18 +91,18 @@ fn resolve_approval_tails(b: &JobBuilder, approval_id: i64, root: Handle<'_>) {
|
||||||
///
|
///
|
||||||
/// Same reason as [`fanned_out_mechanical`] for living here: this was the
|
/// Same reason as [`fanned_out_mechanical`] for living here: this was the
|
||||||
/// second construction site declaring nodes inline in an executor.
|
/// second construction site declaring nodes inline in an executor.
|
||||||
pub(crate) fn grown_rebuilds(b: &JobBuilder, agents: &[String], relock: bool) {
|
pub(crate) fn grown_rebuilds(builder: &JobBuilder, agents: &[String], relock: bool) {
|
||||||
for agent in agents {
|
for agent in agents {
|
||||||
rebuild_nodes(b, agent, relock, None);
|
rebuild_nodes(builder, agent, relock, None);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// As [`grown_rebuilds`], but each agent gets its `Signal` → `Drain` window
|
/// As [`grown_rebuilds`], but each agent gets its `Signal` → `Drain` window
|
||||||
/// before being stopped. The boot sweep's flavour: it stops agents that were
|
/// before being stopped. The boot sweep's flavour: it stops agents that were
|
||||||
/// mid-turn when the host came up, so they drain rather than being cut off.
|
/// mid-turn when the host came up, so they drain rather than being cut off.
|
||||||
pub(crate) fn grown_graceful_rebuilds(b: &JobBuilder, agents: &[String], relock: bool) {
|
pub(crate) fn grown_graceful_rebuilds(builder: &JobBuilder, agents: &[String], relock: bool) {
|
||||||
for agent in agents {
|
for agent in agents {
|
||||||
graceful_rebuild_nodes(b, agent, relock, None);
|
graceful_rebuild_nodes(builder, agent, relock, None);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -117,9 +118,9 @@ pub(crate) fn grown_graceful_rebuilds(b: &JobBuilder, agents: &[String], relock:
|
||||||
/// declaration does: this is the one construction site that was hiding in an
|
/// declaration does: this is the one construction site that was hiding in an
|
||||||
/// executor, which meant the only test of it had to re-declare the same two
|
/// executor, which meant the only test of it had to re-declare the same two
|
||||||
/// calls itself and would have kept passing if the executor changed.
|
/// calls itself and would have kept passing if the executor changed.
|
||||||
pub(crate) fn fanned_out_mechanical(b: &JobBuilder, kind: NodeKind) {
|
pub(crate) fn fanned_out_mechanical(builder: &JobBuilder, kind: NodeKind) {
|
||||||
let lease = Resource::Agent(kind.agent().to_owned());
|
let lease = Resource::Agent(kind.agent().to_owned());
|
||||||
let _ = b.node(kind).needs(lease);
|
let _ = builder.node(kind).needs(lease);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The group-roots a [`rebuild_nodes`] subgraph exposes to its caller: what a
|
/// The group-roots a [`rebuild_nodes`] subgraph exposes to its caller: what a
|
||||||
|
|
@ -176,7 +177,7 @@ impl<'a> RebuildRoots<'a> {
|
||||||
/// takes a fresh lease; the tiny gap is harmless — `Reconcile` converges to
|
/// takes a fresh lease; the tiny gap is harmless — `Reconcile` converges to
|
||||||
/// the persisted `wanted` idempotently.
|
/// the persisted `wanted` idempotently.
|
||||||
fn rebuild_subtree<'a>(
|
fn rebuild_subtree<'a>(
|
||||||
b: &'a JobBuilder,
|
builder: &'a JobBuilder,
|
||||||
agent: &str,
|
agent: &str,
|
||||||
relock: bool,
|
relock: bool,
|
||||||
graceful: bool,
|
graceful: bool,
|
||||||
|
|
@ -184,13 +185,13 @@ fn rebuild_subtree<'a>(
|
||||||
) -> RebuildRoots<'a> {
|
) -> RebuildRoots<'a> {
|
||||||
let a = || agent.to_owned();
|
let a = || agent.to_owned();
|
||||||
|
|
||||||
let mut meta_sync = b
|
let mut meta_sync = builder
|
||||||
.node(NodeKind::MetaSync { agent: a(), relock })
|
.node(NodeKind::MetaSync { agent: a(), relock })
|
||||||
.needs(Resource::MetaWindow);
|
.needs(Resource::MetaWindow);
|
||||||
if let Some(after) = after {
|
if let Some(after) = after {
|
||||||
meta_sync = meta_sync.after_ok(after);
|
meta_sync = meta_sync.after_ok(after);
|
||||||
}
|
}
|
||||||
let prebuild = b
|
let prebuild = builder
|
||||||
.node(NodeKind::Prebuild { agent: a() })
|
.node(NodeKind::Prebuild { agent: a() })
|
||||||
.needs(Resource::BuildSlot)
|
.needs(Resource::BuildSlot)
|
||||||
.after_ok(meta_sync);
|
.after_ok(meta_sync);
|
||||||
|
|
@ -198,27 +199,29 @@ fn rebuild_subtree<'a>(
|
||||||
// The stop root hangs off `Prebuild` and owns the agent lease for
|
// The stop root hangs off `Prebuild` and owns the agent lease for
|
||||||
// everything below it. `StopForUpdate` parents the swap pair either way.
|
// everything below it. `StopForUpdate` parents the swap pair either way.
|
||||||
let stop_for_update = if graceful {
|
let stop_for_update = if graceful {
|
||||||
let signal = b
|
let signal = builder
|
||||||
.node(NodeKind::Signal { agent: a() })
|
.node(NodeKind::Signal { agent: a() })
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.part_of(prebuild);
|
.part_of(prebuild);
|
||||||
// `Drain` is a *child* of `Signal`, so the parent gate already orders
|
// `Drain` is a *child* of `Signal`, so the parent gate already orders
|
||||||
// it — a child must not dep on its own parent (dep-scope).
|
// it — a child must not dep on its own parent (dep-scope).
|
||||||
let drain = b
|
let drain = builder
|
||||||
.node(NodeKind::Drain { agent: a() })
|
.node(NodeKind::Drain { agent: a() })
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.part_of(signal);
|
.part_of(signal);
|
||||||
b.node(NodeKind::StopForUpdate { agent: a() })
|
builder
|
||||||
|
.node(NodeKind::StopForUpdate { agent: a() })
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.part_of(signal)
|
.part_of(signal)
|
||||||
.after_ok(drain)
|
.after_ok(drain)
|
||||||
} else {
|
} else {
|
||||||
b.node(NodeKind::StopForUpdate { agent: a() })
|
builder
|
||||||
|
.node(NodeKind::StopForUpdate { agent: a() })
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.part_of(prebuild)
|
.part_of(prebuild)
|
||||||
};
|
};
|
||||||
|
|
||||||
let swap = b
|
let swap = builder
|
||||||
.node(NodeKind::Swap { agent: a() })
|
.node(NodeKind::Swap { agent: a() })
|
||||||
.needs(Resource::BuildSlot)
|
.needs(Resource::BuildSlot)
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
|
|
@ -227,13 +230,13 @@ fn rebuild_subtree<'a>(
|
||||||
// `StopForUpdate`, which holds it, so this is a re-entrant borrow — no
|
// `StopForUpdate`, which holds it, so this is a re-entrant borrow — no
|
||||||
// second unit, no deadlock. Declaring it is what stops the requirement
|
// second unit, no deadlock. Declaring it is what stops the requirement
|
||||||
// being true only of this one DAG shape.
|
// being true only of this one DAG shape.
|
||||||
let _post_swap = b
|
let _post_swap = builder
|
||||||
.node(NodeKind::PostSwap { agent: a() })
|
.node(NodeKind::PostSwap { agent: a() })
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.part_of(stop_for_update)
|
.part_of(stop_for_update)
|
||||||
.after_ok(swap);
|
.after_ok(swap);
|
||||||
|
|
||||||
let reconcile = b
|
let reconcile = builder
|
||||||
.node(NodeKind::Reconcile { agent: a() })
|
.node(NodeKind::Reconcile { agent: a() })
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.after_any(prebuild);
|
.after_any(prebuild);
|
||||||
|
|
@ -250,12 +253,12 @@ fn rebuild_subtree<'a>(
|
||||||
/// when given, is the node this subgraph chains behind. See
|
/// when given, is the node this subgraph chains behind. See
|
||||||
/// [`rebuild_subtree`] for the structure.
|
/// [`rebuild_subtree`] for the structure.
|
||||||
pub(crate) fn rebuild_nodes<'a>(
|
pub(crate) fn rebuild_nodes<'a>(
|
||||||
b: &'a JobBuilder,
|
builder: &'a JobBuilder,
|
||||||
agent: &str,
|
agent: &str,
|
||||||
relock: bool,
|
relock: bool,
|
||||||
after: Option<Handle<'a>>,
|
after: Option<Handle<'a>>,
|
||||||
) -> RebuildRoots<'a> {
|
) -> RebuildRoots<'a> {
|
||||||
rebuild_subtree(b, agent, relock, false, after)
|
rebuild_subtree(builder, agent, relock, false, after)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// As [`rebuild_nodes`], but the agent gets a `Signal` → `Drain` window to
|
/// As [`rebuild_nodes`], but the agent gets a `Signal` → `Drain` window to
|
||||||
|
|
@ -266,12 +269,12 @@ pub(crate) fn rebuild_nodes<'a>(
|
||||||
/// *prepend* nodes — it **re-parents** the stop root, so a caller cannot
|
/// *prepend* nodes — it **re-parents** the stop root, so a caller cannot
|
||||||
/// declare it without being handed the internals. Only the boot sweep wants it.
|
/// declare it without being handed the internals. Only the boot sweep wants it.
|
||||||
pub(crate) fn graceful_rebuild_nodes<'a>(
|
pub(crate) fn graceful_rebuild_nodes<'a>(
|
||||||
b: &'a JobBuilder,
|
builder: &'a JobBuilder,
|
||||||
agent: &str,
|
agent: &str,
|
||||||
relock: bool,
|
relock: bool,
|
||||||
after: Option<Handle<'a>>,
|
after: Option<Handle<'a>>,
|
||||||
) -> RebuildRoots<'a> {
|
) -> RebuildRoots<'a> {
|
||||||
rebuild_subtree(b, agent, relock, true, after)
|
rebuild_subtree(builder, agent, relock, true, after)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The rebuild subgraph a [`NodeKind::DeployApply`] grows into its own DAG once
|
/// The rebuild subgraph a [`NodeKind::DeployApply`] grows into its own DAG once
|
||||||
|
|
@ -298,9 +301,9 @@ pub(crate) fn graceful_rebuild_nodes<'a>(
|
||||||
/// `DeployWindow`'s subtree — so the `MetaWindow` this subgraph's `MetaSync`
|
/// `DeployWindow`'s subtree — so the `MetaWindow` this subgraph's `MetaSync`
|
||||||
/// and `FinalizeDeploy` declare is re-entered from the ancestor already holding
|
/// and `FinalizeDeploy` declare is re-entered from the ancestor already holding
|
||||||
/// it rather than deadlocking against it.
|
/// it rather than deadlocking against it.
|
||||||
pub(crate) fn deploy_rebuild_nodes(b: &JobBuilder, agent: &str, approval_id: i64) {
|
pub(crate) fn deploy_rebuild_nodes(builder: &JobBuilder, agent: &str, approval_id: i64) {
|
||||||
let roots = rebuild_nodes(b, agent, false, None);
|
let roots = rebuild_nodes(builder, agent, false, None);
|
||||||
let _finalize = b
|
let _finalize = builder
|
||||||
.node(NodeKind::FinalizeDeploy {
|
.node(NodeKind::FinalizeDeploy {
|
||||||
agent: agent.to_owned(),
|
agent: agent.to_owned(),
|
||||||
approval_id,
|
approval_id,
|
||||||
|
|
@ -321,9 +324,9 @@ pub(crate) fn deploy_rebuild_nodes(b: &JobBuilder, agent: &str, approval_id: i64
|
||||||
/// whole `StopForUpdate`→`Swap`→`PostSwap` subtree, so those three cover every
|
/// whole `StopForUpdate`→`Swap`→`PostSwap` subtree, so those three cover every
|
||||||
/// node. Edging `Reconcile` alone would not do: it is `AfterAny` `Prebuild`, so
|
/// node. Edging `Reconcile` alone would not do: it is `AfterAny` `Prebuild`, so
|
||||||
/// it reaches `Done` even after a failed swap and the tail would report success.
|
/// it reaches `Done` even after a failed swap and the tail would report success.
|
||||||
pub fn rebuild(b: &JobBuilder, agent: &str, relock: bool) {
|
pub fn rebuild(builder: &JobBuilder, agent: &str, relock: bool) {
|
||||||
let roots = rebuild_nodes(b, agent, relock, None);
|
let roots = rebuild_nodes(builder, agent, relock, None);
|
||||||
emit_rebuilt_tails(b, agent, &roots.all());
|
emit_rebuilt_tails(builder, agent, &roots.all());
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Approval-driven deploy (`MergeConfigPr`) as a phase subtree rather than the
|
/// Approval-driven deploy (`MergeConfigPr`) as a phase subtree rather than the
|
||||||
|
|
@ -351,7 +354,7 @@ pub fn rebuild(b: &JobBuilder, agent: &str, relock: bool) {
|
||||||
///
|
///
|
||||||
/// The window still spans the container build, as it must: `prepare_deploy`
|
/// The window still spans the container build, as it must: `prepare_deploy`
|
||||||
/// leaves `flake.lock` staged-uncommitted for the build's whole duration.
|
/// leaves `flake.lock` staged-uncommitted for the build's whole duration.
|
||||||
pub fn approval_deploy(b: &JobBuilder, agent: &str, approval_id: i64) {
|
pub fn approval_deploy(builder: &JobBuilder, agent: &str, approval_id: i64) {
|
||||||
let a = || agent.to_owned();
|
let a = || agent.to_owned();
|
||||||
// The window is the widest holder in the tree: it brackets a nix
|
// The window is the widest holder in the tree: it brackets a nix
|
||||||
// build (`BuildSlot`), takes the container down across the swap
|
// build (`BuildSlot`), takes the container down across the swap
|
||||||
|
|
@ -359,7 +362,7 @@ pub fn approval_deploy(b: &JobBuilder, agent: &str, approval_id: i64) {
|
||||||
// (`MetaWindow`). All three are held for its whole subtree, which
|
// (`MetaWindow`). All three are held for its whole subtree, which
|
||||||
// is what lets the appended rebuild's `MetaSync` and the
|
// is what lets the appended rebuild's `MetaSync` and the
|
||||||
// `FinalizeDeploy` re-enter rather than contend.
|
// `FinalizeDeploy` re-enter rather than contend.
|
||||||
let window = b
|
let window = builder
|
||||||
.node(NodeKind::DeployWindow {
|
.node(NodeKind::DeployWindow {
|
||||||
agent: a(),
|
agent: a(),
|
||||||
approval_id,
|
approval_id,
|
||||||
|
|
@ -367,20 +370,20 @@ pub fn approval_deploy(b: &JobBuilder, agent: &str, approval_id: i64) {
|
||||||
.needs(Resource::BuildSlot)
|
.needs(Resource::BuildSlot)
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.needs(Resource::MetaWindow);
|
.needs(Resource::MetaWindow);
|
||||||
let verify = b
|
let verify = builder
|
||||||
.node(NodeKind::MergeVerify {
|
.node(NodeKind::MergeVerify {
|
||||||
agent: a(),
|
agent: a(),
|
||||||
approval_id,
|
approval_id,
|
||||||
})
|
})
|
||||||
.part_of(window);
|
.part_of(window);
|
||||||
let apply = b
|
let apply = builder
|
||||||
.node(NodeKind::DeployApply {
|
.node(NodeKind::DeployApply {
|
||||||
agent: a(),
|
agent: a(),
|
||||||
approval_id,
|
approval_id,
|
||||||
})
|
})
|
||||||
.part_of(window)
|
.part_of(window)
|
||||||
.after_ok(verify);
|
.after_ok(verify);
|
||||||
let _tail = b
|
let _tail = builder
|
||||||
.node(NodeKind::DeployTail {
|
.node(NodeKind::DeployTail {
|
||||||
agent: a(),
|
agent: a(),
|
||||||
approval_id,
|
approval_id,
|
||||||
|
|
@ -388,7 +391,7 @@ pub fn approval_deploy(b: &JobBuilder, agent: &str, approval_id: i64) {
|
||||||
.part_of(window)
|
.part_of(window)
|
||||||
.after_any(apply);
|
.after_any(apply);
|
||||||
|
|
||||||
resolve_approval_tails(b, approval_id, window);
|
resolve_approval_tails(builder, approval_id, window);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// First-deploy spawn (approval-driven): `Provision` (proposed/applied
|
/// First-deploy spawn (approval-driven): `Provision` (proposed/applied
|
||||||
|
|
@ -402,27 +405,27 @@ pub fn approval_deploy(b: &JobBuilder, agent: &str, approval_id: i64) {
|
||||||
/// container was never created). Closed by a `ResolveApproval` tail root edged
|
/// container was never created). Closed by a `ResolveApproval` tail root edged
|
||||||
/// `AfterAny` onto `Provision` — the DAG's only other group-root, so its roll-up
|
/// `AfterAny` onto `Provision` — the DAG's only other group-root, so its roll-up
|
||||||
/// already carries the whole cascade.
|
/// already carries the whole cascade.
|
||||||
pub fn spawn(b: &JobBuilder, agent: &str, approval_id: i64) {
|
pub fn spawn(builder: &JobBuilder, agent: &str, approval_id: i64) {
|
||||||
let a = || agent.to_owned();
|
let a = || agent.to_owned();
|
||||||
let provision = b
|
let provision = builder
|
||||||
.node(NodeKind::Provision { agent: a() })
|
.node(NodeKind::Provision { agent: a() })
|
||||||
.needs(Resource::MetaWindow);
|
.needs(Resource::MetaWindow);
|
||||||
let create = b
|
let create = builder
|
||||||
.node(NodeKind::Create { agent: a() })
|
.node(NodeKind::Create { agent: a() })
|
||||||
.needs(Resource::BuildSlot)
|
.needs(Resource::BuildSlot)
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.part_of(provision);
|
.part_of(provision);
|
||||||
let dropin = b
|
let dropin = builder
|
||||||
.node(NodeKind::WriteDropin { agent: a() })
|
.node(NodeKind::WriteDropin { agent: a() })
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.part_of(create);
|
.part_of(create);
|
||||||
let _reconcile = b
|
let _reconcile = builder
|
||||||
.node(NodeKind::Reconcile { agent: a() })
|
.node(NodeKind::Reconcile { agent: a() })
|
||||||
.needs(Resource::Agent(a()))
|
.needs(Resource::Agent(a()))
|
||||||
.part_of(create)
|
.part_of(create)
|
||||||
.after_ok(dropin);
|
.after_ok(dropin);
|
||||||
|
|
||||||
resolve_approval_tails(b, approval_id, provision);
|
resolve_approval_tails(builder, approval_id, provision);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Perm change: commit the JSON file(s), then the rebuild subgraph so
|
/// Perm change: commit the JSON file(s), then the rebuild subgraph so
|
||||||
|
|
@ -430,16 +433,16 @@ pub fn spawn(b: &JobBuilder, agent: &str, approval_id: i64) {
|
||||||
/// effect in the container. Group-roots are `WritePermFile` plus the rebuild
|
/// effect in the container. Group-roots are `WritePermFile` plus the rebuild
|
||||||
/// subgraph's `MetaSync` / `Prebuild` / `Reconcile`, so the `EmitRebuilt` tail
|
/// subgraph's `MetaSync` / `Prebuild` / `Reconcile`, so the `EmitRebuilt` tail
|
||||||
/// edges all four.
|
/// edges all four.
|
||||||
pub fn perm_change(b: &JobBuilder, agent: &str, payload: PermPayload) {
|
pub fn perm_change(builder: &JobBuilder, agent: &str, payload: PermPayload) {
|
||||||
let write = b
|
let write = builder
|
||||||
.node(NodeKind::WritePermFile {
|
.node(NodeKind::WritePermFile {
|
||||||
agent: agent.to_owned(),
|
agent: agent.to_owned(),
|
||||||
payload,
|
payload,
|
||||||
})
|
})
|
||||||
.needs(Resource::MetaWindow);
|
.needs(Resource::MetaWindow);
|
||||||
let roots = rebuild_nodes(b, agent, true, Some(write));
|
let roots = rebuild_nodes(builder, agent, true, Some(write));
|
||||||
emit_rebuilt_tails(
|
emit_rebuilt_tails(
|
||||||
b,
|
builder,
|
||||||
agent,
|
agent,
|
||||||
&[write, roots.meta_sync, roots.prebuild, roots.reconcile],
|
&[write, roots.meta_sync, roots.prebuild, roots.reconcile],
|
||||||
);
|
);
|
||||||
|
|
@ -455,8 +458,8 @@ pub fn perm_change(b: &JobBuilder, agent: &str, payload: PermPayload) {
|
||||||
/// so the "hyperhive" pseudo-agent gets no pill), giving each cascade agent
|
/// so the "hyperhive" pseudo-agent gets no pill), giving each cascade agent
|
||||||
/// crash-watch suppression during its `Swap` — the property the old child
|
/// crash-watch suppression during its `Swap` — the property the old child
|
||||||
/// `Rebuild` DAGs carried via their own transient.
|
/// `Rebuild` DAGs carried via their own transient.
|
||||||
pub fn meta_update(b: &JobBuilder, inputs: Vec<String>, approval_id: Option<i64>) {
|
pub fn meta_update(builder: &JobBuilder, inputs: Vec<String>, approval_id: Option<i64>) {
|
||||||
let lock = b
|
let lock = builder
|
||||||
.node(NodeKind::MetaLock {
|
.node(NodeKind::MetaLock {
|
||||||
sweep: false,
|
sweep: false,
|
||||||
fanout: None,
|
fanout: None,
|
||||||
|
|
@ -470,7 +473,7 @@ pub fn meta_update(b: &JobBuilder, inputs: Vec<String>, approval_id: Option<i64>
|
||||||
// group-root — whose roll-up covers the rebuild subgraphs `MetaLock`
|
// group-root — whose roll-up covers the rebuild subgraphs `MetaLock`
|
||||||
// grows into itself.
|
// grows into itself.
|
||||||
if let Some(approval_id) = approval_id {
|
if let Some(approval_id) = approval_id {
|
||||||
resolve_approval_tails(b, approval_id, lock);
|
resolve_approval_tails(builder, approval_id, lock);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -483,8 +486,8 @@ pub fn meta_update(b: &JobBuilder, inputs: Vec<String>, approval_id: Option<i64>
|
||||||
/// checks), so a parent move needs no container rebuild to take effect.
|
/// checks), so a parent move needs no container rebuild to take effect.
|
||||||
/// No transient pill either — the node is agentless (no lease to hang one
|
/// No transient pill either — the node is agentless (no lease to hang one
|
||||||
/// off of) and near-instant. No tail node: the write is the whole effect.
|
/// off of) and near-instant. No tail node: the write is the whole effect.
|
||||||
pub fn reparent(b: &JobBuilder, moves: Vec<(hive_types::Ident, Option<hive_types::Ident>)>) {
|
pub fn reparent(builder: &JobBuilder, moves: Vec<(hive_types::Ident, Option<hive_types::Ident>)>) {
|
||||||
let _reparent = b
|
let _reparent = builder
|
||||||
.node(NodeKind::Reparent { moves })
|
.node(NodeKind::Reparent { moves })
|
||||||
.needs(Resource::MetaWindow);
|
.needs(Resource::MetaWindow);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -29,24 +29,24 @@ fn ident(s: &str) -> hive_types::Ident {
|
||||||
hive_types::Ident::parse(s).expect("valid test ident")
|
hive_types::Ident::parse(s).expect("valid test ident")
|
||||||
}
|
}
|
||||||
|
|
||||||
fn rebuild(b: &JobBuilder, agent: &str) {
|
fn rebuild(builder: &JobBuilder, agent: &str) {
|
||||||
templates::rebuild(b, agent, true);
|
templates::rebuild(builder, agent, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Restart shape with every agent treated as **running** — the online
|
/// Restart shape with every agent treated as **running** — the online
|
||||||
/// shape (`[Signal→Drain→] StopForUpdate → Reconcile`, no `SetWanted` head)
|
/// shape (`[Signal→Drain→] StopForUpdate → Reconcile`, no `SetWanted` head)
|
||||||
/// most queue-mechanics tests assume. Mirrors the pre-dynamic
|
/// most queue-mechanics tests assume. Mirrors the pre-dynamic
|
||||||
/// `templates::restart` (which is now the state-aware `submit::restart_nodes`).
|
/// `templates::restart` (which is now the state-aware `submit::restart_nodes`).
|
||||||
fn restart_online(b: &JobBuilder, agents: &[&str], graceful: bool) {
|
fn restart_online(builder: &JobBuilder, agents: &[&str], graceful: bool) {
|
||||||
let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
|
let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
|
||||||
submit::restart_nodes(b, &targets, graceful);
|
submit::restart_nodes(builder, &targets, graceful);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Stop shape with every agent treated as **running** — the online shape
|
/// Stop shape with every agent treated as **running** — the online shape
|
||||||
/// (`SetWanted → [Signal→Drain→](graceful) Reconcile`).
|
/// (`SetWanted → [Signal→Drain→](graceful) Reconcile`).
|
||||||
fn stop_online(b: &JobBuilder, agents: &[&str], graceful: bool) {
|
fn stop_online(builder: &JobBuilder, agents: &[&str], graceful: bool) {
|
||||||
let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
|
let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
|
||||||
submit::stop_nodes(b, &targets, graceful);
|
submit::stop_nodes(builder, &targets, graceful);
|
||||||
}
|
}
|
||||||
|
|
||||||
// `Claimed` / `ClaimReady` / `CompleteNode` lived here: a claim snapshot type
|
// `Claimed` / `ClaimReady` / `CompleteNode` lived here: a claim snapshot type
|
||||||
|
|
@ -299,8 +299,8 @@ fn state_of(q: &JobQueue, dag_id: u64) -> State {
|
||||||
#[test]
|
#[test]
|
||||||
fn submit_assigns_distinct_ids() {
|
fn submit_assigns_distinct_ids() {
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let first = submit(&q, "first", |b| rebuild(b, "agent-a"));
|
let first = submit(&q, "first", |builder| rebuild(builder, "agent-a"));
|
||||||
let second = submit(&q, "second", |b| rebuild(b, "agent-b"));
|
let second = submit(&q, "second", |builder| rebuild(builder, "agent-b"));
|
||||||
assert_ne!(first, second);
|
assert_ne!(first, second);
|
||||||
assert_eq!(q.snapshot().len(), 2);
|
assert_eq!(q.snapshot().len(), 2);
|
||||||
}
|
}
|
||||||
|
|
@ -313,8 +313,8 @@ fn submit_assigns_distinct_ids() {
|
||||||
#[test]
|
#[test]
|
||||||
fn identical_resubmit_is_a_distinct_dag() {
|
fn identical_resubmit_is_a_distinct_dag() {
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let first = submit(&q, "first", |b| rebuild(b, "agent-a"));
|
let first = submit(&q, "first", |builder| rebuild(builder, "agent-a"));
|
||||||
let resubmit = submit(&q, "again", |b| rebuild(b, "agent-a"));
|
let resubmit = submit(&q, "again", |builder| rebuild(builder, "agent-a"));
|
||||||
assert_ne!(first, resubmit, "no dedup: identical resubmit is a new DAG");
|
assert_ne!(first, resubmit, "no dedup: identical resubmit is a new DAG");
|
||||||
assert_eq!(q.snapshot().len(), 2);
|
assert_eq!(q.snapshot().len(), 2);
|
||||||
}
|
}
|
||||||
|
|
@ -322,9 +322,11 @@ fn identical_resubmit_is_a_distinct_dag() {
|
||||||
#[test]
|
#[test]
|
||||||
fn distinct_submits_never_collapse() {
|
fn distinct_submits_never_collapse() {
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let rebuild_a = submit(&q, "r", |b| rebuild(b, "agent-a"));
|
let rebuild_a = submit(&q, "r", |builder| rebuild(builder, "agent-a"));
|
||||||
let rebuild_b = submit(&q, "r", |b| rebuild(b, "agent-b"));
|
let rebuild_b = submit(&q, "r", |builder| rebuild(builder, "agent-b"));
|
||||||
let restart_a = submit(&q, "r", |b| restart_online(b, &["agent-a"], false));
|
let restart_a = submit(&q, "r", |builder| {
|
||||||
|
restart_online(builder, &["agent-a"], false);
|
||||||
|
});
|
||||||
assert_ne!(rebuild_a, rebuild_b);
|
assert_ne!(rebuild_a, rebuild_b);
|
||||||
assert_ne!(rebuild_a, restart_a);
|
assert_ne!(rebuild_a, restart_a);
|
||||||
assert_eq!(q.snapshot().len(), 3);
|
assert_eq!(q.snapshot().len(), 3);
|
||||||
|
|
@ -342,8 +344,10 @@ fn resubmit_while_running_is_new_dag() {
|
||||||
// swallowed" is the scenario people worry about, and a reader looking for
|
// swallowed" is the scenario people worry about, and a reader looking for
|
||||||
// it should find it.
|
// it should find it.
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let a = submit(&q, "first", |b| rebuild(b, "agent-a"));
|
let a = submit(&q, "first", |builder| rebuild(builder, "agent-a"));
|
||||||
let again = submit(&q, "config bumped during build", |b| rebuild(b, "agent-a"));
|
let again = submit(&q, "config bumped during build", |builder| {
|
||||||
|
rebuild(builder, "agent-a");
|
||||||
|
});
|
||||||
assert_ne!(a, again);
|
assert_ne!(a, again);
|
||||||
assert_eq!(q.snapshot().len(), 2);
|
assert_eq!(q.snapshot().len(), 2);
|
||||||
}
|
}
|
||||||
|
|
@ -379,7 +383,7 @@ fn rebuild_chain_is_declared_serial() {
|
||||||
// logic"). Both axes are asserted below because a template can break either
|
// logic"). Both axes are asserted below because a template can break either
|
||||||
// one independently.
|
// one independently.
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "r", |b| rebuild(b, "agent-a"));
|
let id = submit(&q, "r", |builder| rebuild(builder, "agent-a"));
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
declared_shape(&q, id),
|
declared_shape(&q, id),
|
||||||
vec![
|
vec![
|
||||||
|
|
@ -426,9 +430,13 @@ fn rebuild_chain_is_declared_serial() {
|
||||||
fn graceful_rebuild_chain_drains_before_stopping() {
|
fn graceful_rebuild_chain_drains_before_stopping() {
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = q
|
let id = q
|
||||||
.submit(Source::AutoUpdate, "sweep".to_owned(), |b: &JobBuilder| {
|
.submit(
|
||||||
templates::graceful_rebuild_nodes(b, "agent-a", true, None);
|
Source::AutoUpdate,
|
||||||
})
|
"sweep".to_owned(),
|
||||||
|
|builder: &JobBuilder| {
|
||||||
|
templates::graceful_rebuild_nodes(builder, "agent-a", true, None);
|
||||||
|
},
|
||||||
|
)
|
||||||
.expect("valid shape");
|
.expect("valid shape");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
declared_shape(&q, id)
|
declared_shape(&q, id)
|
||||||
|
|
@ -460,8 +468,8 @@ fn non_graceful_rebuild_has_no_signal_or_drain() {
|
||||||
// job keeps its nodes to itself and inserts them, so what it built is
|
// job keeps its nodes to itself and inserts them, so what it built is
|
||||||
// observable where it matters — in what the scheduler runs.
|
// observable where it matters — in what the scheduler runs.
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "manual", |b| {
|
let id = submit(&q, "manual", |builder| {
|
||||||
templates::rebuild_nodes(b, "agent-a", true, None);
|
templates::rebuild_nodes(builder, "agent-a", true, None);
|
||||||
});
|
});
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
declared_shape(&q, id)
|
declared_shape(&q, id)
|
||||||
|
|
@ -541,7 +549,7 @@ fn rebuild_chain_declares_the_slot_where_the_nix_work_is() {
|
||||||
// a resource unit is held for the acquirer's whole subtree, so the slot
|
// a resource unit is held for the acquirer's whole subtree, so the slot
|
||||||
// `Prebuild` takes covers `StopForUpdate` → `Swap` → `PostSwap` beneath it.
|
// `Prebuild` takes covers `StopForUpdate` → `Swap` → `PostSwap` beneath it.
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "r", |b| rebuild(b, "agent-a"));
|
let id = submit(&q, "r", |builder| rebuild(builder, "agent-a"));
|
||||||
let res = |kind: &str| declared_resources(&q, node_of(&q, id, kind));
|
let res = |kind: &str| declared_resources(&q, node_of(&q, id, kind));
|
||||||
|
|
||||||
let agent = || Resource::Agent("agent-a".to_owned());
|
let agent = || Resource::Agent("agent-a".to_owned());
|
||||||
|
|
@ -578,8 +586,8 @@ fn rebuild_chain_declares_the_slot_where_the_nix_work_is() {
|
||||||
#[test]
|
#[test]
|
||||||
fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() {
|
fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() {
|
||||||
let q = JobQueue::new(4);
|
let q = JobQueue::new(4);
|
||||||
let id = submit(&q, "hive-wide", |b| {
|
let id = submit(&q, "hive-wide", |builder| {
|
||||||
restart_online(b, &["agent-a", "agent-b"], false);
|
restart_online(builder, &["agent-a", "agent-b"], false);
|
||||||
});
|
});
|
||||||
// A hive-wide restart is ONE DAG, not one-per-agent.
|
// A hive-wide restart is ONE DAG, not one-per-agent.
|
||||||
assert_eq!(q.snapshot().len(), 1);
|
assert_eq!(q.snapshot().len(), 1);
|
||||||
|
|
@ -614,8 +622,8 @@ fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() {
|
||||||
#[test]
|
#[test]
|
||||||
fn multi_agent_stop_is_one_dag_with_concurrent_per_agent_subgraphs() {
|
fn multi_agent_stop_is_one_dag_with_concurrent_per_agent_subgraphs() {
|
||||||
let q = JobQueue::new(4);
|
let q = JobQueue::new(4);
|
||||||
let id = submit(&q, "hive-wide stop", |b| {
|
let id = submit(&q, "hive-wide stop", |builder| {
|
||||||
stop_online(b, &["agent-a", "agent-b"], false);
|
stop_online(builder, &["agent-a", "agent-b"], false);
|
||||||
});
|
});
|
||||||
// A hive-wide stop is ONE DAG, not one-per-agent.
|
// A hive-wide stop is ONE DAG, not one-per-agent.
|
||||||
assert_eq!(q.snapshot().len(), 1);
|
assert_eq!(q.snapshot().len(), 1);
|
||||||
|
|
@ -647,9 +655,9 @@ fn multi_agent_start_one_dag_folds_per_agent_stale_rebuild() {
|
||||||
let q = JobQueue::new(4);
|
let q = JobQueue::new(4);
|
||||||
// fresh: offline + not stale → SetWanted → Reconcile.
|
// fresh: offline + not stale → SetWanted → Reconcile.
|
||||||
// stale: offline + stale → SetWanted → «rebuild subgraph».
|
// stale: offline + stale → SetWanted → «rebuild subgraph».
|
||||||
let id = submit(&q, "hive-wide start", |b| {
|
let id = submit(&q, "hive-wide start", |builder| {
|
||||||
submit::start_nodes(
|
submit::start_nodes(
|
||||||
b,
|
builder,
|
||||||
&[
|
&[
|
||||||
("fresh".to_owned(), false, false),
|
("fresh".to_owned(), false, false),
|
||||||
("stale".to_owned(), false, true),
|
("stale".to_owned(), false, true),
|
||||||
|
|
@ -694,14 +702,14 @@ fn offline_agents_skip_mechanical_nodes_but_keep_reconcile() {
|
||||||
// read and node exec.
|
// read and node exec.
|
||||||
let q = JobQueue::new(4);
|
let q = JobQueue::new(4);
|
||||||
// Offline graceful stop → SetWanted(Off) → Reconcile (no Signal/Drain).
|
// Offline graceful stop → SetWanted(Off) → Reconcile (no Signal/Drain).
|
||||||
let stop = submit(&q, "stop down", |b| {
|
let stop = submit(&q, "stop down", |builder| {
|
||||||
submit::stop_nodes(b, &[("down".to_owned(), false)], true);
|
submit::stop_nodes(builder, &[("down".to_owned(), false)], true);
|
||||||
});
|
});
|
||||||
// Offline restart → a lone Reconcile (no SetWanted, no StopForUpdate):
|
// Offline restart → a lone Reconcile (no SetWanted, no StopForUpdate):
|
||||||
// nothing to bounce, and restart never rewrites intent, so the tail
|
// nothing to bounce, and restart never rewrites intent, so the tail
|
||||||
// Reconcile converges the down agent to its existing `wanted`.
|
// Reconcile converges the down agent to its existing `wanted`.
|
||||||
let restart = submit(&q, "restart down", |b| {
|
let restart = submit(&q, "restart down", |builder| {
|
||||||
submit::restart_nodes(b, &[("down2".to_owned(), false)], true);
|
submit::restart_nodes(builder, &[("down2".to_owned(), false)], true);
|
||||||
});
|
});
|
||||||
let shape = |id: u64| -> Vec<String> {
|
let shape = |id: u64| -> Vec<String> {
|
||||||
q.snapshot()
|
q.snapshot()
|
||||||
|
|
@ -737,14 +745,18 @@ fn boot_sweep_nodes_declare_their_own_resources() {
|
||||||
// compile; only an exhaustive caller list would have caught it.
|
// compile; only an exhaustive caller list would have caught it.
|
||||||
let q = JobQueue::new(4);
|
let q = JobQueue::new(4);
|
||||||
let id = q
|
let id = q
|
||||||
.submit(Source::AutoUpdate, "boot".to_owned(), |b: &JobBuilder| {
|
.submit(
|
||||||
crate::workers::auto_update::boot_nodes(
|
Source::AutoUpdate,
|
||||||
b,
|
"boot".to_owned(),
|
||||||
true,
|
|builder: &JobBuilder| {
|
||||||
vec!["stale-agent".to_owned()],
|
crate::workers::auto_update::boot_nodes(
|
||||||
vec!["drifted-agent".to_owned()],
|
builder,
|
||||||
);
|
true,
|
||||||
})
|
vec!["stale-agent".to_owned()],
|
||||||
|
vec!["drifted-agent".to_owned()],
|
||||||
|
);
|
||||||
|
},
|
||||||
|
)
|
||||||
.expect("valid shape");
|
.expect("valid shape");
|
||||||
|
|
||||||
let mut lock = declared_resources(&q, node_of(&q, id, "meta_lock"));
|
let mut lock = declared_resources(&q, node_of(&q, id, "meta_lock"));
|
||||||
|
|
@ -846,7 +858,7 @@ fn rebuild_reconcile_waits_for_the_whole_build_subtree() {
|
||||||
// easy thing to break — someone flattening the chain would keep every edge
|
// easy thing to break — someone flattening the chain would keep every edge
|
||||||
// and still lose the guarantee.
|
// and still lose the guarantee.
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "r", |b| rebuild(b, "agent-a"));
|
let id = submit(&q, "r", |builder| rebuild(builder, "agent-a"));
|
||||||
let shape = declared_shape(&q, id);
|
let shape = declared_shape(&q, id);
|
||||||
let parent_of = |kind: &str| {
|
let parent_of = |kind: &str| {
|
||||||
shape
|
shape
|
||||||
|
|
@ -918,9 +930,9 @@ fn rebuild_reconcile_waits_for_the_whole_build_subtree() {
|
||||||
#[test]
|
#[test]
|
||||||
fn a_fanned_out_mechanical_node_declares_its_agent_lease() {
|
fn a_fanned_out_mechanical_node_declares_its_agent_lease() {
|
||||||
let q = JobQueue::new(4);
|
let q = JobQueue::new(4);
|
||||||
let id = submit(&q, "fan-out", |b| {
|
let id = submit(&q, "fan-out", |builder| {
|
||||||
templates::fanned_out_mechanical(
|
templates::fanned_out_mechanical(
|
||||||
b,
|
builder,
|
||||||
NodeKind::Start {
|
NodeKind::Start {
|
||||||
agent: "agent-a".to_owned(),
|
agent: "agent-a".to_owned(),
|
||||||
},
|
},
|
||||||
|
|
@ -954,9 +966,13 @@ fn a_meta_lock_grows_one_rebuild_subgraph_per_agent() {
|
||||||
let q = JobQueue::new(4);
|
let q = JobQueue::new(4);
|
||||||
let agents = vec!["alice".to_owned(), "bob".to_owned()];
|
let agents = vec!["alice".to_owned(), "bob".to_owned()];
|
||||||
let id = q
|
let id = q
|
||||||
.submit(Source::AutoUpdate, "sweep".to_owned(), |b: &JobBuilder| {
|
.submit(
|
||||||
templates::grown_graceful_rebuilds(b, &agents, true);
|
Source::AutoUpdate,
|
||||||
})
|
"sweep".to_owned(),
|
||||||
|
|builder: &JobBuilder| {
|
||||||
|
templates::grown_graceful_rebuilds(builder, &agents, true);
|
||||||
|
},
|
||||||
|
)
|
||||||
.expect("valid shape");
|
.expect("valid shape");
|
||||||
|
|
||||||
// One chain per agent, each an independent group root — so the two rebuild
|
// One chain per agent, each an independent group root — so the two rebuild
|
||||||
|
|
@ -987,7 +1003,7 @@ fn a_meta_lock_grows_one_rebuild_subgraph_per_agent() {
|
||||||
#[test]
|
#[test]
|
||||||
fn cancel_clears_queued_dag() {
|
fn cancel_clears_queued_dag() {
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "r", |b| rebuild(b, "agent-a"));
|
let id = submit(&q, "r", |builder| rebuild(builder, "agent-a"));
|
||||||
assert!(q.cancel(id), "fully-queued dag cancels");
|
assert!(q.cancel(id), "fully-queued dag cancels");
|
||||||
// The operator sees `Cancelled` the moment the cancel returns — the spared
|
// The operator sees `Cancelled` the moment the cancel returns — the spared
|
||||||
// tail is still `Pending`, and a DAG must not read `Queued` back to the
|
// tail is still `Pending`, and a DAG must not read `Queued` back to the
|
||||||
|
|
@ -1016,8 +1032,8 @@ fn cancel_clears_queued_dag() {
|
||||||
#[test]
|
#[test]
|
||||||
fn cancel_drops_one_agents_branch_leaving_the_rest() {
|
fn cancel_drops_one_agents_branch_leaving_the_rest() {
|
||||||
let q = JobQueue::new(2);
|
let q = JobQueue::new(2);
|
||||||
let id = submit(&q, "r", |b| {
|
let id = submit(&q, "r", |builder| {
|
||||||
restart_online(b, &["agent-a", "agent-b"], false);
|
restart_online(builder, &["agent-a", "agent-b"], false);
|
||||||
});
|
});
|
||||||
// Per-agent subgraphs are independent roots; find agent-a's.
|
// Per-agent subgraphs are independent roots; find agent-a's.
|
||||||
let snap = q.snapshot();
|
let snap = q.snapshot();
|
||||||
|
|
@ -1085,20 +1101,20 @@ fn cancelled_power_op_runs_no_compensating_node() {
|
||||||
let case = format!("graceful={graceful} running={running}");
|
let case = format!("graceful={graceful} running={running}");
|
||||||
|
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "bounce", |b| {
|
let id = submit(&q, "bounce", |builder| {
|
||||||
submit::restart_nodes(b, &targets, graceful);
|
submit::restart_nodes(builder, &targets, graceful);
|
||||||
});
|
});
|
||||||
assert_cancels_clean(&q, id, false, &format!("restart {case}"));
|
assert_cancels_clean(&q, id, false, &format!("restart {case}"));
|
||||||
|
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "stop", |b| {
|
let id = submit(&q, "stop", |builder| {
|
||||||
submit::stop_nodes(b, &targets, graceful);
|
submit::stop_nodes(builder, &targets, graceful);
|
||||||
});
|
});
|
||||||
assert_cancels_clean(&q, id, true, &format!("stop {case}"));
|
assert_cancels_clean(&q, id, true, &format!("stop {case}"));
|
||||||
|
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "start", |b| {
|
let id = submit(&q, "start", |builder| {
|
||||||
submit::start_nodes(b, &[("agent-a".to_owned(), running, false)]);
|
submit::start_nodes(builder, &[("agent-a".to_owned(), running, false)]);
|
||||||
});
|
});
|
||||||
assert_cancels_clean(&q, id, true, &format!("start {case}"));
|
assert_cancels_clean(&q, id, true, &format!("start {case}"));
|
||||||
}
|
}
|
||||||
|
|
@ -1118,8 +1134,8 @@ fn cancelled_power_op_runs_no_compensating_node() {
|
||||||
#[test]
|
#[test]
|
||||||
fn cancelled_dag_still_runs_its_approval_tail() {
|
fn cancelled_dag_still_runs_its_approval_tail() {
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "approval #7", |b| {
|
let id = submit(&q, "approval #7", |builder| {
|
||||||
templates::approval_deploy(b, "agent-a", 7);
|
templates::approval_deploy(builder, "agent-a", 7);
|
||||||
});
|
});
|
||||||
assert!(q.cancel(id), "fully-queued dag cancels");
|
assert!(q.cancel(id), "fully-queued dag cancels");
|
||||||
// The `Cancelled` tail is the only node whose edge accepts a dropped
|
// The `Cancelled` tail is the only node whose edge accepts a dropped
|
||||||
|
|
@ -1141,7 +1157,7 @@ fn cancelled_dag_still_runs_its_approval_tail() {
|
||||||
assert_eq!(state_of(&q, id), State::Cancelled);
|
assert_eq!(state_of(&q, id), State::Cancelled);
|
||||||
// An unrelated DAG landing in the same graph doesn't disturb this one's
|
// An unrelated DAG landing in the same graph doesn't disturb this one's
|
||||||
// roll-up — the snapshot is per-DAG, not a global state machine.
|
// roll-up — the snapshot is per-DAG, not a global state machine.
|
||||||
let _other = submit(&q, "r", |b| rebuild(b, "agent-b"));
|
let _other = submit(&q, "r", |builder| rebuild(builder, "agent-b"));
|
||||||
assert_eq!(state_of(&q, id), State::Cancelled);
|
assert_eq!(state_of(&q, id), State::Cancelled);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1155,8 +1171,8 @@ fn cancelled_dag_still_runs_its_approval_tail() {
|
||||||
#[test]
|
#[test]
|
||||||
fn deploy_dag_runs_phases_in_order_and_tails_a_failed_apply() {
|
fn deploy_dag_runs_phases_in_order_and_tails_a_failed_apply() {
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "approval #7", |b| {
|
let id = submit(&q, "approval #7", |builder| {
|
||||||
templates::approval_deploy(b, "agent-a", 7);
|
templates::approval_deploy(builder, "agent-a", 7);
|
||||||
});
|
});
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|
@ -1213,8 +1229,8 @@ fn deploy_apply_grows_rebuild_subgraph_and_finalizes_after_it() {
|
||||||
// children run (`a_completing_node_grows_the_work_it_declared`,
|
// children run (`a_completing_node_grows_the_work_it_declared`,
|
||||||
// `parent_parks_in_finishing_until_children_roll_up`).
|
// `parent_parks_in_finishing_until_children_roll_up`).
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "deploy graft", |b| {
|
let id = submit(&q, "deploy graft", |builder| {
|
||||||
templates::deploy_rebuild_nodes(b, "agent-a", 11);
|
templates::deploy_rebuild_nodes(builder, "agent-a", 11);
|
||||||
});
|
});
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|
@ -1380,7 +1396,9 @@ fn error_truncation_cuts_on_a_char_boundary() {
|
||||||
#[test]
|
#[test]
|
||||||
fn graceful_stop_shape_signal_drain_reconcile() {
|
fn graceful_stop_shape_signal_drain_reconcile() {
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "graceful", |b| stop_online(b, &["agent-a"], true));
|
let id = submit(&q, "graceful", |builder| {
|
||||||
|
stop_online(builder, &["agent-a"], true);
|
||||||
|
});
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
declared_shape(&q, id),
|
declared_shape(&q, id),
|
||||||
vec![
|
vec![
|
||||||
|
|
@ -1410,8 +1428,8 @@ fn graceful_stop_shape_signal_drain_reconcile() {
|
||||||
#[test]
|
#[test]
|
||||||
fn spawn_shape_provision_create_dropin_reconcile() {
|
fn spawn_shape_provision_create_dropin_reconcile() {
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "approval #7 spawn", |b| {
|
let id = submit(&q, "approval #7 spawn", |builder| {
|
||||||
templates::spawn(b, "newbie", 7);
|
templates::spawn(builder, "newbie", 7);
|
||||||
});
|
});
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
declared_shape(&q, id),
|
declared_shape(&q, id),
|
||||||
|
|
@ -1435,9 +1453,9 @@ fn spawn_shape_provision_create_dropin_reconcile() {
|
||||||
#[test]
|
#[test]
|
||||||
fn perm_change_shape_prefixes_rebuild_chain() {
|
fn perm_change_shape_prefixes_rebuild_chain() {
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "perm", |b| {
|
let id = submit(&q, "perm", |builder| {
|
||||||
templates::perm_change(
|
templates::perm_change(
|
||||||
b,
|
builder,
|
||||||
"agent-a",
|
"agent-a",
|
||||||
PermPayload::Combined {
|
PermPayload::Combined {
|
||||||
groups: Some(vec![]),
|
groups: Some(vec![]),
|
||||||
|
|
@ -1473,8 +1491,8 @@ fn reparent_shape_is_a_lone_agentless_meta_window_node() {
|
||||||
// `MetaLock`, and it must declare the meta window — a topology commit
|
// `MetaLock`, and it must declare the meta window — a topology commit
|
||||||
// must not land inside another node's staged deploy window.
|
// must not land inside another node's staged deploy window.
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "set-parent", |b| {
|
let id = submit(&q, "set-parent", |builder| {
|
||||||
templates::reparent(b, vec![(ident("alice"), Some(ident("bob")))]);
|
templates::reparent(builder, vec![(ident("alice"), Some(ident("bob")))]);
|
||||||
});
|
});
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
declared_shape(&q, id),
|
declared_shape(&q, id),
|
||||||
|
|
@ -1497,8 +1515,8 @@ fn reparent_bulk_shape_carries_every_move_on_one_node() {
|
||||||
// request is the reason a single node was chosen in the first place.
|
// request is the reason a single node was chosen in the first place.
|
||||||
let moves = vec![(ident("alice"), Some(ident("bob"))), (ident("carol"), None)];
|
let moves = vec![(ident("alice"), Some(ident("bob"))), (ident("carol"), None)];
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "set-parent-bulk", |b| {
|
let id = submit(&q, "set-parent-bulk", |builder| {
|
||||||
templates::reparent(b, moves.clone());
|
templates::reparent(builder, moves.clone());
|
||||||
});
|
});
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
declared_shape(&q, id),
|
declared_shape(&q, id),
|
||||||
|
|
|
||||||
|
|
@ -327,7 +327,7 @@ pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
|
||||||
/// kind-derived resources were removed, which drops the agent lease a boot
|
/// kind-derived resources were removed, which drops the agent lease a boot
|
||||||
/// reconcile needs to not race another DAG's container ops.
|
/// reconcile needs to not race another DAG's container ops.
|
||||||
pub(crate) fn boot_nodes(
|
pub(crate) fn boot_nodes(
|
||||||
b: &crate::job_queue::JobBuilder,
|
builder: &crate::job_queue::JobBuilder,
|
||||||
any_stale: bool,
|
any_stale: bool,
|
||||||
fanout: Vec<String>,
|
fanout: Vec<String>,
|
||||||
drifted: Vec<String>,
|
drifted: Vec<String>,
|
||||||
|
|
@ -341,7 +341,7 @@ pub(crate) fn boot_nodes(
|
||||||
// ⇒ no meta commit on a no-change boot. The `fanout` list rides the
|
// ⇒ no meta commit on a no-change boot. The `fanout` list rides the
|
||||||
// MetaLock into `run_meta_lock`, which appends the rebuild subgraphs.
|
// MetaLock into `run_meta_lock`, which appends the rebuild subgraphs.
|
||||||
if any_stale {
|
if any_stale {
|
||||||
let _ = b
|
let _ = builder
|
||||||
.node(NodeKind::MetaLock {
|
.node(NodeKind::MetaLock {
|
||||||
sweep: true,
|
sweep: true,
|
||||||
fanout: Some(fanout),
|
fanout: Some(fanout),
|
||||||
|
|
@ -356,7 +356,9 @@ pub(crate) fn boot_nodes(
|
||||||
for name in drifted {
|
for name in drifted {
|
||||||
// Name the lease before the agent string moves into the kind.
|
// Name the lease before the agent string moves into the kind.
|
||||||
let lease = Resource::Agent(name.clone());
|
let lease = Resource::Agent(name.clone());
|
||||||
let _ = b.node(NodeKind::Reconcile { agent: name }).needs(lease);
|
let _ = builder
|
||||||
|
.node(NodeKind::Reconcile { agent: name })
|
||||||
|
.needs(lease);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue