refactor(#2949): kill Declare — a running node declares onto its own builder

A node no longer hands back a recipe for the scheduler to replay later. It
declares straight onto a builder it was given, and that builder is inserted
as part of completing the node.

Deleted: `pub type Declare`, `struct NodeOutput` (+ its hand-written `Debug`),
`JobQueue::append_subgraph`. Nothing added to `Dag` / `DagView`.

jobq gains `Scheduler::new_job()` (the only way to obtain a `JobBuilder`) and
`complete_growing(id, outcome, grown)`, which inserts under `id` and *then*
completes it, so a DAG cannot roll terminal while grown work is still pending.
`complete()` and `complete_growing()` share a private `finish()` rather than
one redirecting through the other. The DAG-gone guard lives beside the graph
now, where it cannot be skipped, instead of being a caller-side lookup.

The growth executors return data (`run_meta_lock -> (Vec<String>, RebuildOpts)`,
`run_reconcile -> Option<NodeKind>`) rather than taking the builder: a `&Job`
parameter is live for the whole function body, and `&RefCell<T>` is never
`Send`, so an async fn taking one cannot be spawned. `run_node` threads the
builder by value and hands it back.

A node can now declare work and then fail, which was previously inexpressible.
`grown` is dropped in that case — failure cancel-cascades downstream, so
inserting it would only add nodes to immediately cancel — and the log line
carries `grown_nodes` so the drop is visible.
This commit is contained in:
atlas 2026-08-02 17:20:43 +02:00 committed by mara
commit 82ef06f445
8 changed files with 388 additions and 344 deletions

View file

@ -10,7 +10,7 @@ use std::sync::Arc;
use anyhow::{Context as _, Result};
use super::{Claim, Declare};
use super::Claim;
use hive_jobq::TerminalState;
use super::model::NodeKind;
@ -26,36 +26,6 @@ use crate::power::{ReconcileAction, reconcile_action};
/// N × this timeout.
pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(3);
/// Extra signal an executor hands back to the scheduler alongside
/// success.
#[derive(Default)]
pub struct NodeOutput {
/// Whole per-agent *subgraphs* to append into *this same* DAG at
/// runtime — the single in-DAG-growth channel. Each [`Job`] is one
/// independent subgraph, declared but not yet inserted: an executor cannot
/// reach the queue, so it hands the declaration back and the scheduler
/// inserts it via [`super::JobQueue::append_subgraph`] under its own lock,
/// rooted on the emitting node. Used both for the multi-node case
/// (`MetaLock` growing one rebuild subgraph per agent — the startup
/// sweep's stale agents, the meta-update cascade's affected agents) and
/// the single-node case (a `Reconcile` planner emitting its mechanical
/// `Start` / `Stop` as a one-node subgraph). The scheduler applies these
/// *before* the emitting node's completion so the DAG never rolls terminal
/// with the appended work still pending — keeping the lease-window
/// transient held across the sub-step.
pub append_subgraph: Vec<Declare>,
}
impl std::fmt::Debug for NodeOutput {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
// The subgraphs are closures — how many were emitted is the only thing
// there is to say about them before the queue runs them.
f.debug_struct("NodeOutput")
.field("append_subgraph", &self.append_subgraph.len())
.finish()
}
}
/// Build-log sink for one claimed node.
struct Ctx<'a> {
coord: &'a Arc<Coordinator>,
@ -78,13 +48,35 @@ impl Ctx<'_> {
/// Run one claimed node to completion. Called from a task the
/// scheduler spawns per claim; the `Result` (stringified) becomes the
/// node's terminal state.
pub(super) async fn run_node(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
///
/// `job` is the node's own growth channel: an executor that decides more work
/// is needed declares it here, and the scheduler inserts it under this node
/// when the node completes. Most executors never touch it. Nothing is inserted
/// while the node runs — the builder is local state, so this stays outside the
/// queue's lock for the whole (often multi-minute) execution.
///
/// ⚠️ Taken **by value and handed back**, not by reference. A `JobBuilder` is
/// `RefCell`-backed: owned it is `Send`, but `&JobBuilder` is not (a shared ref
/// is `Send` only if the referent is `Sync`, and `RefCell` never is). A `&Job`
/// parameter would be live across every `.await` in this fn and make the whole
/// future non-`Send`, which the scheduler's `tokio::spawn` rejects. So the
/// growth executors below return *what to grow* and the declaration happens
/// here, synchronously, between awaits.
pub(super) async fn run_node(
coord: &Arc<Coordinator>,
job: super::Job,
claim: &Claim,
) -> (super::Job, Result<()>) {
let ctx = Ctx {
coord,
dag_id: claim.dag_id,
node_id: claim.node_id,
};
match &claim.kind {
// Every arm is `Result<()>`; the three that grow work declare into `job`
// *synchronously*, after their own awaits have finished. Borrowing `&job`
// inside an `.await` would make this future non-`Send` (see above), so the
// growth executors return what to grow rather than taking the builder.
let result = match &claim.kind {
NodeKind::MetaSync { relock, .. } => run_meta_sync(coord, claim, *relock).await,
NodeKind::Prebuild { .. } => run_prebuild(claim, &ctx).await,
NodeKind::Swap { .. } => run_swap(coord, claim, &ctx).await,
@ -95,19 +87,40 @@ pub(super) async fn run_node(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
sweep,
fanout,
inputs,
} => run_meta_lock(coord, *sweep, fanout.clone(), inputs).await,
NodeKind::Reconcile { .. } => run_reconcile(coord, claim).await,
} => run_meta_lock(coord, *sweep, fanout.clone(), inputs)
.await
.map(|(agents, opts)| {
for agent in agents {
super::templates::rebuild_nodes(&job, &agent, opts, None);
}
}),
NodeKind::Reconcile { .. } => run_reconcile(coord, claim).await.map(|sub| {
if let Some(kind) = sub {
// `Start` / `Stop` declare the lease they run under. This node
// is their parent and holds it, so the declaration is a
// re-entrant borrow — no second unit, no deadlock. It exists so
// the requirement belongs to the node rather than to the fact
// that a `Reconcile` happens to fan it out.
let lease = Resource::Agent(kind.agent().to_owned());
let _ = job.node(kind).needs(lease);
}
}),
NodeKind::Start { .. } => run_start(coord, claim).await,
NodeKind::Stop { .. } => run_stop(coord, claim).await,
NodeKind::StopForUpdate { .. } => run_stop_for_update(coord, claim).await,
NodeKind::Signal { .. } => Ok(run_signal(coord, claim)),
NodeKind::Signal { .. } => {
run_signal(coord, claim);
Ok(())
}
NodeKind::Drain { .. } => run_drain(coord, claim).await,
NodeKind::WriteDropin { .. } => run_write_dropin(coord, claim).await,
NodeKind::WritePermFile { .. } => run_write_perm_file(coord, claim).await,
NodeKind::Reparent { .. } => run_reparent(coord, claim).await,
NodeKind::MergeVerify { approval_id, .. } => run_merge_verify(coord, *approval_id).await,
NodeKind::DeployApply { approval_id, .. } => {
run_deploy_apply(coord, claim, *approval_id).await
run_deploy_apply(coord, *approval_id).await.map(|()| {
super::templates::deploy_rebuild_nodes(&job, claim.kind.agent(), *approval_id);
})
}
NodeKind::FinalizeDeploy { approval_id, .. } => {
run_finalize_deploy(coord, *approval_id).await
@ -119,7 +132,10 @@ pub(super) async fn run_node(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
approval_id,
outcome,
} => run_resolve_approval(coord, claim, *approval_id, *outcome).await,
NodeKind::EmitRebuilt { ok, .. } => Ok(run_emit_rebuilt(coord, claim, *ok)),
NodeKind::EmitRebuilt { ok, .. } => {
run_emit_rebuilt(coord, claim, *ok);
Ok(())
}
NodeKind::SetWanted { up, .. } => run_set_wanted(coord, claim, *up),
// The two nodes that carry no work of their own; completing either
// lets it reach `Finishing` so the nodes under it start.
@ -127,8 +143,9 @@ pub(super) async fn run_node(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
// any, is its own tail node in the graph.
// - `DeployWindow`: pure resource holder — the meta window, agent lease
// and build slot it declares stay held until its subtree settles.
NodeKind::Dag { .. } | NodeKind::DeployWindow { .. } => Ok(NodeOutput::default()),
}
NodeKind::Dag { .. } | NodeKind::DeployWindow { .. } => Ok(()),
};
(job, result)
}
/// Resolve the DAG's approval row the way this node's own `outcome` says.
@ -143,18 +160,18 @@ async fn run_resolve_approval(
claim: &Claim,
approval_id: i64,
outcome: TerminalState,
) -> Result<NodeOutput> {
) -> Result<()> {
let reason = (outcome == TerminalState::Failed)
.then(|| coord.job_queue.first_error(claim.dag_id))
.flatten();
crate::actions::resolve_approval_dag(coord, approval_id, outcome, reason.as_deref()).await;
Ok(NodeOutput::default())
Ok(())
}
/// Emit this agent's `Rebuilt` manager event. `ok` is not computed — it is which
/// of the tail pair the graph let run. The failure note comes from the DAG's
/// first failing node, since the branch knows *that* it failed but not *why*.
fn run_emit_rebuilt(coord: &Arc<Coordinator>, claim: &Claim, ok: bool) -> NodeOutput {
fn run_emit_rebuilt(coord: &Arc<Coordinator>, claim: &Claim, ok: bool) {
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
agent: claim.agent.clone(),
ok,
@ -164,7 +181,6 @@ fn run_emit_rebuilt(coord: &Arc<Coordinator>, claim: &Claim, ok: bool) -> NodeOu
sha: None,
tag: None,
});
NodeOutput::default()
}
/// Write the agent's durable power intent — the DAG-node form of the old
@ -175,7 +191,7 @@ fn run_emit_rebuilt(coord: &Arc<Coordinator>, claim: &Claim, ok: bool) -> NodeOu
/// warn-and-continue write, a failed write fails the node (cancel-downstream
/// cancels the `Reconcile`) rather than letting it converge to a stale
/// intent — that atomicity is the point of moving it into the DAG.
fn run_set_wanted(coord: &Arc<Coordinator>, claim: &Claim, up: bool) -> Result<NodeOutput> {
fn run_set_wanted(coord: &Arc<Coordinator>, claim: &Claim, up: bool) -> Result<()> {
let wanted = if up {
crate::power::Wanted::Up
} else {
@ -185,7 +201,7 @@ fn run_set_wanted(coord: &Arc<Coordinator>, claim: &Claim, up: bool) -> Result<N
.power
.set(&claim.agent, wanted)
.with_context(|| format!("set wanted={} for agent {}", wanted.as_str(), claim.agent))?;
Ok(NodeOutput::default())
Ok(())
}
/// The rebuild's meta preamble: runtime-dir prep, an idempotent meta
@ -197,11 +213,7 @@ fn run_set_wanted(coord: &Arc<Coordinator>, claim: &Claim, up: bool) -> Result<N
/// Deliberately a separate node from the [`run_prebuild`] it feeds: that
/// build takes minutes and only *reads* the store, so keeping the global
/// window off it is what lets rebuilds of different agents overlap.
async fn run_meta_sync(
coord: &Arc<Coordinator>,
claim: &Claim,
relock: bool,
) -> Result<NodeOutput> {
async fn run_meta_sync(coord: &Arc<Coordinator>, claim: &Claim, relock: bool) -> Result<()> {
let name = &claim.agent;
// Runs while the agent is still up — the runtime dir and MCP listener
// already exist. Use the pure path accessor; no need to re-register the
@ -219,7 +231,7 @@ async fn run_meta_sync(
if relock {
crate::meta::lock_update_for_rebuild(name).await?;
}
Ok(NodeOutput::default())
Ok(())
}
/// Out-of-band toplevel build while the container keeps serving: warm
@ -229,7 +241,7 @@ async fn run_meta_sync(
/// container is already down: its only purpose is to shrink the swap's
/// downtime window, so a stopped agent (no uptime to preserve) doesn't
/// pay the double eval — `Swap` builds inline instead.
async fn run_prebuild(claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
async fn run_prebuild(claim: &Claim, ctx: &Ctx<'_>) -> Result<()> {
let name = &claim.agent;
// Warm the toplevel build only when the container is up — the whole
// point of prebuild is to shrink the swap's downtime window. A
@ -240,7 +252,7 @@ async fn run_prebuild(claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
crate::lifecycle::prebuild_toplevel(name, &flake_ref, &|log_id| ctx.build_log(log_id))
.await?;
}
Ok(NodeOutput::default())
Ok(())
}
/// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb),
@ -248,7 +260,7 @@ async fn run_prebuild(claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
/// (rev marker, `Rebuilt` event, forge/matrix sync, kick, rescan).
/// The recovery-start on failure is NOT here — the DAG's tail
/// `Reconcile` runs after this node terminal ok *or* fail.
async fn run_swap(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
async fn run_swap(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Result<()> {
let name = &claim.agent;
// Swap runs on an already-existing (stopped) container — runtime dir
// and listener were created earlier. Pure path accessor suffices.
@ -269,7 +281,7 @@ async fn run_swap(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Res
if result.is_err() {
coord.rescan_containers_and_emit().await;
}
result.map(|()| NodeOutput::default())
result
}
/// The post-`Swap` bookkeeping tail, split into its own node for dashboard
@ -277,7 +289,7 @@ async fn run_swap(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Res
/// means the profile swap succeeded. Store/forge/matrix work only — no nix
/// build (build-slot-exempt); the agent lease taken at `Swap` is still held
/// (the whole chain up to `Reconcile` is one agent's subgraph).
async fn run_post_swap(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
async fn run_post_swap(coord: &Arc<Coordinator>, claim: &Claim) -> Result<()> {
let name = &claim.agent;
if let Some(rev) = crate::auto_update::current_flake_rev(&coord.hyperhive_flake)
&& let Err(e) = std::fs::write(crate::paths::applied_rev_marker(name), rev)
@ -298,20 +310,20 @@ async fn run_post_swap(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOu
coord.kick_agent(name, "container rebuilt");
coord.rescan_containers_and_emit().await;
crate::dashboard::emit_meta_inputs_snapshot(coord);
Ok(NodeOutput::default())
Ok(())
}
/// First-spawn pre-create provisioning: proposed/applied repos, state
/// subvolume, and the meta `sync_agents` registration. Runs under the
/// deploy window (it declares `Resource::MetaWindow`) so its commit can't
/// land inside another node's staged deploy window.
async fn run_provision(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
async fn run_provision(coord: &Arc<Coordinator>, claim: &Claim) -> Result<()> {
let name = &claim.agent;
let agent_dir = crate::paths::agent_runtime_dir(name);
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(name, agent_dir);
crate::lifecycle::provision_container(name, &hive, &paths).await?;
Ok(NodeOutput::default())
Ok(())
}
/// `nixos-container create` proper — the upstream `Provision` node
@ -320,21 +332,24 @@ async fn run_provision(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOu
/// dir creation and MCP listener registration are deferred to the tail
/// `Reconcile` (`converge_start_preamble` + `register_agent`) so this
/// node stays purely "create", not "create + start".
async fn run_create(claim: &Claim) -> Result<NodeOutput> {
async fn run_create(claim: &Claim) -> Result<()> {
crate::lifecycle::create_only(&claim.agent).await?;
Ok(NodeOutput::default())
Ok(())
}
/// Meta flake lock bump. Boot-sweep flavour is non-fatal (a failed
/// bump must not cancel the fan-out rebuilds — they proceed against
/// the current lock, exactly like today's sweep); the meta-update
/// flavour propagates errors, and a failed bump fans out nothing.
/// Returns the agents whose rebuild subgraphs the caller should grow into this
/// node, and the options to build them with — rather than declaring them here.
/// The declaration has to happen outside any `.await` (see [`run_node`]).
async fn run_meta_lock(
coord: &Arc<Coordinator>,
sweep: bool,
fanout: Option<Vec<String>>,
inputs: &[String],
) -> Result<NodeOutput> {
) -> Result<(Vec<String>, super::templates::RebuildOpts)> {
if sweep {
if let Err(e) = crate::meta::lock_update_hyperhive().await {
tracing::warn!(error = ?e, "startup sweep: meta lock_update_hyperhive failed");
@ -349,25 +364,13 @@ async fn run_meta_lock(
// drain window rather than being cut off. The per-agent drains overlap,
// so the sweep's cost ceiling is one `GRACEFUL_STOP_TIMEOUT` in total,
// not one per agent.
let append_subgraph = fanout
.unwrap_or_default()
.iter()
.map(|agent| {
let agent = agent.clone();
Box::new(move |b: &super::Job| {
super::templates::rebuild_nodes(
b,
&agent,
super::templates::RebuildOpts {
relock: true,
graceful: true,
},
None,
);
}) as Declare
})
.collect();
return Ok(NodeOutput { append_subgraph });
return Ok((
fanout.unwrap_or_default(),
super::templates::RebuildOpts {
relock: true,
graceful: true,
},
));
}
let _progress = coord.meta_update_guard();
crate::meta::lock_update(inputs).await?;
@ -383,68 +386,47 @@ async fn run_meta_lock(
// cascade children must NOT re-lock, which would revert the bump this
// node just committed (the property the old `fanout_specs` meta-update
// branch encoded).
let append_subgraph = cascade
.iter()
.map(|agent| {
let agent = agent.clone();
Box::new(move |b: &super::Job| {
super::templates::rebuild_nodes(
b,
&agent,
super::templates::RebuildOpts {
relock: false,
graceful: false,
},
None,
);
}) as Declare
})
.collect();
Ok(NodeOutput { append_subgraph })
Ok((
cascade,
super::templates::RebuildOpts {
relock: false,
graceful: false,
},
))
}
/// Idempotent power-converge *planner*: compare `wanted` (durable
/// intent) against observed state and, when they diverge, fan the
/// mechanical `Start` / `Stop` out as a first-class node appended to
/// *this* DAG (a single-node `NodeOutput::append_subgraph` rooted on
/// this node). Does no container work itself — the sub-step becomes
/// visible in the DAG and the lease-window transient (or the sub-step's
/// own node-local guard) rides across it.
async fn run_reconcile(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
/// *this* DAG (a single node declared into `job`, rooted on this node).
/// Does no container work itself — the sub-step becomes visible in the
/// DAG and the lease-window transient (or the sub-step's own node-local
/// guard) rides across it.
/// Returns the mechanical node to fan out (`None` on a noop) rather than
/// declaring it — the declaration has to happen outside any `.await`, see
/// [`run_node`]. `NodeKind` carries the agent it targets, so `claim.agent` is
/// stamped into the kind here.
async fn run_reconcile(coord: &Arc<Coordinator>, claim: &Claim) -> Result<Option<NodeKind>> {
let name = &claim.agent;
let running = crate::lifecycle::is_running(name).await;
let wanted = coord.power.get_or_seed(name, running)?;
// One node targeting this agent, rooted on this reconcile node. `NodeKind`
// carries the agent it targets, so stamp `claim.agent` into the fanned-out
// Start/Stop kind (one in-DAG-growth channel).
let sub = |kind: NodeKind| {
// `Start` / `Stop` declare the lease they run under. This node is their
// parent and holds it, so the declaration is a re-entrant borrow — no
// second unit, no deadlock. It exists so the requirement belongs to the
// node rather than to the fact that a `Reconcile` happens to fan it out.
let lease = Resource::Agent(kind.agent().to_owned());
vec![Box::new(move |b: &super::Job| {
let _ = b.node(kind).needs(lease);
}) as Declare]
};
let append_subgraph = match reconcile_action(wanted, running) {
ReconcileAction::Start => sub(NodeKind::Start {
Ok(match reconcile_action(wanted, running) {
ReconcileAction::Start => Some(NodeKind::Start {
agent: name.clone(),
}),
ReconcileAction::Stop => sub(NodeKind::Stop {
ReconcileAction::Stop => Some(NodeKind::Stop {
agent: name.clone(),
}),
ReconcileAction::Noop => {
tracing::debug!(%name, wanted = wanted.as_str(), running, "reconcile: noop");
Vec::new()
None
}
};
Ok(NodeOutput { append_subgraph })
})
}
/// Mechanical container start — the sub-step a `Reconcile` planner fans
/// out when it observes `wanted = Up` and the container down.
async fn run_start(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
async fn run_start(coord: &Arc<Coordinator>, claim: &Claim) -> Result<()> {
let name = &claim.agent;
// No node-local transient guard: the pill is derived from the running node
// set, and `Start` reports `Starting` via `NodeKind::transient_kind`. This
@ -468,12 +450,12 @@ async fn run_start(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput
coord.register_agent(name)?;
coord.kick_agent(name, "container started");
coord.rescan_containers_and_emit().await;
Ok(NodeOutput::default())
Ok(())
}
/// Mechanical container stop — the sub-step a `Reconcile` planner fans
/// out when it observes `wanted = Offline` and the container up.
async fn run_stop(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
async fn run_stop(coord: &Arc<Coordinator>, claim: &Claim) -> Result<()> {
let name = &claim.agent;
// See `run_start`: no node-local guard — `Stop` reports `Stopping` from its
// own kind now.
@ -483,12 +465,12 @@ async fn run_stop(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput>
agent: name.clone(),
});
coord.rescan_containers_and_emit().await;
Ok(NodeOutput::default())
Ok(())
}
/// Mechanical stop for the profile swap. Never *changes* `wanted`;
/// noop when already stopped.
async fn run_stop_for_update(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
async fn run_stop_for_update(coord: &Arc<Coordinator>, claim: &Claim) -> Result<()> {
let name = &claim.agent;
if crate::lifecycle::is_running(name).await {
// Seed a missing agent_power row from the PRE-stop observation
@ -501,7 +483,7 @@ async fn run_stop_for_update(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
crate::lifecycle::kill(name).await?;
coord.rescan_containers_and_emit().await;
}
Ok(NodeOutput::default())
Ok(())
}
/// Set the graceful fence + kick so the harness sees it promptly and
@ -513,19 +495,18 @@ async fn run_stop_for_update(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
/// `GRACEFUL_STOP_TIMEOUT`. Safe because the harness tests the marker at
/// the top of its loop — a paused agent has no turn in flight, so there
/// is nothing to checkpoint.
fn run_signal(coord: &Arc<Coordinator>, claim: &Claim) -> NodeOutput {
fn run_signal(coord: &Arc<Coordinator>, claim: &Claim) {
if hive_types::Ident::parse(&claim.agent).is_ok_and(|a| Coordinator::is_paused(&a)) {
return NodeOutput::default();
return;
}
coord.mark_graceful_stop(&claim.agent);
coord.kick_agent(&claim.agent, "graceful stop requested");
NodeOutput::default()
}
/// Await the harness clearing the fence (`GracefulStopComplete`) or
/// the timeout — either way the downstream `Reconcile` proceeds with
/// the actual stop.
async fn run_drain(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
async fn run_drain(coord: &Arc<Coordinator>, claim: &Claim) -> Result<()> {
let name = &claim.agent;
let deadline = std::time::Instant::now() + GRACEFUL_STOP_TIMEOUT;
while coord.is_graceful_stop_pending(name) {
@ -536,11 +517,11 @@ async fn run_drain(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
}
coord.clear_graceful_stop(name);
Ok(NodeOutput::default())
Ok(())
}
/// `set_nspawn_flags` + `set_resource_limits` + daemon-reload.
async fn run_write_dropin(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
async fn run_write_dropin(coord: &Arc<Coordinator>, claim: &Claim) -> Result<()> {
let name = &claim.agent;
// write_dropins only needs the path value to build AgentPaths; the
// dir doesn't need to exist at this point (created by ensure_agent_runtime_dir
@ -549,13 +530,13 @@ async fn run_write_dropin(coord: &Arc<Coordinator>, claim: &Claim) -> Result<Nod
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(name, agent_dir);
crate::lifecycle::write_dropins(name, &hive, &paths).await?;
Ok(NodeOutput::default())
Ok(())
}
/// Write + commit the perm file(s) (fused under `META_LOCK` so the
/// working tree is never left dirty), then emit the P3RM1SS10NS-tab
/// snapshots so the dashboard reflects the new assignment.
async fn run_write_perm_file(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
async fn run_write_perm_file(coord: &Arc<Coordinator>, claim: &Claim) -> Result<()> {
use super::model::PermPayload;
let name = &claim.agent;
// The perm file payload rides the node itself (the only consumer).
@ -591,7 +572,7 @@ async fn run_write_perm_file(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
}
}
}
Ok(NodeOutput::default())
Ok(())
}
/// Apply the node's `(child, new_parent)` moves as one `META_LOCK`-fused
@ -601,7 +582,7 @@ async fn run_write_perm_file(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
/// the deploy window (it declares `Resource::MetaWindow`), same reasoning as
/// `run_write_perm_file`: a topology commit landing inside another node's
/// staged deploy window would sweep the staged lock into its commit.
async fn run_reparent(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
async fn run_reparent(coord: &Arc<Coordinator>, claim: &Claim) -> Result<()> {
let NodeKind::Reparent { moves } = &claim.kind else {
anyhow::bail!("run_reparent on a non-Reparent node");
};
@ -618,16 +599,14 @@ async fn run_reparent(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOut
.reparent_bulk_with_notify(&refs)
.await
.map_err(|e| anyhow::anyhow!(e))?;
Ok(NodeOutput::default())
Ok(())
}
/// Deploy phase 1 — drift gate, fetch, eval-verify. Mutates nothing, so a
/// failure here cancel-cascades the rest of the subtree with the forge and the
/// applied repo exactly as they were.
async fn run_merge_verify(coord: &Arc<Coordinator>, approval_id: i64) -> Result<NodeOutput> {
crate::actions::run_deploy_merge_verify(coord, approval_id)
.await
.map(|()| NodeOutput::default())
async fn run_merge_verify(coord: &Arc<Coordinator>, approval_id: i64) -> Result<()> {
crate::actions::run_deploy_merge_verify(coord, approval_id).await
}
/// Deploy phase 2 — the irreversible half: ff-merge, then phase 1 of the
@ -639,27 +618,15 @@ async fn run_merge_verify(coord: &Arc<Coordinator>, approval_id: i64) -> Result<
/// their `MetaSync` declares is re-entered rather than deadlocked against the
/// ancestor already holding it. On failure nothing is appended and the tail
/// compensates, exactly as before.
async fn run_deploy_apply(
coord: &Arc<Coordinator>,
claim: &Claim,
approval_id: i64,
) -> Result<NodeOutput> {
crate::actions::run_deploy_apply(coord, approval_id).await?;
Ok(NodeOutput {
append_subgraph: vec![super::templates::deploy_rebuild_nodes(
claim.kind.agent(),
approval_id,
)],
})
async fn run_deploy_apply(coord: &Arc<Coordinator>, approval_id: i64) -> Result<()> {
crate::actions::run_deploy_apply(coord, approval_id).await
}
/// Deploy phase 3 — close the staged-lock window once the appended rebuild has
/// come up clean: drop the rollback ref, plant the `deployed/<id>` tag, commit
/// the staged lock.
async fn run_finalize_deploy(coord: &Arc<Coordinator>, approval_id: i64) -> Result<NodeOutput> {
crate::actions::run_finalize_deploy(coord, approval_id)
.await
.map(|()| NodeOutput::default())
async fn run_finalize_deploy(coord: &Arc<Coordinator>, approval_id: i64) -> Result<()> {
crate::actions::run_finalize_deploy(coord, approval_id).await
}
/// Deploy compensation + bookkeeping tail. `AfterAny` the apply node, so it
@ -669,14 +636,10 @@ async fn run_finalize_deploy(coord: &Arc<Coordinator>, approval_id: i64) -> Resu
///
/// Takes the agent from the node payload so the tail can still compensate when
/// the approval row is gone (deny race, purge).
async fn run_deploy_tail(
coord: &Arc<Coordinator>,
claim: &Claim,
approval_id: i64,
) -> Result<NodeOutput> {
async fn run_deploy_tail(coord: &Arc<Coordinator>, claim: &Claim, approval_id: i64) -> Result<()> {
crate::actions::run_deploy_tail(coord, Some(claim.dag_id), claim.kind.agent(), approval_id)
.await;
Ok(NodeOutput::default())
Ok(())
}
/// Compute which agents a `nix flake update <inputs>` on the meta