vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(#2349): fan reconcile's start/stop out as first-class dag nodes

Browse source
This commit is contained in:
damocles 2026-07-11 23:33:45 +02:00 • committed by mara
commit 79d4c345bb
4 changed files with 156 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

133
hive-c0re/src/job_queue/exec.rs
View file

@ -29,6 +29,15 @@ pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from
pub struct NodeOutput {
/// Agents to fan child `Rebuild` DAGs out for (`MetaLock` only).
pub fanout: Vec<String>,
/// Mechanical sub-step nodes to append into *this same* DAG at
/// runtime, each depending `AfterOk` on the emitting node — e.g. a
/// `Reconcile` planner emitting a `Start` / `Stop`. A separate
/// channel from `fanout` (which appends whole *child* DAGs) so the
/// sub-step stays a first-class node in the same DAG and the
/// lease-window transient is held across it. The scheduler applies
/// these *before* the emitting node's completion so the DAG never
/// rolls terminal with the appended work still pending.
pub append_nodes: Vec<NodeKind>,
}
/// Step-label + build-log sink for one claimed node.
@ -76,7 +85,9 @@ pub(super) async fn run_node(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
NodeKind::MetaLock { sweep, fanout } => {
run_meta_lock(coord, claim, &ctx, *sweep, fanout.clone()).await
}
NodeKind::Reconcile => run_reconcile(coord, claim, &ctx).await,
NodeKind::Reconcile => run_reconcile(coord, claim).await,
NodeKind::Start => run_start(coord, claim, &ctx).await,
NodeKind::Stop => run_stop(coord, claim, &ctx).await,
NodeKind::StopForUpdate => run_stop_for_update(coord, claim, &ctx).await,
NodeKind::Signal => Ok(run_signal(coord, claim, &ctx)),
NodeKind::Drain => run_drain(coord, claim, &ctx).await,
@ -216,6 +227,7 @@ async fn run_meta_lock(
}
return Ok(NodeOutput {
fanout: fanout.unwrap_or_default(),
..Default::default()
});
}
let _progress = coord.meta_update_guard();
@ -230,63 +242,84 @@ async fn run_meta_lock(
Some(list) => list,
None => meta_update_cascade_agents(&claim.inputs).await,
};
Ok(NodeOutput { fanout: cascade })
Ok(NodeOutput {
fanout: cascade,
..Default::default()
})
}
/// Idempotent power converge: `wanted` (durable intent) vs observed.
async fn run_reconcile(
coord: &Arc<Coordinator>,
claim: &Claim,
ctx: &Ctx<'_>,
) -> Result<NodeOutput> {
/// Idempotent power-converge *planner*: compare `wanted` (durable
/// intent) against observed state and, when they diverge, fan the
/// mechanical `Start` / `Stop` out as a first-class node appended to
/// *this* DAG (`NodeOutput::append_nodes`). Does no container work
/// itself — the sub-step becomes visible in the DAG and the
/// lease-window transient (or the sub-step's own node-local guard)
/// rides across it.
async fn run_reconcile(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
let name = &claim.agent;
let running = crate::lifecycle::is_running(name).await;
let wanted = coord.power.get_or_seed(name, running)?;
match reconcile_action(wanted, running) {
ReconcileAction::Start => {
// Node-local transient only when the DAG holds none (the
// boot-reconcile template); lease-window guards otherwise
// already cover this node.
let _guard = claim
.transient
.is_none()
.then(|| coord.transient_guard(name, crate::coordinator::TransientKind::Starting));
// Run the typed start preamble: ensures the runtime dir
// exists and writes the nspawn/resource-limits drop-ins.
// The returned StartableAgent token is the only way to call
// start_with_fallback — omitting this becomes a compile error.
let agent_dir = crate::paths::agent_runtime_dir(name);
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(name, agent_dir);
let token = crate::lifecycle::converge_start_preamble(name, &hive, &paths).await?;
ctx.step("nixos-container start");
crate::lifecycle::start_with_fallback(token).await?;
// Bind the MCP listener immediately after starting the container.
// The preamble created the runtime dir; the container is now
// coming up and will connect to this socket on its first turn.
// Event-driven (no background poll) — c0re owns the listener
// lifecycle, so register here rather than waiting for a sweep.
coord.register_agent(name)?;
coord.kick_agent(name, "container started");
coord.rescan_containers_and_emit().await;
}
ReconcileAction::Stop => {
let _guard = claim
.transient
.is_none()
.then(|| coord.transient_guard(name, crate::coordinator::TransientKind::Stopping));
ctx.step("nixos-container stop");
crate::lifecycle::kill(name).await?;
coord.unregister_agent(name);
coord.notify_manager(&hive_sh4re::HelperEvent::Killed {
agent: name.clone(),
});
coord.rescan_containers_and_emit().await;
}
let append_nodes = match reconcile_action(wanted, running) {
ReconcileAction::Start => vec![NodeKind::Start],
ReconcileAction::Stop => vec![NodeKind::Stop],
ReconcileAction::Noop => {
tracing::debug!(%name, wanted = wanted.as_str(), running, "reconcile: noop");
Vec::new()
}
}
};
Ok(NodeOutput {
append_nodes,
..Default::default()
})
}
/// Mechanical container start — the sub-step a `Reconcile` planner fans
/// out when it observes `wanted = Up` and the container down.
async fn run_start(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
let name = &claim.agent;
// Node-local transient only when the DAG holds none (the
// boot-reconcile template); a rebuild/spawn/etc. DAG's lease-window
// transient already covers this node.
let _guard = claim
.transient
.is_none()
.then(|| coord.transient_guard(name, crate::coordinator::TransientKind::Starting));
// Run the typed start preamble: ensures the runtime dir exists and
// writes the nspawn/resource-limits drop-ins. The returned
// StartableAgent token is the only way to call start_with_fallback —
// omitting this becomes a compile error.
let agent_dir = crate::paths::agent_runtime_dir(name);
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(name, agent_dir);
let token = crate::lifecycle::converge_start_preamble(name, &hive, &paths).await?;
ctx.step("nixos-container start");
crate::lifecycle::start_with_fallback(token).await?;
// Bind the MCP listener immediately after starting the container.
// The preamble created the runtime dir; the container is now coming
// up and will connect to this socket on its first turn. Event-driven
// (no background poll) — c0re owns the listener lifecycle, so
// register here rather than waiting for a sweep.
coord.register_agent(name)?;
coord.kick_agent(name, "container started");
coord.rescan_containers_and_emit().await;
Ok(NodeOutput::default())
}
/// Mechanical container stop — the sub-step a `Reconcile` planner fans
/// out when it observes `wanted = Offline` and the container up.
async fn run_stop(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
let name = &claim.agent;
let _guard = claim
.transient
.is_none()
.then(|| coord.transient_guard(name, crate::coordinator::TransientKind::Stopping));
ctx.step("nixos-container stop");
crate::lifecycle::kill(name).await?;
coord.unregister_agent(name);
coord.notify_manager(&hive_sh4re::HelperEvent::Killed {
agent: name.clone(),
});
coord.rescan_containers_and_emit().await;
Ok(NodeOutput::default())
}