vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(hive-c0re): close review findings on the job-DAG queue

Browse source 
- deploy-window gate (meta::exclusive) + path-limited meta commits:
  a perm/lock/topology commit can no longer sweep an ApprovalDeploy's
  staged flake.lock and neuter abort_deploy (regression test included)
- cancel surfaces now buffer terminal roll-ups the scheduler drains,
  so a queued approval DAG cancelled by the operator resolves its
  approval instead of dangling, and cancelled power ops revert their
  wanted flip to the observed state
- hivectl restart / restart-all ride the queue (lease serialization,
  transient guard) and restart sets wanted=Up like the old kill+start
- exactly one Rebuilt event per rebuild DAG, emitted at terminal
- StopForUpdate pre-seeds a missing agent_power row from the pre-stop
  observation so a rebuild can't strand an unknown agent offline
- history trim keeps terminal fan-out parents with live children
- audit_log back on db::open; swarm.js badge for reconcile DAGs
This commit is contained in:
müde 2026-07-06 21:44:43 +02:00
commit 084e12503c
12 changed files with 448 additions and 160 deletions
Download patch file Download diff file Expand all files Collapse all files

118
hive-c0re/src/job_queue/exec.rs
View file

@ -106,11 +106,17 @@ async fn run_prebuild(
// Idempotent meta sync so a manual rebuild can also recover from a
// divergent meta repo; then bump just this agent's input. `relock =
// false` only for meta-update cascade children, where re-locking
// would revert the bump the cascade just committed.
let agents = crate::lifecycle::agents_for_meta_listing().await?;
crate::meta::sync_agents(&hive, &agents).await?;
if relock {
crate::meta::lock_update_for_rebuild(name).await?;
// would revert the bump the cascade just committed. Both run under
// the deploy-window gate so they can never land inside another
// node's staged prepare→finalize window; the gate drops before the
// (long) toplevel build, which only reads the store.
{
let _window = crate::meta::exclusive().await;
let agents = crate::lifecycle::agents_for_meta_listing().await?;
crate::meta::sync_agents(&hive, &agents).await?;
if relock {
crate::meta::lock_update_for_rebuild(name).await?;
}
}
ctx.step("nix build");
let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display());
@ -140,13 +146,10 @@ async fn run_swap(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Res
{
tracing::warn!(%name, error = ?e, "write rev marker failed");
}
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
agent: name.clone(),
ok: true,
note: None,
sha: None,
tag: None,
});
// The `Rebuilt` manager event fires exactly once per DAG
// from the terminal hook — emitting ok here and letting a
// failed tail `Reconcile` add a contradictory !ok would
// double-report the same rebuild.
ctx.step("forge sync");
// Full forge + matrix sync on every successful rebuild so
// the rebuild path is equivalent to the startup sweep:
@ -179,6 +182,11 @@ async fn run_create(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> R
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(name, agent_dir);
ctx.step("nixos-container create");
// create_container registers the new agent in the meta flake
// (sync_agents commit) before `nixos-container create` — hold the
// deploy-window gate so that commit can't land inside another
// node's staged deploy window.
let _window = crate::meta::exclusive().await;
crate::lifecycle::create_container(name, &hive, &paths).await?;
Ok(NodeOutput::default())
}
@ -196,6 +204,7 @@ async fn run_meta_lock(
) -> Result<NodeOutput> {
if sweep {
ctx.step("nix flake update hyperhive");
let _window = crate::meta::exclusive().await;
if let Err(e) = crate::meta::lock_update_hyperhive().await {
tracing::warn!(error = ?e, "startup sweep: meta lock_update_hyperhive failed");
}
@ -205,7 +214,10 @@ async fn run_meta_lock(
}
let _progress = coord.meta_update_guard();
ctx.step("nix flake update");
crate::meta::lock_update(&claim.inputs).await?;
{
let _window = crate::meta::exclusive().await;
crate::meta::lock_update(&claim.inputs).await?;
}
// Lock file changed — meta-inputs panel re-renders.
crate::dashboard::emit_meta_inputs_snapshot(coord);
let cascade = match fanout {
@ -258,8 +270,8 @@ async fn run_reconcile(
Ok(NodeOutput::default())
}
/// Mechanical stop for the profile swap. Never touches `wanted`; noop
/// when already stopped.
/// Mechanical stop for the profile swap. Never *changes* `wanted`;
/// noop when already stopped.
async fn run_stop_for_update(
coord: &Arc<Coordinator>,
claim: &Claim,
@ -267,6 +279,13 @@ async fn run_stop_for_update(
) -> Result<NodeOutput> {
let name = &claim.agent;
if crate::lifecycle::is_running(name).await {
// Seed a missing agent_power row from the PRE-stop observation
// — the DAG's tail `Reconcile` observes only the mechanically
// stopped state and would otherwise seed a running-but-unknown
// agent as `Offline`, stranding it down after its own rebuild.
if let Err(e) = coord.power.get_or_seed(name, true) {
tracing::warn!(%name, error = ?e, "agent_power: pre-stop seed failed");
}
ctx.step("nixos-container stop");
crate::lifecycle::kill(name).await?;
coord.rescan_containers_and_emit().await;
@ -322,6 +341,11 @@ async fn run_write_perm_file(
use super::model::PermPayload;
let name = &claim.agent;
ctx.step("writing + committing perm file");
// Deploy-window gate: a perm commit landing inside another node's
// staged prepare→finalize window would sweep the staged deploy
// lock into its commit (the commits are also path-limited in
// meta.rs — belt and braces).
let _window = crate::meta::exclusive().await;
match &claim.perm_payload {
Some(PermPayload::ToolGroups { groups }) => {
crate::meta::commit_tool_groups(name, groups)
@ -363,6 +387,11 @@ async fn run_approval_deploy(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
let approval_id = claim
.approval_id
.with_context(|| format!("approval_deploy dag {} has no approval_id", claim.dag_id))?;
// Hold the deploy-window gate for the whole prepare→finalize span:
// `prepare_deploy` stages `flake.lock` uncommitted for the entire
// container build, and no other meta mutation may land inside that
// window (it would sweep the staged lock and neuter `abort_deploy`).
let _window = crate::meta::exclusive().await;
let kind = coord
.approvals
.get(approval_id)
@ -377,26 +406,55 @@ async fn run_approval_deploy(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
result.map(|()| NodeOutput::default())
}
/// Terminal-roll-up hook, fired exactly once per DAG. Approval DAGs
/// resolve their approval row (except the opaque deploy pipeline,
/// which resolves inside its node); non-approval rebuild-shaped DAGs
/// surface the `Rebuilt { ok: false }` manager event on failure —
/// success fires from the `Swap` tail, matching today's timing.
/// Terminal-roll-up hook, fired exactly once per DAG (node completion
/// and cancel paths alike — the queue buffers roll-ups and the
/// scheduler drains them). Three concerns:
/// - approval DAGs resolve their approval row (except the opaque
/// deploy pipeline, which resolves inside its node — unless it was
/// cancelled while still queued and the node never ran);
/// - non-approval rebuild-shaped DAGs emit exactly one `Rebuilt`
/// manager event: ok on `Done`, !ok on `Failed`, none on cancel;
/// - a cancelled power-op DAG reverts the `wanted` intent its submit
/// wrote: the operator's cancel means "don't do it", so intent
/// snaps back to the observed state instead of the flip executing
/// as a surprise side effect of some later reconcile.
pub(super) async fn on_dag_terminal(coord: &Arc<Coordinator>, terminal: &TerminalDag) {
if terminal.state == State::Cancelled
&& matches!(
terminal.template,
Template::Start | Template::Stop | Template::GracefulStop | Template::Restart
)
{
let running = crate::lifecycle::is_running(&terminal.agent).await;
if let Err(e) = coord
.power
.set(&terminal.agent, crate::power::Wanted::from_running(running))
{
tracing::warn!(agent = %terminal.agent, error = ?e, "agent_power: cancel revert failed");
}
}
if terminal.approval_id.is_some() {
crate::actions::resolve_approval_dag(coord, terminal).await;
return;
}
if matches!(terminal.template, Template::Rebuild | Template::PermChange)
&& terminal.state == State::Failed
{
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
agent: terminal.agent.clone(),
ok: false,
note: terminal.error.clone(),
sha: None,
tag: None,
});
if matches!(terminal.template, Template::Rebuild | Template::PermChange) {
match terminal.state {
State::Done => coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
agent: terminal.agent.clone(),
ok: true,
note: None,
sha: None,
tag: None,
}),
State::Failed => coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
agent: terminal.agent.clone(),
ok: false,
note: terminal.error.clone(),
sha: None,
tag: None,
}),
_ => {}
}
}
}