From f2ff0deb6bfd56bc0cf36d0563d9abff4f61f6d6 Mon Sep 17 00:00:00 2001 From: damocles Date: Thu, 16 Jul 2026 12:09:15 +0200 Subject: [PATCH] split Swap's Ok-tail into a first-class PostSwap DAG node (#2390) --- hive-c0re/src/job_queue/exec.rs | 75 ++++++++++++++++------------ hive-c0re/src/job_queue/model.rs | 15 +++++- hive-c0re/src/job_queue/templates.rs | 19 ++++--- hive-c0re/src/job_queue/tests.rs | 60 ++++++++++++++++++++-- 4 files changed, 126 insertions(+), 43 deletions(-) diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index 7d1bdc6d..66e5b53c 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -84,6 +84,7 @@ pub(super) async fn run_node(coord: &Arc, claim: &Claim) -> Result< match &claim.kind { NodeKind::Prebuild { relock } => run_prebuild(coord, claim, &ctx, *relock).await, NodeKind::Swap => run_swap(coord, claim, &ctx).await, + NodeKind::PostSwap => run_post_swap(coord, claim, &ctx).await, NodeKind::Provision => run_provision(coord, claim, &ctx).await, NodeKind::Create => run_create(claim, &ctx).await, NodeKind::MetaLock { sweep, fanout } => { @@ -192,41 +193,53 @@ async fn run_swap(coord: &Arc, claim: &Claim, ctx: &Ctx<'_>) -> Res ctx.build_log(log_id); }) .await; - match &result { - Ok(()) => { - if let Some(rev) = crate::auto_update::current_flake_rev(&coord.hyperhive_flake) - && let Err(e) = std::fs::write(crate::paths::applied_rev_marker(name), rev) - { - tracing::warn!(%name, error = ?e, "write rev marker failed"); - } - // The `Rebuilt` manager event fires exactly once per DAG - // from the terminal hook — emitting ok here and letting a - // failed tail `Reconcile` add a contradictory !ok would - // double-report the same rebuild. - ctx.step("forge sync"); - // Full forge + matrix sync on every successful rebuild so - // the rebuild path is equivalent to the startup sweep: - // tokens, config-repo mirror, meta access all recover - // without a hive-c0re restart. - crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await; - crate::matrix::sync_agent_standalone(name).await; - // Wake the agent on its next turn so claude sees a "you - // were rebuilt" hint; rescan so dashboards drop the - // "needs update" chip; lock bump → meta-inputs re-render. - coord.kick_agent(name, "container rebuilt"); - coord.rescan_containers_and_emit().await; - crate::dashboard::emit_meta_inputs_snapshot(coord); - } - Err(_) => { - // The `Rebuilt { ok: false }` manager event fires once per - // DAG from the terminal hook (any node may be the one that - // failed); here only refresh the observed state. - coord.rescan_containers_and_emit().await; - } + // On success the Ok-only bookkeeping tail (rev marker, forge/matrix + // sync, kick, rescan, snapshot) runs in the sibling `PostSwap` node, + // which deps `AfterOk(Swap)`. On failure `PostSwap` is cancel-cascaded + // and the tail `Reconcile` (`AfterAny(PostSwap)`) handles recovery; here + // we only refresh the observed state so dashboards reflect the failed + // swap immediately. The `Rebuilt { ok: false }` manager event fires once + // per DAG from the terminal hook (any node may be the one that failed). + if result.is_err() { + coord.rescan_containers_and_emit().await; } result.map(|()| NodeOutput::default()) } +/// The post-`Swap` bookkeeping tail, split into its own node for dashboard +/// visibility + retry granularity. Deps `AfterOk(Swap)`, so reaching here +/// means the profile swap succeeded. Store/forge/matrix work only — no nix +/// build (build-slot-exempt); the agent lease taken at `Swap` is still held +/// (the whole chain up to `Reconcile` is one agent's subgraph). +async fn run_post_swap( + coord: &Arc, + claim: &Claim, + ctx: &Ctx<'_>, +) -> Result { + let name = &claim.agent; + if let Some(rev) = crate::auto_update::current_flake_rev(&coord.hyperhive_flake) + && let Err(e) = std::fs::write(crate::paths::applied_rev_marker(name), rev) + { + tracing::warn!(%name, error = ?e, "write rev marker failed"); + } + // The `Rebuilt` manager event fires exactly once per DAG from the + // terminal hook — emitting ok here and letting a failed tail `Reconcile` + // add a contradictory !ok would double-report the same rebuild. + ctx.step("forge sync"); + // Full forge + matrix sync on every successful rebuild so the rebuild + // path is equivalent to the startup sweep: tokens, config-repo mirror, + // meta access all recover without a hive-c0re restart. + crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await; + crate::matrix::sync_agent_standalone(name).await; + // Wake the agent on its next turn so claude sees a "you were rebuilt" + // hint; rescan so dashboards drop the "needs update" chip; lock bump → + // meta-inputs re-render. + coord.kick_agent(name, "container rebuilt"); + coord.rescan_containers_and_emit().await; + crate::dashboard::emit_meta_inputs_snapshot(coord); + Ok(NodeOutput::default()) +} + /// First-spawn pre-create provisioning: proposed/applied repos, state /// subvolume, and the meta `sync_agents` registration. Holds the /// deploy-window gate for the commit so it can't land inside another diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs index 7ffe4a04..a20a1ed8 100644 --- a/hive-c0re/src/job_queue/model.rs +++ b/hive-c0re/src/job_queue/model.rs @@ -57,9 +57,19 @@ pub enum NodeKind { Prebuild { relock: bool }, /// `nixos-container update` profile-swap (requires the container /// stopped). Re-applies nspawn flags + resource limits first — - /// rebuild is the reconcile verb — and carries the post-rebuild - /// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan). + /// rebuild is the reconcile verb. The post-rebuild bookkeeping tail + /// lives in the sibling `PostSwap` node. Swap, + /// The post-`Swap` bookkeeping tail as a first-class node: rev marker, + /// forge + matrix sync, manager kick, container rescan, meta-inputs + /// snapshot. Split out of `Swap` for dashboard visibility + retry + /// granularity. Deps `AfterOk(Swap)`, so it runs only when the profile + /// swap succeeded; the tail `Reconcile` deps `AfterAny(PostSwap)`, so on + /// swap failure this node is cancel-cascaded (a terminal state) and + /// recovery still runs. Store/forge/matrix work only — no nix build, so + /// build-slot-exempt; the agent lease taken at `Swap` is held across the + /// whole chain until `Reconcile` settles, so it's not re-declared here. + PostSwap, /// First-spawn pre-create provisioning: proposed/applied repos, /// state subvolume, and meta registration (`sync_agents`). Runs /// ahead of `Create` so the `nixos-container create --flake @@ -137,6 +147,7 @@ impl NodeKind { match self { NodeKind::Prebuild { .. } => "prebuild", NodeKind::Swap => "swap", + NodeKind::PostSwap => "post_swap", NodeKind::Provision => "provision", NodeKind::Create => "create", NodeKind::MetaLock { .. } => "meta_lock", diff --git a/hive-c0re/src/job_queue/templates.rs b/hive-c0re/src/job_queue/templates.rs index ca7069b6..df16a2ce 100644 --- a/hive-c0re/src/job_queue/templates.rs +++ b/hive-c0re/src/job_queue/templates.rs @@ -16,7 +16,7 @@ //! intent+reconcile is atomic per-agent). //! //! ```text -//! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a) +//! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(ok) PostSwap(a) →(any) Reconcile(a) //! spawn(a): Provision(a) → Create(a) → WriteDropin(a) → Reconcile(a) [wanted=Up at approve] //! perm-change(a): WritePermFile(a) → «rebuild subgraph» //! meta-update(inp): MetaLock(inp) →«in-DAG rebuild subgraph per affected a» @@ -50,10 +50,16 @@ pub(crate) fn node(agent: &str, kind: NodeKind, deps: Vec) -> NodeSpec { } } -/// The rebuild node chain. `Reconcile` deps on `Swap` with `AfterAny`: -/// it must run even when the profile swap failed, so a previously-up -/// agent comes back on its old config (today's recovery-start). This -/// is the only `AfterAny` edge in v1. +/// The rebuild node chain. `PostSwap` carries the swap's Ok-only +/// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan) and deps +/// `Swap` with `AfterOk`. `Reconcile` then deps on `PostSwap` with +/// `AfterAny`: it must run even when the swap failed, so a previously-up +/// agent comes back on its old config (today's recovery-start). On swap +/// failure the `AfterOk` `PostSwap` is cancel-cascaded to a terminal state, +/// which still satisfies `Reconcile`'s `AfterAny` edge — the only `AfterAny` +/// edge in v1. Pointing `Reconcile` at `PostSwap` (not `Swap`) also +/// serializes the tail ahead of the reconcile, so there's no double +/// rescan/kick race. pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u32) -> Vec { vec![ node( @@ -67,11 +73,12 @@ pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u32) -> Vec