diff --git a/docs/coordinator.md b/docs/coordinator.md index 3a578cd5..03b64e04 100644 --- a/docs/coordinator.md +++ b/docs/coordinator.md @@ -34,7 +34,7 @@ Nix-heavy — hold one of the `buildSlots` permits for the node's duration: | Node | Wraps | | ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | | `Prebuild` | meta `sync_agents` + optional per-agent relock + `lifecycle::prebuild_toplevel` — build the toplevel out-of-band while the container keeps serving | -| `Swap` | drop-in rewrite + `nixos-container update` profile-swap (requires the container stopped); the post-swap bookkeeping tail lives in the sibling `PostSwap` node | +| `Swap` | drop-in rewrite + `nixos-container update` profile-swap (requires the container stopped) + the post-rebuild bookkeeping tail (rev marker, `Rebuilt` event, forge/matrix sync, kick, rescan) | | `Create` | first-spawn provisioning + `nixos-container create` (atomic build+create) | | `MetaLock` | meta flake lock bump (`lock_update` / boot-sweep `lock_update_hyperhive`, commit fused — see below); fans out child `Rebuild` DAGs on completion | | `ApprovalDeploy` | the opaque apply-commit / merge-config-PR pipeline (see _Approvals_ below) | @@ -45,7 +45,6 @@ Cheap — no build slot: | --------------- | ------------------------------------------------------------------------------------------------------------------------------------ | | `Reconcile` | idempotent power converge: read `wanted` (below) + observed state; start if `Up` & down (cold-start fallback included), stop if `Offline` & up, else noop | | `StopForUpdate` | mechanical `nixos-container stop` for the profile swap; never touches `wanted`; noop if already stopped | -| `PostSwap` | the swap's Ok-only bookkeeping tail — rev marker, forge/matrix sync, manager kick, rescan, meta-inputs snapshot; `AfterOk(Swap)` so it runs only on a successful swap (the `Rebuilt` manager event still fires once per DAG from the terminal hook, not here) | | `Signal` | set the graceful fence + kick, so the harness runs one stop-checkpoint turn | | `Drain` | await the harness clearing the fence, bounded by the 3-min graceful-stop timeout; resolves ok either way | | `WriteDropin` | `set_nspawn_flags` + `set_resource_limits` + daemon-reload | @@ -97,7 +96,7 @@ sweep. `start` folds the per-agent stale-rev upgrade in (a *down + stale* agent's subgraph is a rebuild-then-start). ```text -rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(after-ok) PostSwap(a) →(after-any) Reconcile(a) +rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(after-any) Reconcile(a) stop(a..): online a: SetWanted(a,Off) → [Signal→Drain→ if graceful] Reconcile(a) offline a: SetWanted(a,Off) → Reconcile(a) (N subgraphs, 1 DAG) restart(a..): online a: SetWanted(a,Up) → [Signal→Drain→ if graceful] StopForUpdate(a) → Reconcile(a) diff --git a/frontend/packages/dashboard/src/builds.js b/frontend/packages/dashboard/src/builds.js index 7c9b3160..b5b5b756 100644 --- a/frontend/packages/dashboard/src/builds.js +++ b/frontend/packages/dashboard/src/builds.js @@ -145,7 +145,6 @@ const NODE_KIND_LABEL = { prebuild: 'prebuild', stop_for_update: 'stop', swap: 'swap', - post_swap: 'post swap', provision: 'provision', create: 'create', meta_lock: 'meta lock', diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index 66e5b53c..7d1bdc6d 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -84,7 +84,6 @@ pub(super) async fn run_node(coord: &Arc, claim: &Claim) -> Result< match &claim.kind { NodeKind::Prebuild { relock } => run_prebuild(coord, claim, &ctx, *relock).await, NodeKind::Swap => run_swap(coord, claim, &ctx).await, - NodeKind::PostSwap => run_post_swap(coord, claim, &ctx).await, NodeKind::Provision => run_provision(coord, claim, &ctx).await, NodeKind::Create => run_create(claim, &ctx).await, NodeKind::MetaLock { sweep, fanout } => { @@ -193,53 +192,41 @@ async fn run_swap(coord: &Arc, claim: &Claim, ctx: &Ctx<'_>) -> Res ctx.build_log(log_id); }) .await; - // On success the Ok-only bookkeeping tail (rev marker, forge/matrix - // sync, kick, rescan, snapshot) runs in the sibling `PostSwap` node, - // which deps `AfterOk(Swap)`. On failure `PostSwap` is cancel-cascaded - // and the tail `Reconcile` (`AfterAny(PostSwap)`) handles recovery; here - // we only refresh the observed state so dashboards reflect the failed - // swap immediately. The `Rebuilt { ok: false }` manager event fires once - // per DAG from the terminal hook (any node may be the one that failed). - if result.is_err() { - coord.rescan_containers_and_emit().await; + match &result { + Ok(()) => { + if let Some(rev) = crate::auto_update::current_flake_rev(&coord.hyperhive_flake) + && let Err(e) = std::fs::write(crate::paths::applied_rev_marker(name), rev) + { + tracing::warn!(%name, error = ?e, "write rev marker failed"); + } + // The `Rebuilt` manager event fires exactly once per DAG + // from the terminal hook — emitting ok here and letting a + // failed tail `Reconcile` add a contradictory !ok would + // double-report the same rebuild. + ctx.step("forge sync"); + // Full forge + matrix sync on every successful rebuild so + // the rebuild path is equivalent to the startup sweep: + // tokens, config-repo mirror, meta access all recover + // without a hive-c0re restart. + crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await; + crate::matrix::sync_agent_standalone(name).await; + // Wake the agent on its next turn so claude sees a "you + // were rebuilt" hint; rescan so dashboards drop the + // "needs update" chip; lock bump → meta-inputs re-render. + coord.kick_agent(name, "container rebuilt"); + coord.rescan_containers_and_emit().await; + crate::dashboard::emit_meta_inputs_snapshot(coord); + } + Err(_) => { + // The `Rebuilt { ok: false }` manager event fires once per + // DAG from the terminal hook (any node may be the one that + // failed); here only refresh the observed state. + coord.rescan_containers_and_emit().await; + } } result.map(|()| NodeOutput::default()) } -/// The post-`Swap` bookkeeping tail, split into its own node for dashboard -/// visibility + retry granularity. Deps `AfterOk(Swap)`, so reaching here -/// means the profile swap succeeded. Store/forge/matrix work only — no nix -/// build (build-slot-exempt); the agent lease taken at `Swap` is still held -/// (the whole chain up to `Reconcile` is one agent's subgraph). -async fn run_post_swap( - coord: &Arc, - claim: &Claim, - ctx: &Ctx<'_>, -) -> Result { - let name = &claim.agent; - if let Some(rev) = crate::auto_update::current_flake_rev(&coord.hyperhive_flake) - && let Err(e) = std::fs::write(crate::paths::applied_rev_marker(name), rev) - { - tracing::warn!(%name, error = ?e, "write rev marker failed"); - } - // The `Rebuilt` manager event fires exactly once per DAG from the - // terminal hook — emitting ok here and letting a failed tail `Reconcile` - // add a contradictory !ok would double-report the same rebuild. - ctx.step("forge sync"); - // Full forge + matrix sync on every successful rebuild so the rebuild - // path is equivalent to the startup sweep: tokens, config-repo mirror, - // meta access all recover without a hive-c0re restart. - crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await; - crate::matrix::sync_agent_standalone(name).await; - // Wake the agent on its next turn so claude sees a "you were rebuilt" - // hint; rescan so dashboards drop the "needs update" chip; lock bump → - // meta-inputs re-render. - coord.kick_agent(name, "container rebuilt"); - coord.rescan_containers_and_emit().await; - crate::dashboard::emit_meta_inputs_snapshot(coord); - Ok(NodeOutput::default()) -} - /// First-spawn pre-create provisioning: proposed/applied repos, state /// subvolume, and the meta `sync_agents` registration. Holds the /// deploy-window gate for the commit so it can't land inside another diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs index a20a1ed8..7ffe4a04 100644 --- a/hive-c0re/src/job_queue/model.rs +++ b/hive-c0re/src/job_queue/model.rs @@ -57,19 +57,9 @@ pub enum NodeKind { Prebuild { relock: bool }, /// `nixos-container update` profile-swap (requires the container /// stopped). Re-applies nspawn flags + resource limits first — - /// rebuild is the reconcile verb. The post-rebuild bookkeeping tail - /// lives in the sibling `PostSwap` node. + /// rebuild is the reconcile verb — and carries the post-rebuild + /// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan). Swap, - /// The post-`Swap` bookkeeping tail as a first-class node: rev marker, - /// forge + matrix sync, manager kick, container rescan, meta-inputs - /// snapshot. Split out of `Swap` for dashboard visibility + retry - /// granularity. Deps `AfterOk(Swap)`, so it runs only when the profile - /// swap succeeded; the tail `Reconcile` deps `AfterAny(PostSwap)`, so on - /// swap failure this node is cancel-cascaded (a terminal state) and - /// recovery still runs. Store/forge/matrix work only — no nix build, so - /// build-slot-exempt; the agent lease taken at `Swap` is held across the - /// whole chain until `Reconcile` settles, so it's not re-declared here. - PostSwap, /// First-spawn pre-create provisioning: proposed/applied repos, /// state subvolume, and meta registration (`sync_agents`). Runs /// ahead of `Create` so the `nixos-container create --flake @@ -147,7 +137,6 @@ impl NodeKind { match self { NodeKind::Prebuild { .. } => "prebuild", NodeKind::Swap => "swap", - NodeKind::PostSwap => "post_swap", NodeKind::Provision => "provision", NodeKind::Create => "create", NodeKind::MetaLock { .. } => "meta_lock", diff --git a/hive-c0re/src/job_queue/templates.rs b/hive-c0re/src/job_queue/templates.rs index df16a2ce..ca7069b6 100644 --- a/hive-c0re/src/job_queue/templates.rs +++ b/hive-c0re/src/job_queue/templates.rs @@ -16,7 +16,7 @@ //! intent+reconcile is atomic per-agent). //! //! ```text -//! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(ok) PostSwap(a) →(any) Reconcile(a) +//! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a) //! spawn(a): Provision(a) → Create(a) → WriteDropin(a) → Reconcile(a) [wanted=Up at approve] //! perm-change(a): WritePermFile(a) → «rebuild subgraph» //! meta-update(inp): MetaLock(inp) →«in-DAG rebuild subgraph per affected a» @@ -50,16 +50,10 @@ pub(crate) fn node(agent: &str, kind: NodeKind, deps: Vec) -> NodeSpec { } } -/// The rebuild node chain. `PostSwap` carries the swap's Ok-only -/// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan) and deps -/// `Swap` with `AfterOk`. `Reconcile` then deps on `PostSwap` with -/// `AfterAny`: it must run even when the swap failed, so a previously-up -/// agent comes back on its old config (today's recovery-start). On swap -/// failure the `AfterOk` `PostSwap` is cancel-cascaded to a terminal state, -/// which still satisfies `Reconcile`'s `AfterAny` edge — the only `AfterAny` -/// edge in v1. Pointing `Reconcile` at `PostSwap` (not `Swap`) also -/// serializes the tail ahead of the reconcile, so there's no double -/// rescan/kick race. +/// The rebuild node chain. `Reconcile` deps on `Swap` with `AfterAny`: +/// it must run even when the profile swap failed, so a previously-up +/// agent comes back on its old config (today's recovery-start). This +/// is the only `AfterAny` edge in v1. pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u32) -> Vec { vec![ node( @@ -73,12 +67,11 @@ pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u32) -> Vec