diff --git a/docs/coordinator.md b/docs/coordinator.md index 03b64e04..3a578cd5 100644 --- a/docs/coordinator.md +++ b/docs/coordinator.md @@ -34,7 +34,7 @@ Nix-heavy — hold one of the `buildSlots` permits for the node's duration: | Node | Wraps | | ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | | `Prebuild` | meta `sync_agents` + optional per-agent relock + `lifecycle::prebuild_toplevel` — build the toplevel out-of-band while the container keeps serving | -| `Swap` | drop-in rewrite + `nixos-container update` profile-swap (requires the container stopped) + the post-rebuild bookkeeping tail (rev marker, `Rebuilt` event, forge/matrix sync, kick, rescan) | +| `Swap` | drop-in rewrite + `nixos-container update` profile-swap (requires the container stopped); the post-swap bookkeeping tail lives in the sibling `PostSwap` node | | `Create` | first-spawn provisioning + `nixos-container create` (atomic build+create) | | `MetaLock` | meta flake lock bump (`lock_update` / boot-sweep `lock_update_hyperhive`, commit fused — see below); fans out child `Rebuild` DAGs on completion | | `ApprovalDeploy` | the opaque apply-commit / merge-config-PR pipeline (see _Approvals_ below) | @@ -45,6 +45,7 @@ Cheap — no build slot: | --------------- | ------------------------------------------------------------------------------------------------------------------------------------ | | `Reconcile` | idempotent power converge: read `wanted` (below) + observed state; start if `Up` & down (cold-start fallback included), stop if `Offline` & up, else noop | | `StopForUpdate` | mechanical `nixos-container stop` for the profile swap; never touches `wanted`; noop if already stopped | +| `PostSwap` | the swap's Ok-only bookkeeping tail — rev marker, forge/matrix sync, manager kick, rescan, meta-inputs snapshot; `AfterOk(Swap)` so it runs only on a successful swap (the `Rebuilt` manager event still fires once per DAG from the terminal hook, not here) | | `Signal` | set the graceful fence + kick, so the harness runs one stop-checkpoint turn | | `Drain` | await the harness clearing the fence, bounded by the 3-min graceful-stop timeout; resolves ok either way | | `WriteDropin` | `set_nspawn_flags` + `set_resource_limits` + daemon-reload | @@ -96,7 +97,7 @@ sweep. `start` folds the per-agent stale-rev upgrade in (a *down + stale* agent's subgraph is a rebuild-then-start). ```text -rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(after-any) Reconcile(a) +rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(after-ok) PostSwap(a) →(after-any) Reconcile(a) stop(a..): online a: SetWanted(a,Off) → [Signal→Drain→ if graceful] Reconcile(a) offline a: SetWanted(a,Off) → Reconcile(a) (N subgraphs, 1 DAG) restart(a..): online a: SetWanted(a,Up) → [Signal→Drain→ if graceful] StopForUpdate(a) → Reconcile(a) diff --git a/frontend/packages/dashboard/src/builds.js b/frontend/packages/dashboard/src/builds.js index b5b5b756..7c9b3160 100644 --- a/frontend/packages/dashboard/src/builds.js +++ b/frontend/packages/dashboard/src/builds.js @@ -145,6 +145,7 @@ const NODE_KIND_LABEL = { prebuild: 'prebuild', stop_for_update: 'stop', swap: 'swap', + post_swap: 'post swap', provision: 'provision', create: 'create', meta_lock: 'meta lock', diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index 7d1bdc6d..66e5b53c 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -84,6 +84,7 @@ pub(super) async fn run_node(coord: &Arc, claim: &Claim) -> Result< match &claim.kind { NodeKind::Prebuild { relock } => run_prebuild(coord, claim, &ctx, *relock).await, NodeKind::Swap => run_swap(coord, claim, &ctx).await, + NodeKind::PostSwap => run_post_swap(coord, claim, &ctx).await, NodeKind::Provision => run_provision(coord, claim, &ctx).await, NodeKind::Create => run_create(claim, &ctx).await, NodeKind::MetaLock { sweep, fanout } => { @@ -192,41 +193,53 @@ async fn run_swap(coord: &Arc, claim: &Claim, ctx: &Ctx<'_>) -> Res ctx.build_log(log_id); }) .await; - match &result { - Ok(()) => { - if let Some(rev) = crate::auto_update::current_flake_rev(&coord.hyperhive_flake) - && let Err(e) = std::fs::write(crate::paths::applied_rev_marker(name), rev) - { - tracing::warn!(%name, error = ?e, "write rev marker failed"); - } - // The `Rebuilt` manager event fires exactly once per DAG - // from the terminal hook — emitting ok here and letting a - // failed tail `Reconcile` add a contradictory !ok would - // double-report the same rebuild. - ctx.step("forge sync"); - // Full forge + matrix sync on every successful rebuild so - // the rebuild path is equivalent to the startup sweep: - // tokens, config-repo mirror, meta access all recover - // without a hive-c0re restart. - crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await; - crate::matrix::sync_agent_standalone(name).await; - // Wake the agent on its next turn so claude sees a "you - // were rebuilt" hint; rescan so dashboards drop the - // "needs update" chip; lock bump → meta-inputs re-render. - coord.kick_agent(name, "container rebuilt"); - coord.rescan_containers_and_emit().await; - crate::dashboard::emit_meta_inputs_snapshot(coord); - } - Err(_) => { - // The `Rebuilt { ok: false }` manager event fires once per - // DAG from the terminal hook (any node may be the one that - // failed); here only refresh the observed state. - coord.rescan_containers_and_emit().await; - } + // On success the Ok-only bookkeeping tail (rev marker, forge/matrix + // sync, kick, rescan, snapshot) runs in the sibling `PostSwap` node, + // which deps `AfterOk(Swap)`. On failure `PostSwap` is cancel-cascaded + // and the tail `Reconcile` (`AfterAny(PostSwap)`) handles recovery; here + // we only refresh the observed state so dashboards reflect the failed + // swap immediately. The `Rebuilt { ok: false }` manager event fires once + // per DAG from the terminal hook (any node may be the one that failed). + if result.is_err() { + coord.rescan_containers_and_emit().await; } result.map(|()| NodeOutput::default()) } +/// The post-`Swap` bookkeeping tail, split into its own node for dashboard +/// visibility + retry granularity. Deps `AfterOk(Swap)`, so reaching here +/// means the profile swap succeeded. Store/forge/matrix work only — no nix +/// build (build-slot-exempt); the agent lease taken at `Swap` is still held +/// (the whole chain up to `Reconcile` is one agent's subgraph). +async fn run_post_swap( + coord: &Arc, + claim: &Claim, + ctx: &Ctx<'_>, +) -> Result { + let name = &claim.agent; + if let Some(rev) = crate::auto_update::current_flake_rev(&coord.hyperhive_flake) + && let Err(e) = std::fs::write(crate::paths::applied_rev_marker(name), rev) + { + tracing::warn!(%name, error = ?e, "write rev marker failed"); + } + // The `Rebuilt` manager event fires exactly once per DAG from the + // terminal hook — emitting ok here and letting a failed tail `Reconcile` + // add a contradictory !ok would double-report the same rebuild. + ctx.step("forge sync"); + // Full forge + matrix sync on every successful rebuild so the rebuild + // path is equivalent to the startup sweep: tokens, config-repo mirror, + // meta access all recover without a hive-c0re restart. + crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await; + crate::matrix::sync_agent_standalone(name).await; + // Wake the agent on its next turn so claude sees a "you were rebuilt" + // hint; rescan so dashboards drop the "needs update" chip; lock bump → + // meta-inputs re-render. + coord.kick_agent(name, "container rebuilt"); + coord.rescan_containers_and_emit().await; + crate::dashboard::emit_meta_inputs_snapshot(coord); + Ok(NodeOutput::default()) +} + /// First-spawn pre-create provisioning: proposed/applied repos, state /// subvolume, and the meta `sync_agents` registration. Holds the /// deploy-window gate for the commit so it can't land inside another diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs index 7ffe4a04..a20a1ed8 100644 --- a/hive-c0re/src/job_queue/model.rs +++ b/hive-c0re/src/job_queue/model.rs @@ -57,9 +57,19 @@ pub enum NodeKind { Prebuild { relock: bool }, /// `nixos-container update` profile-swap (requires the container /// stopped). Re-applies nspawn flags + resource limits first — - /// rebuild is the reconcile verb — and carries the post-rebuild - /// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan). + /// rebuild is the reconcile verb. The post-rebuild bookkeeping tail + /// lives in the sibling `PostSwap` node. Swap, + /// The post-`Swap` bookkeeping tail as a first-class node: rev marker, + /// forge + matrix sync, manager kick, container rescan, meta-inputs + /// snapshot. Split out of `Swap` for dashboard visibility + retry + /// granularity. Deps `AfterOk(Swap)`, so it runs only when the profile + /// swap succeeded; the tail `Reconcile` deps `AfterAny(PostSwap)`, so on + /// swap failure this node is cancel-cascaded (a terminal state) and + /// recovery still runs. Store/forge/matrix work only — no nix build, so + /// build-slot-exempt; the agent lease taken at `Swap` is held across the + /// whole chain until `Reconcile` settles, so it's not re-declared here. + PostSwap, /// First-spawn pre-create provisioning: proposed/applied repos, /// state subvolume, and meta registration (`sync_agents`). Runs /// ahead of `Create` so the `nixos-container create --flake @@ -137,6 +147,7 @@ impl NodeKind { match self { NodeKind::Prebuild { .. } => "prebuild", NodeKind::Swap => "swap", + NodeKind::PostSwap => "post_swap", NodeKind::Provision => "provision", NodeKind::Create => "create", NodeKind::MetaLock { .. } => "meta_lock", diff --git a/hive-c0re/src/job_queue/templates.rs b/hive-c0re/src/job_queue/templates.rs index ca7069b6..df16a2ce 100644 --- a/hive-c0re/src/job_queue/templates.rs +++ b/hive-c0re/src/job_queue/templates.rs @@ -16,7 +16,7 @@ //! intent+reconcile is atomic per-agent). //! //! ```text -//! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a) +//! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(ok) PostSwap(a) →(any) Reconcile(a) //! spawn(a): Provision(a) → Create(a) → WriteDropin(a) → Reconcile(a) [wanted=Up at approve] //! perm-change(a): WritePermFile(a) → «rebuild subgraph» //! meta-update(inp): MetaLock(inp) →«in-DAG rebuild subgraph per affected a» @@ -50,10 +50,16 @@ pub(crate) fn node(agent: &str, kind: NodeKind, deps: Vec) -> NodeSpec { } } -/// The rebuild node chain. `Reconcile` deps on `Swap` with `AfterAny`: -/// it must run even when the profile swap failed, so a previously-up -/// agent comes back on its old config (today's recovery-start). This -/// is the only `AfterAny` edge in v1. +/// The rebuild node chain. `PostSwap` carries the swap's Ok-only +/// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan) and deps +/// `Swap` with `AfterOk`. `Reconcile` then deps on `PostSwap` with +/// `AfterAny`: it must run even when the swap failed, so a previously-up +/// agent comes back on its old config (today's recovery-start). On swap +/// failure the `AfterOk` `PostSwap` is cancel-cascaded to a terminal state, +/// which still satisfies `Reconcile`'s `AfterAny` edge — the only `AfterAny` +/// edge in v1. Pointing `Reconcile` at `PostSwap` (not `Swap`) also +/// serializes the tail ahead of the reconcile, so there's no double +/// rescan/kick race. pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u32) -> Vec { vec![ node( @@ -67,11 +73,12 @@ pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u32) -> Vec