//! DAG shape builders — every operation as a template over the shared //! node primitives. //! //! Every node carries its own `agent` (there is no DAG-level agent) — the //! [`node`] helper stamps each node's agent and declares the resources that //! node's kind needs. This module holds the *pure* shape builders (no I/O). //! The hive-wide **power ops** (`stop` / `start` / `restart`) are NOT here: //! their per-agent shape depends on each agent's live running state (an async //! `lifecycle::is_running` read), so they are assembled dynamically in //! `submit.rs` out of the shared pure primitives this module exports //! ([`node`], [`rebuild_nodes`]) — one independent per-agent subgraph each, //! concurrent on its own lease, ONE DAG for the whole hive-wide op. //! `stop`/`start` write the durable `wanted` intent via a head `SetWanted(w)` //! node (holding the agent lease, so intent+reconcile is atomic per-agent); //! `restart` writes no intent — it bounces the container and lets the tail //! `Reconcile` converge to the agent's existing `wanted`. //! //! ```text //! rebuild(a): MetaSync(a) → Prebuild(a) → StopForUpdate(a) → Swap(a) →(ok) PostSwap(a) →(any) Reconcile(a) //! rebuild(a) graceful: … Prebuild(a) → Signal(a) → Drain(a) → StopForUpdate(a) → … [boot sweep only] //! spawn(a): Provision(a) → Create(a) → WriteDropin(a) → Reconcile(a) [wanted=Up at approve] //! perm-change(a): WritePermFile(a) → «rebuild subgraph» //! meta-update(inp): MetaLock(inp) →«in-DAG rebuild subgraph per affected a» //! reparent(moves): Reparent(moves) [no rebuild — topology.json is read live] //! ``` //! //! Nodes are **named, not counted**: a template declares a node and holds the //! handle it gets back, so an edge says which node it waits on instead of //! computing where that node landed. There is no submit-time cycle validation //! left to do — `hive_jobq`'s builder inserts in declaration order and rejects //! a reference to a node declared later, so every edge points backwards and a //! cycle is unrepresentable rather than merely rejected. //! //! For the dynamic power-op shapes (`stop` / `start` / `restart`, built from //! live online/offline state), see `submit.rs`. use hive_jobq::TerminalState; use super::model::{DagSpec, NodeKind, PermPayload, Source}; use super::{Handle, Job}; /// Declare one node carrying `kind`, with the resources that kind needs. /// /// The resource declaration is [`NodeKind::resource_deps`] applied at the /// construction site — a build slot for nix-heavy kinds, the agent lease for /// container-affecting ones, the global meta window for meta-mutating ones. A /// node that declares a resource an ancestor already holds re-enters that /// grant rather than taking a fresh unit, so declaring costs nothing. /// /// The returned handle is where edges and grouping are declared, and is `Copy` /// — naming a node as a dependency does not consume the ability to name it /// again. pub(crate) fn node(b: &Job, kind: NodeKind) -> Handle<'_> { // Read the resources off the kind before handing it over — the payload is // moved into the node, not cloned for it. let resources = kind.resource_deps(); let mut handle = b.node(kind); for (name, count) in resources { handle = handle.needs_units(name, count); } handle } /// The `Rebuilt`-reporting tail pair for a rebuild-shaped DAG: the success node /// gated on every group-root in `roots`, and the failure node gated on *its* /// elimination. /// /// Exactly one runs on a DAG that executed, and neither runs on one the operator /// dropped — see [`hive_jobq::NodeRef::on_elimination_of`]. fn emit_rebuilt_tails(b: &Job, agent: &str, roots: &[Handle<'_>]) { let ok = roots.iter().fold( node( b, NodeKind::EmitRebuilt { agent: agent.to_owned(), ok: true, }, ), hive_jobq::NodeRef::after_ok, ); // The failure branch needs *both*: the ok branch being ruled out (that is the // "something went wrong" signal) **and** every root actually finished. The // second half is easy to forget and gets the ordering wrong without it — a // failed `Prebuild` eliminates the ok branch immediately, while the recovery // `Reconcile` is still bringing the container back up, so reporting straight // off the elimination would announce the failure mid-recovery. let _failed = roots.iter().fold( node( b, NodeKind::EmitRebuilt { agent: agent.to_owned(), ok: false, }, ) .on_elimination_of(ok), hive_jobq::NodeRef::after_any, ); } /// The approval-resolving tails for an approval-carrying DAG: one per outcome of /// the DAG's single group-root `root`, each accepting only its own. /// /// The `Cancelled` node is what keeps a dropped approval DAG from dangling its /// row forever — its edge is the only one [`super::JobQueue::cancel`] spares. fn resolve_approval_tails(b: &Job, approval_id: i64, root: Handle<'_>) { for outcome in [ TerminalState::Done, TerminalState::Failed, TerminalState::Cancelled, ] { let _ = node( b, NodeKind::ResolveApproval { approval_id, outcome, }, ) .on_outcome(root, &[outcome]); } } /// Knobs for [`rebuild_nodes`]. A struct rather than two positional `bool`s so /// a call site cannot silently swap them. #[derive(Debug, Clone, Copy)] pub(crate) struct RebuildOpts { /// Re-lock the meta flake inside `MetaSync`. pub relock: bool, /// Give the agent its `Signal` → `Drain` window to finish the turn in /// flight before the container is stopped, instead of stopping it /// outright. Costs up to one `GRACEFUL_STOP_TIMEOUT` per subgraph, and /// those overlap across agents. pub graceful: bool, } /// The group-roots a [`rebuild_nodes`] subgraph exposes to its caller: what a /// tail node edges onto, and what a follow-up node waits for. /// /// Only the roots — a root's state *is* its subtree's roll-up, so these three /// cover every node in the subgraph without the caller knowing its shape. #[derive(Debug, Clone, Copy)] pub(crate) struct RebuildRoots<'a> { /// The meta-repo preamble. pub meta_sync: Handle<'a>, /// The build root — its roll-up carries the whole /// `StopForUpdate` → `Swap` → `PostSwap` subtree. pub prebuild: Handle<'a>, /// The recovery/convergence tail root. pub reconcile: Handle<'a>, } impl<'a> RebuildRoots<'a> { /// The three roots as a slice, for edging a tail onto all of them. fn all(self) -> [Handle<'a>; 3] { [self.meta_sync, self.prebuild, self.reconcile] } } /// The rebuild node subtree (nested, three group roots). `after`, when given, is /// the node this subgraph chains behind. Structure: /// - `MetaSync` (**root**): the meta-repo preamble (dir prep, agent sync, /// optional relock). Owns the global `MetaWindow` — and *only* for its own /// short duration, which is why it is a sibling root rather than `Prebuild`'s /// parent: a resource is held across the holder's whole subtree, so parenting /// the build under it would extend a hive-global window over every rebuild's /// nix build. /// - `Prebuild` (**root**): `AfterOk` `MetaSync`. Owns the build slot for the /// whole mechanical subtree below it. Lease-exempt — the nix build overlaps /// other DAGs on the same agent. /// - the **stop root** (child of `Prebuild`): owns the agent lease and runs once /// `Prebuild` reaches `Finishing` (parent gate). Non-graceful that is /// `StopForUpdate` itself; graceful it is `Signal`, with `Drain` and then /// `StopForUpdate` as its children so the lease stays continuous across the /// whole stop — siblings would each take the lease separately and leave a gap /// another DAG could claim the agent in, mid-bounce. /// - `Swap` (child of `StopForUpdate`): borrows the agent lease from its /// ancestors and the build slot from `Prebuild` — both continuous. /// - `PostSwap` (child of `StopForUpdate`): the swap's Ok-only bookkeeping tail /// (rev marker, forge/matrix sync, kick, rescan), `AfterOk` its sibling /// `Swap`. /// - `Reconcile` (**last, root**): `AfterAny` `Prebuild`, which rolls up /// terminal only once its whole mechanical subtree (SFU→Swap→PostSwap) has /// settled — so `Reconcile` runs after the swap regardless of outcome, and as /// a top-level root it survives the cancel-cascade of a failed `Prebuild` /// (recovery-start invariant, which also covers a failed `MetaSync`: that /// cancel-cascades `Prebuild`, i.e. terminal, so the tail still runs). It /// takes a fresh lease; the tiny gap is harmless — `Reconcile` converges to /// the persisted `wanted` idempotently. pub(crate) fn rebuild_nodes<'a>( b: &'a Job, agent: &str, opts: RebuildOpts, after: Option>, ) -> RebuildRoots<'a> { let a = || agent.to_owned(); let RebuildOpts { relock, graceful } = opts; let mut meta_sync = node(b, NodeKind::MetaSync { agent: a(), relock }); if let Some(after) = after { meta_sync = meta_sync.after_ok(after); } let prebuild = node(b, NodeKind::Prebuild { agent: a() }).after_ok(meta_sync); // The stop root hangs off `Prebuild` and owns the agent lease for // everything below it. `StopForUpdate` parents the swap pair either way. let stop_for_update = if graceful { let signal = node(b, NodeKind::Signal { agent: a() }).part_of(prebuild); // `Drain` is a *child* of `Signal`, so the parent gate already orders // it — a child must not dep on its own parent (dep-scope). let drain = node(b, NodeKind::Drain { agent: a() }).part_of(signal); node(b, NodeKind::StopForUpdate { agent: a() }) .part_of(signal) .after_ok(drain) } else { node(b, NodeKind::StopForUpdate { agent: a() }).part_of(prebuild) }; let swap = node(b, NodeKind::Swap { agent: a() }).part_of(stop_for_update); let _post_swap = node(b, NodeKind::PostSwap { agent: a() }) .part_of(stop_for_update) .after_ok(swap); let reconcile = node(b, NodeKind::Reconcile { agent: a() }).after_any(prebuild); RebuildRoots { meta_sync, prebuild, reconcile, } } /// The rebuild subgraph a [`NodeKind::DeployApply`] grows into its own DAG once /// the merge has landed and `prepare_deploy` has staged the lock, plus the /// [`NodeKind::FinalizeDeploy`] that closes the window behind it. /// /// `relock = false` is the whole reason this composes: `prepare_deploy` already /// relocked and staged `flake.lock`, so the appended `MetaSync` must do the dir /// prep + `sync_agents` *without* re-locking over it. /// /// `FinalizeDeploy` waits on **two** roots, which together reproduce the gate /// the old fused node had around its inline `rebuild_no_meta` call: /// - `AfterOk` `Prebuild` — a parent's state is its roll-up, so this is `Done` /// only once `StopForUpdate` → `Swap` → `PostSwap` all are (a failed *or* /// cancelled child rolls the parent up `Failed`). That's the old /// `build_result`. /// - `AfterOk` `Reconcile` — the old call passed `deferred_start = false` on /// purpose: the container had to come back up *before* the deploy was /// finalized. `Reconcile` alone would not do, being `AfterAny` — it reaches /// `Done` even after a failed `Swap`. /// /// Appended, not submitted: the roots below become children of the emitting /// `DeployApply` (see [`super::JobQueue::append_subgraph`]), which puts them /// inside the `DeployWindow`'s subtree — so the `MetaWindow` this subgraph's /// `MetaSync` and `FinalizeDeploy` declare is re-entered from the ancestor /// already holding it rather than deadlocking against it. pub(crate) fn deploy_rebuild_nodes(agent: &str, approval_id: i64) -> Job { let b = Job::new(); let roots = rebuild_nodes( &b, agent, RebuildOpts { relock: false, graceful: false, }, None, ); let _finalize = node( &b, NodeKind::FinalizeDeploy { agent: agent.to_owned(), approval_id, }, ) .after_ok(roots.prebuild) .after_ok(roots.reconcile); b } /// One uniform rebuild shape — no `was_running` branch. `StopForUpdate` /// noops when already down; the tail `Reconcile` auto-noops the start /// when `wanted = Offline` (a rebuild of a deliberately-stopped agent /// leaves it stopped). `relock = false` only for meta-update cascade /// children. /// /// Closed by an [`NodeKind::EmitRebuilt`] tail edged onto all three group-roots /// (`MetaSync`, `Prebuild`, `Reconcile`) — `Prebuild`'s roll-up carries the /// whole `StopForUpdate`→`Swap`→`PostSwap` subtree, so those three cover every /// node. Edging `Reconcile` alone would not do: it is `AfterAny` `Prebuild`, so /// it reaches `Done` even after a failed swap and the tail would report success. pub fn rebuild(agent: &str, source: Source, reason: String, relock: bool) -> DagSpec { let job = Job::new(); let roots = rebuild_nodes( &job, agent, RebuildOpts { relock, graceful: false, }, None, ); emit_rebuilt_tails(&job, agent, &roots.all()); DagSpec { source, reason, job, } } /// Approval-driven deploy (`MergeConfigPr`) as a phase subtree rather than the /// single opaque node it used to be. Structure: /// - `DeployWindow` (**root**): the resource holder — global meta window, agent /// lease, build slot — held across every child below. No work of its own; it /// reaches `Finishing` immediately and the children run inside it. /// - `MergeVerify` (child): drift-gate + fetch + eval-verify. Mutates nothing, /// so a failure here cancel-cascades its siblings with the forge and the /// applied repo exactly as they were. /// - `DeployApply` (child, `AfterOk` `MergeVerify`): the irreversible half — /// ff-merge + `prepare_deploy`. It doesn't rebuild inline; it grows /// [`deploy_rebuild_nodes`] into this DAG as its own children, so the build /// and the closing `FinalizeDeploy` are real nodes under the same window. /// - `DeployTail` (child, `AfterAny` `DeployApply`): the compensation + /// bookkeeping tail — rollback when a merge landed unfinalized, forge tag /// mirror, PR failure comment (see [`NodeKind::DeployTail`]). /// /// - `ResolveApproval` (**root**, `AfterAny` `DeployWindow`): resolves the /// approval row. A root rather than another child, so it isn't inside the /// window's resource subtree — it runs once the window has released the meta /// window, lease and build slot. One edge suffices here: `DeployWindow` is the /// DAG's only other group-root, so its roll-up already *is* the whole /// pipeline's outcome. /// /// The window still spans the container build, as it must: `prepare_deploy` /// leaves `flake.lock` staged-uncommitted for the build's whole duration. pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec { let a = || agent.to_owned(); let job = Job::new(); let window = node( &job, NodeKind::DeployWindow { agent: a(), approval_id, }, ); let verify = node( &job, NodeKind::MergeVerify { agent: a(), approval_id, }, ) .part_of(window); let apply = node( &job, NodeKind::DeployApply { agent: a(), approval_id, }, ) .part_of(window) .after_ok(verify); let _tail = node( &job, NodeKind::DeployTail { agent: a(), approval_id, }, ) .part_of(window) .after_any(apply); resolve_approval_tails(&job, approval_id, window); DagSpec { source: Source::Approval, reason, job, } } /// A single `Reconcile` node that converges observed power state to the /// persisted intent — `wanted` is untouched (no `SetWanted`), unlike the /// operator `start`/`stop` templates. Test-only helper now (used to build /// single-node lifecycle DAGs that exercise per-agent lease serialization /// in the queue tests); production paths no longer emit a bare reconcile. #[cfg(test)] pub fn reconcile_only(agent: &str, source: Source, reason: String) -> DagSpec { let job = Job::new(); let _reconcile = node( &job, NodeKind::Reconcile { agent: agent.to_owned(), }, ); DagSpec { source, reason, job, } } /// First-deploy spawn (approval-driven): `Provision` (proposed/applied /// repos, state subvolume, meta registration) then `Create` /// (`nixos-container create`), drop-in write, then `Reconcile` starts /// the container (`wanted = Up` written at approve time). All-or-nothing: /// `Provision` (lease-exempt, precedes the container) is the group root; /// `Create` (child) owns the agent lease; `WriteDropin` + `Reconcile` /// (children of `Create`) borrow it. A failure cancel-cascades the rest — /// unlike rebuild there's no recovery-reconcile (nothing to converge if the /// container was never created). Closed by a `ResolveApproval` tail root edged /// `AfterAny` onto `Provision` — the DAG's only other group-root, so its roll-up /// already carries the whole cascade. pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec { let a = || agent.to_owned(); let job = Job::new(); let provision = node(&job, NodeKind::Provision { agent: a() }); let create = node(&job, NodeKind::Create { agent: a() }).part_of(provision); let dropin = node(&job, NodeKind::WriteDropin { agent: a() }).part_of(create); let _reconcile = node(&job, NodeKind::Reconcile { agent: a() }) .part_of(create) .after_ok(dropin); resolve_approval_tails(&job, approval_id, provision); DagSpec { source: Source::Approval, reason, job, } } /// Perm change: commit the JSON file(s), then the rebuild subgraph so /// the updated `HIVE_TOOL_GROUPS` / `HIVE_CAPABILITIES` env var takes /// effect in the container. Group-roots are `WritePermFile` plus the rebuild /// subgraph's `MetaSync` / `Prebuild` / `Reconcile`, so the `EmitRebuilt` tail /// edges all four. pub fn perm_change(agent: &str, source: Source, reason: String, payload: PermPayload) -> DagSpec { let job = Job::new(); let write = node( &job, NodeKind::WritePermFile { agent: agent.to_owned(), payload, }, ); let roots = rebuild_nodes( &job, agent, RebuildOpts { relock: true, graceful: false, }, Some(write), ); emit_rebuilt_tails( &job, agent, &[write, roots.meta_sync, roots.prebuild, roots.reconcile], ); DagSpec { source, reason, job, } } /// Meta-input lock bump. The `MetaLock` executor grows one rebuild subgraph /// per affected agent into *this same* DAG on completion (via /// `append_subgraph`) — appended *after* the bump lands so their prebuilds /// run against the post-bump lock, and a failed bump appends nothing /// (replacing the old fan-out-child-DAGs dance). /// `transient = Rebuilding` because those appended subgraphs are rebuilds: /// it's applied per-agent at claim time (the `MetaLock` head needs no lease, /// so the "hyperhive" pseudo-agent gets no pill), giving each cascade agent /// crash-watch suppression during its `Swap` — the property the old child /// `Rebuild` DAGs carried via their own transient. pub fn meta_update( inputs: Vec, source: Source, reason: String, approval_id: Option, ) -> DagSpec { let job = Job::new(); let lock = node( &job, NodeKind::MetaLock { sweep: false, fanout: None, inputs, }, ); // The bump itself has no side effect, so an operator-driven one ends at the // `MetaLock`; an approval-driven one still has its row to resolve and gets the // per-outcome tails edged onto that single group-root — whose roll-up covers // the rebuild subgraphs `MetaLock` grows into itself. if let Some(approval_id) = approval_id { resolve_approval_tails(&job, approval_id, lock); } DagSpec { source, reason, job, } } /// Topology move(s) as a single-node DAG. `moves` is `(child, new_parent)` /// pairs — len 1 for `set-parent`, len N for `set-parent-bulk`, applied /// uniformly by the one [`NodeKind::Reparent`] node (which holds the global /// meta window for its duration, same precedent as [`NodeKind::WritePermFile`]). /// No rebuild subgraph: `topology.json` is read live by every consumer /// (dashboard tree, ``/`` sentinel routing, permission /// checks), so a parent move needs no container rebuild to take effect. /// No transient pill either — the node is agentless (no lease to hang one /// off of) and near-instant. No tail node: the write is the whole effect. pub fn reparent( moves: Vec<(hive_types::Ident, Option)>, source: Source, reason: String, ) -> DagSpec { let job = Job::new(); let _reparent = node(&job, NodeKind::Reparent { moves }); DagSpec { source, reason, job, } } // The boot is assembled inline in `workers/auto_update.rs::submit_boot_tree` // as ONE `Boot` DAG (a sweep `MetaLock` root that grows rebuild subgraphs // in-DAG, plus a `Reconcile` root per drifted agent) — no anchor node and no // per-agent child DAGs.