hyperhive/hive-c0re/src/job_queue/templates.rs

//! DAG shape builders — every operation as a template over the shared
//! node primitives.
//!
//! Every node carries its own `agent` (there is no DAG-level agent) — the
//! [`node`] helper stamps each node's agent and declares the resources that
//! node's kind needs. This module holds the *pure* shape builders (no I/O).
//! The hive-wide **power ops** (`stop` / `start` / `restart`) are NOT here:
//! their per-agent shape depends on each agent's live running state (an async
//! `lifecycle::is_running` read), so they are assembled dynamically in
//! `submit.rs` out of the shared pure primitives this module exports
//! ([`node`], [`rebuild_nodes`]) — one independent per-agent subgraph each,
//! concurrent on its own lease, ONE DAG for the whole hive-wide op.
//! `stop`/`start` write the durable `wanted` intent via a head `SetWanted(w)`
//! node (holding the agent lease, so intent+reconcile is atomic per-agent);
//! `restart` writes no intent — it bounces the container and lets the tail
//! `Reconcile` converge to the agent's existing `wanted`.
//!
//! ```text
//! rebuild(a):        MetaSync(a) → Prebuild(a) → StopForUpdate(a) → Swap(a) →(ok) PostSwap(a) →(any) Reconcile(a)
//! rebuild(a) graceful: … Prebuild(a) → Signal(a) → Drain(a) → StopForUpdate(a) → …   [boot sweep only]
//! spawn(a):          Provision(a) → Create(a) → WriteDropin(a) → Reconcile(a)  [wanted=Up at approve]
//! perm-change(a):    WritePermFile(a) → «rebuild subgraph»
//! meta-update(inp):  MetaLock(inp) →«in-DAG rebuild subgraph per affected a»
//! reparent(moves):   Reparent(moves)  [no rebuild — topology.json is read live]
//! ```
//!
//! Nodes are **named, not counted**: a template declares a node and holds the
//! handle it gets back, so an edge says which node it waits on instead of
//! computing where that node landed. There is no submit-time cycle validation
//! left to do — `hive_jobq`'s builder inserts in declaration order and rejects
//! a reference to a node declared later, so every edge points backwards and a
//! cycle is unrepresentable rather than merely rejected.
//!
//! For the dynamic power-op shapes (`stop` / `start` / `restart`, built from
//! live online/offline state), see `submit.rs`.

use hive_jobq::TerminalState;

use super::model::{DagSpec, NodeKind, PermPayload, Source};
use super::{Handle, Job};

/// Declare one node carrying `kind`, with the resources that kind needs.
///
/// The resource declaration is [`NodeKind::resource_deps`] applied at the
/// construction site — a build slot for nix-heavy kinds, the agent lease for
/// container-affecting ones, the global meta window for meta-mutating ones. A
/// node that declares a resource an ancestor already holds re-enters that
/// grant rather than taking a fresh unit, so declaring costs nothing.
///
/// The returned handle is where edges and grouping are declared, and is `Copy`
/// — naming a node as a dependency does not consume the ability to name it
/// again.
pub(crate) fn node(b: &Job, kind: NodeKind) -> Handle<'_> {
    // Read the resources off the kind before handing it over — the payload is
    // moved into the node, not cloned for it.
    let resources = kind.resource_deps();
    let mut handle = b.node(kind);
    for (name, count) in resources {
        handle = handle.needs_units(name, count);
    }
    handle
}

/// The `Rebuilt`-reporting tail pair for a rebuild-shaped DAG: the success node
/// gated on every group-root in `roots`, and the failure node gated on *its*
/// elimination.
///
/// Exactly one runs on a DAG that executed, and neither runs on one the operator
/// dropped — see [`hive_jobq::NodeRef::on_elimination_of`].
fn emit_rebuilt_tails(b: &Job, agent: &str, roots: &[Handle<'_>]) {
    let ok = roots.iter().fold(
        node(
            b,
            NodeKind::EmitRebuilt {
                agent: agent.to_owned(),
                ok: true,
            },
        ),
        hive_jobq::NodeRef::after_ok,
    );
    // The failure branch needs *both*: the ok branch being ruled out (that is the
    // "something went wrong" signal) **and** every root actually finished. The
    // second half is easy to forget and gets the ordering wrong without it — a
    // failed `Prebuild` eliminates the ok branch immediately, while the recovery
    // `Reconcile` is still bringing the container back up, so reporting straight
    // off the elimination would announce the failure mid-recovery.
    let _failed = roots.iter().fold(
        node(
            b,
            NodeKind::EmitRebuilt {
                agent: agent.to_owned(),
                ok: false,
            },
        )
        .on_elimination_of(ok),
        hive_jobq::NodeRef::after_any,
    );
}

/// The approval-resolving tails for an approval-carrying DAG: one per outcome of
/// the DAG's single group-root `root`, each accepting only its own.
///
/// The `Cancelled` node is what keeps a dropped approval DAG from dangling its
/// row forever — its edge is the only one [`super::JobQueue::cancel`] spares.
fn resolve_approval_tails(b: &Job, approval_id: i64, root: Handle<'_>) {
    for outcome in [
        TerminalState::Done,
        TerminalState::Failed,
        TerminalState::Cancelled,
    ] {
        let _ = node(
            b,
            NodeKind::ResolveApproval {
                approval_id,
                outcome,
            },
        )
        .on_outcome(root, &[outcome]);
    }
}

/// Knobs for [`rebuild_nodes`]. A struct rather than two positional `bool`s so
/// a call site cannot silently swap them.
#[derive(Debug, Clone, Copy)]
pub(crate) struct RebuildOpts {
    /// Re-lock the meta flake inside `MetaSync`.
    pub relock: bool,
    /// Give the agent its `Signal` → `Drain` window to finish the turn in
    /// flight before the container is stopped, instead of stopping it
    /// outright. Costs up to one `GRACEFUL_STOP_TIMEOUT` per subgraph, and
    /// those overlap across agents.
    pub graceful: bool,
}

/// The group-roots a [`rebuild_nodes`] subgraph exposes to its caller: what a
/// tail node edges onto, and what a follow-up node waits for.
///
/// Only the roots — a root's state *is* its subtree's roll-up, so these three
/// cover every node in the subgraph without the caller knowing its shape.
#[derive(Debug, Clone, Copy)]
pub(crate) struct RebuildRoots<'a> {
    /// The meta-repo preamble.
    pub meta_sync: Handle<'a>,
    /// The build root — its roll-up carries the whole
    /// `StopForUpdate` → `Swap` → `PostSwap` subtree.
    pub prebuild: Handle<'a>,
    /// The recovery/convergence tail root.
    pub reconcile: Handle<'a>,
}

impl<'a> RebuildRoots<'a> {
    /// The three roots as a slice, for edging a tail onto all of them.
    fn all(self) -> [Handle<'a>; 3] {
        [self.meta_sync, self.prebuild, self.reconcile]
    }
}

/// The rebuild node subtree (nested, three group roots). `after`, when given, is
/// the node this subgraph chains behind. Structure:
/// - `MetaSync` (**root**): the meta-repo preamble (dir prep, agent sync,
///   optional relock). Owns the global `MetaWindow` — and *only* for its own
///   short duration, which is why it is a sibling root rather than `Prebuild`'s
///   parent: a resource is held across the holder's whole subtree, so parenting
///   the build under it would extend a hive-global window over every rebuild's
///   nix build.
/// - `Prebuild` (**root**): `AfterOk` `MetaSync`. Owns the build slot for the
///   whole mechanical subtree below it. Lease-exempt — the nix build overlaps
///   other DAGs on the same agent.
/// - the **stop root** (child of `Prebuild`): owns the agent lease and runs once
///   `Prebuild` reaches `Finishing` (parent gate). Non-graceful that is
///   `StopForUpdate` itself; graceful it is `Signal`, with `Drain` and then
///   `StopForUpdate` as its children so the lease stays continuous across the
///   whole stop — siblings would each take the lease separately and leave a gap
///   another DAG could claim the agent in, mid-bounce.
/// - `Swap` (child of `StopForUpdate`): borrows the agent lease from its
///   ancestors and the build slot from `Prebuild` — both continuous.
/// - `PostSwap` (child of `StopForUpdate`): the swap's Ok-only bookkeeping tail
///   (rev marker, forge/matrix sync, kick, rescan), `AfterOk` its sibling
///   `Swap`.
/// - `Reconcile` (**last, root**): `AfterAny` `Prebuild`, which rolls up
///   terminal only once its whole mechanical subtree (SFU→Swap→PostSwap) has
///   settled — so `Reconcile` runs after the swap regardless of outcome, and as
///   a top-level root it survives the cancel-cascade of a failed `Prebuild`
///   (recovery-start invariant, which also covers a failed `MetaSync`: that
///   cancel-cascades `Prebuild`, i.e. terminal, so the tail still runs). It
///   takes a fresh lease; the tiny gap is harmless — `Reconcile` converges to
///   the persisted `wanted` idempotently.
pub(crate) fn rebuild_nodes<'a>(
    b: &'a Job,
    agent: &str,
    opts: RebuildOpts,
    after: Option<Handle<'a>>,
) -> RebuildRoots<'a> {
    let a = || agent.to_owned();
    let RebuildOpts { relock, graceful } = opts;

    let mut meta_sync = node(b, NodeKind::MetaSync { agent: a(), relock });
    if let Some(after) = after {
        meta_sync = meta_sync.after_ok(after);
    }
    let prebuild = node(b, NodeKind::Prebuild { agent: a() }).after_ok(meta_sync);

    // The stop root hangs off `Prebuild` and owns the agent lease for
    // everything below it. `StopForUpdate` parents the swap pair either way.
    let stop_for_update = if graceful {
        let signal = node(b, NodeKind::Signal { agent: a() }).part_of(prebuild);
        // `Drain` is a *child* of `Signal`, so the parent gate already orders
        // it — a child must not dep on its own parent (dep-scope).
        let drain = node(b, NodeKind::Drain { agent: a() }).part_of(signal);
        node(b, NodeKind::StopForUpdate { agent: a() })
            .part_of(signal)
            .after_ok(drain)
    } else {
        node(b, NodeKind::StopForUpdate { agent: a() }).part_of(prebuild)
    };

    let swap = node(b, NodeKind::Swap { agent: a() }).part_of(stop_for_update);
    let _post_swap = node(b, NodeKind::PostSwap { agent: a() })
        .part_of(stop_for_update)
        .after_ok(swap);

    let reconcile = node(b, NodeKind::Reconcile { agent: a() }).after_any(prebuild);

    RebuildRoots {
        meta_sync,
        prebuild,
        reconcile,
    }
}

/// The rebuild subgraph a [`NodeKind::DeployApply`] grows into its own DAG once
/// the merge has landed and `prepare_deploy` has staged the lock, plus the
/// [`NodeKind::FinalizeDeploy`] that closes the window behind it.
///
/// `relock = false` is the whole reason this composes: `prepare_deploy` already
/// relocked and staged `flake.lock`, so the appended `MetaSync` must do the dir
/// prep + `sync_agents` *without* re-locking over it.
///
/// `FinalizeDeploy` waits on **two** roots, which together reproduce the gate
/// the old fused node had around its inline `rebuild_no_meta` call:
/// - `AfterOk` `Prebuild` — a parent's state is its roll-up, so this is `Done`
///   only once `StopForUpdate` → `Swap` → `PostSwap` all are (a failed *or*
///   cancelled child rolls the parent up `Failed`). That's the old
///   `build_result`.
/// - `AfterOk` `Reconcile` — the old call passed `deferred_start = false` on
///   purpose: the container had to come back up *before* the deploy was
///   finalized. `Reconcile` alone would not do, being `AfterAny` — it reaches
///   `Done` even after a failed `Swap`.
///
/// Appended, not submitted: the roots below become children of the emitting
/// `DeployApply` (see [`super::JobQueue::append_subgraph`]), which puts them
/// inside the `DeployWindow`'s subtree — so the `MetaWindow` this subgraph's
/// `MetaSync` and `FinalizeDeploy` declare is re-entered from the ancestor
/// already holding it rather than deadlocking against it.
pub(crate) fn deploy_rebuild_nodes(agent: &str, approval_id: i64) -> Job {
    let b = Job::new();
    let roots = rebuild_nodes(
        &b,
        agent,
        RebuildOpts {
            relock: false,
            graceful: false,
        },
        None,
    );
    let _finalize = node(
        &b,
        NodeKind::FinalizeDeploy {
            agent: agent.to_owned(),
            approval_id,
        },
    )
    .after_ok(roots.prebuild)
    .after_ok(roots.reconcile);
    b
}

/// One uniform rebuild shape — no `was_running` branch. `StopForUpdate`
/// noops when already down; the tail `Reconcile` auto-noops the start
/// when `wanted = Offline` (a rebuild of a deliberately-stopped agent
/// leaves it stopped). `relock = false` only for meta-update cascade
/// children.
///
/// Closed by an [`NodeKind::EmitRebuilt`] tail edged onto all three group-roots
/// (`MetaSync`, `Prebuild`, `Reconcile`) — `Prebuild`'s roll-up carries the
/// whole `StopForUpdate`→`Swap`→`PostSwap` subtree, so those three cover every
/// node. Edging `Reconcile` alone would not do: it is `AfterAny` `Prebuild`, so
/// it reaches `Done` even after a failed swap and the tail would report success.
pub fn rebuild(agent: &str, source: Source, reason: String, relock: bool) -> DagSpec {
    let job = Job::new();
    let roots = rebuild_nodes(
        &job,
        agent,
        RebuildOpts {
            relock,
            graceful: false,
        },
        None,
    );
    emit_rebuilt_tails(&job, agent, &roots.all());
    DagSpec {
        source,
        reason,
        job,
    }
}

/// Approval-driven deploy (`MergeConfigPr`) as a phase subtree rather than the
/// single opaque node it used to be. Structure:
/// - `DeployWindow` (**root**): the resource holder — global meta window, agent
///   lease, build slot — held across every child below. No work of its own; it
///   reaches `Finishing` immediately and the children run inside it.
/// - `MergeVerify` (child): drift-gate + fetch + eval-verify. Mutates nothing,
///   so a failure here cancel-cascades its siblings with the forge and the
///   applied repo exactly as they were.
/// - `DeployApply` (child, `AfterOk` `MergeVerify`): the irreversible half —
///   ff-merge + `prepare_deploy`. It doesn't rebuild inline; it grows
///   [`deploy_rebuild_nodes`] into this DAG as its own children, so the build
///   and the closing `FinalizeDeploy` are real nodes under the same window.
/// - `DeployTail` (child, `AfterAny` `DeployApply`): the compensation +
///   bookkeeping tail — rollback when a merge landed unfinalized, forge tag
///   mirror, PR failure comment (see [`NodeKind::DeployTail`]).
///
/// - `ResolveApproval` (**root**, `AfterAny` `DeployWindow`): resolves the
///   approval row. A root rather than another child, so it isn't inside the
///   window's resource subtree — it runs once the window has released the meta
///   window, lease and build slot. One edge suffices here: `DeployWindow` is the
///   DAG's only other group-root, so its roll-up already *is* the whole
///   pipeline's outcome.
///
/// The window still spans the container build, as it must: `prepare_deploy`
/// leaves `flake.lock` staged-uncommitted for the build's whole duration.
pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec {
    let a = || agent.to_owned();
    let job = Job::new();

    let window = node(
        &job,
        NodeKind::DeployWindow {
            agent: a(),
            approval_id,
        },
    );
    let verify = node(
        &job,
        NodeKind::MergeVerify {
            agent: a(),
            approval_id,
        },
    )
    .part_of(window);
    let apply = node(
        &job,
        NodeKind::DeployApply {
            agent: a(),
            approval_id,
        },
    )
    .part_of(window)
    .after_ok(verify);
    let _tail = node(
        &job,
        NodeKind::DeployTail {
            agent: a(),
            approval_id,
        },
    )
    .part_of(window)
    .after_any(apply);

    resolve_approval_tails(&job, approval_id, window);
    DagSpec {
        source: Source::Approval,
        reason,
        job,
    }
}

/// A single `Reconcile` node that converges observed power state to the
/// persisted intent — `wanted` is untouched (no `SetWanted`), unlike the
/// operator `start`/`stop` templates. Test-only helper now (used to build
/// single-node lifecycle DAGs that exercise per-agent lease serialization
/// in the queue tests); production paths no longer emit a bare reconcile.
#[cfg(test)]
pub fn reconcile_only(agent: &str, source: Source, reason: String) -> DagSpec {
    let job = Job::new();
    let _reconcile = node(
        &job,
        NodeKind::Reconcile {
            agent: agent.to_owned(),
        },
    );
    DagSpec {
        source,
        reason,
        job,
    }
}

/// First-deploy spawn (approval-driven): `Provision` (proposed/applied
/// repos, state subvolume, meta registration) then `Create`
/// (`nixos-container create`), drop-in write, then `Reconcile` starts
/// the container (`wanted = Up` written at approve time). All-or-nothing:
/// `Provision` (lease-exempt, precedes the container) is the group root;
/// `Create` (child) owns the agent lease; `WriteDropin` + `Reconcile`
/// (children of `Create`) borrow it. A failure cancel-cascades the rest —
/// unlike rebuild there's no recovery-reconcile (nothing to converge if the
/// container was never created). Closed by a `ResolveApproval` tail root edged
/// `AfterAny` onto `Provision` — the DAG's only other group-root, so its roll-up
/// already carries the whole cascade.
pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
    let a = || agent.to_owned();
    let job = Job::new();

    let provision = node(&job, NodeKind::Provision { agent: a() });
    let create = node(&job, NodeKind::Create { agent: a() }).part_of(provision);
    let dropin = node(&job, NodeKind::WriteDropin { agent: a() }).part_of(create);
    let _reconcile = node(&job, NodeKind::Reconcile { agent: a() })
        .part_of(create)
        .after_ok(dropin);

    resolve_approval_tails(&job, approval_id, provision);
    DagSpec {
        source: Source::Approval,
        reason,
        job,
    }
}

/// Perm change: commit the JSON file(s), then the rebuild subgraph so
/// the updated `HIVE_TOOL_GROUPS` / `HIVE_CAPABILITIES` env var takes
/// effect in the container. Group-roots are `WritePermFile` plus the rebuild
/// subgraph's `MetaSync` / `Prebuild` / `Reconcile`, so the `EmitRebuilt` tail
/// edges all four.
pub fn perm_change(agent: &str, source: Source, reason: String, payload: PermPayload) -> DagSpec {
    let job = Job::new();
    let write = node(
        &job,
        NodeKind::WritePermFile {
            agent: agent.to_owned(),
            payload,
        },
    );
    let roots = rebuild_nodes(
        &job,
        agent,
        RebuildOpts {
            relock: true,
            graceful: false,
        },
        Some(write),
    );
    emit_rebuilt_tails(
        &job,
        agent,
        &[write, roots.meta_sync, roots.prebuild, roots.reconcile],
    );
    DagSpec {
        source,
        reason,
        job,
    }
}

/// Meta-input lock bump. The `MetaLock` executor grows one rebuild subgraph
/// per affected agent into *this same* DAG on completion (via
/// `append_subgraph`) — appended *after* the bump lands so their prebuilds
/// run against the post-bump lock, and a failed bump appends nothing
/// (replacing the old fan-out-child-DAGs dance).
/// `transient = Rebuilding` because those appended subgraphs are rebuilds:
/// it's applied per-agent at claim time (the `MetaLock` head needs no lease,
/// so the "hyperhive" pseudo-agent gets no pill), giving each cascade agent
/// crash-watch suppression during its `Swap` — the property the old child
/// `Rebuild` DAGs carried via their own transient.
pub fn meta_update(
    inputs: Vec<String>,
    source: Source,
    reason: String,
    approval_id: Option<i64>,
) -> DagSpec {
    let job = Job::new();
    let lock = node(
        &job,
        NodeKind::MetaLock {
            sweep: false,
            fanout: None,
            inputs,
        },
    );
    // The bump itself has no side effect, so an operator-driven one ends at the
    // `MetaLock`; an approval-driven one still has its row to resolve and gets the
    // per-outcome tails edged onto that single group-root — whose roll-up covers
    // the rebuild subgraphs `MetaLock` grows into itself.
    if let Some(approval_id) = approval_id {
        resolve_approval_tails(&job, approval_id, lock);
    }
    DagSpec {
        source,
        reason,
        job,
    }
}

/// Topology move(s) as a single-node DAG. `moves` is `(child, new_parent)`
/// pairs — len 1 for `set-parent`, len N for `set-parent-bulk`, applied
/// uniformly by the one [`NodeKind::Reparent`] node (which holds the global
/// meta window for its duration, same precedent as [`NodeKind::WritePermFile`]).
/// No rebuild subgraph: `topology.json` is read live by every consumer
/// (dashboard tree, `<parent>`/`<children>` sentinel routing, permission
/// checks), so a parent move needs no container rebuild to take effect.
/// No transient pill either — the node is agentless (no lease to hang one
/// off of) and near-instant. No tail node: the write is the whole effect.
pub fn reparent(
    moves: Vec<(hive_types::Ident, Option<hive_types::Ident>)>,
    source: Source,
    reason: String,
) -> DagSpec {
    let job = Job::new();
    let _reparent = node(&job, NodeKind::Reparent { moves });
    DagSpec {
        source,
        reason,
        job,
    }
}

// The boot is assembled inline in `workers/auto_update.rs::submit_boot_tree`
// as ONE `Boot` DAG (a sweep `MetaLock` root that grows rebuild subgraphs
// in-DAG, plus a `Reconcile` root per drifted agent) — no anchor node and no
// per-agent child DAGs.