hyperhive/hive-c0re/src/job_queue/templates.rs

//! DAG shape builders — every operation as a template over the shared
//! node primitives — plus submit-time cycle validation (petgraph is
//! confined to this validation; the runtime store stays the plain
//! `Vec<Node>` + `deps`).
//!
//! Every node carries its own `agent` (there is no DAG-level agent) — the
//! `node` helper stamps each node's agent. This module holds the *pure*
//! shape builders (no I/O). The hive-wide **power ops** (`stop` / `start` /
//! `restart`) are NOT here: their per-agent shape depends on each agent's
//! live running state (an async `lifecycle::is_running` read), so they are
//! assembled dynamically in `submit.rs` out of the shared pure primitives
//! this module exports (`node`, `after_ok`, `rebuild_nodes`) — one
//! independent per-agent subgraph each, concurrent on its own lease, ONE
//! DAG for the whole hive-wide op. `stop`/`start` write the durable `wanted`
//! intent via a head `SetWanted(w)` node (holding the agent lease, so
//! intent+reconcile is atomic per-agent); `restart` writes no intent — it
//! bounces the container and lets the tail `Reconcile` converge to the
//! agent's existing `wanted`.
//!
//! ```text
//! rebuild(a):        MetaSync(a) → Prebuild(a) → StopForUpdate(a) → Swap(a) →(ok) PostSwap(a) →(any) Reconcile(a)
//! spawn(a):          Provision(a) → Create(a) → WriteDropin(a) → Reconcile(a)  [wanted=Up at approve]
//! perm-change(a):    WritePermFile(a) → «rebuild subgraph»
//! meta-update(inp):  MetaLock(inp) →«in-DAG rebuild subgraph per affected a»
//! reparent(moves):   Reparent(moves)  [no rebuild — topology.json is read live]
//! ```
//!
//! For the dynamic power-op shapes (`stop` / `start` / `restart`, built from
//! live online/offline state), see `submit.rs`.

use anyhow::{Result, bail};

use super::model::{DagSpec, Dep, DepWhen, HookKind, NodeKind, NodeSpec, PermPayload, Source};
use crate::coordinator::TransientKind;

/// After-ok edge on the previous node — the common chain link. Shared with
/// the async power-op builders in `submit.rs` (which assemble per-agent
/// chains dynamically from live container state).
pub(crate) fn after_ok(on: u64) -> Vec<Dep> {
    vec![Dep {
        on,
        when: DepWhen::AfterOk,
    }]
}

/// Build one **top-level (group-root)** node — `parent = None`. `kind` carries
/// the agent it targets ([`NodeKind`] is the payload directly). Shared with
/// `submit.rs`'s dynamic power-op builders. A root owns whatever resource it
/// declares for its whole subtree; its descendants borrow it (agent-lease /
/// build-slot continuity). Ordering vs other nodes is `deps`; grouping is
/// `parent`.
pub(crate) fn node(kind: NodeKind, deps: Vec<Dep>) -> NodeSpec {
    NodeSpec {
        kind,
        deps,
        parent: None,
    }
}

/// Build a **child** node whose structural parent is spec-index `parent`. The
/// child runs once its parent reaches `Finishing` (the parent gate), so it must
/// NOT `deps` on `parent` (dep-scope validation rejects a dep on one's own
/// parent). `deps` here order the child against its *siblings* only.
pub(crate) fn child(parent: u64, kind: NodeKind, deps: Vec<Dep>) -> NodeSpec {
    NodeSpec {
        kind,
        deps,
        parent: Some(parent),
    }
}

/// The rebuild node subtree (nested, three group roots). `base` is the spec
/// index of the first node (`MetaSync`). Structure:
/// - `MetaSync` (base+0, **root**): the meta-repo preamble (dir prep, agent
///   sync, optional relock). Owns the global `MetaWindow` — and *only* for its
///   own short duration, which is why it is a sibling root rather than
///   `Prebuild`'s parent: a resource is held across the holder's whole subtree,
///   so parenting the build under it would extend a hive-global window over
///   every rebuild's nix build.
/// - `Prebuild` (base+1, **root**): `AfterOk` `MetaSync`. Owns the build slot
///   for the whole mechanical subtree below it. Lease-exempt — the nix build
///   overlaps other DAGs on the same agent.
/// - `StopForUpdate` (base+2, child of `Prebuild`): owns the agent lease. Runs
///   once `Prebuild` reaches `Finishing` (parent gate).
/// - `Swap` (base+3, child of `StopForUpdate`): borrows the agent lease from its
///   parent and the build slot from grand-ancestor `Prebuild` — both continuous.
/// - `PostSwap` (base+4, child of `StopForUpdate`): the swap's Ok-only
///   bookkeeping tail (rev marker, forge/matrix sync, kick, rescan), `AfterOk`
///   its sibling `Swap`.
/// - `Reconcile` (base+5, **root**): `AfterAny` `Prebuild`, which rolls up
///   terminal only once its whole mechanical subtree (SFU→Swap→PostSwap) has
///   settled — so `Reconcile` runs after the swap regardless of outcome, and as
///   a top-level root it survives the cancel-cascade of a failed `Prebuild`
///   (recovery-start invariant, which also covers a failed `MetaSync`: that
///   cancel-cascades `Prebuild`, i.e. terminal, so the tail still runs). It
///   takes a fresh lease; the tiny gap is harmless — `Reconcile` converges to
///   the persisted `wanted` idempotently.
pub(crate) fn rebuild_nodes(agent: &str, relock: bool, base: u64) -> Vec<NodeSpec> {
    let a = || agent.to_owned();
    vec![
        node(
            NodeKind::MetaSync { agent: a(), relock },
            if base == 0 {
                Vec::new()
            } else {
                after_ok(base - 1)
            },
        ),
        node(NodeKind::Prebuild { agent: a() }, after_ok(base)),
        child(base + 1, NodeKind::StopForUpdate { agent: a() }, Vec::new()),
        child(base + 2, NodeKind::Swap { agent: a() }, Vec::new()),
        child(
            base + 2,
            NodeKind::PostSwap { agent: a() },
            after_ok(base + 3),
        ),
        node(
            NodeKind::Reconcile { agent: a() },
            vec![Dep {
                on: base + 1,
                when: DepWhen::AfterAny,
            }],
        ),
    ]
}

/// The rebuild subgraph a [`NodeKind::DeployApply`] grows into its own DAG once
/// the merge has landed and `prepare_deploy` has staged the lock, plus the
/// [`NodeKind::FinalizeDeploy`] that closes the window behind it.
///
/// `relock = false` is the whole reason this composes: `prepare_deploy` already
/// relocked and staged `flake.lock`, so the appended `MetaSync` must do the dir
/// prep + `sync_agents` *without* re-locking over it.
///
/// `FinalizeDeploy` waits on **two** siblings, which together reproduce the gate
/// the old fused node had around its inline `rebuild_no_meta` call:
/// - `AfterOk` `Prebuild` — a parent's state is its roll-up, so this is `Done`
///   only once `StopForUpdate` → `Swap` → `PostSwap` all are (a failed *or*
///   cancelled child rolls the parent up `Failed`). That's the old
///   `build_result`.
/// - `AfterOk` `Reconcile` — the old call passed `deferred_start = false` on
///   purpose: the container had to come back up *before* the deploy was
///   finalized. `Reconcile` alone would not do, being `AfterAny` — it reaches
///   `Done` even after a failed `Swap`.
///
/// Appended, not submitted: the roots below become children of the emitting
/// `DeployApply` (see [`super::JobQueue::append_subgraph`]), which puts them
/// inside the `DeployWindow`'s subtree — so the `MetaWindow` this subgraph's
/// `MetaSync` and `FinalizeDeploy` declare is re-entered from the ancestor
/// already holding it rather than deadlocking against it.
pub(crate) fn deploy_rebuild_nodes(agent: &str) -> Vec<NodeSpec> {
    let mut nodes = rebuild_nodes(agent, false, 0);
    nodes.push(node(
        NodeKind::FinalizeDeploy {
            agent: agent.to_owned(),
        },
        vec![
            Dep {
                on: 1,
                when: DepWhen::AfterOk,
            },
            Dep {
                on: 5,
                when: DepWhen::AfterOk,
            },
        ],
    ));
    nodes
}

/// One uniform rebuild shape — no `was_running` branch. `StopForUpdate`
/// noops when already down; the tail `Reconcile` auto-noops the start
/// when `wanted = Offline` (a rebuild of a deliberately-stopped agent
/// leaves it stopped). `relock = false` only for meta-update cascade
/// children.
pub fn rebuild(agent: &str, source: Source, reason: String, relock: bool) -> DagSpec {
    DagSpec {
        hook: Some(HookKind::EmitRebuilt),
        source,
        reason,
        approval_id: None,
        inputs: Vec::new(),
        transient: Some(TransientKind::Rebuilding),
        nodes: rebuild_nodes(agent, relock, 0),
    }
}

/// Approval-driven deploy (`MergeConfigPr`) as a phase subtree rather than the
/// single opaque node it used to be. Structure:
/// - `DeployWindow` (0, **root**): the resource holder — global meta window,
///   agent lease, build slot — held across every child below. No work of its
///   own; it reaches `Finishing` immediately and the children run inside it.
/// - `MergeVerify` (1, child): drift-gate + fetch + eval-verify. Mutates
///   nothing, so a failure here cancel-cascades its siblings with the forge and
///   the applied repo exactly as they were.
/// - `DeployApply` (2, child, `AfterOk` `MergeVerify`): the irreversible half —
///   ff-merge + `prepare_deploy`. It doesn't rebuild inline; it grows
///   [`deploy_rebuild_nodes`] into this DAG as its own children, so the build
///   and the closing `FinalizeDeploy` are real nodes under the same window.
/// - `DeployTail` (3, child, `AfterAny` `DeployApply`): the compensation +
///   bookkeeping tail — rollback when a merge landed unfinalized, forge tag
///   mirror, PR failure comment (see [`NodeKind::DeployTail`]).
///
/// The window still spans the container build, as it must: `prepare_deploy`
/// leaves `flake.lock` staged-uncommitted for the build's whole duration.
pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec {
    let a = || agent.to_owned();
    DagSpec {
        hook: Some(HookKind::ResolveApproval),
        source: Source::Approval,
        reason,
        approval_id: Some(approval_id),
        inputs: Vec::new(),
        transient: Some(TransientKind::Rebuilding),
        nodes: vec![
            node(NodeKind::DeployWindow { agent: a() }, Vec::new()),
            child(0, NodeKind::MergeVerify { agent: a() }, Vec::new()),
            child(0, NodeKind::DeployApply { agent: a() }, after_ok(1)),
            child(
                0,
                NodeKind::DeployTail { agent: a() },
                vec![Dep {
                    on: 2,
                    when: DepWhen::AfterAny,
                }],
            ),
        ],
    }
}

/// A single `Reconcile` node that converges observed power state to the
/// persisted intent — `wanted` is untouched (no `SetWanted`), unlike the
/// operator `start`/`stop` templates. Test-only helper now (used to build
/// single-node lifecycle DAGs that exercise per-agent lease serialization
/// in the queue tests); production paths no longer emit a bare reconcile.
#[cfg(test)]
pub fn reconcile_only(
    agent: &str,
    source: Source,
    reason: String,
    transient: Option<TransientKind>,
) -> DagSpec {
    DagSpec {
        hook: None,
        source,
        reason,
        approval_id: None,
        inputs: Vec::new(),
        transient,
        nodes: vec![node(
            NodeKind::Reconcile {
                agent: agent.to_owned(),
            },
            Vec::new(),
        )],
    }
}

/// First-deploy spawn (approval-driven): `Provision` (proposed/applied
/// repos, state subvolume, meta registration) then `Create`
/// (`nixos-container create`), drop-in write, then `Reconcile` starts
/// the container (`wanted = Up` written at approve time). All-or-nothing:
/// `Provision` (lease-exempt, precedes the container) is the group root;
/// `Create` (child) owns the agent lease; `WriteDropin` + `Reconcile`
/// (children of `Create`) borrow it. A failure cancel-cascades the rest —
/// unlike rebuild there's no recovery-reconcile (nothing to converge if the
/// container was never created).
pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
    DagSpec {
        hook: Some(HookKind::ResolveApproval),
        source: Source::Approval,
        reason,
        approval_id: Some(approval_id),
        inputs: Vec::new(),
        transient: Some(TransientKind::Spawning),
        nodes: {
            let a = || agent.to_owned();
            vec![
                node(NodeKind::Provision { agent: a() }, Vec::new()),
                child(0, NodeKind::Create { agent: a() }, Vec::new()),
                child(1, NodeKind::WriteDropin { agent: a() }, Vec::new()),
                child(1, NodeKind::Reconcile { agent: a() }, after_ok(2)),
            ]
        },
    }
}

/// Perm change: commit the JSON file(s), then the rebuild subgraph so
/// the updated `HIVE_TOOL_GROUPS` / `HIVE_CAPABILITIES` env var takes
/// effect in the container.
pub fn perm_change(agent: &str, source: Source, reason: String, payload: PermPayload) -> DagSpec {
    let mut nodes = vec![node(
        NodeKind::WritePermFile {
            agent: agent.to_owned(),
            payload,
        },
        Vec::new(),
    )];
    nodes.extend(rebuild_nodes(agent, true, 1));
    DagSpec {
        hook: Some(HookKind::EmitRebuilt),
        source,
        reason,
        approval_id: None,
        inputs: Vec::new(),
        transient: Some(TransientKind::Rebuilding),
        nodes,
    }
}

/// Meta-input lock bump. The `MetaLock` executor grows one rebuild subgraph
/// per affected agent into *this same* DAG on completion (via
/// `append_subgraph`) — appended *after* the bump lands so their prebuilds
/// run against the post-bump lock, and a failed bump appends nothing
/// (replacing the old fan-out-child-DAGs dance).
/// `transient = Rebuilding` because those appended subgraphs are rebuilds:
/// it's applied per-agent at claim time (the `MetaLock` head needs no lease,
/// so the "hyperhive" pseudo-agent gets no pill), giving each cascade agent
/// crash-watch suppression during its `Swap` — the property the old child
/// `Rebuild` DAGs carried via their own transient.
pub fn meta_update(
    inputs: Vec<String>,
    source: Source,
    reason: String,
    approval_id: Option<i64>,
) -> DagSpec {
    DagSpec {
        // The bump itself has no side effect; an approval-driven one still has
        // its row to resolve.
        hook: approval_id.map(|_| HookKind::ResolveApproval),
        source,
        reason,
        approval_id,
        inputs,
        transient: Some(TransientKind::Rebuilding),
        nodes: vec![node(
            NodeKind::MetaLock {
                sweep: false,
                fanout: None,
            },
            Vec::new(),
        )],
    }
}

/// Topology move(s) as a single-node DAG. `moves` is `(child, new_parent)`
/// pairs — len 1 for `set-parent`, len N for `set-parent-bulk`, applied
/// uniformly by the one [`NodeKind::Reparent`] node (which holds the global
/// meta window for its duration, same precedent as [`NodeKind::WritePermFile`]).
/// No rebuild subgraph: `topology.json` is read live by every consumer
/// (dashboard tree, `<parent>`/`<children>` sentinel routing, permission
/// checks), so a parent move needs no container rebuild to take effect.
/// No transient pill either — the node is agentless (no lease to hang one
/// off of) and near-instant. No terminal hook: the write is the whole effect.
pub fn reparent(
    moves: Vec<(hive_types::Ident, Option<hive_types::Ident>)>,
    source: Source,
    reason: String,
) -> DagSpec {
    DagSpec {
        hook: None,
        source,
        reason,
        approval_id: None,
        inputs: Vec::new(),
        transient: None,
        nodes: vec![node(NodeKind::Reparent { moves }, Vec::new())],
    }
}

// The boot is assembled inline in `workers/auto_update.rs::submit_boot_tree`
// as ONE `Boot` DAG (a sweep `MetaLock` root that grows rebuild subgraphs
// in-DAG, plus a `Reconcile` root per drifted agent) — no anchor node and no
// per-agent child DAGs.

/// Validate a spec before it enters the queue: node ids are dense
/// (index = id), deps + parents reference existing *earlier* nodes, and the
/// dep graph is acyclic (petgraph `toposort`). Rejecting cycles here fixes the
/// old queue's documented "circular dep silently deadlocks forever" caveat.
pub fn validate(spec: &DagSpec) -> Result<()> {
    if spec.nodes.is_empty() {
        bail!("dag spec {:?} has no nodes", spec.reason);
    }
    let n = spec.nodes.len();
    let mut graph = petgraph::graph::DiGraph::<u32, ()>::new();
    let idx: Vec<_> = (0..n)
        .map(|i| graph.add_node(u32::try_from(i).unwrap_or(u32::MAX)))
        .collect();
    for (i, node) in spec.nodes.iter().enumerate() {
        // A `parent` must index an earlier node — `insert_group` resolves it to
        // an already-inserted `NodeId`, so a forward/out-of-bounds parent would
        // otherwise panic there.
        if let Some(p) = node.parent
            && usize::try_from(p).is_ok_and(|p| p >= i)
        {
            bail!(
                "dag spec {:?} node {i} has invalid parent {p} (must be an earlier node)",
                spec.reason
            );
        }
        for dep in &node.deps {
            let Some(&dep_idx) = usize::try_from(dep.on).ok().and_then(|i| idx.get(i)) else {
                bail!(
                    "dag spec {:?} node {i} depends on unknown node {}",
                    spec.reason,
                    dep.on
                );
            };
            graph.add_edge(dep_idx, idx[i], ());
        }
    }
    if petgraph::algo::toposort(&graph, None).is_err() {
        bail!("dag spec {:?} contains a dependency cycle", spec.reason);
    }
    Ok(())
}