hyperhive/hive-c0re/src/job_queue/templates.rs

//! DAG shape builders — every operation as a template over the shared node
//! primitives. Pure (no I/O); each node carries its own `agent` (there is no
//! DAG-level agent) and **declares its own resources** with `.needs(…)` right
//! where it is constructed, rather than derived from its kind. Deriving made the
//! requirement a property of the *kind*, so a kind running under an ancestor
//! that already held the resource could get away with declaring nothing.
//!
//! **The one sanctioned exception is a brace** — a pure-resource-holder root
//! ([`NodeKind::AgentWindow`]) declaring for a
//! coordinated subtree whose members then declare nothing. See
//! `docs/scheduler/coordinator.md`, _Braces_ — which also carries the per-operation DAG
//! shapes, so they are not restated here.
//!
//! Nodes are **named, not counted** — a template holds the handle
//! [`JobBuilder::node`] hands back, so an edge says which node it waits on. Why that
//! removes submit-time cycle validation: `docs/scheduler/coordinator.md`.
//!
//! The hive-wide **power ops** (`stop` / `start` / `restart`) are NOT here:
//! their per-agent shape depends on live running state (an async
//! `lifecycle::is_running` read), so [`super::power`] assembles them out of
//! the primitives this module exports ([`rebuild_nodes`]).

use super::model::{NodeKind, PermPayload};
use super::resource::Resource;
use super::{Handle, JobBuilder};

/// The `Rebuilt`-reporting tail pair for a rebuild-shaped DAG: the success node
/// gated on every group-root in `roots`, and the failure node gated on *its*
/// elimination.
///
/// Exactly one runs on a DAG that executed, and neither runs on one the operator
/// dropped — see [`hive_jobq::NodeRef::on_elimination_of`].
fn emit_rebuilt_tails(builder: &JobBuilder, agent: &str, roots: &[Handle<'_>]) {
    let ok = roots.iter().fold(
        builder.node(NodeKind::EmitRebuilt {
            agent: agent.to_owned(),
            ok: true,
        }),
        hive_jobq::NodeRef::after_ok,
    );
    // The failure branch needs *both*: the ok branch being ruled out (that is the
    // "something went wrong" signal) **and** every root actually finished. The
    // second half is easy to forget and gets the ordering wrong without it — a
    // failed `Prebuild` eliminates the ok branch immediately, while the recovery
    // `Reconcile` is still bringing the container back up, so reporting straight
    // off the elimination would announce the failure mid-recovery.
    let _failed = roots.iter().fold(
        builder
            .node(NodeKind::EmitRebuilt {
                agent: agent.to_owned(),
                ok: false,
            })
            .on_elimination_of(ok),
        hive_jobq::NodeRef::after_any,
    );
}

/// Declare one rebuild subgraph per agent onto the builder a running
/// [`NodeKind::MetaLock`] was handed.
///
/// **Into the emitter's own builder, not as new DAGs.** Growing in-DAG is what
/// roots each subgraph on the `MetaLock`, so the whole sweep (or meta-update
/// cascade) stays one unit of work the operator can watch and cancel, and every
/// rebuild builds against the lock the emitter just bumped.
///
/// Same reason as [`fanned_out_mechanical`] for living here: this was the
/// second construction site declaring nodes inline in an executor.
pub(crate) fn grown_rebuilds(builder: &JobBuilder, agents: &[String], relock: bool) {
    for agent in agents {
        rebuild_nodes(builder, agent, relock, None);
    }
}

/// As [`grown_rebuilds`], but each agent gets its `Signal` → `Drain` window
/// before being stopped. The boot sweep's flavour: it stops agents that were
/// mid-turn when the host came up, so they drain rather than being cut off.
pub(crate) fn grown_graceful_rebuilds(builder: &JobBuilder, agents: &[String], relock: bool) {
    for agent in agents {
        graceful_rebuild_nodes(builder, agent, relock, None);
    }
}

/// Declare the mechanical node a [`NodeKind::Reconcile`] planner fans out
/// (`Start` / `Stop`) onto the builder it was handed while running.
///
/// `Start` / `Stop` declare the agent lease they run under. Their `Reconcile`
/// parent is holding it already, so the declaration is a **re-entrant borrow**
/// — no second unit, no deadlock. It exists so the requirement belongs to the
/// node rather than to the fact that a `Reconcile` happens to fan it out.
///
/// Lives here rather than inline in `exec.rs` for the same reason every other
/// declaration does: this is the one construction site that was hiding in an
/// executor, which meant the only test of it had to re-declare the same two
/// calls itself and would have kept passing if the executor changed.
pub(crate) fn fanned_out_mechanical(builder: &JobBuilder, kind: NodeKind) {
    let lease = Resource::Agent(kind.agent().to_owned());
    let _ = builder.node(kind).needs(lease);
}

/// The graceful quiesce pair — `Signal` then `Drain`: ask the agent to run its
/// stop-checkpoint turn, then wait for it to go quiet. Returns the `Drain`
/// handle, which is what a caller edges its stop onto. The container stays
/// **up** throughout; the actual stop is the caller's next node.
///
/// `brace` must already hold [`Resource::Agent`] for its subtree; the pair
/// declares nothing and borrows it (see the module header). They are
/// dep-ordered **siblings**, not nested — nesting is only load-bearing where
/// `Signal` itself holds the lease, as in `submit.rs`'s `restart_chain`.
pub(crate) fn quiesce<'a>(builder: &'a JobBuilder, agent: &str, brace: Handle<'a>) -> Handle<'a> {
    let a = || agent.to_owned();
    let signal = builder.node(NodeKind::Signal { agent: a() }).part_of(brace);
    builder
        .node(NodeKind::Drain { agent: a() })
        .part_of(brace)
        .after_ok(signal)
}

/// The pause quiesce pair — `PauseSignal` then `PauseDrain`: write the
/// pause marker, then wait for the harness to acknowledge it. Returns
/// the **group root** — the brace, not the tail. Unlike [`quiesce`],
/// nothing chains onto `PauseDrain` (there's no downstream
/// `Reconcile`-shaped node the way a stop has one), so the handle a
/// caller actually needs is the brace whose roll-up covers the whole
/// pair, same as [`super::power`]'s `stop_chain` returning `wanted`'s
/// guid rather than `quiesce`'s own returned `Drain` handle.
///
/// Unlike [`quiesce`], there's no natural resource-holding head to
/// borrow a `brace` from — pausing isn't a `wanted`-state transition,
/// so there's no `SetWanted`-shaped parent the way `stop_chain` has
/// one. This is exactly the case [`NodeKind::AgentWindow`] exists for
/// (see the module header's _brace_ paragraph): a pure-resource-holder
/// root with no work of its own, so `PauseSignal`/`PauseDrain` can be
/// plain siblings under it, both borrowing its lease via `part_of`.
///
/// ⚠️ `PauseSignal` can *not* hold the lease itself with `PauseDrain`
/// nested under it (`.part_of(signal)`) — that was the first shape
/// tried here, and `hive_jobq` rejects it at insert: `PauseDrain`
/// depending on `PauseSignal` via `after_ok` while also being its
/// *child* reaches outside `PauseDrain`'s own group (its parent, not a
/// sibling) — "an edge must stay within the depender's own group".
/// `AgentWindow` as a separate, actual brace is what makes them
/// siblings instead.
pub(crate) fn pause_quiesce<'a>(builder: &'a JobBuilder, agent: &str) -> Handle<'a> {
    let a = || agent.to_owned();
    let brace = builder
        .node(NodeKind::AgentWindow { agent: a() })
        .needs(Resource::Agent(a()));
    let signal = builder
        .node(NodeKind::PauseSignal { agent: a() })
        .part_of(brace);
    let _drain = builder
        .node(NodeKind::PauseDrain { agent: a() })
        .part_of(brace)
        .after_ok(signal);
    brace
}

/// The group-roots a [`rebuild_nodes`] subgraph exposes to its caller: what a
/// tail node edges onto, and what a follow-up node waits for.
///
/// Only the roots — a root's state *is* its subtree's roll-up, so these three
/// cover every node in the subgraph without the caller knowing its shape.
#[derive(Debug, Clone, Copy)]
pub(crate) struct RebuildRoots<'a> {
    /// The meta-repo preamble.
    pub meta_sync: Handle<'a>,
    /// The brace holding the agent lease and the build slot — its roll-up
    /// carries the whole mechanical subtree (`Prebuild`, the quiesce chain,
    /// `StopForUpdate` → `Swap` → `RebuildBookkeeping`).
    pub agent_window: Handle<'a>,
    /// The recovery/convergence tail root.
    pub reconcile: Handle<'a>,
}

impl<'a> RebuildRoots<'a> {
    /// The three roots as a slice, for edging a tail onto all of them.
    fn all(self) -> [Handle<'a>; 3] {
        [self.meta_sync, self.agent_window, self.reconcile]
    }
}

/// The rebuild node subtree — three group roots (`MetaSync`, the `AgentWindow`
/// brace, `Reconcile`). `after`, when given, is the node it chains behind. The
/// shape itself is in `docs/scheduler/coordinator.md`; the code below is the source of
/// truth for it, so only the three choices a reader would otherwise undo are
/// called out here:
///
/// - **`MetaSync` is a sibling root, not the brace's parent** — it owns the
///   *global* `MetaWindow`, and a resource is held across the holder's whole
///   subtree, so parenting the rebuild under it would serialise every agent's
///   nix build behind one hive-wide window.
/// - **`StopForUpdate` waits on `Prebuild` as well as `Drain`** — running the
///   drain early is the win; taking the container *down* early is pure downtime.
/// - **`Reconcile` is a top-level root, not a child** — so it survives the
///   cancel-cascade of a failed brace and still converges the container
///   (recovery-start invariant). It takes a fresh lease; the gap is harmless
///   because it is idempotent.
fn rebuild_subtree<'a>(
    builder: &'a JobBuilder,
    agent: &str,
    relock: bool,
    graceful: bool,
    after: Option<Handle<'a>>,
) -> RebuildRoots<'a> {
    let a = || agent.to_owned();

    let mut meta_sync = builder
        .node(NodeKind::MetaSync { agent: a(), relock })
        .needs(Resource::MetaWindow);
    if let Some(after) = after {
        meta_sync = meta_sync.after_ok(after);
    }
    // The brace. Both resources are declared here, on one node, on purpose —
    // the queue acquires a node's resources atomically, so a single
    // multi-resource root can never hold one and block on another. Hoisting the
    // slot up costs nothing: a resource is held for the acquirer's whole
    // subtree, and the build slot already spanned the entire rebuild when
    // `Prebuild` was the one holding it.
    let agent_window = builder
        .node(NodeKind::AgentWindow { agent: a() })
        .needs(Resource::BuildSlot)
        .needs(Resource::Agent(a()))
        .after_ok(meta_sync);

    // Siblings under the brace: the build and the quiesce chain run
    // concurrently, borrowing the brace's grants rather than declaring their
    // own. Declaring the lease on both would make them mutually exclusive —
    // it is single-unit — which is exactly what this shape exists to avoid.
    let prebuild = builder
        .node(NodeKind::Prebuild { agent: a() })
        .part_of(agent_window);

    let drain = graceful.then(|| quiesce(builder, agent, agent_window));

    // The container goes down here, not earlier: `AfterOk` the build so a
    // failed build never stops a healthy container, and `AfterOk` the drain so
    // the agent has checkpointed.
    let mut stop_for_update = builder
        .node(NodeKind::StopForUpdate { agent: a() })
        .part_of(agent_window)
        .after_ok(prebuild);
    if let Some(drain) = drain {
        stop_for_update = stop_for_update.after_ok(drain);
    }

    let swap = builder
        .node(NodeKind::Swap { agent: a() })
        .part_of(stop_for_update);
    let _rebuild_bookkeeping = builder
        .node(NodeKind::RebuildBookkeeping { agent: a() })
        .part_of(stop_for_update)
        .after_ok(swap);

    let reconcile = builder
        .node(NodeKind::Reconcile { agent: a() })
        .needs(Resource::Agent(a()))
        .after_any(agent_window);

    RebuildRoots {
        meta_sync,
        agent_window,
        reconcile,
    }
}

/// The rebuild subtree, stopping the agent outright — the shape five of the six
/// call sites want. `relock` re-locks the meta flake inside `MetaSync`; `after`,
/// when given, is the node this subgraph chains behind. See
/// [`rebuild_subtree`] for the structure.
pub(crate) fn rebuild_nodes<'a>(
    builder: &'a JobBuilder,
    agent: &str,
    relock: bool,
    after: Option<Handle<'a>>,
) -> RebuildRoots<'a> {
    rebuild_subtree(builder, agent, relock, false, after)
}

/// As [`rebuild_nodes`], but the agent gets a `Signal` → `Drain` window to
/// finish the turn in flight before it is stopped. Costs up to one
/// `GRACEFUL_STOP_TIMEOUT` per subgraph, and those overlap across agents.
///
/// A separate entry point rather than a flag because `graceful` does not
/// *prepend* nodes — it **re-parents** the stop root, so a caller cannot
/// declare it without being handed the internals. Only the boot sweep wants it.
pub(crate) fn graceful_rebuild_nodes<'a>(
    builder: &'a JobBuilder,
    agent: &str,
    relock: bool,
    after: Option<Handle<'a>>,
) -> RebuildRoots<'a> {
    rebuild_subtree(builder, agent, relock, true, after)
}

/// One uniform rebuild shape — no `was_running` branch. `StopForUpdate`
/// noops when already down; the tail `Reconcile` auto-noops the start
/// when `wanted = Offline` (a rebuild of a deliberately-stopped agent
/// leaves it stopped). `relock = false` only for meta-update cascade
/// children.
///
/// Closed by an [`NodeKind::EmitRebuilt`] tail edged onto all three group-roots
/// (`MetaSync`, the `AgentWindow` brace, `Reconcile`) — the brace's roll-up
/// carries the whole `StopForUpdate`→`Swap`→`RebuildBookkeeping` subtree, so
/// those three cover every node. Edging `Reconcile` alone would not do: it is
/// `AfterAny` the brace, so it reaches `Done` even after a failed swap and the
/// tail would report success.
///
/// Returns those three roots, so a caller that needs to wait on the rebuild can
/// name them.
pub fn rebuild(builder: &JobBuilder, agent: &str, relock: bool) -> Vec<hive_jobq::NodeGuid> {
    let roots = rebuild_nodes(builder, agent, relock, None);
    emit_rebuilt_tails(builder, agent, &roots.all());
    vec![
        roots.meta_sync.guid(),
        roots.agent_window.guid(),
        roots.reconcile.guid(),
    ]
}

/// First deploy of an agent this hive has never seen, asked for by the swarm:
/// `Provision` (proposed/applied repos, state subvolume, meta registration)
/// then `Create` (`nixos-container create`), drop-in write, then `Reconcile`
/// starts the container (`wanted = Up`, seeded by
/// `swarm_status::queue_first_deploy`). All-or-nothing: `Provision`
/// (lease-exempt, precedes the container) is the group root; `Create` (child)
/// owns the agent lease; `WriteDropin` + `Reconcile` (children of `Create`)
/// borrow it. A failure cancel-cascades the rest — unlike rebuild there's no
/// recovery-reconcile (nothing to converge if the container was never created).
///
/// Returns the group root so a caller can wait on the whole subtree.
pub fn first_deploy(builder: &JobBuilder, agent: &str) -> Vec<hive_jobq::NodeGuid> {
    let a = || agent.to_owned();
    let provision = builder
        .node(NodeKind::Provision { agent: a() })
        .needs(Resource::MetaWindow);
    let create = builder
        .node(NodeKind::Create { agent: a() })
        .needs(Resource::BuildSlot)
        .needs(Resource::Agent(a()))
        .part_of(provision);
    let dropin = builder
        .node(NodeKind::WriteDropin { agent: a() })
        .needs(Resource::Agent(a()))
        .part_of(create);
    let _reconcile = builder
        .node(NodeKind::Reconcile { agent: a() })
        .needs(Resource::Agent(a()))
        .part_of(create)
        .after_ok(dropin);
    vec![provision.guid()]
}

/// Teardown: `Stop` → `DestroyContainer` → (`PurgeState`) → `DestroyBookkeeping`.
///
/// The chain is the point, not a decomposition for its own sake. Destroy used
/// to be a straight-line async fn with no queue node behind it, so nothing in
/// the graph could answer "is this container going down on purpose?" — which is
/// why an imperative crash-watch suppression guard existed at all. Reusing the
/// existing [`NodeKind::Stop`] answers it structurally: `Stop` already declares
/// `takes_container_down`, so the suppression is derived from the graph like
/// every other lifecycle op's.
///
/// That also makes the precondition an edge rather than an assertion.
/// `DestroyContainer` runs only after `Stop` succeeded, so it operates on an
/// already-stopped container and carries `takes_container_down = false`
/// permanently — a container still alive at that point is a real bug and stays
/// loud instead of being absorbed by a flag.
///
/// `Stop` is idempotent against an already-down container, so the common
/// "destroy something that isn't running" path costs nothing extra.
///
/// `Stop` is the group root and holds the agent lease for the whole teardown;
/// the rest are `part_of` children that borrow it, so no other op can interleave
/// with a half-destroyed agent. `PurgeState` is inserted only when asked for —
/// the graph shows the irreversible step as its own row when it happens, and
/// omits it entirely when it doesn't.
pub fn destroy(builder: &JobBuilder, agent: &str, purge: bool) {
    let a = || agent.to_owned();
    let stop = builder
        .node(NodeKind::Stop { agent: a() })
        .needs(Resource::Agent(a()));
    // `part_of` IS the ordering: a child runs once its parent reaches
    // `Finishing`, and a node may not also declare a dep on its own parent
    // (dep-scope validation rejects it — it would deadlock). So the
    // "container is already stopped" precondition is the group edge itself,
    // with no explicit `after_ok(stop)` to add.
    let destroy = builder
        .node(NodeKind::DestroyContainer { agent: a() })
        .part_of(stop);
    // The bookkeeping tail hangs off the purge when there is one, so the
    // irreversible delete lands before the meta sync that stops referencing it.
    let last = if purge {
        builder
            .node(NodeKind::PurgeState { agent: a() })
            .part_of(stop)
            .after_ok(destroy)
    } else {
        destroy
    };
    let _tail = builder
        .node(NodeKind::DestroyBookkeeping { agent: a() })
        .part_of(stop)
        .after_ok(last);
}

/// Perm change: commit the JSON file, then the rebuild subgraph so
/// the updated `HIVE_TOOL_GROUPS` env var takes
/// effect in the container. Group-roots are `WritePermFile` plus the rebuild
/// subgraph's `MetaSync` / `Prebuild` / `Reconcile`, so the `EmitRebuilt` tail
/// edges all four.
pub fn perm_change(builder: &JobBuilder, agent: &str, payload: PermPayload) {
    let write = builder
        .node(NodeKind::WritePermFile {
            agent: agent.to_owned(),
            payload,
        })
        .needs(Resource::MetaWindow);
    let roots = rebuild_nodes(builder, agent, true, Some(write));
    emit_rebuilt_tails(
        builder,
        agent,
        &[write, roots.meta_sync, roots.agent_window, roots.reconcile],
    );
}

/// Meta-input lock bump. The `MetaLock` executor grows one rebuild subgraph
/// per affected agent into *this same* DAG on completion (declared onto the
/// builder it was handed) — appended *after* the bump lands so their prebuilds
/// run against the post-bump lock, and a failed bump appends nothing
/// (replacing the old fan-out-child-DAGs dance).
/// `transient = Rebuilding` because those appended subgraphs are rebuilds:
/// it's applied per-agent at claim time (the `MetaLock` head needs no lease,
/// so the "hyperhive" pseudo-agent gets no pill), giving each cascade agent
/// crash-watch suppression during its `Swap` — the property the old child
/// `Rebuild` DAGs carried via their own transient.
pub fn meta_update(builder: &JobBuilder, inputs: Vec<String>) {
    let _lock = builder
        .node(NodeKind::MetaLock {
            sweep: false,
            fanout: None,
            inputs,
        })
        .needs(Resource::BuildSlot)
        .needs(Resource::MetaWindow);
}

// The boot is assembled inline in `workers/auto_update.rs::submit_boot_tree`
// as ONE `Boot` DAG (a sweep `MetaLock` root that grows rebuild subgraphs
// in-DAG, plus a `Reconcile` root per drifted agent) — no anchor node and no
// per-agent child DAGs.

/// One matrix user/space sweep. Holding [`Resource::MatrixSweep`] is what keeps
/// two passes from overlapping, whichever caller submitted each.
pub fn matrix_sweep(builder: &JobBuilder) -> Handle<'_> {
    builder
        .node(NodeKind::MatrixSweep)
        .needs(Resource::MatrixSweep)
}

/// One `/knowledge` pull, holding [`Resource::KnowledgeTree`] for its duration.
pub fn knowledge_pull(builder: &JobBuilder) -> Handle<'_> {
    builder
        .node(NodeKind::KnowledgePull)
        .needs(Resource::KnowledgeTree)
}