hyperhive/hive-c0re/src/job_queue/model.rs

//! Data model for the generic job-DAG queue: node kinds (the primitive
//! operations), dependency edges, and the runtime `Dag` / `Node` store.
//! The serialized *views* — `DagView` / `NodeView` plus the `Source` /
//! `State` / `PermPayload` wire enums — live in `hive_host_sock::jobs`
//! (they travel on the host admin socket and the dashboard channels
//! served off the same snapshot, and nowhere else) and are re-exported
//! here for the queue's internal use.
//!
//! Two levels: the **DAG** is the unit of cancel / approval-resolution
//! and the dashboard group; the **node** is the unit of scheduling /
//! execution / build-log, and carries its own `agent` (a
//! DAG can span agents). See `docs/coordinator.md::Job queue` for the
//! full design.

pub use hive_host_sock::jobs::{DagView, NodeId, PermPayload, Source, State};
use serde::Serialize;

use crate::coordinator::TransientKind;

use hive_jobq::{DepWhen, TerminalState};

/// A dependency edge (intra-DAG only — cross-DAG ordering comes from
/// the per-agent lease + dedup, never from edges between DAGs).
#[derive(Debug, Clone, Copy, Serialize)]
pub struct Dep {
    pub on: NodeId,
    pub when: DepWhen,
}

/// The primitive operations — each kind maps to one executor fn in
/// `exec.rs`, a thin wrapper over existing `lifecycle.rs` / `meta.rs`
/// code. Concurrency is gated by two resource classes (see
/// [`NodeKind::needs_build_slot`] / [`NodeKind::needs_lease`]); the
/// meta *repo* is serialized by `meta::META_LOCK` inside the wrapped
/// functions themselves, which is why there is no `GitCommit` node —
/// a standalone commit node would open a dirty-working-tree window
/// between nodes that the fused `meta.rs` ops deliberately close.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum NodeKind {
    /// The rebuild's meta-repo preamble: `lifecycle::prepare_rebuild_dirs`,
    /// an idempotent meta `sync_agents`, and an optional per-agent relock.
    /// `relock = false` only for meta-update cascade rebuilds (re-locking
    /// would revert the bump the cascade just committed).
    ///
    /// Its own node — ahead of, and *not* an ancestor of, [`NodeKind::Prebuild`]
    /// — precisely because it is the only part of the rebuild that mutates the
    /// meta repo and so holds the global
    /// [`Resource::MetaWindow`](super::resource::Resource::MetaWindow). Fusing
    /// it into `Prebuild` (or making it `Prebuild`'s parent, which holds a
    /// resource across the whole subtree) would extend that global window over
    /// the multi-minute toplevel build and serialize rebuilds hive-wide.
    /// Store/meta work only — build-slot- and lease-exempt.
    MetaSync { agent: String, relock: bool },
    /// Out-of-band toplevel build while the container keeps serving:
    /// `lifecycle::prebuild_toplevel`, reading a meta repo the upstream
    /// [`NodeKind::MetaSync`] has already synced. The warm build is skipped
    /// when the container is already down — it only exists to shrink the
    /// swap's downtime, which a stopped agent doesn't need (`Swap` builds
    /// inline instead).
    Prebuild { agent: String },
    /// `nixos-container update` profile-swap (requires the container
    /// stopped). Re-applies nspawn flags + resource limits first —
    /// rebuild is the reconcile verb. The post-rebuild bookkeeping tail
    /// lives in the sibling `PostSwap` node.
    Swap { agent: String },
    /// The post-`Swap` bookkeeping tail as a first-class node: rev marker,
    /// forge + matrix sync, manager kick, container rescan, meta-inputs
    /// snapshot. Split out of `Swap` for dashboard visibility + retry
    /// granularity. Deps `AfterOk(Swap)`, so it runs only when the profile
    /// swap succeeded; the tail `Reconcile` deps `AfterAny(PostSwap)`, so on
    /// swap failure this node is cancel-cascaded (a terminal state) and
    /// recovery still runs. Store/forge/matrix work only — no nix build, so
    /// build-slot-exempt; the agent lease taken at `Swap` is held across the
    /// whole chain until `Reconcile` settles, so it's not re-declared here.
    PostSwap { agent: String },
    /// First-spawn pre-create provisioning: proposed/applied repos,
    /// state subvolume, and meta registration (`sync_agents`). Runs
    /// ahead of `Create` so the `nixos-container create --flake
    /// meta#<name>` ref resolves. Store/meta-only — no container yet —
    /// so it's lease- and build-slot-exempt like `Prebuild`.
    Provision { agent: String },
    /// First-spawn `nixos-container create` proper. Assumes the
    /// upstream `Provision` node already registered the agent in meta.
    Create { agent: String },
    /// Meta flake lock bump. `sweep = false`: `meta::lock_update`
    /// (commit fused, under `META_LOCK`) with the DAG's `inputs`;
    /// `sweep = true`: `meta::lock_update_hyperhive`, *non-fatal* (a
    /// failed boot-time bump must not cancel the fan-out rebuilds).
    /// On success the scheduler appends child `Rebuild` DAGs: the
    /// precomputed `fanout` list when present (boot sweep), else the
    /// post-bump affected set (`meta_update_cascade_agents`).
    MetaLock {
        sweep: bool,
        fanout: Option<Vec<String>>,
    },
    /// Idempotent power converge *planner*: read `wanted` + observed
    /// state and decide the action (start if `Up` & down, stop if
    /// `Offline` & up, else noop). The mechanical work is not done in
    /// this node — it fans a child [`NodeKind::Start`] / [`NodeKind::Stop`]
    /// DAG out at runtime so the sub-step is a first-class DAG node.
    Reconcile { agent: String },
    /// Mechanical container start: the start preamble (runtime dir +
    /// drop-ins), `start_with_fallback`, MCP listener registration, and
    /// the manager kick. Fanned out by a [`NodeKind::Reconcile`] that
    /// observed `wanted = Up` and the container down.
    Start { agent: String },
    /// Mechanical container stop: `nixos-container` kill, MCP listener
    /// unregister, and the `Killed` manager notify. Fanned out by a
    /// [`NodeKind::Reconcile`] that observed `wanted = Offline` and up.
    Stop { agent: String },
    /// Mechanical `nixos-container stop` for the profile swap. Never
    /// touches `wanted`. Noop if already stopped.
    StopForUpdate { agent: String },
    /// Set the graceful-stop fence + kick the harness so it runs one
    /// stop-checkpoint turn.
    Signal { agent: String },
    /// Await the harness clearing the fence, bounded by
    /// `GRACEFUL_STOP_TIMEOUT`. Resolves ok either way — the
    /// downstream `Reconcile` performs the actual stop.
    Drain { agent: String },
    /// `set_nspawn_flags` + `set_resource_limits` + daemon-reload.
    WriteDropin { agent: String },
    /// Commit `tool-groups.json` / `capabilities.json` per its `payload`
    /// (commit fused under `META_LOCK`). The payload rides this node — the only
    /// consumer — rather than the generic DAG container.
    WritePermFile { agent: String, payload: PermPayload },
    /// Topology move(s) — `set-parent` (len 1) or `set-parent-bulk` (len N) —
    /// as a single queue node. Agentless like [`NodeKind::MetaLock`]: a
    /// reparent touches the meta repo, not any one container, and a bulk
    /// move spans multiple agents anyway. `needs_meta_window() = true`, same
    /// precedent as [`NodeKind::WritePermFile`] (also a small
    /// git-commit-under-`META_LOCK` op) — a reparent's commit must not land
    /// inside another node's staged deploy `prepare_deploy`→`finalize_deploy`
    /// window. `(child, new_parent)` pairs, applied in order under one
    /// `META_LOCK` acquisition / one git commit (`meta::bulk_commit_topology`
    /// handles both the single- and multi-move case uniformly).
    Reparent {
        moves: Vec<(hive_types::Ident, Option<hive_types::Ident>)>,
    },
    /// Group root of the approval-deploy (`MergeConfigPr`) subtree, and the
    /// node that **owns the deploy window**. It performs no work of its own —
    /// it exists so the resources it declares (the global
    /// [`Resource::MetaWindow`](super::resource::Resource::MetaWindow), the
    /// agent lease, a build slot) are held continuously across every child
    /// phase, which a per-node acquisition could not guarantee.
    ///
    /// All three resources are declared *here*, on one node, on purpose. The
    /// queue acquires a node's resources atomically (all-or-nothing), so a
    /// single multi-resource root can never hold one and block on another —
    /// whereas letting a child take the build slot while its parent held the
    /// meta window would introduce exactly that pattern, and with it a
    /// lock-ordering argument that has to be re-verified on every future edit.
    /// Cheap, too: the window has to span the container build regardless (see
    /// [`NodeKind::DeployApply`]), so nothing is over-serialised by hoisting
    /// the slot and the lease up alongside it.
    ///
    /// Carries the approval row every phase below it re-reads, like each of
    /// those phases does — the id is the node's own payload, not something a
    /// DAG-level catch-all hands down.
    DeployWindow { agent: String, approval_id: i64 },
    /// Deploy phase 1 — **verify only, mutates nothing.** Drift-gate the
    /// approval's PR head, fetch it into the applied repo, and eval-verify the
    /// merge head. Any failure here aborts the deploy with the forge state
    /// untouched, so it is safely retryable and cancel-safe: nothing downstream
    /// has happened yet.
    MergeVerify { agent: String, approval_id: i64 },
    /// Deploy phase 2 — the irreversible fast-forward plus the *opening* half of
    /// the two-phase meta deploy: park the rollback ref, ff-merge the reviewed
    /// head to `main` via the forge API, ff `applied/main`, and
    /// `meta::prepare_deploy` (which stages `flake.lock` uncommitted).
    ///
    /// It does **not** run the container rebuild itself. It grows the ordinary
    /// rebuild subgraph into this DAG as its own children
    /// ([`super::templates::deploy_rebuild_nodes`], `relock = false` — the lock
    /// is already staged), so the multi-minute build renders as the same real
    /// nodes every other rebuild does instead of one opaque box. Closing the
    /// staged-lock window is likewise its own node
    /// ([`NodeKind::FinalizeDeploy`]), and the compensation path is
    /// [`NodeKind::DeployTail`].
    DeployApply { agent: String, approval_id: i64 },
    /// Deploy phase 3 — close the two-phase meta deploy once the rebuild
    /// subgraph under [`NodeKind::DeployApply`] has come up clean: drop the
    /// rollback ref, plant the `deployed/<id>` tag, commit the staged
    /// `flake.lock` (`meta::finalize_deploy`).
    ///
    /// Its two git steps are **fatal**, deliberately. They are the writes that
    /// tell [`NodeKind::DeployTail`] a deploy confirmed good, so a node that
    /// merely warned on them could report success while leaving the tail
    /// looking at the git state of a failure — and the tail would then roll a
    /// *good* deploy back. Failing loudly keeps the node's outcome and the
    /// repo's state saying the same thing.
    ///
    /// The trailing `meta::finalize_deploy` stays warn-only: by then the
    /// container already runs the new config, and an uncommitted staged lock is
    /// something the operator can commit by hand.
    FinalizeDeploy { agent: String, approval_id: i64 },
    /// Deploy compensation **and bookkeeping** tail — `AfterAny`
    /// [`NodeKind::DeployApply`], so it runs on success, failure, and cancel
    /// alike, in the same spirit as the rebuild template's tail `Reconcile`
    /// ("always runs, decides internally"). It:
    /// 1. compensates a merge that landed but was never finalized — roll `main`
    ///    back, reset the tree, `meta::abort_deploy`, plant `failed/<id>`;
    /// 2. mirrors whichever deploy tag got planted to the forge config repo
    ///    (`forge::push_config`), always, best-effort;
    /// 3. posts the failing build log back onto the config PR when the deploy
    ///    failed, so the manager sees the rejection without leaving the forge.
    ///
    /// Steps 2 and 3 are why this is `DeployTail` and not `AbortDeploy`: it has
    /// work to do on the success path too, and a node name that claims
    /// otherwise would be a lie on the dashboard.
    ///
    /// For (1) it needs no knowledge of how far the deploy got, because that state is
    /// parked in the applied repo rather than passed between nodes:
    /// `DeployApply` writes the pre-merge `main` sha to
    /// `refs/hyperhive/rollback/<approval-id>` before the fast-forward and
    /// [`NodeKind::FinalizeDeploy`] deletes it. So the ref existing *is*
    /// the "a merge landed and was not finalized" signal, and its absence makes
    /// this node a no-op. Parking it in git rather than in a node payload also
    /// means it survives a `hive-c0re` restart mid-deploy, which an in-memory
    /// queue does not.
    DeployTail { agent: String, approval_id: i64 },
    /// Tail node of an approval-carrying DAG (spawn / opaque deploy / config-PR
    /// merge): resolve the approval row from how the work actually ended.
    ///
    /// Weak-edged (`DepWhen::AFTER_ANY`) like [`NodeKind::DeployTail`], so it runs on
    /// success, failure **and cancel** alike and decides internally. It reads its
    /// dependencies' terminal states off its own [`Claim::deps`] rather than
    /// re-deriving them from the world the way `DeployTail` reads git: a node is
    /// *told* how the work it follows ended, it does not go back out and ask.
    ///
    /// Agentless on purpose: the approval row already names its agent, so
    /// carrying one here would be a second copy free to drift. Like
    /// [`NodeKind::MetaLock`] it reports `""` from [`NodeKind::agent`] and takes
    /// no lease — which is also what lets one close a multi-agent DAG.
    ResolveApproval {
        approval_id: i64,
        /// Which outcome this node reports. A template emits **one per outcome**,
        /// each edged to accept only that one, so exactly one is ever runnable
        /// and the executor has nothing to decide — it resolves the row the way
        /// its own variant says. The `Cancelled` one is also the node that
        /// [`super::JobQueue::cancel`] spares, since its edge is the only one
        /// that accepts a dropped dependency.
        outcome: TerminalState,
    },
    /// Tail node of a rebuild / perm-change: emit this agent's `Rebuilt` manager
    /// event — `ok` when its deps are `Done`, `!ok` carrying the failure note when
    /// they `Failed`, and **nothing at all** when they `Cancelled` (a cancelled DAG
    /// never ran, so there is no rebuild to report).
    ///
    /// One node per **agent** — a multi-agent DAG reports each agent's own
    /// outcome rather than painting all of them with the whole DAG's roll-up —
    /// and one per **outcome**: `ok` isn't computed here, it's which of the pair
    /// the graph let run.
    ///
    /// No cancel variant, deliberately: a DAG dropped before it started has no
    /// rebuild to report, and neither tail's edge accepts `Cancelled`, so both
    /// are cancelled with the rest and nothing is emitted.
    EmitRebuilt { agent: String, ok: bool },
    /// Write the agent's durable power intent (`wanted = Up` when `up`, else
    /// `Offline`) as a first-class DAG node, at the head of a power-op
    /// template so the downstream `Reconcile` reads it. Replaces the old
    /// pre-submit `set_wanted` side effect: the intent write is now part of
    /// the atomic DAG (crash-safe, per-agent — a multi-agent DAG carries one
    /// `SetWanted` per agent). Build-slot-exempt (a store write), but
    /// **lease-needing**: it takes the agent's lifecycle lease so the whole
    /// power-op DAG (intent write → reconcile) is atomic per-agent — two
    /// racing ops (e.g. restart vs stop) can't clobber each other's intent
    /// before either reconciles, which is the point of moving the write into
    /// the DAG. (In `stale_start` the lease is thus held across the head
    /// `Prebuild`, but that's a no-op there — the agent is down, so prebuild
    /// is skipped.)
    SetWanted { agent: String, up: bool },
    /// The **DAG container** node: one per submitted DAG, carrying the group's
    /// domain metadata. Every node hangs *under* it (its subtree), so
    /// the container's `NodeId` **is** the DAG id and its rolled-up state **is**
    /// the DAG state. Pure grouping — lease- and
    /// build-slot-exempt; the executor instant-completes it (`Done`) so it
    /// reaches `Finishing` and its children start.
    Dag {
        source: Source,
        reason: String,
        transient: Option<TransientKind>,
        inputs: Vec<String>,
        created_at: i64,
    },
}

impl NodeKind {
    /// Wire string for `NodeView.kind`.
    pub fn as_str(&self) -> &'static str {
        match self {
            NodeKind::MetaSync { .. } => "meta_sync",
            NodeKind::Prebuild { .. } => "prebuild",
            NodeKind::Swap { .. } => "swap",
            NodeKind::PostSwap { .. } => "post_swap",
            NodeKind::Provision { .. } => "provision",
            NodeKind::Create { .. } => "create",
            NodeKind::MetaLock { .. } => "meta_lock",
            NodeKind::Reconcile { .. } => "reconcile",
            NodeKind::Start { .. } => "start",
            NodeKind::Stop { .. } => "stop",
            NodeKind::StopForUpdate { .. } => "stop_for_update",
            NodeKind::Signal { .. } => "signal",
            NodeKind::Drain { .. } => "drain",
            NodeKind::WriteDropin { .. } => "write_dropin",
            NodeKind::WritePermFile { .. } => "write_perm_file",
            NodeKind::Reparent { .. } => "reparent",
            NodeKind::DeployWindow { .. } => "deploy_window",
            NodeKind::MergeVerify { .. } => "merge_verify",
            NodeKind::DeployApply { .. } => "deploy_apply",
            NodeKind::FinalizeDeploy { .. } => "finalize_deploy",
            NodeKind::DeployTail { .. } => "deploy_tail",
            NodeKind::ResolveApproval { .. } => "resolve_approval",
            NodeKind::EmitRebuilt { .. } => "emit_rebuilt",
            NodeKind::SetWanted { .. } => "set_wanted",
            NodeKind::Dag { .. } => "dag",
        }
    }

    /// The agent this node targets, or `""` for agentless kinds
    /// ([`NodeKind::MetaLock`] on the `hyperhive` pseudo-agent,
    /// [`NodeKind::Reparent`] which can span multiple agents, and the
    /// [`NodeKind::Dag`] container).
    #[must_use]
    pub fn agent(&self) -> &str {
        match self {
            NodeKind::MetaSync { agent, .. }
            | NodeKind::Prebuild { agent }
            | NodeKind::Swap { agent }
            | NodeKind::PostSwap { agent }
            | NodeKind::Provision { agent }
            | NodeKind::Create { agent }
            | NodeKind::Reconcile { agent }
            | NodeKind::Start { agent }
            | NodeKind::Stop { agent }
            | NodeKind::StopForUpdate { agent }
            | NodeKind::Signal { agent }
            | NodeKind::Drain { agent }
            | NodeKind::WriteDropin { agent }
            | NodeKind::WritePermFile { agent, .. }
            | NodeKind::DeployWindow { agent, .. }
            | NodeKind::MergeVerify { agent, .. }
            | NodeKind::DeployApply { agent, .. }
            | NodeKind::FinalizeDeploy { agent, .. }
            | NodeKind::DeployTail { agent, .. }
            | NodeKind::EmitRebuilt { agent, .. }
            | NodeKind::SetWanted { agent, .. } => agent,
            NodeKind::MetaLock { .. }
            | NodeKind::Reparent { .. }
            | NodeKind::ResolveApproval { .. }
            | NodeKind::Dag { .. } => "",
        }
    }

    /// Nix-heavy kinds hold one of the `buildSlots` semaphore permits
    /// for the node's duration.
    pub fn needs_build_slot(&self) -> bool {
        matches!(
            self,
            NodeKind::Prebuild { .. }
                | NodeKind::Swap { .. }
                | NodeKind::Create { .. }
                | NodeKind::MetaLock { .. }
                | NodeKind::DeployWindow { .. }
        )
    }

    /// Container-affecting kinds require the DAG to hold the agent's
    /// lifecycle lease (acquired at the first such node, held until the
    /// DAG is terminal). Lease-exempt kinds (`MetaSync`, `Prebuild`,
    /// `Provision`, `MetaLock`, `WritePermFile`, `Reparent`) touch the store / meta repo, not the
    /// running container — which is exactly why a `Prebuild` can overlap
    /// another DAG's work on the same agent. `Provision` precedes the
    /// container's existence entirely, so the lease is first taken at the
    /// `Create` node it feeds.
    pub fn needs_lease(&self) -> bool {
        matches!(
            self,
            NodeKind::Swap { .. }
                | NodeKind::Create { .. }
                | NodeKind::Reconcile { .. }
                | NodeKind::StopForUpdate { .. }
                | NodeKind::Signal { .. }
                | NodeKind::Drain { .. }
                | NodeKind::WriteDropin { .. }
                | NodeKind::DeployWindow { .. }
                | NodeKind::SetWanted { .. }
        )
    }

    /// Kinds that **mutate the meta repo** and so must hold the global
    /// [`Resource::MetaWindow`](super::resource::Resource::MetaWindow) for
    /// their duration: no two meta mutations may interleave, because a commit
    /// landing inside another node's staged `prepare_deploy`→`finalize_deploy`
    /// window would sweep the staged `flake.lock` into its own commit and
    /// neuter `abort_deploy`.
    ///
    /// This is the queue-primitive replacement for the former runtime
    /// `meta::exclusive()` mutex — same global serialisation, but held by the
    /// scheduler and therefore able to span a whole subtree, which a
    /// `MutexGuard` cannot.
    ///
    /// Note what is **not** here: [`NodeKind::Prebuild`]. The window must stay
    /// off the multi-minute toplevel build, which only *reads* the store — the
    /// old mutex was scoped to drop before it, and holding a cap-1 global
    /// across it would serialize every agent's rebuild behind every other's.
    /// That is why the meta preamble is its own [`NodeKind::MetaSync`] node,
    /// and why that node is a sibling rather than `Prebuild`'s parent (a
    /// resource held by a parent covers its whole subtree).
    ///
    /// Two of these kinds run *inside* a [`NodeKind::DeployWindow`]'s subtree
    /// (the appended rebuild's `MetaSync`, and [`NodeKind::FinalizeDeploy`]).
    /// They still declare the window: a descendant re-enters an ancestor's hold
    /// through the crate's recursive lock, exactly as `Start` / `Stop` re-enter
    /// a `Reconcile`'s agent lease. Declaring it is what keeps the requirement
    /// true of the *node* rather than of one particular DAG shape.
    pub fn needs_meta_window(&self) -> bool {
        matches!(
            self,
            NodeKind::MetaSync { .. }
                | NodeKind::Provision { .. }
                | NodeKind::MetaLock { .. }
                | NodeKind::WritePermFile { .. }
                | NodeKind::Reparent { .. }
                | NodeKind::DeployWindow { .. }
                | NodeKind::FinalizeDeploy { .. }
        )
    }
}

/// Submit-time spec for one node.
#[derive(Debug, Clone)]
pub struct NodeSpec {
    /// The node's payload — [`NodeKind`] is the queue's payload type directly,
    /// and each variant carries the agent it targets (a DAG can span agents;
    /// the queue derives per-agent leasing from [`NodeKind::agent`]).
    pub kind: NodeKind,
    pub deps: Vec<Dep>,
    /// The **structural parent** axis — the spec-local index of this node's
    /// group parent, or `None` for a top-level (group-root) node. Independent
    /// of `deps`: `deps` order execution, `parent` groups nodes into a subtree
    /// whose resource the whole subtree borrows (the agent lease is owned by a
    /// group root and re-entered by its descendants for continuity). A child
    /// runs once its parent reaches `Finishing` (the parent gate), so a child
    /// never `deps` on its own parent (that would deadlock — dep-scope
    /// validation rejects it).
    pub parent: Option<u64>,
}

/// Submit-time spec for a whole DAG. Built by `templates.rs`; validated
/// (cycle rejection) by `JobQueue::submit`. No DAG-level `agent` — every
/// node carries its own (a DAG can span agents), and the queue derives
/// per-agent leasing from [`NodeKind::agent`]. Type-specific payloads
/// (`PermChange`'s file payload) ride the node that consumes them
/// ([`NodeKind::WritePermFile`]), not this generic spec.
#[derive(Debug, Clone)]
pub struct DagSpec {
    pub source: Source,
    /// Free-form "why".
    pub reason: String,
    /// Meta-update only: the inputs to bump. Display copy lives on the DAG.
    pub inputs: Vec<String>,
    /// Dashboard transient pill (and crash-watch suppression) held for
    /// the lease window — from lease acquisition to DAG terminal.
    pub transient: Option<crate::coordinator::TransientKind>,
    pub nodes: Vec<NodeSpec>,
}