hyperhive/hive-c0re/src/job_queue/model.rs

//! Data model for the generic job-DAG queue: node kinds (the primitive
//! operations), dependency edges, and the runtime `Dag` / `Node` store.
//! The serialized *views* — `DagView` / `NodeView` plus the `Template`
//! / `Source` / `State` / `PermPayload` wire enums — live in
//! `hive_sh4re::jobs` (wire types belong to the shared crate) and are
//! re-exported here for the queue's internal use.
//!
//! Two levels: the **DAG** is the unit of cancel / approval-resolution
//! and the dashboard group; the **node** is the unit of scheduling /
//! execution / build-log / step label, and carries its own `agent` (a
//! DAG can span agents). See `docs/coordinator.md::Job queue` for the
//! full design.

pub use hive_sh4re::jobs::{DagView, NodeId, NodeView, PermPayload, Source, State, Template};
use serde::Serialize;

/// When a dependency edge is considered satisfied.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum DepWhen {
    /// Dep must reach `Done`. A `Failed` / `Cancelled` dep cancels this
    /// node (cancel-downstream).
    AfterOk,
    /// Dep must merely reach a terminal state (ok *or* fail). Used only
    /// by `rebuild`'s tail `Reconcile` so the recovery-start runs even
    /// when `Swap` failed.
    AfterAny,
}

/// A dependency edge (intra-DAG only — cross-DAG ordering comes from
/// the per-agent lease + dedup, never from edges between DAGs).
#[derive(Debug, Clone, Copy, Serialize)]
pub struct Dep {
    pub on: NodeId,
    pub when: DepWhen,
}

/// The primitive operations — each kind maps to one executor fn in
/// `exec.rs`, a thin wrapper over existing `lifecycle.rs` / `meta.rs`
/// code. Concurrency is gated by two resource classes (see
/// [`NodeKind::needs_build_slot`] / [`NodeKind::needs_lease`]); the
/// meta *repo* is serialized by `meta::META_LOCK` inside the wrapped
/// functions themselves, which is why there is no `GitCommit` node —
/// a standalone commit node would open a dirty-working-tree window
/// between nodes that the fused `meta.rs` ops deliberately close.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum NodeKind {
    /// Out-of-band toplevel build while the container keeps serving:
    /// meta `sync_agents`, optional per-agent relock, then
    /// `lifecycle::prebuild_toplevel`. `relock = false` only for
    /// meta-update cascade rebuilds (re-locking would revert the bump
    /// the cascade just committed). The `prebuild_toplevel` warm is
    /// skipped when the container is already down — it only exists to
    /// shrink the swap's downtime, which a stopped agent doesn't need
    /// (the sync + dir prep still run; `Swap` builds inline).
    Prebuild { relock: bool },
    /// `nixos-container update` profile-swap (requires the container
    /// stopped). Re-applies nspawn flags + resource limits first —
    /// rebuild is the reconcile verb — and carries the post-rebuild
    /// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan).
    Swap,
    /// First-spawn pre-create provisioning: proposed/applied repos,
    /// state subvolume, and meta registration (`sync_agents`). Runs
    /// ahead of `Create` so the `nixos-container create --flake
    /// meta#<name>` ref resolves. Store/meta-only — no container yet —
    /// so it's lease- and build-slot-exempt like `Prebuild`.
    Provision,
    /// First-spawn `nixos-container create` proper. Assumes the
    /// upstream `Provision` node already registered the agent in meta.
    Create,
    /// Meta flake lock bump. `sweep = false`: `meta::lock_update`
    /// (commit fused, under `META_LOCK`) with the DAG's `inputs`;
    /// `sweep = true`: `meta::lock_update_hyperhive`, *non-fatal* (a
    /// failed boot-time bump must not cancel the fan-out rebuilds).
    /// On success the scheduler appends child `Rebuild` DAGs: the
    /// precomputed `fanout` list when present (boot sweep), else the
    /// post-bump affected set (`meta_update_cascade_agents`).
    MetaLock {
        sweep: bool,
        fanout: Option<Vec<String>>,
    },
    /// Idempotent power converge *planner*: read `wanted` + observed
    /// state and decide the action (start if `Up` & down, stop if
    /// `Offline` & up, else noop). The mechanical work is not done in
    /// this node — it fans a child [`NodeKind::Start`] / [`NodeKind::Stop`]
    /// DAG out at runtime so the sub-step is a first-class DAG node.
    Reconcile,
    /// Mechanical container start: the start preamble (runtime dir +
    /// drop-ins), `start_with_fallback`, MCP listener registration, and
    /// the manager kick. Fanned out by a [`NodeKind::Reconcile`] that
    /// observed `wanted = Up` and the container down.
    Start,
    /// Mechanical container stop: `nixos-container` kill, MCP listener
    /// unregister, and the `Killed` manager notify. Fanned out by a
    /// [`NodeKind::Reconcile`] that observed `wanted = Offline` and up.
    Stop,
    /// Mechanical `nixos-container stop` for the profile swap. Never
    /// touches `wanted`. Noop if already stopped.
    StopForUpdate,
    /// Set the graceful-stop fence + kick the harness so it runs one
    /// stop-checkpoint turn.
    Signal,
    /// Await the harness clearing the fence, bounded by
    /// `GRACEFUL_STOP_TIMEOUT`. Resolves ok either way — the
    /// downstream `Reconcile` performs the actual stop.
    Drain,
    /// `set_nspawn_flags` + `set_resource_limits` + daemon-reload.
    WriteDropin,
    /// Commit `tool-groups.json` / `capabilities.json` per the DAG's
    /// `perm_payload` (commit fused under `META_LOCK`).
    WritePermFile,
    /// Opaque approval deploy pipeline (`ApplyCommit` /
    /// `MergeConfigPr`): the two-phase prepare/finalize/abort meta
    /// deploy stays inside `actions.rs` in v1 — deliberately not
    /// modeled as scheduler nodes (see the design doc §9).
    ApprovalDeploy,
    /// No-op anchor that completes immediately with no work. It exists so a
    /// boot's `StartupSweep` + per-agent `Reconcile` child DAGs can hang off
    /// one root (`parent_id`) and render as a single boot tree on the
    /// dashboard. Holds no lease and no build slot; the child DAGs it anchors
    /// still run concurrently — the grouping is a display link, not a dep edge.
    Noop,
    /// Write the agent's durable power intent (`wanted = Up` when `up`, else
    /// `Offline`) as a first-class DAG node, at the head of a power-op
    /// template so the downstream `Reconcile` reads it. Replaces the old
    /// pre-submit `set_wanted` side effect: the intent write is now part of
    /// the atomic DAG (crash-safe, per-agent — a multi-agent DAG carries one
    /// `SetWanted` per agent). Build-slot-exempt (a store write), but
    /// **lease-needing**: it takes the agent's lifecycle lease so the whole
    /// power-op DAG (intent write → reconcile) is atomic per-agent — two
    /// racing ops (e.g. restart vs stop) can't clobber each other's intent
    /// before either reconciles, which is the point of moving the write into
    /// the DAG. (In `stale_start` the lease is thus held across the head
    /// `Prebuild`, but that's a no-op there — the agent is down, so prebuild
    /// is skipped.)
    SetWanted { up: bool },
}

impl NodeKind {
    /// Wire string for `NodeView.kind`.
    pub fn as_str(&self) -> &'static str {
        match self {
            NodeKind::Prebuild { .. } => "prebuild",
            NodeKind::Swap => "swap",
            NodeKind::Provision => "provision",
            NodeKind::Create => "create",
            NodeKind::MetaLock { .. } => "meta_lock",
            NodeKind::Reconcile => "reconcile",
            NodeKind::Start => "start",
            NodeKind::Stop => "stop",
            NodeKind::StopForUpdate => "stop_for_update",
            NodeKind::Signal => "signal",
            NodeKind::Drain => "drain",
            NodeKind::WriteDropin => "write_dropin",
            NodeKind::WritePermFile => "write_perm_file",
            NodeKind::ApprovalDeploy => "approval_deploy",
            NodeKind::Noop => "noop",
            NodeKind::SetWanted { .. } => "set_wanted",
        }
    }

    /// Nix-heavy kinds hold one of the `buildSlots` semaphore permits
    /// for the node's duration.
    pub fn needs_build_slot(&self) -> bool {
        matches!(
            self,
            NodeKind::Prebuild { .. }
                | NodeKind::Swap
                | NodeKind::Create
                | NodeKind::MetaLock { .. }
                | NodeKind::ApprovalDeploy
        )
    }

    /// Container-affecting kinds require the DAG to hold the agent's
    /// lifecycle lease (acquired at the first such node, held until the
    /// DAG is terminal). Lease-exempt kinds (`Prebuild`, `Provision`,
    /// `MetaLock`, `WritePermFile`) touch the store / meta repo, not the
    /// running container — which is exactly why a `Prebuild` can overlap
    /// another DAG's work on the same agent. `Provision` precedes the
    /// container's existence entirely, so the lease is first taken at the
    /// `Create` node it feeds.
    pub fn needs_lease(&self) -> bool {
        matches!(
            self,
            NodeKind::Swap
                | NodeKind::Create
                | NodeKind::Reconcile
                | NodeKind::StopForUpdate
                | NodeKind::Signal
                | NodeKind::Drain
                | NodeKind::WriteDropin
                | NodeKind::ApprovalDeploy
                | NodeKind::SetWanted { .. }
        )
    }
}

/// One schedulable unit inside a DAG.
#[derive(Debug, Clone)]
pub struct Node {
    pub id: NodeId,
    /// The agent this node's work targets. Per-node so a single DAG can
    /// span agents (e.g. a hive-wide restart); the lifecycle lease is
    /// acquired against *this* agent (still globally exclusive per agent
    /// across all DAGs). `"hyperhive"` for meta-level nodes.
    pub agent: String,
    pub kind: NodeKind,
    pub deps: Vec<Dep>,
    pub state: State,
    /// Live sub-label while `Running` (kept for parity with the old
    /// per-entry `step`).
    pub step: Option<String>,
    /// Row id of the `build_logs` entry this node opened (`Prebuild` /
    /// `Swap` / `ApprovalDeploy`), for the dashboard's live-stream link.
    pub build_log_id: Option<i64>,
    pub started_at: Option<i64>,
    pub finished_at: Option<i64>,
    /// Populated when `state == Failed` (truncated by the queue).
    pub error: Option<String>,
}

/// Submit-time spec for one node.
#[derive(Debug, Clone)]
pub struct NodeSpec {
    /// The agent this node targets (see [`Node::agent`]). Built by the
    /// `templates.rs` `node` helper, which stamps the template's agent
    /// onto every node.
    pub agent: String,
    pub kind: NodeKind,
    pub deps: Vec<Dep>,
}

/// Submit-time spec for a whole DAG. Built by `templates.rs`; validated
/// (cycle rejection) by `JobQueue::submit`. No DAG-level `agent` — every
/// node carries its own (a DAG can span agents), and the queue derives
/// per-agent leasing from [`NodeSpec::agent`].
#[derive(Debug, Clone)]
pub struct DagSpec {
    pub template: Template,
    pub source: Source,
    /// Free-form "why".
    pub reason: String,
    /// Cascade grouping (meta-update / sweep children).
    pub parent_id: Option<u64>,
    /// Fires the approval-resolution hook on DAG terminal.
    pub approval_id: Option<i64>,
    /// `MetaUpdate`-only: the inputs to bump (also part of the dedup
    /// key for that template). Display copy lives on the DAG.
    pub inputs: Vec<String>,
    /// `PermChange`-only payload.
    pub perm_payload: Option<PermPayload>,
    /// Dashboard transient pill (and crash-watch suppression) held for
    /// the lease window — from lease acquisition to DAG terminal.
    pub transient: Option<crate::coordinator::TransientKind>,
    pub nodes: Vec<NodeSpec>,
}

/// A live DAG in the queue. No DAG-level `agent`: agent is per-[`Node`],
/// so a DAG can span agents. Per-agent leasing is derived from the
/// nodes' agents.
#[derive(Debug, Clone)]
pub struct Dag {
    pub id: u64,
    pub template: Template,
    pub source: Source,
    pub reason: String,
    pub parent_id: Option<u64>,
    pub approval_id: Option<i64>,
    pub inputs: Vec<String>,
    pub perm_payload: Option<PermPayload>,
    pub transient: Option<crate::coordinator::TransientKind>,
    pub created_at: i64,
    pub nodes: Vec<Node>,
    /// Terminal roll-up already reported to the scheduler's hooks
    /// (approval resolution, transient release). Internal bookkeeping,
    /// never serialized.
    pub terminal_reported: bool,
}

impl Dag {
    /// Roll-up state: `Failed` if any node failed; else `Running` if
    /// any running; else `Queued` if any queued; else `Cancelled` if
    /// any cancelled; else `Done`.
    pub fn rollup(&self) -> State {
        let mut any_cancelled = false;
        let mut any_queued = false;
        let mut any_running = false;
        for n in &self.nodes {
            match n.state {
                State::Failed => return State::Failed,
                State::Running => any_running = true,
                State::Queued => any_queued = true,
                State::Cancelled => any_cancelled = true,
                State::Done => {}
            }
        }
        if any_running {
            State::Running
        } else if any_queued {
            State::Queued
        } else if any_cancelled {
            State::Cancelled
        } else {
            State::Done
        }
    }

    /// True when every node is terminal.
    pub fn is_terminal(&self) -> bool {
        self.nodes.iter().all(|n| n.state.is_terminal())
    }

    /// First failed node's error, for the roll-up `error` field.
    pub fn first_error(&self) -> Option<&str> {
        self.nodes
            .iter()
            .find(|n| n.state == State::Failed)
            .and_then(|n| n.error.as_deref())
    }

    pub fn node(&self, id: NodeId) -> Option<&Node> {
        self.nodes.iter().find(|n| n.id == id)
    }

    pub fn node_mut(&mut self, id: NodeId) -> Option<&mut Node> {
        self.nodes.iter_mut().find(|n| n.id == id)
    }

    /// Distinct agents this DAG's nodes target, in first-seen order.
    /// Used for per-agent lease release and the terminal cancel-revert —
    /// a single-agent DAG yields one, a multi-agent DAG yields several.
    pub fn agents(&self) -> Vec<String> {
        let mut seen: Vec<String> = Vec::new();
        for n in &self.nodes {
            if !seen.iter().any(|a| a == &n.agent) {
                seen.push(n.agent.clone());
            }
        }
        seen
    }
}

impl Dag {
    pub fn view(&self) -> DagView {
        let started_at = self.nodes.iter().filter_map(|n| n.started_at).min();
        let finished_at = if self.is_terminal() {
            self.nodes.iter().filter_map(|n| n.finished_at).max()
        } else {
            None
        };
        DagView {
            id: self.id,
            kind: self.template,
            state: self.rollup(),
            source: self.source,
            parent_id: self.parent_id,
            reason: self.reason.clone(),
            enqueued_at: self.created_at,
            started_at,
            finished_at,
            inputs: self.inputs.clone(),
            approval_id: self.approval_id,
            perm_payload: self.perm_payload.clone(),
            nodes: self
                .nodes
                .iter()
                .map(|n| NodeView {
                    id: n.id,
                    agent: n.agent.clone(),
                    kind: n.kind.as_str().to_owned(),
                    deps: n.deps.iter().map(|d| d.on).collect(),
                    state: n.state,
                    step: n.step.clone(),
                    build_log_id: n.build_log_id,
                    started_at: n.started_at,
                    finished_at: n.finished_at,
                    error: n.error.clone(),
                })
                .collect(),
        }
    }
}