feat(hive-c0re): replace rebuild queue with generic job-DAG queue

jobs are now DAGs of primitive nodes (prebuild, stop-for-update, swap,
reconcile, signal, drain, ...) driven by one scheduler with N build
slots + per-agent lifecycle leases. per-agent power intent (wanted
up/offline) is durable in agent_power.sqlite; Reconcile nodes converge
observed state to it. kills the graceful-stop watcher thread, the
deferred-start follow-up, and the cascade pre-enqueue (fan-out on
MetaLock completion instead). tracker: #2166

hive-c0re/src/job_queue/model.rs

@@ -0,0 +1,520 @@
+//! Data model for the generic job-DAG queue: templates (what a DAG
+//! *means*), node kinds (the primitive operations), dependency edges,
+//! states, and the wire-facing `DagView` / `NodeView` snapshot shapes.
+//!
+//! Two levels: the **DAG** is the unit of dedup / cancel /
+//! approval-resolution and the dashboard group; the **node** is the
+//! unit of scheduling / execution / build-log / step label. See
+//! `docs/coordinator.md::Job queue` for the full design.
+
+use serde::{Deserialize, Serialize};
+
+/// What a DAG *means* — the request-level shape. Wire strings match the
+/// old `QueueKind` values (serialized as the `kind` field on `DagView`)
+/// so the dashboard's glyph map keeps working.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Template {
+    /// Rebuild one agent's container: `Prebuild → StopForUpdate → Swap
+    /// → Reconcile` (the tail `Reconcile` runs after `Swap` terminal,
+    /// ok *or* fail — the recovery-start).
+    Rebuild,
+    /// Bump meta flake locks: one `MetaLock` node; child `Rebuild`
+    /// DAGs fan out on completion for every affected agent.
+    MetaUpdate,
+    /// First-deploy spawn (approval-driven): `Create → WriteDropin →
+    /// Reconcile`.
+    Spawn,
+    /// Reserved for a future destroy integration — kept so the wire
+    /// shape doesn't need to change later.
+    #[allow(dead_code, reason = "wire shape — routed by a future PR")]
+    Destroy,
+    /// Boot-time config sweep: one `MetaLock` (hyperhive input,
+    /// non-fatal) node; stale agents' `Rebuild` DAGs fan out on
+    /// completion.
+    StartupSweep,
+    /// `StopForUpdate → Reconcile` — stop + converge back to `wanted`
+    /// (unchanged), i.e. a restart for a wanted-up agent.
+    Restart,
+    /// `WritePermFile → Prebuild → StopForUpdate → Swap → Reconcile` —
+    /// perm-file commit followed by the rebuild subgraph.
+    PermChange,
+    /// `Signal → Drain → Reconcile` with `wanted` set to `Offline` at
+    /// submit time: quiesce the harness, await the drain (bounded),
+    /// then the tail `Reconcile` performs the actual container stop.
+    GracefulStop,
+    /// Single `Reconcile` with `wanted` set to `Up` at submit time.
+    Start,
+    /// Single `Reconcile` with `wanted` set to `Offline` at submit time.
+    Stop,
+    /// Single `Reconcile` with `wanted` untouched — boot-time converge
+    /// of observed state to the persisted intent.
+    Reconcile,
+}
+
+impl Template {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Template::Rebuild => "rebuild",
+            Template::MetaUpdate => "meta_update",
+            Template::Spawn => "spawn",
+            Template::Destroy => "destroy",
+            Template::StartupSweep => "startup_sweep",
+            Template::Restart => "restart",
+            Template::PermChange => "perm_change",
+            Template::GracefulStop => "graceful_stop",
+            Template::Start => "start",
+            Template::Stop => "stop",
+            Template::Reconcile => "reconcile",
+        }
+    }
+}
+
+/// Where the submit request originated. Same variants + wire strings
+/// as the old `QueueSource` — drives the "why" chip on the dashboard.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Source {
+    /// Operator action (dashboard button, CLI, manager tool).
+    Manual,
+    /// Cascade child of a `MetaUpdate` DAG's fan-out; `parent_id`
+    /// points back at the originating meta-update.
+    MetaUpdate,
+    /// Boot-time submission (sweep parent, boot reconciles).
+    AutoUpdate,
+    /// Cascade child of a `StartupSweep` DAG's fan-out.
+    StartupSweep,
+    /// Crash recovery path (future use).
+    #[allow(dead_code, reason = "wire shape — used by a future feature")]
+    CrashRecover,
+    /// Operator approved a pending `Approval` row; `approval_id` on
+    /// the DAG points back at the source row.
+    Approval,
+}
+
+impl Source {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Source::Manual => "manual",
+            Source::MetaUpdate => "meta_update",
+            Source::AutoUpdate => "auto_update",
+            Source::StartupSweep => "startup_sweep",
+            Source::CrashRecover => "crash_recover",
+            Source::Approval => "approval",
+        }
+    }
+}
+
+/// Lifecycle state of a node — and, rolled up, of a DAG. Same wire
+/// strings as the old `QueueState`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum State {
+    Queued,
+    Running,
+    Done,
+    Failed,
+    Cancelled,
+}
+
+impl State {
+    pub fn is_terminal(self) -> bool {
+        matches!(self, State::Done | State::Failed | State::Cancelled)
+    }
+}
+
+/// Kind-specific payload for `Template::PermChange` DAGs. Carried on
+/// the DAG (not the node) so dedup can compare the perm *type*
+/// discriminant. Identical to the old `rebuild_queue::PermPayload`.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum PermPayload {
+    /// Set the tool groups for one agent (`tool-groups.json`).
+    ToolGroups { groups: Vec<String> },
+    /// Set the capabilities for one agent (`capabilities.json`).
+    Capabilities { caps: Vec<String> },
+    /// Set both perm-types in one entry — the batch
+    /// `POST /api/permissions` path. `None` leaves that file untouched;
+    /// the worker commits whichever are present in one git commit,
+    /// then rebuilds once.
+    Combined {
+        groups: Option<Vec<String>>,
+        caps: Option<Vec<String>>,
+    },
+}
+
+impl PermPayload {
+    /// Dedup compares the perm *type*, not the value — a tool-groups
+    /// change and a capabilities change for the same agent are
+    /// distinct operations that must not collapse.
+    pub fn same_type(a: Option<&PermPayload>, b: Option<&PermPayload>) -> bool {
+        matches!(
+            (a, b),
+            (
+                Some(PermPayload::ToolGroups { .. }),
+                Some(PermPayload::ToolGroups { .. })
+            ) | (
+                Some(PermPayload::Capabilities { .. }),
+                Some(PermPayload::Capabilities { .. })
+            ) | (
+                Some(PermPayload::Combined { .. }),
+                Some(PermPayload::Combined { .. })
+            ) | (None, None)
+        )
+    }
+}
+
+/// Node id, unique within its DAG (dense small ints assigned by the
+/// template builders).
+pub type NodeId = u32;
+
+/// When a dependency edge is considered satisfied.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum DepWhen {
+    /// Dep must reach `Done`. A `Failed` / `Cancelled` dep cancels this
+    /// node (cancel-downstream).
+    AfterOk,
+    /// Dep must merely reach a terminal state (ok *or* fail). Used only
+    /// by `rebuild`'s tail `Reconcile` so the recovery-start runs even
+    /// when `Swap` failed.
+    AfterAny,
+}
+
+/// A dependency edge (intra-DAG only — cross-DAG ordering comes from
+/// the per-agent lease + dedup, never from edges between DAGs).
+#[derive(Debug, Clone, Copy, Serialize)]
+pub struct Dep {
+    pub on: NodeId,
+    pub when: DepWhen,
+}
+
+/// The primitive operations — each kind maps to one executor fn in
+/// `exec.rs`, a thin wrapper over existing `lifecycle.rs` / `meta.rs`
+/// code. Concurrency is gated by two resource classes (see
+/// [`NodeKind::needs_build_slot`] / [`NodeKind::needs_lease`]); the
+/// meta *repo* is serialized by `meta::META_LOCK` inside the wrapped
+/// functions themselves, which is why there is no `GitCommit` node —
+/// a standalone commit node would open a dirty-working-tree window
+/// between nodes that the fused `meta.rs` ops deliberately close.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(tag = "kind", rename_all = "snake_case")]
+pub enum NodeKind {
+    /// Out-of-band toplevel build while the container keeps serving:
+    /// meta `sync_agents`, optional per-agent relock, then
+    /// `lifecycle::prebuild_toplevel`. `relock = false` only for
+    /// meta-update cascade rebuilds (re-locking would revert the bump
+    /// the cascade just committed).
+    Prebuild { relock: bool },
+    /// `nixos-container update` profile-swap (requires the container
+    /// stopped). Re-applies nspawn flags + resource limits first —
+    /// rebuild is the reconcile verb — and carries the post-rebuild
+    /// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan).
+    Swap,
+    /// First-spawn `nixos-container create` plus the pre-create
+    /// provisioning (proposed/applied repos, state subvolume, meta
+    /// registration).
+    Create,
+    /// Meta flake lock bump. `sweep = false`: `meta::lock_update`
+    /// (commit fused, under `META_LOCK`) with the DAG's `inputs`;
+    /// `sweep = true`: `meta::lock_update_hyperhive`, *non-fatal* (a
+    /// failed boot-time bump must not cancel the fan-out rebuilds).
+    /// On success the scheduler appends child `Rebuild` DAGs: the
+    /// precomputed `fanout` list when present (boot sweep), else the
+    /// post-bump affected set (`meta_update_cascade_agents`).
+    MetaLock {
+        sweep: bool,
+        fanout: Option<Vec<String>>,
+    },
+    /// Idempotent power converge: read `wanted` + observed state;
+    /// start if `Up` & down (with cold-start fallback), stop if
+    /// `Offline` & up, else noop.
+    Reconcile,
+    /// Mechanical `nixos-container stop` for the profile swap. Never
+    /// touches `wanted`. Noop if already stopped.
+    StopForUpdate,
+    /// Set the graceful-stop fence + kick the harness so it runs one
+    /// stop-checkpoint turn.
+    Signal,
+    /// Await the harness clearing the fence, bounded by
+    /// `GRACEFUL_STOP_TIMEOUT`. Resolves ok either way — the
+    /// downstream `Reconcile` performs the actual stop.
+    Drain,
+    /// `set_nspawn_flags` + `set_resource_limits` + daemon-reload.
+    WriteDropin,
+    /// Commit `tool-groups.json` / `capabilities.json` per the DAG's
+    /// `perm_payload` (commit fused under `META_LOCK`).
+    WritePermFile,
+    /// Opaque approval deploy pipeline (`ApplyCommit` /
+    /// `MergeConfigPr`): the two-phase prepare/finalize/abort meta
+    /// deploy stays inside `actions.rs` in v1 — deliberately not
+    /// modeled as scheduler nodes (see the design doc §9).
+    ApprovalDeploy,
+}
+
+impl NodeKind {
+    /// Wire string for `NodeView.kind`.
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            NodeKind::Prebuild { .. } => "prebuild",
+            NodeKind::Swap => "swap",
+            NodeKind::Create => "create",
+            NodeKind::MetaLock { .. } => "meta_lock",
+            NodeKind::Reconcile => "reconcile",
+            NodeKind::StopForUpdate => "stop_for_update",
+            NodeKind::Signal => "signal",
+            NodeKind::Drain => "drain",
+            NodeKind::WriteDropin => "write_dropin",
+            NodeKind::WritePermFile => "write_perm_file",
+            NodeKind::ApprovalDeploy => "approval_deploy",
+        }
+    }
+
+    /// Nix-heavy kinds hold one of the `buildSlots` semaphore permits
+    /// for the node's duration.
+    pub fn needs_build_slot(&self) -> bool {
+        matches!(
+            self,
+            NodeKind::Prebuild { .. }
+                | NodeKind::Swap
+                | NodeKind::Create
+                | NodeKind::MetaLock { .. }
+                | NodeKind::ApprovalDeploy
+        )
+    }
+
+    /// Container-affecting kinds require the DAG to hold the agent's
+    /// lifecycle lease (acquired at the first such node, held until the
+    /// DAG is terminal). Lease-exempt kinds (`Prebuild`, `MetaLock`,
+    /// `WritePermFile`) touch the store / meta repo, not the running
+    /// container — which is exactly why a `Prebuild` can overlap
+    /// another DAG's work on the same agent.
+    pub fn needs_lease(&self) -> bool {
+        matches!(
+            self,
+            NodeKind::Swap
+                | NodeKind::Create
+                | NodeKind::Reconcile
+                | NodeKind::StopForUpdate
+                | NodeKind::Signal
+                | NodeKind::Drain
+                | NodeKind::WriteDropin
+                | NodeKind::ApprovalDeploy
+        )
+    }
+}
+
+/// One schedulable unit inside a DAG.
+#[derive(Debug, Clone)]
+pub struct Node {
+    pub id: NodeId,
+    pub kind: NodeKind,
+    pub deps: Vec<Dep>,
+    pub state: State,
+    /// Live sub-label while `Running` (kept for parity with the old
+    /// per-entry `step`).
+    pub step: Option<String>,
+    /// Row id of the `build_logs` entry this node opened (`Prebuild` /
+    /// `Swap` / `ApprovalDeploy`), for the dashboard's live-stream link.
+    pub build_log_id: Option<i64>,
+    pub started_at: Option<i64>,
+    pub finished_at: Option<i64>,
+    /// Populated when `state == Failed` (truncated by the queue).
+    pub error: Option<String>,
+}
+
+/// Submit-time spec for one node.
+#[derive(Debug, Clone)]
+pub struct NodeSpec {
+    pub kind: NodeKind,
+    pub deps: Vec<Dep>,
+}
+
+/// Submit-time spec for a whole DAG. Built by `templates.rs`; validated
+/// (cycle rejection) and dedup'd by `JobQueue::submit`.
+#[derive(Debug, Clone)]
+pub struct DagSpec {
+    pub template: Template,
+    /// Primary target agent, or `"hyperhive"` for meta-level DAGs.
+    pub agent: String,
+    pub source: Source,
+    /// Free-form "why"; dedup appends "also requested by …" lines.
+    pub reason: String,
+    /// Cascade grouping (meta-update / sweep children).
+    pub parent_id: Option<u64>,
+    /// Fires the approval-resolution hook on DAG terminal.
+    pub approval_id: Option<i64>,
+    /// `MetaUpdate`-only: the inputs to bump (also part of the dedup
+    /// key for that template). Display copy lives on the DAG.
+    pub inputs: Vec<String>,
+    /// `PermChange`-only payload.
+    pub perm_payload: Option<PermPayload>,
+    /// Dashboard transient pill (and crash-watch suppression) held for
+    /// the lease window — from lease acquisition to DAG terminal.
+    pub transient: Option<crate::coordinator::TransientKind>,
+    pub nodes: Vec<NodeSpec>,
+}
+
+/// A live DAG in the queue.
+#[derive(Debug, Clone)]
+pub struct Dag {
+    pub id: u64,
+    pub template: Template,
+    pub agent: String,
+    pub source: Source,
+    pub reason: String,
+    pub parent_id: Option<u64>,
+    pub approval_id: Option<i64>,
+    pub inputs: Vec<String>,
+    pub perm_payload: Option<PermPayload>,
+    pub transient: Option<crate::coordinator::TransientKind>,
+    pub created_at: i64,
+    pub nodes: Vec<Node>,
+    /// Terminal roll-up already reported to the scheduler's hooks
+    /// (approval resolution, transient release). Internal bookkeeping,
+    /// never serialized.
+    pub terminal_reported: bool,
+}
+
+impl Dag {
+    /// Roll-up state: `Failed` if any node failed; else `Running` if
+    /// any running; else `Queued` if any queued; else `Cancelled` if
+    /// any cancelled; else `Done`.
+    pub fn rollup(&self) -> State {
+        let mut any_cancelled = false;
+        let mut any_queued = false;
+        let mut any_running = false;
+        for n in &self.nodes {
+            match n.state {
+                State::Failed => return State::Failed,
+                State::Running => any_running = true,
+                State::Queued => any_queued = true,
+                State::Cancelled => any_cancelled = true,
+                State::Done => {}
+            }
+        }
+        if any_running {
+            State::Running
+        } else if any_queued {
+            State::Queued
+        } else if any_cancelled {
+            State::Cancelled
+        } else {
+            State::Done
+        }
+    }
+
+    /// True when every node is terminal.
+    pub fn is_terminal(&self) -> bool {
+        self.nodes.iter().all(|n| n.state.is_terminal())
+    }
+
+    /// First failed node's error, for the roll-up `error` field.
+    pub fn first_error(&self) -> Option<&str> {
+        self.nodes
+            .iter()
+            .find(|n| n.state == State::Failed)
+            .and_then(|n| n.error.as_deref())
+    }
+
+    pub fn node(&self, id: NodeId) -> Option<&Node> {
+        self.nodes.iter().find(|n| n.id == id)
+    }
+
+    pub fn node_mut(&mut self, id: NodeId) -> Option<&mut Node> {
+        self.nodes.iter_mut().find(|n| n.id == id)
+    }
+}
+
+/// Wire shape of one node inside a `DagView`.
+#[derive(Debug, Clone, Serialize)]
+pub struct NodeView {
+    pub id: NodeId,
+    /// Flattened `NodeKind` tag ("prebuild", "swap", …).
+    pub kind: &'static str,
+    /// Ids of the nodes this one waits for.
+    pub deps: Vec<NodeId>,
+    pub state: State,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub step: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub build_log_id: Option<i64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub started_at: Option<i64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub finished_at: Option<i64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub error: Option<String>,
+}
+
+/// Wire shape of a DAG, serialized onto `RebuildQueueChanged` and the
+/// `/api/state` snapshot. DAG-level fields mirror the old `QueueEntry`
+/// names (`kind` = template string, roll-up `state`); everything
+/// per-node — step labels, build-log links, errors, timestamps —
+/// appears exactly once, inside `nodes`.
+#[derive(Debug, Clone, Serialize)]
+pub struct DagView {
+    pub id: u64,
+    pub agent: String,
+    /// Template wire string — same values the old `kind` field used.
+    pub kind: Template,
+    /// Roll-up state (see [`Dag::rollup`]).
+    pub state: State,
+    pub source: Source,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub parent_id: Option<u64>,
+    pub reason: String,
+    pub enqueued_at: i64,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub started_at: Option<i64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub finished_at: Option<i64>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub inputs: Vec<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub approval_id: Option<i64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub perm_payload: Option<PermPayload>,
+    pub nodes: Vec<NodeView>,
+}
+
+impl Dag {
+    pub fn view(&self) -> DagView {
+        let started_at = self.nodes.iter().filter_map(|n| n.started_at).min();
+        let finished_at = if self.is_terminal() {
+            self.nodes.iter().filter_map(|n| n.finished_at).max()
+        } else {
+            None
+        };
+        DagView {
+            id: self.id,
+            agent: self.agent.clone(),
+            kind: self.template,
+            state: self.rollup(),
+            source: self.source,
+            parent_id: self.parent_id,
+            reason: self.reason.clone(),
+            enqueued_at: self.created_at,
+            started_at,
+            finished_at,
+            inputs: self.inputs.clone(),
+            approval_id: self.approval_id,
+            perm_payload: self.perm_payload.clone(),
+            nodes: self
+                .nodes
+                .iter()
+                .map(|n| NodeView {
+                    id: n.id,
+                    kind: n.kind.as_str(),
+                    deps: n.deps.iter().map(|d| d.on).collect(),
+                    state: n.state,
+                    step: n.step.clone(),
+                    build_log_id: n.build_log_id,
+                    started_at: n.started_at,
+                    finished_at: n.finished_at,
+                    error: n.error.clone(),
+                })
+                .collect(),
+        }
+    }
+}