feat(hive-c0re): replace rebuild queue with generic job-DAG queue

jobs are now DAGs of primitive nodes (prebuild, stop-for-update, swap,
reconcile, signal, drain, ...) driven by one scheduler with N build
slots + per-agent lifecycle leases. per-agent power intent (wanted
up/offline) is durable in agent_power.sqlite; Reconcile nodes converge
observed state to it. kills the graceful-stop watcher thread, the
deferred-start follow-up, and the cascade pre-enqueue (fan-out on
MetaLock completion instead). tracker: #2166
This commit is contained in:
müde 2026-07-06 20:13:14 +02:00
commit 7946e03fde
25 changed files with 3673 additions and 2731 deletions

View file

@ -173,12 +173,17 @@ pub struct Coordinator {
/// tokio mutex so the rescan can `await` `lifecycle::list` /
/// `is_running` without blocking other coordinator paths.
last_containers: tokio::sync::Mutex<HashMap<String, ContainerView>>,
/// Global rebuild queue. Every long-running container/meta op
/// (rebuild, meta-update, first-spawn) goes through this queue so
/// hive-c0re runs at most one at a time and the dashboard can
/// render a single ordered view of pending + running work. See
/// `rebuild_queue.rs` for the dedup rules + history retention.
pub rebuild_queue: Arc<crate::rebuild_queue::RebuildQueue>,
/// Global job-DAG queue. Every container/meta op (rebuild,
/// meta-update, first-spawn, power changes) is submitted as a DAG
/// of primitive nodes; a single scheduler drives them with
/// build-slot + per-agent-lease gating so the dashboard renders one
/// ordered view of pending + running work. See `job_queue/` for
/// the dedup rules, resource classes, and history retention.
pub job_queue: Arc<crate::job_queue::JobQueue>,
/// Durable per-agent power intent (`wanted: Up | Offline`) — the
/// spec half of desired-state reconciliation; the queue's
/// `Reconcile` nodes converge observed state to it.
pub power: Arc<crate::power::PowerStore>,
/// Shutdown signal broadcast to all background tasks. Sending
/// `true` asks every loop to exit after its current work item.
/// Use `shutdown_rx()` to subscribe; `request_shutdown()` to fire.
@ -244,12 +249,27 @@ impl Default for HiveEnv {
/// instead of every host-level setting as its own JSON-blob argument.
/// `#[serde(default)]` lets any field be omitted and fall back to its
/// canonical default.
#[derive(Clone, Debug, Default, serde::Deserialize)]
#[derive(Clone, Debug, serde::Deserialize)]
#[serde(default)]
pub struct ServeConfig {
#[serde(flatten)]
pub env: HiveEnv,
pub model_prices: crate::hive_stats::PriceTable,
/// Number of concurrent nix-heavy job-queue nodes (prebuild /
/// profile-swap / create / meta lock). hive-c0re-local like
/// `model_prices` — never injected into containers. Set via
/// `services.hyperhive.c0re.buildSlots`.
pub build_slots: usize,
}
impl Default for ServeConfig {
fn default() -> Self {
Self {
env: HiveEnv::default(),
model_prices: crate::hive_stats::PriceTable::default(),
build_slots: 1,
}
}
}
#[cfg(test)]
@ -433,6 +453,7 @@ impl Coordinator {
db_path: &Path,
env: HiveEnv,
model_prices: crate::hive_stats::PriceTable,
build_slots: usize,
) -> Result<Self> {
let HiveEnv {
hyperhive_flake,
@ -469,6 +490,8 @@ impl Coordinator {
let audit_log =
Arc::new(crate::audit_log::AuditLog::open(build_logs_dir).context("open audit_log")?);
crate::audit_log::install(audit_log.clone());
let power =
Arc::new(crate::power::PowerStore::open(build_logs_dir).context("open agent_power")?);
let (dashboard_events, _) = broadcast::channel(DASHBOARD_CHANNEL);
let (shutdown_tx, _) = watch::channel(false);
Ok(Self {
@ -497,7 +520,8 @@ impl Coordinator {
event_seq: AtomicU64::new(0),
meta_updates_active: AtomicU64::new(0),
last_containers: tokio::sync::Mutex::new(HashMap::new()),
rebuild_queue: Arc::new(crate::rebuild_queue::RebuildQueue::new()),
job_queue: Arc::new(crate::job_queue::JobQueue::new(build_slots)),
power,
shutdown_tx,
})
}
@ -543,7 +567,7 @@ impl Coordinator {
/// wrappers below) and the worker so every state transition
/// surfaces on the dashboard without extra plumbing.
pub fn emit_rebuild_queue_snapshot(self: &Arc<Self>) {
let queue = self.rebuild_queue.snapshot();
let queue = self.job_queue.snapshot();
self.emit_dashboard_event(DashboardEvent::RebuildQueueChanged {
seq: self.next_seq(),
queue,
@ -665,15 +689,27 @@ impl Coordinator {
});
}
/// Update the `step` label on a running queue entry and (if it
/// actually changed) re-emit the queue snapshot so the dashboard
/// renders the new phase. Returns `true` when the label was new
/// and an emit fired, mostly for tracing/logging callers; safe to
/// ignore. No-op when `id` is `None` (e.g. callers that aren't
/// running from the queue worker) or when the row isn't `Running`.
/// Update the `step` label on the currently-running node of DAG
/// `id` and (if it actually changed) re-emit the queue snapshot so
/// the dashboard renders the new phase. DAG-id-only surface for
/// the opaque approval pipeline in `actions.rs`, whose callbacks
/// don't know node ids (its DAGs are single-node, so the lookup is
/// exact); queue executors use the precise per-node sink in
/// `job_queue::exec` instead. No-op when `id` is `None` (callers
/// not running from the queue) or when nothing is `Running`.
pub fn set_queue_step(self: &Arc<Self>, id: Option<u64>, step: &str) {
let Some(id) = id else { return };
if self.rebuild_queue.set_step(id, step) {
if self.job_queue.set_step_running(id, step) {
self.emit_rebuild_queue_snapshot();
}
}
/// Link a `build_logs` row to the currently-running node of DAG
/// `id` and re-emit the snapshot. Same DAG-id-only compatibility
/// surface as [`Self::set_queue_step`].
pub fn set_queue_build_log(self: &Arc<Self>, id: Option<u64>, log_id: i64) {
let Some(id) = id else { return };
if self.job_queue.set_build_log_id_running(id, log_id) {
self.emit_rebuild_queue_snapshot();
}
}