feat(hive-c0re): replace rebuild queue with generic job-DAG queue
jobs are now DAGs of primitive nodes (prebuild, stop-for-update, swap, reconcile, signal, drain, ...) driven by one scheduler with N build slots + per-agent lifecycle leases. per-agent power intent (wanted up/offline) is durable in agent_power.sqlite; Reconcile nodes converge observed state to it. kills the graceful-stop watcher thread, the deferred-start follow-up, and the cascade pre-enqueue (fan-out on MetaLock completion instead). tracker: #2166
This commit is contained in:
parent
79a3993def
commit
7946e03fde
25 changed files with 3673 additions and 2731 deletions
|
|
@ -173,12 +173,17 @@ pub struct Coordinator {
|
|||
/// tokio mutex so the rescan can `await` `lifecycle::list` /
|
||||
/// `is_running` without blocking other coordinator paths.
|
||||
last_containers: tokio::sync::Mutex<HashMap<String, ContainerView>>,
|
||||
/// Global rebuild queue. Every long-running container/meta op
|
||||
/// (rebuild, meta-update, first-spawn) goes through this queue so
|
||||
/// hive-c0re runs at most one at a time and the dashboard can
|
||||
/// render a single ordered view of pending + running work. See
|
||||
/// `rebuild_queue.rs` for the dedup rules + history retention.
|
||||
pub rebuild_queue: Arc<crate::rebuild_queue::RebuildQueue>,
|
||||
/// Global job-DAG queue. Every container/meta op (rebuild,
|
||||
/// meta-update, first-spawn, power changes) is submitted as a DAG
|
||||
/// of primitive nodes; a single scheduler drives them with
|
||||
/// build-slot + per-agent-lease gating so the dashboard renders one
|
||||
/// ordered view of pending + running work. See `job_queue/` for
|
||||
/// the dedup rules, resource classes, and history retention.
|
||||
pub job_queue: Arc<crate::job_queue::JobQueue>,
|
||||
/// Durable per-agent power intent (`wanted: Up | Offline`) — the
|
||||
/// spec half of desired-state reconciliation; the queue's
|
||||
/// `Reconcile` nodes converge observed state to it.
|
||||
pub power: Arc<crate::power::PowerStore>,
|
||||
/// Shutdown signal broadcast to all background tasks. Sending
|
||||
/// `true` asks every loop to exit after its current work item.
|
||||
/// Use `shutdown_rx()` to subscribe; `request_shutdown()` to fire.
|
||||
|
|
@ -244,12 +249,27 @@ impl Default for HiveEnv {
|
|||
/// instead of every host-level setting as its own JSON-blob argument.
|
||||
/// `#[serde(default)]` lets any field be omitted and fall back to its
|
||||
/// canonical default.
|
||||
#[derive(Clone, Debug, Default, serde::Deserialize)]
|
||||
#[derive(Clone, Debug, serde::Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct ServeConfig {
|
||||
#[serde(flatten)]
|
||||
pub env: HiveEnv,
|
||||
pub model_prices: crate::hive_stats::PriceTable,
|
||||
/// Number of concurrent nix-heavy job-queue nodes (prebuild /
|
||||
/// profile-swap / create / meta lock). hive-c0re-local like
|
||||
/// `model_prices` — never injected into containers. Set via
|
||||
/// `services.hyperhive.c0re.buildSlots`.
|
||||
pub build_slots: usize,
|
||||
}
|
||||
|
||||
impl Default for ServeConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
env: HiveEnv::default(),
|
||||
model_prices: crate::hive_stats::PriceTable::default(),
|
||||
build_slots: 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -433,6 +453,7 @@ impl Coordinator {
|
|||
db_path: &Path,
|
||||
env: HiveEnv,
|
||||
model_prices: crate::hive_stats::PriceTable,
|
||||
build_slots: usize,
|
||||
) -> Result<Self> {
|
||||
let HiveEnv {
|
||||
hyperhive_flake,
|
||||
|
|
@ -469,6 +490,8 @@ impl Coordinator {
|
|||
let audit_log =
|
||||
Arc::new(crate::audit_log::AuditLog::open(build_logs_dir).context("open audit_log")?);
|
||||
crate::audit_log::install(audit_log.clone());
|
||||
let power =
|
||||
Arc::new(crate::power::PowerStore::open(build_logs_dir).context("open agent_power")?);
|
||||
let (dashboard_events, _) = broadcast::channel(DASHBOARD_CHANNEL);
|
||||
let (shutdown_tx, _) = watch::channel(false);
|
||||
Ok(Self {
|
||||
|
|
@ -497,7 +520,8 @@ impl Coordinator {
|
|||
event_seq: AtomicU64::new(0),
|
||||
meta_updates_active: AtomicU64::new(0),
|
||||
last_containers: tokio::sync::Mutex::new(HashMap::new()),
|
||||
rebuild_queue: Arc::new(crate::rebuild_queue::RebuildQueue::new()),
|
||||
job_queue: Arc::new(crate::job_queue::JobQueue::new(build_slots)),
|
||||
power,
|
||||
shutdown_tx,
|
||||
})
|
||||
}
|
||||
|
|
@ -543,7 +567,7 @@ impl Coordinator {
|
|||
/// wrappers below) and the worker so every state transition
|
||||
/// surfaces on the dashboard without extra plumbing.
|
||||
pub fn emit_rebuild_queue_snapshot(self: &Arc<Self>) {
|
||||
let queue = self.rebuild_queue.snapshot();
|
||||
let queue = self.job_queue.snapshot();
|
||||
self.emit_dashboard_event(DashboardEvent::RebuildQueueChanged {
|
||||
seq: self.next_seq(),
|
||||
queue,
|
||||
|
|
@ -665,15 +689,27 @@ impl Coordinator {
|
|||
});
|
||||
}
|
||||
|
||||
/// Update the `step` label on a running queue entry and (if it
|
||||
/// actually changed) re-emit the queue snapshot so the dashboard
|
||||
/// renders the new phase. Returns `true` when the label was new
|
||||
/// and an emit fired, mostly for tracing/logging callers; safe to
|
||||
/// ignore. No-op when `id` is `None` (e.g. callers that aren't
|
||||
/// running from the queue worker) or when the row isn't `Running`.
|
||||
/// Update the `step` label on the currently-running node of DAG
|
||||
/// `id` and (if it actually changed) re-emit the queue snapshot so
|
||||
/// the dashboard renders the new phase. DAG-id-only surface for
|
||||
/// the opaque approval pipeline in `actions.rs`, whose callbacks
|
||||
/// don't know node ids (its DAGs are single-node, so the lookup is
|
||||
/// exact); queue executors use the precise per-node sink in
|
||||
/// `job_queue::exec` instead. No-op when `id` is `None` (callers
|
||||
/// not running from the queue) or when nothing is `Running`.
|
||||
pub fn set_queue_step(self: &Arc<Self>, id: Option<u64>, step: &str) {
|
||||
let Some(id) = id else { return };
|
||||
if self.rebuild_queue.set_step(id, step) {
|
||||
if self.job_queue.set_step_running(id, step) {
|
||||
self.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
}
|
||||
|
||||
/// Link a `build_logs` row to the currently-running node of DAG
|
||||
/// `id` and re-emit the snapshot. Same DAG-id-only compatibility
|
||||
/// surface as [`Self::set_queue_step`].
|
||||
pub fn set_queue_build_log(self: &Arc<Self>, id: Option<u64>, log_id: i64) {
|
||||
let Some(id) = id else { return };
|
||||
if self.job_queue.set_build_log_id_running(id, log_id) {
|
||||
self.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue