vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat(hive-c0re): replace rebuild queue with generic job-DAG queue

Browse source 
jobs are now DAGs of primitive nodes (prebuild, stop-for-update, swap,
reconcile, signal, drain, ...) driven by one scheduler with N build
slots + per-agent lifecycle leases. per-agent power intent (wanted
up/offline) is durable in agent_power.sqlite; Reconcile nodes converge
observed state to it. kills the graceful-stop watcher thread, the
deferred-start follow-up, and the cascade pre-enqueue (fan-out on
MetaLock completion instead). tracker: #2166
This commit is contained in:
müde 2026-07-06 20:13:14 +02:00
commit 7946e03fde
25 changed files with 3673 additions and 2731 deletions
Download patch file Download diff file Expand all files Collapse all files

42
hive-c0re/src/main.rs
View file

@ -13,8 +13,8 @@ use hive_sh4re::{HostRequest, HostResponse};
use hive_c0re::coordinator::{Coordinator, HiveEnv, ServeConfig};
use hive_c0re::{
agent_sockets, auto_update, broker, client, crash_watch, dashboard, dashboard_events, forge,
knowledge, matrix, migrate, rebuild_queue, reminder_scheduler, scheduled_prompts_worker,
server, socket_server,
job_queue, knowledge, matrix, migrate, reminder_scheduler, scheduled_prompts_worker, server,
socket_server,
};
#[derive(Parser)]
@ -85,6 +85,11 @@ enum Cmd {
/// option.
#[arg(long)]
model_prices: Option<String>,
/// Override: number of concurrent nix-heavy job-queue nodes
/// (prebuild / profile-swap / create / meta lock). Set via the
/// `services.hyperhive.c0re.buildSlots` NixOS option.
#[arg(long)]
build_slots: Option<usize>,
},
/// Spawn a new agent container directly (`hive-agent-<name>`). Bypasses
/// the approval queue — use only as an operator on the host. For
@ -156,6 +161,7 @@ async fn main() -> Result<()> {
agent_cpu_quota,
agent_memory_max,
model_prices,
build_slots,
} => {
// Base config from the --config file (or the built-in
// defaults), then apply any per-flag overrides — config
@ -195,7 +201,10 @@ async fn main() -> Result<()> {
sc.model_prices =
serde_json::from_str(&v).context("--model-prices: invalid JSON")?;
}
cmd_serve(sc.env, sc.model_prices, db, &cli.socket).await
if let Some(v) = build_slots {
sc.build_slots = v;
}
cmd_serve(sc.env, sc.model_prices, sc.build_slots, db, &cli.socket).await
}
Cmd::Spawn { name } => {
render(client::request(&cli.socket, HostRequest::Spawn { name }).await?)
@ -244,6 +253,7 @@ async fn main() -> Result<()> {
async fn cmd_serve(
env: HiveEnv,
model_prices: hive_c0re::hive_stats::PriceTable,
build_slots: usize,
db: std::path::PathBuf,
socket: &std::path::Path,
) -> Result<()> {
@ -255,7 +265,7 @@ async fn cmd_serve(
// `dashboard_port` is consumed into the Coordinator below; capture the
// Copy value first for the dashboard + knowledge-webhook tasks.
let dashboard_port = env.dashboard_port;
let coord = Arc::new(Coordinator::open(&db, env, model_prices)?);
let coord = Arc::new(Coordinator::open(&db, env, model_prices, build_slots)?);
socket_server::start_manager(coord.clone())?;
// Idempotent pre-flight: rewrite pre-meta-layout applied
// repos, ensure proposed repos carry the `applied`
@ -413,25 +423,17 @@ async fn cmd_serve(
// and fans the body out to each active target's inbox. See
// scheduled_prompts_worker.rs.
scheduled_prompts_worker::spawn(coord.clone());
// Rebuild-queue worker: drains the global rebuild/meta-update/
// spawn queue FIFO so hive-c0re never runs two heavyweight
// container ops concurrently. Existing rebuild call sites
// (auto_update, dashboard, manager, approval handler) enqueue
// here instead of awaiting `rebuild_agent` inline. See
// `rebuild_queue.rs`.
// Job-queue scheduler: drives the global DAG queue (rebuild /
// meta-update / spawn / power ops). Concurrency comes from the
// build-slot count + per-agent leases inside the queue, not from
// multiple workers — cheap nodes (graceful signals, drains,
// reconciles) overlap nix-heavy ones structurally. Call sites
// (auto_update, dashboard, manager, approval handler) submit DAGs
// instead of awaiting lifecycle work inline. See `job_queue/`.
{
let q_coord = coord.clone();
tokio::spawn(async move {
rebuild_queue::run_worker(q_coord).await;
});
// Fast lane: a second serial worker for hard Start/Stop, running
// concurrently with the build worker above so a stop/start never
// waits behind a slow build for another container. Per-agent
// ordering vs that agent's own build is enforced in the queue's
// claim logic (a fast op defers behind its agent's running build).
let fast_coord = coord.clone();
tokio::spawn(async move {
rebuild_queue::run_fast_worker(fast_coord).await;
job_queue::scheduler::run_worker(q_coord).await;
});
}
// Forward every broker event onto the unified dashboard