diff --git a/Cargo.lock b/Cargo.lock index 446ea995..2e4e59c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1029,6 +1029,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.9" @@ -1381,6 +1387,7 @@ dependencies = [ "hive-sh4re", "libc", "listenfd", + "petgraph", "problem_details", "reqwest", "rusqlite", @@ -2554,6 +2561,17 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + [[package]] name = "phf" version = "0.11.3" diff --git a/Cargo.toml b/Cargo.toml index 128ba120..873c0434 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,7 @@ reqwest = { version = "0.12", default-features = false, features = [ "json", "rustls-tls", ] } +petgraph = { version = "0.8", default-features = false, features = ["std"] } matrix-sdk = { version = "0.14", default-features = false, features = [ "rustls-tls", "sqlite", diff --git a/hive-c0re/Cargo.toml b/hive-c0re/Cargo.toml index 86502340..afa5ef54 100644 --- a/hive-c0re/Cargo.toml +++ b/hive-c0re/Cargo.toml @@ -18,6 +18,7 @@ clap-markdown = "0.1" hive-sh4re.workspace = true libc.workspace = true listenfd = "1" +petgraph.workspace = true rusqlite.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/hive-c0re/src/actions.rs b/hive-c0re/src/actions.rs index 5fc88933..e2291c5f 100644 --- a/hive-c0re/src/actions.rs +++ b/hive-c0re/src/actions.rs @@ -13,19 +13,20 @@ use crate::lifecycle; /// Approve a pending request. Marks the approval row durably, then /// either runs the work inline (`InitConfig`, sub-second git ops) or -/// enqueues it into `rebuild_queue` so the dashboard POST returns +/// submits it to the job queue so the dashboard POST returns /// immediately while the long-running pipeline runs off-thread /// (operator no longer blocks on a 30-90s spinner for `ApplyCommit`). /// /// Dispatch: -/// - `ApplyCommit` → `QueueKind::Rebuild` (~30-90s wall time) -/// - `UpdateMetaInputs` → `QueueKind::MetaUpdate` (~3-15s) -/// - `Spawn` → `QueueKind::Spawn` (~30-90s) +/// - `ApplyCommit` / `MergeConfigPr` → a single-node `ApprovalDeploy` +/// DAG (the two-phase meta deploy stays opaque in v1; ~30-90s) +/// - `UpdateMetaInputs` → a `MetaUpdate` DAG (fan-out on completion) +/// - `Spawn` → a `Spawn` DAG (`Create → WriteDropin → Reconcile`) /// - `InitConfig` → inline (<1s; queue card would be noise) /// -/// The queue worker re-fetches the approval row on dispatch, runs -/// the kind-specific pipeline, and fires `ApprovalResolved` / -/// `Spawned` / `Rebuilt` / `ConfigReady` via `finish_approval`. +/// `ApprovalDeploy` resolves the approval inside its pipeline; the +/// `MetaUpdate` / `Spawn` DAGs resolve via [`resolve_approval_dag`] +/// when their DAG settles terminal. pub async fn approve(coord: Arc, id: i64) -> Result<()> { let approval = coord.approvals.mark_approved(id)?; tracing::info!( @@ -56,54 +57,41 @@ pub async fn approve(coord: Arc, id: i64) -> Result<()> { } ApprovalKind::UpdateMetaInputs => { // Inputs JSON-encoded into commit_ref by the manager's - // submit path — surface them on the queue entry so the - // dashboard can show *which* inputs are about to bump. + // submit path — surface them on the DAG so the dashboard + // can show *which* inputs are about to bump. The cascade + // rebuilds fan out when the lock bump lands (so they build + // against the post-bump lock, and a failed bump fans out + // nothing). let inputs: Vec = serde_json::from_str(&approval.commit_ref).unwrap_or_default(); - let parent_id = coord - .rebuild_queue - .enqueue_full(crate::rebuild_queue::FullEnqueue { - kind: crate::rebuild_queue::QueueKind::MetaUpdate, - agent: approval.agent.clone(), - source: crate::rebuild_queue::QueueSource::Approval, - reason: format!("approval #{id} meta input update"), - parent_id: None, - inputs: inputs.clone(), - approval_id: Some(id), - perm_payload: None, - depends_on: Vec::new(), - }); - // Pre-enqueue cascade rebuilds in topological order so - // agents depending on updated inputs are rebuilt after the - // lock bump, matching the dashboard post_meta_update path. - let cascade_agents = crate::rebuild_queue::meta_update_cascade_agents(&inputs).await; - let cascade_reason = format!("approval #{id} meta input cascade"); - for name in cascade_agents { - coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Rebuild, - name, - crate::rebuild_queue::QueueSource::MetaUpdate, - cascade_reason.clone(), - Some(parent_id), - ); + let submitted = coord + .job_queue + .submit(crate::job_queue::templates::meta_update( + inputs, + crate::job_queue::Source::Approval, + format!("approval #{id} meta input update"), + Some(id), + )); + if let Err(e) = submitted { + return Err(e.context("submit meta-update dag")); } coord.emit_rebuild_queue_snapshot(); Ok(()) } ApprovalKind::Spawn => { - coord - .rebuild_queue - .enqueue_full(crate::rebuild_queue::FullEnqueue { - kind: crate::rebuild_queue::QueueKind::Spawn, - agent: approval.agent.clone(), - source: crate::rebuild_queue::QueueSource::Approval, - reason: format!("approval #{id} spawn"), - parent_id: None, - inputs: Vec::new(), - approval_id: Some(id), - perm_payload: None, - depends_on: Vec::new(), - }); + // The spawn's tail `Reconcile` starts the container, so the + // new agent's power intent is `Up` from the outset. + if let Err(e) = coord.power.set(&approval.agent, crate::power::Wanted::Up) { + tracing::warn!(agent = %approval.agent, error = ?e, "agent_power: seed on spawn failed"); + } + let submitted = coord.job_queue.submit(crate::job_queue::templates::spawn( + &approval.agent, + id, + format!("approval #{id} spawn"), + )); + if let Err(e) = submitted { + return Err(e.context("submit spawn dag")); + } coord.emit_rebuild_queue_snapshot(); Ok(()) } @@ -137,29 +125,27 @@ pub async fn approve(coord: Arc, id: i64) -> Result<()> { } } -/// Enqueue a `Rebuild` queue entry tied to an approval id. Shared by the -/// `ApplyCommit` and `MergeConfigPr` dispatch arms — both end in a container -/// rebuild routed through the queue, differing only in the queue `reason`. -/// The queue worker branches on the approval's kind to pick the right handler. +/// Submit the single-node `ApprovalDeploy` DAG tied to an approval id. +/// Shared by the `ApplyCommit` and `MergeConfigPr` dispatch arms — both +/// end in a container rebuild routed through the queue, differing only +/// in the `reason`. The node executor branches on the approval's kind +/// to pick the right handler. fn enqueue_approval_rebuild( coord: &Arc, agent: &str, approval_id: i64, reason: String, ) { - coord - .rebuild_queue - .enqueue_full(crate::rebuild_queue::FullEnqueue { - kind: crate::rebuild_queue::QueueKind::Rebuild, - agent: agent.to_owned(), - source: crate::rebuild_queue::QueueSource::Approval, + if let Err(e) = coord + .job_queue + .submit(crate::job_queue::templates::approval_deploy( + agent, + approval_id, reason, - parent_id: None, - inputs: Vec::new(), - approval_id: Some(approval_id), - perm_payload: None, - depends_on: Vec::new(), - }); + )) + { + tracing::error!(%agent, approval_id, error = ?e, "submit approval deploy dag failed"); + } coord.emit_rebuild_queue_snapshot(); } @@ -375,69 +361,60 @@ async fn run_approval_schedule_prompt( finish_approval(coord, &approval, result, None, false) } -/// Worker entry point for `ApprovalKind::UpdateMetaInputs` queue -/// entries. Inputs come from the approval row's `commit_ref` field -/// (JSON-encoded by the manager submit path), not the queue entry's -/// `inputs` — the queue copy is for dashboard display only. -pub async fn run_approval_update_meta_inputs( +/// Terminal hook for approval-carrying DAGs — the job queue's +/// scheduler calls this exactly once when such a DAG settles terminal. +/// `MetaUpdate` and `Spawn` approval DAGs resolve here (their work is +/// ordinary queue nodes); the opaque `ApprovalDeploy` pipeline resolves +/// *inside* its node, so its DAG is skipped — unless it was cancelled +/// while still queued, in which case the node never ran and the row +/// would otherwise dangle forever. +pub(crate) async fn resolve_approval_dag( coord: &Arc, - queue_entry_id: Option, - approval_id: i64, -) -> Result<()> { - let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::UpdateMetaInputs)?; - let inputs: Vec = serde_json::from_str(&approval.commit_ref).unwrap_or_default(); - coord.set_queue_step(queue_entry_id, "nix flake update"); - let result = crate::meta::lock_update(&inputs).await; - finish_approval(coord, &approval, result, None, false) -} - -/// Worker entry point for `ApprovalKind::Spawn` queue entries. -/// Differs from `run_approval_apply_commit` only in routing through -/// `lifecycle::spawn` (the deprecated direct-spawn path). Synchronous -/// in the queue worker — the previous `tokio::spawn` wrapper is gone -/// (the queue worker itself is the async task). -pub async fn run_approval_spawn( - coord: &Arc, - queue_entry_id: Option, - approval_id: i64, -) -> Result<()> { - let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::Spawn)?; - let agent_dir = coord.ensure_runtime(&approval.agent)?; - let hive = coord.hive_env(); - let paths = Coordinator::agent_paths(&approval.agent, agent_dir); - // Transient guard keeps the per-container "Spawning" pill lit while - // the worker is doing the actual nixos-container create. Auto-clears - // on the function's scope exit (success or panic). - let _guard = coord.transient_guard(&approval.agent, TransientKind::Spawning); - coord.set_queue_step(queue_entry_id, "lifecycle::spawn"); - let result = lifecycle::spawn(&approval.agent, &hive, &paths).await; - if result.is_ok() { - coord.set_queue_step(queue_entry_id, "forge user"); - if let Err(e) = crate::forge::ensure_user_for(&approval.agent).await { - tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_user after spawn failed"); + terminal: &crate::job_queue::TerminalDag, +) { + use crate::job_queue::{State, Template}; + let Some(approval_id) = terminal.approval_id else { + return; + }; + if terminal.template == Template::Rebuild && terminal.state != State::Cancelled { + return; // ApprovalDeploy resolved inside the node. + } + let approval = match coord.approvals.get(approval_id) { + Ok(Some(a)) => a, + Ok(None) => { + tracing::warn!(approval_id, "approval dag terminal: row no longer exists"); + return; } - coord.set_queue_step(queue_entry_id, "forge config repo"); - if let Err(e) = crate::forge::ensure_config_repo(&approval.agent).await { - tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_config_repo after spawn failed"); + Err(e) => { + tracing::warn!(approval_id, error = ?e, "approval dag terminal: row read failed"); + return; } - coord.set_queue_step(queue_entry_id, "forge push"); - if let Err(e) = crate::forge::push_config(&approval.agent).await { - tracing::warn!(agent = %approval.agent, error = ?e, "forge: push_config after spawn failed"); - } - coord.set_queue_step(queue_entry_id, "forge meta access"); - if let Some(core_token) = crate::forge::core_token() - && let Err(e) = crate::forge::meta_read_access(&approval.agent, &core_token).await - { - tracing::warn!(agent = %approval.agent, error = ?e, "forge: meta_read_access after spawn failed"); - } - if let Err(e) = crate::forge::ensure_meta_remote(&approval.agent).await { - tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_meta_remote after spawn failed"); + }; + let result: Result<()> = match terminal.state { + State::Done => Ok(()), + State::Cancelled => Err(anyhow::anyhow!("cancelled before completion")), + _ => Err(anyhow::anyhow!( + "{}", + terminal + .error + .clone() + .unwrap_or_else(|| "job dag failed".to_owned()) + )), + }; + if approval.kind == ApprovalKind::Spawn { + // Post-spawn forge bookkeeping (user, config repo mirror, meta + // access) — warn-only, then the resolution events + a rescan so + // the dashboard reflects the post-spawn state either way. + if result.is_ok() { + forge_after_first_spawn(coord, &approval.agent).await; + } else { + coord.rescan_containers_and_emit().await; + crate::dashboard::emit_tombstones_snapshot(coord).await; } } - let final_result = finish_approval(coord, &approval, result, None, false); - coord.rescan_containers_and_emit().await; - crate::dashboard::emit_tombstones_snapshot(coord).await; - final_result + if let Err(e) = finish_approval(coord, &approval, result, None, false) { + tracing::warn!(approval_id, error = ?e, "approval dag resolved with failure"); + } } /// Re-fetch an approval row from sqlite for a queue-worker dispatch. @@ -867,13 +844,7 @@ async fn deploy_applied_target( // part of this entry rather than a deferred fast-lane follow-up. false, &|step| coord.set_queue_step(queue_entry_id, step), - &|log_id| { - if let Some(qid) = queue_entry_id - && coord.rebuild_queue.set_build_log_id(qid, log_id) - { - coord.emit_rebuild_queue_snapshot(); - } - }, + &|log_id| coord.set_queue_build_log(queue_entry_id, log_id), ) .await; @@ -984,6 +955,11 @@ pub async fn destroy(coord: &Arc, name: &str, purge: bool) -> Resul "agent destroyed" }, ); + // Drop the durable power intent — a future agent of the same name + // seeds fresh from its observed state. + if let Err(e) = coord.power.remove(name) { + tracing::warn!(%name, error = ?e, "agent_power: remove on destroy failed"); + } drop(guard); coord.notify_manager(&HelperEvent::Destroyed { agent: name.to_owned(), diff --git a/hive-c0re/src/auto_update.rs b/hive-c0re/src/auto_update.rs index d17a1887..590ff450 100644 --- a/hive-c0re/src/auto_update.rs +++ b/hive-c0re/src/auto_update.rs @@ -1,18 +1,25 @@ -//! Startup auto-update: on `hive-c0re serve` boot, rebuild containers that -//! actually need it. Two skip rules keep boot-time work minimal: +//! Boot reconcile: on `hive-c0re serve` boot, (a) run the config path +//! for agents whose per-agent rev marker is stale — a `StartupSweep` +//! DAG (meta hyperhive lock bump) fanning out `Rebuild` children for +//! the stale agents whose `wanted` power intent is `Up` — and (b) +//! converge every other drifted agent to its persisted `wanted` via +//! `Reconcile` DAGs. Two rules keep boot-time nix work minimal: //! -//! 1. **Stopped containers** are deferred — they will be rebuilt the first -//! time the operator starts them (see `rebuild_queue::run_start` and -//! `socket_server::handle_start`). -//! 2. **Running containers whose rev marker matches** the current hyperhive -//! flake path are skipped — nothing changed, no nix work to do. +//! 1. **Stale but wanted-offline agents** get no rebuild — their +//! rebuild happens the first time they're started (the start +//! submit path upgrades a stale start to rebuild+start). The sweep +//! parent still runs whenever *any* marker is stale so the meta +//! hyperhive lock is bumped for those later start-upgrades. +//! 2. **Agents whose rev marker matches** the current hyperhive flake +//! path are skipped — nothing changed, no nix work to do. //! -//! See `docs/coordinator.md::Auto-update sweep`. +//! Booting with no config change performs no meta commit — only +//! reconciles. See `docs/coordinator.md::Boot reconcile`. use std::path::{Path, PathBuf}; use std::sync::Arc; -use anyhow::{Context, Result}; +use anyhow::Result; use crate::coordinator::Coordinator; use crate::lifecycle::{self, AGENT_PREFIX, MANAGER_NAME}; @@ -71,143 +78,6 @@ pub async fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> boo } } -/// Rebuild one sub-agent and refresh its marker. Used by both the startup -/// scanner and the dashboard's manual "update" button so the two paths -/// can't diverge. -/// -/// `queue_entry_id` is `Some(id)` when the rebuild was dispatched from -/// the `rebuild_queue` worker (lets the function annotate its phase via -/// `coord.set_queue_step`) and `None` when called directly (e.g. the -/// root-agent migration nudge in `ensure_root_agent`). -/// -/// `relock` bumps the agent's meta input to `applied//main` before -/// the container rebuild. Pass `false` only for meta-update cascade -/// rebuilds, where re-locking would revert the bump the cascade just -/// committed (see `lifecycle::rebuild`). -/// -/// `defer_start_source` is `Some(source)` for queue-dispatched rebuilds: -/// instead of holding the serialized build lane through the container -/// boot, the start-after-rebuild is enqueued as a fast-lane `Start` -/// entry (grouped under this rebuild via `parent_id`, same split as the -/// graceful-stop follow-up). Pass `None` for direct callers to keep the -/// start inline. -/// -/// # Errors -/// -/// Propagates errors from `coord.ensure_runtime` and `lifecycle::rebuild`. -pub async fn rebuild_agent( - coord: &Arc, - name: &str, - current_rev: &str, - queue_entry_id: Option, - relock: bool, - defer_start_source: Option, -) -> Result<()> { - tracing::info!(%name, rev = %current_rev, "rebuild agent"); - let agent_dir = coord - .ensure_runtime(name) - .with_context(|| format!("ensure_runtime {name}"))?; - let hive = coord.hive_env(); - let paths = Coordinator::agent_paths(name, agent_dir); - // Suppress crash_watch during the stop+start window inside - // lifecycle::rebuild. Dashboard rebuilds already do this via - // lifecycle_action; this catches the auto-update scan + any - // other direct caller. - let guard = coord.transient_guard(name, crate::coordinator::TransientKind::Rebuilding); - let result = lifecycle::rebuild( - name, - &hive, - &paths, - relock, - defer_start_source.is_some(), - &|step| coord.set_queue_step(queue_entry_id, step), - &|log_id| { - if let Some(qid) = queue_entry_id - && coord.rebuild_queue.set_build_log_id(qid, log_id) - { - coord.emit_rebuild_queue_snapshot(); - } - }, - ) - .await; - drop(guard); - match &result { - Ok(needs_start) => { - if let Err(e) = std::fs::write(rev_marker_path(name), current_rev) { - tracing::warn!(%name, error = ?e, "write rev marker failed"); - } - // Deferred start: hand the container boot to the fast lane so - // this build-lane entry completes now and the next queued - // rebuild's nix build overlaps with the boot. `parent_id` - // groups the follow-up under this rebuild on the dashboard — - // same split the graceful-stop path uses for its container - // stop. A start failure surfaces on the Start entry (with - // the cold-start fallback) instead of failing the rebuild. - if *needs_start && let Some(source) = defer_start_source { - coord - .rebuild_queue - .enqueue_full(crate::rebuild_queue::FullEnqueue { - kind: crate::rebuild_queue::QueueKind::Start, - agent: name.to_owned(), - source, - reason: format!("start after rebuild of {name}"), - parent_id: queue_entry_id, - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: Vec::new(), - }); - coord.emit_rebuild_queue_snapshot(); - } - coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { - agent: name.to_owned(), - ok: true, - note: None, - sha: None, - tag: None, - }); - coord.set_queue_step(queue_entry_id, "forge sync"); - // Run the full forge sync on every successful rebuild so - // the rebuild path is equivalent to the hive-c0re startup - // sweep: token, config-repo mirror, meta read access, and - // meta remote are all kept in sync. Recovers missing tokens - // (e.g. first-spawn seeding failed transiently) without - // requiring a full hive-c0re restart. - crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await; - // Mirror the matrix side of the startup sweep: if hive-matrix - // is present, ensure this agent has a registered user + token. - // Idempotent (skips if token file already exists). Keeps the - // rebuild path equivalent to the startup sweep for newly-spawned - // agents that missed ensure_all(). - crate::matrix::sync_agent_standalone(name).await; - // Wake the agent on its next turn so claude sees a - // "you were rebuilt — check /state/ for notes, --continue - // session intact" hint. Covers dashboard rebuild, admin - // CLI rebuild, auto-update startup scan, and the - // dashboard's meta-input update path — all of which - // route through rebuild_agent. - coord.kick_agent(name, "container rebuilt"); - // Container state (needs_update, deployed_sha) may have - // shifted — rescan so dashboards drop the "needs update" - // chip without waiting for the next /api/state poll. - coord.rescan_containers_and_emit().await; - // Lock bump → meta-inputs panel needs to re-render. - crate::dashboard::emit_meta_inputs_snapshot(coord); - } - Err(e) => { - coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { - agent: name.to_owned(), - ok: false, - note: Some(format!("{e:#}")), - sha: None, - tag: None, - }); - coord.rescan_containers_and_emit().await; - } - } - result.map(|_| ()) -} - /// Whether this hive is "ruthless" — running with no root/manager agent at /// all (no ruth). When true, hive-c0re skips the root-agent create/start /// sweep entirely. Controlled by the host option @@ -246,17 +116,18 @@ pub async fn ensure_root_agent(coord: &Arc) -> Result<()> { // running whatever the host-declarative config was at create // time, with a wrong systemd unit and port. let applied_flake = Coordinator::agent_applied_dir(MANAGER_NAME).join("flake.nix"); - if !applied_flake.exists() - && let Some(rev) = current_rev.as_ref() - { + if !applied_flake.exists() && current_rev.is_some() { tracing::warn!( "manager container exists but no applied flake — forcing rebuild to migrate" ); - let coord_clone = coord.clone(); - if let Err(e) = - rebuild_agent(&coord_clone, MANAGER_NAME, rev.as_str(), None, true, None).await - { - tracing::warn!(error = ?e, "manager migration rebuild failed"); + if let Err(e) = coord.job_queue.submit(crate::job_queue::templates::rebuild( + MANAGER_NAME, + crate::job_queue::Source::AutoUpdate, + "manager migration: no applied flake".to_owned(), + None, + true, + )) { + tracing::warn!(error = ?e, "manager migration rebuild submit failed"); } } else { tracing::debug!("manager container already present"); @@ -272,6 +143,9 @@ pub async fn ensure_root_agent(coord: &Arc) -> Result<()> { // this function.) if !lifecycle::is_running(MANAGER_NAME).await { tracing::info!("manager container present but not running — starting"); + if let Err(e) = coord.power.set(MANAGER_NAME, crate::power::Wanted::Up) { + tracing::warn!(error = ?e, "agent_power: set manager wanted=up failed"); + } if let Err(e) = lifecycle::start(MANAGER_NAME).await { tracing::warn!(error = ?e, "manager start failed"); } @@ -283,6 +157,9 @@ pub async fn ensure_root_agent(coord: &Arc) -> Result<()> { let hive = coord.hive_env(); let paths = Coordinator::agent_paths(MANAGER_NAME, runtime); lifecycle::spawn(MANAGER_NAME, &hive, &paths).await?; + if let Err(e) = coord.power.set(MANAGER_NAME, crate::power::Wanted::Up) { + tracing::warn!(error = ?e, "agent_power: set manager wanted=up failed"); + } if let Some(rev) = current_rev { let _ = std::fs::write(rev_marker_path(MANAGER_NAME), &rev); } @@ -327,18 +204,17 @@ pub fn topology_sort( }); } -/// Rebuild containers that need it on startup. Skips: -/// - **Stopped containers**: deferred to on-start (`run_start` / `handle_start` -/// upgrades a plain start to rebuild+start when the rev marker is stale). -/// - **Running containers with a matching rev marker**: no nix work needed. -/// -/// Enqueues a `StartupSweep` parent entry followed by per-agent `Rebuild` -/// children linked via `parent_id`. Returns Ok even if some rebuilds failed. +/// Boot reconcile (see the module doc): classify every agent by rev +/// freshness + persisted `wanted` intent, submit one `StartupSweep` +/// DAG (hyperhive lock bump → fan-out rebuilds for stale wanted-up +/// agents) when anything is stale, and `Reconcile` DAGs for agents +/// whose observed power state drifted from `wanted`. Returns Ok even +/// if some submissions failed. pub async fn run(coord: Arc) -> Result<()> { let containers = match lifecycle::list().await { Ok(c) => c, Err(e) => { - tracing::warn!(error = ?e, "auto-update: nixos-container list failed"); + tracing::warn!(error = ?e, "boot reconcile: nixos-container list failed"); return Ok(()); } }; @@ -356,62 +232,88 @@ pub async fn run(coord: Arc) -> Result<()> { let topo = crate::topology::read(); topology_sort(&mut logical_names, &topo); - // Pre-classify: decide which agents need a rebuild now vs can be skipped. - let mut to_rebuild: Vec = Vec::new(); + // Classify. `get_or_seed` doubles as the one-time migration: an + // agent without an `agent_power` row is seeded from its observed + // state (running ⇒ Up), after which the DB is authoritative. + let mut any_stale = false; + let mut fanout: Vec = Vec::new(); // stale ∧ wanted=Up → sweep rebuild + let mut drifted: Vec = Vec::new(); // fresh ∧ wanted≠observed → reconcile let mut n_deferred = 0usize; let mut n_skipped = 0usize; for name in &logical_names { - // Idea 2: stopped containers are deferred — rebuild happens the first - // time the operator starts them. - if !lifecycle::is_running(name).await { - n_deferred += 1; - tracing::debug!(%name, "startup sweep: stopped — deferring rebuild to on-start"); - continue; - } - // Idea 1: running containers with a matching rev marker need no rebuild. - if let Some(ref rev) = current_rev { - let stored = std::fs::read_to_string(rev_marker_path(name)).ok(); - if stored.as_deref() == Some(rev.as_str()) { - n_skipped += 1; - tracing::debug!(%name, "startup sweep: rev unchanged — skipping rebuild"); + let running = lifecycle::is_running(name).await; + let wanted = match coord.power.get_or_seed(name, running) { + Ok(w) => w, + Err(e) => { + tracing::warn!(%name, error = ?e, "agent_power read failed — assuming observed"); + crate::power::Wanted::from_running(running) + } + }; + let fresh = current_rev.as_ref().is_some_and(|rev| { + std::fs::read_to_string(rev_marker_path(name)) + .is_ok_and(|stored| stored == rev.as_str()) + }); + if fresh { + n_skipped += 1; + } else { + any_stale = true; + if wanted == crate::power::Wanted::Up { + // Rebuild against the post-bump lock; the DAG's tail + // Reconcile brings the agent (back) up — covering both + // the running-stale and stopped-but-wanted-up cases. + fanout.push(name.clone()); continue; } + // Stale but wanted offline: no boot-time nix work — the + // start submit path upgrades a stale start to a rebuild. + n_deferred += 1; + tracing::debug!(%name, "boot reconcile: stale but offline — deferring rebuild to on-start"); + } + if crate::power::reconcile_action(wanted, running) != crate::power::ReconcileAction::Noop { + drifted.push(name.clone()); } - to_rebuild.push(name.clone()); } - // Enqueue the parent sweep entry. The worker processes it trivially - // (no-op dispatch); its purpose is to give the dashboard a "why" header. - let sweep_id = coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::StartupSweep, - "hyperhive".to_owned(), - crate::rebuild_queue::QueueSource::AutoUpdate, - format!( - "startup sweep: {} rebuild(s), {} deferred (stopped), {} skipped (up-to-date)", - to_rebuild.len(), - n_deferred, - n_skipped, - ), - None, - ); - tracing::info!( total = containers.len(), - rebuilds = to_rebuild.len(), + rebuilds = fanout.len(), + reconciles = drifted.len(), deferred = n_deferred, - skipped = n_skipped, - sweep_id, - "auto-update: startup sweep" + up_to_date = n_skipped, + "boot reconcile" ); - for name in to_rebuild { - coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Rebuild, - name, - crate::rebuild_queue::QueueSource::StartupSweep, - "startup sweep".to_owned(), - Some(sweep_id), + // Sweep parent whenever ANY marker is stale — even when every + // stale agent is wanted-offline: the hyperhive lock bump must land + // now so their later start-upgrade rebuilds build against it. + // No stale agents ⇒ no sweep ⇒ no meta commit on a no-change boot. + if any_stale { + let reason = format!( + "startup sweep: {} rebuild(s), {} deferred (offline), {} up-to-date", + fanout.len(), + n_deferred, + n_skipped, ); + if let Err(e) = coord + .job_queue + .submit(crate::job_queue::templates::startup_sweep(reason, fanout)) + { + tracing::warn!(error = ?e, "boot reconcile: sweep submit failed"); + } + } + for name in drifted { + if let Err(e) = coord + .job_queue + .submit(crate::job_queue::templates::reconcile_only( + crate::job_queue::Template::Reconcile, + &name, + crate::job_queue::Source::AutoUpdate, + "boot reconcile".to_owned(), + None, + )) + { + tracing::warn!(%name, error = ?e, "boot reconcile: submit failed"); + } } coord.emit_rebuild_queue_snapshot(); Ok(()) diff --git a/hive-c0re/src/coordinator.rs b/hive-c0re/src/coordinator.rs index c19d1629..2dbf4dd1 100644 --- a/hive-c0re/src/coordinator.rs +++ b/hive-c0re/src/coordinator.rs @@ -173,12 +173,17 @@ pub struct Coordinator { /// tokio mutex so the rescan can `await` `lifecycle::list` / /// `is_running` without blocking other coordinator paths. last_containers: tokio::sync::Mutex>, - /// Global rebuild queue. Every long-running container/meta op - /// (rebuild, meta-update, first-spawn) goes through this queue so - /// hive-c0re runs at most one at a time and the dashboard can - /// render a single ordered view of pending + running work. See - /// `rebuild_queue.rs` for the dedup rules + history retention. - pub rebuild_queue: Arc, + /// Global job-DAG queue. Every container/meta op (rebuild, + /// meta-update, first-spawn, power changes) is submitted as a DAG + /// of primitive nodes; a single scheduler drives them with + /// build-slot + per-agent-lease gating so the dashboard renders one + /// ordered view of pending + running work. See `job_queue/` for + /// the dedup rules, resource classes, and history retention. + pub job_queue: Arc, + /// Durable per-agent power intent (`wanted: Up | Offline`) — the + /// spec half of desired-state reconciliation; the queue's + /// `Reconcile` nodes converge observed state to it. + pub power: Arc, /// Shutdown signal broadcast to all background tasks. Sending /// `true` asks every loop to exit after its current work item. /// Use `shutdown_rx()` to subscribe; `request_shutdown()` to fire. @@ -244,12 +249,27 @@ impl Default for HiveEnv { /// instead of every host-level setting as its own JSON-blob argument. /// `#[serde(default)]` lets any field be omitted and fall back to its /// canonical default. -#[derive(Clone, Debug, Default, serde::Deserialize)] +#[derive(Clone, Debug, serde::Deserialize)] #[serde(default)] pub struct ServeConfig { #[serde(flatten)] pub env: HiveEnv, pub model_prices: crate::hive_stats::PriceTable, + /// Number of concurrent nix-heavy job-queue nodes (prebuild / + /// profile-swap / create / meta lock). hive-c0re-local like + /// `model_prices` — never injected into containers. Set via + /// `services.hyperhive.c0re.buildSlots`. + pub build_slots: usize, +} + +impl Default for ServeConfig { + fn default() -> Self { + Self { + env: HiveEnv::default(), + model_prices: crate::hive_stats::PriceTable::default(), + build_slots: 1, + } + } } #[cfg(test)] @@ -433,6 +453,7 @@ impl Coordinator { db_path: &Path, env: HiveEnv, model_prices: crate::hive_stats::PriceTable, + build_slots: usize, ) -> Result { let HiveEnv { hyperhive_flake, @@ -469,6 +490,8 @@ impl Coordinator { let audit_log = Arc::new(crate::audit_log::AuditLog::open(build_logs_dir).context("open audit_log")?); crate::audit_log::install(audit_log.clone()); + let power = + Arc::new(crate::power::PowerStore::open(build_logs_dir).context("open agent_power")?); let (dashboard_events, _) = broadcast::channel(DASHBOARD_CHANNEL); let (shutdown_tx, _) = watch::channel(false); Ok(Self { @@ -497,7 +520,8 @@ impl Coordinator { event_seq: AtomicU64::new(0), meta_updates_active: AtomicU64::new(0), last_containers: tokio::sync::Mutex::new(HashMap::new()), - rebuild_queue: Arc::new(crate::rebuild_queue::RebuildQueue::new()), + job_queue: Arc::new(crate::job_queue::JobQueue::new(build_slots)), + power, shutdown_tx, }) } @@ -543,7 +567,7 @@ impl Coordinator { /// wrappers below) and the worker so every state transition /// surfaces on the dashboard without extra plumbing. pub fn emit_rebuild_queue_snapshot(self: &Arc) { - let queue = self.rebuild_queue.snapshot(); + let queue = self.job_queue.snapshot(); self.emit_dashboard_event(DashboardEvent::RebuildQueueChanged { seq: self.next_seq(), queue, @@ -665,15 +689,27 @@ impl Coordinator { }); } - /// Update the `step` label on a running queue entry and (if it - /// actually changed) re-emit the queue snapshot so the dashboard - /// renders the new phase. Returns `true` when the label was new - /// and an emit fired, mostly for tracing/logging callers; safe to - /// ignore. No-op when `id` is `None` (e.g. callers that aren't - /// running from the queue worker) or when the row isn't `Running`. + /// Update the `step` label on the currently-running node of DAG + /// `id` and (if it actually changed) re-emit the queue snapshot so + /// the dashboard renders the new phase. DAG-id-only surface for + /// the opaque approval pipeline in `actions.rs`, whose callbacks + /// don't know node ids (its DAGs are single-node, so the lookup is + /// exact); queue executors use the precise per-node sink in + /// `job_queue::exec` instead. No-op when `id` is `None` (callers + /// not running from the queue) or when nothing is `Running`. pub fn set_queue_step(self: &Arc, id: Option, step: &str) { let Some(id) = id else { return }; - if self.rebuild_queue.set_step(id, step) { + if self.job_queue.set_step_running(id, step) { + self.emit_rebuild_queue_snapshot(); + } + } + + /// Link a `build_logs` row to the currently-running node of DAG + /// `id` and re-emit the snapshot. Same DAG-id-only compatibility + /// surface as [`Self::set_queue_step`]. + pub fn set_queue_build_log(self: &Arc, id: Option, log_id: i64) { + let Some(id) = id else { return }; + if self.job_queue.set_build_log_id_running(id, log_id) { self.emit_rebuild_queue_snapshot(); } } diff --git a/hive-c0re/src/dashboard.rs b/hive-c0re/src/dashboard.rs index 9ba989db..e9e70ed8 100644 --- a/hive-c0re/src/dashboard.rs +++ b/hive-c0re/src/dashboard.rs @@ -299,12 +299,13 @@ struct StateSnapshot { /// disabled "updating…" state; live transitions arrive via the /// `MetaUpdateRunning` event. meta_update_running: bool, - /// Current state of the global rebuild queue — pending + running - /// long-lived ops (rebuild / meta-update / spawn) plus the most - /// recent few terminal entries the queue retains for history. - /// Live transitions arrive via the `RebuildQueueChanged` event. - /// See `rebuild_queue.rs`. - rebuild_queue: Vec, + /// Current state of the global job queue — pending + running DAGs + /// (rebuild / meta-update / spawn / power ops) with their per-node + /// breakdowns, plus the most recent few terminal DAGs the queue + /// retains for history. Live transitions arrive via the + /// `RebuildQueueChanged` event. See `job_queue/`. Field name kept + /// from the old flat queue for wire compatibility. + rebuild_queue: Vec, /// Whether the hive-forge container is up. When true the dashboard /// links each container's config + each approval's commit into the /// forge's `agent-configs` repos. @@ -607,7 +608,7 @@ async fn api_state(headers: HeaderMap, State(state): State) -> axum::J question_history, tombstones, port_conflicts, - rebuild_queue: state.coord.rebuild_queue.snapshot(), + rebuild_queue: state.coord.job_queue.snapshot(), forge_present: crate::forge::is_present().await, matrix_gui_enabled: std::env::var_os("HIVE_MATRIX_GUI_ENABLED").is_some_and(|v| { // Accept any truthy string ("1", "true", "yes") since the @@ -1602,30 +1603,15 @@ async fn post_meta_update( return error_response("meta-update: no inputs selected"); } let inputs_label = inputs.join(", "); - let parent_id = state.coord.rebuild_queue.enqueue_with_inputs( - crate::rebuild_queue::QueueKind::MetaUpdate, - "hyperhive".to_owned(), - crate::rebuild_queue::QueueSource::Manual, + // Cascade rebuild children fan out from the MetaLock node when the + // lock bump lands — appended by the scheduler so they build against + // the post-bump lock, and a failed bump simply fans out nothing. + crate::job_queue::submit::meta_update( + &state.coord, + inputs, + crate::job_queue::Source::Manual, format!("meta-update via dashboard ({inputs_label})"), - None, - inputs.clone(), ); - // Pre-enqueue cascade rebuilds NOW so they're visible in the queue - // alongside the parent. The worker's MetaUpdate arm - // no longer enqueues children — it just runs the lock bump and - // (on failure) cancels these pre-queued children. - let cascade_agents = crate::rebuild_queue::meta_update_cascade_agents(&inputs).await; - let cascade_reason = format!("meta-update cascade ({inputs_label})"); - for name in cascade_agents { - state.coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Rebuild, - name, - crate::rebuild_queue::QueueSource::MetaUpdate, - cascade_reason.clone(), - Some(parent_id), - ); - } - state.coord.emit_rebuild_queue_snapshot(); (StatusCode::OK, "ok").into_response() } diff --git a/hive-c0re/src/dashboard/lifecycle_ops.rs b/hive-c0re/src/dashboard/lifecycle_ops.rs index b8b8b3a4..c139feaa 100644 --- a/hive-c0re/src/dashboard/lifecycle_ops.rs +++ b/hive-c0re/src/dashboard/lifecycle_ops.rs @@ -1,10 +1,12 @@ //! Container lifecycle endpoints for the dashboard. //! //! Rebuild / restart / start / stop (hard + graceful) / update-all all -//! enqueue onto the rebuild queue, so each shows a visible queued→running -//! transient on the dashboard — a direct sub-second start/stop only flashed -//! the badge; destroy delegates to `actions::destroy` (optionally -//! purging). +//! submit DAGs to the job queue (`job_queue::submit`), so each shows a +//! visible queued→running transient on the dashboard — a direct +//! sub-second start/stop only flashed the badge. Start/stop also +//! persist the agent's `wanted` power intent before submitting; the +//! DAG's `Reconcile` converges to it. Destroy delegates to +//! `actions::destroy` (optionally purging). use axum::{ extract::{Form, Path as AxumPath, Query, State}, @@ -23,6 +25,7 @@ pub(super) struct KillParams { } use super::{AppState, error_response, guard_agent_name, strip_container_prefix}; +use crate::job_queue::{Source, submit}; use crate::{actions, lifecycle}; pub(super) async fn post_rebuild( @@ -33,14 +36,12 @@ pub(super) async fn post_rebuild( if let Some(reject) = guard_agent_name(&state, &logical).await { return reject; } - state.coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Rebuild, - logical, - crate::rebuild_queue::QueueSource::Manual, + submit::rebuild( + &state.coord, + &logical, + Source::Manual, "manual via dashboard ↻ R3BU1LD button".to_owned(), - None, ); - state.coord.emit_rebuild_queue_snapshot(); (StatusCode::OK, "ok").into_response() } @@ -54,19 +55,17 @@ pub(super) async fn post_kill( return reject; } if params.graceful { - // Graceful stop: enqueue the quiesce orchestration (signal the harness - // → one stop-checkpoint turn → drain → container stop, with a timeout - // fallback to a hard stop). Serialised through the rebuild queue so it - // can't race an in-flight rebuild for the same agent, and its per-step - // progress surfaces on the queue snapshot + build log. - state.coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::GracefulStop, - logical, - crate::rebuild_queue::QueueSource::Manual, + // Graceful stop: submit the quiesce DAG (signal the harness → + // one stop-checkpoint turn → drain → container stop, with a + // timeout fallback to a hard stop). The agent's lifecycle + // lease keeps it from racing an in-flight rebuild for the same + // agent, and per-node progress surfaces on the queue snapshot. + submit::graceful_stop( + &state.coord, + &logical, + Source::Manual, "manual via dashboard graceful stop".to_owned(), - None, ); - state.coord.emit_rebuild_queue_snapshot(); return (StatusCode::OK, "ok").into_response(); } // Manager is stoppable from the dashboard like any other @@ -79,14 +78,12 @@ pub(super) async fn post_kill( // `socket_server.rs::ManagerRequest::Kill` stays in place: a // manager calling Kill on its own container is self-suicide // mid-call, not a legitimate operator action. - state.coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Stop, - logical, - crate::rebuild_queue::QueueSource::Manual, + submit::stop( + &state.coord, + &logical, + Source::Manual, "manual via dashboard stop".to_owned(), - None, ); - state.coord.emit_rebuild_queue_snapshot(); (StatusCode::OK, "ok").into_response() } @@ -98,14 +95,12 @@ pub(super) async fn post_restart( if let Some(reject) = guard_agent_name(&state, &logical).await { return reject; } - state.coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Restart, - logical, - crate::rebuild_queue::QueueSource::Manual, + submit::restart( + &state.coord, + &logical, + Source::Manual, "manual via dashboard ↺ R3START button".to_owned(), - None, ); - state.coord.emit_rebuild_queue_snapshot(); (StatusCode::OK, "ok").into_response() } @@ -117,14 +112,12 @@ pub(super) async fn post_start( if let Some(reject) = guard_agent_name(&state, &logical).await { return reject; } - state.coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Start, - logical, - crate::rebuild_queue::QueueSource::Manual, + submit::start( + &state.coord, + &logical, + Source::Manual, "manual via dashboard start".to_owned(), - None, ); - state.coord.emit_rebuild_queue_snapshot(); (StatusCode::OK, "ok").into_response() } @@ -137,15 +130,13 @@ pub(super) async fn post_update_all(State(state): State) -> Response { else { continue; }; - state.coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Rebuild, - logical, - crate::rebuild_queue::QueueSource::Manual, + submit::rebuild( + &state.coord, + &logical, + Source::Manual, "manual via dashboard 🌀 UPDATE ALL".to_owned(), - None, ); } - state.coord.emit_rebuild_queue_snapshot(); (StatusCode::OK, "ok").into_response() } diff --git a/hive-c0re/src/dashboard/permissions.rs b/hive-c0re/src/dashboard/permissions.rs index 1c41637c..1f133354 100644 --- a/hive-c0re/src/dashboard/permissions.rs +++ b/hive-c0re/src/dashboard/permissions.rs @@ -128,18 +128,19 @@ pub(super) async fn post_tool_groups( return Err(ProblemDetails::from_status_code(StatusCode::BAD_REQUEST) .with_detail(format!("invalid tool-groups for {logical}: {e}"))); } - // Enqueue a PermChange so the JSON file write is serialised through - // the FIFO worker. Prevents concurrent batch-apply actions for - // different agents from racing on the shared tool-groups.json. - state.coord.rebuild_queue.enqueue_with_perm( - logical.clone(), - crate::rebuild_queue::QueueSource::Manual, + // Submit a PermChange DAG: the JSON file write commits under + // META_LOCK inside the WritePermFile node, so concurrent + // batch-apply actions for different agents never race on the + // shared tool-groups.json. + crate::job_queue::submit::perm_change( + &state.coord, + &logical, + crate::job_queue::Source::Manual, "tool-group change via permissions UI".to_owned(), - crate::rebuild_queue::PermPayload::ToolGroups { + crate::job_queue::PermPayload::ToolGroups { groups: body.groups.clone(), }, ); - state.coord.emit_rebuild_queue_snapshot(); tracing::info!(agent = %logical, groups = ?body.groups, "operator: set tool-groups via dashboard"); Ok((StatusCode::OK, "ok").into_response()) } @@ -214,18 +215,19 @@ pub(super) async fn post_capabilities( .with_detail(format!("unknown capability: {cap}"))); } } - // Enqueue a PermChange so the JSON file write is serialised through - // the FIFO worker. Prevents concurrent batch-apply actions for - // different agents from racing on the shared capabilities.json. - state.coord.rebuild_queue.enqueue_with_perm( - logical.clone(), - crate::rebuild_queue::QueueSource::Manual, + // Submit a PermChange DAG: the JSON file write commits under + // META_LOCK inside the WritePermFile node, so concurrent + // batch-apply actions for different agents never race on the + // shared capabilities.json. + crate::job_queue::submit::perm_change( + &state.coord, + &logical, + crate::job_queue::Source::Manual, "capability change via dashboard".to_owned(), - crate::rebuild_queue::PermPayload::Capabilities { + crate::job_queue::PermPayload::Capabilities { caps: body.caps.clone(), }, ); - state.coord.emit_rebuild_queue_snapshot(); tracing::info!(agent = %logical, caps = ?body.caps, "operator: set capabilities via dashboard"); Ok((StatusCode::OK, "ok").into_response()) } @@ -298,17 +300,17 @@ pub(super) async fn post_permissions( )); } } - // Phase 2 — enqueue one combined PermChange per affected agent. + // Phase 2 — submit one combined PermChange DAG per affected agent. for (logical, groups, caps) in staged { - state.coord.rebuild_queue.enqueue_with_perm( - logical.clone(), - crate::rebuild_queue::QueueSource::Manual, + crate::job_queue::submit::perm_change( + &state.coord, + &logical, + crate::job_queue::Source::Manual, "batch permission change via permissions UI".to_owned(), - crate::rebuild_queue::PermPayload::Combined { groups, caps }, + crate::job_queue::PermPayload::Combined { groups, caps }, ); tracing::info!(agent = %logical, "operator: batch perm change via dashboard"); } - state.coord.emit_rebuild_queue_snapshot(); Ok((StatusCode::OK, "ok").into_response()) } diff --git a/hive-c0re/src/dashboard/schedules.rs b/hive-c0re/src/dashboard/schedules.rs index 967146f2..8ad220d9 100644 --- a/hive-c0re/src/dashboard/schedules.rs +++ b/hive-c0re/src/dashboard/schedules.rs @@ -115,20 +115,19 @@ pub(super) async fn post_schedule_fire_now( } } -/// `POST /api/rebuild-queue/{id}/cancel` — drop a `Queued` entry -/// from the rebuild queue. Refuses `Running` / terminal -/// entries: an in-flight rebuild owns the agent's nix store + -/// nixos-container update lock and can't be safely interrupted -/// from the queue side. Always returns 200; the body is -/// `{"cancelled": true}` on a successful flip from Queued → -/// Cancelled, `{"cancelled": false}` when the row was Running / -/// terminal / gone. On success a fresh `RebuildQueueChanged` -/// snapshot fires so the row's state flip surfaces live. +/// `POST /api/rebuild-queue/{id}/cancel` — drop a still-fully-queued +/// DAG from the job queue. Refuses `Running` / terminal DAGs: an +/// in-flight node owns the agent's nix store + nixos-container update +/// lock and can't be safely interrupted from the queue side. Always +/// returns 200; the body is `{"cancelled": true}` on a successful +/// flip to Cancelled, `{"cancelled": false}` when the DAG was +/// Running / terminal / gone. On success a fresh `RebuildQueueChanged` +/// snapshot fires so the state flip surfaces live. pub(super) async fn post_rebuild_queue_cancel( State(state): State, AxumPath(id): AxumPath, ) -> Response { - let cancelled = state.coord.rebuild_queue.cancel(id); + let cancelled = state.coord.job_queue.cancel(id); if cancelled { state.coord.emit_rebuild_queue_snapshot(); axum::Json(serde_json::json!({"cancelled": true})).into_response() diff --git a/hive-c0re/src/dashboard_events.rs b/hive-c0re/src/dashboard_events.rs index 21f5d820..04590dca 100644 --- a/hive-c0re/src/dashboard_events.rs +++ b/hive-c0re/src/dashboard_events.rs @@ -8,7 +8,7 @@ use serde::Serialize; use crate::container_view::ContainerView; use crate::dashboard::{MetaInputView, TombstoneView}; -use crate::rebuild_queue::QueueEntry; +use crate::job_queue::DagView; use chrono::{DateTime, Utc}; #[derive(Debug, Clone, Serialize)] @@ -210,7 +210,7 @@ pub enum DashboardEvent { /// the add/remove races a per-row event would have, and the /// dashboard's grouping (`parent_id`) is most naturally re-derived /// from the full list. - RebuildQueueChanged { seq: u64, queue: Vec }, + RebuildQueueChanged { seq: u64, queue: Vec }, /// Full snapshot of all scheduled prompts. Emitted after every /// operator mutation (new / edit / cancel / fire-now) and after the /// worker fires or rearms a row. Same snapshot-shape rationale as diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs new file mode 100644 index 00000000..0e9810e8 --- /dev/null +++ b/hive-c0re/src/job_queue/exec.rs @@ -0,0 +1,433 @@ +//! Node executors — one async fn per [`NodeKind`], each a thin wrapper +//! over existing `lifecycle.rs` / `meta.rs` / `actions.rs` code. Node +//! executors keep their own internal error handling where it exists +//! today (cold-start fallback inside `Reconcile`, non-fatal boot-time +//! lock bump inside the sweep `MetaLock`, warn-only forge sync in the +//! `Swap` tail); DAG-level failure handling is cancel-downstream in +//! the queue. + +use std::sync::Arc; + +use anyhow::{Context as _, Result}; + +use super::model::{NodeKind, State, Template}; +use super::{Claim, TerminalDag}; +use crate::coordinator::Coordinator; +use crate::power::{ReconcileAction, reconcile_action}; + +/// Max time `Drain` waits for the harness to run its stop-checkpoint +/// turn before falling back to the hard stop. Generous — a checkpoint +/// turn can take a while — but bounded so a wedged agent never blocks +/// the stop indefinitely. Drains hold no build slot, so a whole-hive +/// graceful stop overlaps every agent's drain instead of serialising +/// N × this timeout. +pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(3); + +/// Extra signal an executor hands back to the scheduler alongside +/// success. +#[derive(Debug, Default)] +pub struct NodeOutput { + /// Agents to fan child `Rebuild` DAGs out for (`MetaLock` only). + pub fanout: Vec, +} + +/// Step-label + build-log sink for one claimed node. +struct Ctx<'a> { + coord: &'a Arc, + dag_id: u64, + node_id: super::NodeId, +} + +impl Ctx<'_> { + fn step(&self, step: &str) { + if self + .coord + .job_queue + .set_step(self.dag_id, self.node_id, step) + { + self.coord.emit_rebuild_queue_snapshot(); + } + } + + fn build_log(&self, log_id: i64) { + if self + .coord + .job_queue + .set_build_log_id(self.dag_id, self.node_id, log_id) + { + self.coord.emit_rebuild_queue_snapshot(); + } + } +} + +/// Run one claimed node to completion. Called from a task the +/// scheduler spawns per claim; the `Result` (stringified) becomes the +/// node's terminal state. +pub(super) async fn run_node(coord: &Arc, claim: &Claim) -> Result { + let ctx = Ctx { + coord, + dag_id: claim.dag_id, + node_id: claim.node_id, + }; + match &claim.kind { + NodeKind::Prebuild { relock } => run_prebuild(coord, claim, &ctx, *relock).await, + NodeKind::Swap => run_swap(coord, claim, &ctx).await, + NodeKind::Create => run_create(coord, claim, &ctx).await, + NodeKind::MetaLock { sweep, fanout } => { + run_meta_lock(coord, claim, &ctx, *sweep, fanout.clone()).await + } + NodeKind::Reconcile => run_reconcile(coord, claim, &ctx).await, + NodeKind::StopForUpdate => run_stop_for_update(coord, claim, &ctx).await, + NodeKind::Signal => Ok(run_signal(coord, claim, &ctx)), + NodeKind::Drain => run_drain(coord, claim, &ctx).await, + NodeKind::WriteDropin => run_write_dropin(coord, claim).await, + NodeKind::WritePermFile => run_write_perm_file(coord, claim, &ctx).await, + NodeKind::ApprovalDeploy => run_approval_deploy(coord, claim).await, + } +} + +/// Out-of-band toplevel build while the container keeps serving: meta +/// sync + optional per-agent relock, then warm +/// `system.build.toplevel` so the later `Swap` hits cache and skips +/// straight to the profile-swap. +async fn run_prebuild( + coord: &Arc, + claim: &Claim, + ctx: &Ctx<'_>, + relock: bool, +) -> Result { + let name = &claim.agent; + let agent_dir = coord + .ensure_runtime(name) + .with_context(|| format!("ensure_runtime {name}"))?; + let hive = coord.hive_env(); + let paths = Coordinator::agent_paths(name, agent_dir); + crate::lifecycle::prepare_rebuild_dirs(name, &paths).await?; + // Idempotent meta sync so a manual rebuild can also recover from a + // divergent meta repo; then bump just this agent's input. `relock = + // false` only for meta-update cascade children, where re-locking + // would revert the bump the cascade just committed. + let agents = crate::lifecycle::agents_for_meta_listing().await?; + crate::meta::sync_agents(&hive, &agents).await?; + if relock { + crate::meta::lock_update_for_rebuild(name).await?; + } + ctx.step("nix build"); + let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display()); + crate::lifecycle::prebuild_toplevel(name, &flake_ref, &|log_id| ctx.build_log(log_id)).await?; + Ok(NodeOutput::default()) +} + +/// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb), +/// `nixos-container update`, then the post-rebuild bookkeeping tail +/// (rev marker, `Rebuilt` event, forge/matrix sync, kick, rescan). +/// The recovery-start on failure is NOT here — the DAG's tail +/// `Reconcile` runs after this node terminal ok *or* fail. +async fn run_swap(coord: &Arc, claim: &Claim, ctx: &Ctx<'_>) -> Result { + let name = &claim.agent; + let agent_dir = coord.ensure_runtime(name)?; + let hive = coord.hive_env(); + let paths = Coordinator::agent_paths(name, agent_dir); + let result = + crate::lifecycle::swap_update(name, &hive, &paths, &|step| ctx.step(step), &|log_id| { + ctx.build_log(log_id) + }) + .await; + match &result { + Ok(()) => { + if let Some(rev) = crate::auto_update::current_flake_rev(&coord.hyperhive_flake) + && let Err(e) = std::fs::write(crate::auto_update::rev_marker_path(name), rev) + { + tracing::warn!(%name, error = ?e, "write rev marker failed"); + } + coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { + agent: name.clone(), + ok: true, + note: None, + sha: None, + tag: None, + }); + ctx.step("forge sync"); + // Full forge + matrix sync on every successful rebuild so + // the rebuild path is equivalent to the startup sweep: + // tokens, config-repo mirror, meta access all recover + // without a hive-c0re restart. + crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await; + crate::matrix::sync_agent_standalone(name).await; + // Wake the agent on its next turn so claude sees a "you + // were rebuilt" hint; rescan so dashboards drop the + // "needs update" chip; lock bump → meta-inputs re-render. + coord.kick_agent(name, "container rebuilt"); + coord.rescan_containers_and_emit().await; + crate::dashboard::emit_meta_inputs_snapshot(coord); + } + Err(_) => { + // The `Rebuilt { ok: false }` manager event fires once per + // DAG from the terminal hook (any node may be the one that + // failed); here only refresh the observed state. + coord.rescan_containers_and_emit().await; + } + } + result.map(|()| NodeOutput::default()) +} + +/// First-spawn provisioning + `nixos-container create` (atomic +/// build+create — no prebuild needed). +async fn run_create(coord: &Arc, claim: &Claim, ctx: &Ctx<'_>) -> Result { + let name = &claim.agent; + let agent_dir = coord.ensure_runtime(name)?; + let hive = coord.hive_env(); + let paths = Coordinator::agent_paths(name, agent_dir); + ctx.step("nixos-container create"); + crate::lifecycle::create_container(name, &hive, &paths).await?; + Ok(NodeOutput::default()) +} + +/// Meta flake lock bump. Boot-sweep flavour is non-fatal (a failed +/// bump must not cancel the fan-out rebuilds — they proceed against +/// the current lock, exactly like today's sweep); the meta-update +/// flavour propagates errors, and a failed bump fans out nothing. +async fn run_meta_lock( + coord: &Arc, + claim: &Claim, + ctx: &Ctx<'_>, + sweep: bool, + fanout: Option>, +) -> Result { + if sweep { + ctx.step("nix flake update hyperhive"); + if let Err(e) = crate::meta::lock_update_hyperhive().await { + tracing::warn!(error = ?e, "startup sweep: meta lock_update_hyperhive failed"); + } + return Ok(NodeOutput { + fanout: fanout.unwrap_or_default(), + }); + } + let _progress = coord.meta_update_guard(); + ctx.step("nix flake update"); + crate::meta::lock_update(&claim.inputs).await?; + // Lock file changed — meta-inputs panel re-renders. + crate::dashboard::emit_meta_inputs_snapshot(coord); + let cascade = match fanout { + Some(list) => list, + None => meta_update_cascade_agents(&claim.inputs).await, + }; + Ok(NodeOutput { fanout: cascade }) +} + +/// Idempotent power converge: `wanted` (durable intent) vs observed. +async fn run_reconcile( + coord: &Arc, + claim: &Claim, + ctx: &Ctx<'_>, +) -> Result { + let name = &claim.agent; + let running = crate::lifecycle::is_running(name).await; + let wanted = coord.power.get_or_seed(name, running)?; + match reconcile_action(wanted, running) { + ReconcileAction::Start => { + // Node-local transient only when the DAG holds none (the + // boot-reconcile template); lease-window guards otherwise + // already cover this node. + let _guard = claim + .transient + .is_none() + .then(|| coord.transient_guard(name, crate::coordinator::TransientKind::Starting)); + ctx.step("nixos-container start"); + crate::lifecycle::start_with_fallback(name).await?; + coord.kick_agent(name, "container started"); + coord.rescan_containers_and_emit().await; + } + ReconcileAction::Stop => { + let _guard = claim + .transient + .is_none() + .then(|| coord.transient_guard(name, crate::coordinator::TransientKind::Stopping)); + ctx.step("nixos-container stop"); + crate::lifecycle::kill(name).await?; + coord.unregister_agent(name); + coord.notify_manager(&hive_sh4re::HelperEvent::Killed { + agent: name.clone(), + }); + coord.rescan_containers_and_emit().await; + } + ReconcileAction::Noop => { + tracing::debug!(%name, wanted = wanted.as_str(), running, "reconcile: noop"); + } + } + Ok(NodeOutput::default()) +} + +/// Mechanical stop for the profile swap. Never touches `wanted`; noop +/// when already stopped. +async fn run_stop_for_update( + coord: &Arc, + claim: &Claim, + ctx: &Ctx<'_>, +) -> Result { + let name = &claim.agent; + if crate::lifecycle::is_running(name).await { + ctx.step("nixos-container stop"); + crate::lifecycle::kill(name).await?; + coord.rescan_containers_and_emit().await; + } + Ok(NodeOutput::default()) +} + +/// Set the graceful fence + kick so the harness sees it promptly and +/// runs its one stop-checkpoint turn. +fn run_signal(coord: &Arc, claim: &Claim, ctx: &Ctx<'_>) -> NodeOutput { + ctx.step("graceful stop: signalling agent"); + coord.mark_graceful_stop(&claim.agent); + coord.kick_agent(&claim.agent, "graceful stop requested"); + NodeOutput::default() +} + +/// Await the harness clearing the fence (`GracefulStopComplete`) or +/// the timeout — either way the downstream `Reconcile` proceeds with +/// the actual stop. +async fn run_drain(coord: &Arc, claim: &Claim, ctx: &Ctx<'_>) -> Result { + let name = &claim.agent; + ctx.step("graceful stop: draining"); + let deadline = std::time::Instant::now() + GRACEFUL_STOP_TIMEOUT; + while coord.is_graceful_stop_pending(name) { + if std::time::Instant::now() >= deadline { + tracing::warn!(agent = %name, "graceful stop: drain timed out — hard-stopping"); + break; + } + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + } + coord.clear_graceful_stop(name); + Ok(NodeOutput::default()) +} + +/// `set_nspawn_flags` + `set_resource_limits` + daemon-reload. +async fn run_write_dropin(coord: &Arc, claim: &Claim) -> Result { + let name = &claim.agent; + let agent_dir = coord.ensure_runtime(name)?; + let hive = coord.hive_env(); + let paths = Coordinator::agent_paths(name, agent_dir); + crate::lifecycle::write_dropins(name, &hive, &paths).await?; + Ok(NodeOutput::default()) +} + +/// Write + commit the perm file(s) (fused under `META_LOCK` so the +/// working tree is never left dirty), then emit the P3RM1SS10NS-tab +/// snapshots so the dashboard reflects the new assignment. +async fn run_write_perm_file( + coord: &Arc, + claim: &Claim, + ctx: &Ctx<'_>, +) -> Result { + use super::model::PermPayload; + let name = &claim.agent; + ctx.step("writing + committing perm file"); + match &claim.perm_payload { + Some(PermPayload::ToolGroups { groups }) => { + crate::meta::commit_tool_groups(name, groups) + .await + .with_context(|| format!("commit tool-groups for {name}"))?; + coord.emit_tool_groups_snapshot(); + } + Some(PermPayload::Capabilities { caps }) => { + crate::meta::commit_capabilities(name, caps) + .await + .with_context(|| format!("commit capabilities for {name}"))?; + coord.emit_capabilities_snapshot(); + } + Some(PermPayload::Combined { groups, caps }) => { + crate::meta::commit_perms(name, groups.as_deref(), caps.as_deref()) + .await + .with_context(|| format!("commit perms for {name}"))?; + if groups.is_some() { + coord.emit_tool_groups_snapshot(); + } + if caps.is_some() { + coord.emit_capabilities_snapshot(); + } + } + None => anyhow::bail!( + "perm_change dag {} for {name} is missing perm_payload", + claim.dag_id + ), + } + Ok(NodeOutput::default()) +} + +/// Opaque approval deploy pipeline: `ApplyCommit` and `MergeConfigPr` +/// both end in a container rebuild; branch on the approval row's kind +/// (the authoritative source). The two-phase prepare/finalize/abort +/// meta deploy — and the approval resolution — stay inside +/// `actions.rs` in v1 (design doc §9). +async fn run_approval_deploy(coord: &Arc, claim: &Claim) -> Result { + let approval_id = claim + .approval_id + .with_context(|| format!("approval_deploy dag {} has no approval_id", claim.dag_id))?; + let kind = coord + .approvals + .get(approval_id) + .ok() + .flatten() + .map(|a| a.kind); + let result = if kind == Some(hive_sh4re::ApprovalKind::MergeConfigPr) { + crate::actions::run_approval_merge_config_pr(coord, Some(claim.dag_id), approval_id).await + } else { + crate::actions::run_approval_apply_commit(coord, Some(claim.dag_id), approval_id).await + }; + result.map(|()| NodeOutput::default()) +} + +/// Terminal-roll-up hook, fired exactly once per DAG. Approval DAGs +/// resolve their approval row (except the opaque deploy pipeline, +/// which resolves inside its node); non-approval rebuild-shaped DAGs +/// surface the `Rebuilt { ok: false }` manager event on failure — +/// success fires from the `Swap` tail, matching today's timing. +pub(super) async fn on_dag_terminal(coord: &Arc, terminal: &TerminalDag) { + if terminal.approval_id.is_some() { + crate::actions::resolve_approval_dag(coord, terminal).await; + return; + } + if matches!(terminal.template, Template::Rebuild | Template::PermChange) + && terminal.state == State::Failed + { + coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { + agent: terminal.agent.clone(), + ok: false, + note: terminal.error.clone(), + sha: None, + tag: None, + }); + } +} + +/// Compute which agents a `nix flake update ` on the meta +/// flake affects — the fan-out set for `MetaUpdate` DAGs. Empty +/// `inputs` or any input under `hyperhive` → every container; +/// otherwise just the agents named by `agent-` inputs. +/// Topology-sorted so parents rebuild before their children. +pub async fn meta_update_cascade_agents(inputs: &[String]) -> Vec { + let touched_hyperhive = inputs + .iter() + .any(|i| i == "hyperhive" || i.starts_with("hyperhive/")); + let touched_agents: Vec = inputs + .iter() + .filter_map(|i| i.strip_prefix("agent-")) + .map(|rest| rest.split('/').next().unwrap_or(rest).to_owned()) + .collect(); + let mut names = if touched_hyperhive || inputs.is_empty() { + crate::lifecycle::list() + .await + .unwrap_or_default() + .into_iter() + .filter_map(|c| { + c.strip_prefix(crate::lifecycle::AGENT_PREFIX) + .map(str::to_owned) + }) + .collect() + } else { + touched_agents + }; + let topo = crate::topology::read(); + crate::auto_update::topology_sort(&mut names, &topo); + names +} diff --git a/hive-c0re/src/job_queue/mod.rs b/hive-c0re/src/job_queue/mod.rs new file mode 100644 index 00000000..9aa91d07 --- /dev/null +++ b/hive-c0re/src/job_queue/mod.rs @@ -0,0 +1,565 @@ +//! Generic job-DAG queue + desired-state reconciliation — replaces the +//! old flat `rebuild_queue`. Jobs are nodes in per-request DAGs (see +//! [`templates`]); the special cases (graceful-stop watcher thread, +//! deferred-start follow-up, meta-update cascade) collapse into DAG +//! *shapes* over a shared set of primitive nodes ([`model::NodeKind`]). +//! +//! Concurrency is gated by two resource classes: +//! 1. **Build slots** — N permits (`services.hyperhive.c0re.buildSlots`, +//! default 1) held by nix-heavy nodes for the node's duration. +//! 2. **Per-agent lifecycle lease** — DAG-scoped: acquired before the +//! DAG's first container-affecting node runs, held until the DAG is +//! terminal, so two lifecycle DAGs for one agent never interleave +//! their container ops. +//! +//! The meta *repo* is serialized by `meta::META_LOCK` inside the +//! executors themselves. Per-agent power *intent* (`wanted`) lives in +//! the durable [`crate::power`] store; the DAGs are the reconcile +//! mechanism. Design + rationale: `docs/coordinator.md::Job queue`. + +pub mod exec; +pub mod model; +pub mod scheduler; +pub mod submit; +pub mod templates; +#[cfg(test)] +mod tests; + +use std::collections::{HashMap, VecDeque}; +use std::sync::Mutex; + +use tokio::sync::Notify; + +pub use model::{ + Dag, DagSpec, DagView, DepWhen, Node, NodeId, NodeKind, PermPayload, Source, State, Template, +}; + +/// How many terminal DAGs (`Done` / `Failed` / `Cancelled`) to retain +/// per template in the snapshot, matching the old per-kind history cap. +const MAX_HISTORY_PER_TEMPLATE: usize = 5; + +/// Cap on stored node error strings. +const MAX_ERROR_LEN: usize = 2_000; + +/// A node claimed for execution — everything the executor needs, +/// snapshotted at claim time. +#[derive(Debug, Clone)] +pub struct Claim { + pub dag_id: u64, + pub node_id: NodeId, + pub kind: NodeKind, + pub agent: String, + pub template: Template, + pub source: Source, + pub approval_id: Option, + pub inputs: Vec, + pub perm_payload: Option, + /// True when claiming this node acquired the DAG's agent lease — + /// the scheduler creates the DAG-scoped transient guard on this + /// edge. + pub lease_acquired: bool, + /// Transient pill kind for the lease window (from the spec). + pub transient: Option, +} + +/// Summary of a DAG that just reached its terminal roll-up state — +/// input to the approval-resolution hook and the lease/transient +/// release. +#[derive(Debug, Clone)] +pub struct TerminalDag { + pub dag_id: u64, + pub template: Template, + pub agent: String, + pub approval_id: Option, + pub state: State, + /// First failed node's error when `state == Failed`. + pub error: Option, +} + +/// Report from [`JobQueue::complete_node`]. +#[derive(Debug, Default)] +pub struct CompletionReport { + /// DAGs that became terminal as a result of this completion + /// (the completed node's own DAG, plus none others — but kept as a + /// Vec so cancel paths can reuse the same settle plumbing). + pub terminal: Vec, +} + +#[derive(Debug, Default)] +struct Inner { + dags: VecDeque, + next_id: u64, + build_slots: usize, + slots_used: usize, + /// agent → dag id currently holding that agent's lifecycle lease. + leases: HashMap, +} + +/// The queue. Lives on `Coordinator` (one per hive-c0re process); a +/// single scheduler task ([`scheduler::run_worker`]) drives it — +/// concurrency comes from the build-slot count, not multiple workers. +#[derive(Debug)] +pub struct JobQueue { + inner: Mutex, + /// Wakes the scheduler when something new arrives or state changed. + pub(crate) notify: Notify, +} + +impl Default for JobQueue { + fn default() -> Self { + Self::new(1) + } +} + +impl JobQueue { + pub fn new(build_slots: usize) -> Self { + Self { + inner: Mutex::new(Inner { + build_slots: build_slots.max(1), + ..Inner::default() + }), + notify: Notify::new(), + } + } + + /// Submit a DAG. Validates the spec (cycle rejection) and dedups + /// against non-started DAGs; returns the DAG id (newly-allocated, + /// or the existing DAG's id with the new reason appended). + /// + /// Dedup: a DAG whose roll-up is still `Queued` (no node started) + /// with the same `(template, agent, parent_id, approval_id)` — plus + /// `inputs` for `MetaUpdate` and the perm-type discriminant for + /// `PermChange` — swallows the repeat. `parent_id` is part of the + /// key so a meta-update cascade rebuild never collapses into a + /// standalone or sweep rebuild. Running / terminal DAGs never + /// dedup — operators are free to re-queue. + pub fn submit(&self, spec: DagSpec) -> anyhow::Result { + templates::validate(&spec)?; + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + if let Some(existing) = Self::dedup_target(&mut inner, &spec) { + if !existing.reason.contains(&spec.reason) { + use std::fmt::Write as _; + let _ = write!(existing.reason, "\nalso requested by: {}", spec.reason); + } + return Ok(existing.id); + } + let id = Self::push_dag(&mut inner, spec); + drop(inner); + self.notify.notify_one(); + Ok(id) + } + + /// Append fan-out children under a parent DAG (meta-update / sweep + /// cascade). Applies the same dedup as [`Self::submit`]; returns + /// the child ids actually created or coalesced into. + pub fn append_children(&self, specs: Vec) -> Vec { + let mut ids = Vec::with_capacity(specs.len()); + for spec in specs { + match self.submit(spec) { + Ok(id) => ids.push(id), + Err(e) => tracing::error!(error = ?e, "job_queue: invalid fan-out child spec"), + } + } + ids + } + + fn dedup_target<'a>(inner: &'a mut Inner, spec: &DagSpec) -> Option<&'a mut Dag> { + inner.dags.iter_mut().find(|d| { + d.rollup() == State::Queued + && d.template == spec.template + && d.agent == spec.agent + && d.parent_id == spec.parent_id + && d.approval_id == spec.approval_id + && (d.template != Template::MetaUpdate || d.inputs == spec.inputs) + && PermPayload::same_type(d.perm_payload.as_ref(), spec.perm_payload.as_ref()) + }) + } + + fn push_dag(inner: &mut Inner, spec: DagSpec) -> u64 { + inner.next_id += 1; + let id = inner.next_id; + let nodes = spec + .nodes + .into_iter() + .enumerate() + .map(|(i, n)| Node { + id: u32::try_from(i).unwrap_or(u32::MAX), + kind: n.kind, + deps: n.deps, + state: State::Queued, + step: None, + build_log_id: None, + started_at: None, + finished_at: None, + error: None, + }) + .collect(); + inner.dags.push_back(Dag { + id, + template: spec.template, + agent: spec.agent, + source: spec.source, + reason: spec.reason, + parent_id: spec.parent_id, + approval_id: spec.approval_id, + inputs: spec.inputs, + perm_payload: spec.perm_payload, + transient: spec.transient, + created_at: now_unix(), + nodes, + terminal_reported: false, + }); + id + } + + /// Claim every currently-ready node, acquiring resources, and mark + /// them `Running`. A node is ready when it's `Queued`, every dep is + /// satisfied (`AfterOk`: dep `Done`; `AfterAny`: dep terminal), and + /// its resources are free (build slot; agent lease free or already + /// held by this DAG). Iteration is in DAG-submit order, so + /// simultaneously-ready nodes compete FIFO — bulk operations drain + /// predictably. + pub fn claim_ready(&self) -> Vec { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + Self::propagate_cancellations(&mut inner); + let mut claims = Vec::new(); + let inner = &mut *inner; + for di in 0..inner.dags.len() { + // Split-borrow dance: deps are checked against the same + // DAG's other nodes, so snapshot the states first. + let dag = &inner.dags[di]; + let dag_id = dag.id; + let ready_ids: Vec = dag + .nodes + .iter() + .filter(|n| n.state == State::Queued && Self::deps_satisfied(dag, n)) + .map(|n| n.id) + .collect(); + for node_id in ready_ids { + let dag = &inner.dags[di]; + let node = dag.node(node_id).expect("node id from same dag"); + let needs_slot = node.kind.needs_build_slot(); + if needs_slot && inner.slots_used >= inner.build_slots { + continue; + } + let mut lease_acquired = false; + if node.kind.needs_lease() { + match inner.leases.get(dag.agent.as_str()) { + Some(&holder) if holder != dag_id => continue, + Some(_) => {} + None => { + inner.leases.insert(dag.agent.clone(), dag_id); + lease_acquired = true; + } + } + } + if needs_slot { + inner.slots_used += 1; + } + let dag = &mut inner.dags[di]; + let claim = Claim { + dag_id, + node_id, + kind: dag.node(node_id).expect("node").kind.clone(), + agent: dag.agent.clone(), + template: dag.template, + source: dag.source, + approval_id: dag.approval_id, + inputs: dag.inputs.clone(), + perm_payload: dag.perm_payload.clone(), + lease_acquired, + transient: dag.transient, + }; + let node = dag.node_mut(node_id).expect("node"); + node.state = State::Running; + node.started_at = Some(now_unix()); + claims.push(claim); + } + } + claims + } + + fn deps_satisfied(dag: &Dag, node: &Node) -> bool { + node.deps.iter().all(|dep| { + dag.node(dep.on).is_some_and(|d| match dep.when { + DepWhen::AfterOk => d.state == State::Done, + DepWhen::AfterAny => d.state.is_terminal(), + }) + }) + } + + /// Cancel-downstream: a `Queued` node with an `AfterOk` dep that + /// `Failed` / `Cancelled` becomes `Cancelled` itself. Loops to a + /// fixpoint so the cancellation cascades through chains. + fn propagate_cancellations(inner: &mut Inner) { + for dag in &mut inner.dags { + loop { + let doomed: Vec = dag + .nodes + .iter() + .filter(|n| { + n.state == State::Queued + && n.deps.iter().any(|dep| { + dep.when == DepWhen::AfterOk + && dag.node(dep.on).is_some_and(|d| { + matches!(d.state, State::Failed | State::Cancelled) + }) + }) + }) + .map(|n| n.id) + .collect(); + if doomed.is_empty() { + break; + } + let now = now_unix(); + for id in doomed { + if let Some(n) = dag.node_mut(id) { + n.state = State::Cancelled; + n.finished_at = Some(now); + } + } + } + } + } + + /// Mark a claimed node terminal, release its build slot, cascade + /// cancellations, and settle terminal DAGs (lease release + history + /// trim). `error` is stored (truncated) when `result` is `Err`. + pub fn complete_node( + &self, + dag_id: u64, + node_id: NodeId, + result: Result<(), String>, + ) -> CompletionReport { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + if let Some(dag) = inner.dags.iter_mut().find(|d| d.id == dag_id) + && let Some(node) = dag.node_mut(node_id) + && node.state == State::Running + { + let needs_slot = node.kind.needs_build_slot(); + node.finished_at = Some(now_unix()); + node.step = None; + match result { + Ok(()) => node.state = State::Done, + Err(e) => { + node.state = State::Failed; + let mut msg = e; + if msg.len() > MAX_ERROR_LEN { + msg.truncate( + (0..=MAX_ERROR_LEN) + .rev() + .find(|i| msg.is_char_boundary(*i)) + .unwrap_or(0), + ); + msg.push('…'); + } + node.error = Some(msg); + } + } + if needs_slot { + inner.slots_used = inner.slots_used.saturating_sub(1); + } + } + let report = Self::settle(&mut inner); + drop(inner); + self.notify.notify_one(); + report + } + + /// Propagate cancellations, release the leases of newly-terminal + /// DAGs, and trim history. Each terminal DAG is reported exactly + /// once (the `terminal_reported` flag) so the scheduler's hooks — + /// approval resolution, transient-guard release — fire once per + /// DAG. + fn settle(inner: &mut Inner) -> CompletionReport { + Self::propagate_cancellations(inner); + let mut report = CompletionReport::default(); + let mut freed: Vec = Vec::new(); + for dag in &mut inner.dags { + if !dag.is_terminal() || dag.terminal_reported { + continue; + } + dag.terminal_reported = true; + if inner.leases.get(dag.agent.as_str()) == Some(&dag.id) { + freed.push(dag.agent.clone()); + } + report.terminal.push(TerminalDag { + dag_id: dag.id, + template: dag.template, + agent: dag.agent.clone(), + approval_id: dag.approval_id, + state: dag.rollup(), + error: dag.first_error().map(str::to_owned), + }); + } + for agent in freed { + inner.leases.remove(&agent); + } + Self::trim_history(inner); + report + } + + /// Cancel a DAG that hasn't started yet (roll-up `Queued`): every + /// node flips to `Cancelled`. No-op (returns `false`) once any node + /// is running or terminal — an in-flight nix build isn't + /// interruptible, matching the old queue's rule. + pub fn cancel(&self, dag_id: u64) -> bool { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + let Some(dag) = inner.dags.iter_mut().find(|d| d.id == dag_id) else { + return false; + }; + if dag.rollup() != State::Queued { + return false; + } + let now = now_unix(); + for n in &mut dag.nodes { + n.state = State::Cancelled; + n.finished_at = Some(now); + } + let _ = Self::settle(&mut inner); + drop(inner); + self.notify.notify_one(); + true + } + + /// Cancel every still-fully-queued child DAG of `parent`. Running + /// children are left alone. Returns the count of cancelled DAGs. + pub fn cancel_children(&self, parent: u64) -> usize { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + let now = now_unix(); + let mut count = 0; + for dag in &mut inner.dags { + if dag.parent_id == Some(parent) && dag.rollup() == State::Queued { + for n in &mut dag.nodes { + n.state = State::Cancelled; + n.finished_at = Some(now); + } + count += 1; + } + } + if count > 0 { + let _ = Self::settle(&mut inner); + drop(inner); + self.notify.notify_one(); + } + count + } + + /// Set the step label on a `Running` node. Returns `true` when the + /// label actually changed (callers emit a snapshot only then). + pub fn set_step(&self, dag_id: u64, node_id: NodeId, step: &str) -> bool { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + let Some(node) = inner + .dags + .iter_mut() + .find(|d| d.id == dag_id) + .and_then(|d| d.node_mut(node_id)) + else { + return false; + }; + if node.state != State::Running || node.step.as_deref() == Some(step) { + return false; + } + node.step = Some(step.to_owned()); + true + } + + /// Set the step label on the DAG's currently-running node — + /// compatibility surface for the opaque approval pipeline, whose + /// callbacks only know the DAG id. Single-node approval DAGs make + /// this exact. + pub fn set_step_running(&self, dag_id: u64, step: &str) -> bool { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + let Some(node) = inner + .dags + .iter_mut() + .find(|d| d.id == dag_id) + .and_then(|d| d.nodes.iter_mut().find(|n| n.state == State::Running)) + else { + return false; + }; + if node.step.as_deref() == Some(step) { + return false; + } + node.step = Some(step.to_owned()); + true + } + + /// Link a `build_logs` row to a specific `Running` node. + pub fn set_build_log_id(&self, dag_id: u64, node_id: NodeId, log_id: i64) -> bool { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + let Some(node) = inner + .dags + .iter_mut() + .find(|d| d.id == dag_id) + .and_then(|d| d.node_mut(node_id)) + else { + return false; + }; + if node.state != State::Running { + return false; + } + node.build_log_id = Some(log_id); + true + } + + /// Link a `build_logs` row to the DAG's currently-running node — + /// DAG-id-only compatibility surface (approval pipeline callbacks). + pub fn set_build_log_id_running(&self, dag_id: u64, log_id: i64) -> bool { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + let Some(node) = inner + .dags + .iter_mut() + .find(|d| d.id == dag_id) + .and_then(|d| d.nodes.iter_mut().find(|n| n.state == State::Running)) + else { + return false; + }; + node.build_log_id = Some(log_id); + true + } + + /// Snapshot every DAG for `/api/state` + `RebuildQueueChanged`. + pub fn snapshot(&self) -> Vec { + let inner = self.inner.lock().expect("job_queue mutex poisoned"); + inner.dags.iter().map(Dag::view).collect() + } + + /// Number of live (non-terminal) DAGs — used by tests and + /// diagnostics. + #[cfg(test)] + pub fn live_count(&self) -> usize { + let inner = self.inner.lock().expect("job_queue mutex poisoned"); + inner.dags.iter().filter(|d| !d.is_terminal()).count() + } + + /// Keep only the newest `MAX_HISTORY_PER_TEMPLATE` terminal DAGs + /// per template; live DAGs are never evicted. + fn trim_history(inner: &mut Inner) { + let mut counts: HashMap = HashMap::new(); + let kept: Vec = inner + .dags + .iter() + .rev() + .filter(|d| { + if !d.is_terminal() { + return true; + } + let n = counts.entry(d.template).or_insert(0); + *n += 1; + *n <= MAX_HISTORY_PER_TEMPLATE + }) + .cloned() + .collect(); + inner.dags = kept.into_iter().rev().collect(); + } +} + +/// Current unix timestamp in seconds. +fn now_unix() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .ok() + .and_then(|d| i64::try_from(d.as_secs()).ok()) + .unwrap_or(0) +} diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs new file mode 100644 index 00000000..8c4ebcf6 --- /dev/null +++ b/hive-c0re/src/job_queue/model.rs @@ -0,0 +1,520 @@ +//! Data model for the generic job-DAG queue: templates (what a DAG +//! *means*), node kinds (the primitive operations), dependency edges, +//! states, and the wire-facing `DagView` / `NodeView` snapshot shapes. +//! +//! Two levels: the **DAG** is the unit of dedup / cancel / +//! approval-resolution and the dashboard group; the **node** is the +//! unit of scheduling / execution / build-log / step label. See +//! `docs/coordinator.md::Job queue` for the full design. + +use serde::{Deserialize, Serialize}; + +/// What a DAG *means* — the request-level shape. Wire strings match the +/// old `QueueKind` values (serialized as the `kind` field on `DagView`) +/// so the dashboard's glyph map keeps working. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Template { + /// Rebuild one agent's container: `Prebuild → StopForUpdate → Swap + /// → Reconcile` (the tail `Reconcile` runs after `Swap` terminal, + /// ok *or* fail — the recovery-start). + Rebuild, + /// Bump meta flake locks: one `MetaLock` node; child `Rebuild` + /// DAGs fan out on completion for every affected agent. + MetaUpdate, + /// First-deploy spawn (approval-driven): `Create → WriteDropin → + /// Reconcile`. + Spawn, + /// Reserved for a future destroy integration — kept so the wire + /// shape doesn't need to change later. + #[allow(dead_code, reason = "wire shape — routed by a future PR")] + Destroy, + /// Boot-time config sweep: one `MetaLock` (hyperhive input, + /// non-fatal) node; stale agents' `Rebuild` DAGs fan out on + /// completion. + StartupSweep, + /// `StopForUpdate → Reconcile` — stop + converge back to `wanted` + /// (unchanged), i.e. a restart for a wanted-up agent. + Restart, + /// `WritePermFile → Prebuild → StopForUpdate → Swap → Reconcile` — + /// perm-file commit followed by the rebuild subgraph. + PermChange, + /// `Signal → Drain → Reconcile` with `wanted` set to `Offline` at + /// submit time: quiesce the harness, await the drain (bounded), + /// then the tail `Reconcile` performs the actual container stop. + GracefulStop, + /// Single `Reconcile` with `wanted` set to `Up` at submit time. + Start, + /// Single `Reconcile` with `wanted` set to `Offline` at submit time. + Stop, + /// Single `Reconcile` with `wanted` untouched — boot-time converge + /// of observed state to the persisted intent. + Reconcile, +} + +impl Template { + pub fn as_str(self) -> &'static str { + match self { + Template::Rebuild => "rebuild", + Template::MetaUpdate => "meta_update", + Template::Spawn => "spawn", + Template::Destroy => "destroy", + Template::StartupSweep => "startup_sweep", + Template::Restart => "restart", + Template::PermChange => "perm_change", + Template::GracefulStop => "graceful_stop", + Template::Start => "start", + Template::Stop => "stop", + Template::Reconcile => "reconcile", + } + } +} + +/// Where the submit request originated. Same variants + wire strings +/// as the old `QueueSource` — drives the "why" chip on the dashboard. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Source { + /// Operator action (dashboard button, CLI, manager tool). + Manual, + /// Cascade child of a `MetaUpdate` DAG's fan-out; `parent_id` + /// points back at the originating meta-update. + MetaUpdate, + /// Boot-time submission (sweep parent, boot reconciles). + AutoUpdate, + /// Cascade child of a `StartupSweep` DAG's fan-out. + StartupSweep, + /// Crash recovery path (future use). + #[allow(dead_code, reason = "wire shape — used by a future feature")] + CrashRecover, + /// Operator approved a pending `Approval` row; `approval_id` on + /// the DAG points back at the source row. + Approval, +} + +impl Source { + pub fn as_str(self) -> &'static str { + match self { + Source::Manual => "manual", + Source::MetaUpdate => "meta_update", + Source::AutoUpdate => "auto_update", + Source::StartupSweep => "startup_sweep", + Source::CrashRecover => "crash_recover", + Source::Approval => "approval", + } + } +} + +/// Lifecycle state of a node — and, rolled up, of a DAG. Same wire +/// strings as the old `QueueState`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum State { + Queued, + Running, + Done, + Failed, + Cancelled, +} + +impl State { + pub fn is_terminal(self) -> bool { + matches!(self, State::Done | State::Failed | State::Cancelled) + } +} + +/// Kind-specific payload for `Template::PermChange` DAGs. Carried on +/// the DAG (not the node) so dedup can compare the perm *type* +/// discriminant. Identical to the old `rebuild_queue::PermPayload`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum PermPayload { + /// Set the tool groups for one agent (`tool-groups.json`). + ToolGroups { groups: Vec }, + /// Set the capabilities for one agent (`capabilities.json`). + Capabilities { caps: Vec }, + /// Set both perm-types in one entry — the batch + /// `POST /api/permissions` path. `None` leaves that file untouched; + /// the worker commits whichever are present in one git commit, + /// then rebuilds once. + Combined { + groups: Option>, + caps: Option>, + }, +} + +impl PermPayload { + /// Dedup compares the perm *type*, not the value — a tool-groups + /// change and a capabilities change for the same agent are + /// distinct operations that must not collapse. + pub fn same_type(a: Option<&PermPayload>, b: Option<&PermPayload>) -> bool { + matches!( + (a, b), + ( + Some(PermPayload::ToolGroups { .. }), + Some(PermPayload::ToolGroups { .. }) + ) | ( + Some(PermPayload::Capabilities { .. }), + Some(PermPayload::Capabilities { .. }) + ) | ( + Some(PermPayload::Combined { .. }), + Some(PermPayload::Combined { .. }) + ) | (None, None) + ) + } +} + +/// Node id, unique within its DAG (dense small ints assigned by the +/// template builders). +pub type NodeId = u32; + +/// When a dependency edge is considered satisfied. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum DepWhen { + /// Dep must reach `Done`. A `Failed` / `Cancelled` dep cancels this + /// node (cancel-downstream). + AfterOk, + /// Dep must merely reach a terminal state (ok *or* fail). Used only + /// by `rebuild`'s tail `Reconcile` so the recovery-start runs even + /// when `Swap` failed. + AfterAny, +} + +/// A dependency edge (intra-DAG only — cross-DAG ordering comes from +/// the per-agent lease + dedup, never from edges between DAGs). +#[derive(Debug, Clone, Copy, Serialize)] +pub struct Dep { + pub on: NodeId, + pub when: DepWhen, +} + +/// The primitive operations — each kind maps to one executor fn in +/// `exec.rs`, a thin wrapper over existing `lifecycle.rs` / `meta.rs` +/// code. Concurrency is gated by two resource classes (see +/// [`NodeKind::needs_build_slot`] / [`NodeKind::needs_lease`]); the +/// meta *repo* is serialized by `meta::META_LOCK` inside the wrapped +/// functions themselves, which is why there is no `GitCommit` node — +/// a standalone commit node would open a dirty-working-tree window +/// between nodes that the fused `meta.rs` ops deliberately close. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum NodeKind { + /// Out-of-band toplevel build while the container keeps serving: + /// meta `sync_agents`, optional per-agent relock, then + /// `lifecycle::prebuild_toplevel`. `relock = false` only for + /// meta-update cascade rebuilds (re-locking would revert the bump + /// the cascade just committed). + Prebuild { relock: bool }, + /// `nixos-container update` profile-swap (requires the container + /// stopped). Re-applies nspawn flags + resource limits first — + /// rebuild is the reconcile verb — and carries the post-rebuild + /// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan). + Swap, + /// First-spawn `nixos-container create` plus the pre-create + /// provisioning (proposed/applied repos, state subvolume, meta + /// registration). + Create, + /// Meta flake lock bump. `sweep = false`: `meta::lock_update` + /// (commit fused, under `META_LOCK`) with the DAG's `inputs`; + /// `sweep = true`: `meta::lock_update_hyperhive`, *non-fatal* (a + /// failed boot-time bump must not cancel the fan-out rebuilds). + /// On success the scheduler appends child `Rebuild` DAGs: the + /// precomputed `fanout` list when present (boot sweep), else the + /// post-bump affected set (`meta_update_cascade_agents`). + MetaLock { + sweep: bool, + fanout: Option>, + }, + /// Idempotent power converge: read `wanted` + observed state; + /// start if `Up` & down (with cold-start fallback), stop if + /// `Offline` & up, else noop. + Reconcile, + /// Mechanical `nixos-container stop` for the profile swap. Never + /// touches `wanted`. Noop if already stopped. + StopForUpdate, + /// Set the graceful-stop fence + kick the harness so it runs one + /// stop-checkpoint turn. + Signal, + /// Await the harness clearing the fence, bounded by + /// `GRACEFUL_STOP_TIMEOUT`. Resolves ok either way — the + /// downstream `Reconcile` performs the actual stop. + Drain, + /// `set_nspawn_flags` + `set_resource_limits` + daemon-reload. + WriteDropin, + /// Commit `tool-groups.json` / `capabilities.json` per the DAG's + /// `perm_payload` (commit fused under `META_LOCK`). + WritePermFile, + /// Opaque approval deploy pipeline (`ApplyCommit` / + /// `MergeConfigPr`): the two-phase prepare/finalize/abort meta + /// deploy stays inside `actions.rs` in v1 — deliberately not + /// modeled as scheduler nodes (see the design doc §9). + ApprovalDeploy, +} + +impl NodeKind { + /// Wire string for `NodeView.kind`. + pub fn as_str(&self) -> &'static str { + match self { + NodeKind::Prebuild { .. } => "prebuild", + NodeKind::Swap => "swap", + NodeKind::Create => "create", + NodeKind::MetaLock { .. } => "meta_lock", + NodeKind::Reconcile => "reconcile", + NodeKind::StopForUpdate => "stop_for_update", + NodeKind::Signal => "signal", + NodeKind::Drain => "drain", + NodeKind::WriteDropin => "write_dropin", + NodeKind::WritePermFile => "write_perm_file", + NodeKind::ApprovalDeploy => "approval_deploy", + } + } + + /// Nix-heavy kinds hold one of the `buildSlots` semaphore permits + /// for the node's duration. + pub fn needs_build_slot(&self) -> bool { + matches!( + self, + NodeKind::Prebuild { .. } + | NodeKind::Swap + | NodeKind::Create + | NodeKind::MetaLock { .. } + | NodeKind::ApprovalDeploy + ) + } + + /// Container-affecting kinds require the DAG to hold the agent's + /// lifecycle lease (acquired at the first such node, held until the + /// DAG is terminal). Lease-exempt kinds (`Prebuild`, `MetaLock`, + /// `WritePermFile`) touch the store / meta repo, not the running + /// container — which is exactly why a `Prebuild` can overlap + /// another DAG's work on the same agent. + pub fn needs_lease(&self) -> bool { + matches!( + self, + NodeKind::Swap + | NodeKind::Create + | NodeKind::Reconcile + | NodeKind::StopForUpdate + | NodeKind::Signal + | NodeKind::Drain + | NodeKind::WriteDropin + | NodeKind::ApprovalDeploy + ) + } +} + +/// One schedulable unit inside a DAG. +#[derive(Debug, Clone)] +pub struct Node { + pub id: NodeId, + pub kind: NodeKind, + pub deps: Vec, + pub state: State, + /// Live sub-label while `Running` (kept for parity with the old + /// per-entry `step`). + pub step: Option, + /// Row id of the `build_logs` entry this node opened (`Prebuild` / + /// `Swap` / `ApprovalDeploy`), for the dashboard's live-stream link. + pub build_log_id: Option, + pub started_at: Option, + pub finished_at: Option, + /// Populated when `state == Failed` (truncated by the queue). + pub error: Option, +} + +/// Submit-time spec for one node. +#[derive(Debug, Clone)] +pub struct NodeSpec { + pub kind: NodeKind, + pub deps: Vec, +} + +/// Submit-time spec for a whole DAG. Built by `templates.rs`; validated +/// (cycle rejection) and dedup'd by `JobQueue::submit`. +#[derive(Debug, Clone)] +pub struct DagSpec { + pub template: Template, + /// Primary target agent, or `"hyperhive"` for meta-level DAGs. + pub agent: String, + pub source: Source, + /// Free-form "why"; dedup appends "also requested by …" lines. + pub reason: String, + /// Cascade grouping (meta-update / sweep children). + pub parent_id: Option, + /// Fires the approval-resolution hook on DAG terminal. + pub approval_id: Option, + /// `MetaUpdate`-only: the inputs to bump (also part of the dedup + /// key for that template). Display copy lives on the DAG. + pub inputs: Vec, + /// `PermChange`-only payload. + pub perm_payload: Option, + /// Dashboard transient pill (and crash-watch suppression) held for + /// the lease window — from lease acquisition to DAG terminal. + pub transient: Option, + pub nodes: Vec, +} + +/// A live DAG in the queue. +#[derive(Debug, Clone)] +pub struct Dag { + pub id: u64, + pub template: Template, + pub agent: String, + pub source: Source, + pub reason: String, + pub parent_id: Option, + pub approval_id: Option, + pub inputs: Vec, + pub perm_payload: Option, + pub transient: Option, + pub created_at: i64, + pub nodes: Vec, + /// Terminal roll-up already reported to the scheduler's hooks + /// (approval resolution, transient release). Internal bookkeeping, + /// never serialized. + pub terminal_reported: bool, +} + +impl Dag { + /// Roll-up state: `Failed` if any node failed; else `Running` if + /// any running; else `Queued` if any queued; else `Cancelled` if + /// any cancelled; else `Done`. + pub fn rollup(&self) -> State { + let mut any_cancelled = false; + let mut any_queued = false; + let mut any_running = false; + for n in &self.nodes { + match n.state { + State::Failed => return State::Failed, + State::Running => any_running = true, + State::Queued => any_queued = true, + State::Cancelled => any_cancelled = true, + State::Done => {} + } + } + if any_running { + State::Running + } else if any_queued { + State::Queued + } else if any_cancelled { + State::Cancelled + } else { + State::Done + } + } + + /// True when every node is terminal. + pub fn is_terminal(&self) -> bool { + self.nodes.iter().all(|n| n.state.is_terminal()) + } + + /// First failed node's error, for the roll-up `error` field. + pub fn first_error(&self) -> Option<&str> { + self.nodes + .iter() + .find(|n| n.state == State::Failed) + .and_then(|n| n.error.as_deref()) + } + + pub fn node(&self, id: NodeId) -> Option<&Node> { + self.nodes.iter().find(|n| n.id == id) + } + + pub fn node_mut(&mut self, id: NodeId) -> Option<&mut Node> { + self.nodes.iter_mut().find(|n| n.id == id) + } +} + +/// Wire shape of one node inside a `DagView`. +#[derive(Debug, Clone, Serialize)] +pub struct NodeView { + pub id: NodeId, + /// Flattened `NodeKind` tag ("prebuild", "swap", …). + pub kind: &'static str, + /// Ids of the nodes this one waits for. + pub deps: Vec, + pub state: State, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub step: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub build_log_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finished_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +/// Wire shape of a DAG, serialized onto `RebuildQueueChanged` and the +/// `/api/state` snapshot. DAG-level fields mirror the old `QueueEntry` +/// names (`kind` = template string, roll-up `state`); everything +/// per-node — step labels, build-log links, errors, timestamps — +/// appears exactly once, inside `nodes`. +#[derive(Debug, Clone, Serialize)] +pub struct DagView { + pub id: u64, + pub agent: String, + /// Template wire string — same values the old `kind` field used. + pub kind: Template, + /// Roll-up state (see [`Dag::rollup`]). + pub state: State, + pub source: Source, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent_id: Option, + pub reason: String, + pub enqueued_at: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finished_at: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub inputs: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub approval_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub perm_payload: Option, + pub nodes: Vec, +} + +impl Dag { + pub fn view(&self) -> DagView { + let started_at = self.nodes.iter().filter_map(|n| n.started_at).min(); + let finished_at = if self.is_terminal() { + self.nodes.iter().filter_map(|n| n.finished_at).max() + } else { + None + }; + DagView { + id: self.id, + agent: self.agent.clone(), + kind: self.template, + state: self.rollup(), + source: self.source, + parent_id: self.parent_id, + reason: self.reason.clone(), + enqueued_at: self.created_at, + started_at, + finished_at, + inputs: self.inputs.clone(), + approval_id: self.approval_id, + perm_payload: self.perm_payload.clone(), + nodes: self + .nodes + .iter() + .map(|n| NodeView { + id: n.id, + kind: n.kind.as_str(), + deps: n.deps.iter().map(|d| d.on).collect(), + state: n.state, + step: n.step.clone(), + build_log_id: n.build_log_id, + started_at: n.started_at, + finished_at: n.finished_at, + error: n.error.clone(), + }) + .collect(), + } + } +} diff --git a/hive-c0re/src/job_queue/scheduler.rs b/hive-c0re/src/job_queue/scheduler.rs new file mode 100644 index 00000000..3962cbc2 --- /dev/null +++ b/hive-c0re/src/job_queue/scheduler.rs @@ -0,0 +1,150 @@ +//! The single scheduler task that drives all DAGs: claim every ready +//! node (as many as the build slots / leases allow), spawn one +//! executor task per claim, and on any completion re-evaluate. +//! Concurrency comes from the build-slot count, not multiple workers. +//! +//! Also owns the two DAG-lifetime side channels the sync queue core +//! can't hold itself: +//! - the per-DAG transient guard (dashboard pill + crash-watch +//! suppression), created when a DAG acquires its agent lease and +//! dropped when the DAG settles terminal; +//! - the `MetaLock` fan-out: appending child `Rebuild` DAGs once the +//! lock bump lands, so children build against the post-bump lock +//! (and a failed bump fans out nothing — replacing the old +//! pre-enqueue + cancel-children dance). + +use std::collections::HashMap; +use std::sync::Arc; + +use super::exec::{self, NodeOutput}; +use super::{Claim, Source, Template, templates}; +use crate::coordinator::Coordinator; + +struct NodeDone { + claim: Claim, + result: anyhow::Result, +} + +/// Scheduler loop. Spawned once at hive-c0re startup from `main.rs`. +/// +/// Shutdown semantics: subscribes to `coord.shutdown_rx()`. On a true +/// signal the loop exits immediately; already-running node tasks ride +/// the runtime down with the process, and pending `Queued` DAGs are +/// dropped — desired state is re-derived on next boot (boot sweep + +/// reconcile), so the in-memory queue is deliberately not durable. +pub async fn run_worker(coord: Arc) { + let mut shutdown = coord.shutdown_rx(); + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); + // DAG id → transient guard held for the lease window. + let mut transients: HashMap = HashMap::new(); + loop { + let claims = coord.job_queue.claim_ready(); + if !claims.is_empty() { + for claim in claims { + if claim.lease_acquired + && let Some(kind) = claim.transient + { + transients.insert(claim.dag_id, coord.transient_guard(&claim.agent, kind)); + } + tracing::info!( + dag = claim.dag_id, + node = claim.node_id, + kind = claim.kind.as_str(), + agent = %claim.agent, + template = claim.template.as_str(), + "job_queue: node running" + ); + let coord = Arc::clone(&coord); + let tx = tx.clone(); + tokio::spawn(async move { + let result = exec::run_node(&coord, &claim).await; + // Send failure = scheduler gone (shutdown); drop. + let _ = tx.send(NodeDone { claim, result }); + }); + } + coord.emit_rebuild_queue_snapshot(); + continue; + } + tokio::select! { + biased; + res = shutdown.changed() => { + if res.is_err() || *shutdown.borrow() { + tracing::info!("job_queue: scheduler exiting on shutdown"); + return; + } + } + Some(done) = rx.recv() => { + handle_completion(&coord, &mut transients, done).await; + } + () = coord.job_queue.notify.notified() => {} + } + } +} + +async fn handle_completion( + coord: &Arc, + transients: &mut HashMap, + done: NodeDone, +) { + let NodeDone { claim, result } = done; + let (queue_result, fanout) = match result { + Ok(output) => { + tracing::info!( + dag = claim.dag_id, + node = claim.node_id, + "job_queue: node done" + ); + (Ok(()), output.fanout) + } + Err(e) => { + let msg = format!("{e:#}"); + tracing::warn!( + dag = claim.dag_id, + node = claim.node_id, + kind = claim.kind.as_str(), + agent = %claim.agent, + error = %msg, + "job_queue: node failed" + ); + (Err(msg), Vec::new()) + } + }; + let report = coord + .job_queue + .complete_node(claim.dag_id, claim.node_id, queue_result); + if !fanout.is_empty() { + let specs = fanout_specs(&claim, fanout); + coord.job_queue.append_children(specs); + } + for terminal in report.terminal { + // Drop the lease-window transient guard, then let the hook + // fire approval resolution / failure events. + transients.remove(&terminal.dag_id); + exec::on_dag_terminal(coord, &terminal).await; + } + coord.emit_rebuild_queue_snapshot(); +} + +/// Child `Rebuild` specs for a completed `MetaLock` fan-out, grouped +/// under the parent via `parent_id`. Meta-update children skip the +/// per-agent relock (it would revert the bump the parent just +/// committed); sweep children relock like a manual rebuild. +fn fanout_specs(claim: &Claim, agents: Vec) -> Vec { + let sweep = claim.template == Template::StartupSweep; + let (source, relock) = if sweep { + (Source::StartupSweep, true) + } else { + (Source::MetaUpdate, false) + }; + let reason = if sweep { + "startup sweep".to_owned() + } else if let Some(approval_id) = claim.approval_id { + format!("approval #{approval_id} meta input cascade") + } else { + "meta-update cascade".to_owned() + }; + agents + .into_iter() + .map(|agent| templates::rebuild(&agent, source, reason.clone(), Some(claim.dag_id), relock)) + .collect() +} diff --git a/hive-c0re/src/job_queue/submit.rs b/hive-c0re/src/job_queue/submit.rs new file mode 100644 index 00000000..324a1c0c --- /dev/null +++ b/hive-c0re/src/job_queue/submit.rs @@ -0,0 +1,121 @@ +//! Request-level submit API — the surface the dashboard POST handlers, +//! the MCP socket handlers, and `hivectl` paths call. Owns the +//! submit-time side effects the DAG templates deliberately don't: +//! writing the durable `wanted` power intent (synchronously, +//! last-writer-wins) before the DAG whose `Reconcile` reads it, and +//! upgrading a stale start to a full rebuild. Every helper emits a +//! fresh queue snapshot so the dashboard shows the new DAG immediately. + +use std::sync::Arc; + +use super::{Source, Template, templates}; +use crate::coordinator::Coordinator; +use crate::power::Wanted; + +fn submit_and_emit(coord: &Arc, spec: super::DagSpec) -> u64 { + let id = coord + .job_queue + .submit(spec) + .expect("template-built dag specs are acyclic"); + coord.emit_rebuild_queue_snapshot(); + id +} + +fn set_wanted(coord: &Arc, agent: &str, wanted: Wanted) { + if let Err(e) = coord.power.set(agent, wanted) { + tracing::warn!(%agent, wanted = wanted.as_str(), error = ?e, "agent_power: set failed"); + } +} + +/// Manual/approval-independent rebuild (always relocks the agent's +/// meta input — cascade children are built by the scheduler's fan-out +/// instead of this surface). +pub fn rebuild(coord: &Arc, agent: &str, source: Source, reason: String) -> u64 { + submit_and_emit(coord, templates::rebuild(agent, source, reason, None, true)) +} + +/// Restart: mechanical stop + converge back to `wanted` (unchanged). +pub fn restart(coord: &Arc, agent: &str, source: Source, reason: String) -> u64 { + submit_and_emit(coord, templates::restart(agent, source, reason)) +} + +/// Start: persist `wanted = Up`, then reconcile. A stale rev marker +/// upgrades the start to a full rebuild (whose tail `Reconcile` does +/// the start) so the container always comes up on current derivations +/// — the old fast-lane `run_start` upgrade, moved to submit time. +pub fn start(coord: &Arc, agent: &str, source: Source, reason: String) -> u64 { + set_wanted(coord, agent, Wanted::Up); + let stored = std::fs::read_to_string(crate::auto_update::rev_marker_path(agent)).ok(); + let stale = crate::auto_update::current_flake_rev(&coord.hyperhive_flake) + .is_some_and(|rev| stored.as_deref() != Some(rev.as_str())); + if stale { + tracing::info!(%agent, "start: rev stale — upgrading to rebuild+start"); + return submit_and_emit( + coord, + templates::rebuild( + agent, + source, + format!("{reason} (stale — rebuild+start)"), + None, + true, + ), + ); + } + submit_and_emit( + coord, + templates::reconcile_only( + Template::Start, + agent, + source, + reason, + Some(crate::coordinator::TransientKind::Starting), + ), + ) +} + +/// Hard stop: persist `wanted = Offline`, then reconcile (kill + +/// unregister + `Killed` event). +pub fn stop(coord: &Arc, agent: &str, source: Source, reason: String) -> u64 { + set_wanted(coord, agent, Wanted::Offline); + submit_and_emit( + coord, + templates::reconcile_only( + Template::Stop, + agent, + source, + reason, + Some(crate::coordinator::TransientKind::Stopping), + ), + ) +} + +/// Graceful stop: persist `wanted = Offline`, then signal → drain → +/// reconcile (the actual stop). +pub fn graceful_stop(coord: &Arc, agent: &str, source: Source, reason: String) -> u64 { + set_wanted(coord, agent, Wanted::Offline); + submit_and_emit(coord, templates::graceful_stop(agent, source, reason)) +} + +/// Perm change: commit the JSON file(s) then rebuild. +pub fn perm_change( + coord: &Arc, + agent: &str, + source: Source, + reason: String, + payload: super::PermPayload, +) -> u64 { + submit_and_emit( + coord, + templates::perm_change(agent, source, reason, payload), + ) +} + +/// Meta-input lock bump; cascade rebuilds fan out on completion. +pub fn meta_update( + coord: &Arc, + inputs: Vec, + source: Source, + reason: String, +) -> u64 { + submit_and_emit(coord, templates::meta_update(inputs, source, reason, None)) +} diff --git a/hive-c0re/src/job_queue/templates.rs b/hive-c0re/src/job_queue/templates.rs new file mode 100644 index 00000000..536a7e50 --- /dev/null +++ b/hive-c0re/src/job_queue/templates.rs @@ -0,0 +1,333 @@ +//! DAG shape builders — every operation as a template over the shared +//! node primitives — plus submit-time cycle validation (petgraph is +//! confined to this validation; the runtime store stays the plain +//! `Vec` + `deps`). +//! +//! ```text +//! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a) +//! graceful-stop(a): [wanted=Offline] Signal(a) → Drain(a) → Reconcile(a) +//! restart(a): StopForUpdate(a) → Reconcile(a) (wanted unchanged) +//! start(a): [wanted=Up] Reconcile(a) +//! stop(a): [wanted=Offline] Reconcile(a) +//! spawn(a): [wanted=Up] Create(a) → WriteDropin(a) → Reconcile(a) +//! perm-change(a): WritePermFile(a) → «rebuild subgraph» +//! meta-update(inp): MetaLock(inp) → «fan-out rebuild(a) per affected a» +//! startup sweep: MetaLock(hyperhive, non-fatal) → «fan-out rebuild(stale a)» +//! ``` + +use anyhow::{Result, bail}; + +use super::model::{DagSpec, Dep, DepWhen, NodeKind, NodeSpec, PermPayload, Source, Template}; +use crate::coordinator::TransientKind; + +/// After-ok edge on the previous node — the common chain link. +fn after_ok(on: u32) -> Vec { + vec![Dep { + on, + when: DepWhen::AfterOk, + }] +} + +/// The rebuild node chain. `Reconcile` deps on `Swap` with `AfterAny`: +/// it must run even when the profile swap failed, so a previously-up +/// agent comes back on its old config (today's recovery-start). This +/// is the only `AfterAny` edge in v1. +fn rebuild_nodes(relock: bool, base: u32) -> Vec { + vec![ + NodeSpec { + kind: NodeKind::Prebuild { relock }, + deps: if base == 0 { + Vec::new() + } else { + after_ok(base - 1) + }, + }, + NodeSpec { + kind: NodeKind::StopForUpdate, + deps: after_ok(base), + }, + NodeSpec { + kind: NodeKind::Swap, + deps: after_ok(base + 1), + }, + NodeSpec { + kind: NodeKind::Reconcile, + deps: vec![Dep { + on: base + 2, + when: DepWhen::AfterAny, + }], + }, + ] +} + +/// One uniform rebuild shape — no `was_running` branch. `StopForUpdate` +/// noops when already down; the tail `Reconcile` auto-noops the start +/// when `wanted = Offline` (a rebuild of a deliberately-stopped agent +/// leaves it stopped). `relock = false` only for meta-update cascade +/// children. +pub fn rebuild( + agent: &str, + source: Source, + reason: String, + parent_id: Option, + relock: bool, +) -> DagSpec { + DagSpec { + template: Template::Rebuild, + agent: agent.to_owned(), + source, + reason, + parent_id, + approval_id: None, + inputs: Vec::new(), + perm_payload: None, + transient: Some(TransientKind::Rebuilding), + nodes: rebuild_nodes(relock, 0), + } +} + +/// Approval-driven deploy (`ApplyCommit` / `MergeConfigPr`): the whole +/// two-phase pipeline stays one opaque node in v1 (design doc §9) — +/// wire-visible as a `rebuild` card like today. +pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec { + DagSpec { + template: Template::Rebuild, + agent: agent.to_owned(), + source: Source::Approval, + reason, + parent_id: None, + approval_id: Some(approval_id), + inputs: Vec::new(), + perm_payload: None, + transient: Some(TransientKind::Rebuilding), + nodes: vec![NodeSpec { + kind: NodeKind::ApprovalDeploy, + deps: Vec::new(), + }], + } +} + +/// Graceful stop: cheap `Signal` fires immediately (no build slot), the +/// `Drain` awaits the harness checkpoint (bounded), and the tail +/// `Reconcile` performs the actual container stop — the caller sets +/// `wanted = Offline` at submit time. A whole-hive graceful stop +/// therefore signals every agent up front and overlaps every drain, +/// replacing the old detached-watcher thread structurally. +pub fn graceful_stop(agent: &str, source: Source, reason: String) -> DagSpec { + DagSpec { + template: Template::GracefulStop, + agent: agent.to_owned(), + source, + reason, + parent_id: None, + approval_id: None, + inputs: Vec::new(), + perm_payload: None, + transient: Some(TransientKind::Stopping), + nodes: vec![ + NodeSpec { + kind: NodeKind::Signal, + deps: Vec::new(), + }, + NodeSpec { + kind: NodeKind::Drain, + deps: after_ok(0), + }, + NodeSpec { + kind: NodeKind::Reconcile, + deps: after_ok(1), + }, + ], + } +} + +/// Restart: mechanical stop, then converge back to `wanted` +/// (unchanged) — a stop + start for a wanted-up agent. +pub fn restart(agent: &str, source: Source, reason: String) -> DagSpec { + DagSpec { + template: Template::Restart, + agent: agent.to_owned(), + source, + reason, + parent_id: None, + approval_id: None, + inputs: Vec::new(), + perm_payload: None, + transient: Some(TransientKind::Restarting), + nodes: vec![ + NodeSpec { + kind: NodeKind::StopForUpdate, + deps: Vec::new(), + }, + NodeSpec { + kind: NodeKind::Reconcile, + deps: after_ok(0), + }, + ], + } +} + +/// Single-`Reconcile` DAG: `Start` / `Stop` (caller writes `wanted` +/// first) and the boot-time `Reconcile` converge (wanted untouched). +pub fn reconcile_only( + template: Template, + agent: &str, + source: Source, + reason: String, + transient: Option, +) -> DagSpec { + DagSpec { + template, + agent: agent.to_owned(), + source, + reason, + parent_id: None, + approval_id: None, + inputs: Vec::new(), + perm_payload: None, + transient, + nodes: vec![NodeSpec { + kind: NodeKind::Reconcile, + deps: Vec::new(), + }], + } +} + +/// First-deploy spawn (approval-driven): pre-start provisioning + +/// `nixos-container create`, drop-in write, then `Reconcile` starts the +/// container (`wanted = Up` written at approve time). +pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec { + DagSpec { + template: Template::Spawn, + agent: agent.to_owned(), + source: Source::Approval, + reason, + parent_id: None, + approval_id: Some(approval_id), + inputs: Vec::new(), + perm_payload: None, + transient: Some(TransientKind::Spawning), + nodes: vec![ + NodeSpec { + kind: NodeKind::Create, + deps: Vec::new(), + }, + NodeSpec { + kind: NodeKind::WriteDropin, + deps: after_ok(0), + }, + NodeSpec { + kind: NodeKind::Reconcile, + deps: after_ok(1), + }, + ], + } +} + +/// Perm change: commit the JSON file(s), then the rebuild subgraph so +/// the updated `HIVE_TOOL_GROUPS` / `HIVE_CAPABILITIES` env var takes +/// effect in the container. +pub fn perm_change(agent: &str, source: Source, reason: String, payload: PermPayload) -> DagSpec { + let mut nodes = vec![NodeSpec { + kind: NodeKind::WritePermFile, + deps: Vec::new(), + }]; + nodes.extend(rebuild_nodes(true, 1)); + DagSpec { + template: Template::PermChange, + agent: agent.to_owned(), + source, + reason, + parent_id: None, + approval_id: None, + inputs: Vec::new(), + perm_payload: Some(payload), + transient: Some(TransientKind::Rebuilding), + nodes, + } +} + +/// Meta-input lock bump. Child `Rebuild` DAGs fan out on completion — +/// appended *after* the bump lands so their prebuilds run against the +/// post-bump lock (and so a failed bump simply fans out nothing, +/// replacing the old pre-enqueue + `cancel_children` dance). +pub fn meta_update( + inputs: Vec, + source: Source, + reason: String, + approval_id: Option, +) -> DagSpec { + DagSpec { + template: Template::MetaUpdate, + agent: "hyperhive".to_owned(), + source, + reason, + parent_id: None, + approval_id, + inputs, + perm_payload: None, + transient: None, + nodes: vec![NodeSpec { + kind: NodeKind::MetaLock { + sweep: false, + fanout: None, + }, + deps: Vec::new(), + }], + } +} + +/// Boot-time sweep parent: bump meta's hyperhive input (non-fatal), +/// then fan out `Rebuild` children for the precomputed stale agent +/// list (topology-sorted by the caller). +pub fn startup_sweep(reason: String, stale_agents: Vec) -> DagSpec { + DagSpec { + template: Template::StartupSweep, + agent: "hyperhive".to_owned(), + source: Source::AutoUpdate, + reason, + parent_id: None, + approval_id: None, + inputs: Vec::new(), + perm_payload: None, + transient: None, + nodes: vec![NodeSpec { + kind: NodeKind::MetaLock { + sweep: true, + fanout: Some(stale_agents), + }, + deps: Vec::new(), + }], + } +} + +/// Validate a spec before it enters the queue: node ids are dense +/// (index = id), deps reference existing nodes, and the dep graph is +/// acyclic (petgraph `toposort`). Rejecting cycles here fixes the old +/// queue's documented "circular dep silently deadlocks forever" caveat. +pub fn validate(spec: &DagSpec) -> Result<()> { + if spec.nodes.is_empty() { + bail!("dag spec {:?} has no nodes", spec.template); + } + let n = spec.nodes.len(); + let mut graph = petgraph::graph::DiGraph::::new(); + let idx: Vec<_> = (0..n) + .map(|i| graph.add_node(u32::try_from(i).unwrap_or(u32::MAX))) + .collect(); + for (i, node) in spec.nodes.iter().enumerate() { + for dep in &node.deps { + let Some(&dep_idx) = idx.get(dep.on as usize) else { + bail!( + "dag spec {:?} node {i} depends on unknown node {}", + spec.template, + dep.on + ); + }; + graph.add_edge(dep_idx, idx[i], ()); + } + } + if petgraph::algo::toposort(&graph, None).is_err() { + bail!("dag spec {:?} contains a dependency cycle", spec.template); + } + Ok(()) +} diff --git a/hive-c0re/src/job_queue/tests.rs b/hive-c0re/src/job_queue/tests.rs new file mode 100644 index 00000000..128a352e --- /dev/null +++ b/hive-c0re/src/job_queue/tests.rs @@ -0,0 +1,823 @@ +//! Queue-core unit tests: dedup, cycle rejection, resource +//! serialization (build slots / per-agent leases), lease-exempt +//! overlap, FIFO fairness, cancel semantics, `AfterAny` failure +//! routing, fan-out, and history retention. All synchronous — the +//! scheduler's async loop is a thin claim/complete pump over the same +//! methods exercised here. + +use super::model::{Dep, DepWhen, NodeKind, NodeSpec}; +use super::*; + +fn submit(q: &JobQueue, spec: DagSpec) -> u64 { + q.submit(spec).expect("valid spec") +} + +fn rebuild(agent: &str, reason: &str) -> DagSpec { + templates::rebuild(agent, Source::Manual, reason.to_owned(), None, true) +} + +/// Claim helper asserting exactly one node comes back. +fn claim_one(q: &JobQueue) -> Claim { + let mut claims = q.claim_ready(); + assert_eq!( + claims.len(), + 1, + "expected exactly one claim, got {claims:?}" + ); + claims.pop().expect("one claim") +} + +fn state_of(q: &JobQueue, dag_id: u64) -> State { + q.snapshot() + .iter() + .find(|d| d.id == dag_id) + .expect("dag present") + .state +} + +// ---- submit / dedup ---- + +#[test] +fn submit_assigns_distinct_ids() { + let q = JobQueue::new(1); + let a = submit(&q, rebuild("agent-a", "first")); + let b = submit(&q, rebuild("agent-b", "second")); + assert_ne!(a, b); + assert_eq!(q.snapshot().len(), 2); +} + +#[test] +fn dedup_pending_same_template_and_agent() { + let q = JobQueue::new(1); + let a = submit(&q, rebuild("agent-a", "first")); + let b = submit(&q, rebuild("agent-a", "auto sweep")); + assert_eq!(a, b, "dedup should return existing id"); + let snap = q.snapshot(); + assert_eq!(snap.len(), 1); + assert!(snap[0].reason.contains("first")); + assert!(snap[0].reason.contains("auto sweep")); +} + +#[test] +fn dedup_does_not_apply_across_templates_or_agents() { + let q = JobQueue::new(1); + let a = submit(&q, rebuild("agent-a", "r")); + let b = submit(&q, rebuild("agent-b", "r")); + let c = submit( + &q, + templates::restart("agent-a", Source::Manual, "r".to_owned()), + ); + assert_ne!(a, b); + assert_ne!(a, c); + assert_eq!(q.snapshot().len(), 3); +} + +#[test] +fn dedup_skips_running_dags() { + let q = JobQueue::new(1); + let a = submit(&q, rebuild("agent-a", "first")); + let claim = claim_one(&q); // Prebuild running + assert_eq!(claim.dag_id, a); + // While the original runs, re-submit is legitimate new work. + let again = submit(&q, rebuild("agent-a", "config bumped during build")); + assert_ne!(a, again); + assert_eq!(q.snapshot().len(), 2); +} + +#[test] +fn meta_update_dedup_matches_inputs() { + let q = JobQueue::new(1); + let a = submit( + &q, + templates::meta_update( + vec!["nixpkgs".to_owned()], + Source::Manual, + "first".to_owned(), + None, + ), + ); + let b = submit( + &q, + templates::meta_update( + vec!["nixpkgs".to_owned()], + Source::Manual, + "duplicate click".to_owned(), + None, + ), + ); + assert_eq!(a, b, "identical-inputs meta-updates should dedup"); + let c = submit( + &q, + templates::meta_update( + vec!["agent-bitburner/bitburner-agent".to_owned()], + Source::Manual, + "bump agent".to_owned(), + None, + ), + ); + assert_ne!(a, c, "different-inputs meta-updates must NOT dedup"); + assert_eq!(q.snapshot().len(), 2); +} + +#[test] +fn approval_dags_dedup_only_on_matching_id() { + let q = JobQueue::new(1); + let a = submit( + &q, + templates::approval_deploy("agent-a", 1, "approval #1".to_owned()), + ); + let b = submit( + &q, + templates::approval_deploy("agent-a", 2, "approval #2".to_owned()), + ); + assert_ne!(a, b, "distinct approvals must not collapse"); + // Rapid double-click on the same approval IS a single op. + let c = submit( + &q, + templates::approval_deploy("agent-a", 1, "approval #1 (dup)".to_owned()), + ); + assert_eq!(a, c); + assert_eq!(q.snapshot().len(), 2); +} + +#[test] +fn perm_change_dedup_respects_perm_type() { + let q = JobQueue::new(1); + let groups = templates::perm_change( + "agent-a", + Source::Manual, + "groups".to_owned(), + PermPayload::ToolGroups { groups: vec![] }, + ); + let caps = templates::perm_change( + "agent-a", + Source::Manual, + "caps".to_owned(), + PermPayload::Capabilities { caps: vec![] }, + ); + let a = submit(&q, groups.clone()); + let b = submit(&q, caps); + assert_ne!(a, b, "tool-groups vs capabilities must not collapse"); + let c = submit(&q, groups); + assert_eq!(a, c, "same perm type dedups"); +} + +/// A `MetaUpdate` cascade `Rebuild` (with `parent_id = Some(meta_id)`) +/// must NOT dedup into a queued `Rebuild` with a different +/// `parent_id` (e.g. from a startup sweep) — without the guard the +/// cascade child would be swallowed and the agent never rebuilt +/// against the post-bump meta. +#[test] +fn dedup_respects_parent_id() { + let q = JobQueue::new(1); + let sweep = submit(&q, templates::startup_sweep("boot".to_owned(), vec![])); + let sweep_child = submit( + &q, + templates::rebuild( + "alice", + Source::StartupSweep, + "startup sweep".to_owned(), + Some(sweep), + true, + ), + ); + let meta = submit( + &q, + templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None), + ); + let cascade_child = submit( + &q, + templates::rebuild( + "alice", + Source::MetaUpdate, + "meta-update cascade".to_owned(), + Some(meta), + false, + ), + ); + assert_ne!(sweep_child, cascade_child); + let rebuilds = q + .snapshot() + .iter() + .filter(|d| d.kind == Template::Rebuild && d.agent == "alice") + .count(); + assert_eq!(rebuilds, 2, "both rebuilds must be present"); +} + +// ---- cycle rejection ---- + +#[test] +fn cyclic_dag_is_rejected_at_submit() { + let q = JobQueue::new(1); + let mut spec = rebuild("agent-a", "cyclic"); + // 0 → 1 → 0 cycle. + spec.nodes = vec![ + NodeSpec { + kind: NodeKind::StopForUpdate, + deps: vec![Dep { + on: 1, + when: DepWhen::AfterOk, + }], + }, + NodeSpec { + kind: NodeKind::Reconcile, + deps: vec![Dep { + on: 0, + when: DepWhen::AfterOk, + }], + }, + ]; + assert!(q.submit(spec).is_err(), "cyclic spec must be refused"); + assert!(q.snapshot().is_empty()); +} + +#[test] +fn unknown_dep_is_rejected_at_submit() { + let q = JobQueue::new(1); + let mut spec = rebuild("agent-a", "bad dep"); + spec.nodes = vec![NodeSpec { + kind: NodeKind::Reconcile, + deps: vec![Dep { + on: 9, + when: DepWhen::AfterOk, + }], + }]; + assert!(q.submit(spec).is_err()); +} + +// ---- dependency order within a DAG ---- + +#[test] +fn rebuild_chain_claims_in_dep_order() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + for expected in ["prebuild", "stop_for_update", "swap", "reconcile"] { + let c = claim_one(&q); + assert_eq!(c.dag_id, id); + assert_eq!(c.kind.as_str(), expected); + assert!( + q.claim_ready().is_empty(), + "chain must serialize: nothing ready while {expected} runs" + ); + q.complete_node(id, c.node_id, Ok(())); + } + assert_eq!(state_of(&q, id), State::Done); +} + +// ---- build slots ---- + +#[test] +fn build_slot_serializes_nix_heavy_nodes() { + let q = JobQueue::new(1); + let a = submit(&q, rebuild("agent-a", "r")); + let b = submit(&q, rebuild("agent-b", "r")); + let first = claim_one(&q); // a's Prebuild takes the only slot + assert_eq!(first.dag_id, a); + assert_eq!(first.kind.as_str(), "prebuild"); + q.complete_node(a, first.node_id, Ok(())); + // With the slot free again, FIFO gives... a's StopForUpdate is + // slot-free (lease) and b's Prebuild takes the slot — both run. + let claims = q.claim_ready(); + let kinds: Vec<(u64, &str)> = claims.iter().map(|c| (c.dag_id, c.kind.as_str())).collect(); + assert!(kinds.contains(&(a, "stop_for_update"))); + assert!(kinds.contains(&(b, "prebuild"))); + assert_eq!(claims.len(), 2); +} + +#[test] +fn two_build_slots_run_two_prebuilds() { + let q = JobQueue::new(2); + submit(&q, rebuild("agent-a", "r")); + submit(&q, rebuild("agent-b", "r")); + let claims = q.claim_ready(); + assert_eq!(claims.len(), 2, "two slots → two concurrent prebuilds"); + assert!(claims.iter().all(|c| c.kind.as_str() == "prebuild")); +} + +#[test] +fn fifo_fairness_for_the_slot() { + let q = JobQueue::new(1); + let a = submit(&q, rebuild("agent-a", "r")); + let b = submit(&q, rebuild("agent-b", "r")); + let c = submit(&q, rebuild("agent-c", "r")); + let first = claim_one(&q); + assert_eq!(first.dag_id, a, "submit order wins the slot"); + q.complete_node(a, first.node_id, Ok(())); + let next: Vec = q.claim_ready().iter().map(|cl| cl.dag_id).collect(); + assert!(next.contains(&b), "b's prebuild before c's"); + assert!(!next.contains(&c)); +} + +// ---- per-agent lease ---- + +#[test] +fn lease_serializes_two_lifecycle_dags_for_same_agent() { + let q = JobQueue::new(4); + let restart = submit( + &q, + templates::restart("agent-a", Source::Manual, "restart".to_owned()), + ); + let stop = submit( + &q, + templates::reconcile_only( + Template::Stop, + "agent-a", + Source::Manual, + "stop".to_owned(), + None, + ), + ); + // Restart's StopForUpdate acquires the lease; stop's Reconcile + // must wait even though slots are free. + let first = claim_one(&q); + assert_eq!(first.dag_id, restart); + assert!(first.lease_acquired); + q.complete_node(restart, first.node_id, Ok(())); + // Same DAG keeps the lease for its Reconcile. + let second = claim_one(&q); + assert_eq!(second.dag_id, restart); + assert!(!second.lease_acquired, "lease already held by this DAG"); + q.complete_node(restart, second.node_id, Ok(())); + // Restart terminal → lease released → stop's Reconcile runs. + let third = claim_one(&q); + assert_eq!(third.dag_id, stop); + q.complete_node(stop, third.node_id, Ok(())); + assert_eq!(state_of(&q, restart), State::Done); + assert_eq!(state_of(&q, stop), State::Done); +} + +#[test] +fn lease_exempt_prebuild_overlaps_other_dag_on_same_agent() { + let q = JobQueue::new(2); + submit(&q, rebuild("agent-a", "rebuild")); + let stop = submit( + &q, + templates::reconcile_only( + Template::Stop, + "agent-a", + Source::Manual, + "stop".to_owned(), + None, + ), + ); + // Prebuild is lease-exempt: the stop's Reconcile takes the lease + // and runs concurrently with the rebuild's out-of-band nix build. + let claims = q.claim_ready(); + let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect(); + assert!(kinds.contains(&"prebuild")); + assert!(kinds.contains(&"reconcile")); + // But the rebuild's StopForUpdate must then wait for the stop DAG + // to finish (lease). + let prebuild = claims + .iter() + .find(|c| c.kind.as_str() == "prebuild") + .expect("prebuild claim") + .clone(); + q.complete_node(prebuild.dag_id, prebuild.node_id, Ok(())); + assert!( + q.claim_ready().is_empty(), + "StopForUpdate blocked while stop DAG holds the lease" + ); + let reconcile = claims + .iter() + .find(|c| c.kind.as_str() == "reconcile") + .expect("reconcile claim") + .clone(); + q.complete_node(stop, reconcile.node_id, Ok(())); + let next = claim_one(&q); + assert_eq!(next.kind.as_str(), "stop_for_update"); +} + +#[test] +fn agents_do_not_contend_on_each_others_leases() { + let q = JobQueue::new(4); + submit( + &q, + templates::restart("agent-a", Source::Manual, "r".to_owned()), + ); + submit( + &q, + templates::restart("agent-b", Source::Manual, "r".to_owned()), + ); + let claims = q.claim_ready(); + assert_eq!(claims.len(), 2, "different agents run concurrently"); +} + +// ---- failure: cancel-downstream + AfterAny ---- + +#[test] +fn failed_node_cancels_downstream_but_afterany_reconcile_runs() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + let prebuild = claim_one(&q); + q.complete_node(id, prebuild.node_id, Err("nix build exploded".to_owned())); + // StopForUpdate + Swap are cancelled (AfterOk on a failed chain); + // the AfterAny Reconcile still runs once Swap is terminal. + let reconcile = claim_one(&q); + assert_eq!(reconcile.kind.as_str(), "reconcile"); + q.complete_node(id, reconcile.node_id, Ok(())); + let snap = q.snapshot(); + let dag = snap.iter().find(|d| d.id == id).expect("dag"); + assert_eq!(dag.state, State::Failed, "roll-up failed"); + let by_kind = |k: &str| { + dag.nodes + .iter() + .find(|n| n.kind == k) + .expect("node present") + .state + }; + assert_eq!(by_kind("prebuild"), State::Failed); + assert_eq!(by_kind("stop_for_update"), State::Cancelled); + assert_eq!(by_kind("swap"), State::Cancelled); + assert_eq!(by_kind("reconcile"), State::Done); + assert_eq!( + dag.nodes + .iter() + .find(|n| n.kind == "prebuild") + .and_then(|n| n.error.as_deref()), + Some("nix build exploded") + ); +} + +/// The swap-failure recovery: `Swap` fails → the `AfterAny` edge still +/// runs `Reconcile`, which brings a wanted-up agent back on its old +/// config. +#[test] +fn swap_failure_still_runs_reconcile() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + for _ in 0..2 { + let c = claim_one(&q); + q.complete_node(id, c.node_id, Ok(())); + } + let swap = claim_one(&q); + assert_eq!(swap.kind.as_str(), "swap"); + q.complete_node(id, swap.node_id, Err("update failed".to_owned())); + let reconcile = claim_one(&q); + assert_eq!(reconcile.kind.as_str(), "reconcile"); + q.complete_node(id, reconcile.node_id, Ok(())); + assert_eq!(state_of(&q, id), State::Failed); +} + +#[test] +fn failed_reconcile_marks_dag_failed() { + let q = JobQueue::new(1); + let id = submit( + &q, + templates::reconcile_only( + Template::Start, + "agent-a", + Source::Manual, + "start".to_owned(), + None, + ), + ); + let c = claim_one(&q); + q.complete_node(id, c.node_id, Err("start failed".to_owned())); + assert_eq!(state_of(&q, id), State::Failed); +} + +// ---- cancel ---- + +#[test] +fn cancel_clears_queued_dag() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + assert!(q.cancel(id)); + assert_eq!(state_of(&q, id), State::Cancelled); + assert!(q.claim_ready().is_empty()); +} + +#[test] +fn cancel_refuses_running_dag() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + let _ = claim_one(&q); + assert!(!q.cancel(id)); + assert_eq!(state_of(&q, id), State::Running); +} + +#[test] +fn cancel_children_marks_queued_children_only() { + let q = JobQueue::new(1); + let meta = submit( + &q, + templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None), + ); + // Parent's MetaLock is running while children exist. + let lock = claim_one(&q); + assert_eq!(lock.dag_id, meta); + let child_a = submit( + &q, + templates::rebuild( + "agent-a", + Source::MetaUpdate, + "cascade".to_owned(), + Some(meta), + false, + ), + ); + let child_b = submit( + &q, + templates::rebuild( + "agent-b", + Source::MetaUpdate, + "cascade".to_owned(), + Some(meta), + false, + ), + ); + let unrelated = submit(&q, rebuild("agent-c", "operator queued")); + // MetaLock holds the single build slot, so both children (and the + // unrelated rebuild) are still fully queued here. + let cancelled = q.cancel_children(meta); + assert_eq!(cancelled, 2); + assert_eq!(state_of(&q, child_a), State::Cancelled); + assert_eq!(state_of(&q, child_b), State::Cancelled); + assert_eq!(state_of(&q, unrelated), State::Queued); +} + +#[test] +fn cancel_children_skips_running_child() { + let q = JobQueue::new(2); + let meta = submit( + &q, + templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None), + ); + let lock = claim_one(&q); + let running_child = submit( + &q, + templates::rebuild( + "agent-a", + Source::MetaUpdate, + "cascade".to_owned(), + Some(meta), + false, + ), + ); + let queued_child = submit( + &q, + templates::rebuild( + "agent-b", + Source::MetaUpdate, + "cascade".to_owned(), + Some(meta), + false, + ), + ); + // Second slot lets running_child's prebuild start. + let child_claim = claim_one(&q); + assert_eq!(child_claim.dag_id, running_child); + let n = q.cancel_children(meta); + assert_eq!(n, 1); + assert_eq!(state_of(&q, running_child), State::Running); + assert_eq!(state_of(&q, queued_child), State::Cancelled); + q.complete_node(meta, lock.node_id, Ok(())); +} + +// ---- fan-out ---- + +#[test] +fn append_children_sets_parent_and_dedups() { + let q = JobQueue::new(1); + let meta = submit( + &q, + templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None), + ); + let specs = vec![ + templates::rebuild( + "alice", + Source::MetaUpdate, + "cascade".to_owned(), + Some(meta), + false, + ), + templates::rebuild( + "bob", + Source::MetaUpdate, + "cascade".to_owned(), + Some(meta), + false, + ), + // Duplicate — must coalesce into the first alice child. + templates::rebuild( + "alice", + Source::MetaUpdate, + "cascade again".to_owned(), + Some(meta), + false, + ), + ]; + let ids = q.append_children(specs); + assert_eq!(ids.len(), 3); + assert_eq!(ids[0], ids[2], "duplicate child dedups"); + let snap = q.snapshot(); + let children: Vec<_> = snap.iter().filter(|d| d.parent_id == Some(meta)).collect(); + assert_eq!(children.len(), 2); +} + +// ---- terminal reporting + lease release ---- + +#[test] +fn terminal_dag_reported_exactly_once_and_lease_released() { + let q = JobQueue::new(1); + let id = submit( + &q, + templates::restart("agent-a", Source::Manual, "r".to_owned()), + ); + let stop = claim_one(&q); + let r1 = q.complete_node(id, stop.node_id, Ok(())); + assert!(r1.terminal.is_empty(), "dag not terminal yet"); + let rec = claim_one(&q); + let r2 = q.complete_node(id, rec.node_id, Ok(())); + assert_eq!(r2.terminal.len(), 1); + assert_eq!(r2.terminal[0].dag_id, id); + assert_eq!(r2.terminal[0].state, State::Done); + // Lease released: a new DAG for the agent can claim immediately. + let next = submit( + &q, + templates::reconcile_only( + Template::Stop, + "agent-a", + Source::Manual, + "stop".to_owned(), + None, + ), + ); + let c = claim_one(&q); + assert_eq!(c.dag_id, next); + assert!(c.lease_acquired); +} + +#[test] +fn cancelled_dag_reports_terminal() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + assert!(q.cancel(id)); + // The cancel path settles internally; a subsequent completion + // report must not re-report it. Verify via a second dag's cycle. + let other = submit(&q, rebuild("agent-b", "r")); + let c = claim_one(&q); + assert_eq!(c.dag_id, other); + let report = q.complete_node(other, c.node_id, Err("boom".to_owned())); + // agent-b's dag isn't terminal (reconcile still pending) and + // agent-a's was already reported by cancel → nothing here. + assert!(report.terminal.iter().all(|t| t.dag_id != id)); +} + +// ---- steps, build logs, history ---- + +#[test] +fn set_step_only_on_running_and_signals_change() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + assert!(!q.set_step(id, 0, "too early"), "queued node refuses step"); + let c = claim_one(&q); + assert!(q.set_step(id, c.node_id, "nix build")); + assert!( + !q.set_step(id, c.node_id, "nix build"), + "same label → false" + ); + assert!(q.set_step(id, c.node_id, "next phase")); + assert!(q.set_step_running(id, "via running lookup")); + q.complete_node(id, c.node_id, Ok(())); + let snap = q.snapshot(); + let node = &snap.iter().find(|d| d.id == id).expect("dag").nodes[0]; + assert_eq!(node.step, None, "step cleared on completion"); +} + +#[test] +fn set_build_log_id_links_running_node() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + assert!(!q.set_build_log_id(id, 0, 41), "queued node refuses log id"); + let c = claim_one(&q); + assert!(q.set_build_log_id(id, c.node_id, 42)); + assert!(q.set_build_log_id_running(id, 43)); + q.complete_node(id, c.node_id, Ok(())); + let snap = q.snapshot(); + let node = &snap.iter().find(|d| d.id == id).expect("dag").nodes[0]; + assert_eq!(node.build_log_id, Some(43), "log id survives completion"); +} + +#[test] +fn history_evicts_old_terminals_per_template() { + let q = JobQueue::new(1); + for i in 0..8 { + let id = submit( + &q, + templates::reconcile_only( + Template::Start, + &format!("agent-{i}"), + Source::Manual, + "start".to_owned(), + None, + ), + ); + let c = claim_one(&q); + q.complete_node(id, c.node_id, Ok(())); + } + assert_eq!(q.snapshot().len(), 5, "per-template history cap"); + assert_eq!(q.live_count(), 0); +} + +#[test] +fn error_is_truncated() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + let c = claim_one(&q); + q.complete_node(id, c.node_id, Err("x".repeat(5000))); + let snap = q.snapshot(); + let err = snap.iter().find(|d| d.id == id).expect("dag").nodes[0] + .error + .clone() + .expect("error stored"); + assert!(err.chars().count() <= 2001, "truncated + ellipsis"); + assert!(err.ends_with('…')); +} + +// ---- template shapes ---- + +#[test] +fn graceful_stop_shape_signal_drain_reconcile() { + let q = JobQueue::new(1); + let id = submit( + &q, + templates::graceful_stop("agent-a", Source::Manual, "graceful".to_owned()), + ); + for expected in ["signal", "drain", "reconcile"] { + let c = claim_one(&q); + assert_eq!(c.kind.as_str(), expected); + q.complete_node(id, c.node_id, Ok(())); + } + assert_eq!(state_of(&q, id), State::Done); +} + +#[test] +fn graceful_signal_and_drain_hold_no_build_slot() { + // A whole-hive graceful stop overlaps every drain even at + // buildSlots = 1 while a rebuild hogs the slot. + let q = JobQueue::new(1); + submit(&q, rebuild("builder", "slot hog")); + submit( + &q, + templates::graceful_stop("agent-a", Source::Manual, "g".to_owned()), + ); + submit( + &q, + templates::graceful_stop("agent-b", Source::Manual, "g".to_owned()), + ); + let claims = q.claim_ready(); + let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect(); + assert_eq!( + kinds, + vec!["prebuild", "signal", "signal"], + "both agents' signals fire while the slot is held" + ); +} + +#[test] +fn spawn_shape_create_dropin_reconcile() { + let q = JobQueue::new(1); + let id = submit( + &q, + templates::spawn("newbie", 7, "approval #7 spawn".to_owned()), + ); + for expected in ["create", "write_dropin", "reconcile"] { + let c = claim_one(&q); + assert_eq!(c.kind.as_str(), expected); + assert_eq!(c.approval_id, Some(7)); + q.complete_node(id, c.node_id, Ok(())); + } + let report_terminal = state_of(&q, id); + assert_eq!(report_terminal, State::Done); +} + +#[test] +fn perm_change_shape_prefixes_rebuild_chain() { + let q = JobQueue::new(1); + let id = submit( + &q, + templates::perm_change( + "agent-a", + Source::Manual, + "perm".to_owned(), + PermPayload::Combined { + groups: Some(vec![]), + caps: None, + }, + ), + ); + for expected in [ + "write_perm_file", + "prebuild", + "stop_for_update", + "swap", + "reconcile", + ] { + let c = claim_one(&q); + assert_eq!(c.kind.as_str(), expected); + q.complete_node(id, c.node_id, Ok(())); + } + assert_eq!(state_of(&q, id), State::Done); +} diff --git a/hive-c0re/src/lib.rs b/hive-c0re/src/lib.rs index e58f2d0f..c02819a7 100644 --- a/hive-c0re/src/lib.rs +++ b/hive-c0re/src/lib.rs @@ -32,6 +32,7 @@ pub mod forge; pub mod gateway_nginx; pub mod hive_stats; pub mod host_stats; +pub mod job_queue; pub mod knowledge; pub mod lifecycle; pub mod limits; @@ -41,9 +42,9 @@ pub mod meta; pub mod migrate; pub mod operator_questions; pub mod paths; +pub mod power; pub mod priv_client; pub mod questions; -pub mod rebuild_queue; pub mod reminder_scheduler; pub mod scheduled_prompts; pub mod scheduled_prompts_worker; diff --git a/hive-c0re/src/lifecycle.rs b/hive-c0re/src/lifecycle.rs index 3296866a..4102a839 100644 --- a/hive-c0re/src/lifecycle.rs +++ b/hive-c0re/src/lifecycle.rs @@ -260,6 +260,16 @@ async fn port_collision(self_name: &str) -> Option { } pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> { + create_container(name, hive, paths).await?; + write_dropins(name, hive, paths).await?; + priv_run("start", name).await +} + +/// First-spawn provisioning + `nixos-container create`, without the +/// drop-in write or the start — the job queue's `Create` node. +/// `spawn` composes this with `write_dropins` + start for direct +/// callers (root-agent bootstrap). +pub async fn create_container(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> { validate(name)?; if let Some(other) = port_collision(name).await { bail!( @@ -277,8 +287,17 @@ pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> // ref resolves. let agents = agents_after_spawn(name).await?; crate::meta::sync_agents(hive, &agents).await?; + priv_run("create", name).await +} + +/// Re-apply the per-container host-side config: nspawn flags (bind +/// mounts etc.), the systemd resource-limits drop-in, and a daemon +/// reload so both take effect on the next unit (re)start. Idempotent — +/// the job queue's `WriteDropin` node, also folded into every `Swap` +/// (rebuild is the reconcile verb). +pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> { + validate(name)?; let container = container_name(name); - priv_run("create", name).await?; set_nspawn_flags( &container, &paths.agent_dir, @@ -287,8 +306,42 @@ pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> ) .await?; set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?; - systemd_daemon_reload().await?; - priv_run("start", name).await + systemd_daemon_reload().await +} + +/// Rebuild-path preamble shared by the job queue's `Prebuild` node and +/// `rebuild_no_meta`: fail fast on a port collision, then make sure +/// the applied repo + state dirs exist. Container untouched. +pub async fn prepare_rebuild_dirs(name: &str, paths: &AgentPaths) -> Result<()> { + validate(name)?; + if let Some(other) = port_collision(name).await { + bail!( + "port {} is already taken by '{other}' — rename one of them and retry", + agent_web_port(name) + ); + } + setup_applied(&paths.applied_dir, None, name).await?; + ensure_agent_state_subvolume(name).await?; + ensure_claude_dir(&paths.claude_dir)?; + ensure_state_dir(&paths.notes_dir)?; + Ok(()) +} + +/// Profile-swap for an existing, stopped container: re-apply the +/// drop-ins, then `nixos-container update`. The job queue's `Swap` +/// node. Requires the container stopped (the queue's `StopForUpdate` +/// upstream); does NOT start it — the DAG's tail `Reconcile` owns +/// bringing the agent back to its wanted power state. +pub async fn swap_update( + name: &str, + hive: &HiveEnv, + paths: &AgentPaths, + on_step: &(dyn Fn(&str) + Send + Sync), + on_build_log_id: &(dyn Fn(i64) + Send + Sync), +) -> Result<()> { + write_dropins(name, hive, paths).await?; + on_step("nixos-container update"); + priv_run_inner("update", name, Some(on_build_log_id)).await } /// Build the `AgentSpec` list for the meta flake from `nixos-container @@ -532,35 +585,16 @@ pub async fn rebuild_no_meta( on_step: &(dyn Fn(&str) + Send + Sync), on_build_log_id: &(dyn Fn(i64) + Send + Sync), ) -> Result { - validate(name)?; - if let Some(other) = port_collision(name).await { - bail!( - "port {} is already taken by '{other}' — rename one of them and retry", - agent_web_port(name) - ); - } - setup_applied(&paths.applied_dir, None, name).await?; - ensure_agent_state_subvolume(name).await?; - ensure_claude_dir(&paths.claude_dir)?; - ensure_state_dir(&paths.notes_dir)?; - let container = container_name(name); + prepare_rebuild_dirs(name, paths).await?; let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display()); if container_exists(name).await { // Rebuild strategy: stop-before-update + pre-build. // See `docs/coordinator.md::Container lifecycle`. let was_running = is_running(name).await; - set_nspawn_flags( - &container, - &paths.agent_dir, - &paths.claude_dir, - &paths.notes_dir, - ) - .await?; - set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?; - systemd_daemon_reload().await?; + write_dropins(name, hive, paths).await?; if was_running { on_step("nix build"); - prebuild_toplevel(name, &flake_ref).await?; + prebuild_toplevel(name, &flake_ref, &|_| ()).await?; on_step("nixos-container stop"); priv_run("stop", name).await?; } @@ -601,15 +635,7 @@ pub async fn rebuild_no_meta( // See `docs/coordinator.md::Spawn path`. on_step("nixos-container create"); priv_run("create", name).await?; - set_nspawn_flags( - &container, - &paths.agent_dir, - &paths.claude_dir, - &paths.notes_dir, - ) - .await?; - set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?; - systemd_daemon_reload().await?; + write_dropins(name, hive, paths).await?; on_step("nixos-container start"); priv_run("start", name).await?; Ok(false) @@ -622,7 +648,15 @@ pub async fn rebuild_no_meta( /// is untouched. See `docs/coordinator.md::Rebuild path` for why /// the prebuild happens before stop, and `docs/coordinator.md::Prebuild /// attr path` for why the explicit nixosConfigurations attr is required. -async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> { +/// +/// `on_build_log_id` fires with the `build_logs` row id as soon as the +/// row opens, so queue-side callers can link their node to the live +/// stream. Pass `&|_| ()` when not needed. +pub async fn prebuild_toplevel( + name: &str, + flake_ref: &str, + on_build_log_id: &(dyn Fn(i64) + Send + Sync), +) -> Result<()> { use tokio::io::{AsyncBufReadExt, BufReader}; // Split `#` so we can re-emit with the explicit // `nixosConfigurations.` segment. The flake_ref shape is @@ -663,6 +697,9 @@ async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> { }) .ok() }); + if let Some(id) = log_id { + on_build_log_id(id); + } let mut child = Command::new("nix") .args(&args) diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index f39fb1e4..2b61082f 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -13,8 +13,8 @@ use hive_sh4re::{HostRequest, HostResponse}; use hive_c0re::coordinator::{Coordinator, HiveEnv, ServeConfig}; use hive_c0re::{ agent_sockets, auto_update, broker, client, crash_watch, dashboard, dashboard_events, forge, - knowledge, matrix, migrate, rebuild_queue, reminder_scheduler, scheduled_prompts_worker, - server, socket_server, + job_queue, knowledge, matrix, migrate, reminder_scheduler, scheduled_prompts_worker, server, + socket_server, }; #[derive(Parser)] @@ -85,6 +85,11 @@ enum Cmd { /// option. #[arg(long)] model_prices: Option, + /// Override: number of concurrent nix-heavy job-queue nodes + /// (prebuild / profile-swap / create / meta lock). Set via the + /// `services.hyperhive.c0re.buildSlots` NixOS option. + #[arg(long)] + build_slots: Option, }, /// Spawn a new agent container directly (`hive-agent-`). Bypasses /// the approval queue — use only as an operator on the host. For @@ -156,6 +161,7 @@ async fn main() -> Result<()> { agent_cpu_quota, agent_memory_max, model_prices, + build_slots, } => { // Base config from the --config file (or the built-in // defaults), then apply any per-flag overrides — config @@ -195,7 +201,10 @@ async fn main() -> Result<()> { sc.model_prices = serde_json::from_str(&v).context("--model-prices: invalid JSON")?; } - cmd_serve(sc.env, sc.model_prices, db, &cli.socket).await + if let Some(v) = build_slots { + sc.build_slots = v; + } + cmd_serve(sc.env, sc.model_prices, sc.build_slots, db, &cli.socket).await } Cmd::Spawn { name } => { render(client::request(&cli.socket, HostRequest::Spawn { name }).await?) @@ -244,6 +253,7 @@ async fn main() -> Result<()> { async fn cmd_serve( env: HiveEnv, model_prices: hive_c0re::hive_stats::PriceTable, + build_slots: usize, db: std::path::PathBuf, socket: &std::path::Path, ) -> Result<()> { @@ -255,7 +265,7 @@ async fn cmd_serve( // `dashboard_port` is consumed into the Coordinator below; capture the // Copy value first for the dashboard + knowledge-webhook tasks. let dashboard_port = env.dashboard_port; - let coord = Arc::new(Coordinator::open(&db, env, model_prices)?); + let coord = Arc::new(Coordinator::open(&db, env, model_prices, build_slots)?); socket_server::start_manager(coord.clone())?; // Idempotent pre-flight: rewrite pre-meta-layout applied // repos, ensure proposed repos carry the `applied` @@ -413,25 +423,17 @@ async fn cmd_serve( // and fans the body out to each active target's inbox. See // scheduled_prompts_worker.rs. scheduled_prompts_worker::spawn(coord.clone()); - // Rebuild-queue worker: drains the global rebuild/meta-update/ - // spawn queue FIFO so hive-c0re never runs two heavyweight - // container ops concurrently. Existing rebuild call sites - // (auto_update, dashboard, manager, approval handler) enqueue - // here instead of awaiting `rebuild_agent` inline. See - // `rebuild_queue.rs`. + // Job-queue scheduler: drives the global DAG queue (rebuild / + // meta-update / spawn / power ops). Concurrency comes from the + // build-slot count + per-agent leases inside the queue, not from + // multiple workers — cheap nodes (graceful signals, drains, + // reconciles) overlap nix-heavy ones structurally. Call sites + // (auto_update, dashboard, manager, approval handler) submit DAGs + // instead of awaiting lifecycle work inline. See `job_queue/`. { let q_coord = coord.clone(); tokio::spawn(async move { - rebuild_queue::run_worker(q_coord).await; - }); - // Fast lane: a second serial worker for hard Start/Stop, running - // concurrently with the build worker above so a stop/start never - // waits behind a slow build for another container. Per-agent - // ordering vs that agent's own build is enforced in the queue's - // claim logic (a fast op defers behind its agent's running build). - let fast_coord = coord.clone(); - tokio::spawn(async move { - rebuild_queue::run_fast_worker(fast_coord).await; + job_queue::scheduler::run_worker(q_coord).await; }); } // Forward every broker event onto the unified dashboard diff --git a/hive-c0re/src/power.rs b/hive-c0re/src/power.rs new file mode 100644 index 00000000..dbf71f69 --- /dev/null +++ b/hive-c0re/src/power.rs @@ -0,0 +1,203 @@ +//! Durable per-agent power *intent* (`wanted: Up | Offline`) — the +//! spec half of spec-vs-status desired-state reconciliation. +//! `container_view` remains the observed *status*; the job queue's +//! `Reconcile` nodes are the mechanism that converges the two. +//! +//! Stored in `/var/lib/hyperhive/db/agent_power.sqlite` (one tiny row +//! per agent). Intent persists across hive-c0re restarts; in-flight +//! queue work deliberately does not. Setting `wanted` is never a +//! queued node: operator/intent actions update the row synchronously +//! at request time, then submit the DAG whose terminal `Reconcile` +//! reads the fresh value — rapid toggles are last-writer-wins and the +//! reconciles converge. Power toggles never commit to the meta repo. + +use std::path::Path; +use std::sync::Mutex; + +use anyhow::{Context, Result}; +use rusqlite::{Connection, OptionalExtension, params}; + +const SCHEMA: &str = " +CREATE TABLE IF NOT EXISTS agent_power ( + agent TEXT PRIMARY KEY, + wanted TEXT NOT NULL, + updated_at INTEGER NOT NULL +); +"; + +/// Per-agent power intent. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Wanted { + Up, + Offline, +} + +impl Wanted { + pub fn as_str(self) -> &'static str { + match self { + Wanted::Up => "up", + Wanted::Offline => "offline", + } + } + + fn parse(s: &str) -> Option { + match s { + "up" => Some(Wanted::Up), + "offline" => Some(Wanted::Offline), + _ => None, + } + } + + /// Seed value from an observed running state (first boot after + /// this store lands, or an agent spawned outside the normal path). + pub fn from_running(running: bool) -> Self { + if running { Wanted::Up } else { Wanted::Offline } + } +} + +/// What a `Reconcile` should do given intent + observation. Pure so +/// the `{Up,Offline} × {up,down}` matrix is unit-testable. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReconcileAction { + Start, + Stop, + Noop, +} + +#[must_use] +pub fn reconcile_action(wanted: Wanted, running: bool) -> ReconcileAction { + match (wanted, running) { + (Wanted::Up, false) => ReconcileAction::Start, + (Wanted::Offline, true) => ReconcileAction::Stop, + (Wanted::Up, true) | (Wanted::Offline, false) => ReconcileAction::Noop, + } +} + +/// Sqlite-backed store. `Arc`-friendly: all methods take `&self`, the +/// internal `Mutex` serializes access. +pub struct PowerStore { + conn: Mutex, +} + +impl PowerStore { + pub fn open(db_dir: &Path) -> Result { + std::fs::create_dir_all(db_dir) + .with_context(|| format!("create agent_power db parent {}", db_dir.display()))?; + let path = db_dir.join("agent_power.sqlite"); + let conn = Connection::open(&path) + .with_context(|| format!("open agent_power db {}", path.display()))?; + conn.execute_batch(SCHEMA) + .context("apply agent_power schema")?; + Ok(Self { + conn: Mutex::new(conn), + }) + } + + /// In-memory store for tests. + #[cfg(test)] + pub fn open_in_memory() -> Result { + let conn = Connection::open_in_memory().context("open in-memory agent_power db")?; + conn.execute_batch(SCHEMA) + .context("apply agent_power schema")?; + Ok(Self { + conn: Mutex::new(conn), + }) + } + + /// Read an agent's intent. `None` when the agent has no row yet + /// (callers seed from observed state via [`Self::get_or_seed`]). + pub fn get(&self, agent: &str) -> Result> { + let conn = self.conn.lock().expect("agent_power mutex poisoned"); + let row: Option = conn + .query_row( + "SELECT wanted FROM agent_power WHERE agent = ?1", + params![agent], + |r| r.get(0), + ) + .optional() + .context("select agent_power")?; + Ok(row.and_then(|s| Wanted::parse(&s))) + } + + /// Write an agent's intent (last-writer-wins, synchronous at + /// request time). + pub fn set(&self, agent: &str, wanted: Wanted) -> Result<()> { + let conn = self.conn.lock().expect("agent_power mutex poisoned"); + conn.execute( + "INSERT INTO agent_power (agent, wanted, updated_at) VALUES (?1, ?2, ?3) + ON CONFLICT(agent) DO UPDATE SET wanted = ?2, updated_at = ?3", + params![agent, wanted.as_str(), now_secs()], + ) + .context("upsert agent_power")?; + Ok(()) + } + + /// Read an agent's intent, seeding the row from the observed + /// running state when absent — the migration rule for agents that + /// predate this store (running ⇒ `Up`, stopped ⇒ `Offline`), after + /// which the DB is authoritative. + pub fn get_or_seed(&self, agent: &str, running: bool) -> Result { + if let Some(w) = self.get(agent)? { + return Ok(w); + } + let seeded = Wanted::from_running(running); + self.set(agent, seeded)?; + tracing::info!(%agent, wanted = seeded.as_str(), "agent_power: seeded from observed state"); + Ok(seeded) + } + + /// Drop an agent's row (container destroyed). + pub fn remove(&self, agent: &str) -> Result<()> { + let conn = self.conn.lock().expect("agent_power mutex poisoned"); + conn.execute("DELETE FROM agent_power WHERE agent = ?1", params![agent]) + .context("delete agent_power")?; + Ok(()) + } +} + +fn now_secs() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .ok() + .and_then(|d| i64::try_from(d.as_secs()).ok()) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The full `{Up,Offline} × {up,down}` reconcile matrix: + /// start / stop / noop / noop. + #[test] + fn reconcile_matrix() { + assert_eq!(reconcile_action(Wanted::Up, false), ReconcileAction::Start); + assert_eq!( + reconcile_action(Wanted::Offline, true), + ReconcileAction::Stop + ); + assert_eq!(reconcile_action(Wanted::Up, true), ReconcileAction::Noop); + assert_eq!( + reconcile_action(Wanted::Offline, false), + ReconcileAction::Noop + ); + } + + #[test] + fn get_set_roundtrip_and_seed() { + let store = PowerStore::open_in_memory().expect("open"); + assert_eq!(store.get("alice").expect("get"), None); + // Seed from observed running state, once. + assert_eq!(store.get_or_seed("alice", true).expect("seed"), Wanted::Up); + // Thereafter the DB is authoritative — observed state no longer + // overrides. + assert_eq!( + store.get_or_seed("alice", false).expect("seeded"), + Wanted::Up + ); + store.set("alice", Wanted::Offline).expect("set"); + assert_eq!(store.get("alice").expect("get"), Some(Wanted::Offline)); + store.remove("alice").expect("remove"); + assert_eq!(store.get("alice").expect("get"), None); + } +} diff --git a/hive-c0re/src/rebuild_queue.rs b/hive-c0re/src/rebuild_queue.rs deleted file mode 100644 index b6853698..00000000 --- a/hive-c0re/src/rebuild_queue.rs +++ /dev/null @@ -1,2158 +0,0 @@ -//! Global rebuild queue — serialises all long-running container/meta -//! operations (rebuild, meta-update, first-spawn) through a single -//! background worker. Design rationale, kind taxonomy, dedup rules, -//! cascade parent tracking, and step labels: -//! `docs/coordinator.md::Rebuild queue`. - -use std::collections::VecDeque; -use std::sync::Mutex; - -use anyhow::Context as _; -use serde::{Deserialize, Serialize}; -use tokio::sync::Notify; - -/// What the queue can run. Each variant maps to a specific worker -/// execution path; `agent` (in `QueueEntry`) names the target where -/// relevant. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum QueueKind { - /// Rebuild a single agent's container (`auto_update::rebuild_agent`). - Rebuild, - /// Run `nix flake update` on the meta flake. Triggers cascade - /// `Rebuild` entries (with `parent_id`) once the lock bump lands. - MetaUpdate, - /// First-deploy spawn of a new agent (approval-driven). - Spawn, - /// Destroy with `--purge` (real fs work). Not yet routed here; the - /// variant exists so the wire shape doesn't need to change later. - #[allow(dead_code, reason = "wire shape — routed by a future PR")] - Destroy, - /// hive-c0re boot-time sweep: bumps the meta hyperhive lock then - /// enqueues a `Rebuild` child for every managed container. Completes - /// after the lock bump; children run as independent queue entries - /// grouped under this parent's `id`. `agent` = `"hyperhive"`. - StartupSweep, - /// Stop + start a container without touching config. Fast op (~5-10s). - /// Queued so it serialises against in-flight rebuilds for the same - /// agent — prevents a restart racing a rebuild mid-flight. - Restart, - /// Write a tool-group or capability change to the shared JSON file, - /// then rebuild the agent so the new env var takes effect. - /// Serialised through the queue so concurrent dashboard batch-apply - /// actions for different agents never race on the shared JSON file. - PermChange, - /// Gracefully stop a container: signal the harness to run one - /// stop-checkpoint turn (flush durable `/state`), then hand the drain-wait - /// to a detached watcher (freeing the build lane) which, once the agent - /// drains or `GRACEFUL_STOP_TIMEOUT` elapses, enqueues a fast-lane `Stop` - /// for the actual `nixos-container stop`. The build worker only does the - /// cheap signal, so whole-hive graceful stops overlap every agent's drain. - GracefulStop, - /// Start a stopped container (`lifecycle::start`). Routed through the - /// queue so the dashboard shows a visible queued→running transient — a - /// direct sub-second start only flashes the badge — and bulk starts - /// serialise legibly on the queue. Fast op. - Start, - /// Hard-stop a container (`lifecycle::kill`), no quiesce. Routed through - /// the queue for the same visible-progress reason as `Start`; the - /// quiescing variant is `GracefulStop`. Fast op. - Stop, -} - -impl QueueKind { - pub fn as_str(self) -> &'static str { - match self { - QueueKind::Rebuild => "rebuild", - QueueKind::MetaUpdate => "meta_update", - QueueKind::Spawn => "spawn", - QueueKind::Destroy => "destroy", - QueueKind::StartupSweep => "startup_sweep", - QueueKind::Restart => "restart", - QueueKind::PermChange => "perm_change", - QueueKind::GracefulStop => "graceful_stop", - QueueKind::Start => "start", - QueueKind::Stop => "stop", - } - } - - /// Fast-lane kinds: hard `Start` / `Stop`. These run on a separate - /// serial fast worker concurrently with the build lane (so a stop/start - /// never waits behind another container's slow build). `GracefulStop` - /// and `Restart` are deliberately NOT fast — they go through the build - /// lane (`GracefulStop` does the cheap harness signal then detaches the - /// drain-wait, enqueueing a fast-lane `Stop` for the real container stop; - /// `Restart` is a stop+start). - pub fn is_fast(self) -> bool { - matches!(self, QueueKind::Start | QueueKind::Stop) - } -} - -/// Kind-specific payload for `QueueKind::PermChange` entries. -/// Carries the desired new value so the worker can apply the file -/// write (serialised, in FIFO order) without racing concurrent HTTP -/// handlers writing to the same shared JSON file. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum PermPayload { - /// Set the tool groups for one agent (`tool-groups.json`). - ToolGroups { groups: Vec }, - /// Set the capabilities for one agent (`capabilities.json`). - Capabilities { caps: Vec }, - /// Set both perm-types for one agent in a single entry — the batch - /// `POST /api/permissions` path. Either field `None` leaves that - /// file untouched (no write, no commit); the worker commits whichever - /// are present in one git commit, then rebuilds once. Collapses the - /// dedup key to `(kind, agent)` so caps + groups for one agent - /// produce a single rebuild rather than two. - Combined { - groups: Option>, - caps: Option>, - }, -} - -/// Where the enqueue request originated. Drives the "why" chip on the -/// dashboard and lets the UI group cascade entries under their parent -/// without parsing the reason text. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum QueueSource { - /// Operator clicked rebuild / update-all / meta-update on the - /// dashboard, or any other direct human action (CLI, manager tool). - Manual, - /// Spawned as a cascade from a `MetaUpdate` entry's lock-bump - /// fan-out. The `parent_id` on the `QueueEntry` points back at - /// the originating meta-update. - MetaUpdate, - /// `auto_update::run` startup sweep — rebuild every container on - /// hive-c0re boot. Legacy flat source (no parent); replaced by - /// `StartupSweep` for the parent entry and child rebuilds once the - /// queue introduced `parent_id` grouping. Kept for wire compatibility - /// with entries logged before the migration. - AutoUpdate, - /// Direct child of a `StartupSweep` queue entry — one per agent in - /// the boot-time rebuild sweep. Carries `parent_id` back-link so - /// the dashboard renders the sweep's per-agent rebuilds nested under - /// the parent header. The parent entry itself uses `QueueSource::AutoUpdate` - /// (automated boot action, not operator-driven). - StartupSweep, - /// Crash recovery path (future use — currently no auto-rebuild on - /// crash, but the variant exists for the imminent feature). - #[allow(dead_code, reason = "wire shape — used by a future feature")] - CrashRecover, - /// Operator approved a pending `Approval` row on the dashboard. - /// `QueueEntry.approval_id` points back at the source row so the - /// worker can fetch the kind-specific payload (`commit_ref`, inputs, - /// description) before dispatching. - Approval, -} - -impl QueueSource { - pub fn as_str(self) -> &'static str { - match self { - QueueSource::Manual => "manual", - QueueSource::MetaUpdate => "meta_update", - QueueSource::AutoUpdate => "auto_update", - QueueSource::StartupSweep => "startup_sweep", - QueueSource::CrashRecover => "crash_recover", - QueueSource::Approval => "approval", - } - } -} - -/// Lifecycle state of an entry. `Done` / `Failed` / `Cancelled` are -/// retained in the queue snapshot for a short tail (`MAX_HISTORY_PER_KIND`) -/// so the dashboard can show "last few" runs alongside live state. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum QueueState { - Queued, - Running, - Done, - Failed, - Cancelled, -} - -impl QueueState { - pub fn is_terminal(self) -> bool { - matches!( - self, - QueueState::Done | QueueState::Failed | QueueState::Cancelled - ) - } -} - -/// A single queue entry — what's pending, running, or recently finished. -/// Serialised verbatim onto the dashboard event channel and the -/// `/api/state` snapshot. -#[derive(Debug, Clone, Serialize)] -pub struct QueueEntry { - /// Monotonic per-process id. Stable for the lifetime of the entry - /// so SSE upserts land in place rather than churning the list. - pub id: u64, - /// Target agent name, or the literal `"hyperhive"` for entries - /// (`MetaUpdate`) that affect the meta flake rather than a single - /// agent. - pub agent: String, - pub kind: QueueKind, - pub state: QueueState, - pub source: QueueSource, - /// Groups cascade entries under their originating parent. For a - /// `MetaUpdate` entry this is `None`; for the per-agent rebuilds - /// the worker enqueues after the lock bump it's `Some(meta_id)`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub parent_id: Option, - /// Human-readable "why" — populated by the enqueuer (`"manual via - /// dashboard"`, `"meta-update cascade (hyperhive bumped)"`, - /// `"startup sweep"`). Free-form; dedup appends `(also requested - /// by …)` lines on repeated enqueues. - pub reason: String, - pub enqueued_at: i64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub started_at: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub finished_at: Option, - /// Populated when `state == Failed`. Carries the worker's error - /// string (already truncated to a reasonable length by the caller). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub error: Option, - /// `MetaUpdate`-only payload: the list of meta flake inputs to run - /// through `nix flake update`. Empty / absent on `Rebuild` / - /// `Spawn` / `Destroy` entries; absent on the wire (never - /// serialised) when the entry kind doesn't have meaningful inputs. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub inputs: Vec, - /// Source approval row id when this entry was created by an - /// operator-approve POST (`source == Approval`). The worker uses - /// it to re-fetch the kind-specific payload (`commit_ref` / inputs / - /// description / `fetched_sha`) and to fire `ApprovalResolved` on - /// completion. `None` for non-approval entries — preserved on - /// the wire that way too. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub approval_id: Option, - /// Current sub-step inside the running entry. Worker mutates this - /// as the kind-specific pipeline - /// advances through phases (e.g. `"plant tags"` → - /// `"nixos-container update"` → `"finalize deploy"`). `None` while - /// `Queued` and after terminal — only meaningful with - /// `state == Running`. Each transition fires a fresh - /// `RebuildQueueChanged` snapshot so the dashboard can render - /// the label as a sub-line on the queue card. Free-form per - /// pipeline; the kind-specific worker is the source of truth. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub step: Option, - /// `PermChange`-only payload: the desired new permission value to - /// apply. Absent (`None`) on all other entry kinds — omitted from - /// the wire in those cases. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub perm_payload: Option, - /// Entries this entry must wait for before it can run. The worker - /// skips this entry until every id in the list has reached a - /// terminal state (`Done` / `Failed` / `Cancelled`) — or no longer - /// exists in the queue (evicted terminal entries are treated as - /// resolved, since `trim_history` only evicts terminals). Empty on - /// most entries; serialised only when non-empty. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub depends_on: Vec, - /// Row id of the associated `build_logs` entry (opened by the - /// lifecycle worker when `nixos-container update` starts). Set - /// shortly after `state` transitions to `Running`; `None` while - /// `Queued` or for entries that don't open a build log (`Restart`, - /// `PermChange` file-write phase, etc.). Links the queue card to - /// the live-streaming `/api/build-logs/id/{id}/stream` endpoint so - /// the operator can follow the nix build output in real time. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub build_log_id: Option, -} - -/// How many terminal-state entries (`Done` / `Failed` / `Cancelled`) -/// to retain per kind in the snapshot. Older entries get evicted to -/// keep `/api/state` tight; the live event channel is unaffected. -const MAX_HISTORY_PER_KIND: usize = 5; - -/// Inner state guarded by a single mutex. Held briefly — every -/// operation is constant-time relative to the queue's depth, and -/// the depths in practice are tiny (single-digit). -#[derive(Debug, Default)] -struct Inner { - entries: VecDeque, - next_id: u64, -} - -/// Global rebuild queue. Lives on `Coordinator` (one per hive-c0re -/// process). The associated `Notify` wakes the worker when something -/// new arrives. -#[derive(Debug)] -pub struct RebuildQueue { - inner: Mutex, - /// Build-lane worker wakes on this signal. The worker checks the queue - /// and loops back to `notified().await` when there's nothing to run. - /// Also nudged by the fast worker when a fast op finishes (a build may - /// have been deferred behind a `Running` fast op for the same agent). - pub(crate) notify: Notify, - /// Fast-lane worker wakes on this signal. Nudged on a fast `Start` / - /// `Stop` enqueue and by the build worker when a build finishes (a - /// deferred `Start` may now be runnable). - pub(crate) fast_notify: Notify, -} - -impl Default for RebuildQueue { - fn default() -> Self { - Self { - inner: Mutex::new(Inner::default()), - notify: Notify::new(), - fast_notify: Notify::new(), - } - } -} - -/// Full-shape submit spec for [`RebuildQueue::enqueue_full`] — every -/// `QueueEntry` field settable at submit time. The thinner `enqueue` -/// / `enqueue_with_inputs` / `enqueue_with_perm` wrappers build this -/// for the common cases. -pub struct FullEnqueue { - pub kind: QueueKind, - pub agent: String, - pub source: QueueSource, - pub reason: String, - pub parent_id: Option, - pub inputs: Vec, - pub approval_id: Option, - pub perm_payload: Option, - pub depends_on: Vec, -} - -impl RebuildQueue { - pub fn new() -> Self { - Self::default() - } - - /// Add an entry to the queue. Returns the entry's id (newly-allocated - /// or — on dedup — the existing entry's id with the new reason - /// appended). - /// - /// Dedup rule: - /// - `Rebuild` / `Spawn` / `Destroy`: a `Queued` entry with the same - /// `(kind, agent, parent_id)` swallows the new request. `parent_id` - /// is part of the key so that a `MetaUpdate` cascade rebuild (with a - /// specific `parent_id`) never collapses into a standalone rebuild or - /// a cascade from a different `MetaUpdate`. Without this guard a - /// cascade rebuild pre-enqueued before the lock bump would be swallowed - /// by an existing `Queued` startup-sweep rebuild, causing the agent to - /// never rebuild against the post-bump meta. - /// - `MetaUpdate`: dedup ALSO requires the `inputs` field to match — - /// two meta-updates with different input lists are distinct work - /// and must queue separately, otherwise the second meta-update - /// would silently collapse into the first whenever it was still - /// `Queued`, losing the second's input set. - /// - /// Running and terminal entries never dedup — operators are free - /// to re-queue a rebuild that's currently running (something - /// changed since it started) or re-run one that just finished. - pub fn enqueue( - &self, - kind: QueueKind, - agent: String, - source: QueueSource, - reason: String, - parent_id: Option, - ) -> u64 { - self.enqueue_full(FullEnqueue { - kind, - agent, - source, - reason, - parent_id, - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: Vec::new(), - }) - } - - /// Same as `enqueue` but carries an `inputs` payload — used by - /// `MetaUpdate` enqueues to tell the worker which meta-flake - /// inputs to bump. For `MetaUpdate` the `inputs` value is part of - /// the dedup key (two meta-updates with different inputs are - /// distinct operations). - pub fn enqueue_with_inputs( - &self, - kind: QueueKind, - agent: String, - source: QueueSource, - reason: String, - parent_id: Option, - inputs: Vec, - ) -> u64 { - self.enqueue_full(FullEnqueue { - kind, - agent, - source, - reason, - parent_id, - inputs, - approval_id: None, - perm_payload: None, - depends_on: Vec::new(), - }) - } - - /// Enqueue a `PermChange` entry for `agent`. The worker applies the - /// JSON file write (serialised through FIFO) then rebuilds the - /// container so the updated env var takes effect. - pub fn enqueue_with_perm( - &self, - agent: String, - source: QueueSource, - reason: String, - payload: PermPayload, - ) -> u64 { - self.enqueue_full(FullEnqueue { - kind: QueueKind::PermChange, - agent, - source, - reason, - parent_id: None, - inputs: Vec::new(), - approval_id: None, - perm_payload: Some(payload), - depends_on: Vec::new(), - }) - } - - /// Full-shape enqueue — every `QueueEntry` field that's settable - /// at submit time. Existing `enqueue` / `enqueue_with_inputs` / - /// `enqueue_with_perm` delegate to this; the approval-driven POST - /// handlers call it directly with the source row's id so the - /// worker can re-fetch the kind-specific payload. - pub fn enqueue_full(&self, spec: FullEnqueue) -> u64 { - let FullEnqueue { - kind, - agent, - source, - reason, - parent_id, - inputs, - approval_id, - perm_payload, - depends_on, - } = spec; - let mut inner = self.inner.lock().expect("rebuild_queue mutex poisoned"); - // Dedup against a pending entry with the same (kind, agent) — - // and, for MetaUpdate, the same `inputs` list (see method - // docstring for why). Approval-driven entries also require the - // approval_id to match so two distinct approvals for the same - // agent never collapse into one queue slot. Rebuild (and Spawn / - // Destroy) entries also require parent_id to match so a - // MetaUpdate cascade rebuild is never swallowed by an unrelated - // queued rebuild (e.g. from the startup sweep). PermChange - // entries additionally check the perm type discriminant — a - // tool-groups change and a capabilities change for the same - // agent are distinct operations and must not collapse into one. - for entry in &mut inner.entries { - let perm_type_matches = matches!( - (&entry.perm_payload, &perm_payload), - ( - Some(PermPayload::ToolGroups { .. }), - Some(PermPayload::ToolGroups { .. }) - ) | ( - Some(PermPayload::Capabilities { .. }), - Some(PermPayload::Capabilities { .. }) - ) | ( - Some(PermPayload::Combined { .. }), - Some(PermPayload::Combined { .. }) - ) | (None, None) - ); - if entry.state == QueueState::Queued - && entry.kind == kind - && entry.agent == agent - && (kind != QueueKind::MetaUpdate || entry.inputs == inputs) - && entry.approval_id == approval_id - && entry.parent_id == parent_id - && perm_type_matches - && entry.depends_on == depends_on - { - if !entry.reason.contains(&reason) { - use std::fmt::Write as _; - let _ = write!(entry.reason, "\nalso requested by: {reason}"); - } - return entry.id; - } - } - inner.next_id += 1; - let id = inner.next_id; - let entry = QueueEntry { - id, - agent, - kind, - state: QueueState::Queued, - source, - parent_id, - reason, - enqueued_at: now_unix(), - started_at: None, - finished_at: None, - error: None, - inputs, - approval_id, - step: None, - perm_payload, - depends_on, - build_log_id: None, - }; - inner.entries.push_back(entry); - // Wake the worker for this entry's lane (fast = Start/Stop, build = - // everything else). `notify_one` is a no-op when there's no waiter; - // the next `notified().await` returns immediately. - if kind.is_fast() { - self.fast_notify.notify_one(); - } else { - self.notify.notify_one(); - } - id - } - - /// Claim the next runnable `Queued` entry for the **build** lane (every - /// kind except the fast `Start` / `Stop`) and mark it `Running`. See - /// [`Self::claim`] for the dependency + per-agent rules. - pub fn take_next_build(&self) -> Option { - self.claim(false) - } - - /// Claim the next runnable `Queued` entry for the **fast** lane (hard - /// `Start` / `Stop`) and mark it `Running`. Runs on its own serial - /// worker concurrently with the build lane. See [`Self::claim`]. - pub fn take_next_fast(&self) -> Option { - self.claim(true) - } - - /// Pop the next `Queued` entry for one lane whose dependencies are - /// resolved and which doesn't race the same agent's other-lane work, - /// and mark it `Running`. Returns the entry (a clone — the original - /// stays in the queue so live state reflects "this is currently - /// running"). Returns `None` when nothing in this lane is runnable. - /// - /// `want_fast` selects the lane: `true` = fast (`Start` / `Stop`), - /// `false` = build (everything else). The two lanes run on separate - /// serial workers, so this is called from both — the lane filter keeps - /// each worker to its own kinds. - /// - /// A dependency is "resolved" when the dep's id is either: - /// - still in the queue AND in a terminal state (`Done` / `Failed` - /// / `Cancelled`), OR - /// - no longer in the queue (evicted by `trim_history` — only - /// terminal entries are ever evicted, so missing == completed). - /// - /// Per-agent cross-lane guard (so a fast op never races that agent's - /// own build, and vice versa): - /// - a fast op waits while its agent has a build entry `Running`; - /// a `Start` additionally waits while its agent has a build entry - /// `Queued` (a start of a soon-to-be-rebuilt container is pointless); - /// - a build op waits while its agent has a fast op `Running`. - fn claim(&self, want_fast: bool) -> Option { - let mut inner = self.inner.lock().expect("rebuild_queue mutex poisoned"); - // Collect ids that are still in the queue and terminal. Entries - // absent from the queue are also considered resolved (see above). - let terminal_ids: std::collections::HashSet = inner - .entries - .iter() - .filter(|e| e.state.is_terminal()) - .map(|e| e.id) - .collect(); - // Active (non-terminal) ids: Queued + Running. Named `active_ids` - // rather than `queued_ids` because Running entries are included; - // used to distinguish "still in flight" from "evicted (= resolved)". - let active_ids: std::collections::HashSet = inner - .entries - .iter() - .filter(|e| !e.state.is_terminal()) - .map(|e| e.id) - .collect(); - let pos = { - let entries = &inner.entries; - entries.iter().position(|e| { - e.state == QueueState::Queued - && e.kind.is_fast() == want_fast - && e.depends_on.iter().all(|dep_id| { - // Resolved if terminal in queue OR not in queue at all. - // Note: circular deps (A depends on B, B depends on A) - // silently deadlock — neither entry ever becomes - // runnable. Callers must ensure acyclic dep graphs. - terminal_ids.contains(dep_id) || !active_ids.contains(dep_id) - }) - && lane_clear(entries, e) - }) - }?; - let entry = &mut inner.entries[pos]; - entry.state = QueueState::Running; - entry.started_at = Some(now_unix()); - Some(entry.clone()) - } - - /// Mark an entry terminal. `error` is populated for `Failed`; - /// `Done` / `Cancelled` ignore it. Trims the history tail. - /// Clears `step` — the field is only meaningful while `Running`, - /// and leaving a stale "in flight" label after a terminal - /// transition would mislead the dashboard render. - pub fn finish(&self, id: u64, state: QueueState, error: Option) { - debug_assert!( - state.is_terminal(), - "finish() called with non-terminal {state:?}" - ); - let mut inner = self.inner.lock().expect("rebuild_queue mutex poisoned"); - if let Some(entry) = inner.entries.iter_mut().find(|e| e.id == id) { - entry.state = state; - entry.finished_at = Some(now_unix()); - entry.error = error.filter(|_| state == QueueState::Failed); - entry.step = None; - } - Self::trim_history(&mut inner); - } - - /// Set the current sub-step label on a `Running` entry. - /// Returns `true` when the row was found AND the label changed - /// (caller should emit a `RebuildQueueChanged` snapshot only on - /// `true` to avoid noisy duplicate frames). No-op for entries not - /// in `Running` — the field is conceptually undefined outside - /// that state. - pub fn set_step(&self, id: u64, step: impl Into) -> bool { - let new_step = step.into(); - let mut inner = self.inner.lock().expect("rebuild_queue mutex poisoned"); - let Some(entry) = inner.entries.iter_mut().find(|e| e.id == id) else { - return false; - }; - if entry.state != QueueState::Running { - return false; - } - if entry.step.as_deref() == Some(new_step.as_str()) { - return false; - } - entry.step = Some(new_step); - true - } - - /// Link a `build_logs` row to a `Running` entry. Called by the - /// lifecycle worker when `nixos-container update` opens a build log - /// row so the dashboard can surface a "view logs" link while the - /// build is in flight. Returns `true` when the row was found and - /// the id was stored; `false` when the entry is no longer in the - /// queue or is not `Running`. - pub fn set_build_log_id(&self, id: u64, log_id: i64) -> bool { - let mut inner = self.inner.lock().expect("rebuild_queue mutex poisoned"); - let Some(entry) = inner.entries.iter_mut().find(|e| e.id == id) else { - return false; - }; - if entry.state != QueueState::Running { - return false; - } - entry.build_log_id = Some(log_id); - true - } - - /// Snapshot the queue for `/api/state` and `RebuildQueueChanged`. - /// Cheap clone — entries are small (~hundreds of bytes each). - pub fn snapshot(&self) -> Vec { - let inner = self.inner.lock().expect("rebuild_queue mutex poisoned"); - inner.entries.iter().cloned().collect() - } - - /// Cancel every `Queued` entry whose `parent_id` matches `parent`. - /// Used when a `MetaUpdate` parent fails its lock bump — the - /// cascade rebuilds the enqueuer pre-queued no longer apply - /// (nothing actually changed, so they'd be wasted work). Running - /// children are left alone — they were started under the parent's - /// assumption and can't be cleanly aborted from the queue side. - /// Returns the count of cancelled entries. - pub fn cancel_children(&self, parent: u64) -> usize { - let mut inner = self.inner.lock().expect("rebuild_queue mutex poisoned"); - let mut count = 0; - for entry in &mut inner.entries { - if entry.parent_id == Some(parent) && entry.state == QueueState::Queued { - entry.state = QueueState::Cancelled; - entry.finished_at = Some(now_unix()); - count += 1; - } - } - if count > 0 { - Self::trim_history(&mut inner); - } - count - } - - /// Cancel a `Queued` entry (no-op for `Running` / terminal — the - /// in-flight rebuild owns the agent's nix store and can't be - /// safely interrupted). Returns true when an entry was cancelled. - pub fn cancel(&self, id: u64) -> bool { - let mut inner = self.inner.lock().expect("rebuild_queue mutex poisoned"); - if let Some(entry) = inner.entries.iter_mut().find(|e| e.id == id) - && entry.state == QueueState::Queued - { - entry.state = QueueState::Cancelled; - entry.finished_at = Some(now_unix()); - Self::trim_history(&mut inner); - return true; - } - false - } - - /// Keep only the most recent `MAX_HISTORY_PER_KIND` terminal entries - /// per kind. Pending + running entries are never evicted. - fn trim_history(inner: &mut Inner) { - let mut counts: std::collections::HashMap = - std::collections::HashMap::new(); - // Walk newest-first; keep the first MAX_HISTORY_PER_KIND - // terminals per kind, evict the rest. - let entries: Vec = inner - .entries - .iter() - .rev() - .filter(|e| { - if !e.state.is_terminal() { - return true; - } - let n = counts.entry(e.kind).or_insert(0); - *n += 1; - *n <= MAX_HISTORY_PER_KIND - }) - .cloned() - .collect(); - inner.entries = entries.into_iter().rev().collect(); - } -} - -/// Per-agent cross-lane guard for [`RebuildQueue::claim`]: returns true when -/// entry `e` is safe to start given the same agent's other-lane work in -/// `entries`. A fast op waits for the agent's `Running` build (and a `Start` -/// also for a `Queued` build); a build op waits for the agent's `Running` -/// fast op. Keeps a stop/start from racing that container's own rebuild. -fn lane_clear(entries: &VecDeque, e: &QueueEntry) -> bool { - let agent = e.agent.as_str(); - if e.kind.is_fast() { - let build_blocking = entries.iter().any(|b| { - !b.kind.is_fast() - && b.agent == agent - && (b.state == QueueState::Running - || (e.kind == QueueKind::Start && b.state == QueueState::Queued)) - }); - !build_blocking - } else { - !entries - .iter() - .any(|f| f.kind.is_fast() && f.agent == agent && f.state == QueueState::Running) - } -} - -/// Background worker that drains the queue. Spawned once at hive-c0re -/// startup from `main.rs`. Loops forever: -/// 1. Pop the next `Queued` entry (`take_next` marks it `Running` and -/// fires a `RebuildQueueChanged` snapshot via the caller). -/// 2. Dispatch by kind — single-agent rebuild, meta-update + cascade, -/// or first-spawn. -/// 3. Mark the entry terminal (`finish`) and emit another snapshot. -/// 4. When the queue is empty, `await` on `notify` until something -/// new lands. -/// -/// Shutdown semantics: subscribes to `coord.shutdown_rx()`. On a true -/// signal the worker exits after its current entry finishes; pending -/// `Queued` entries are dropped (they'll either be replayed by the -/// startup sweep on next boot or left for an operator to re-queue). -/// Max time the `GracefulStop` drain watcher waits for the harness to run its -/// stop-checkpoint turn + drain before falling back to a hard container stop. -/// Generous — a checkpoint turn can take a while — but bounded so a wedged -/// agent never blocks the stop indefinitely. The wait runs in a detached -/// watcher task (not the build worker), so a whole-hive graceful stop overlaps -/// every agent's drain instead of serialising N × this timeout. -const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(3); - -/// Run one claimed queue entry to completion: snapshot, dispatch, mark -/// terminal, snapshot. Shared by both lane workers. -async fn run_one(coord: &std::sync::Arc, entry: &QueueEntry) { - coord.emit_rebuild_queue_snapshot(); - tracing::info!( - id = entry.id, - kind = entry.kind.as_str(), - agent = %entry.agent, - source = entry.source.as_str(), - "rebuild_queue: running" - ); - match dispatch(coord, entry).await { - Ok(()) => { - coord.rebuild_queue.finish(entry.id, QueueState::Done, None); - tracing::info!(id = entry.id, "rebuild_queue: done"); - } - Err(e) => { - let msg = format!("{e:#}"); - let truncated = if msg.len() > 2_000 { - format!("{}…", &msg[..2_000]) - } else { - msg.clone() - }; - coord - .rebuild_queue - .finish(entry.id, QueueState::Failed, Some(truncated)); - tracing::warn!(id = entry.id, error = %msg, "rebuild_queue: failed"); - } - } - coord.emit_rebuild_queue_snapshot(); -} - -/// Build-lane worker: drains every non-fast kind serially. Spawned once at -/// hive-c0re startup from `main.rs`, alongside [`run_fast_worker`] which -/// drains the fast `Start` / `Stop` lane concurrently. -/// -/// Shutdown semantics: subscribes to `coord.shutdown_rx()`. On a true -/// signal the worker exits after its current entry finishes; pending -/// `Queued` entries are dropped (replayed by the startup sweep on next boot -/// or left for an operator to re-queue). -pub async fn run_worker(coord: std::sync::Arc) { - let mut shutdown = coord.shutdown_rx(); - loop { - while let Some(entry) = coord.rebuild_queue.take_next_build() { - run_one(&coord, &entry).await; - // A finished build may unblock a fast op that was deferred behind - // this agent's build — nudge the fast lane to re-check. - coord.rebuild_queue.fast_notify.notify_one(); - } - tokio::select! { - biased; - res = shutdown.changed() => { - if res.is_err() || *shutdown.borrow() { - tracing::info!("rebuild_queue: build worker exiting on shutdown"); - return; - } - } - () = coord.rebuild_queue.notify.notified() => {} - } - } -} - -/// Fast-lane worker: drains hard `Start` / `Stop` serially, concurrently -/// with [`run_worker`], so a stop/start never waits behind another -/// container's slow build. Per-agent ordering vs that agent's own build is -/// enforced in [`RebuildQueue::claim`]. -pub async fn run_fast_worker(coord: std::sync::Arc) { - let mut shutdown = coord.shutdown_rx(); - loop { - while let Some(entry) = coord.rebuild_queue.take_next_fast() { - run_one(&coord, &entry).await; - // A finished fast op may unblock a build deferred behind it. - coord.rebuild_queue.notify.notify_one(); - } - tokio::select! { - biased; - res = shutdown.changed() => { - if res.is_err() || *shutdown.borrow() { - tracing::info!("rebuild_queue: fast worker exiting on shutdown"); - return; - } - } - () = coord.rebuild_queue.fast_notify.notified() => {} - } - } -} - -/// Run a single queue entry to completion. Kind-dispatched; failures -/// bubble up to the worker which marks the entry `Failed`. -/// -/// Approval-driven entries (`approval_id.is_some()`) route through -/// `actions::run_approval_*` which carry the kind-specific commit -/// pipeline + the `ApprovalResolved` event fan-out. Non-approval -/// entries hit the original auto/manual rebuild paths. -/// Pick the right approval pipeline for a `Rebuild` queue entry. Both -/// `ApplyCommit` and `MergeConfigPr` approvals enqueue a `Rebuild` entry -/// (both end in a container rebuild); branch on the approval kind. Falls -/// back to the apply-commit path if the row can't be read — it re-fetches -/// + surfaces a clean error itself. -async fn dispatch_rebuild_approval( - coord: &std::sync::Arc, - entry: &QueueEntry, - approval_id: i64, -) -> anyhow::Result<()> { - let kind = coord - .approvals - .get(approval_id) - .ok() - .flatten() - .map(|a| a.kind); - if kind == Some(hive_sh4re::ApprovalKind::MergeConfigPr) { - crate::actions::run_approval_merge_config_pr(coord, Some(entry.id), approval_id).await - } else { - crate::actions::run_approval_apply_commit(coord, Some(entry.id), approval_id).await - } -} - -async fn dispatch( - coord: &std::sync::Arc, - entry: &QueueEntry, -) -> anyhow::Result<()> { - match (entry.kind, entry.approval_id) { - (QueueKind::Rebuild, Some(approval_id)) => { - dispatch_rebuild_approval(coord, entry, approval_id).await - } - (QueueKind::Rebuild, None) => { - // A meta-update cascade has just set the meta lock; re-locking - // in the per-agent rebuild would revert it (the agent's own - // flake.lock wins). Every other source wants the relock so it - // advances to applied//main. - let relock = entry.source != QueueSource::MetaUpdate; - rebuild_for_entry(coord, entry, relock).await - } - (QueueKind::MetaUpdate, Some(approval_id)) => { - crate::actions::run_approval_update_meta_inputs(coord, Some(entry.id), approval_id) - .await - } - (QueueKind::MetaUpdate, None) => run_meta_update(coord, entry).await, - (QueueKind::Spawn, Some(approval_id)) => { - crate::actions::run_approval_spawn(coord, Some(entry.id), approval_id).await - } - (QueueKind::Spawn, None) => { - // Unreachable today: every Spawn entry is born from an - // approval (HostRequest::RequestSpawn → submit_kind → - // approve → enqueue with approval_id). The manager-side - // `RequestSpawn` surface that used to bypass approvals - // was removed; if a future direct-spawn admin path needs - // to skip the approval ride it should wire its own action - // call rather than route through here. - anyhow::bail!( - "rebuild_queue: Spawn entry id={} agent={} arrived without an approval_id — \ - nothing should enqueue this shape today", - entry.id, - entry.agent, - ) - } - (QueueKind::Destroy, _) => { - // Reserved for future `destroy --purge` integration. - anyhow::bail!("Destroy kind not yet implemented in rebuild_queue worker"); - } - (QueueKind::StartupSweep, _) => { - // Bump meta's hyperhive input before per-agent child rebuilds - // run so they build against the latest base. Non-fatal on - // failure — child rebuilds proceed regardless. After the bump - // (or failure) this entry transitions to Done and the worker - // drains the pre-enqueued child Rebuild entries. - coord.set_queue_step(Some(entry.id), "nix flake update hyperhive"); - if let Err(e) = crate::meta::lock_update_hyperhive().await { - tracing::warn!(error = ?e, "startup_sweep: meta lock_update_hyperhive failed"); - } - // `finish` clears the step label; no explicit clear needed here. - Ok(()) - } - (QueueKind::Restart, _) => { - let name = &entry.agent; - let _guard = coord.transient_guard(name, crate::coordinator::TransientKind::Restarting); - coord.set_queue_step(Some(entry.id), "nixos-container restart"); - crate::lifecycle::restart(name).await?; - coord.kick_agent(name, "container restarted"); - coord.rescan_containers_and_emit().await; - Ok(()) - } - (QueueKind::PermChange, _) => { - let name = &entry.agent; - // Write + commit the perm file under META_LOCK so the - // working tree is never left dirty between the file write - // and the subsequent prepare_deploy git operations. - coord.set_queue_step(Some(entry.id), "writing + committing perm file"); - match &entry.perm_payload { - Some(PermPayload::ToolGroups { groups }) => { - crate::meta::commit_tool_groups(name, groups) - .await - .with_context(|| format!("commit tool-groups for {name}"))?; - // Emit after the commit so the P3RM1SS10NS tab - // reflects the new assignment without the operator - // needing to navigate away and back. - coord.emit_tool_groups_snapshot(); - } - Some(PermPayload::Capabilities { caps }) => { - crate::meta::commit_capabilities(name, caps) - .await - .with_context(|| format!("commit capabilities for {name}"))?; - coord.emit_capabilities_snapshot(); - } - Some(PermPayload::Combined { groups, caps }) => { - // Batch perm change: commit whichever file(s) are - // present in a single git commit, then the rebuild - // below runs once — no double-rebuild for an agent - // whose caps AND groups both changed. - crate::meta::commit_perms(name, groups.as_deref(), caps.as_deref()) - .await - .with_context(|| format!("commit perms for {name}"))?; - if groups.is_some() { - coord.emit_tool_groups_snapshot(); - } - if caps.is_some() { - coord.emit_capabilities_snapshot(); - } - } - None => { - anyhow::bail!( - "PermChange entry id={} agent={} is missing perm_payload", - entry.id, - entry.agent, - ); - } - } - // Now rebuild so the updated HIVE_TOOL_GROUPS / HIVE_CAPABILITIES - // env var takes effect in the container. - rebuild_for_entry(coord, entry, true).await - } - (QueueKind::GracefulStop, _) => { - run_graceful_stop(coord, entry); - Ok(()) - } - (QueueKind::Start, _) => run_start(coord, entry).await, - (QueueKind::Stop, _) => run_stop(coord, entry).await, - } -} - -/// Queue-side container rebuild for `entry.agent`: resolves the current -/// flake rev and hands off to `rebuild_agent` with the entry's id + -/// source. Passing the source defers the start-after-rebuild to a -/// fast-lane `Start` follow-up (grouped under this entry via -/// `parent_id`), so the build lane is freed for the next entry instead -/// of waiting out the container boot. -async fn rebuild_for_entry( - coord: &std::sync::Arc, - entry: &QueueEntry, - relock: bool, -) -> anyhow::Result<()> { - let current_rev = - crate::auto_update::current_flake_rev(&coord.hyperhive_flake).unwrap_or_default(); - crate::auto_update::rebuild_agent( - coord, - &entry.agent, - ¤t_rev, - Some(entry.id), - relock, - Some(entry.source), - ) - .await -} - -/// Start a stopped container off the queue (`QueueKind::Start`), with a -/// `Starting` transient so the dashboard shows a visible queued→running -/// progression rather than the sub-second flash of a direct start. -/// Uses the cold-start fallback (stop + kill + start retry) so the -/// deferred start-after-rebuild keeps the same activation-error recovery -/// it had when it ran inline on the build lane. -/// -/// If the hyperhive flake rev has changed since the container was last built -/// (i.e. the rev marker is stale or missing), the start is upgraded to a full -/// rebuild so the container runs current nix derivations. This is the -/// "deferred stopped container" path from `auto_update::run`. -async fn run_start( - coord: &std::sync::Arc, - entry: &QueueEntry, -) -> anyhow::Result<()> { - let name = &entry.agent; - // Upgrade to rebuild+start if the rev marker is stale. - let current_rev = crate::auto_update::current_flake_rev(&coord.hyperhive_flake); - if let Some(ref rev) = current_rev { - let stored = std::fs::read_to_string(crate::auto_update::rev_marker_path(name)).ok(); - if stored.as_deref() != Some(rev.as_str()) { - tracing::info!(%name, "start: rev stale — upgrading to rebuild+start"); - return crate::auto_update::rebuild_agent( - coord, - name, - rev, - Some(entry.id), - true, - Some(entry.source), - ) - .await; - } - } - let _guard = coord.transient_guard(name, crate::coordinator::TransientKind::Starting); - coord.set_queue_step(Some(entry.id), "nixos-container start"); - crate::lifecycle::start_with_fallback(name).await?; - coord.kick_agent(name, "container started"); - coord.rescan_containers_and_emit().await; - Ok(()) -} - -/// Hard-stop a container off the queue (`QueueKind::Stop`) — same teardown as -/// a direct kill (unregister + `Killed` event), but with a `Stopping` -/// transient for visible queue progress. The quiescing variant is -/// `run_graceful_stop`. -async fn run_stop( - coord: &std::sync::Arc, - entry: &QueueEntry, -) -> anyhow::Result<()> { - let name = &entry.agent; - let _guard = coord.transient_guard(name, crate::coordinator::TransientKind::Stopping); - coord.set_queue_step(Some(entry.id), "nixos-container stop"); - crate::lifecycle::kill(name).await?; - coord.unregister_agent(name); - coord.notify_manager(&hive_sh4re::HelperEvent::Killed { - agent: name.clone(), - }); - coord.rescan_containers_and_emit().await; - Ok(()) -} - -/// Run one `GracefulStop` entry: signal the harness to quiesce (it returns -/// `GracefulStop` on its next `Recv`, runs one stop-checkpoint turn to flush -/// durable `/state`, then exits), then hand the drain-wait + container stop to -/// a detached watcher and return — freeing the build lane immediately. -/// -/// This is the concurrency split: the build worker only does the cheap signal, -/// so a whole-hive graceful stop signals every agent up front and their -/// checkpoint drains overlap. The watcher waits for this agent's drain -/// (bounded by `GRACEFUL_STOP_TIMEOUT` so a wedged agent can't block forever), -/// then enqueues a fast-lane `Stop` for the actual `nixos-container stop`. -/// Routing the real stop through the fast lane means the container stops -/// serialise there (one stop at a time) while the drains ran in parallel. -fn run_graceful_stop(coord: &std::sync::Arc, entry: &QueueEntry) { - let name = entry.agent.clone(); - let parent_id = entry.id; - let source = entry.source; - // Signal the harness; the kick breaks an idle long-poll so it's seen promptly. - coord.set_queue_step(Some(entry.id), "graceful stop: signalling agent"); - coord.mark_graceful_stop(&name); - coord.kick_agent(&name, "graceful stop requested"); - // Detached watcher: wait for the drain (or timeout), then enqueue the - // container stop on the fast lane. The build entry itself is now Done — - // the dashboard groups the follow-up `Stop` under it via `parent_id`. - let coord = std::sync::Arc::clone(coord); - tokio::spawn(async move { - // Hold the `Stopping` transient across the drain so the dashboard keeps - // showing the agent quiescing; dropped before the fast `Stop` is - // enqueued (its `run_stop` re-establishes the transient) so the two - // never clobber each other's clear-on-drop. - let guard = coord.transient_guard(&name, crate::coordinator::TransientKind::Stopping); - // Wait for the harness to drain (it clears the flag via - // `GracefulStopComplete`) or fall back to a hard stop after the timeout. - let deadline = std::time::Instant::now() + GRACEFUL_STOP_TIMEOUT; - while coord.is_graceful_stop_pending(&name) { - if std::time::Instant::now() >= deadline { - tracing::warn!(agent = %name, "graceful stop: drain timed out — hard-stopping"); - break; - } - tokio::time::sleep(std::time::Duration::from_millis(500)).await; - } - coord.clear_graceful_stop(&name); - drop(guard); - // Enqueue the actual container stop on the fast lane (same teardown as a - // plain kill — `run_stop`). `parent_id` links it to the graceful entry - // for dashboard grouping. `enqueue_full` nudges the fast worker itself. - coord.rebuild_queue.enqueue_full(FullEnqueue { - kind: QueueKind::Stop, - agent: name.clone(), - source, - reason: format!("container stop after graceful drain of {name}"), - parent_id: Some(parent_id), - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: Vec::new(), - }); - coord.emit_rebuild_queue_snapshot(); - }); -} - -/// Run one `MetaUpdate` entry: bump the meta flake's locks for the -/// requested inputs, then enqueue a cascade of `Rebuild` entries -/// (with `parent_id` set to this entry's id) for every agent affected -/// by the bump. Mirrors the previous `dashboard::run_meta_update` -/// semantics; that path now enqueues into this queue rather than -/// running the bump + rebuild loop inline. -async fn run_meta_update( - coord: &std::sync::Arc, - entry: &QueueEntry, -) -> anyhow::Result<()> { - let _progress = coord.meta_update_guard(); - let inputs = entry.inputs.clone(); - tracing::info!( - ?inputs, - parent = entry.id, - "rebuild_queue: meta-update starting" - ); - coord.set_queue_step(Some(entry.id), "nix flake update"); - let result = if inputs.is_empty() { - crate::meta::lock_update(&[]).await - } else { - crate::meta::lock_update(&inputs).await - }; - if let Err(e) = result { - // Lock bump failed — cancel any pending cascade rebuilds the - // enqueuer pre-queued for this MetaUpdate. Their parent_id - // matches this entry; the children no longer make sense (we - // never bumped the lock that justified them). - let cancelled = coord.rebuild_queue.cancel_children(entry.id); - if cancelled > 0 { - tracing::warn!( - cancelled, - parent = entry.id, - "rebuild_queue: meta-update failed; cancelled cascade rebuilds" - ); - coord.emit_rebuild_queue_snapshot(); - } - return Err(e); - } - // Lock file changed — meta-inputs panel re-renders. The cascade - // rebuilds were already enqueued at MetaUpdate submission time, - // so no further enqueue is needed here. - crate::dashboard::emit_meta_inputs_snapshot(coord.as_ref()); - Ok(()) -} - -/// Compute which agents a `nix flake update ` on the meta -/// flake would affect. Used by callers that pre-enqueue cascade -/// `Rebuild` entries at `MetaUpdate` submission time so the dashboard -/// can render the dependent work alongside its parent before the lock -/// bump actually runs. -/// -/// Mirrors `run_meta_update`'s post-bump fan-out logic. Empty `inputs` -/// or any input under `hyperhive` → every container; otherwise just -/// the agents named by `agent-` inputs. -pub async fn meta_update_cascade_agents(inputs: &[String]) -> Vec { - let touched_hyperhive = inputs - .iter() - .any(|i| i == "hyperhive" || i.starts_with("hyperhive/")); - let touched_agents: Vec = inputs - .iter() - .filter_map(|i| i.strip_prefix("agent-")) - .map(|rest| rest.split('/').next().unwrap_or(rest).to_owned()) - .collect(); - let mut names = if touched_hyperhive || inputs.is_empty() { - crate::lifecycle::list() - .await - .unwrap_or_default() - .into_iter() - .filter_map(|c| { - c.strip_prefix(crate::lifecycle::AGENT_PREFIX) - .map(str::to_owned) - }) - .collect() - } else { - touched_agents - }; - // Sort parents before children so the sequential queue worker - // always rebuilds a parent before any of its dependents. - let topo = crate::topology::read(); - crate::auto_update::topology_sort(&mut names, &topo); - names -} - -/// Current unix timestamp in seconds. `now()` calls are pulled into a -/// helper so tests can swap them out later. -fn now_unix() -> i64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .ok() - .and_then(|d| i64::try_from(d.as_secs()).ok()) - .unwrap_or(0) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn enqueue_and_take_in_order() { - let q = RebuildQueue::new(); - let a = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "first".to_owned(), - None, - ); - let b = q.enqueue( - QueueKind::Rebuild, - "agent-b".to_owned(), - QueueSource::Manual, - "second".to_owned(), - None, - ); - assert_ne!(a, b); - let next = q.take_next_build().expect("queued"); - assert_eq!(next.id, a); - assert_eq!(next.state, QueueState::Running); - let next = q.take_next_build().expect("queued"); - assert_eq!(next.id, b); - assert!(q.take_next_build().is_none()); - } - - #[test] - fn dedup_pending_same_kind_and_agent() { - let q = RebuildQueue::new(); - let a = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "first".to_owned(), - None, - ); - let b = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::AutoUpdate, - "auto sweep".to_owned(), - None, - ); - assert_eq!(a, b, "dedup should return existing id"); - let snap = q.snapshot(); - assert_eq!(snap.len(), 1); - assert!(snap[0].reason.contains("first")); - assert!(snap[0].reason.contains("auto sweep")); - } - - #[test] - fn meta_update_dedup_matches_inputs() { - // Two MetaUpdate enqueues with identical inputs → dedup. - let q = RebuildQueue::new(); - let a = q.enqueue_with_inputs( - QueueKind::MetaUpdate, - "hyperhive".to_owned(), - QueueSource::Manual, - "first".to_owned(), - None, - vec!["nixpkgs".to_owned()], - ); - let b = q.enqueue_with_inputs( - QueueKind::MetaUpdate, - "hyperhive".to_owned(), - QueueSource::Manual, - "duplicate click".to_owned(), - None, - vec!["nixpkgs".to_owned()], - ); - assert_eq!(a, b, "identical-inputs meta-updates should dedup"); - assert_eq!(q.snapshot().len(), 1); - } - - #[test] - fn meta_update_dedup_separates_distinct_inputs() { - // Two MetaUpdate enqueues with DIFFERENT inputs → distinct - // entries, not deduped. - let q = RebuildQueue::new(); - let a = q.enqueue_with_inputs( - QueueKind::MetaUpdate, - "hyperhive".to_owned(), - QueueSource::Manual, - "bump nixpkgs".to_owned(), - None, - vec!["nixpkgs".to_owned()], - ); - let b = q.enqueue_with_inputs( - QueueKind::MetaUpdate, - "hyperhive".to_owned(), - QueueSource::Manual, - "bump bitburner-agent".to_owned(), - None, - vec!["agent-bitburner/bitburner-agent".to_owned()], - ); - assert_ne!(a, b, "different-inputs meta-updates must NOT dedup"); - let snap = q.snapshot(); - assert_eq!(snap.len(), 2); - // Both inputs lists are preserved. - let inputs: Vec<&[String]> = snap.iter().map(|e| e.inputs.as_slice()).collect(); - assert!(inputs.iter().any(|i| *i == ["nixpkgs"])); - assert!( - inputs - .iter() - .any(|i| *i == ["agent-bitburner/bitburner-agent"]) - ); - } - - #[test] - fn dedup_does_not_apply_across_kinds_or_agents() { - let q = RebuildQueue::new(); - let a = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "r".to_owned(), - None, - ); - let b = q.enqueue( - QueueKind::Rebuild, - "agent-b".to_owned(), - QueueSource::Manual, - "r".to_owned(), - None, - ); - let c = q.enqueue( - QueueKind::Spawn, - "agent-a".to_owned(), - QueueSource::Manual, - "s".to_owned(), - None, - ); - assert_ne!(a, b); - assert_ne!(a, c); - assert_eq!(q.snapshot().len(), 3); - } - - #[test] - fn dedup_skips_running_entries() { - let q = RebuildQueue::new(); - q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "first".to_owned(), - None, - ); - let running = q.take_next_build().expect("queued"); - assert_eq!(running.state, QueueState::Running); - // While the original is running, re-enqueue is legitimate. - let again = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "config bumped during build".to_owned(), - None, - ); - assert_ne!(running.id, again); - let snap = q.snapshot(); - assert_eq!(snap.len(), 2); - } - - #[test] - fn finish_marks_state_and_keeps_history() { - let q = RebuildQueue::new(); - let id = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "r".to_owned(), - None, - ); - q.take_next_build(); - q.finish(id, QueueState::Done, None); - let snap = q.snapshot(); - assert_eq!(snap.len(), 1); - assert_eq!(snap[0].state, QueueState::Done); - assert!(snap[0].finished_at.is_some()); - assert!(snap[0].error.is_none()); - } - - #[test] - fn finish_with_failure_records_error() { - let q = RebuildQueue::new(); - let id = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "r".to_owned(), - None, - ); - q.take_next_build(); - q.finish(id, QueueState::Failed, Some("nix build failed".to_owned())); - let snap = q.snapshot(); - assert_eq!(snap[0].state, QueueState::Failed); - assert_eq!(snap[0].error.as_deref(), Some("nix build failed")); - } - - #[test] - fn history_evicts_old_terminals_per_kind() { - let q = RebuildQueue::new(); - for i in 0..(MAX_HISTORY_PER_KIND + 3) { - let id = q.enqueue( - QueueKind::Rebuild, - format!("agent-{i}"), - QueueSource::Manual, - "r".to_owned(), - None, - ); - q.take_next_build(); - q.finish(id, QueueState::Done, None); - } - let snap = q.snapshot(); - assert_eq!(snap.len(), MAX_HISTORY_PER_KIND); - } - - #[test] - fn cancel_clears_queued_entry() { - let q = RebuildQueue::new(); - let id = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "r".to_owned(), - None, - ); - assert!(q.cancel(id)); - let snap = q.snapshot(); - assert_eq!(snap[0].state, QueueState::Cancelled); - assert!(q.take_next_build().is_none()); - } - - #[test] - fn cancel_refuses_running_entry() { - let q = RebuildQueue::new(); - let id = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "r".to_owned(), - None, - ); - q.take_next_build(); - assert!(!q.cancel(id)); - let snap = q.snapshot(); - assert_eq!(snap[0].state, QueueState::Running); - } - - #[test] - fn parent_id_groups_cascade() { - let q = RebuildQueue::new(); - let meta = q.enqueue( - QueueKind::MetaUpdate, - "hyperhive".to_owned(), - QueueSource::Manual, - "lock bump".to_owned(), - None, - ); - let child = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::MetaUpdate, - "cascade".to_owned(), - Some(meta), - ); - let snap = q.snapshot(); - let child_entry = snap.iter().find(|e| e.id == child).expect("child queued"); - assert_eq!(child_entry.parent_id, Some(meta)); - } - - #[test] - fn cancel_children_marks_queued_descendants() { - let q = RebuildQueue::new(); - let meta = q.enqueue( - QueueKind::MetaUpdate, - "hyperhive".to_owned(), - QueueSource::Manual, - "lock bump".to_owned(), - None, - ); - let a = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::MetaUpdate, - "cascade".to_owned(), - Some(meta), - ); - let b = q.enqueue( - QueueKind::Rebuild, - "agent-b".to_owned(), - QueueSource::MetaUpdate, - "cascade".to_owned(), - Some(meta), - ); - // An unrelated queued entry must not be cancelled. - let c = q.enqueue( - QueueKind::Rebuild, - "agent-c".to_owned(), - QueueSource::Manual, - "operator queued".to_owned(), - None, - ); - let cancelled = q.cancel_children(meta); - assert_eq!(cancelled, 2); - let snap = q.snapshot(); - let find = |id: u64| snap.iter().find(|e| e.id == id).expect("present"); - assert_eq!(find(a).state, QueueState::Cancelled); - assert_eq!(find(b).state, QueueState::Cancelled); - assert_eq!(find(c).state, QueueState::Queued); - } - - #[test] - fn approval_entries_keep_approval_id() { - let q = RebuildQueue::new(); - let id = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "agent-a".to_owned(), - source: QueueSource::Approval, - reason: "approval 42 apply commit".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: Some(42), - perm_payload: None, - depends_on: Vec::new(), - }); - let snap = q.snapshot(); - let entry = snap.iter().find(|e| e.id == id).expect("entry present"); - assert_eq!(entry.approval_id, Some(42)); - assert_eq!(entry.source, QueueSource::Approval); - } - - #[test] - fn approval_entries_dedup_only_on_matching_id() { - // Two pending approval-driven entries for the same agent but - // DIFFERENT approval ids must NOT collapse — each operator - // approve click is a separate piece of work even when the - // (kind, agent) pair matches. - let q = RebuildQueue::new(); - let a = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "agent-a".to_owned(), - source: QueueSource::Approval, - reason: "approval #1".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: Some(1), - perm_payload: None, - depends_on: Vec::new(), - }); - let b = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "agent-a".to_owned(), - source: QueueSource::Approval, - reason: "approval #2".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: Some(2), - perm_payload: None, - depends_on: Vec::new(), - }); - assert_ne!(a, b); - assert_eq!(q.snapshot().len(), 2); - // Same approval_id submitted twice DOES dedup (rapid double- - // click on the dashboard's approve button is a single op). - let c = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "agent-a".to_owned(), - source: QueueSource::Approval, - reason: "approval #1 (duplicate)".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: Some(1), - perm_payload: None, - depends_on: Vec::new(), - }); - assert_eq!(a, c); - assert_eq!(q.snapshot().len(), 2); - } - - #[test] - fn cancel_children_skips_running_and_terminal() { - let q = RebuildQueue::new(); - let meta = q.enqueue( - QueueKind::MetaUpdate, - "hyperhive".to_owned(), - QueueSource::Manual, - "lock bump".to_owned(), - None, - ); - // Running child — must NOT be cancelled. - let running = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::MetaUpdate, - "cascade".to_owned(), - Some(meta), - ); - q.take_next_build(); // pops meta, marks it Running - q.take_next_build(); // pops `running`, marks it Running - // Terminal child — must NOT be re-cancelled (its state stays Done). - let done = q.enqueue( - QueueKind::Rebuild, - "agent-b".to_owned(), - QueueSource::MetaUpdate, - "cascade".to_owned(), - Some(meta), - ); - q.take_next_build(); - q.finish(done, QueueState::Done, None); - // Queued child that should be cancelled. - let queued = q.enqueue( - QueueKind::Rebuild, - "agent-c".to_owned(), - QueueSource::MetaUpdate, - "cascade".to_owned(), - Some(meta), - ); - let n = q.cancel_children(meta); - assert_eq!(n, 1); - let snap = q.snapshot(); - let find = |id: u64| snap.iter().find(|e| e.id == id).expect("present"); - assert_eq!(find(running).state, QueueState::Running); - assert_eq!(find(done).state, QueueState::Done); - assert_eq!(find(queued).state, QueueState::Cancelled); - } - - #[test] - fn set_step_updates_running_entry_and_signals_change() { - let q = RebuildQueue::new(); - let id = q.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - "test".to_owned(), - None, - ); - // Queued — set_step should refuse (returns false). - assert!(!q.set_step(id, "plant tags")); - // Promote to Running. - let entry = q.take_next_build().expect("queued entry"); - assert_eq!(entry.id, id); - // First label transition — true. - assert!(q.set_step(id, "plant tags")); - assert_eq!( - q.snapshot() - .iter() - .find(|e| e.id == id) - .and_then(|e| e.step.as_deref()), - Some("plant tags") - ); - // Same label again — false (caller can skip the snapshot emit). - assert!(!q.set_step(id, "plant tags")); - // Different label — true. - assert!(q.set_step(id, "nixos-container update")); - assert_eq!( - q.snapshot() - .iter() - .find(|e| e.id == id) - .and_then(|e| e.step.as_deref()), - Some("nixos-container update") - ); - } - - #[test] - fn set_step_no_op_on_unknown_id() { - let q = RebuildQueue::new(); - assert!(!q.set_step(999, "anything")); - } - - /// A `MetaUpdate` cascade `Rebuild` (with `parent_id` = `Some(meta_id)`) must - /// NOT dedup into a pre-existing `Queued` `Rebuild` with a different `parent_id` - /// (e.g. from a startup sweep). Without the `parent_id` dedup guard the - /// cascade rebuild would be swallowed and the agent would never rebuild - /// against the post-lock-bump meta. - #[test] - fn meta_update_cascade_does_not_dedup_into_startup_sweep_rebuild() { - let q = RebuildQueue::new(); - // Startup sweep enqueues a Rebuild for alice with its own parent_id. - let sweep = q.enqueue( - QueueKind::StartupSweep, - "hyperhive".to_owned(), - QueueSource::AutoUpdate, - "boot sweep".to_owned(), - None, - ); - let sweep_rebuild = q.enqueue( - QueueKind::Rebuild, - "alice".to_owned(), - QueueSource::StartupSweep, - "startup sweep".to_owned(), - Some(sweep), - ); - // MetaUpdate cascade pre-enqueues another Rebuild for alice. - let meta = q.enqueue( - QueueKind::MetaUpdate, - "hyperhive".to_owned(), - QueueSource::Manual, - "bump nixpkgs".to_owned(), - None, - ); - let cascade_rebuild = q.enqueue( - QueueKind::Rebuild, - "alice".to_owned(), - QueueSource::MetaUpdate, - "meta-update cascade".to_owned(), - Some(meta), - ); - // The two Rebuilds have different parent_ids — must NOT dedup. - assert_ne!( - sweep_rebuild, cascade_rebuild, - "cascade rebuild must be distinct from startup-sweep rebuild" - ); - let snap = q.snapshot(); - let rebuilds: Vec<_> = snap - .iter() - .filter(|e| e.kind == QueueKind::Rebuild && e.agent == "alice") - .collect(); - assert_eq!( - rebuilds.len(), - 2, - "both rebuilds must be present in the queue" - ); - } - - #[test] - fn finish_clears_step() { - let q = RebuildQueue::new(); - let id = q.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - "test".to_owned(), - None, - ); - q.take_next_build(); - assert!(q.set_step(id, "running phase")); - q.finish(id, QueueState::Done, None); - assert_eq!( - q.snapshot() - .iter() - .find(|e| e.id == id) - .and_then(|e| e.step.as_deref()), - None - ); - } - - // --- depends_on tests --- - - /// An entry whose dep is not yet terminal must be skipped by - /// `take_next`; it runs only after the dep finishes. - #[test] - fn depends_on_blocks_until_dep_is_terminal() { - let q = RebuildQueue::new(); - let a = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "first".to_owned(), - None, - ); - let b = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "agent-b".to_owned(), - source: QueueSource::Manual, - reason: "second (blocked on a)".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: vec![a], - }); - // B depends on A — take_next should give A first. - let first = q.take_next_build().expect("a is ready"); - assert_eq!(first.id, a); - // A is Running, not terminal — B must still be blocked. - assert!( - q.take_next_build().is_none(), - "b must be blocked while a runs" - ); - // Finish A → B should now be available. - q.finish(a, QueueState::Done, None); - let second = q.take_next_build().expect("b unblocked after a done"); - assert_eq!(second.id, b); - } - - /// An entry whose dep finished and was evicted from history is - /// treated as resolved (eviction only happens to terminal entries). - #[test] - fn depends_on_evicted_dep_counts_as_resolved() { - let q = RebuildQueue::new(); - // Fill the history cap for Rebuild so old terminals get evicted. - for i in 0..MAX_HISTORY_PER_KIND { - let id = q.enqueue( - QueueKind::Rebuild, - format!("filler-{i}"), - QueueSource::Manual, - "filler".to_owned(), - None, - ); - q.take_next_build(); - q.finish(id, QueueState::Done, None); - } - // `dep` gets enqueued, run, finished, and evicted by the - // next history-trimming call. - let dep = q.enqueue( - QueueKind::Rebuild, - "dep-agent".to_owned(), - QueueSource::Manual, - "dep".to_owned(), - None, - ); - q.take_next_build(); - q.finish(dep, QueueState::Done, None); - // Push `dep` out of the per-kind history window: `trim_history` - // keeps the newest MAX_HISTORY_PER_KIND terminals per kind, so it - // takes that many newer terminals to evict `dep`. - for i in 0..MAX_HISTORY_PER_KIND { - let extra = q.enqueue( - QueueKind::Rebuild, - format!("extra-{i}"), - QueueSource::Manual, - format!("extra-{i}"), - None, - ); - q.take_next_build(); - q.finish(extra, QueueState::Done, None); - } - // `dep` should now be evicted. - assert!( - q.snapshot().iter().all(|e| e.id != dep), - "dep must be evicted from history" - ); - // An entry that depends on the (evicted) dep must be immediately runnable. - let downstream = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "downstream".to_owned(), - source: QueueSource::Manual, - reason: "downstream (dep evicted = resolved)".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: vec![dep], - }); - let got = q - .take_next_build() - .expect("downstream runnable when dep evicted"); - assert_eq!(got.id, downstream); - } - - /// Dedup respects `depends_on`: two otherwise-identical entries with - /// different dep sets are distinct and must NOT collapse. - #[test] - fn depends_on_is_part_of_dedup_key() { - let q = RebuildQueue::new(); - let dep1 = q.enqueue( - QueueKind::Rebuild, - "dep1".to_owned(), - QueueSource::Manual, - "d1".to_owned(), - None, - ); - let dep2 = q.enqueue( - QueueKind::Rebuild, - "dep2".to_owned(), - QueueSource::Manual, - "d2".to_owned(), - None, - ); - let a = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "target".to_owned(), - source: QueueSource::Manual, - reason: "r".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: vec![dep1], - }); - // Same kind+agent but different depends_on — must NOT dedup. - let b = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "target".to_owned(), - source: QueueSource::Manual, - reason: "r".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: vec![dep2], - }); - assert_ne!(a, b, "different depends_on must produce distinct entries"); - // Same depends_on as a — must dedup. - let c = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "target".to_owned(), - source: QueueSource::Manual, - reason: "r again".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: vec![dep1], - }); - assert_eq!(a, c, "identical depends_on must dedup"); - } - - /// An entry with Failed dep is still resolved — the dependent runs - /// regardless of whether its upstream succeeded or not. Callers that - /// need to abort on dep failure should cancel the downstream manually. - #[test] - fn depends_on_failed_dep_counts_as_resolved() { - let q = RebuildQueue::new(); - let a = q.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - "a".to_owned(), - None, - ); - let b = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "b".to_owned(), - source: QueueSource::Manual, - reason: "b (blocked on a)".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: vec![a], - }); - q.take_next_build(); // pop a, mark Running - q.finish(a, QueueState::Failed, Some("nix build exploded".to_owned())); - let got = q.take_next_build().expect("b runnable after a failed"); - assert_eq!(got.id, b); - } - - // ---- fast lane (Start / Stop run on a separate concurrent worker) ---- - - #[test] - fn lanes_claim_only_their_own_kinds() { - let q = RebuildQueue::new(); - let r = q.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - let s = q.enqueue( - QueueKind::Stop, - "b".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - let build = q.take_next_build().expect("build entry"); - assert_eq!(build.id, r); - let fast = q.take_next_fast().expect("fast entry"); - assert_eq!(fast.id, s); - assert!(q.take_next_build().is_none()); - assert!(q.take_next_fast().is_none()); - } - - #[test] - fn start_defers_behind_same_agent_queued_build() { - let q = RebuildQueue::new(); - let b = q.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - q.enqueue( - QueueKind::Start, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - assert!( - q.take_next_fast().is_none(), - "start blocked while same agent has a queued build" - ); - q.take_next_build().expect("build runs"); - q.finish(b, QueueState::Done, None); - let started = q - .take_next_fast() - .expect("start unblocked after build done"); - assert_eq!(started.kind, QueueKind::Start); - } - - #[test] - fn start_for_other_agent_runs_concurrently_with_a_build() { - let q = RebuildQueue::new(); - q.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - q.enqueue( - QueueKind::Start, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - q.enqueue( - QueueKind::Start, - "b".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - q.take_next_build().expect("a's build running"); - let got = q - .take_next_fast() - .expect("start for b runs while a's build runs"); - assert_eq!(got.agent, "b"); - assert!( - q.take_next_fast().is_none(), - "start for a still blocked by a's running build" - ); - } - - #[test] - fn stop_jumps_queued_build_but_waits_running_build() { - // Stop jumps ahead of a *queued* build for the same agent. - let q = RebuildQueue::new(); - q.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - q.enqueue( - QueueKind::Stop, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - let got = q - .take_next_fast() - .expect("stop jumps ahead of a's queued build"); - assert_eq!(got.kind, QueueKind::Stop); - - // But a stop waits for a *running* build of the same agent. - let q2 = RebuildQueue::new(); - let b = q2.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - q2.enqueue( - QueueKind::Stop, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - q2.take_next_build().expect("a's build running"); - assert!( - q2.take_next_fast().is_none(), - "stop waits for a's running build (no kill mid-rebuild)" - ); - q2.finish(b, QueueState::Done, None); - assert!( - q2.take_next_fast().is_some(), - "stop runs once a's build is done" - ); - } - - #[test] - fn build_defers_behind_same_agent_running_fast_op() { - let q = RebuildQueue::new(); - q.enqueue( - QueueKind::Stop, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - q.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - let s = q.take_next_fast().expect("stop running"); - assert!( - q.take_next_build().is_none(), - "build waits while a's fast op is running" - ); - q.finish(s.id, QueueState::Done, None); - assert!( - q.take_next_build().is_some(), - "build runs once the fast op is done" - ); - } -} diff --git a/hive-c0re/src/server.rs b/hive-c0re/src/server.rs index a711bf6b..0572306c 100644 --- a/hive-c0re/src/server.rs +++ b/hive-c0re/src/server.rs @@ -131,7 +131,7 @@ async fn dispatch(req: &HostRequest, coord: Arc) -> HostResponse { agents.retain(|a| prev.contains(a)); } let infra = scoped_infra(scope); - handle_start(&agents, &infra).await? + handle_start(&coord, &agents, &infra).await? } HostRequest::Destroy { name, purge } => { actions::destroy(&coord, name, *purge).await?; @@ -201,6 +201,9 @@ async fn handle_spawn(coord: &Arc, name: &str) -> Result { + if let Err(e) = coord.power.set(name, crate::power::Wanted::Up) { + tracing::warn!(%name, error = ?e, "agent_power: set wanted=up failed"); + } coord.notify_manager(&hive_sh4re::HelperEvent::Spawned { agent: name.to_owned(), ok: true, @@ -224,8 +227,12 @@ async fn handle_spawn(coord: &Arc, name: &str) -> Result, name: &str) -> Result { tracing::info!(%name, "kill"); + if let Err(e) = coord.power.set(name, crate::power::Wanted::Offline) { + tracing::warn!(%name, error = ?e, "agent_power: set wanted=offline failed"); + } lifecycle::kill(name).await?; coord.unregister_agent(name); coord.notify_manager(&hive_sh4re::HelperEvent::Killed { @@ -284,25 +291,29 @@ async fn handle_stop( tracing::info!(?agents, ?infra, graceful, "stop"); let mut ok_items: Vec = Vec::new(); let mut errors: Vec = Vec::new(); - let mut enqueued_graceful = false; for agent in agents { if graceful { - // Graceful stop: enqueue the quiesce orchestration rather than a - // hard kill. Serialised through the rebuild queue so it can't race - // an in-flight rebuild for the same agent, and its per-step - // progress surfaces on the queue snapshot + build log. - coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::GracefulStop, - agent.clone(), - crate::rebuild_queue::QueueSource::Manual, + // Graceful stop: submit the quiesce DAG rather than a hard + // kill. The per-agent lease keeps it from racing an + // in-flight rebuild for the same agent; the cheap Signal + // nodes all fire immediately so every agent's drain + // overlaps. `submit::graceful_stop` also persists + // `wanted = Offline` and emits the queue snapshot. + crate::job_queue::submit::graceful_stop( + coord, + agent, + crate::job_queue::Source::Manual, "manual via hivectl graceful stop".to_owned(), - None, ); ok_items.push(agent.clone()); - enqueued_graceful = true; continue; } + // Persist the intent even if the kill itself fails — otherwise + // the next boot reconcile would restart the agent. + if let Err(e) = coord.power.set(agent, crate::power::Wanted::Offline) { + tracing::warn!(%agent, error = ?e, "agent_power: set wanted=offline failed"); + } match lifecycle::kill(agent).await { Ok(()) => ok_items.push(agent.clone()), Err(e) => { @@ -311,9 +322,6 @@ async fn handle_stop( } } } - if enqueued_graceful { - coord.emit_rebuild_queue_snapshot(); - } for &container in infra { let name = container.unit_name(); @@ -333,7 +341,11 @@ async fn handle_stop( /// inverse of [`handle_stop`]. Infra comes up before agents so the agents /// find forge/matrix/gateway ready. Per-target failures aggregated. Callers /// resolve the [`LifecycleScope`] to these explicit name lists up front. -async fn handle_start(agents: &[String], infra: &[InfraContainer]) -> Result { +async fn handle_start( + coord: &Arc, + agents: &[String], + infra: &[InfraContainer], +) -> Result { tracing::info!(?agents, ?infra, "start"); let mut ok_items: Vec = Vec::new(); let mut errors: Vec = Vec::new(); @@ -350,6 +362,11 @@ async fn handle_start(agents: &[String], infra: &[InfraContainer]) -> Result ok_items.push(agent.clone()), Err(e) => { diff --git a/hive-c0re/src/socket_server.rs b/hive-c0re/src/socket_server.rs index 8e192239..f49b0ea6 100644 --- a/hive-c0re/src/socket_server.rs +++ b/hive-c0re/src/socket_server.rs @@ -510,7 +510,7 @@ async fn dispatch(req: &AgentRequest, agent: &str, coord: &Arc) -> match req { // Lifecycle + config: caller must be an ancestor of the target // (a parent owns its whole subtree; the root covers every agent). - AgentRequest::Start { name } => handle_start(coord, agent, name).await, + AgentRequest::Start { name } => handle_start(coord, agent, name), AgentRequest::Restart { name } => handle_restart(coord, agent, name).await, AgentRequest::Kill { name } => handle_kill(coord, agent, name).await, AgentRequest::Update { name } => handle_update(coord, agent, name), @@ -786,39 +786,21 @@ fn handle_reminder_rollup( /// `Start` — start a container, kicking its next turn. The caller must be an /// ancestor of `name` in the topology (the root covers every agent). -async fn handle_start(coord: &Arc, agent: &str, name: &str) -> AgentResponse { +fn handle_start(coord: &Arc, agent: &str, name: &str) -> AgentResponse { if let Some(err) = require_descendant(agent, name, "start") { return err; } tracing::info!(%agent, %name, "start container"); - // If the hyperhive rev is stale, route through the rebuild queue so the - // container runs current nix derivations before it starts. Same logic as - // `run_start`; this covers the MCP `start` tool path. - let current_rev = crate::auto_update::current_flake_rev(&coord.hyperhive_flake); - if let Some(ref rev) = current_rev { - let stored = std::fs::read_to_string(crate::auto_update::rev_marker_path(name)).ok(); - if stored.as_deref() != Some(rev.as_str()) { - tracing::info!(%agent, %name, "start: rev stale — enqueuing rebuild"); - coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Rebuild, - name.to_owned(), - crate::rebuild_queue::QueueSource::Manual, - format!("start {name}: rev stale — rebuilding first"), - None, - ); - coord.emit_rebuild_queue_snapshot(); - return AgentResponse::Ok; - } - } - match crate::lifecycle::start(name).await { - Ok(()) => { - coord.kick_agent(name, "container started"); - AgentResponse::Ok - } - Err(e) => AgentResponse::Err { - message: format!("{e:#}"), - }, - } + // Persist `wanted = Up` and submit the Start DAG; the submit layer + // upgrades a stale-rev start to a full rebuild so the container + // runs current nix derivations before it starts. + crate::job_queue::submit::start( + coord, + name, + crate::job_queue::Source::Manual, + format!("agent `{agent}` start tool"), + ); + AgentResponse::Ok } /// `Restart` — enqueue a restart for a container. The caller must be an @@ -838,15 +820,13 @@ async fn handle_restart(coord: &Arc, agent: &str, name: &str) -> Ag if let Some(err) = require_descendant(agent, name, "restart") { return err; } - tracing::info!(%agent, %name, "enqueue restart"); - coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Restart, - name.to_owned(), - crate::rebuild_queue::QueueSource::Manual, + tracing::info!(%agent, %name, "submit restart"); + crate::job_queue::submit::restart( + coord, + name, + crate::job_queue::Source::Manual, format!("agent `{agent}` restart tool"), - None, ); - coord.emit_rebuild_queue_snapshot(); AgentResponse::Ok } @@ -908,6 +888,11 @@ async fn handle_kill(coord: &Arc, agent: &str, name: &str) -> Agent return err; } tracing::info!(%agent, %name, "kill container"); + // Persist the intent even if the kill fails — otherwise the next + // reconcile would restart the container. + if let Err(e) = coord.power.set(name, crate::power::Wanted::Offline) { + tracing::warn!(%name, error = ?e, "agent_power: set wanted=offline failed"); + } let result: anyhow::Result<()> = async { crate::lifecycle::kill(name).await?; coord.unregister_agent(name); @@ -933,15 +918,13 @@ fn handle_update(coord: &Arc, agent: &str, name: &str) -> AgentResp if let Some(err) = require_descendant(agent, name, "rebuild") { return err; } - tracing::info!(%agent, %name, "enqueue rebuild"); - coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Rebuild, - name.to_owned(), - crate::rebuild_queue::QueueSource::Manual, + tracing::info!(%agent, %name, "submit rebuild"); + crate::job_queue::submit::rebuild( + coord, + name, + crate::job_queue::Source::Manual, format!("agent `{agent}` update tool"), - None, ); - coord.emit_rebuild_queue_snapshot(); AgentResponse::Ok }