feat(hive-c0re): replace rebuild queue with generic job-DAG queue
jobs are now DAGs of primitive nodes (prebuild, stop-for-update, swap, reconcile, signal, drain, ...) driven by one scheduler with N build slots + per-agent lifecycle leases. per-agent power intent (wanted up/offline) is durable in agent_power.sqlite; Reconcile nodes converge observed state to it. kills the graceful-stop watcher thread, the deferred-start follow-up, and the cascade pre-enqueue (fan-out on MetaLock completion instead). tracker: #2166
This commit is contained in:
parent
79a3993def
commit
7946e03fde
25 changed files with 3673 additions and 2731 deletions
121
hive-c0re/src/job_queue/submit.rs
Normal file
121
hive-c0re/src/job_queue/submit.rs
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
//! Request-level submit API — the surface the dashboard POST handlers,
|
||||
//! the MCP socket handlers, and `hivectl` paths call. Owns the
|
||||
//! submit-time side effects the DAG templates deliberately don't:
|
||||
//! writing the durable `wanted` power intent (synchronously,
|
||||
//! last-writer-wins) before the DAG whose `Reconcile` reads it, and
|
||||
//! upgrading a stale start to a full rebuild. Every helper emits a
|
||||
//! fresh queue snapshot so the dashboard shows the new DAG immediately.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::{Source, Template, templates};
|
||||
use crate::coordinator::Coordinator;
|
||||
use crate::power::Wanted;
|
||||
|
||||
fn submit_and_emit(coord: &Arc<Coordinator>, spec: super::DagSpec) -> u64 {
|
||||
let id = coord
|
||||
.job_queue
|
||||
.submit(spec)
|
||||
.expect("template-built dag specs are acyclic");
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
id
|
||||
}
|
||||
|
||||
fn set_wanted(coord: &Arc<Coordinator>, agent: &str, wanted: Wanted) {
|
||||
if let Err(e) = coord.power.set(agent, wanted) {
|
||||
tracing::warn!(%agent, wanted = wanted.as_str(), error = ?e, "agent_power: set failed");
|
||||
}
|
||||
}
|
||||
|
||||
/// Manual/approval-independent rebuild (always relocks the agent's
|
||||
/// meta input — cascade children are built by the scheduler's fan-out
|
||||
/// instead of this surface).
|
||||
pub fn rebuild(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||
submit_and_emit(coord, templates::rebuild(agent, source, reason, None, true))
|
||||
}
|
||||
|
||||
/// Restart: mechanical stop + converge back to `wanted` (unchanged).
|
||||
pub fn restart(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||
submit_and_emit(coord, templates::restart(agent, source, reason))
|
||||
}
|
||||
|
||||
/// Start: persist `wanted = Up`, then reconcile. A stale rev marker
|
||||
/// upgrades the start to a full rebuild (whose tail `Reconcile` does
|
||||
/// the start) so the container always comes up on current derivations
|
||||
/// — the old fast-lane `run_start` upgrade, moved to submit time.
|
||||
pub fn start(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||
set_wanted(coord, agent, Wanted::Up);
|
||||
let stored = std::fs::read_to_string(crate::auto_update::rev_marker_path(agent)).ok();
|
||||
let stale = crate::auto_update::current_flake_rev(&coord.hyperhive_flake)
|
||||
.is_some_and(|rev| stored.as_deref() != Some(rev.as_str()));
|
||||
if stale {
|
||||
tracing::info!(%agent, "start: rev stale — upgrading to rebuild+start");
|
||||
return submit_and_emit(
|
||||
coord,
|
||||
templates::rebuild(
|
||||
agent,
|
||||
source,
|
||||
format!("{reason} (stale — rebuild+start)"),
|
||||
None,
|
||||
true,
|
||||
),
|
||||
);
|
||||
}
|
||||
submit_and_emit(
|
||||
coord,
|
||||
templates::reconcile_only(
|
||||
Template::Start,
|
||||
agent,
|
||||
source,
|
||||
reason,
|
||||
Some(crate::coordinator::TransientKind::Starting),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
/// Hard stop: persist `wanted = Offline`, then reconcile (kill +
|
||||
/// unregister + `Killed` event).
|
||||
pub fn stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||
set_wanted(coord, agent, Wanted::Offline);
|
||||
submit_and_emit(
|
||||
coord,
|
||||
templates::reconcile_only(
|
||||
Template::Stop,
|
||||
agent,
|
||||
source,
|
||||
reason,
|
||||
Some(crate::coordinator::TransientKind::Stopping),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
/// Graceful stop: persist `wanted = Offline`, then signal → drain →
|
||||
/// reconcile (the actual stop).
|
||||
pub fn graceful_stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||
set_wanted(coord, agent, Wanted::Offline);
|
||||
submit_and_emit(coord, templates::graceful_stop(agent, source, reason))
|
||||
}
|
||||
|
||||
/// Perm change: commit the JSON file(s) then rebuild.
|
||||
pub fn perm_change(
|
||||
coord: &Arc<Coordinator>,
|
||||
agent: &str,
|
||||
source: Source,
|
||||
reason: String,
|
||||
payload: super::PermPayload,
|
||||
) -> u64 {
|
||||
submit_and_emit(
|
||||
coord,
|
||||
templates::perm_change(agent, source, reason, payload),
|
||||
)
|
||||
}
|
||||
|
||||
/// Meta-input lock bump; cascade rebuilds fan out on completion.
|
||||
pub fn meta_update(
|
||||
coord: &Arc<Coordinator>,
|
||||
inputs: Vec<String>,
|
||||
source: Source,
|
||||
reason: String,
|
||||
) -> u64 {
|
||||
submit_and_emit(coord, templates::meta_update(inputs, source, reason, None))
|
||||
}
|
||||
Loading…
Reference in a new issue