feat(hive-c0re): replace rebuild queue with generic job-DAG queue
jobs are now DAGs of primitive nodes (prebuild, stop-for-update, swap, reconcile, signal, drain, ...) driven by one scheduler with N build slots + per-agent lifecycle leases. per-agent power intent (wanted up/offline) is durable in agent_power.sqlite; Reconcile nodes converge observed state to it. kills the graceful-stop watcher thread, the deferred-start follow-up, and the cascade pre-enqueue (fan-out on MetaLock completion instead). tracker: #2166
This commit is contained in:
müde
parent
79a3993def
commit
7946e03fde
25 changed files with 3673 additions and 2731 deletions
|
|
@ -13,19 +13,20 @@ use crate::lifecycle;
|
|||
|
||||
/// Approve a pending request. Marks the approval row durably, then
|
||||
/// either runs the work inline (`InitConfig`, sub-second git ops) or
|
||||
/// enqueues it into `rebuild_queue` so the dashboard POST returns
|
||||
/// submits it to the job queue so the dashboard POST returns
|
||||
/// immediately while the long-running pipeline runs off-thread
|
||||
/// (operator no longer blocks on a 30-90s spinner for `ApplyCommit`).
|
||||
///
|
||||
/// Dispatch:
|
||||
/// - `ApplyCommit` → `QueueKind::Rebuild` (~30-90s wall time)
|
||||
/// - `UpdateMetaInputs` → `QueueKind::MetaUpdate` (~3-15s)
|
||||
/// - `Spawn` → `QueueKind::Spawn` (~30-90s)
|
||||
/// - `ApplyCommit` / `MergeConfigPr` → a single-node `ApprovalDeploy`
|
||||
/// DAG (the two-phase meta deploy stays opaque in v1; ~30-90s)
|
||||
/// - `UpdateMetaInputs` → a `MetaUpdate` DAG (fan-out on completion)
|
||||
/// - `Spawn` → a `Spawn` DAG (`Create → WriteDropin → Reconcile`)
|
||||
/// - `InitConfig` → inline (<1s; queue card would be noise)
|
||||
///
|
||||
/// The queue worker re-fetches the approval row on dispatch, runs
|
||||
/// the kind-specific pipeline, and fires `ApprovalResolved` /
|
||||
/// `Spawned` / `Rebuilt` / `ConfigReady` via `finish_approval`.
|
||||
/// `ApprovalDeploy` resolves the approval inside its pipeline; the
|
||||
/// `MetaUpdate` / `Spawn` DAGs resolve via [`resolve_approval_dag`]
|
||||
/// when their DAG settles terminal.
|
||||
pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
|
||||
let approval = coord.approvals.mark_approved(id)?;
|
||||
tracing::info!(
|
||||
|
|
@ -56,54 +57,41 @@ pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
|
|||
}
|
||||
ApprovalKind::UpdateMetaInputs => {
|
||||
// Inputs JSON-encoded into commit_ref by the manager's
|
||||
// submit path — surface them on the queue entry so the
|
||||
// dashboard can show *which* inputs are about to bump.
|
||||
// submit path — surface them on the DAG so the dashboard
|
||||
// can show *which* inputs are about to bump. The cascade
|
||||
// rebuilds fan out when the lock bump lands (so they build
|
||||
// against the post-bump lock, and a failed bump fans out
|
||||
// nothing).
|
||||
let inputs: Vec<String> =
|
||||
serde_json::from_str(&approval.commit_ref).unwrap_or_default();
|
||||
let parent_id = coord
|
||||
.rebuild_queue
|
||||
.enqueue_full(crate::rebuild_queue::FullEnqueue {
|
||||
kind: crate::rebuild_queue::QueueKind::MetaUpdate,
|
||||
agent: approval.agent.clone(),
|
||||
source: crate::rebuild_queue::QueueSource::Approval,
|
||||
reason: format!("approval #{id} meta input update"),
|
||||
parent_id: None,
|
||||
inputs: inputs.clone(),
|
||||
approval_id: Some(id),
|
||||
perm_payload: None,
|
||||
depends_on: Vec::new(),
|
||||
});
|
||||
// Pre-enqueue cascade rebuilds in topological order so
|
||||
// agents depending on updated inputs are rebuilt after the
|
||||
// lock bump, matching the dashboard post_meta_update path.
|
||||
let cascade_agents = crate::rebuild_queue::meta_update_cascade_agents(&inputs).await;
|
||||
let cascade_reason = format!("approval #{id} meta input cascade");
|
||||
for name in cascade_agents {
|
||||
coord.rebuild_queue.enqueue(
|
||||
crate::rebuild_queue::QueueKind::Rebuild,
|
||||
name,
|
||||
crate::rebuild_queue::QueueSource::MetaUpdate,
|
||||
cascade_reason.clone(),
|
||||
Some(parent_id),
|
||||
);
|
||||
let submitted = coord
|
||||
.job_queue
|
||||
.submit(crate::job_queue::templates::meta_update(
|
||||
inputs,
|
||||
crate::job_queue::Source::Approval,
|
||||
format!("approval #{id} meta input update"),
|
||||
Some(id),
|
||||
));
|
||||
if let Err(e) = submitted {
|
||||
return Err(e.context("submit meta-update dag"));
|
||||
}
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
Ok(())
|
||||
}
|
||||
ApprovalKind::Spawn => {
|
||||
coord
|
||||
.rebuild_queue
|
||||
.enqueue_full(crate::rebuild_queue::FullEnqueue {
|
||||
kind: crate::rebuild_queue::QueueKind::Spawn,
|
||||
agent: approval.agent.clone(),
|
||||
source: crate::rebuild_queue::QueueSource::Approval,
|
||||
reason: format!("approval #{id} spawn"),
|
||||
parent_id: None,
|
||||
inputs: Vec::new(),
|
||||
approval_id: Some(id),
|
||||
perm_payload: None,
|
||||
depends_on: Vec::new(),
|
||||
});
|
||||
// The spawn's tail `Reconcile` starts the container, so the
|
||||
// new agent's power intent is `Up` from the outset.
|
||||
if let Err(e) = coord.power.set(&approval.agent, crate::power::Wanted::Up) {
|
||||
tracing::warn!(agent = %approval.agent, error = ?e, "agent_power: seed on spawn failed");
|
||||
}
|
||||
let submitted = coord.job_queue.submit(crate::job_queue::templates::spawn(
|
||||
&approval.agent,
|
||||
id,
|
||||
format!("approval #{id} spawn"),
|
||||
));
|
||||
if let Err(e) = submitted {
|
||||
return Err(e.context("submit spawn dag"));
|
||||
}
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -137,29 +125,27 @@ pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
|
|||
}
|
||||
}
|
||||
|
||||
/// Enqueue a `Rebuild` queue entry tied to an approval id. Shared by the
|
||||
/// `ApplyCommit` and `MergeConfigPr` dispatch arms — both end in a container
|
||||
/// rebuild routed through the queue, differing only in the queue `reason`.
|
||||
/// The queue worker branches on the approval's kind to pick the right handler.
|
||||
/// Submit the single-node `ApprovalDeploy` DAG tied to an approval id.
|
||||
/// Shared by the `ApplyCommit` and `MergeConfigPr` dispatch arms — both
|
||||
/// end in a container rebuild routed through the queue, differing only
|
||||
/// in the `reason`. The node executor branches on the approval's kind
|
||||
/// to pick the right handler.
|
||||
fn enqueue_approval_rebuild(
|
||||
coord: &Arc<Coordinator>,
|
||||
agent: &str,
|
||||
approval_id: i64,
|
||||
reason: String,
|
||||
) {
|
||||
coord
|
||||
.rebuild_queue
|
||||
.enqueue_full(crate::rebuild_queue::FullEnqueue {
|
||||
kind: crate::rebuild_queue::QueueKind::Rebuild,
|
||||
agent: agent.to_owned(),
|
||||
source: crate::rebuild_queue::QueueSource::Approval,
|
||||
if let Err(e) = coord
|
||||
.job_queue
|
||||
.submit(crate::job_queue::templates::approval_deploy(
|
||||
agent,
|
||||
approval_id,
|
||||
reason,
|
||||
parent_id: None,
|
||||
inputs: Vec::new(),
|
||||
approval_id: Some(approval_id),
|
||||
perm_payload: None,
|
||||
depends_on: Vec::new(),
|
||||
});
|
||||
))
|
||||
{
|
||||
tracing::error!(%agent, approval_id, error = ?e, "submit approval deploy dag failed");
|
||||
}
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
|
||||
|
|
@ -375,69 +361,60 @@ async fn run_approval_schedule_prompt(
|
|||
finish_approval(coord, &approval, result, None, false)
|
||||
}
|
||||
|
||||
/// Worker entry point for `ApprovalKind::UpdateMetaInputs` queue
|
||||
/// entries. Inputs come from the approval row's `commit_ref` field
|
||||
/// (JSON-encoded by the manager submit path), not the queue entry's
|
||||
/// `inputs` — the queue copy is for dashboard display only.
|
||||
pub async fn run_approval_update_meta_inputs(
|
||||
/// Terminal hook for approval-carrying DAGs — the job queue's
|
||||
/// scheduler calls this exactly once when such a DAG settles terminal.
|
||||
/// `MetaUpdate` and `Spawn` approval DAGs resolve here (their work is
|
||||
/// ordinary queue nodes); the opaque `ApprovalDeploy` pipeline resolves
|
||||
/// *inside* its node, so its DAG is skipped — unless it was cancelled
|
||||
/// while still queued, in which case the node never ran and the row
|
||||
/// would otherwise dangle forever.
|
||||
pub(crate) async fn resolve_approval_dag(
|
||||
coord: &Arc<Coordinator>,
|
||||
queue_entry_id: Option<u64>,
|
||||
approval_id: i64,
|
||||
) -> Result<()> {
|
||||
let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::UpdateMetaInputs)?;
|
||||
let inputs: Vec<String> = serde_json::from_str(&approval.commit_ref).unwrap_or_default();
|
||||
coord.set_queue_step(queue_entry_id, "nix flake update");
|
||||
let result = crate::meta::lock_update(&inputs).await;
|
||||
finish_approval(coord, &approval, result, None, false)
|
||||
}
|
||||
|
||||
/// Worker entry point for `ApprovalKind::Spawn` queue entries.
|
||||
/// Differs from `run_approval_apply_commit` only in routing through
|
||||
/// `lifecycle::spawn` (the deprecated direct-spawn path). Synchronous
|
||||
/// in the queue worker — the previous `tokio::spawn` wrapper is gone
|
||||
/// (the queue worker itself is the async task).
|
||||
pub async fn run_approval_spawn(
|
||||
coord: &Arc<Coordinator>,
|
||||
queue_entry_id: Option<u64>,
|
||||
approval_id: i64,
|
||||
) -> Result<()> {
|
||||
let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::Spawn)?;
|
||||
let agent_dir = coord.ensure_runtime(&approval.agent)?;
|
||||
let hive = coord.hive_env();
|
||||
let paths = Coordinator::agent_paths(&approval.agent, agent_dir);
|
||||
// Transient guard keeps the per-container "Spawning" pill lit while
|
||||
// the worker is doing the actual nixos-container create. Auto-clears
|
||||
// on the function's scope exit (success or panic).
|
||||
let _guard = coord.transient_guard(&approval.agent, TransientKind::Spawning);
|
||||
coord.set_queue_step(queue_entry_id, "lifecycle::spawn");
|
||||
let result = lifecycle::spawn(&approval.agent, &hive, &paths).await;
|
||||
if result.is_ok() {
|
||||
coord.set_queue_step(queue_entry_id, "forge user");
|
||||
if let Err(e) = crate::forge::ensure_user_for(&approval.agent).await {
|
||||
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_user after spawn failed");
|
||||
terminal: &crate::job_queue::TerminalDag,
|
||||
) {
|
||||
use crate::job_queue::{State, Template};
|
||||
let Some(approval_id) = terminal.approval_id else {
|
||||
return;
|
||||
};
|
||||
if terminal.template == Template::Rebuild && terminal.state != State::Cancelled {
|
||||
return; // ApprovalDeploy resolved inside the node.
|
||||
}
|
||||
let approval = match coord.approvals.get(approval_id) {
|
||||
Ok(Some(a)) => a,
|
||||
Ok(None) => {
|
||||
tracing::warn!(approval_id, "approval dag terminal: row no longer exists");
|
||||
return;
|
||||
}
|
||||
coord.set_queue_step(queue_entry_id, "forge config repo");
|
||||
if let Err(e) = crate::forge::ensure_config_repo(&approval.agent).await {
|
||||
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_config_repo after spawn failed");
|
||||
Err(e) => {
|
||||
tracing::warn!(approval_id, error = ?e, "approval dag terminal: row read failed");
|
||||
return;
|
||||
}
|
||||
coord.set_queue_step(queue_entry_id, "forge push");
|
||||
if let Err(e) = crate::forge::push_config(&approval.agent).await {
|
||||
tracing::warn!(agent = %approval.agent, error = ?e, "forge: push_config after spawn failed");
|
||||
}
|
||||
coord.set_queue_step(queue_entry_id, "forge meta access");
|
||||
if let Some(core_token) = crate::forge::core_token()
|
||||
&& let Err(e) = crate::forge::meta_read_access(&approval.agent, &core_token).await
|
||||
{
|
||||
tracing::warn!(agent = %approval.agent, error = ?e, "forge: meta_read_access after spawn failed");
|
||||
}
|
||||
if let Err(e) = crate::forge::ensure_meta_remote(&approval.agent).await {
|
||||
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_meta_remote after spawn failed");
|
||||
};
|
||||
let result: Result<()> = match terminal.state {
|
||||
State::Done => Ok(()),
|
||||
State::Cancelled => Err(anyhow::anyhow!("cancelled before completion")),
|
||||
_ => Err(anyhow::anyhow!(
|
||||
"{}",
|
||||
terminal
|
||||
.error
|
||||
.clone()
|
||||
.unwrap_or_else(|| "job dag failed".to_owned())
|
||||
)),
|
||||
};
|
||||
if approval.kind == ApprovalKind::Spawn {
|
||||
// Post-spawn forge bookkeeping (user, config repo mirror, meta
|
||||
// access) — warn-only, then the resolution events + a rescan so
|
||||
// the dashboard reflects the post-spawn state either way.
|
||||
if result.is_ok() {
|
||||
forge_after_first_spawn(coord, &approval.agent).await;
|
||||
} else {
|
||||
coord.rescan_containers_and_emit().await;
|
||||
crate::dashboard::emit_tombstones_snapshot(coord).await;
|
||||
}
|
||||
}
|
||||
let final_result = finish_approval(coord, &approval, result, None, false);
|
||||
coord.rescan_containers_and_emit().await;
|
||||
crate::dashboard::emit_tombstones_snapshot(coord).await;
|
||||
final_result
|
||||
if let Err(e) = finish_approval(coord, &approval, result, None, false) {
|
||||
tracing::warn!(approval_id, error = ?e, "approval dag resolved with failure");
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-fetch an approval row from sqlite for a queue-worker dispatch.
|
||||
|
|
@ -867,13 +844,7 @@ async fn deploy_applied_target(
|
|||
// part of this entry rather than a deferred fast-lane follow-up.
|
||||
false,
|
||||
&|step| coord.set_queue_step(queue_entry_id, step),
|
||||
&|log_id| {
|
||||
if let Some(qid) = queue_entry_id
|
||||
&& coord.rebuild_queue.set_build_log_id(qid, log_id)
|
||||
{
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
},
|
||||
&|log_id| coord.set_queue_build_log(queue_entry_id, log_id),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -984,6 +955,11 @@ pub async fn destroy(coord: &Arc<Coordinator>, name: &str, purge: bool) -> Resul
|
|||
"agent destroyed"
|
||||
},
|
||||
);
|
||||
// Drop the durable power intent — a future agent of the same name
|
||||
// seeds fresh from its observed state.
|
||||
if let Err(e) = coord.power.remove(name) {
|
||||
tracing::warn!(%name, error = ?e, "agent_power: remove on destroy failed");
|
||||
}
|
||||
drop(guard);
|
||||
coord.notify_manager(&HelperEvent::Destroyed {
|
||||
agent: name.to_owned(),
|
||||
|
|
|
|||
Loading…
Reference in a new issue