vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(hive-c0re): replace rebuild queue with generic job-DAG queue

Browse source 
jobs are now DAGs of primitive nodes (prebuild, stop-for-update, swap,
reconcile, signal, drain, ...) driven by one scheduler with N build
slots + per-agent lifecycle leases. per-agent power intent (wanted
up/offline) is durable in agent_power.sqlite; Reconcile nodes converge
observed state to it. kills the graceful-stop watcher thread, the
deferred-start follow-up, and the cascade pre-enqueue (fan-out on
MetaLock completion instead). tracker: #2166
This commit is contained in:
müde 2026-07-06 20:13:14 +02:00
commit 7946e03fde
25 changed files with 3673 additions and 2731 deletions
Download patch file Download diff file Expand all files Collapse all files

18
Cargo.lock generated
View file

@ -1029,6 +1029,12 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "fixedbitset"
version = "0.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
[[package]]
name = "flate2"
version = "1.1.9"
@ -1381,6 +1387,7 @@ dependencies = [
"hive-sh4re",
"libc",
"listenfd",
"petgraph",
"problem_details",
"reqwest",
"rusqlite",
@ -2554,6 +2561,17 @@ version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "petgraph"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455"
dependencies = [
"fixedbitset",
"hashbrown 0.15.5",
"indexmap",
]
[[package]]
name = "phf"
version = "0.11.3"

1
Cargo.toml
View file

@ -68,6 +68,7 @@ reqwest = { version = "0.12", default-features = false, features = [
"json",
"rustls-tls",
] }
petgraph = { version = "0.8", default-features = false, features = ["std"] }
matrix-sdk = { version = "0.14", default-features = false, features = [
"rustls-tls",
"sqlite",

1
hive-c0re/Cargo.toml
View file

@ -18,6 +18,7 @@ clap-markdown = "0.1"
hive-sh4re.workspace = true
libc.workspace = true
listenfd = "1"
petgraph.workspace = true
rusqlite.workspace = true
serde.workspace = true
serde_json.workspace = true

234
hive-c0re/src/actions.rs
View file

@ -13,19 +13,20 @@ use crate::lifecycle;
/// Approve a pending request. Marks the approval row durably, then
/// either runs the work inline (`InitConfig`, sub-second git ops) or
/// enqueues it into `rebuild_queue` so the dashboard POST returns
/// submits it to the job queue so the dashboard POST returns
/// immediately while the long-running pipeline runs off-thread
/// (operator no longer blocks on a 30-90s spinner for `ApplyCommit`).
///
/// Dispatch:
/// - `ApplyCommit` → `QueueKind::Rebuild` (~30-90s wall time)
/// - `UpdateMetaInputs` → `QueueKind::MetaUpdate` (~3-15s)
/// - `Spawn` → `QueueKind::Spawn` (~30-90s)
/// - `ApplyCommit` / `MergeConfigPr` → a single-node `ApprovalDeploy`
/// DAG (the two-phase meta deploy stays opaque in v1; ~30-90s)
/// - `UpdateMetaInputs` → a `MetaUpdate` DAG (fan-out on completion)
/// - `Spawn` → a `Spawn` DAG (`Create → WriteDropin → Reconcile`)
/// - `InitConfig` → inline (<1s; queue card would be noise)
///
/// The queue worker re-fetches the approval row on dispatch, runs
/// the kind-specific pipeline, and fires `ApprovalResolved` /
/// `Spawned` / `Rebuilt` / `ConfigReady` via `finish_approval`.
/// `ApprovalDeploy` resolves the approval inside its pipeline; the
/// `MetaUpdate` / `Spawn` DAGs resolve via [`resolve_approval_dag`]
/// when their DAG settles terminal.
pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
let approval = coord.approvals.mark_approved(id)?;
tracing::info!(
@ -56,54 +57,41 @@ pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
}
ApprovalKind::UpdateMetaInputs => {
// Inputs JSON-encoded into commit_ref by the manager's
// submit path — surface them on the queue entry so the
// dashboard can show *which* inputs are about to bump.
// submit path — surface them on the DAG so the dashboard
// can show *which* inputs are about to bump. The cascade
// rebuilds fan out when the lock bump lands (so they build
// against the post-bump lock, and a failed bump fans out
// nothing).
let inputs: Vec<String> =
serde_json::from_str(&approval.commit_ref).unwrap_or_default();
let parent_id = coord
.rebuild_queue
.enqueue_full(crate::rebuild_queue::FullEnqueue {
kind: crate::rebuild_queue::QueueKind::MetaUpdate,
agent: approval.agent.clone(),
source: crate::rebuild_queue::QueueSource::Approval,
reason: format!("approval #{id} meta input update"),
parent_id: None,
inputs: inputs.clone(),
approval_id: Some(id),
perm_payload: None,
depends_on: Vec::new(),
});
// Pre-enqueue cascade rebuilds in topological order so
// agents depending on updated inputs are rebuilt after the
// lock bump, matching the dashboard post_meta_update path.
let cascade_agents = crate::rebuild_queue::meta_update_cascade_agents(&inputs).await;
let cascade_reason = format!("approval #{id} meta input cascade");
for name in cascade_agents {
coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Rebuild,
name,
crate::rebuild_queue::QueueSource::MetaUpdate,
cascade_reason.clone(),
Some(parent_id),
);
let submitted = coord
.job_queue
.submit(crate::job_queue::templates::meta_update(
inputs,
crate::job_queue::Source::Approval,
format!("approval #{id} meta input update"),
Some(id),
));
if let Err(e) = submitted {
return Err(e.context("submit meta-update dag"));
}
coord.emit_rebuild_queue_snapshot();
Ok(())
}
ApprovalKind::Spawn => {
coord
.rebuild_queue
.enqueue_full(crate::rebuild_queue::FullEnqueue {
kind: crate::rebuild_queue::QueueKind::Spawn,
agent: approval.agent.clone(),
source: crate::rebuild_queue::QueueSource::Approval,
reason: format!("approval #{id} spawn"),
parent_id: None,
inputs: Vec::new(),
approval_id: Some(id),
perm_payload: None,
depends_on: Vec::new(),
});
// The spawn's tail `Reconcile` starts the container, so the
// new agent's power intent is `Up` from the outset.
if let Err(e) = coord.power.set(&approval.agent, crate::power::Wanted::Up) {
tracing::warn!(agent = %approval.agent, error = ?e, "agent_power: seed on spawn failed");
}
let submitted = coord.job_queue.submit(crate::job_queue::templates::spawn(
&approval.agent,
id,
format!("approval #{id} spawn"),
));
if let Err(e) = submitted {
return Err(e.context("submit spawn dag"));
}
coord.emit_rebuild_queue_snapshot();
Ok(())
}
@ -137,29 +125,27 @@ pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
}
}
/// Enqueue a `Rebuild` queue entry tied to an approval id. Shared by the
/// `ApplyCommit` and `MergeConfigPr` dispatch arms — both end in a container
/// rebuild routed through the queue, differing only in the queue `reason`.
/// The queue worker branches on the approval's kind to pick the right handler.
/// Submit the single-node `ApprovalDeploy` DAG tied to an approval id.
/// Shared by the `ApplyCommit` and `MergeConfigPr` dispatch arms — both
/// end in a container rebuild routed through the queue, differing only
/// in the `reason`. The node executor branches on the approval's kind
/// to pick the right handler.
fn enqueue_approval_rebuild(
coord: &Arc<Coordinator>,
agent: &str,
approval_id: i64,
reason: String,
) {
coord
.rebuild_queue
.enqueue_full(crate::rebuild_queue::FullEnqueue {
kind: crate::rebuild_queue::QueueKind::Rebuild,
agent: agent.to_owned(),
source: crate::rebuild_queue::QueueSource::Approval,
if let Err(e) = coord
.job_queue
.submit(crate::job_queue::templates::approval_deploy(
agent,
approval_id,
reason,
parent_id: None,
inputs: Vec::new(),
approval_id: Some(approval_id),
perm_payload: None,
depends_on: Vec::new(),
});
))
{
tracing::error!(%agent, approval_id, error = ?e, "submit approval deploy dag failed");
}
coord.emit_rebuild_queue_snapshot();
}
@ -375,69 +361,60 @@ async fn run_approval_schedule_prompt(
finish_approval(coord, &approval, result, None, false)
}
/// Worker entry point for `ApprovalKind::UpdateMetaInputs` queue
/// entries. Inputs come from the approval row's `commit_ref` field
/// (JSON-encoded by the manager submit path), not the queue entry's
/// `inputs` — the queue copy is for dashboard display only.
pub async fn run_approval_update_meta_inputs(
/// Terminal hook for approval-carrying DAGs — the job queue's
/// scheduler calls this exactly once when such a DAG settles terminal.
/// `MetaUpdate` and `Spawn` approval DAGs resolve here (their work is
/// ordinary queue nodes); the opaque `ApprovalDeploy` pipeline resolves
/// *inside* its node, so its DAG is skipped — unless it was cancelled
/// while still queued, in which case the node never ran and the row
/// would otherwise dangle forever.
pub(crate) async fn resolve_approval_dag(
coord: &Arc<Coordinator>,
queue_entry_id: Option<u64>,
approval_id: i64,
) -> Result<()> {
let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::UpdateMetaInputs)?;
let inputs: Vec<String> = serde_json::from_str(&approval.commit_ref).unwrap_or_default();
coord.set_queue_step(queue_entry_id, "nix flake update");
let result = crate::meta::lock_update(&inputs).await;
finish_approval(coord, &approval, result, None, false)
}
/// Worker entry point for `ApprovalKind::Spawn` queue entries.
/// Differs from `run_approval_apply_commit` only in routing through
/// `lifecycle::spawn` (the deprecated direct-spawn path). Synchronous
/// in the queue worker — the previous `tokio::spawn` wrapper is gone
/// (the queue worker itself is the async task).
pub async fn run_approval_spawn(
coord: &Arc<Coordinator>,
queue_entry_id: Option<u64>,
approval_id: i64,
) -> Result<()> {
let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::Spawn)?;
let agent_dir = coord.ensure_runtime(&approval.agent)?;
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(&approval.agent, agent_dir);
// Transient guard keeps the per-container "Spawning" pill lit while
// the worker is doing the actual nixos-container create. Auto-clears
// on the function's scope exit (success or panic).
let _guard = coord.transient_guard(&approval.agent, TransientKind::Spawning);
coord.set_queue_step(queue_entry_id, "lifecycle::spawn");
let result = lifecycle::spawn(&approval.agent, &hive, &paths).await;
if result.is_ok() {
coord.set_queue_step(queue_entry_id, "forge user");
if let Err(e) = crate::forge::ensure_user_for(&approval.agent).await {
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_user after spawn failed");
terminal: &crate::job_queue::TerminalDag,
) {
use crate::job_queue::{State, Template};
let Some(approval_id) = terminal.approval_id else {
return;
};
if terminal.template == Template::Rebuild && terminal.state != State::Cancelled {
return; // ApprovalDeploy resolved inside the node.
}
let approval = match coord.approvals.get(approval_id) {
Ok(Some(a)) => a,
Ok(None) => {
tracing::warn!(approval_id, "approval dag terminal: row no longer exists");
return;
}
coord.set_queue_step(queue_entry_id, "forge config repo");
if let Err(e) = crate::forge::ensure_config_repo(&approval.agent).await {
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_config_repo after spawn failed");
Err(e) => {
tracing::warn!(approval_id, error = ?e, "approval dag terminal: row read failed");
return;
}
coord.set_queue_step(queue_entry_id, "forge push");
if let Err(e) = crate::forge::push_config(&approval.agent).await {
tracing::warn!(agent = %approval.agent, error = ?e, "forge: push_config after spawn failed");
}
coord.set_queue_step(queue_entry_id, "forge meta access");
if let Some(core_token) = crate::forge::core_token()
&& let Err(e) = crate::forge::meta_read_access(&approval.agent, &core_token).await
{
tracing::warn!(agent = %approval.agent, error = ?e, "forge: meta_read_access after spawn failed");
}
if let Err(e) = crate::forge::ensure_meta_remote(&approval.agent).await {
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_meta_remote after spawn failed");
};
let result: Result<()> = match terminal.state {
State::Done => Ok(()),
State::Cancelled => Err(anyhow::anyhow!("cancelled before completion")),
_ => Err(anyhow::anyhow!(
"{}",
terminal
.error
.clone()
.unwrap_or_else(|| "job dag failed".to_owned())
)),
};
if approval.kind == ApprovalKind::Spawn {
// Post-spawn forge bookkeeping (user, config repo mirror, meta
// access) — warn-only, then the resolution events + a rescan so
// the dashboard reflects the post-spawn state either way.
if result.is_ok() {
forge_after_first_spawn(coord, &approval.agent).await;
} else {
coord.rescan_containers_and_emit().await;
crate::dashboard::emit_tombstones_snapshot(coord).await;
}
}
let final_result = finish_approval(coord, &approval, result, None, false);
coord.rescan_containers_and_emit().await;
crate::dashboard::emit_tombstones_snapshot(coord).await;
final_result
if let Err(e) = finish_approval(coord, &approval, result, None, false) {
tracing::warn!(approval_id, error = ?e, "approval dag resolved with failure");
}
}
/// Re-fetch an approval row from sqlite for a queue-worker dispatch.
@ -867,13 +844,7 @@ async fn deploy_applied_target(
// part of this entry rather than a deferred fast-lane follow-up.
false,
&|step| coord.set_queue_step(queue_entry_id, step),
&|log_id| {
if let Some(qid) = queue_entry_id
&& coord.rebuild_queue.set_build_log_id(qid, log_id)
{
coord.emit_rebuild_queue_snapshot();
}
},
&|log_id| coord.set_queue_build_log(queue_entry_id, log_id),
)
.await;
@ -984,6 +955,11 @@ pub async fn destroy(coord: &Arc<Coordinator>, name: &str, purge: bool) -> Resul
"agent destroyed"
},
);
// Drop the durable power intent — a future agent of the same name
// seeds fresh from its observed state.
if let Err(e) = coord.power.remove(name) {
tracing::warn!(%name, error = ?e, "agent_power: remove on destroy failed");
}
drop(guard);
coord.notify_manager(&HelperEvent::Destroyed {
agent: name.to_owned(),

310
hive-c0re/src/auto_update.rs
View file

@ -1,18 +1,25 @@
//! Startup auto-update: on `hive-c0re serve` boot, rebuild containers that
//! actually need it. Two skip rules keep boot-time work minimal:
//! Boot reconcile: on `hive-c0re serve` boot, (a) run the config path
//! for agents whose per-agent rev marker is stale — a `StartupSweep`
//! DAG (meta hyperhive lock bump) fanning out `Rebuild` children for
//! the stale agents whose `wanted` power intent is `Up` — and (b)
//! converge every other drifted agent to its persisted `wanted` via
//! `Reconcile` DAGs. Two rules keep boot-time nix work minimal:
//!
//! 1. **Stopped containers** are deferred — they will be rebuilt the first
//! time the operator starts them (see `rebuild_queue::run_start` and
//! `socket_server::handle_start`).
//! 2. **Running containers whose rev marker matches** the current hyperhive
//! flake path are skipped — nothing changed, no nix work to do.
//! 1. **Stale but wanted-offline agents** get no rebuild — their
//! rebuild happens the first time they're started (the start
//! submit path upgrades a stale start to rebuild+start). The sweep
//! parent still runs whenever *any* marker is stale so the meta
//! hyperhive lock is bumped for those later start-upgrades.
//! 2. **Agents whose rev marker matches** the current hyperhive flake
//! path are skipped — nothing changed, no nix work to do.
//!
//! See `docs/coordinator.md::Auto-update sweep`.
//! Booting with no config change performs no meta commit — only
//! reconciles. See `docs/coordinator.md::Boot reconcile`.
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::{Context, Result};
use anyhow::Result;
use crate::coordinator::Coordinator;
use crate::lifecycle::{self, AGENT_PREFIX, MANAGER_NAME};
@ -71,143 +78,6 @@ pub async fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> boo
}
}
/// Rebuild one sub-agent and refresh its marker. Used by both the startup
/// scanner and the dashboard's manual "update" button so the two paths
/// can't diverge.
///
/// `queue_entry_id` is `Some(id)` when the rebuild was dispatched from
/// the `rebuild_queue` worker (lets the function annotate its phase via
/// `coord.set_queue_step`) and `None` when called directly (e.g. the
/// root-agent migration nudge in `ensure_root_agent`).
///
/// `relock` bumps the agent's meta input to `applied/<n>/main` before
/// the container rebuild. Pass `false` only for meta-update cascade
/// rebuilds, where re-locking would revert the bump the cascade just
/// committed (see `lifecycle::rebuild`).
///
/// `defer_start_source` is `Some(source)` for queue-dispatched rebuilds:
/// instead of holding the serialized build lane through the container
/// boot, the start-after-rebuild is enqueued as a fast-lane `Start`
/// entry (grouped under this rebuild via `parent_id`, same split as the
/// graceful-stop follow-up). Pass `None` for direct callers to keep the
/// start inline.
///
/// # Errors
///
/// Propagates errors from `coord.ensure_runtime` and `lifecycle::rebuild`.
pub async fn rebuild_agent(
coord: &Arc<Coordinator>,
name: &str,
current_rev: &str,
queue_entry_id: Option<u64>,
relock: bool,
defer_start_source: Option<crate::rebuild_queue::QueueSource>,
) -> Result<()> {
tracing::info!(%name, rev = %current_rev, "rebuild agent");
let agent_dir = coord
.ensure_runtime(name)
.with_context(|| format!("ensure_runtime {name}"))?;
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(name, agent_dir);
// Suppress crash_watch during the stop+start window inside
// lifecycle::rebuild. Dashboard rebuilds already do this via
// lifecycle_action; this catches the auto-update scan + any
// other direct caller.
let guard = coord.transient_guard(name, crate::coordinator::TransientKind::Rebuilding);
let result = lifecycle::rebuild(
name,
&hive,
&paths,
relock,
defer_start_source.is_some(),
&|step| coord.set_queue_step(queue_entry_id, step),
&|log_id| {
if let Some(qid) = queue_entry_id
&& coord.rebuild_queue.set_build_log_id(qid, log_id)
{
coord.emit_rebuild_queue_snapshot();
}
},
)
.await;
drop(guard);
match &result {
Ok(needs_start) => {
if let Err(e) = std::fs::write(rev_marker_path(name), current_rev) {
tracing::warn!(%name, error = ?e, "write rev marker failed");
}
// Deferred start: hand the container boot to the fast lane so
// this build-lane entry completes now and the next queued
// rebuild's nix build overlaps with the boot. `parent_id`
// groups the follow-up under this rebuild on the dashboard —
// same split the graceful-stop path uses for its container
// stop. A start failure surfaces on the Start entry (with
// the cold-start fallback) instead of failing the rebuild.
if *needs_start && let Some(source) = defer_start_source {
coord
.rebuild_queue
.enqueue_full(crate::rebuild_queue::FullEnqueue {
kind: crate::rebuild_queue::QueueKind::Start,
agent: name.to_owned(),
source,
reason: format!("start after rebuild of {name}"),
parent_id: queue_entry_id,
inputs: Vec::new(),
approval_id: None,
perm_payload: None,
depends_on: Vec::new(),
});
coord.emit_rebuild_queue_snapshot();
}
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
agent: name.to_owned(),
ok: true,
note: None,
sha: None,
tag: None,
});
coord.set_queue_step(queue_entry_id, "forge sync");
// Run the full forge sync on every successful rebuild so
// the rebuild path is equivalent to the hive-c0re startup
// sweep: token, config-repo mirror, meta read access, and
// meta remote are all kept in sync. Recovers missing tokens
// (e.g. first-spawn seeding failed transiently) without
// requiring a full hive-c0re restart.
crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await;
// Mirror the matrix side of the startup sweep: if hive-matrix
// is present, ensure this agent has a registered user + token.
// Idempotent (skips if token file already exists). Keeps the
// rebuild path equivalent to the startup sweep for newly-spawned
// agents that missed ensure_all().
crate::matrix::sync_agent_standalone(name).await;
// Wake the agent on its next turn so claude sees a
// "you were rebuilt — check /state/ for notes, --continue
// session intact" hint. Covers dashboard rebuild, admin
// CLI rebuild, auto-update startup scan, and the
// dashboard's meta-input update path — all of which
// route through rebuild_agent.
coord.kick_agent(name, "container rebuilt");
// Container state (needs_update, deployed_sha) may have
// shifted — rescan so dashboards drop the "needs update"
// chip without waiting for the next /api/state poll.
coord.rescan_containers_and_emit().await;
// Lock bump → meta-inputs panel needs to re-render.
crate::dashboard::emit_meta_inputs_snapshot(coord);
}
Err(e) => {
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
agent: name.to_owned(),
ok: false,
note: Some(format!("{e:#}")),
sha: None,
tag: None,
});
coord.rescan_containers_and_emit().await;
}
}
result.map(|_| ())
}
/// Whether this hive is "ruthless" — running with no root/manager agent at
/// all (no ruth). When true, hive-c0re skips the root-agent create/start
/// sweep entirely. Controlled by the host option
@ -246,17 +116,18 @@ pub async fn ensure_root_agent(coord: &Arc<Coordinator>) -> Result<()> {
// running whatever the host-declarative config was at create
// time, with a wrong systemd unit and port.
let applied_flake = Coordinator::agent_applied_dir(MANAGER_NAME).join("flake.nix");
if !applied_flake.exists()
&& let Some(rev) = current_rev.as_ref()
{
if !applied_flake.exists() && current_rev.is_some() {
tracing::warn!(
"manager container exists but no applied flake — forcing rebuild to migrate"
);
let coord_clone = coord.clone();
if let Err(e) =
rebuild_agent(&coord_clone, MANAGER_NAME, rev.as_str(), None, true, None).await
{
tracing::warn!(error = ?e, "manager migration rebuild failed");
if let Err(e) = coord.job_queue.submit(crate::job_queue::templates::rebuild(
MANAGER_NAME,
crate::job_queue::Source::AutoUpdate,
"manager migration: no applied flake".to_owned(),
None,
true,
)) {
tracing::warn!(error = ?e, "manager migration rebuild submit failed");
}
} else {
tracing::debug!("manager container already present");
@ -272,6 +143,9 @@ pub async fn ensure_root_agent(coord: &Arc<Coordinator>) -> Result<()> {
// this function.)
if !lifecycle::is_running(MANAGER_NAME).await {
tracing::info!("manager container present but not running — starting");
if let Err(e) = coord.power.set(MANAGER_NAME, crate::power::Wanted::Up) {
tracing::warn!(error = ?e, "agent_power: set manager wanted=up failed");
}
if let Err(e) = lifecycle::start(MANAGER_NAME).await {
tracing::warn!(error = ?e, "manager start failed");
}
@ -283,6 +157,9 @@ pub async fn ensure_root_agent(coord: &Arc<Coordinator>) -> Result<()> {
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(MANAGER_NAME, runtime);
lifecycle::spawn(MANAGER_NAME, &hive, &paths).await?;
if let Err(e) = coord.power.set(MANAGER_NAME, crate::power::Wanted::Up) {
tracing::warn!(error = ?e, "agent_power: set manager wanted=up failed");
}
if let Some(rev) = current_rev {
let _ = std::fs::write(rev_marker_path(MANAGER_NAME), &rev);
}
@ -327,18 +204,17 @@ pub fn topology_sort(
});
}
/// Rebuild containers that need it on startup. Skips:
/// - **Stopped containers**: deferred to on-start (`run_start` / `handle_start`
/// upgrades a plain start to rebuild+start when the rev marker is stale).
/// - **Running containers with a matching rev marker**: no nix work needed.
///
/// Enqueues a `StartupSweep` parent entry followed by per-agent `Rebuild`
/// children linked via `parent_id`. Returns Ok even if some rebuilds failed.
/// Boot reconcile (see the module doc): classify every agent by rev
/// freshness + persisted `wanted` intent, submit one `StartupSweep`
/// DAG (hyperhive lock bump → fan-out rebuilds for stale wanted-up
/// agents) when anything is stale, and `Reconcile` DAGs for agents
/// whose observed power state drifted from `wanted`. Returns Ok even
/// if some submissions failed.
pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
let containers = match lifecycle::list().await {
Ok(c) => c,
Err(e) => {
tracing::warn!(error = ?e, "auto-update: nixos-container list failed");
tracing::warn!(error = ?e, "boot reconcile: nixos-container list failed");
return Ok(());
}
};
@ -356,62 +232,88 @@ pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
let topo = crate::topology::read();
topology_sort(&mut logical_names, &topo);
// Pre-classify: decide which agents need a rebuild now vs can be skipped.
let mut to_rebuild: Vec<String> = Vec::new();
// Classify. `get_or_seed` doubles as the one-time migration: an
// agent without an `agent_power` row is seeded from its observed
// state (running ⇒ Up), after which the DB is authoritative.
let mut any_stale = false;
let mut fanout: Vec<String> = Vec::new(); // stale ∧ wanted=Up → sweep rebuild
let mut drifted: Vec<String> = Vec::new(); // fresh ∧ wanted≠observed → reconcile
let mut n_deferred = 0usize;
let mut n_skipped = 0usize;
for name in &logical_names {
// Idea 2: stopped containers are deferred — rebuild happens the first
// time the operator starts them.
if !lifecycle::is_running(name).await {
n_deferred += 1;
tracing::debug!(%name, "startup sweep: stopped — deferring rebuild to on-start");
continue;
}
// Idea 1: running containers with a matching rev marker need no rebuild.
if let Some(ref rev) = current_rev {
let stored = std::fs::read_to_string(rev_marker_path(name)).ok();
if stored.as_deref() == Some(rev.as_str()) {
n_skipped += 1;
tracing::debug!(%name, "startup sweep: rev unchanged — skipping rebuild");
let running = lifecycle::is_running(name).await;
let wanted = match coord.power.get_or_seed(name, running) {
Ok(w) => w,
Err(e) => {
tracing::warn!(%name, error = ?e, "agent_power read failed — assuming observed");
crate::power::Wanted::from_running(running)
}
};
let fresh = current_rev.as_ref().is_some_and(|rev| {
std::fs::read_to_string(rev_marker_path(name))
.is_ok_and(|stored| stored == rev.as_str())
});
if fresh {
n_skipped += 1;
} else {
any_stale = true;
if wanted == crate::power::Wanted::Up {
// Rebuild against the post-bump lock; the DAG's tail
// Reconcile brings the agent (back) up — covering both
// the running-stale and stopped-but-wanted-up cases.
fanout.push(name.clone());
continue;
}
// Stale but wanted offline: no boot-time nix work — the
// start submit path upgrades a stale start to a rebuild.
n_deferred += 1;
tracing::debug!(%name, "boot reconcile: stale but offline — deferring rebuild to on-start");
}
if crate::power::reconcile_action(wanted, running) != crate::power::ReconcileAction::Noop {
drifted.push(name.clone());
}
to_rebuild.push(name.clone());
}
// Enqueue the parent sweep entry. The worker processes it trivially
// (no-op dispatch); its purpose is to give the dashboard a "why" header.
let sweep_id = coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::StartupSweep,
"hyperhive".to_owned(),
crate::rebuild_queue::QueueSource::AutoUpdate,
format!(
"startup sweep: {} rebuild(s), {} deferred (stopped), {} skipped (up-to-date)",
to_rebuild.len(),
n_deferred,
n_skipped,
),
None,
);
tracing::info!(
total = containers.len(),
rebuilds = to_rebuild.len(),
rebuilds = fanout.len(),
reconciles = drifted.len(),
deferred = n_deferred,
skipped = n_skipped,
sweep_id,
"auto-update: startup sweep"
up_to_date = n_skipped,
"boot reconcile"
);
for name in to_rebuild {
coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Rebuild,
name,
crate::rebuild_queue::QueueSource::StartupSweep,
"startup sweep".to_owned(),
Some(sweep_id),
// Sweep parent whenever ANY marker is stale — even when every
// stale agent is wanted-offline: the hyperhive lock bump must land
// now so their later start-upgrade rebuilds build against it.
// No stale agents ⇒ no sweep ⇒ no meta commit on a no-change boot.
if any_stale {
let reason = format!(
"startup sweep: {} rebuild(s), {} deferred (offline), {} up-to-date",
fanout.len(),
n_deferred,
n_skipped,
);
if let Err(e) = coord
.job_queue
.submit(crate::job_queue::templates::startup_sweep(reason, fanout))
{
tracing::warn!(error = ?e, "boot reconcile: sweep submit failed");
}
}
for name in drifted {
if let Err(e) = coord
.job_queue
.submit(crate::job_queue::templates::reconcile_only(
crate::job_queue::Template::Reconcile,
&name,
crate::job_queue::Source::AutoUpdate,
"boot reconcile".to_owned(),
None,
))
{
tracing::warn!(%name, error = ?e, "boot reconcile: submit failed");
}
}
coord.emit_rebuild_queue_snapshot();
Ok(())

68
hive-c0re/src/coordinator.rs
View file

@ -173,12 +173,17 @@ pub struct Coordinator {
/// tokio mutex so the rescan can `await` `lifecycle::list` /
/// `is_running` without blocking other coordinator paths.
last_containers: tokio::sync::Mutex<HashMap<String, ContainerView>>,
/// Global rebuild queue. Every long-running container/meta op
/// (rebuild, meta-update, first-spawn) goes through this queue so
/// hive-c0re runs at most one at a time and the dashboard can
/// render a single ordered view of pending + running work. See
/// `rebuild_queue.rs` for the dedup rules + history retention.
pub rebuild_queue: Arc<crate::rebuild_queue::RebuildQueue>,
/// Global job-DAG queue. Every container/meta op (rebuild,
/// meta-update, first-spawn, power changes) is submitted as a DAG
/// of primitive nodes; a single scheduler drives them with
/// build-slot + per-agent-lease gating so the dashboard renders one
/// ordered view of pending + running work. See `job_queue/` for
/// the dedup rules, resource classes, and history retention.
pub job_queue: Arc<crate::job_queue::JobQueue>,
/// Durable per-agent power intent (`wanted: Up | Offline`) — the
/// spec half of desired-state reconciliation; the queue's
/// `Reconcile` nodes converge observed state to it.
pub power: Arc<crate::power::PowerStore>,
/// Shutdown signal broadcast to all background tasks. Sending
/// `true` asks every loop to exit after its current work item.
/// Use `shutdown_rx()` to subscribe; `request_shutdown()` to fire.
@ -244,12 +249,27 @@ impl Default for HiveEnv {
/// instead of every host-level setting as its own JSON-blob argument.
/// `#[serde(default)]` lets any field be omitted and fall back to its
/// canonical default.
#[derive(Clone, Debug, Default, serde::Deserialize)]
#[derive(Clone, Debug, serde::Deserialize)]
#[serde(default)]
pub struct ServeConfig {
#[serde(flatten)]
pub env: HiveEnv,
pub model_prices: crate::hive_stats::PriceTable,
/// Number of concurrent nix-heavy job-queue nodes (prebuild /
/// profile-swap / create / meta lock). hive-c0re-local like
/// `model_prices` — never injected into containers. Set via
/// `services.hyperhive.c0re.buildSlots`.
pub build_slots: usize,
}
impl Default for ServeConfig {
fn default() -> Self {
Self {
env: HiveEnv::default(),
model_prices: crate::hive_stats::PriceTable::default(),
build_slots: 1,
}
}
}
#[cfg(test)]
@ -433,6 +453,7 @@ impl Coordinator {
db_path: &Path,
env: HiveEnv,
model_prices: crate::hive_stats::PriceTable,
build_slots: usize,
) -> Result<Self> {
let HiveEnv {
hyperhive_flake,
@ -469,6 +490,8 @@ impl Coordinator {
let audit_log =
Arc::new(crate::audit_log::AuditLog::open(build_logs_dir).context("open audit_log")?);
crate::audit_log::install(audit_log.clone());
let power =
Arc::new(crate::power::PowerStore::open(build_logs_dir).context("open agent_power")?);
let (dashboard_events, _) = broadcast::channel(DASHBOARD_CHANNEL);
let (shutdown_tx, _) = watch::channel(false);
Ok(Self {
@ -497,7 +520,8 @@ impl Coordinator {
event_seq: AtomicU64::new(0),
meta_updates_active: AtomicU64::new(0),
last_containers: tokio::sync::Mutex::new(HashMap::new()),
rebuild_queue: Arc::new(crate::rebuild_queue::RebuildQueue::new()),
job_queue: Arc::new(crate::job_queue::JobQueue::new(build_slots)),
power,
shutdown_tx,
})
}
@ -543,7 +567,7 @@ impl Coordinator {
/// wrappers below) and the worker so every state transition
/// surfaces on the dashboard without extra plumbing.
pub fn emit_rebuild_queue_snapshot(self: &Arc<Self>) {
let queue = self.rebuild_queue.snapshot();
let queue = self.job_queue.snapshot();
self.emit_dashboard_event(DashboardEvent::RebuildQueueChanged {
seq: self.next_seq(),
queue,
@ -665,15 +689,27 @@ impl Coordinator {
});
}
/// Update the `step` label on a running queue entry and (if it
/// actually changed) re-emit the queue snapshot so the dashboard
/// renders the new phase. Returns `true` when the label was new
/// and an emit fired, mostly for tracing/logging callers; safe to
/// ignore. No-op when `id` is `None` (e.g. callers that aren't
/// running from the queue worker) or when the row isn't `Running`.
/// Update the `step` label on the currently-running node of DAG
/// `id` and (if it actually changed) re-emit the queue snapshot so
/// the dashboard renders the new phase. DAG-id-only surface for
/// the opaque approval pipeline in `actions.rs`, whose callbacks
/// don't know node ids (its DAGs are single-node, so the lookup is
/// exact); queue executors use the precise per-node sink in
/// `job_queue::exec` instead. No-op when `id` is `None` (callers
/// not running from the queue) or when nothing is `Running`.
pub fn set_queue_step(self: &Arc<Self>, id: Option<u64>, step: &str) {
let Some(id) = id else { return };
if self.rebuild_queue.set_step(id, step) {
if self.job_queue.set_step_running(id, step) {
self.emit_rebuild_queue_snapshot();
}
}
/// Link a `build_logs` row to the currently-running node of DAG
/// `id` and re-emit the snapshot. Same DAG-id-only compatibility
/// surface as [`Self::set_queue_step`].
pub fn set_queue_build_log(self: &Arc<Self>, id: Option<u64>, log_id: i64) {
let Some(id) = id else { return };
if self.job_queue.set_build_log_id_running(id, log_id) {
self.emit_rebuild_queue_snapshot();
}
}

44
hive-c0re/src/dashboard.rs
View file

@ -299,12 +299,13 @@ struct StateSnapshot {
/// disabled "updating…" state; live transitions arrive via the
/// `MetaUpdateRunning` event.
meta_update_running: bool,
/// Current state of the global rebuild queue — pending + running
/// long-lived ops (rebuild / meta-update / spawn) plus the most
/// recent few terminal entries the queue retains for history.
/// Live transitions arrive via the `RebuildQueueChanged` event.
/// See `rebuild_queue.rs`.
rebuild_queue: Vec<crate::rebuild_queue::QueueEntry>,
/// Current state of the global job queue — pending + running DAGs
/// (rebuild / meta-update / spawn / power ops) with their per-node
/// breakdowns, plus the most recent few terminal DAGs the queue
/// retains for history. Live transitions arrive via the
/// `RebuildQueueChanged` event. See `job_queue/`. Field name kept
/// from the old flat queue for wire compatibility.
rebuild_queue: Vec<crate::job_queue::DagView>,
/// Whether the hive-forge container is up. When true the dashboard
/// links each container's config + each approval's commit into the
/// forge's `agent-configs` repos.
@ -607,7 +608,7 @@ async fn api_state(headers: HeaderMap, State(state): State<AppState>) -> axum::J
question_history,
tombstones,
port_conflicts,
rebuild_queue: state.coord.rebuild_queue.snapshot(),
rebuild_queue: state.coord.job_queue.snapshot(),
forge_present: crate::forge::is_present().await,
matrix_gui_enabled: std::env::var_os("HIVE_MATRIX_GUI_ENABLED").is_some_and(|v| {
// Accept any truthy string ("1", "true", "yes") since the
@ -1602,30 +1603,15 @@ async fn post_meta_update(
return error_response("meta-update: no inputs selected");
}
let inputs_label = inputs.join(", ");
let parent_id = state.coord.rebuild_queue.enqueue_with_inputs(
crate::rebuild_queue::QueueKind::MetaUpdate,
"hyperhive".to_owned(),
crate::rebuild_queue::QueueSource::Manual,
// Cascade rebuild children fan out from the MetaLock node when the
// lock bump lands — appended by the scheduler so they build against
// the post-bump lock, and a failed bump simply fans out nothing.
crate::job_queue::submit::meta_update(
&state.coord,
inputs,
crate::job_queue::Source::Manual,
format!("meta-update via dashboard ({inputs_label})"),
None,
inputs.clone(),
);
// Pre-enqueue cascade rebuilds NOW so they're visible in the queue
// alongside the parent. The worker's MetaUpdate arm
// no longer enqueues children — it just runs the lock bump and
// (on failure) cancels these pre-queued children.
let cascade_agents = crate::rebuild_queue::meta_update_cascade_agents(&inputs).await;
let cascade_reason = format!("meta-update cascade ({inputs_label})");
for name in cascade_agents {
state.coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Rebuild,
name,
crate::rebuild_queue::QueueSource::MetaUpdate,
cascade_reason.clone(),
Some(parent_id),
);
}
state.coord.emit_rebuild_queue_snapshot();
(StatusCode::OK, "ok").into_response()
}

81
hive-c0re/src/dashboard/lifecycle_ops.rs
View file

@ -1,10 +1,12 @@
//! Container lifecycle endpoints for the dashboard.
//!
//! Rebuild / restart / start / stop (hard + graceful) / update-all all
//! enqueue onto the rebuild queue, so each shows a visible queued→running
//! transient on the dashboard — a direct sub-second start/stop only flashed
//! the badge; destroy delegates to `actions::destroy` (optionally
//! purging).
//! submit DAGs to the job queue (`job_queue::submit`), so each shows a
//! visible queued→running transient on the dashboard — a direct
//! sub-second start/stop only flashed the badge. Start/stop also
//! persist the agent's `wanted` power intent before submitting; the
//! DAG's `Reconcile` converges to it. Destroy delegates to
//! `actions::destroy` (optionally purging).
use axum::{
extract::{Form, Path as AxumPath, Query, State},
@ -23,6 +25,7 @@ pub(super) struct KillParams {
}
use super::{AppState, error_response, guard_agent_name, strip_container_prefix};
use crate::job_queue::{Source, submit};
use crate::{actions, lifecycle};
pub(super) async fn post_rebuild(
@ -33,14 +36,12 @@ pub(super) async fn post_rebuild(
if let Some(reject) = guard_agent_name(&state, &logical).await {
return reject;
}
state.coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Rebuild,
logical,
crate::rebuild_queue::QueueSource::Manual,
submit::rebuild(
&state.coord,
&logical,
Source::Manual,
"manual via dashboard ↻ R3BU1LD button".to_owned(),
None,
);
state.coord.emit_rebuild_queue_snapshot();
(StatusCode::OK, "ok").into_response()
}
@ -54,19 +55,17 @@ pub(super) async fn post_kill(
return reject;
}
if params.graceful {
// Graceful stop: enqueue the quiesce orchestration (signal the harness
// → one stop-checkpoint turn → drain → container stop, with a timeout
// fallback to a hard stop). Serialised through the rebuild queue so it
// can't race an in-flight rebuild for the same agent, and its per-step
// progress surfaces on the queue snapshot + build log.
state.coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::GracefulStop,
logical,
crate::rebuild_queue::QueueSource::Manual,
// Graceful stop: submit the quiesce DAG (signal the harness →
// one stop-checkpoint turn → drain → container stop, with a
// timeout fallback to a hard stop). The agent's lifecycle
// lease keeps it from racing an in-flight rebuild for the same
// agent, and per-node progress surfaces on the queue snapshot.
submit::graceful_stop(
&state.coord,
&logical,
Source::Manual,
"manual via dashboard graceful stop".to_owned(),
None,
);
state.coord.emit_rebuild_queue_snapshot();
return (StatusCode::OK, "ok").into_response();
}
// Manager is stoppable from the dashboard like any other
@ -79,14 +78,12 @@ pub(super) async fn post_kill(
// `socket_server.rs::ManagerRequest::Kill` stays in place: a
// manager calling Kill on its own container is self-suicide
// mid-call, not a legitimate operator action.
state.coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Stop,
logical,
crate::rebuild_queue::QueueSource::Manual,
submit::stop(
&state.coord,
&logical,
Source::Manual,
"manual via dashboard stop".to_owned(),
None,
);
state.coord.emit_rebuild_queue_snapshot();
(StatusCode::OK, "ok").into_response()
}
@ -98,14 +95,12 @@ pub(super) async fn post_restart(
if let Some(reject) = guard_agent_name(&state, &logical).await {
return reject;
}
state.coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Restart,
logical,
crate::rebuild_queue::QueueSource::Manual,
submit::restart(
&state.coord,
&logical,
Source::Manual,
"manual via dashboard ↺ R3START button".to_owned(),
None,
);
state.coord.emit_rebuild_queue_snapshot();
(StatusCode::OK, "ok").into_response()
}
@ -117,14 +112,12 @@ pub(super) async fn post_start(
if let Some(reject) = guard_agent_name(&state, &logical).await {
return reject;
}
state.coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Start,
logical,
crate::rebuild_queue::QueueSource::Manual,
submit::start(
&state.coord,
&logical,
Source::Manual,
"manual via dashboard start".to_owned(),
None,
);
state.coord.emit_rebuild_queue_snapshot();
(StatusCode::OK, "ok").into_response()
}
@ -137,15 +130,13 @@ pub(super) async fn post_update_all(State(state): State<AppState>) -> Response {
else {
continue;
};
state.coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Rebuild,
logical,
crate::rebuild_queue::QueueSource::Manual,
submit::rebuild(
&state.coord,
&logical,
Source::Manual,
"manual via dashboard 🌀 UPDATE ALL".to_owned(),
None,
);
}
state.coord.emit_rebuild_queue_snapshot();
(StatusCode::OK, "ok").into_response()
}

46
hive-c0re/src/dashboard/permissions.rs
View file

@ -128,18 +128,19 @@ pub(super) async fn post_tool_groups(
return Err(ProblemDetails::from_status_code(StatusCode::BAD_REQUEST)
.with_detail(format!("invalid tool-groups for {logical}: {e}")));
}
// Enqueue a PermChange so the JSON file write is serialised through
// the FIFO worker. Prevents concurrent batch-apply actions for
// different agents from racing on the shared tool-groups.json.
state.coord.rebuild_queue.enqueue_with_perm(
logical.clone(),
crate::rebuild_queue::QueueSource::Manual,
// Submit a PermChange DAG: the JSON file write commits under
// META_LOCK inside the WritePermFile node, so concurrent
// batch-apply actions for different agents never race on the
// shared tool-groups.json.
crate::job_queue::submit::perm_change(
&state.coord,
&logical,
crate::job_queue::Source::Manual,
"tool-group change via permissions UI".to_owned(),
crate::rebuild_queue::PermPayload::ToolGroups {
crate::job_queue::PermPayload::ToolGroups {
groups: body.groups.clone(),
},
);
state.coord.emit_rebuild_queue_snapshot();
tracing::info!(agent = %logical, groups = ?body.groups, "operator: set tool-groups via dashboard");
Ok((StatusCode::OK, "ok").into_response())
}
@ -214,18 +215,19 @@ pub(super) async fn post_capabilities(
.with_detail(format!("unknown capability: {cap}")));
}
}
// Enqueue a PermChange so the JSON file write is serialised through
// the FIFO worker. Prevents concurrent batch-apply actions for
// different agents from racing on the shared capabilities.json.
state.coord.rebuild_queue.enqueue_with_perm(
logical.clone(),
crate::rebuild_queue::QueueSource::Manual,
// Submit a PermChange DAG: the JSON file write commits under
// META_LOCK inside the WritePermFile node, so concurrent
// batch-apply actions for different agents never race on the
// shared capabilities.json.
crate::job_queue::submit::perm_change(
&state.coord,
&logical,
crate::job_queue::Source::Manual,
"capability change via dashboard".to_owned(),
crate::rebuild_queue::PermPayload::Capabilities {
crate::job_queue::PermPayload::Capabilities {
caps: body.caps.clone(),
},
);
state.coord.emit_rebuild_queue_snapshot();
tracing::info!(agent = %logical, caps = ?body.caps, "operator: set capabilities via dashboard");
Ok((StatusCode::OK, "ok").into_response())
}
@ -298,17 +300,17 @@ pub(super) async fn post_permissions(
));
}
}
// Phase 2 — enqueue one combined PermChange per affected agent.
// Phase 2 — submit one combined PermChange DAG per affected agent.
for (logical, groups, caps) in staged {
state.coord.rebuild_queue.enqueue_with_perm(
logical.clone(),
crate::rebuild_queue::QueueSource::Manual,
crate::job_queue::submit::perm_change(
&state.coord,
&logical,
crate::job_queue::Source::Manual,
"batch permission change via permissions UI".to_owned(),
crate::rebuild_queue::PermPayload::Combined { groups, caps },
crate::job_queue::PermPayload::Combined { groups, caps },
);
tracing::info!(agent = %logical, "operator: batch perm change via dashboard");
}
state.coord.emit_rebuild_queue_snapshot();
Ok((StatusCode::OK, "ok").into_response())
}

19
hive-c0re/src/dashboard/schedules.rs
View file

@ -115,20 +115,19 @@ pub(super) async fn post_schedule_fire_now(
}
}
/// `POST /api/rebuild-queue/{id}/cancel` — drop a `Queued` entry
/// from the rebuild queue. Refuses `Running` / terminal
/// entries: an in-flight rebuild owns the agent's nix store +
/// nixos-container update lock and can't be safely interrupted
/// from the queue side. Always returns 200; the body is
/// `{"cancelled": true}` on a successful flip from Queued →
/// Cancelled, `{"cancelled": false}` when the row was Running /
/// terminal / gone. On success a fresh `RebuildQueueChanged`
/// snapshot fires so the row's state flip surfaces live.
/// `POST /api/rebuild-queue/{id}/cancel` — drop a still-fully-queued
/// DAG from the job queue. Refuses `Running` / terminal DAGs: an
/// in-flight node owns the agent's nix store + nixos-container update
/// lock and can't be safely interrupted from the queue side. Always
/// returns 200; the body is `{"cancelled": true}` on a successful
/// flip to Cancelled, `{"cancelled": false}` when the DAG was
/// Running / terminal / gone. On success a fresh `RebuildQueueChanged`
/// snapshot fires so the state flip surfaces live.
pub(super) async fn post_rebuild_queue_cancel(
State(state): State<AppState>,
AxumPath(id): AxumPath<u64>,
) -> Response {
let cancelled = state.coord.rebuild_queue.cancel(id);
let cancelled = state.coord.job_queue.cancel(id);
if cancelled {
state.coord.emit_rebuild_queue_snapshot();
axum::Json(serde_json::json!({"cancelled": true})).into_response()

4
hive-c0re/src/dashboard_events.rs
View file

@ -8,7 +8,7 @@ use serde::Serialize;
use crate::container_view::ContainerView;
use crate::dashboard::{MetaInputView, TombstoneView};
use crate::rebuild_queue::QueueEntry;
use crate::job_queue::DagView;
use chrono::{DateTime, Utc};
#[derive(Debug, Clone, Serialize)]
@ -210,7 +210,7 @@ pub enum DashboardEvent {
/// the add/remove races a per-row event would have, and the
/// dashboard's grouping (`parent_id`) is most naturally re-derived
/// from the full list.
RebuildQueueChanged { seq: u64, queue: Vec<QueueEntry> },
RebuildQueueChanged { seq: u64, queue: Vec<DagView> },
/// Full snapshot of all scheduled prompts. Emitted after every
/// operator mutation (new / edit / cancel / fire-now) and after the
/// worker fires or rearms a row. Same snapshot-shape rationale as

433
hive-c0re/src/job_queue/exec.rs Normal file
View file

@ -0,0 +1,433 @@
//! Node executors — one async fn per [`NodeKind`], each a thin wrapper
//! over existing `lifecycle.rs` / `meta.rs` / `actions.rs` code. Node
//! executors keep their own internal error handling where it exists
//! today (cold-start fallback inside `Reconcile`, non-fatal boot-time
//! lock bump inside the sweep `MetaLock`, warn-only forge sync in the
//! `Swap` tail); DAG-level failure handling is cancel-downstream in
//! the queue.
use std::sync::Arc;
use anyhow::{Context as _, Result};
use super::model::{NodeKind, State, Template};
use super::{Claim, TerminalDag};
use crate::coordinator::Coordinator;
use crate::power::{ReconcileAction, reconcile_action};
/// Max time `Drain` waits for the harness to run its stop-checkpoint
/// turn before falling back to the hard stop. Generous — a checkpoint
/// turn can take a while — but bounded so a wedged agent never blocks
/// the stop indefinitely. Drains hold no build slot, so a whole-hive
/// graceful stop overlaps every agent's drain instead of serialising
/// N × this timeout.
pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(3);
/// Extra signal an executor hands back to the scheduler alongside
/// success.
#[derive(Debug, Default)]
pub struct NodeOutput {
/// Agents to fan child `Rebuild` DAGs out for (`MetaLock` only).
pub fanout: Vec<String>,
}
/// Step-label + build-log sink for one claimed node.
struct Ctx<'a> {
coord: &'a Arc<Coordinator>,
dag_id: u64,
node_id: super::NodeId,
}
impl Ctx<'_> {
fn step(&self, step: &str) {
if self
.coord
.job_queue
.set_step(self.dag_id, self.node_id, step)
{
self.coord.emit_rebuild_queue_snapshot();
}
}
fn build_log(&self, log_id: i64) {
if self
.coord
.job_queue
.set_build_log_id(self.dag_id, self.node_id, log_id)
{
self.coord.emit_rebuild_queue_snapshot();
}
}
}
/// Run one claimed node to completion. Called from a task the
/// scheduler spawns per claim; the `Result` (stringified) becomes the
/// node's terminal state.
pub(super) async fn run_node(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
let ctx = Ctx {
coord,
dag_id: claim.dag_id,
node_id: claim.node_id,
};
match &claim.kind {
NodeKind::Prebuild { relock } => run_prebuild(coord, claim, &ctx, *relock).await,
NodeKind::Swap => run_swap(coord, claim, &ctx).await,
NodeKind::Create => run_create(coord, claim, &ctx).await,
NodeKind::MetaLock { sweep, fanout } => {
run_meta_lock(coord, claim, &ctx, *sweep, fanout.clone()).await
}
NodeKind::Reconcile => run_reconcile(coord, claim, &ctx).await,
NodeKind::StopForUpdate => run_stop_for_update(coord, claim, &ctx).await,
NodeKind::Signal => Ok(run_signal(coord, claim, &ctx)),
NodeKind::Drain => run_drain(coord, claim, &ctx).await,
NodeKind::WriteDropin => run_write_dropin(coord, claim).await,
NodeKind::WritePermFile => run_write_perm_file(coord, claim, &ctx).await,
NodeKind::ApprovalDeploy => run_approval_deploy(coord, claim).await,
}
}
/// Out-of-band toplevel build while the container keeps serving: meta
/// sync + optional per-agent relock, then warm
/// `system.build.toplevel` so the later `Swap` hits cache and skips
/// straight to the profile-swap.
async fn run_prebuild(
coord: &Arc<Coordinator>,
claim: &Claim,
ctx: &Ctx<'_>,
relock: bool,
) -> Result<NodeOutput> {
let name = &claim.agent;
let agent_dir = coord
.ensure_runtime(name)
.with_context(|| format!("ensure_runtime {name}"))?;
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(name, agent_dir);
crate::lifecycle::prepare_rebuild_dirs(name, &paths).await?;
// Idempotent meta sync so a manual rebuild can also recover from a
// divergent meta repo; then bump just this agent's input. `relock =
// false` only for meta-update cascade children, where re-locking
// would revert the bump the cascade just committed.
let agents = crate::lifecycle::agents_for_meta_listing().await?;
crate::meta::sync_agents(&hive, &agents).await?;
if relock {
crate::meta::lock_update_for_rebuild(name).await?;
}
ctx.step("nix build");
let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display());
crate::lifecycle::prebuild_toplevel(name, &flake_ref, &|log_id| ctx.build_log(log_id)).await?;
Ok(NodeOutput::default())
}
/// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb),
/// `nixos-container update`, then the post-rebuild bookkeeping tail
/// (rev marker, `Rebuilt` event, forge/matrix sync, kick, rescan).
/// The recovery-start on failure is NOT here — the DAG's tail
/// `Reconcile` runs after this node terminal ok *or* fail.
async fn run_swap(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
let name = &claim.agent;
let agent_dir = coord.ensure_runtime(name)?;
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(name, agent_dir);
let result =
crate::lifecycle::swap_update(name, &hive, &paths, &|step| ctx.step(step), &|log_id| {
ctx.build_log(log_id)
})
.await;
match &result {
Ok(()) => {
if let Some(rev) = crate::auto_update::current_flake_rev(&coord.hyperhive_flake)
&& let Err(e) = std::fs::write(crate::auto_update::rev_marker_path(name), rev)
{
tracing::warn!(%name, error = ?e, "write rev marker failed");
}
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
agent: name.clone(),
ok: true,
note: None,
sha: None,
tag: None,
});
ctx.step("forge sync");
// Full forge + matrix sync on every successful rebuild so
// the rebuild path is equivalent to the startup sweep:
// tokens, config-repo mirror, meta access all recover
// without a hive-c0re restart.
crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await;
crate::matrix::sync_agent_standalone(name).await;
// Wake the agent on its next turn so claude sees a "you
// were rebuilt" hint; rescan so dashboards drop the
// "needs update" chip; lock bump → meta-inputs re-render.
coord.kick_agent(name, "container rebuilt");
coord.rescan_containers_and_emit().await;
crate::dashboard::emit_meta_inputs_snapshot(coord);
}
Err(_) => {
// The `Rebuilt { ok: false }` manager event fires once per
// DAG from the terminal hook (any node may be the one that
// failed); here only refresh the observed state.
coord.rescan_containers_and_emit().await;
}
}
result.map(|()| NodeOutput::default())
}
/// First-spawn provisioning + `nixos-container create` (atomic
/// build+create — no prebuild needed).
async fn run_create(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
let name = &claim.agent;
let agent_dir = coord.ensure_runtime(name)?;
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(name, agent_dir);
ctx.step("nixos-container create");
crate::lifecycle::create_container(name, &hive, &paths).await?;
Ok(NodeOutput::default())
}
/// Meta flake lock bump. Boot-sweep flavour is non-fatal (a failed
/// bump must not cancel the fan-out rebuilds — they proceed against
/// the current lock, exactly like today's sweep); the meta-update
/// flavour propagates errors, and a failed bump fans out nothing.
async fn run_meta_lock(
coord: &Arc<Coordinator>,
claim: &Claim,
ctx: &Ctx<'_>,
sweep: bool,
fanout: Option<Vec<String>>,
) -> Result<NodeOutput> {
if sweep {
ctx.step("nix flake update hyperhive");
if let Err(e) = crate::meta::lock_update_hyperhive().await {
tracing::warn!(error = ?e, "startup sweep: meta lock_update_hyperhive failed");
}
return Ok(NodeOutput {
fanout: fanout.unwrap_or_default(),
});
}
let _progress = coord.meta_update_guard();
ctx.step("nix flake update");
crate::meta::lock_update(&claim.inputs).await?;
// Lock file changed — meta-inputs panel re-renders.
crate::dashboard::emit_meta_inputs_snapshot(coord);
let cascade = match fanout {
Some(list) => list,
None => meta_update_cascade_agents(&claim.inputs).await,
};
Ok(NodeOutput { fanout: cascade })
}
/// Idempotent power converge: `wanted` (durable intent) vs observed.
async fn run_reconcile(
coord: &Arc<Coordinator>,
claim: &Claim,
ctx: &Ctx<'_>,
) -> Result<NodeOutput> {
let name = &claim.agent;
let running = crate::lifecycle::is_running(name).await;
let wanted = coord.power.get_or_seed(name, running)?;
match reconcile_action(wanted, running) {
ReconcileAction::Start => {
// Node-local transient only when the DAG holds none (the
// boot-reconcile template); lease-window guards otherwise
// already cover this node.
let _guard = claim
.transient
.is_none()
.then(|| coord.transient_guard(name, crate::coordinator::TransientKind::Starting));
ctx.step("nixos-container start");
crate::lifecycle::start_with_fallback(name).await?;
coord.kick_agent(name, "container started");
coord.rescan_containers_and_emit().await;
}
ReconcileAction::Stop => {
let _guard = claim
.transient
.is_none()
.then(|| coord.transient_guard(name, crate::coordinator::TransientKind::Stopping));
ctx.step("nixos-container stop");
crate::lifecycle::kill(name).await?;
coord.unregister_agent(name);
coord.notify_manager(&hive_sh4re::HelperEvent::Killed {
agent: name.clone(),
});
coord.rescan_containers_and_emit().await;
}
ReconcileAction::Noop => {
tracing::debug!(%name, wanted = wanted.as_str(), running, "reconcile: noop");
}
}
Ok(NodeOutput::default())
}
/// Mechanical stop for the profile swap. Never touches `wanted`; noop
/// when already stopped.
async fn run_stop_for_update(
coord: &Arc<Coordinator>,
claim: &Claim,
ctx: &Ctx<'_>,
) -> Result<NodeOutput> {
let name = &claim.agent;
if crate::lifecycle::is_running(name).await {
ctx.step("nixos-container stop");
crate::lifecycle::kill(name).await?;
coord.rescan_containers_and_emit().await;
}
Ok(NodeOutput::default())
}
/// Set the graceful fence + kick so the harness sees it promptly and
/// runs its one stop-checkpoint turn.
fn run_signal(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> NodeOutput {
ctx.step("graceful stop: signalling agent");
coord.mark_graceful_stop(&claim.agent);
coord.kick_agent(&claim.agent, "graceful stop requested");
NodeOutput::default()
}
/// Await the harness clearing the fence (`GracefulStopComplete`) or
/// the timeout — either way the downstream `Reconcile` proceeds with
/// the actual stop.
async fn run_drain(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
let name = &claim.agent;
ctx.step("graceful stop: draining");
let deadline = std::time::Instant::now() + GRACEFUL_STOP_TIMEOUT;
while coord.is_graceful_stop_pending(name) {
if std::time::Instant::now() >= deadline {
tracing::warn!(agent = %name, "graceful stop: drain timed out — hard-stopping");
break;
}
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
}
coord.clear_graceful_stop(name);
Ok(NodeOutput::default())
}
/// `set_nspawn_flags` + `set_resource_limits` + daemon-reload.
async fn run_write_dropin(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
let name = &claim.agent;
let agent_dir = coord.ensure_runtime(name)?;
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(name, agent_dir);
crate::lifecycle::write_dropins(name, &hive, &paths).await?;
Ok(NodeOutput::default())
}
/// Write + commit the perm file(s) (fused under `META_LOCK` so the
/// working tree is never left dirty), then emit the P3RM1SS10NS-tab
/// snapshots so the dashboard reflects the new assignment.
async fn run_write_perm_file(
coord: &Arc<Coordinator>,
claim: &Claim,
ctx: &Ctx<'_>,
) -> Result<NodeOutput> {
use super::model::PermPayload;
let name = &claim.agent;
ctx.step("writing + committing perm file");
match &claim.perm_payload {
Some(PermPayload::ToolGroups { groups }) => {
crate::meta::commit_tool_groups(name, groups)
.await
.with_context(|| format!("commit tool-groups for {name}"))?;
coord.emit_tool_groups_snapshot();
}
Some(PermPayload::Capabilities { caps }) => {
crate::meta::commit_capabilities(name, caps)
.await
.with_context(|| format!("commit capabilities for {name}"))?;
coord.emit_capabilities_snapshot();
}
Some(PermPayload::Combined { groups, caps }) => {
crate::meta::commit_perms(name, groups.as_deref(), caps.as_deref())
.await
.with_context(|| format!("commit perms for {name}"))?;
if groups.is_some() {
coord.emit_tool_groups_snapshot();
}
if caps.is_some() {
coord.emit_capabilities_snapshot();
}
}
None => anyhow::bail!(
"perm_change dag {} for {name} is missing perm_payload",
claim.dag_id
),
}
Ok(NodeOutput::default())
}
/// Opaque approval deploy pipeline: `ApplyCommit` and `MergeConfigPr`
/// both end in a container rebuild; branch on the approval row's kind
/// (the authoritative source). The two-phase prepare/finalize/abort
/// meta deploy — and the approval resolution — stay inside
/// `actions.rs` in v1 (design doc §9).
async fn run_approval_deploy(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
let approval_id = claim
.approval_id
.with_context(|| format!("approval_deploy dag {} has no approval_id", claim.dag_id))?;
let kind = coord
.approvals
.get(approval_id)
.ok()
.flatten()
.map(|a| a.kind);
let result = if kind == Some(hive_sh4re::ApprovalKind::MergeConfigPr) {
crate::actions::run_approval_merge_config_pr(coord, Some(claim.dag_id), approval_id).await
} else {
crate::actions::run_approval_apply_commit(coord, Some(claim.dag_id), approval_id).await
};
result.map(|()| NodeOutput::default())
}
/// Terminal-roll-up hook, fired exactly once per DAG. Approval DAGs
/// resolve their approval row (except the opaque deploy pipeline,
/// which resolves inside its node); non-approval rebuild-shaped DAGs
/// surface the `Rebuilt { ok: false }` manager event on failure —
/// success fires from the `Swap` tail, matching today's timing.
pub(super) async fn on_dag_terminal(coord: &Arc<Coordinator>, terminal: &TerminalDag) {
if terminal.approval_id.is_some() {
crate::actions::resolve_approval_dag(coord, terminal).await;
return;
}
if matches!(terminal.template, Template::Rebuild | Template::PermChange)
&& terminal.state == State::Failed
{
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
agent: terminal.agent.clone(),
ok: false,
note: terminal.error.clone(),
sha: None,
tag: None,
});
}
}
/// Compute which agents a `nix flake update <inputs>` on the meta
/// flake affects — the fan-out set for `MetaUpdate` DAGs. Empty
/// `inputs` or any input under `hyperhive` → every container;
/// otherwise just the agents named by `agent-<name>` inputs.
/// Topology-sorted so parents rebuild before their children.
pub async fn meta_update_cascade_agents(inputs: &[String]) -> Vec<String> {
let touched_hyperhive = inputs
.iter()
.any(|i| i == "hyperhive" || i.starts_with("hyperhive/"));
let touched_agents: Vec<String> = inputs
.iter()
.filter_map(|i| i.strip_prefix("agent-"))
.map(|rest| rest.split('/').next().unwrap_or(rest).to_owned())
.collect();
let mut names = if touched_hyperhive || inputs.is_empty() {
crate::lifecycle::list()
.await
.unwrap_or_default()
.into_iter()
.filter_map(|c| {
c.strip_prefix(crate::lifecycle::AGENT_PREFIX)
.map(str::to_owned)
})
.collect()
} else {
touched_agents
};
let topo = crate::topology::read();
crate::auto_update::topology_sort(&mut names, &topo);
names
}

565
hive-c0re/src/job_queue/mod.rs Normal file
View file

@ -0,0 +1,565 @@
//! Generic job-DAG queue + desired-state reconciliation — replaces the
//! old flat `rebuild_queue`. Jobs are nodes in per-request DAGs (see
//! [`templates`]); the special cases (graceful-stop watcher thread,
//! deferred-start follow-up, meta-update cascade) collapse into DAG
//! *shapes* over a shared set of primitive nodes ([`model::NodeKind`]).
//!
//! Concurrency is gated by two resource classes:
//! 1. **Build slots** — N permits (`services.hyperhive.c0re.buildSlots`,
//! default 1) held by nix-heavy nodes for the node's duration.
//! 2. **Per-agent lifecycle lease** — DAG-scoped: acquired before the
//! DAG's first container-affecting node runs, held until the DAG is
//! terminal, so two lifecycle DAGs for one agent never interleave
//! their container ops.
//!
//! The meta *repo* is serialized by `meta::META_LOCK` inside the
//! executors themselves. Per-agent power *intent* (`wanted`) lives in
//! the durable [`crate::power`] store; the DAGs are the reconcile
//! mechanism. Design + rationale: `docs/coordinator.md::Job queue`.
pub mod exec;
pub mod model;
pub mod scheduler;
pub mod submit;
pub mod templates;
#[cfg(test)]
mod tests;
use std::collections::{HashMap, VecDeque};
use std::sync::Mutex;
use tokio::sync::Notify;
pub use model::{
Dag, DagSpec, DagView, DepWhen, Node, NodeId, NodeKind, PermPayload, Source, State, Template,
};
/// How many terminal DAGs (`Done` / `Failed` / `Cancelled`) to retain
/// per template in the snapshot, matching the old per-kind history cap.
const MAX_HISTORY_PER_TEMPLATE: usize = 5;
/// Cap on stored node error strings.
const MAX_ERROR_LEN: usize = 2_000;
/// A node claimed for execution — everything the executor needs,
/// snapshotted at claim time.
#[derive(Debug, Clone)]
pub struct Claim {
pub dag_id: u64,
pub node_id: NodeId,
pub kind: NodeKind,
pub agent: String,
pub template: Template,
pub source: Source,
pub approval_id: Option<i64>,
pub inputs: Vec<String>,
pub perm_payload: Option<PermPayload>,
/// True when claiming this node acquired the DAG's agent lease —
/// the scheduler creates the DAG-scoped transient guard on this
/// edge.
pub lease_acquired: bool,
/// Transient pill kind for the lease window (from the spec).
pub transient: Option<crate::coordinator::TransientKind>,
}
/// Summary of a DAG that just reached its terminal roll-up state —
/// input to the approval-resolution hook and the lease/transient
/// release.
#[derive(Debug, Clone)]
pub struct TerminalDag {
pub dag_id: u64,
pub template: Template,
pub agent: String,
pub approval_id: Option<i64>,
pub state: State,
/// First failed node's error when `state == Failed`.
pub error: Option<String>,
}
/// Report from [`JobQueue::complete_node`].
#[derive(Debug, Default)]
pub struct CompletionReport {
/// DAGs that became terminal as a result of this completion
/// (the completed node's own DAG, plus none others — but kept as a
/// Vec so cancel paths can reuse the same settle plumbing).
pub terminal: Vec<TerminalDag>,
}
#[derive(Debug, Default)]
struct Inner {
dags: VecDeque<Dag>,
next_id: u64,
build_slots: usize,
slots_used: usize,
/// agent → dag id currently holding that agent's lifecycle lease.
leases: HashMap<String, u64>,
}
/// The queue. Lives on `Coordinator` (one per hive-c0re process); a
/// single scheduler task ([`scheduler::run_worker`]) drives it —
/// concurrency comes from the build-slot count, not multiple workers.
#[derive(Debug)]
pub struct JobQueue {
inner: Mutex<Inner>,
/// Wakes the scheduler when something new arrives or state changed.
pub(crate) notify: Notify,
}
impl Default for JobQueue {
fn default() -> Self {
Self::new(1)
}
}
impl JobQueue {
pub fn new(build_slots: usize) -> Self {
Self {
inner: Mutex::new(Inner {
build_slots: build_slots.max(1),
..Inner::default()
}),
notify: Notify::new(),
}
}
/// Submit a DAG. Validates the spec (cycle rejection) and dedups
/// against non-started DAGs; returns the DAG id (newly-allocated,
/// or the existing DAG's id with the new reason appended).
///
/// Dedup: a DAG whose roll-up is still `Queued` (no node started)
/// with the same `(template, agent, parent_id, approval_id)` — plus
/// `inputs` for `MetaUpdate` and the perm-type discriminant for
/// `PermChange` — swallows the repeat. `parent_id` is part of the
/// key so a meta-update cascade rebuild never collapses into a
/// standalone or sweep rebuild. Running / terminal DAGs never
/// dedup — operators are free to re-queue.
pub fn submit(&self, spec: DagSpec) -> anyhow::Result<u64> {
templates::validate(&spec)?;
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
if let Some(existing) = Self::dedup_target(&mut inner, &spec) {
if !existing.reason.contains(&spec.reason) {
use std::fmt::Write as _;
let _ = write!(existing.reason, "\nalso requested by: {}", spec.reason);
}
return Ok(existing.id);
}
let id = Self::push_dag(&mut inner, spec);
drop(inner);
self.notify.notify_one();
Ok(id)
}
/// Append fan-out children under a parent DAG (meta-update / sweep
/// cascade). Applies the same dedup as [`Self::submit`]; returns
/// the child ids actually created or coalesced into.
pub fn append_children(&self, specs: Vec<DagSpec>) -> Vec<u64> {
let mut ids = Vec::with_capacity(specs.len());
for spec in specs {
match self.submit(spec) {
Ok(id) => ids.push(id),
Err(e) => tracing::error!(error = ?e, "job_queue: invalid fan-out child spec"),
}
}
ids
}
fn dedup_target<'a>(inner: &'a mut Inner, spec: &DagSpec) -> Option<&'a mut Dag> {
inner.dags.iter_mut().find(|d| {
d.rollup() == State::Queued
&& d.template == spec.template
&& d.agent == spec.agent
&& d.parent_id == spec.parent_id
&& d.approval_id == spec.approval_id
&& (d.template != Template::MetaUpdate || d.inputs == spec.inputs)
&& PermPayload::same_type(d.perm_payload.as_ref(), spec.perm_payload.as_ref())
})
}
fn push_dag(inner: &mut Inner, spec: DagSpec) -> u64 {
inner.next_id += 1;
let id = inner.next_id;
let nodes = spec
.nodes
.into_iter()
.enumerate()
.map(|(i, n)| Node {
id: u32::try_from(i).unwrap_or(u32::MAX),
kind: n.kind,
deps: n.deps,
state: State::Queued,
step: None,
build_log_id: None,
started_at: None,
finished_at: None,
error: None,
})
.collect();
inner.dags.push_back(Dag {
id,
template: spec.template,
agent: spec.agent,
source: spec.source,
reason: spec.reason,
parent_id: spec.parent_id,
approval_id: spec.approval_id,
inputs: spec.inputs,
perm_payload: spec.perm_payload,
transient: spec.transient,
created_at: now_unix(),
nodes,
terminal_reported: false,
});
id
}
/// Claim every currently-ready node, acquiring resources, and mark
/// them `Running`. A node is ready when it's `Queued`, every dep is
/// satisfied (`AfterOk`: dep `Done`; `AfterAny`: dep terminal), and
/// its resources are free (build slot; agent lease free or already
/// held by this DAG). Iteration is in DAG-submit order, so
/// simultaneously-ready nodes compete FIFO — bulk operations drain
/// predictably.
pub fn claim_ready(&self) -> Vec<Claim> {
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
Self::propagate_cancellations(&mut inner);
let mut claims = Vec::new();
let inner = &mut *inner;
for di in 0..inner.dags.len() {
// Split-borrow dance: deps are checked against the same
// DAG's other nodes, so snapshot the states first.
let dag = &inner.dags[di];
let dag_id = dag.id;
let ready_ids: Vec<NodeId> = dag
.nodes
.iter()
.filter(|n| n.state == State::Queued && Self::deps_satisfied(dag, n))
.map(|n| n.id)
.collect();
for node_id in ready_ids {
let dag = &inner.dags[di];
let node = dag.node(node_id).expect("node id from same dag");
let needs_slot = node.kind.needs_build_slot();
if needs_slot && inner.slots_used >= inner.build_slots {
continue;
}
let mut lease_acquired = false;
if node.kind.needs_lease() {
match inner.leases.get(dag.agent.as_str()) {
Some(&holder) if holder != dag_id => continue,
Some(_) => {}
None => {
inner.leases.insert(dag.agent.clone(), dag_id);
lease_acquired = true;
}
}
}
if needs_slot {
inner.slots_used += 1;
}
let dag = &mut inner.dags[di];
let claim = Claim {
dag_id,
node_id,
kind: dag.node(node_id).expect("node").kind.clone(),
agent: dag.agent.clone(),
template: dag.template,
source: dag.source,
approval_id: dag.approval_id,
inputs: dag.inputs.clone(),
perm_payload: dag.perm_payload.clone(),
lease_acquired,
transient: dag.transient,
};
let node = dag.node_mut(node_id).expect("node");
node.state = State::Running;
node.started_at = Some(now_unix());
claims.push(claim);
}
}
claims
}
fn deps_satisfied(dag: &Dag, node: &Node) -> bool {
node.deps.iter().all(|dep| {
dag.node(dep.on).is_some_and(|d| match dep.when {
DepWhen::AfterOk => d.state == State::Done,
DepWhen::AfterAny => d.state.is_terminal(),
})
})
}
/// Cancel-downstream: a `Queued` node with an `AfterOk` dep that
/// `Failed` / `Cancelled` becomes `Cancelled` itself. Loops to a
/// fixpoint so the cancellation cascades through chains.
fn propagate_cancellations(inner: &mut Inner) {
for dag in &mut inner.dags {
loop {
let doomed: Vec<NodeId> = dag
.nodes
.iter()
.filter(|n| {
n.state == State::Queued
&& n.deps.iter().any(|dep| {
dep.when == DepWhen::AfterOk
&& dag.node(dep.on).is_some_and(|d| {
matches!(d.state, State::Failed | State::Cancelled)
})
})
})
.map(|n| n.id)
.collect();
if doomed.is_empty() {
break;
}
let now = now_unix();
for id in doomed {
if let Some(n) = dag.node_mut(id) {
n.state = State::Cancelled;
n.finished_at = Some(now);
}
}
}
}
}
/// Mark a claimed node terminal, release its build slot, cascade
/// cancellations, and settle terminal DAGs (lease release + history
/// trim). `error` is stored (truncated) when `result` is `Err`.
pub fn complete_node(
&self,
dag_id: u64,
node_id: NodeId,
result: Result<(), String>,
) -> CompletionReport {
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
if let Some(dag) = inner.dags.iter_mut().find(|d| d.id == dag_id)
&& let Some(node) = dag.node_mut(node_id)
&& node.state == State::Running
{
let needs_slot = node.kind.needs_build_slot();
node.finished_at = Some(now_unix());
node.step = None;
match result {
Ok(()) => node.state = State::Done,
Err(e) => {
node.state = State::Failed;
let mut msg = e;
if msg.len() > MAX_ERROR_LEN {
msg.truncate(
(0..=MAX_ERROR_LEN)
.rev()
.find(|i| msg.is_char_boundary(*i))
.unwrap_or(0),
);
msg.push('…');
}
node.error = Some(msg);
}
}
if needs_slot {
inner.slots_used = inner.slots_used.saturating_sub(1);
}
}
let report = Self::settle(&mut inner);
drop(inner);
self.notify.notify_one();
report
}
/// Propagate cancellations, release the leases of newly-terminal
/// DAGs, and trim history. Each terminal DAG is reported exactly
/// once (the `terminal_reported` flag) so the scheduler's hooks —
/// approval resolution, transient-guard release — fire once per
/// DAG.
fn settle(inner: &mut Inner) -> CompletionReport {
Self::propagate_cancellations(inner);
let mut report = CompletionReport::default();
let mut freed: Vec<String> = Vec::new();
for dag in &mut inner.dags {
if !dag.is_terminal() || dag.terminal_reported {
continue;
}
dag.terminal_reported = true;
if inner.leases.get(dag.agent.as_str()) == Some(&dag.id) {
freed.push(dag.agent.clone());
}
report.terminal.push(TerminalDag {
dag_id: dag.id,
template: dag.template,
agent: dag.agent.clone(),
approval_id: dag.approval_id,
state: dag.rollup(),
error: dag.first_error().map(str::to_owned),
});
}
for agent in freed {
inner.leases.remove(&agent);
}
Self::trim_history(inner);
report
}
/// Cancel a DAG that hasn't started yet (roll-up `Queued`): every
/// node flips to `Cancelled`. No-op (returns `false`) once any node
/// is running or terminal — an in-flight nix build isn't
/// interruptible, matching the old queue's rule.
pub fn cancel(&self, dag_id: u64) -> bool {
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
let Some(dag) = inner.dags.iter_mut().find(|d| d.id == dag_id) else {
return false;
};
if dag.rollup() != State::Queued {
return false;
}
let now = now_unix();
for n in &mut dag.nodes {
n.state = State::Cancelled;
n.finished_at = Some(now);
}
let _ = Self::settle(&mut inner);
drop(inner);
self.notify.notify_one();
true
}
/// Cancel every still-fully-queued child DAG of `parent`. Running
/// children are left alone. Returns the count of cancelled DAGs.
pub fn cancel_children(&self, parent: u64) -> usize {
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
let now = now_unix();
let mut count = 0;
for dag in &mut inner.dags {
if dag.parent_id == Some(parent) && dag.rollup() == State::Queued {
for n in &mut dag.nodes {
n.state = State::Cancelled;
n.finished_at = Some(now);
}
count += 1;
}
}
if count > 0 {
let _ = Self::settle(&mut inner);
drop(inner);
self.notify.notify_one();
}
count
}
/// Set the step label on a `Running` node. Returns `true` when the
/// label actually changed (callers emit a snapshot only then).
pub fn set_step(&self, dag_id: u64, node_id: NodeId, step: &str) -> bool {
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
let Some(node) = inner
.dags
.iter_mut()
.find(|d| d.id == dag_id)
.and_then(|d| d.node_mut(node_id))
else {
return false;
};
if node.state != State::Running || node.step.as_deref() == Some(step) {
return false;
}
node.step = Some(step.to_owned());
true
}
/// Set the step label on the DAG's currently-running node —
/// compatibility surface for the opaque approval pipeline, whose
/// callbacks only know the DAG id. Single-node approval DAGs make
/// this exact.
pub fn set_step_running(&self, dag_id: u64, step: &str) -> bool {
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
let Some(node) = inner
.dags
.iter_mut()
.find(|d| d.id == dag_id)
.and_then(|d| d.nodes.iter_mut().find(|n| n.state == State::Running))
else {
return false;
};
if node.step.as_deref() == Some(step) {
return false;
}
node.step = Some(step.to_owned());
true
}
/// Link a `build_logs` row to a specific `Running` node.
pub fn set_build_log_id(&self, dag_id: u64, node_id: NodeId, log_id: i64) -> bool {
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
let Some(node) = inner
.dags
.iter_mut()
.find(|d| d.id == dag_id)
.and_then(|d| d.node_mut(node_id))
else {
return false;
};
if node.state != State::Running {
return false;
}
node.build_log_id = Some(log_id);
true
}
/// Link a `build_logs` row to the DAG's currently-running node —
/// DAG-id-only compatibility surface (approval pipeline callbacks).
pub fn set_build_log_id_running(&self, dag_id: u64, log_id: i64) -> bool {
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
let Some(node) = inner
.dags
.iter_mut()
.find(|d| d.id == dag_id)
.and_then(|d| d.nodes.iter_mut().find(|n| n.state == State::Running))
else {
return false;
};
node.build_log_id = Some(log_id);
true
}
/// Snapshot every DAG for `/api/state` + `RebuildQueueChanged`.
pub fn snapshot(&self) -> Vec<DagView> {
let inner = self.inner.lock().expect("job_queue mutex poisoned");
inner.dags.iter().map(Dag::view).collect()
}
/// Number of live (non-terminal) DAGs — used by tests and
/// diagnostics.
#[cfg(test)]
pub fn live_count(&self) -> usize {
let inner = self.inner.lock().expect("job_queue mutex poisoned");
inner.dags.iter().filter(|d| !d.is_terminal()).count()
}
/// Keep only the newest `MAX_HISTORY_PER_TEMPLATE` terminal DAGs
/// per template; live DAGs are never evicted.
fn trim_history(inner: &mut Inner) {
let mut counts: HashMap<Template, usize> = HashMap::new();
let kept: Vec<Dag> = inner
.dags
.iter()
.rev()
.filter(|d| {
if !d.is_terminal() {
return true;
}
let n = counts.entry(d.template).or_insert(0);
*n += 1;
*n <= MAX_HISTORY_PER_TEMPLATE
})
.cloned()
.collect();
inner.dags = kept.into_iter().rev().collect();
}
}
/// Current unix timestamp in seconds.
fn now_unix() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.ok()
.and_then(|d| i64::try_from(d.as_secs()).ok())
.unwrap_or(0)
}

520
hive-c0re/src/job_queue/model.rs Normal file
View file

@ -0,0 +1,520 @@
//! Data model for the generic job-DAG queue: templates (what a DAG
//! *means*), node kinds (the primitive operations), dependency edges,
//! states, and the wire-facing `DagView` / `NodeView` snapshot shapes.
//!
//! Two levels: the **DAG** is the unit of dedup / cancel /
//! approval-resolution and the dashboard group; the **node** is the
//! unit of scheduling / execution / build-log / step label. See
//! `docs/coordinator.md::Job queue` for the full design.
use serde::{Deserialize, Serialize};
/// What a DAG *means* — the request-level shape. Wire strings match the
/// old `QueueKind` values (serialized as the `kind` field on `DagView`)
/// so the dashboard's glyph map keeps working.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum Template {
/// Rebuild one agent's container: `Prebuild → StopForUpdate → Swap
/// → Reconcile` (the tail `Reconcile` runs after `Swap` terminal,
/// ok *or* fail — the recovery-start).
Rebuild,
/// Bump meta flake locks: one `MetaLock` node; child `Rebuild`
/// DAGs fan out on completion for every affected agent.
MetaUpdate,
/// First-deploy spawn (approval-driven): `Create → WriteDropin →
/// Reconcile`.
Spawn,
/// Reserved for a future destroy integration — kept so the wire
/// shape doesn't need to change later.
#[allow(dead_code, reason = "wire shape — routed by a future PR")]
Destroy,
/// Boot-time config sweep: one `MetaLock` (hyperhive input,
/// non-fatal) node; stale agents' `Rebuild` DAGs fan out on
/// completion.
StartupSweep,
/// `StopForUpdate → Reconcile` — stop + converge back to `wanted`
/// (unchanged), i.e. a restart for a wanted-up agent.
Restart,
/// `WritePermFile → Prebuild → StopForUpdate → Swap → Reconcile` —
/// perm-file commit followed by the rebuild subgraph.
PermChange,
/// `Signal → Drain → Reconcile` with `wanted` set to `Offline` at
/// submit time: quiesce the harness, await the drain (bounded),
/// then the tail `Reconcile` performs the actual container stop.
GracefulStop,
/// Single `Reconcile` with `wanted` set to `Up` at submit time.
Start,
/// Single `Reconcile` with `wanted` set to `Offline` at submit time.
Stop,
/// Single `Reconcile` with `wanted` untouched — boot-time converge
/// of observed state to the persisted intent.
Reconcile,
}
impl Template {
pub fn as_str(self) -> &'static str {
match self {
Template::Rebuild => "rebuild",
Template::MetaUpdate => "meta_update",
Template::Spawn => "spawn",
Template::Destroy => "destroy",
Template::StartupSweep => "startup_sweep",
Template::Restart => "restart",
Template::PermChange => "perm_change",
Template::GracefulStop => "graceful_stop",
Template::Start => "start",
Template::Stop => "stop",
Template::Reconcile => "reconcile",
}
}
}
/// Where the submit request originated. Same variants + wire strings
/// as the old `QueueSource` — drives the "why" chip on the dashboard.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum Source {
/// Operator action (dashboard button, CLI, manager tool).
Manual,
/// Cascade child of a `MetaUpdate` DAG's fan-out; `parent_id`
/// points back at the originating meta-update.
MetaUpdate,
/// Boot-time submission (sweep parent, boot reconciles).
AutoUpdate,
/// Cascade child of a `StartupSweep` DAG's fan-out.
StartupSweep,
/// Crash recovery path (future use).
#[allow(dead_code, reason = "wire shape — used by a future feature")]
CrashRecover,
/// Operator approved a pending `Approval` row; `approval_id` on
/// the DAG points back at the source row.
Approval,
}
impl Source {
pub fn as_str(self) -> &'static str {
match self {
Source::Manual => "manual",
Source::MetaUpdate => "meta_update",
Source::AutoUpdate => "auto_update",
Source::StartupSweep => "startup_sweep",
Source::CrashRecover => "crash_recover",
Source::Approval => "approval",
}
}
}
/// Lifecycle state of a node — and, rolled up, of a DAG. Same wire
/// strings as the old `QueueState`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum State {
Queued,
Running,
Done,
Failed,
Cancelled,
}
impl State {
pub fn is_terminal(self) -> bool {
matches!(self, State::Done | State::Failed | State::Cancelled)
}
}
/// Kind-specific payload for `Template::PermChange` DAGs. Carried on
/// the DAG (not the node) so dedup can compare the perm *type*
/// discriminant. Identical to the old `rebuild_queue::PermPayload`.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum PermPayload {
/// Set the tool groups for one agent (`tool-groups.json`).
ToolGroups { groups: Vec<String> },
/// Set the capabilities for one agent (`capabilities.json`).
Capabilities { caps: Vec<String> },
/// Set both perm-types in one entry — the batch
/// `POST /api/permissions` path. `None` leaves that file untouched;
/// the worker commits whichever are present in one git commit,
/// then rebuilds once.
Combined {
groups: Option<Vec<String>>,
caps: Option<Vec<String>>,
},
}
impl PermPayload {
/// Dedup compares the perm *type*, not the value — a tool-groups
/// change and a capabilities change for the same agent are
/// distinct operations that must not collapse.
pub fn same_type(a: Option<&PermPayload>, b: Option<&PermPayload>) -> bool {
matches!(
(a, b),
(
Some(PermPayload::ToolGroups { .. }),
Some(PermPayload::ToolGroups { .. })
) | (
Some(PermPayload::Capabilities { .. }),
Some(PermPayload::Capabilities { .. })
) | (
Some(PermPayload::Combined { .. }),
Some(PermPayload::Combined { .. })
) | (None, None)
)
}
}
/// Node id, unique within its DAG (dense small ints assigned by the
/// template builders).
pub type NodeId = u32;
/// When a dependency edge is considered satisfied.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum DepWhen {
/// Dep must reach `Done`. A `Failed` / `Cancelled` dep cancels this
/// node (cancel-downstream).
AfterOk,
/// Dep must merely reach a terminal state (ok *or* fail). Used only
/// by `rebuild`'s tail `Reconcile` so the recovery-start runs even
/// when `Swap` failed.
AfterAny,
}
/// A dependency edge (intra-DAG only — cross-DAG ordering comes from
/// the per-agent lease + dedup, never from edges between DAGs).
#[derive(Debug, Clone, Copy, Serialize)]
pub struct Dep {
pub on: NodeId,
pub when: DepWhen,
}
/// The primitive operations — each kind maps to one executor fn in
/// `exec.rs`, a thin wrapper over existing `lifecycle.rs` / `meta.rs`
/// code. Concurrency is gated by two resource classes (see
/// [`NodeKind::needs_build_slot`] / [`NodeKind::needs_lease`]); the
/// meta *repo* is serialized by `meta::META_LOCK` inside the wrapped
/// functions themselves, which is why there is no `GitCommit` node —
/// a standalone commit node would open a dirty-working-tree window
/// between nodes that the fused `meta.rs` ops deliberately close.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum NodeKind {
/// Out-of-band toplevel build while the container keeps serving:
/// meta `sync_agents`, optional per-agent relock, then
/// `lifecycle::prebuild_toplevel`. `relock = false` only for
/// meta-update cascade rebuilds (re-locking would revert the bump
/// the cascade just committed).
Prebuild { relock: bool },
/// `nixos-container update` profile-swap (requires the container
/// stopped). Re-applies nspawn flags + resource limits first —
/// rebuild is the reconcile verb — and carries the post-rebuild
/// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan).
Swap,
/// First-spawn `nixos-container create` plus the pre-create
/// provisioning (proposed/applied repos, state subvolume, meta
/// registration).
Create,
/// Meta flake lock bump. `sweep = false`: `meta::lock_update`
/// (commit fused, under `META_LOCK`) with the DAG's `inputs`;
/// `sweep = true`: `meta::lock_update_hyperhive`, *non-fatal* (a
/// failed boot-time bump must not cancel the fan-out rebuilds).
/// On success the scheduler appends child `Rebuild` DAGs: the
/// precomputed `fanout` list when present (boot sweep), else the
/// post-bump affected set (`meta_update_cascade_agents`).
MetaLock {
sweep: bool,
fanout: Option<Vec<String>>,
},
/// Idempotent power converge: read `wanted` + observed state;
/// start if `Up` & down (with cold-start fallback), stop if
/// `Offline` & up, else noop.
Reconcile,
/// Mechanical `nixos-container stop` for the profile swap. Never
/// touches `wanted`. Noop if already stopped.
StopForUpdate,
/// Set the graceful-stop fence + kick the harness so it runs one
/// stop-checkpoint turn.
Signal,
/// Await the harness clearing the fence, bounded by
/// `GRACEFUL_STOP_TIMEOUT`. Resolves ok either way — the
/// downstream `Reconcile` performs the actual stop.
Drain,
/// `set_nspawn_flags` + `set_resource_limits` + daemon-reload.
WriteDropin,
/// Commit `tool-groups.json` / `capabilities.json` per the DAG's
/// `perm_payload` (commit fused under `META_LOCK`).
WritePermFile,
/// Opaque approval deploy pipeline (`ApplyCommit` /
/// `MergeConfigPr`): the two-phase prepare/finalize/abort meta
/// deploy stays inside `actions.rs` in v1 — deliberately not
/// modeled as scheduler nodes (see the design doc §9).
ApprovalDeploy,
}
impl NodeKind {
/// Wire string for `NodeView.kind`.
pub fn as_str(&self) -> &'static str {
match self {
NodeKind::Prebuild { .. } => "prebuild",
NodeKind::Swap => "swap",
NodeKind::Create => "create",
NodeKind::MetaLock { .. } => "meta_lock",
NodeKind::Reconcile => "reconcile",
NodeKind::StopForUpdate => "stop_for_update",
NodeKind::Signal => "signal",
NodeKind::Drain => "drain",
NodeKind::WriteDropin => "write_dropin",
NodeKind::WritePermFile => "write_perm_file",
NodeKind::ApprovalDeploy => "approval_deploy",
}
}
/// Nix-heavy kinds hold one of the `buildSlots` semaphore permits
/// for the node's duration.
pub fn needs_build_slot(&self) -> bool {
matches!(
self,
NodeKind::Prebuild { .. }
| NodeKind::Swap
| NodeKind::Create
| NodeKind::MetaLock { .. }
| NodeKind::ApprovalDeploy
)
}
/// Container-affecting kinds require the DAG to hold the agent's
/// lifecycle lease (acquired at the first such node, held until the
/// DAG is terminal). Lease-exempt kinds (`Prebuild`, `MetaLock`,
/// `WritePermFile`) touch the store / meta repo, not the running
/// container — which is exactly why a `Prebuild` can overlap
/// another DAG's work on the same agent.
pub fn needs_lease(&self) -> bool {
matches!(
self,
NodeKind::Swap
| NodeKind::Create
| NodeKind::Reconcile
| NodeKind::StopForUpdate
| NodeKind::Signal
| NodeKind::Drain
| NodeKind::WriteDropin
| NodeKind::ApprovalDeploy
)
}
}
/// One schedulable unit inside a DAG.
#[derive(Debug, Clone)]
pub struct Node {
pub id: NodeId,
pub kind: NodeKind,
pub deps: Vec<Dep>,
pub state: State,
/// Live sub-label while `Running` (kept for parity with the old
/// per-entry `step`).
pub step: Option<String>,
/// Row id of the `build_logs` entry this node opened (`Prebuild` /
/// `Swap` / `ApprovalDeploy`), for the dashboard's live-stream link.
pub build_log_id: Option<i64>,
pub started_at: Option<i64>,
pub finished_at: Option<i64>,
/// Populated when `state == Failed` (truncated by the queue).
pub error: Option<String>,
}
/// Submit-time spec for one node.
#[derive(Debug, Clone)]
pub struct NodeSpec {
pub kind: NodeKind,
pub deps: Vec<Dep>,
}
/// Submit-time spec for a whole DAG. Built by `templates.rs`; validated
/// (cycle rejection) and dedup'd by `JobQueue::submit`.
#[derive(Debug, Clone)]
pub struct DagSpec {
pub template: Template,
/// Primary target agent, or `"hyperhive"` for meta-level DAGs.
pub agent: String,
pub source: Source,
/// Free-form "why"; dedup appends "also requested by …" lines.
pub reason: String,
/// Cascade grouping (meta-update / sweep children).
pub parent_id: Option<u64>,
/// Fires the approval-resolution hook on DAG terminal.
pub approval_id: Option<i64>,
/// `MetaUpdate`-only: the inputs to bump (also part of the dedup
/// key for that template). Display copy lives on the DAG.
pub inputs: Vec<String>,
/// `PermChange`-only payload.
pub perm_payload: Option<PermPayload>,
/// Dashboard transient pill (and crash-watch suppression) held for
/// the lease window — from lease acquisition to DAG terminal.
pub transient: Option<crate::coordinator::TransientKind>,
pub nodes: Vec<NodeSpec>,
}
/// A live DAG in the queue.
#[derive(Debug, Clone)]
pub struct Dag {
pub id: u64,
pub template: Template,
pub agent: String,
pub source: Source,
pub reason: String,
pub parent_id: Option<u64>,
pub approval_id: Option<i64>,
pub inputs: Vec<String>,
pub perm_payload: Option<PermPayload>,
pub transient: Option<crate::coordinator::TransientKind>,
pub created_at: i64,
pub nodes: Vec<Node>,
/// Terminal roll-up already reported to the scheduler's hooks
/// (approval resolution, transient release). Internal bookkeeping,
/// never serialized.
pub terminal_reported: bool,
}
impl Dag {
/// Roll-up state: `Failed` if any node failed; else `Running` if
/// any running; else `Queued` if any queued; else `Cancelled` if
/// any cancelled; else `Done`.
pub fn rollup(&self) -> State {
let mut any_cancelled = false;
let mut any_queued = false;
let mut any_running = false;
for n in &self.nodes {
match n.state {
State::Failed => return State::Failed,
State::Running => any_running = true,
State::Queued => any_queued = true,
State::Cancelled => any_cancelled = true,
State::Done => {}
}
}
if any_running {
State::Running
} else if any_queued {
State::Queued
} else if any_cancelled {
State::Cancelled
} else {
State::Done
}
}
/// True when every node is terminal.
pub fn is_terminal(&self) -> bool {
self.nodes.iter().all(|n| n.state.is_terminal())
}
/// First failed node's error, for the roll-up `error` field.
pub fn first_error(&self) -> Option<&str> {
self.nodes
.iter()
.find(|n| n.state == State::Failed)
.and_then(|n| n.error.as_deref())
}
pub fn node(&self, id: NodeId) -> Option<&Node> {
self.nodes.iter().find(|n| n.id == id)
}
pub fn node_mut(&mut self, id: NodeId) -> Option<&mut Node> {
self.nodes.iter_mut().find(|n| n.id == id)
}
}
/// Wire shape of one node inside a `DagView`.
#[derive(Debug, Clone, Serialize)]
pub struct NodeView {
pub id: NodeId,
/// Flattened `NodeKind` tag ("prebuild", "swap", …).
pub kind: &'static str,
/// Ids of the nodes this one waits for.
pub deps: Vec<NodeId>,
pub state: State,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub step: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub build_log_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub started_at: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub finished_at: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
}
/// Wire shape of a DAG, serialized onto `RebuildQueueChanged` and the
/// `/api/state` snapshot. DAG-level fields mirror the old `QueueEntry`
/// names (`kind` = template string, roll-up `state`); everything
/// per-node — step labels, build-log links, errors, timestamps —
/// appears exactly once, inside `nodes`.
#[derive(Debug, Clone, Serialize)]
pub struct DagView {
pub id: u64,
pub agent: String,
/// Template wire string — same values the old `kind` field used.
pub kind: Template,
/// Roll-up state (see [`Dag::rollup`]).
pub state: State,
pub source: Source,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub parent_id: Option<u64>,
pub reason: String,
pub enqueued_at: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub started_at: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub finished_at: Option<i64>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub inputs: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub approval_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub perm_payload: Option<PermPayload>,
pub nodes: Vec<NodeView>,
}
impl Dag {
pub fn view(&self) -> DagView {
let started_at = self.nodes.iter().filter_map(|n| n.started_at).min();
let finished_at = if self.is_terminal() {
self.nodes.iter().filter_map(|n| n.finished_at).max()
} else {
None
};
DagView {
id: self.id,
agent: self.agent.clone(),
kind: self.template,
state: self.rollup(),
source: self.source,
parent_id: self.parent_id,
reason: self.reason.clone(),
enqueued_at: self.created_at,
started_at,
finished_at,
inputs: self.inputs.clone(),
approval_id: self.approval_id,
perm_payload: self.perm_payload.clone(),
nodes: self
.nodes
.iter()
.map(|n| NodeView {
id: n.id,
kind: n.kind.as_str(),
deps: n.deps.iter().map(|d| d.on).collect(),
state: n.state,
step: n.step.clone(),
build_log_id: n.build_log_id,
started_at: n.started_at,
finished_at: n.finished_at,
error: n.error.clone(),
})
.collect(),
}
}
}

150
hive-c0re/src/job_queue/scheduler.rs Normal file
View file

@ -0,0 +1,150 @@
//! The single scheduler task that drives all DAGs: claim every ready
//! node (as many as the build slots / leases allow), spawn one
//! executor task per claim, and on any completion re-evaluate.
//! Concurrency comes from the build-slot count, not multiple workers.
//!
//! Also owns the two DAG-lifetime side channels the sync queue core
//! can't hold itself:
//! - the per-DAG transient guard (dashboard pill + crash-watch
//! suppression), created when a DAG acquires its agent lease and
//! dropped when the DAG settles terminal;
//! - the `MetaLock` fan-out: appending child `Rebuild` DAGs once the
//! lock bump lands, so children build against the post-bump lock
//! (and a failed bump fans out nothing — replacing the old
//! pre-enqueue + cancel-children dance).
use std::collections::HashMap;
use std::sync::Arc;
use super::exec::{self, NodeOutput};
use super::{Claim, Source, Template, templates};
use crate::coordinator::Coordinator;
struct NodeDone {
claim: Claim,
result: anyhow::Result<NodeOutput>,
}
/// Scheduler loop. Spawned once at hive-c0re startup from `main.rs`.
///
/// Shutdown semantics: subscribes to `coord.shutdown_rx()`. On a true
/// signal the loop exits immediately; already-running node tasks ride
/// the runtime down with the process, and pending `Queued` DAGs are
/// dropped — desired state is re-derived on next boot (boot sweep +
/// reconcile), so the in-memory queue is deliberately not durable.
pub async fn run_worker(coord: Arc<Coordinator>) {
let mut shutdown = coord.shutdown_rx();
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<NodeDone>();
// DAG id → transient guard held for the lease window.
let mut transients: HashMap<u64, crate::coordinator::TransientGuard> = HashMap::new();
loop {
let claims = coord.job_queue.claim_ready();
if !claims.is_empty() {
for claim in claims {
if claim.lease_acquired
&& let Some(kind) = claim.transient
{
transients.insert(claim.dag_id, coord.transient_guard(&claim.agent, kind));
}
tracing::info!(
dag = claim.dag_id,
node = claim.node_id,
kind = claim.kind.as_str(),
agent = %claim.agent,
template = claim.template.as_str(),
"job_queue: node running"
);
let coord = Arc::clone(&coord);
let tx = tx.clone();
tokio::spawn(async move {
let result = exec::run_node(&coord, &claim).await;
// Send failure = scheduler gone (shutdown); drop.
let _ = tx.send(NodeDone { claim, result });
});
}
coord.emit_rebuild_queue_snapshot();
continue;
}
tokio::select! {
biased;
res = shutdown.changed() => {
if res.is_err() || *shutdown.borrow() {
tracing::info!("job_queue: scheduler exiting on shutdown");
return;
}
}
Some(done) = rx.recv() => {
handle_completion(&coord, &mut transients, done).await;
}
() = coord.job_queue.notify.notified() => {}
}
}
}
async fn handle_completion(
coord: &Arc<Coordinator>,
transients: &mut HashMap<u64, crate::coordinator::TransientGuard>,
done: NodeDone,
) {
let NodeDone { claim, result } = done;
let (queue_result, fanout) = match result {
Ok(output) => {
tracing::info!(
dag = claim.dag_id,
node = claim.node_id,
"job_queue: node done"
);
(Ok(()), output.fanout)
}
Err(e) => {
let msg = format!("{e:#}");
tracing::warn!(
dag = claim.dag_id,
node = claim.node_id,
kind = claim.kind.as_str(),
agent = %claim.agent,
error = %msg,
"job_queue: node failed"
);
(Err(msg), Vec::new())
}
};
let report = coord
.job_queue
.complete_node(claim.dag_id, claim.node_id, queue_result);
if !fanout.is_empty() {
let specs = fanout_specs(&claim, fanout);
coord.job_queue.append_children(specs);
}
for terminal in report.terminal {
// Drop the lease-window transient guard, then let the hook
// fire approval resolution / failure events.
transients.remove(&terminal.dag_id);
exec::on_dag_terminal(coord, &terminal).await;
}
coord.emit_rebuild_queue_snapshot();
}
/// Child `Rebuild` specs for a completed `MetaLock` fan-out, grouped
/// under the parent via `parent_id`. Meta-update children skip the
/// per-agent relock (it would revert the bump the parent just
/// committed); sweep children relock like a manual rebuild.
fn fanout_specs(claim: &Claim, agents: Vec<String>) -> Vec<super::DagSpec> {
let sweep = claim.template == Template::StartupSweep;
let (source, relock) = if sweep {
(Source::StartupSweep, true)
} else {
(Source::MetaUpdate, false)
};
let reason = if sweep {
"startup sweep".to_owned()
} else if let Some(approval_id) = claim.approval_id {
format!("approval #{approval_id} meta input cascade")
} else {
"meta-update cascade".to_owned()
};
agents
.into_iter()
.map(|agent| templates::rebuild(&agent, source, reason.clone(), Some(claim.dag_id), relock))
.collect()
}

121
hive-c0re/src/job_queue/submit.rs Normal file
View file

@ -0,0 +1,121 @@
//! Request-level submit API — the surface the dashboard POST handlers,
//! the MCP socket handlers, and `hivectl` paths call. Owns the
//! submit-time side effects the DAG templates deliberately don't:
//! writing the durable `wanted` power intent (synchronously,
//! last-writer-wins) before the DAG whose `Reconcile` reads it, and
//! upgrading a stale start to a full rebuild. Every helper emits a
//! fresh queue snapshot so the dashboard shows the new DAG immediately.
use std::sync::Arc;
use super::{Source, Template, templates};
use crate::coordinator::Coordinator;
use crate::power::Wanted;
fn submit_and_emit(coord: &Arc<Coordinator>, spec: super::DagSpec) -> u64 {
let id = coord
.job_queue
.submit(spec)
.expect("template-built dag specs are acyclic");
coord.emit_rebuild_queue_snapshot();
id
}
fn set_wanted(coord: &Arc<Coordinator>, agent: &str, wanted: Wanted) {
if let Err(e) = coord.power.set(agent, wanted) {
tracing::warn!(%agent, wanted = wanted.as_str(), error = ?e, "agent_power: set failed");
}
}
/// Manual/approval-independent rebuild (always relocks the agent's
/// meta input — cascade children are built by the scheduler's fan-out
/// instead of this surface).
pub fn rebuild(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
submit_and_emit(coord, templates::rebuild(agent, source, reason, None, true))
}
/// Restart: mechanical stop + converge back to `wanted` (unchanged).
pub fn restart(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
submit_and_emit(coord, templates::restart(agent, source, reason))
}
/// Start: persist `wanted = Up`, then reconcile. A stale rev marker
/// upgrades the start to a full rebuild (whose tail `Reconcile` does
/// the start) so the container always comes up on current derivations
/// — the old fast-lane `run_start` upgrade, moved to submit time.
pub fn start(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
set_wanted(coord, agent, Wanted::Up);
let stored = std::fs::read_to_string(crate::auto_update::rev_marker_path(agent)).ok();
let stale = crate::auto_update::current_flake_rev(&coord.hyperhive_flake)
.is_some_and(|rev| stored.as_deref() != Some(rev.as_str()));
if stale {
tracing::info!(%agent, "start: rev stale — upgrading to rebuild+start");
return submit_and_emit(
coord,
templates::rebuild(
agent,
source,
format!("{reason} (stale — rebuild+start)"),
None,
true,
),
);
}
submit_and_emit(
coord,
templates::reconcile_only(
Template::Start,
agent,
source,
reason,
Some(crate::coordinator::TransientKind::Starting),
),
)
}
/// Hard stop: persist `wanted = Offline`, then reconcile (kill +
/// unregister + `Killed` event).
pub fn stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
set_wanted(coord, agent, Wanted::Offline);
submit_and_emit(
coord,
templates::reconcile_only(
Template::Stop,
agent,
source,
reason,
Some(crate::coordinator::TransientKind::Stopping),
),
)
}
/// Graceful stop: persist `wanted = Offline`, then signal → drain →
/// reconcile (the actual stop).
pub fn graceful_stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
set_wanted(coord, agent, Wanted::Offline);
submit_and_emit(coord, templates::graceful_stop(agent, source, reason))
}
/// Perm change: commit the JSON file(s) then rebuild.
pub fn perm_change(
coord: &Arc<Coordinator>,
agent: &str,
source: Source,
reason: String,
payload: super::PermPayload,
) -> u64 {
submit_and_emit(
coord,
templates::perm_change(agent, source, reason, payload),
)
}
/// Meta-input lock bump; cascade rebuilds fan out on completion.
pub fn meta_update(
coord: &Arc<Coordinator>,
inputs: Vec<String>,
source: Source,
reason: String,
) -> u64 {
submit_and_emit(coord, templates::meta_update(inputs, source, reason, None))
}

333
hive-c0re/src/job_queue/templates.rs Normal file
View file

@ -0,0 +1,333 @@
//! DAG shape builders — every operation as a template over the shared
//! node primitives — plus submit-time cycle validation (petgraph is
//! confined to this validation; the runtime store stays the plain
//! `Vec<Node>` + `deps`).
//!
//! ```text
//! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a)
//! graceful-stop(a): [wanted=Offline] Signal(a) → Drain(a) → Reconcile(a)
//! restart(a): StopForUpdate(a) → Reconcile(a) (wanted unchanged)
//! start(a): [wanted=Up] Reconcile(a)
//! stop(a): [wanted=Offline] Reconcile(a)
//! spawn(a): [wanted=Up] Create(a) → WriteDropin(a) → Reconcile(a)
//! perm-change(a): WritePermFile(a) → «rebuild subgraph»
//! meta-update(inp): MetaLock(inp) → «fan-out rebuild(a) per affected a»
//! startup sweep: MetaLock(hyperhive, non-fatal) → «fan-out rebuild(stale a)»
//! ```
use anyhow::{Result, bail};
use super::model::{DagSpec, Dep, DepWhen, NodeKind, NodeSpec, PermPayload, Source, Template};
use crate::coordinator::TransientKind;
/// After-ok edge on the previous node — the common chain link.
fn after_ok(on: u32) -> Vec<Dep> {
vec![Dep {
on,
when: DepWhen::AfterOk,
}]
}
/// The rebuild node chain. `Reconcile` deps on `Swap` with `AfterAny`:
/// it must run even when the profile swap failed, so a previously-up
/// agent comes back on its old config (today's recovery-start). This
/// is the only `AfterAny` edge in v1.
fn rebuild_nodes(relock: bool, base: u32) -> Vec<NodeSpec> {
vec![
NodeSpec {
kind: NodeKind::Prebuild { relock },
deps: if base == 0 {
Vec::new()
} else {
after_ok(base - 1)
},
},
NodeSpec {
kind: NodeKind::StopForUpdate,
deps: after_ok(base),
},
NodeSpec {
kind: NodeKind::Swap,
deps: after_ok(base + 1),
},
NodeSpec {
kind: NodeKind::Reconcile,
deps: vec![Dep {
on: base + 2,
when: DepWhen::AfterAny,
}],
},
]
}
/// One uniform rebuild shape — no `was_running` branch. `StopForUpdate`
/// noops when already down; the tail `Reconcile` auto-noops the start
/// when `wanted = Offline` (a rebuild of a deliberately-stopped agent
/// leaves it stopped). `relock = false` only for meta-update cascade
/// children.
pub fn rebuild(
agent: &str,
source: Source,
reason: String,
parent_id: Option<u64>,
relock: bool,
) -> DagSpec {
DagSpec {
template: Template::Rebuild,
agent: agent.to_owned(),
source,
reason,
parent_id,
approval_id: None,
inputs: Vec::new(),
perm_payload: None,
transient: Some(TransientKind::Rebuilding),
nodes: rebuild_nodes(relock, 0),
}
}
/// Approval-driven deploy (`ApplyCommit` / `MergeConfigPr`): the whole
/// two-phase pipeline stays one opaque node in v1 (design doc §9) —
/// wire-visible as a `rebuild` card like today.
pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec {
DagSpec {
template: Template::Rebuild,
agent: agent.to_owned(),
source: Source::Approval,
reason,
parent_id: None,
approval_id: Some(approval_id),
inputs: Vec::new(),
perm_payload: None,
transient: Some(TransientKind::Rebuilding),
nodes: vec![NodeSpec {
kind: NodeKind::ApprovalDeploy,
deps: Vec::new(),
}],
}
}
/// Graceful stop: cheap `Signal` fires immediately (no build slot), the
/// `Drain` awaits the harness checkpoint (bounded), and the tail
/// `Reconcile` performs the actual container stop — the caller sets
/// `wanted = Offline` at submit time. A whole-hive graceful stop
/// therefore signals every agent up front and overlaps every drain,
/// replacing the old detached-watcher thread structurally.
pub fn graceful_stop(agent: &str, source: Source, reason: String) -> DagSpec {
DagSpec {
template: Template::GracefulStop,
agent: agent.to_owned(),
source,
reason,
parent_id: None,
approval_id: None,
inputs: Vec::new(),
perm_payload: None,
transient: Some(TransientKind::Stopping),
nodes: vec![
NodeSpec {
kind: NodeKind::Signal,
deps: Vec::new(),
},
NodeSpec {
kind: NodeKind::Drain,
deps: after_ok(0),
},
NodeSpec {
kind: NodeKind::Reconcile,
deps: after_ok(1),
},
],
}
}
/// Restart: mechanical stop, then converge back to `wanted`
/// (unchanged) — a stop + start for a wanted-up agent.
pub fn restart(agent: &str, source: Source, reason: String) -> DagSpec {
DagSpec {
template: Template::Restart,
agent: agent.to_owned(),
source,
reason,
parent_id: None,
approval_id: None,
inputs: Vec::new(),
perm_payload: None,
transient: Some(TransientKind::Restarting),
nodes: vec![
NodeSpec {
kind: NodeKind::StopForUpdate,
deps: Vec::new(),
},
NodeSpec {
kind: NodeKind::Reconcile,
deps: after_ok(0),
},
],
}
}
/// Single-`Reconcile` DAG: `Start` / `Stop` (caller writes `wanted`
/// first) and the boot-time `Reconcile` converge (wanted untouched).
pub fn reconcile_only(
template: Template,
agent: &str,
source: Source,
reason: String,
transient: Option<TransientKind>,
) -> DagSpec {
DagSpec {
template,
agent: agent.to_owned(),
source,
reason,
parent_id: None,
approval_id: None,
inputs: Vec::new(),
perm_payload: None,
transient,
nodes: vec![NodeSpec {
kind: NodeKind::Reconcile,
deps: Vec::new(),
}],
}
}
/// First-deploy spawn (approval-driven): pre-start provisioning +
/// `nixos-container create`, drop-in write, then `Reconcile` starts the
/// container (`wanted = Up` written at approve time).
pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
DagSpec {
template: Template::Spawn,
agent: agent.to_owned(),
source: Source::Approval,
reason,
parent_id: None,
approval_id: Some(approval_id),
inputs: Vec::new(),
perm_payload: None,
transient: Some(TransientKind::Spawning),
nodes: vec![
NodeSpec {
kind: NodeKind::Create,
deps: Vec::new(),
},
NodeSpec {
kind: NodeKind::WriteDropin,
deps: after_ok(0),
},
NodeSpec {
kind: NodeKind::Reconcile,
deps: after_ok(1),
},
],
}
}
/// Perm change: commit the JSON file(s), then the rebuild subgraph so
/// the updated `HIVE_TOOL_GROUPS` / `HIVE_CAPABILITIES` env var takes
/// effect in the container.
pub fn perm_change(agent: &str, source: Source, reason: String, payload: PermPayload) -> DagSpec {
let mut nodes = vec![NodeSpec {
kind: NodeKind::WritePermFile,
deps: Vec::new(),
}];
nodes.extend(rebuild_nodes(true, 1));
DagSpec {
template: Template::PermChange,
agent: agent.to_owned(),
source,
reason,
parent_id: None,
approval_id: None,
inputs: Vec::new(),
perm_payload: Some(payload),
transient: Some(TransientKind::Rebuilding),
nodes,
}
}
/// Meta-input lock bump. Child `Rebuild` DAGs fan out on completion —
/// appended *after* the bump lands so their prebuilds run against the
/// post-bump lock (and so a failed bump simply fans out nothing,
/// replacing the old pre-enqueue + `cancel_children` dance).
pub fn meta_update(
inputs: Vec<String>,
source: Source,
reason: String,
approval_id: Option<i64>,
) -> DagSpec {
DagSpec {
template: Template::MetaUpdate,
agent: "hyperhive".to_owned(),
source,
reason,
parent_id: None,
approval_id,
inputs,
perm_payload: None,
transient: None,
nodes: vec![NodeSpec {
kind: NodeKind::MetaLock {
sweep: false,
fanout: None,
},
deps: Vec::new(),
}],
}
}
/// Boot-time sweep parent: bump meta's hyperhive input (non-fatal),
/// then fan out `Rebuild` children for the precomputed stale agent
/// list (topology-sorted by the caller).
pub fn startup_sweep(reason: String, stale_agents: Vec<String>) -> DagSpec {
DagSpec {
template: Template::StartupSweep,
agent: "hyperhive".to_owned(),
source: Source::AutoUpdate,
reason,
parent_id: None,
approval_id: None,
inputs: Vec::new(),
perm_payload: None,
transient: None,
nodes: vec![NodeSpec {
kind: NodeKind::MetaLock {
sweep: true,
fanout: Some(stale_agents),
},
deps: Vec::new(),
}],
}
}
/// Validate a spec before it enters the queue: node ids are dense
/// (index = id), deps reference existing nodes, and the dep graph is
/// acyclic (petgraph `toposort`). Rejecting cycles here fixes the old
/// queue's documented "circular dep silently deadlocks forever" caveat.
pub fn validate(spec: &DagSpec) -> Result<()> {
if spec.nodes.is_empty() {
bail!("dag spec {:?} has no nodes", spec.template);
}
let n = spec.nodes.len();
let mut graph = petgraph::graph::DiGraph::<u32, ()>::new();
let idx: Vec<_> = (0..n)
.map(|i| graph.add_node(u32::try_from(i).unwrap_or(u32::MAX)))
.collect();
for (i, node) in spec.nodes.iter().enumerate() {
for dep in &node.deps {
let Some(&dep_idx) = idx.get(dep.on as usize) else {
bail!(
"dag spec {:?} node {i} depends on unknown node {}",
spec.template,
dep.on
);
};
graph.add_edge(dep_idx, idx[i], ());
}
}
if petgraph::algo::toposort(&graph, None).is_err() {
bail!("dag spec {:?} contains a dependency cycle", spec.template);
}
Ok(())
}

823
hive-c0re/src/job_queue/tests.rs Normal file
View file

@ -0,0 +1,823 @@
//! Queue-core unit tests: dedup, cycle rejection, resource
//! serialization (build slots / per-agent leases), lease-exempt
//! overlap, FIFO fairness, cancel semantics, `AfterAny` failure
//! routing, fan-out, and history retention. All synchronous — the
//! scheduler's async loop is a thin claim/complete pump over the same
//! methods exercised here.
use super::model::{Dep, DepWhen, NodeKind, NodeSpec};
use super::*;
fn submit(q: &JobQueue, spec: DagSpec) -> u64 {
q.submit(spec).expect("valid spec")
}
fn rebuild(agent: &str, reason: &str) -> DagSpec {
templates::rebuild(agent, Source::Manual, reason.to_owned(), None, true)
}
/// Claim helper asserting exactly one node comes back.
fn claim_one(q: &JobQueue) -> Claim {
let mut claims = q.claim_ready();
assert_eq!(
claims.len(),
1,
"expected exactly one claim, got {claims:?}"
);
claims.pop().expect("one claim")
}
fn state_of(q: &JobQueue, dag_id: u64) -> State {
q.snapshot()
.iter()
.find(|d| d.id == dag_id)
.expect("dag present")
.state
}
// ---- submit / dedup ----
#[test]
fn submit_assigns_distinct_ids() {
let q = JobQueue::new(1);
let a = submit(&q, rebuild("agent-a", "first"));
let b = submit(&q, rebuild("agent-b", "second"));
assert_ne!(a, b);
assert_eq!(q.snapshot().len(), 2);
}
#[test]
fn dedup_pending_same_template_and_agent() {
let q = JobQueue::new(1);
let a = submit(&q, rebuild("agent-a", "first"));
let b = submit(&q, rebuild("agent-a", "auto sweep"));
assert_eq!(a, b, "dedup should return existing id");
let snap = q.snapshot();
assert_eq!(snap.len(), 1);
assert!(snap[0].reason.contains("first"));
assert!(snap[0].reason.contains("auto sweep"));
}
#[test]
fn dedup_does_not_apply_across_templates_or_agents() {
let q = JobQueue::new(1);
let a = submit(&q, rebuild("agent-a", "r"));
let b = submit(&q, rebuild("agent-b", "r"));
let c = submit(
&q,
templates::restart("agent-a", Source::Manual, "r".to_owned()),
);
assert_ne!(a, b);
assert_ne!(a, c);
assert_eq!(q.snapshot().len(), 3);
}
#[test]
fn dedup_skips_running_dags() {
let q = JobQueue::new(1);
let a = submit(&q, rebuild("agent-a", "first"));
let claim = claim_one(&q); // Prebuild running
assert_eq!(claim.dag_id, a);
// While the original runs, re-submit is legitimate new work.
let again = submit(&q, rebuild("agent-a", "config bumped during build"));
assert_ne!(a, again);
assert_eq!(q.snapshot().len(), 2);
}
#[test]
fn meta_update_dedup_matches_inputs() {
let q = JobQueue::new(1);
let a = submit(
&q,
templates::meta_update(
vec!["nixpkgs".to_owned()],
Source::Manual,
"first".to_owned(),
None,
),
);
let b = submit(
&q,
templates::meta_update(
vec!["nixpkgs".to_owned()],
Source::Manual,
"duplicate click".to_owned(),
None,
),
);
assert_eq!(a, b, "identical-inputs meta-updates should dedup");
let c = submit(
&q,
templates::meta_update(
vec!["agent-bitburner/bitburner-agent".to_owned()],
Source::Manual,
"bump agent".to_owned(),
None,
),
);
assert_ne!(a, c, "different-inputs meta-updates must NOT dedup");
assert_eq!(q.snapshot().len(), 2);
}
#[test]
fn approval_dags_dedup_only_on_matching_id() {
let q = JobQueue::new(1);
let a = submit(
&q,
templates::approval_deploy("agent-a", 1, "approval #1".to_owned()),
);
let b = submit(
&q,
templates::approval_deploy("agent-a", 2, "approval #2".to_owned()),
);
assert_ne!(a, b, "distinct approvals must not collapse");
// Rapid double-click on the same approval IS a single op.
let c = submit(
&q,
templates::approval_deploy("agent-a", 1, "approval #1 (dup)".to_owned()),
);
assert_eq!(a, c);
assert_eq!(q.snapshot().len(), 2);
}
#[test]
fn perm_change_dedup_respects_perm_type() {
let q = JobQueue::new(1);
let groups = templates::perm_change(
"agent-a",
Source::Manual,
"groups".to_owned(),
PermPayload::ToolGroups { groups: vec![] },
);
let caps = templates::perm_change(
"agent-a",
Source::Manual,
"caps".to_owned(),
PermPayload::Capabilities { caps: vec![] },
);
let a = submit(&q, groups.clone());
let b = submit(&q, caps);
assert_ne!(a, b, "tool-groups vs capabilities must not collapse");
let c = submit(&q, groups);
assert_eq!(a, c, "same perm type dedups");
}
/// A `MetaUpdate` cascade `Rebuild` (with `parent_id = Some(meta_id)`)
/// must NOT dedup into a queued `Rebuild` with a different
/// `parent_id` (e.g. from a startup sweep) — without the guard the
/// cascade child would be swallowed and the agent never rebuilt
/// against the post-bump meta.
#[test]
fn dedup_respects_parent_id() {
let q = JobQueue::new(1);
let sweep = submit(&q, templates::startup_sweep("boot".to_owned(), vec![]));
let sweep_child = submit(
&q,
templates::rebuild(
"alice",
Source::StartupSweep,
"startup sweep".to_owned(),
Some(sweep),
true,
),
);
let meta = submit(
&q,
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
);
let cascade_child = submit(
&q,
templates::rebuild(
"alice",
Source::MetaUpdate,
"meta-update cascade".to_owned(),
Some(meta),
false,
),
);
assert_ne!(sweep_child, cascade_child);
let rebuilds = q
.snapshot()
.iter()
.filter(|d| d.kind == Template::Rebuild && d.agent == "alice")
.count();
assert_eq!(rebuilds, 2, "both rebuilds must be present");
}
// ---- cycle rejection ----
#[test]
fn cyclic_dag_is_rejected_at_submit() {
let q = JobQueue::new(1);
let mut spec = rebuild("agent-a", "cyclic");
// 0 → 1 → 0 cycle.
spec.nodes = vec![
NodeSpec {
kind: NodeKind::StopForUpdate,
deps: vec![Dep {
on: 1,
when: DepWhen::AfterOk,
}],
},
NodeSpec {
kind: NodeKind::Reconcile,
deps: vec![Dep {
on: 0,
when: DepWhen::AfterOk,
}],
},
];
assert!(q.submit(spec).is_err(), "cyclic spec must be refused");
assert!(q.snapshot().is_empty());
}
#[test]
fn unknown_dep_is_rejected_at_submit() {
let q = JobQueue::new(1);
let mut spec = rebuild("agent-a", "bad dep");
spec.nodes = vec![NodeSpec {
kind: NodeKind::Reconcile,
deps: vec![Dep {
on: 9,
when: DepWhen::AfterOk,
}],
}];
assert!(q.submit(spec).is_err());
}
// ---- dependency order within a DAG ----
#[test]
fn rebuild_chain_claims_in_dep_order() {
let q = JobQueue::new(1);
let id = submit(&q, rebuild("agent-a", "r"));
for expected in ["prebuild", "stop_for_update", "swap", "reconcile"] {
let c = claim_one(&q);
assert_eq!(c.dag_id, id);
assert_eq!(c.kind.as_str(), expected);
assert!(
q.claim_ready().is_empty(),
"chain must serialize: nothing ready while {expected} runs"
);
q.complete_node(id, c.node_id, Ok(()));
}
assert_eq!(state_of(&q, id), State::Done);
}
// ---- build slots ----
#[test]
fn build_slot_serializes_nix_heavy_nodes() {
let q = JobQueue::new(1);
let a = submit(&q, rebuild("agent-a", "r"));
let b = submit(&q, rebuild("agent-b", "r"));
let first = claim_one(&q); // a's Prebuild takes the only slot
assert_eq!(first.dag_id, a);
assert_eq!(first.kind.as_str(), "prebuild");
q.complete_node(a, first.node_id, Ok(()));
// With the slot free again, FIFO gives... a's StopForUpdate is
// slot-free (lease) and b's Prebuild takes the slot — both run.
let claims = q.claim_ready();
let kinds: Vec<(u64, &str)> = claims.iter().map(|c| (c.dag_id, c.kind.as_str())).collect();
assert!(kinds.contains(&(a, "stop_for_update")));
assert!(kinds.contains(&(b, "prebuild")));
assert_eq!(claims.len(), 2);
}
#[test]
fn two_build_slots_run_two_prebuilds() {
let q = JobQueue::new(2);
submit(&q, rebuild("agent-a", "r"));
submit(&q, rebuild("agent-b", "r"));
let claims = q.claim_ready();
assert_eq!(claims.len(), 2, "two slots → two concurrent prebuilds");
assert!(claims.iter().all(|c| c.kind.as_str() == "prebuild"));
}
#[test]
fn fifo_fairness_for_the_slot() {
let q = JobQueue::new(1);
let a = submit(&q, rebuild("agent-a", "r"));
let b = submit(&q, rebuild("agent-b", "r"));
let c = submit(&q, rebuild("agent-c", "r"));
let first = claim_one(&q);
assert_eq!(first.dag_id, a, "submit order wins the slot");
q.complete_node(a, first.node_id, Ok(()));
let next: Vec<u64> = q.claim_ready().iter().map(|cl| cl.dag_id).collect();
assert!(next.contains(&b), "b's prebuild before c's");
assert!(!next.contains(&c));
}
// ---- per-agent lease ----
#[test]
fn lease_serializes_two_lifecycle_dags_for_same_agent() {
let q = JobQueue::new(4);
let restart = submit(
&q,
templates::restart("agent-a", Source::Manual, "restart".to_owned()),
);
let stop = submit(
&q,
templates::reconcile_only(
Template::Stop,
"agent-a",
Source::Manual,
"stop".to_owned(),
None,
),
);
// Restart's StopForUpdate acquires the lease; stop's Reconcile
// must wait even though slots are free.
let first = claim_one(&q);
assert_eq!(first.dag_id, restart);
assert!(first.lease_acquired);
q.complete_node(restart, first.node_id, Ok(()));
// Same DAG keeps the lease for its Reconcile.
let second = claim_one(&q);
assert_eq!(second.dag_id, restart);
assert!(!second.lease_acquired, "lease already held by this DAG");
q.complete_node(restart, second.node_id, Ok(()));
// Restart terminal → lease released → stop's Reconcile runs.
let third = claim_one(&q);
assert_eq!(third.dag_id, stop);
q.complete_node(stop, third.node_id, Ok(()));
assert_eq!(state_of(&q, restart), State::Done);
assert_eq!(state_of(&q, stop), State::Done);
}
#[test]
fn lease_exempt_prebuild_overlaps_other_dag_on_same_agent() {
let q = JobQueue::new(2);
submit(&q, rebuild("agent-a", "rebuild"));
let stop = submit(
&q,
templates::reconcile_only(
Template::Stop,
"agent-a",
Source::Manual,
"stop".to_owned(),
None,
),
);
// Prebuild is lease-exempt: the stop's Reconcile takes the lease
// and runs concurrently with the rebuild's out-of-band nix build.
let claims = q.claim_ready();
let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect();
assert!(kinds.contains(&"prebuild"));
assert!(kinds.contains(&"reconcile"));
// But the rebuild's StopForUpdate must then wait for the stop DAG
// to finish (lease).
let prebuild = claims
.iter()
.find(|c| c.kind.as_str() == "prebuild")
.expect("prebuild claim")
.clone();
q.complete_node(prebuild.dag_id, prebuild.node_id, Ok(()));
assert!(
q.claim_ready().is_empty(),
"StopForUpdate blocked while stop DAG holds the lease"
);
let reconcile = claims
.iter()
.find(|c| c.kind.as_str() == "reconcile")
.expect("reconcile claim")
.clone();
q.complete_node(stop, reconcile.node_id, Ok(()));
let next = claim_one(&q);
assert_eq!(next.kind.as_str(), "stop_for_update");
}
#[test]
fn agents_do_not_contend_on_each_others_leases() {
let q = JobQueue::new(4);
submit(
&q,
templates::restart("agent-a", Source::Manual, "r".to_owned()),
);
submit(
&q,
templates::restart("agent-b", Source::Manual, "r".to_owned()),
);
let claims = q.claim_ready();
assert_eq!(claims.len(), 2, "different agents run concurrently");
}
// ---- failure: cancel-downstream + AfterAny ----
#[test]
fn failed_node_cancels_downstream_but_afterany_reconcile_runs() {
let q = JobQueue::new(1);
let id = submit(&q, rebuild("agent-a", "r"));
let prebuild = claim_one(&q);
q.complete_node(id, prebuild.node_id, Err("nix build exploded".to_owned()));
// StopForUpdate + Swap are cancelled (AfterOk on a failed chain);
// the AfterAny Reconcile still runs once Swap is terminal.
let reconcile = claim_one(&q);
assert_eq!(reconcile.kind.as_str(), "reconcile");
q.complete_node(id, reconcile.node_id, Ok(()));
let snap = q.snapshot();
let dag = snap.iter().find(|d| d.id == id).expect("dag");
assert_eq!(dag.state, State::Failed, "roll-up failed");
let by_kind = |k: &str| {
dag.nodes
.iter()
.find(|n| n.kind == k)
.expect("node present")
.state
};
assert_eq!(by_kind("prebuild"), State::Failed);
assert_eq!(by_kind("stop_for_update"), State::Cancelled);
assert_eq!(by_kind("swap"), State::Cancelled);
assert_eq!(by_kind("reconcile"), State::Done);
assert_eq!(
dag.nodes
.iter()
.find(|n| n.kind == "prebuild")
.and_then(|n| n.error.as_deref()),
Some("nix build exploded")
);
}
/// The swap-failure recovery: `Swap` fails → the `AfterAny` edge still
/// runs `Reconcile`, which brings a wanted-up agent back on its old
/// config.
#[test]
fn swap_failure_still_runs_reconcile() {
let q = JobQueue::new(1);
let id = submit(&q, rebuild("agent-a", "r"));
for _ in 0..2 {
let c = claim_one(&q);
q.complete_node(id, c.node_id, Ok(()));
}
let swap = claim_one(&q);
assert_eq!(swap.kind.as_str(), "swap");
q.complete_node(id, swap.node_id, Err("update failed".to_owned()));
let reconcile = claim_one(&q);
assert_eq!(reconcile.kind.as_str(), "reconcile");
q.complete_node(id, reconcile.node_id, Ok(()));
assert_eq!(state_of(&q, id), State::Failed);
}
#[test]
fn failed_reconcile_marks_dag_failed() {
let q = JobQueue::new(1);
let id = submit(
&q,
templates::reconcile_only(
Template::Start,
"agent-a",
Source::Manual,
"start".to_owned(),
None,
),
);
let c = claim_one(&q);
q.complete_node(id, c.node_id, Err("start failed".to_owned()));
assert_eq!(state_of(&q, id), State::Failed);
}
// ---- cancel ----
#[test]
fn cancel_clears_queued_dag() {
let q = JobQueue::new(1);
let id = submit(&q, rebuild("agent-a", "r"));
assert!(q.cancel(id));
assert_eq!(state_of(&q, id), State::Cancelled);
assert!(q.claim_ready().is_empty());
}
#[test]
fn cancel_refuses_running_dag() {
let q = JobQueue::new(1);
let id = submit(&q, rebuild("agent-a", "r"));
let _ = claim_one(&q);
assert!(!q.cancel(id));
assert_eq!(state_of(&q, id), State::Running);
}
#[test]
fn cancel_children_marks_queued_children_only() {
let q = JobQueue::new(1);
let meta = submit(
&q,
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
);
// Parent's MetaLock is running while children exist.
let lock = claim_one(&q);
assert_eq!(lock.dag_id, meta);
let child_a = submit(
&q,
templates::rebuild(
"agent-a",
Source::MetaUpdate,
"cascade".to_owned(),
Some(meta),
false,
),
);
let child_b = submit(
&q,
templates::rebuild(
"agent-b",
Source::MetaUpdate,
"cascade".to_owned(),
Some(meta),
false,
),
);
let unrelated = submit(&q, rebuild("agent-c", "operator queued"));
// MetaLock holds the single build slot, so both children (and the
// unrelated rebuild) are still fully queued here.
let cancelled = q.cancel_children(meta);
assert_eq!(cancelled, 2);
assert_eq!(state_of(&q, child_a), State::Cancelled);
assert_eq!(state_of(&q, child_b), State::Cancelled);
assert_eq!(state_of(&q, unrelated), State::Queued);
}
#[test]
fn cancel_children_skips_running_child() {
let q = JobQueue::new(2);
let meta = submit(
&q,
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
);
let lock = claim_one(&q);
let running_child = submit(
&q,
templates::rebuild(
"agent-a",
Source::MetaUpdate,
"cascade".to_owned(),
Some(meta),
false,
),
);
let queued_child = submit(
&q,
templates::rebuild(
"agent-b",
Source::MetaUpdate,
"cascade".to_owned(),
Some(meta),
false,
),
);
// Second slot lets running_child's prebuild start.
let child_claim = claim_one(&q);
assert_eq!(child_claim.dag_id, running_child);
let n = q.cancel_children(meta);
assert_eq!(n, 1);
assert_eq!(state_of(&q, running_child), State::Running);
assert_eq!(state_of(&q, queued_child), State::Cancelled);
q.complete_node(meta, lock.node_id, Ok(()));
}
// ---- fan-out ----
#[test]
fn append_children_sets_parent_and_dedups() {
let q = JobQueue::new(1);
let meta = submit(
&q,
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
);
let specs = vec![
templates::rebuild(
"alice",
Source::MetaUpdate,
"cascade".to_owned(),
Some(meta),
false,
),
templates::rebuild(
"bob",
Source::MetaUpdate,
"cascade".to_owned(),
Some(meta),
false,
),
// Duplicate — must coalesce into the first alice child.
templates::rebuild(
"alice",
Source::MetaUpdate,
"cascade again".to_owned(),
Some(meta),
false,
),
];
let ids = q.append_children(specs);
assert_eq!(ids.len(), 3);
assert_eq!(ids[0], ids[2], "duplicate child dedups");
let snap = q.snapshot();
let children: Vec<_> = snap.iter().filter(|d| d.parent_id == Some(meta)).collect();
assert_eq!(children.len(), 2);
}
// ---- terminal reporting + lease release ----
#[test]
fn terminal_dag_reported_exactly_once_and_lease_released() {
let q = JobQueue::new(1);
let id = submit(
&q,
templates::restart("agent-a", Source::Manual, "r".to_owned()),
);
let stop = claim_one(&q);
let r1 = q.complete_node(id, stop.node_id, Ok(()));
assert!(r1.terminal.is_empty(), "dag not terminal yet");
let rec = claim_one(&q);
let r2 = q.complete_node(id, rec.node_id, Ok(()));
assert_eq!(r2.terminal.len(), 1);
assert_eq!(r2.terminal[0].dag_id, id);
assert_eq!(r2.terminal[0].state, State::Done);
// Lease released: a new DAG for the agent can claim immediately.
let next = submit(
&q,
templates::reconcile_only(
Template::Stop,
"agent-a",
Source::Manual,
"stop".to_owned(),
None,
),
);
let c = claim_one(&q);
assert_eq!(c.dag_id, next);
assert!(c.lease_acquired);
}
#[test]
fn cancelled_dag_reports_terminal() {
let q = JobQueue::new(1);
let id = submit(&q, rebuild("agent-a", "r"));
assert!(q.cancel(id));
// The cancel path settles internally; a subsequent completion
// report must not re-report it. Verify via a second dag's cycle.
let other = submit(&q, rebuild("agent-b", "r"));
let c = claim_one(&q);
assert_eq!(c.dag_id, other);
let report = q.complete_node(other, c.node_id, Err("boom".to_owned()));
// agent-b's dag isn't terminal (reconcile still pending) and
// agent-a's was already reported by cancel → nothing here.
assert!(report.terminal.iter().all(|t| t.dag_id != id));
}
// ---- steps, build logs, history ----
#[test]
fn set_step_only_on_running_and_signals_change() {
let q = JobQueue::new(1);
let id = submit(&q, rebuild("agent-a", "r"));
assert!(!q.set_step(id, 0, "too early"), "queued node refuses step");
let c = claim_one(&q);
assert!(q.set_step(id, c.node_id, "nix build"));
assert!(
!q.set_step(id, c.node_id, "nix build"),
"same label → false"
);
assert!(q.set_step(id, c.node_id, "next phase"));
assert!(q.set_step_running(id, "via running lookup"));
q.complete_node(id, c.node_id, Ok(()));
let snap = q.snapshot();
let node = &snap.iter().find(|d| d.id == id).expect("dag").nodes[0];
assert_eq!(node.step, None, "step cleared on completion");
}
#[test]
fn set_build_log_id_links_running_node() {
let q = JobQueue::new(1);
let id = submit(&q, rebuild("agent-a", "r"));
assert!(!q.set_build_log_id(id, 0, 41), "queued node refuses log id");
let c = claim_one(&q);
assert!(q.set_build_log_id(id, c.node_id, 42));
assert!(q.set_build_log_id_running(id, 43));
q.complete_node(id, c.node_id, Ok(()));
let snap = q.snapshot();
let node = &snap.iter().find(|d| d.id == id).expect("dag").nodes[0];
assert_eq!(node.build_log_id, Some(43), "log id survives completion");
}
#[test]
fn history_evicts_old_terminals_per_template() {
let q = JobQueue::new(1);
for i in 0..8 {
let id = submit(
&q,
templates::reconcile_only(
Template::Start,
&format!("agent-{i}"),
Source::Manual,
"start".to_owned(),
None,
),
);
let c = claim_one(&q);
q.complete_node(id, c.node_id, Ok(()));
}
assert_eq!(q.snapshot().len(), 5, "per-template history cap");
assert_eq!(q.live_count(), 0);
}
#[test]
fn error_is_truncated() {
let q = JobQueue::new(1);
let id = submit(&q, rebuild("agent-a", "r"));
let c = claim_one(&q);
q.complete_node(id, c.node_id, Err("x".repeat(5000)));
let snap = q.snapshot();
let err = snap.iter().find(|d| d.id == id).expect("dag").nodes[0]
.error
.clone()
.expect("error stored");
assert!(err.chars().count() <= 2001, "truncated + ellipsis");
assert!(err.ends_with('…'));
}
// ---- template shapes ----
#[test]
fn graceful_stop_shape_signal_drain_reconcile() {
let q = JobQueue::new(1);
let id = submit(
&q,
templates::graceful_stop("agent-a", Source::Manual, "graceful".to_owned()),
);
for expected in ["signal", "drain", "reconcile"] {
let c = claim_one(&q);
assert_eq!(c.kind.as_str(), expected);
q.complete_node(id, c.node_id, Ok(()));
}
assert_eq!(state_of(&q, id), State::Done);
}
#[test]
fn graceful_signal_and_drain_hold_no_build_slot() {
// A whole-hive graceful stop overlaps every drain even at
// buildSlots = 1 while a rebuild hogs the slot.
let q = JobQueue::new(1);
submit(&q, rebuild("builder", "slot hog"));
submit(
&q,
templates::graceful_stop("agent-a", Source::Manual, "g".to_owned()),
);
submit(
&q,
templates::graceful_stop("agent-b", Source::Manual, "g".to_owned()),
);
let claims = q.claim_ready();
let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect();
assert_eq!(
kinds,
vec!["prebuild", "signal", "signal"],
"both agents' signals fire while the slot is held"
);
}
#[test]
fn spawn_shape_create_dropin_reconcile() {
let q = JobQueue::new(1);
let id = submit(
&q,
templates::spawn("newbie", 7, "approval #7 spawn".to_owned()),
);
for expected in ["create", "write_dropin", "reconcile"] {
let c = claim_one(&q);
assert_eq!(c.kind.as_str(), expected);
assert_eq!(c.approval_id, Some(7));
q.complete_node(id, c.node_id, Ok(()));
}
let report_terminal = state_of(&q, id);
assert_eq!(report_terminal, State::Done);
}
#[test]
fn perm_change_shape_prefixes_rebuild_chain() {
let q = JobQueue::new(1);
let id = submit(
&q,
templates::perm_change(
"agent-a",
Source::Manual,
"perm".to_owned(),
PermPayload::Combined {
groups: Some(vec![]),
caps: None,
},
),
);
for expected in [
"write_perm_file",
"prebuild",
"stop_for_update",
"swap",
"reconcile",
] {
let c = claim_one(&q);
assert_eq!(c.kind.as_str(), expected);
q.complete_node(id, c.node_id, Ok(()));
}
assert_eq!(state_of(&q, id), State::Done);
}

3
hive-c0re/src/lib.rs
View file

@ -32,6 +32,7 @@ pub mod forge;
pub mod gateway_nginx;
pub mod hive_stats;
pub mod host_stats;
pub mod job_queue;
pub mod knowledge;
pub mod lifecycle;
pub mod limits;
@ -41,9 +42,9 @@ pub mod meta;
pub mod migrate;
pub mod operator_questions;
pub mod paths;
pub mod power;
pub mod priv_client;
pub mod questions;
pub mod rebuild_queue;
pub mod reminder_scheduler;
pub mod scheduled_prompts;
pub mod scheduled_prompts_worker;

107
hive-c0re/src/lifecycle.rs
View file

@ -260,6 +260,16 @@ async fn port_collision(self_name: &str) -> Option<String> {
}
pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
create_container(name, hive, paths).await?;
write_dropins(name, hive, paths).await?;
priv_run("start", name).await
}
/// First-spawn provisioning + `nixos-container create`, without the
/// drop-in write or the start — the job queue's `Create` node.
/// `spawn` composes this with `write_dropins` + start for direct
/// callers (root-agent bootstrap).
pub async fn create_container(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
validate(name)?;
if let Some(other) = port_collision(name).await {
bail!(
@ -277,8 +287,17 @@ pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()>
// ref resolves.
let agents = agents_after_spawn(name).await?;
crate::meta::sync_agents(hive, &agents).await?;
priv_run("create", name).await
}
/// Re-apply the per-container host-side config: nspawn flags (bind
/// mounts etc.), the systemd resource-limits drop-in, and a daemon
/// reload so both take effect on the next unit (re)start. Idempotent —
/// the job queue's `WriteDropin` node, also folded into every `Swap`
/// (rebuild is the reconcile verb).
pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
validate(name)?;
let container = container_name(name);
priv_run("create", name).await?;
set_nspawn_flags(
&container,
&paths.agent_dir,
@ -287,8 +306,42 @@ pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()>
)
.await?;
set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?;
systemd_daemon_reload().await?;
priv_run("start", name).await
systemd_daemon_reload().await
}
/// Rebuild-path preamble shared by the job queue's `Prebuild` node and
/// `rebuild_no_meta`: fail fast on a port collision, then make sure
/// the applied repo + state dirs exist. Container untouched.
pub async fn prepare_rebuild_dirs(name: &str, paths: &AgentPaths) -> Result<()> {
validate(name)?;
if let Some(other) = port_collision(name).await {
bail!(
"port {} is already taken by '{other}' — rename one of them and retry",
agent_web_port(name)
);
}
setup_applied(&paths.applied_dir, None, name).await?;
ensure_agent_state_subvolume(name).await?;
ensure_claude_dir(&paths.claude_dir)?;
ensure_state_dir(&paths.notes_dir)?;
Ok(())
}
/// Profile-swap for an existing, stopped container: re-apply the
/// drop-ins, then `nixos-container update`. The job queue's `Swap`
/// node. Requires the container stopped (the queue's `StopForUpdate`
/// upstream); does NOT start it — the DAG's tail `Reconcile` owns
/// bringing the agent back to its wanted power state.
pub async fn swap_update(
name: &str,
hive: &HiveEnv,
paths: &AgentPaths,
on_step: &(dyn Fn(&str) + Send + Sync),
on_build_log_id: &(dyn Fn(i64) + Send + Sync),
) -> Result<()> {
write_dropins(name, hive, paths).await?;
on_step("nixos-container update");
priv_run_inner("update", name, Some(on_build_log_id)).await
}
/// Build the `AgentSpec` list for the meta flake from `nixos-container
@ -532,35 +585,16 @@ pub async fn rebuild_no_meta(
on_step: &(dyn Fn(&str) + Send + Sync),
on_build_log_id: &(dyn Fn(i64) + Send + Sync),
) -> Result<bool> {
validate(name)?;
if let Some(other) = port_collision(name).await {
bail!(
"port {} is already taken by '{other}' — rename one of them and retry",
agent_web_port(name)
);
}
setup_applied(&paths.applied_dir, None, name).await?;
ensure_agent_state_subvolume(name).await?;
ensure_claude_dir(&paths.claude_dir)?;
ensure_state_dir(&paths.notes_dir)?;
let container = container_name(name);
prepare_rebuild_dirs(name, paths).await?;
let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display());
if container_exists(name).await {
// Rebuild strategy: stop-before-update + pre-build.
// See `docs/coordinator.md::Container lifecycle`.
let was_running = is_running(name).await;
set_nspawn_flags(
&container,
&paths.agent_dir,
&paths.claude_dir,
&paths.notes_dir,
)
.await?;
set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?;
systemd_daemon_reload().await?;
write_dropins(name, hive, paths).await?;
if was_running {
on_step("nix build");
prebuild_toplevel(name, &flake_ref).await?;
prebuild_toplevel(name, &flake_ref, &|_| ()).await?;
on_step("nixos-container stop");
priv_run("stop", name).await?;
}
@ -601,15 +635,7 @@ pub async fn rebuild_no_meta(
// See `docs/coordinator.md::Spawn path`.
on_step("nixos-container create");
priv_run("create", name).await?;
set_nspawn_flags(
&container,
&paths.agent_dir,
&paths.claude_dir,
&paths.notes_dir,
)
.await?;
set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?;
systemd_daemon_reload().await?;
write_dropins(name, hive, paths).await?;
on_step("nixos-container start");
priv_run("start", name).await?;
Ok(false)
@ -622,7 +648,15 @@ pub async fn rebuild_no_meta(
/// is untouched. See `docs/coordinator.md::Rebuild path` for why
/// the prebuild happens before stop, and `docs/coordinator.md::Prebuild
/// attr path` for why the explicit nixosConfigurations attr is required.
async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> {
///
/// `on_build_log_id` fires with the `build_logs` row id as soon as the
/// row opens, so queue-side callers can link their node to the live
/// stream. Pass `&|_| ()` when not needed.
pub async fn prebuild_toplevel(
name: &str,
flake_ref: &str,
on_build_log_id: &(dyn Fn(i64) + Send + Sync),
) -> Result<()> {
use tokio::io::{AsyncBufReadExt, BufReader};
// Split `<root>#<name>` so we can re-emit with the explicit
// `nixosConfigurations.<name>` segment. The flake_ref shape is
@ -663,6 +697,9 @@ async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> {
})
.ok()
});
if let Some(id) = log_id {
on_build_log_id(id);
}
let mut child = Command::new("nix")
.args(&args)

42
hive-c0re/src/main.rs
View file

@ -13,8 +13,8 @@ use hive_sh4re::{HostRequest, HostResponse};
use hive_c0re::coordinator::{Coordinator, HiveEnv, ServeConfig};
use hive_c0re::{
agent_sockets, auto_update, broker, client, crash_watch, dashboard, dashboard_events, forge,
knowledge, matrix, migrate, rebuild_queue, reminder_scheduler, scheduled_prompts_worker,
server, socket_server,
job_queue, knowledge, matrix, migrate, reminder_scheduler, scheduled_prompts_worker, server,
socket_server,
};
#[derive(Parser)]
@ -85,6 +85,11 @@ enum Cmd {
/// option.
#[arg(long)]
model_prices: Option<String>,
/// Override: number of concurrent nix-heavy job-queue nodes
/// (prebuild / profile-swap / create / meta lock). Set via the
/// `services.hyperhive.c0re.buildSlots` NixOS option.
#[arg(long)]
build_slots: Option<usize>,
},
/// Spawn a new agent container directly (`hive-agent-<name>`). Bypasses
/// the approval queue — use only as an operator on the host. For
@ -156,6 +161,7 @@ async fn main() -> Result<()> {
agent_cpu_quota,
agent_memory_max,
model_prices,
build_slots,
} => {
// Base config from the --config file (or the built-in
// defaults), then apply any per-flag overrides — config
@ -195,7 +201,10 @@ async fn main() -> Result<()> {
sc.model_prices =
serde_json::from_str(&v).context("--model-prices: invalid JSON")?;
}
cmd_serve(sc.env, sc.model_prices, db, &cli.socket).await
if let Some(v) = build_slots {
sc.build_slots = v;
}
cmd_serve(sc.env, sc.model_prices, sc.build_slots, db, &cli.socket).await
}
Cmd::Spawn { name } => {
render(client::request(&cli.socket, HostRequest::Spawn { name }).await?)
@ -244,6 +253,7 @@ async fn main() -> Result<()> {
async fn cmd_serve(
env: HiveEnv,
model_prices: hive_c0re::hive_stats::PriceTable,
build_slots: usize,
db: std::path::PathBuf,
socket: &std::path::Path,
) -> Result<()> {
@ -255,7 +265,7 @@ async fn cmd_serve(
// `dashboard_port` is consumed into the Coordinator below; capture the
// Copy value first for the dashboard + knowledge-webhook tasks.
let dashboard_port = env.dashboard_port;
let coord = Arc::new(Coordinator::open(&db, env, model_prices)?);
let coord = Arc::new(Coordinator::open(&db, env, model_prices, build_slots)?);
socket_server::start_manager(coord.clone())?;
// Idempotent pre-flight: rewrite pre-meta-layout applied
// repos, ensure proposed repos carry the `applied`
@ -413,25 +423,17 @@ async fn cmd_serve(
// and fans the body out to each active target's inbox. See
// scheduled_prompts_worker.rs.
scheduled_prompts_worker::spawn(coord.clone());
// Rebuild-queue worker: drains the global rebuild/meta-update/
// spawn queue FIFO so hive-c0re never runs two heavyweight
// container ops concurrently. Existing rebuild call sites
// (auto_update, dashboard, manager, approval handler) enqueue
// here instead of awaiting `rebuild_agent` inline. See
// `rebuild_queue.rs`.
// Job-queue scheduler: drives the global DAG queue (rebuild /
// meta-update / spawn / power ops). Concurrency comes from the
// build-slot count + per-agent leases inside the queue, not from
// multiple workers — cheap nodes (graceful signals, drains,
// reconciles) overlap nix-heavy ones structurally. Call sites
// (auto_update, dashboard, manager, approval handler) submit DAGs
// instead of awaiting lifecycle work inline. See `job_queue/`.
{
let q_coord = coord.clone();
tokio::spawn(async move {
rebuild_queue::run_worker(q_coord).await;
});
// Fast lane: a second serial worker for hard Start/Stop, running
// concurrently with the build worker above so a stop/start never
// waits behind a slow build for another container. Per-agent
// ordering vs that agent's own build is enforced in the queue's
// claim logic (a fast op defers behind its agent's running build).
let fast_coord = coord.clone();
tokio::spawn(async move {
rebuild_queue::run_fast_worker(fast_coord).await;
job_queue::scheduler::run_worker(q_coord).await;
});
}
// Forward every broker event onto the unified dashboard

203
hive-c0re/src/power.rs Normal file
View file

@ -0,0 +1,203 @@
//! Durable per-agent power *intent* (`wanted: Up | Offline`) — the
//! spec half of spec-vs-status desired-state reconciliation.
//! `container_view` remains the observed *status*; the job queue's
//! `Reconcile` nodes are the mechanism that converges the two.
//!
//! Stored in `/var/lib/hyperhive/db/agent_power.sqlite` (one tiny row
//! per agent). Intent persists across hive-c0re restarts; in-flight
//! queue work deliberately does not. Setting `wanted` is never a
//! queued node: operator/intent actions update the row synchronously
//! at request time, then submit the DAG whose terminal `Reconcile`
//! reads the fresh value — rapid toggles are last-writer-wins and the
//! reconciles converge. Power toggles never commit to the meta repo.
use std::path::Path;
use std::sync::Mutex;
use anyhow::{Context, Result};
use rusqlite::{Connection, OptionalExtension, params};
const SCHEMA: &str = "
CREATE TABLE IF NOT EXISTS agent_power (
agent TEXT PRIMARY KEY,
wanted TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
";
/// Per-agent power intent.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Wanted {
Up,
Offline,
}
impl Wanted {
pub fn as_str(self) -> &'static str {
match self {
Wanted::Up => "up",
Wanted::Offline => "offline",
}
}
fn parse(s: &str) -> Option<Self> {
match s {
"up" => Some(Wanted::Up),
"offline" => Some(Wanted::Offline),
_ => None,
}
}
/// Seed value from an observed running state (first boot after
/// this store lands, or an agent spawned outside the normal path).
pub fn from_running(running: bool) -> Self {
if running { Wanted::Up } else { Wanted::Offline }
}
}
/// What a `Reconcile` should do given intent + observation. Pure so
/// the `{Up,Offline} × {up,down}` matrix is unit-testable.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReconcileAction {
Start,
Stop,
Noop,
}
#[must_use]
pub fn reconcile_action(wanted: Wanted, running: bool) -> ReconcileAction {
match (wanted, running) {
(Wanted::Up, false) => ReconcileAction::Start,
(Wanted::Offline, true) => ReconcileAction::Stop,
(Wanted::Up, true) | (Wanted::Offline, false) => ReconcileAction::Noop,
}
}
/// Sqlite-backed store. `Arc`-friendly: all methods take `&self`, the
/// internal `Mutex<Connection>` serializes access.
pub struct PowerStore {
conn: Mutex<Connection>,
}
impl PowerStore {
pub fn open(db_dir: &Path) -> Result<Self> {
std::fs::create_dir_all(db_dir)
.with_context(|| format!("create agent_power db parent {}", db_dir.display()))?;
let path = db_dir.join("agent_power.sqlite");
let conn = Connection::open(&path)
.with_context(|| format!("open agent_power db {}", path.display()))?;
conn.execute_batch(SCHEMA)
.context("apply agent_power schema")?;
Ok(Self {
conn: Mutex::new(conn),
})
}
/// In-memory store for tests.
#[cfg(test)]
pub fn open_in_memory() -> Result<Self> {
let conn = Connection::open_in_memory().context("open in-memory agent_power db")?;
conn.execute_batch(SCHEMA)
.context("apply agent_power schema")?;
Ok(Self {
conn: Mutex::new(conn),
})
}
/// Read an agent's intent. `None` when the agent has no row yet
/// (callers seed from observed state via [`Self::get_or_seed`]).
pub fn get(&self, agent: &str) -> Result<Option<Wanted>> {
let conn = self.conn.lock().expect("agent_power mutex poisoned");
let row: Option<String> = conn
.query_row(
"SELECT wanted FROM agent_power WHERE agent = ?1",
params![agent],
|r| r.get(0),
)
.optional()
.context("select agent_power")?;
Ok(row.and_then(|s| Wanted::parse(&s)))
}
/// Write an agent's intent (last-writer-wins, synchronous at
/// request time).
pub fn set(&self, agent: &str, wanted: Wanted) -> Result<()> {
let conn = self.conn.lock().expect("agent_power mutex poisoned");
conn.execute(
"INSERT INTO agent_power (agent, wanted, updated_at) VALUES (?1, ?2, ?3)
ON CONFLICT(agent) DO UPDATE SET wanted = ?2, updated_at = ?3",
params![agent, wanted.as_str(), now_secs()],
)
.context("upsert agent_power")?;
Ok(())
}
/// Read an agent's intent, seeding the row from the observed
/// running state when absent — the migration rule for agents that
/// predate this store (running ⇒ `Up`, stopped ⇒ `Offline`), after
/// which the DB is authoritative.
pub fn get_or_seed(&self, agent: &str, running: bool) -> Result<Wanted> {
if let Some(w) = self.get(agent)? {
return Ok(w);
}
let seeded = Wanted::from_running(running);
self.set(agent, seeded)?;
tracing::info!(%agent, wanted = seeded.as_str(), "agent_power: seeded from observed state");
Ok(seeded)
}
/// Drop an agent's row (container destroyed).
pub fn remove(&self, agent: &str) -> Result<()> {
let conn = self.conn.lock().expect("agent_power mutex poisoned");
conn.execute("DELETE FROM agent_power WHERE agent = ?1", params![agent])
.context("delete agent_power")?;
Ok(())
}
}
fn now_secs() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.ok()
.and_then(|d| i64::try_from(d.as_secs()).ok())
.unwrap_or(0)
}
#[cfg(test)]
mod tests {
use super::*;
/// The full `{Up,Offline} × {up,down}` reconcile matrix:
/// start / stop / noop / noop.
#[test]
fn reconcile_matrix() {
assert_eq!(reconcile_action(Wanted::Up, false), ReconcileAction::Start);
assert_eq!(
reconcile_action(Wanted::Offline, true),
ReconcileAction::Stop
);
assert_eq!(reconcile_action(Wanted::Up, true), ReconcileAction::Noop);
assert_eq!(
reconcile_action(Wanted::Offline, false),
ReconcileAction::Noop
);
}
#[test]
fn get_set_roundtrip_and_seed() {
let store = PowerStore::open_in_memory().expect("open");
assert_eq!(store.get("alice").expect("get"), None);
// Seed from observed running state, once.
assert_eq!(store.get_or_seed("alice", true).expect("seed"), Wanted::Up);
// Thereafter the DB is authoritative — observed state no longer
// overrides.
assert_eq!(
store.get_or_seed("alice", false).expect("seeded"),
Wanted::Up
);
store.set("alice", Wanted::Offline).expect("set");
assert_eq!(store.get("alice").expect("get"), Some(Wanted::Offline));
store.remove("alice").expect("remove");
assert_eq!(store.get("alice").expect("get"), None);
}
}

2158
hive-c0re/src/rebuild_queue.rs
View file

File diff suppressed because it is too large Load diff

49
hive-c0re/src/server.rs
View file

@ -131,7 +131,7 @@ async fn dispatch(req: &HostRequest, coord: Arc<Coordinator>) -> HostResponse {
agents.retain(|a| prev.contains(a));
}
let infra = scoped_infra(scope);
handle_start(&agents, &infra).await?
handle_start(&coord, &agents, &infra).await?
}
HostRequest::Destroy { name, purge } => {
actions::destroy(&coord, name, *purge).await?;
@ -201,6 +201,9 @@ async fn handle_spawn(coord: &Arc<Coordinator>, name: &str) -> Result<HostRespon
let paths = Coordinator::agent_paths(name, agent_dir);
match lifecycle::spawn(name, &hive, &paths).await {
Ok(()) => {
if let Err(e) = coord.power.set(name, crate::power::Wanted::Up) {
tracing::warn!(%name, error = ?e, "agent_power: set wanted=up failed");
}
coord.notify_manager(&hive_sh4re::HelperEvent::Spawned {
agent: name.to_owned(),
ok: true,
@ -224,8 +227,12 @@ async fn handle_spawn(coord: &Arc<Coordinator>, name: &str) -> Result<HostRespon
}
/// Kill `name`'s container, unregister its socket, notify the manager.
/// Persists `wanted = Offline` first so reconciles don't undo the kill.
async fn handle_kill(coord: &Arc<Coordinator>, name: &str) -> Result<HostResponse> {
tracing::info!(%name, "kill");
if let Err(e) = coord.power.set(name, crate::power::Wanted::Offline) {
tracing::warn!(%name, error = ?e, "agent_power: set wanted=offline failed");
}
lifecycle::kill(name).await?;
coord.unregister_agent(name);
coord.notify_manager(&hive_sh4re::HelperEvent::Killed {
@ -284,25 +291,29 @@ async fn handle_stop(
tracing::info!(?agents, ?infra, graceful, "stop");
let mut ok_items: Vec<String> = Vec::new();
let mut errors: Vec<String> = Vec::new();
let mut enqueued_graceful = false;
for agent in agents {
if graceful {
// Graceful stop: enqueue the quiesce orchestration rather than a
// hard kill. Serialised through the rebuild queue so it can't race
// an in-flight rebuild for the same agent, and its per-step
// progress surfaces on the queue snapshot + build log.
coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::GracefulStop,
agent.clone(),
crate::rebuild_queue::QueueSource::Manual,
// Graceful stop: submit the quiesce DAG rather than a hard
// kill. The per-agent lease keeps it from racing an
// in-flight rebuild for the same agent; the cheap Signal
// nodes all fire immediately so every agent's drain
// overlaps. `submit::graceful_stop` also persists
// `wanted = Offline` and emits the queue snapshot.
crate::job_queue::submit::graceful_stop(
coord,
agent,
crate::job_queue::Source::Manual,
"manual via hivectl graceful stop".to_owned(),
None,
);
ok_items.push(agent.clone());
enqueued_graceful = true;
continue;
}
// Persist the intent even if the kill itself fails — otherwise
// the next boot reconcile would restart the agent.
if let Err(e) = coord.power.set(agent, crate::power::Wanted::Offline) {
tracing::warn!(%agent, error = ?e, "agent_power: set wanted=offline failed");
}
match lifecycle::kill(agent).await {
Ok(()) => ok_items.push(agent.clone()),
Err(e) => {
@ -311,9 +322,6 @@ async fn handle_stop(
}
}
}
if enqueued_graceful {
coord.emit_rebuild_queue_snapshot();
}
for &container in infra {
let name = container.unit_name();
@ -333,7 +341,11 @@ async fn handle_stop(
/// inverse of [`handle_stop`]. Infra comes up before agents so the agents
/// find forge/matrix/gateway ready. Per-target failures aggregated. Callers
/// resolve the [`LifecycleScope`] to these explicit name lists up front.
async fn handle_start(agents: &[String], infra: &[InfraContainer]) -> Result<HostResponse> {
async fn handle_start(
coord: &Arc<Coordinator>,
agents: &[String],
infra: &[InfraContainer],
) -> Result<HostResponse> {
tracing::info!(?agents, ?infra, "start");
let mut ok_items: Vec<String> = Vec::new();
let mut errors: Vec<String> = Vec::new();
@ -350,6 +362,11 @@ async fn handle_start(agents: &[String], infra: &[InfraContainer]) -> Result<Hos
}
for agent in agents {
// Persist the intent even if the start itself fails — the next
// reconcile (boot or queued) retries toward `Up`.
if let Err(e) = coord.power.set(agent, crate::power::Wanted::Up) {
tracing::warn!(%agent, error = ?e, "agent_power: set wanted=up failed");
}
match lifecycle::start(agent).await {
Ok(()) => ok_items.push(agent.clone()),
Err(e) => {

71
hive-c0re/src/socket_server.rs
View file

@ -510,7 +510,7 @@ async fn dispatch(req: &AgentRequest, agent: &str, coord: &Arc<Coordinator>) ->
match req {
// Lifecycle + config: caller must be an ancestor of the target
// (a parent owns its whole subtree; the root covers every agent).
AgentRequest::Start { name } => handle_start(coord, agent, name).await,
AgentRequest::Start { name } => handle_start(coord, agent, name),
AgentRequest::Restart { name } => handle_restart(coord, agent, name).await,
AgentRequest::Kill { name } => handle_kill(coord, agent, name).await,
AgentRequest::Update { name } => handle_update(coord, agent, name),
@ -786,39 +786,21 @@ fn handle_reminder_rollup(
/// `Start` — start a container, kicking its next turn. The caller must be an
/// ancestor of `name` in the topology (the root covers every agent).
async fn handle_start(coord: &Arc<Coordinator>, agent: &str, name: &str) -> AgentResponse {
fn handle_start(coord: &Arc<Coordinator>, agent: &str, name: &str) -> AgentResponse {
if let Some(err) = require_descendant(agent, name, "start") {
return err;
}
tracing::info!(%agent, %name, "start container");
// If the hyperhive rev is stale, route through the rebuild queue so the
// container runs current nix derivations before it starts. Same logic as
// `run_start`; this covers the MCP `start` tool path.
let current_rev = crate::auto_update::current_flake_rev(&coord.hyperhive_flake);
if let Some(ref rev) = current_rev {
let stored = std::fs::read_to_string(crate::auto_update::rev_marker_path(name)).ok();
if stored.as_deref() != Some(rev.as_str()) {
tracing::info!(%agent, %name, "start: rev stale — enqueuing rebuild");
coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Rebuild,
name.to_owned(),
crate::rebuild_queue::QueueSource::Manual,
format!("start {name}: rev stale — rebuilding first"),
None,
);
coord.emit_rebuild_queue_snapshot();
return AgentResponse::Ok;
}
}
match crate::lifecycle::start(name).await {
Ok(()) => {
coord.kick_agent(name, "container started");
AgentResponse::Ok
}
Err(e) => AgentResponse::Err {
message: format!("{e:#}"),
},
}
// Persist `wanted = Up` and submit the Start DAG; the submit layer
// upgrades a stale-rev start to a full rebuild so the container
// runs current nix derivations before it starts.
crate::job_queue::submit::start(
coord,
name,
crate::job_queue::Source::Manual,
format!("agent `{agent}` start tool"),
);
AgentResponse::Ok
}
/// `Restart` — enqueue a restart for a container. The caller must be an
@ -838,15 +820,13 @@ async fn handle_restart(coord: &Arc<Coordinator>, agent: &str, name: &str) -> Ag
if let Some(err) = require_descendant(agent, name, "restart") {
return err;
}
tracing::info!(%agent, %name, "enqueue restart");
coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Restart,
name.to_owned(),
crate::rebuild_queue::QueueSource::Manual,
tracing::info!(%agent, %name, "submit restart");
crate::job_queue::submit::restart(
coord,
name,
crate::job_queue::Source::Manual,
format!("agent `{agent}` restart tool"),
None,
);
coord.emit_rebuild_queue_snapshot();
AgentResponse::Ok
}
@ -908,6 +888,11 @@ async fn handle_kill(coord: &Arc<Coordinator>, agent: &str, name: &str) -> Agent
return err;
}
tracing::info!(%agent, %name, "kill container");
// Persist the intent even if the kill fails — otherwise the next
// reconcile would restart the container.
if let Err(e) = coord.power.set(name, crate::power::Wanted::Offline) {
tracing::warn!(%name, error = ?e, "agent_power: set wanted=offline failed");
}
let result: anyhow::Result<()> = async {
crate::lifecycle::kill(name).await?;
coord.unregister_agent(name);
@ -933,15 +918,13 @@ fn handle_update(coord: &Arc<Coordinator>, agent: &str, name: &str) -> AgentResp
if let Some(err) = require_descendant(agent, name, "rebuild") {
return err;
}
tracing::info!(%agent, %name, "enqueue rebuild");
coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Rebuild,
name.to_owned(),
crate::rebuild_queue::QueueSource::Manual,
tracing::info!(%agent, %name, "submit rebuild");
crate::job_queue::submit::rebuild(
coord,
name,
crate::job_queue::Source::Manual,
format!("agent `{agent}` update tool"),
None,
);
coord.emit_rebuild_queue_snapshot();
AgentResponse::Ok
}