vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat(hive-c0re): replace rebuild queue with generic job-DAG queue

Browse source 
jobs are now DAGs of primitive nodes (prebuild, stop-for-update, swap,
reconcile, signal, drain, ...) driven by one scheduler with N build
slots + per-agent lifecycle leases. per-agent power intent (wanted
up/offline) is durable in agent_power.sqlite; Reconcile nodes converge
observed state to it. kills the graceful-stop watcher thread, the
deferred-start follow-up, and the cascade pre-enqueue (fan-out on
MetaLock completion instead). tracker: #2166
This commit is contained in:
müde 2026-07-06 20:13:14 +02:00
commit 7946e03fde
25 changed files with 3673 additions and 2731 deletions
Download patch file Download diff file Expand all files Collapse all files

310
hive-c0re/src/auto_update.rs
View file

@ -1,18 +1,25 @@
//! Startup auto-update: on `hive-c0re serve` boot, rebuild containers that
//! actually need it. Two skip rules keep boot-time work minimal:
//! Boot reconcile: on `hive-c0re serve` boot, (a) run the config path
//! for agents whose per-agent rev marker is stale — a `StartupSweep`
//! DAG (meta hyperhive lock bump) fanning out `Rebuild` children for
//! the stale agents whose `wanted` power intent is `Up` — and (b)
//! converge every other drifted agent to its persisted `wanted` via
//! `Reconcile` DAGs. Two rules keep boot-time nix work minimal:
//!
//! 1. **Stopped containers** are deferred — they will be rebuilt the first
//! time the operator starts them (see `rebuild_queue::run_start` and
//! `socket_server::handle_start`).
//! 2. **Running containers whose rev marker matches** the current hyperhive
//! flake path are skipped — nothing changed, no nix work to do.
//! 1. **Stale but wanted-offline agents** get no rebuild — their
//! rebuild happens the first time they're started (the start
//! submit path upgrades a stale start to rebuild+start). The sweep
//! parent still runs whenever *any* marker is stale so the meta
//! hyperhive lock is bumped for those later start-upgrades.
//! 2. **Agents whose rev marker matches** the current hyperhive flake
//! path are skipped — nothing changed, no nix work to do.
//!
//! See `docs/coordinator.md::Auto-update sweep`.
//! Booting with no config change performs no meta commit — only
//! reconciles. See `docs/coordinator.md::Boot reconcile`.
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::{Context, Result};
use anyhow::Result;
use crate::coordinator::Coordinator;
use crate::lifecycle::{self, AGENT_PREFIX, MANAGER_NAME};
@ -71,143 +78,6 @@ pub async fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> boo
}
}
/// Rebuild one sub-agent and refresh its marker. Used by both the startup
/// scanner and the dashboard's manual "update" button so the two paths
/// can't diverge.
///
/// `queue_entry_id` is `Some(id)` when the rebuild was dispatched from
/// the `rebuild_queue` worker (lets the function annotate its phase via
/// `coord.set_queue_step`) and `None` when called directly (e.g. the
/// root-agent migration nudge in `ensure_root_agent`).
///
/// `relock` bumps the agent's meta input to `applied/<n>/main` before
/// the container rebuild. Pass `false` only for meta-update cascade
/// rebuilds, where re-locking would revert the bump the cascade just
/// committed (see `lifecycle::rebuild`).
///
/// `defer_start_source` is `Some(source)` for queue-dispatched rebuilds:
/// instead of holding the serialized build lane through the container
/// boot, the start-after-rebuild is enqueued as a fast-lane `Start`
/// entry (grouped under this rebuild via `parent_id`, same split as the
/// graceful-stop follow-up). Pass `None` for direct callers to keep the
/// start inline.
///
/// # Errors
///
/// Propagates errors from `coord.ensure_runtime` and `lifecycle::rebuild`.
pub async fn rebuild_agent(
coord: &Arc<Coordinator>,
name: &str,
current_rev: &str,
queue_entry_id: Option<u64>,
relock: bool,
defer_start_source: Option<crate::rebuild_queue::QueueSource>,
) -> Result<()> {
tracing::info!(%name, rev = %current_rev, "rebuild agent");
let agent_dir = coord
.ensure_runtime(name)
.with_context(|| format!("ensure_runtime {name}"))?;
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(name, agent_dir);
// Suppress crash_watch during the stop+start window inside
// lifecycle::rebuild. Dashboard rebuilds already do this via
// lifecycle_action; this catches the auto-update scan + any
// other direct caller.
let guard = coord.transient_guard(name, crate::coordinator::TransientKind::Rebuilding);
let result = lifecycle::rebuild(
name,
&hive,
&paths,
relock,
defer_start_source.is_some(),
&|step| coord.set_queue_step(queue_entry_id, step),
&|log_id| {
if let Some(qid) = queue_entry_id
&& coord.rebuild_queue.set_build_log_id(qid, log_id)
{
coord.emit_rebuild_queue_snapshot();
}
},
)
.await;
drop(guard);
match &result {
Ok(needs_start) => {
if let Err(e) = std::fs::write(rev_marker_path(name), current_rev) {
tracing::warn!(%name, error = ?e, "write rev marker failed");
}
// Deferred start: hand the container boot to the fast lane so
// this build-lane entry completes now and the next queued
// rebuild's nix build overlaps with the boot. `parent_id`
// groups the follow-up under this rebuild on the dashboard —
// same split the graceful-stop path uses for its container
// stop. A start failure surfaces on the Start entry (with
// the cold-start fallback) instead of failing the rebuild.
if *needs_start && let Some(source) = defer_start_source {
coord
.rebuild_queue
.enqueue_full(crate::rebuild_queue::FullEnqueue {
kind: crate::rebuild_queue::QueueKind::Start,
agent: name.to_owned(),
source,
reason: format!("start after rebuild of {name}"),
parent_id: queue_entry_id,
inputs: Vec::new(),
approval_id: None,
perm_payload: None,
depends_on: Vec::new(),
});
coord.emit_rebuild_queue_snapshot();
}
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
agent: name.to_owned(),
ok: true,
note: None,
sha: None,
tag: None,
});
coord.set_queue_step(queue_entry_id, "forge sync");
// Run the full forge sync on every successful rebuild so
// the rebuild path is equivalent to the hive-c0re startup
// sweep: token, config-repo mirror, meta read access, and
// meta remote are all kept in sync. Recovers missing tokens
// (e.g. first-spawn seeding failed transiently) without
// requiring a full hive-c0re restart.
crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await;
// Mirror the matrix side of the startup sweep: if hive-matrix
// is present, ensure this agent has a registered user + token.
// Idempotent (skips if token file already exists). Keeps the
// rebuild path equivalent to the startup sweep for newly-spawned
// agents that missed ensure_all().
crate::matrix::sync_agent_standalone(name).await;
// Wake the agent on its next turn so claude sees a
// "you were rebuilt — check /state/ for notes, --continue
// session intact" hint. Covers dashboard rebuild, admin
// CLI rebuild, auto-update startup scan, and the
// dashboard's meta-input update path — all of which
// route through rebuild_agent.
coord.kick_agent(name, "container rebuilt");
// Container state (needs_update, deployed_sha) may have
// shifted — rescan so dashboards drop the "needs update"
// chip without waiting for the next /api/state poll.
coord.rescan_containers_and_emit().await;
// Lock bump → meta-inputs panel needs to re-render.
crate::dashboard::emit_meta_inputs_snapshot(coord);
}
Err(e) => {
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
agent: name.to_owned(),
ok: false,
note: Some(format!("{e:#}")),
sha: None,
tag: None,
});
coord.rescan_containers_and_emit().await;
}
}
result.map(|_| ())
}
/// Whether this hive is "ruthless" — running with no root/manager agent at
/// all (no ruth). When true, hive-c0re skips the root-agent create/start
/// sweep entirely. Controlled by the host option
@ -246,17 +116,18 @@ pub async fn ensure_root_agent(coord: &Arc<Coordinator>) -> Result<()> {
// running whatever the host-declarative config was at create
// time, with a wrong systemd unit and port.
let applied_flake = Coordinator::agent_applied_dir(MANAGER_NAME).join("flake.nix");
if !applied_flake.exists()
&& let Some(rev) = current_rev.as_ref()
{
if !applied_flake.exists() && current_rev.is_some() {
tracing::warn!(
"manager container exists but no applied flake — forcing rebuild to migrate"
);
let coord_clone = coord.clone();
if let Err(e) =
rebuild_agent(&coord_clone, MANAGER_NAME, rev.as_str(), None, true, None).await
{
tracing::warn!(error = ?e, "manager migration rebuild failed");
if let Err(e) = coord.job_queue.submit(crate::job_queue::templates::rebuild(
MANAGER_NAME,
crate::job_queue::Source::AutoUpdate,
"manager migration: no applied flake".to_owned(),
None,
true,
)) {
tracing::warn!(error = ?e, "manager migration rebuild submit failed");
}
} else {
tracing::debug!("manager container already present");
@ -272,6 +143,9 @@ pub async fn ensure_root_agent(coord: &Arc<Coordinator>) -> Result<()> {
// this function.)
if !lifecycle::is_running(MANAGER_NAME).await {
tracing::info!("manager container present but not running — starting");
if let Err(e) = coord.power.set(MANAGER_NAME, crate::power::Wanted::Up) {
tracing::warn!(error = ?e, "agent_power: set manager wanted=up failed");
}
if let Err(e) = lifecycle::start(MANAGER_NAME).await {
tracing::warn!(error = ?e, "manager start failed");
}
@ -283,6 +157,9 @@ pub async fn ensure_root_agent(coord: &Arc<Coordinator>) -> Result<()> {
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(MANAGER_NAME, runtime);
lifecycle::spawn(MANAGER_NAME, &hive, &paths).await?;
if let Err(e) = coord.power.set(MANAGER_NAME, crate::power::Wanted::Up) {
tracing::warn!(error = ?e, "agent_power: set manager wanted=up failed");
}
if let Some(rev) = current_rev {
let _ = std::fs::write(rev_marker_path(MANAGER_NAME), &rev);
}
@ -327,18 +204,17 @@ pub fn topology_sort(
});
}
/// Rebuild containers that need it on startup. Skips:
/// - **Stopped containers**: deferred to on-start (`run_start` / `handle_start`
/// upgrades a plain start to rebuild+start when the rev marker is stale).
/// - **Running containers with a matching rev marker**: no nix work needed.
///
/// Enqueues a `StartupSweep` parent entry followed by per-agent `Rebuild`
/// children linked via `parent_id`. Returns Ok even if some rebuilds failed.
/// Boot reconcile (see the module doc): classify every agent by rev
/// freshness + persisted `wanted` intent, submit one `StartupSweep`
/// DAG (hyperhive lock bump → fan-out rebuilds for stale wanted-up
/// agents) when anything is stale, and `Reconcile` DAGs for agents
/// whose observed power state drifted from `wanted`. Returns Ok even
/// if some submissions failed.
pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
let containers = match lifecycle::list().await {
Ok(c) => c,
Err(e) => {
tracing::warn!(error = ?e, "auto-update: nixos-container list failed");
tracing::warn!(error = ?e, "boot reconcile: nixos-container list failed");
return Ok(());
}
};
@ -356,62 +232,88 @@ pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
let topo = crate::topology::read();
topology_sort(&mut logical_names, &topo);
// Pre-classify: decide which agents need a rebuild now vs can be skipped.
let mut to_rebuild: Vec<String> = Vec::new();
// Classify. `get_or_seed` doubles as the one-time migration: an
// agent without an `agent_power` row is seeded from its observed
// state (running ⇒ Up), after which the DB is authoritative.
let mut any_stale = false;
let mut fanout: Vec<String> = Vec::new(); // stale ∧ wanted=Up → sweep rebuild
let mut drifted: Vec<String> = Vec::new(); // fresh ∧ wanted≠observed → reconcile
let mut n_deferred = 0usize;
let mut n_skipped = 0usize;
for name in &logical_names {
// Idea 2: stopped containers are deferred — rebuild happens the first
// time the operator starts them.
if !lifecycle::is_running(name).await {
n_deferred += 1;
tracing::debug!(%name, "startup sweep: stopped — deferring rebuild to on-start");
continue;
}
// Idea 1: running containers with a matching rev marker need no rebuild.
if let Some(ref rev) = current_rev {
let stored = std::fs::read_to_string(rev_marker_path(name)).ok();
if stored.as_deref() == Some(rev.as_str()) {
n_skipped += 1;
tracing::debug!(%name, "startup sweep: rev unchanged — skipping rebuild");
let running = lifecycle::is_running(name).await;
let wanted = match coord.power.get_or_seed(name, running) {
Ok(w) => w,
Err(e) => {
tracing::warn!(%name, error = ?e, "agent_power read failed — assuming observed");
crate::power::Wanted::from_running(running)
}
};
let fresh = current_rev.as_ref().is_some_and(|rev| {
std::fs::read_to_string(rev_marker_path(name))
.is_ok_and(|stored| stored == rev.as_str())
});
if fresh {
n_skipped += 1;
} else {
any_stale = true;
if wanted == crate::power::Wanted::Up {
// Rebuild against the post-bump lock; the DAG's tail
// Reconcile brings the agent (back) up — covering both
// the running-stale and stopped-but-wanted-up cases.
fanout.push(name.clone());
continue;
}
// Stale but wanted offline: no boot-time nix work — the
// start submit path upgrades a stale start to a rebuild.
n_deferred += 1;
tracing::debug!(%name, "boot reconcile: stale but offline — deferring rebuild to on-start");
}
if crate::power::reconcile_action(wanted, running) != crate::power::ReconcileAction::Noop {
drifted.push(name.clone());
}
to_rebuild.push(name.clone());
}
// Enqueue the parent sweep entry. The worker processes it trivially
// (no-op dispatch); its purpose is to give the dashboard a "why" header.
let sweep_id = coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::StartupSweep,
"hyperhive".to_owned(),
crate::rebuild_queue::QueueSource::AutoUpdate,
format!(
"startup sweep: {} rebuild(s), {} deferred (stopped), {} skipped (up-to-date)",
to_rebuild.len(),
n_deferred,
n_skipped,
),
None,
);
tracing::info!(
total = containers.len(),
rebuilds = to_rebuild.len(),
rebuilds = fanout.len(),
reconciles = drifted.len(),
deferred = n_deferred,
skipped = n_skipped,
sweep_id,
"auto-update: startup sweep"
up_to_date = n_skipped,
"boot reconcile"
);
for name in to_rebuild {
coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Rebuild,
name,
crate::rebuild_queue::QueueSource::StartupSweep,
"startup sweep".to_owned(),
Some(sweep_id),
// Sweep parent whenever ANY marker is stale — even when every
// stale agent is wanted-offline: the hyperhive lock bump must land
// now so their later start-upgrade rebuilds build against it.
// No stale agents ⇒ no sweep ⇒ no meta commit on a no-change boot.
if any_stale {
let reason = format!(
"startup sweep: {} rebuild(s), {} deferred (offline), {} up-to-date",
fanout.len(),
n_deferred,
n_skipped,
);
if let Err(e) = coord
.job_queue
.submit(crate::job_queue::templates::startup_sweep(reason, fanout))
{
tracing::warn!(error = ?e, "boot reconcile: sweep submit failed");
}
}
for name in drifted {
if let Err(e) = coord
.job_queue
.submit(crate::job_queue::templates::reconcile_only(
crate::job_queue::Template::Reconcile,
&name,
crate::job_queue::Source::AutoUpdate,
"boot reconcile".to_owned(),
None,
))
{
tracing::warn!(%name, error = ?e, "boot reconcile: submit failed");
}
}
coord.emit_rebuild_queue_snapshot();
Ok(())