vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(#2102): skip startup rebuild for stopped/unchanged containers

Browse source
This commit is contained in:
damocles 2026-07-04 18:16:49 +02:00 • committed by mara
commit 672e77c849
4 changed files with 125 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

95
hive-c0re/src/auto_update.rs
View file

@ -1,8 +1,13 @@
//! Startup auto-update: on `hive-c0re serve` boot, rebuild every known
//! container unconditionally. `nixos-container update` is a no-op at the
//! nix level when nothing changed (same store path), so the cost is low
//! and avoids rev-marker staleness (all agents always need an update pass
//! when any meta commit lands). See `docs/coordinator.md::Auto-update sweep`.
//! Startup auto-update: on `hive-c0re serve` boot, rebuild containers that
//! actually need it. Two skip rules keep boot-time work minimal:
//!
//! 1. **Stopped containers** are deferred — they will be rebuilt the first
//! time the operator starts them (see `rebuild_queue::run_start` and
//! `socket_server::handle_start`).
//! 2. **Running containers whose rev marker matches** the current hyperhive
//! flake path are skipped — nothing changed, no nix work to do.
//!
//! See `docs/coordinator.md::Auto-update sweep`.
use std::path::{Path, PathBuf};
use std::sync::Arc;
@ -322,11 +327,13 @@ pub fn topology_sort(
});
}
/// Rebuild every container on startup. Enqueues a `StartupSweep` parent
/// entry (agent = `"hyperhive"`) followed by per-agent `Rebuild` children
/// linked via `parent_id`. The dashboard renders them nested so the operator
/// can see at a glance "boot N agents, here is each rebuild's status".
/// Returns Ok even if some rebuilds failed.
/// Rebuild containers that need it on startup. Skips:
/// - **Stopped containers**: deferred to on-start (`run_start` / `handle_start`
/// upgrades a plain start to rebuild+start when the rev marker is stale).
/// - **Running containers with a matching rev marker**: no nix work needed.
///
/// Enqueues a `StartupSweep` parent entry followed by per-agent `Rebuild`
/// children linked via `parent_id`. Returns Ok even if some rebuilds failed.
pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
let containers = match lifecycle::list().await {
Ok(c) => c,
@ -336,22 +343,7 @@ pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
}
};
// Enqueue the parent sweep entry. The worker processes it trivially
// (no-op dispatch) so it completes quickly; its purpose is to give the
// dashboard a "why" header for the per-agent child rebuilds below.
let sweep_id = coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::StartupSweep,
"hyperhive".to_owned(),
crate::rebuild_queue::QueueSource::AutoUpdate,
format!("startup sweep ({} containers)", containers.len()),
None,
);
tracing::info!(
agents = containers.len(),
sweep_id,
"auto-update: queueing all on startup"
);
let current_rev = current_flake_rev(&coord.hyperhive_flake);
// Resolve container names to logical agent names, then sort by
// topology depth so parents are always rebuilt before their
@ -363,7 +355,56 @@ pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
.collect();
let topo = crate::topology::read();
topology_sort(&mut logical_names, &topo);
for name in logical_names {
// Pre-classify: decide which agents need a rebuild now vs can be skipped.
let mut to_rebuild: Vec<String> = Vec::new();
let mut n_deferred = 0usize;
let mut n_skipped = 0usize;
for name in &logical_names {
// Idea 2: stopped containers are deferred — rebuild happens the first
// time the operator starts them.
if !lifecycle::is_running(name).await {
n_deferred += 1;
tracing::debug!(%name, "startup sweep: stopped — deferring rebuild to on-start");
continue;
}
// Idea 1: running containers with a matching rev marker need no rebuild.
if let Some(ref rev) = current_rev {
let stored = std::fs::read_to_string(rev_marker_path(name)).ok();
if stored.as_deref() == Some(rev.as_str()) {
n_skipped += 1;
tracing::debug!(%name, "startup sweep: rev unchanged — skipping rebuild");
continue;
}
}
to_rebuild.push(name.clone());
}
// Enqueue the parent sweep entry. The worker processes it trivially
// (no-op dispatch); its purpose is to give the dashboard a "why" header.
let sweep_id = coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::StartupSweep,
"hyperhive".to_owned(),
crate::rebuild_queue::QueueSource::AutoUpdate,
format!(
"startup sweep: {} rebuild(s), {} deferred (stopped), {} skipped (up-to-date)",
to_rebuild.len(),
n_deferred,
n_skipped,
),
None,
);
tracing::info!(
total = containers.len(),
rebuilds = to_rebuild.len(),
deferred = n_deferred,
skipped = n_skipped,
sweep_id,
"auto-update: startup sweep"
);
for name in to_rebuild {
coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Rebuild,
name,

22
hive-c0re/src/rebuild_queue.rs
View file

@ -1034,11 +1034,33 @@ async fn rebuild_for_entry(
/// Uses the cold-start fallback (stop + kill + start retry) so the
/// deferred start-after-rebuild keeps the same activation-error recovery
/// it had when it ran inline on the build lane.
///
/// If the hyperhive flake rev has changed since the container was last built
/// (i.e. the rev marker is stale or missing), the start is upgraded to a full
/// rebuild so the container runs current nix derivations. This is the
/// "deferred stopped container" path from `auto_update::run`.
async fn run_start(
coord: &std::sync::Arc<crate::coordinator::Coordinator>,
entry: &QueueEntry,
) -> anyhow::Result<()> {
let name = &entry.agent;
// Upgrade to rebuild+start if the rev marker is stale.
let current_rev = crate::auto_update::current_flake_rev(&coord.hyperhive_flake);
if let Some(ref rev) = current_rev {
let stored = std::fs::read_to_string(crate::auto_update::rev_marker_path(name)).ok();
if stored.as_deref() != Some(rev.as_str()) {
tracing::info!(%name, "start: rev stale — upgrading to rebuild+start");
return crate::auto_update::rebuild_agent(
coord,
name,
rev,
Some(entry.id),
true,
Some(entry.source),
)
.await;
}
}
let _guard = coord.transient_guard(name, crate::coordinator::TransientKind::Starting);
coord.set_queue_step(Some(entry.id), "nixos-container start");
crate::lifecycle::start_with_fallback(name).await?;

19
hive-c0re/src/socket_server.rs
View file

@ -803,6 +803,25 @@ async fn handle_start(coord: &Arc<Coordinator>, agent: &str, name: &str) -> Agen
return err;
}
tracing::info!(%agent, %name, "start container");
// If the hyperhive rev is stale, route through the rebuild queue so the
// container runs current nix derivations before it starts. Same logic as
// `run_start`; this covers the MCP `start` tool path.
let current_rev = crate::auto_update::current_flake_rev(&coord.hyperhive_flake);
if let Some(ref rev) = current_rev {
let stored = std::fs::read_to_string(crate::auto_update::rev_marker_path(name)).ok();
if stored.as_deref() != Some(rev.as_str()) {
tracing::info!(%agent, %name, "start: rev stale — enqueuing rebuild");
coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Rebuild,
name.to_owned(),
crate::rebuild_queue::QueueSource::Manual,
format!("start {name}: rev stale — rebuilding first"),
None,
);
coord.emit_rebuild_queue_snapshot();
return AgentResponse::Ok;
}
}
match crate::lifecycle::start(name).await {
Ok(()) => {
coord.kick_agent(name, "container started");