From 672e77c849e20bc1d6f67cbf00846f78376ac0be Mon Sep 17 00:00:00 2001 From: damocles Date: Sat, 4 Jul 2026 18:16:49 +0200 Subject: [PATCH] feat(#2102): skip startup rebuild for stopped/unchanged containers --- docs/coordinator.md | 25 ++++++--- hive-c0re/src/auto_update.rs | 95 ++++++++++++++++++++++++---------- hive-c0re/src/rebuild_queue.rs | 22 ++++++++ hive-c0re/src/socket_server.rs | 19 +++++++ 4 files changed, 126 insertions(+), 35 deletions(-) diff --git a/docs/coordinator.md b/docs/coordinator.md index b8ac35d6..539aa159 100644 --- a/docs/coordinator.md +++ b/docs/coordinator.md @@ -117,19 +117,28 @@ render. ## Auto-update sweep -On startup, `auto_update.rs` rebuilds every known container unconditionally. -`nixos-container update` is a no-op at the nix level when nothing changed (same -store path), so the cost is low and avoids rev-marker staleness — all agents always -need an update pass when any meta commit lands. +On startup, `auto_update.rs` rebuilds containers that actually need it. Two skip +rules keep boot-time work minimal: + +1. **Stopped containers** are deferred: the startup sweep enqueues nothing for them. + When the operator later starts a stopped container (via the dashboard or the + `start` MCP tool), both `run_start` (queue path) and `handle_start` (socket path) + check the rev marker first — if it's stale, the start is silently upgraded to a + full rebuild+start so the container runs current nix derivations. + +2. **Running containers with a matching rev marker** are skipped: if the per-agent + `.{name}.hyperhive-rev` file under `/var/lib/hyperhive/applied/` already holds + the current flake rev, no nix work is needed and the entry is omitted entirely. `auto_update::run` enqueues a single `StartupSweep` parent entry (`kind = startup_sweep`, `agent = "hyperhive"`) followed by per-agent `Rebuild` children -(`source = startup_sweep`, `parent_id = sweep_id`). The worker processes the parent -by bumping the meta `hyperhive` input lock, then transitions it to Done. The child +for the agents that do need rebuilding (`source = startup_sweep`, `parent_id = +sweep_id`). The sweep description records the rebuild / deferred / skipped counts +so the operator can see at a glance how much work the boot triggered. The child rebuilds drain sequentially through the queue; the dashboard renders them nested -under the parent so the operator can see the whole boot-time sweep in one group. +under the parent. -Before this change, each boot enqueued flat `Rebuild` entries with +Before the sweep-grouping change, each boot enqueued flat `Rebuild` entries with `source = AutoUpdate` and no parent — visible but ungrouped. ## Meta flake diff --git a/hive-c0re/src/auto_update.rs b/hive-c0re/src/auto_update.rs index 40d81b6e..d17a1887 100644 --- a/hive-c0re/src/auto_update.rs +++ b/hive-c0re/src/auto_update.rs @@ -1,8 +1,13 @@ -//! Startup auto-update: on `hive-c0re serve` boot, rebuild every known -//! container unconditionally. `nixos-container update` is a no-op at the -//! nix level when nothing changed (same store path), so the cost is low -//! and avoids rev-marker staleness (all agents always need an update pass -//! when any meta commit lands). See `docs/coordinator.md::Auto-update sweep`. +//! Startup auto-update: on `hive-c0re serve` boot, rebuild containers that +//! actually need it. Two skip rules keep boot-time work minimal: +//! +//! 1. **Stopped containers** are deferred — they will be rebuilt the first +//! time the operator starts them (see `rebuild_queue::run_start` and +//! `socket_server::handle_start`). +//! 2. **Running containers whose rev marker matches** the current hyperhive +//! flake path are skipped — nothing changed, no nix work to do. +//! +//! See `docs/coordinator.md::Auto-update sweep`. use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -322,11 +327,13 @@ pub fn topology_sort( }); } -/// Rebuild every container on startup. Enqueues a `StartupSweep` parent -/// entry (agent = `"hyperhive"`) followed by per-agent `Rebuild` children -/// linked via `parent_id`. The dashboard renders them nested so the operator -/// can see at a glance "boot N agents, here is each rebuild's status". -/// Returns Ok even if some rebuilds failed. +/// Rebuild containers that need it on startup. Skips: +/// - **Stopped containers**: deferred to on-start (`run_start` / `handle_start` +/// upgrades a plain start to rebuild+start when the rev marker is stale). +/// - **Running containers with a matching rev marker**: no nix work needed. +/// +/// Enqueues a `StartupSweep` parent entry followed by per-agent `Rebuild` +/// children linked via `parent_id`. Returns Ok even if some rebuilds failed. pub async fn run(coord: Arc) -> Result<()> { let containers = match lifecycle::list().await { Ok(c) => c, @@ -336,22 +343,7 @@ pub async fn run(coord: Arc) -> Result<()> { } }; - // Enqueue the parent sweep entry. The worker processes it trivially - // (no-op dispatch) so it completes quickly; its purpose is to give the - // dashboard a "why" header for the per-agent child rebuilds below. - let sweep_id = coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::StartupSweep, - "hyperhive".to_owned(), - crate::rebuild_queue::QueueSource::AutoUpdate, - format!("startup sweep ({} containers)", containers.len()), - None, - ); - - tracing::info!( - agents = containers.len(), - sweep_id, - "auto-update: queueing all on startup" - ); + let current_rev = current_flake_rev(&coord.hyperhive_flake); // Resolve container names to logical agent names, then sort by // topology depth so parents are always rebuilt before their @@ -363,7 +355,56 @@ pub async fn run(coord: Arc) -> Result<()> { .collect(); let topo = crate::topology::read(); topology_sort(&mut logical_names, &topo); - for name in logical_names { + + // Pre-classify: decide which agents need a rebuild now vs can be skipped. + let mut to_rebuild: Vec = Vec::new(); + let mut n_deferred = 0usize; + let mut n_skipped = 0usize; + for name in &logical_names { + // Idea 2: stopped containers are deferred — rebuild happens the first + // time the operator starts them. + if !lifecycle::is_running(name).await { + n_deferred += 1; + tracing::debug!(%name, "startup sweep: stopped — deferring rebuild to on-start"); + continue; + } + // Idea 1: running containers with a matching rev marker need no rebuild. + if let Some(ref rev) = current_rev { + let stored = std::fs::read_to_string(rev_marker_path(name)).ok(); + if stored.as_deref() == Some(rev.as_str()) { + n_skipped += 1; + tracing::debug!(%name, "startup sweep: rev unchanged — skipping rebuild"); + continue; + } + } + to_rebuild.push(name.clone()); + } + + // Enqueue the parent sweep entry. The worker processes it trivially + // (no-op dispatch); its purpose is to give the dashboard a "why" header. + let sweep_id = coord.rebuild_queue.enqueue( + crate::rebuild_queue::QueueKind::StartupSweep, + "hyperhive".to_owned(), + crate::rebuild_queue::QueueSource::AutoUpdate, + format!( + "startup sweep: {} rebuild(s), {} deferred (stopped), {} skipped (up-to-date)", + to_rebuild.len(), + n_deferred, + n_skipped, + ), + None, + ); + + tracing::info!( + total = containers.len(), + rebuilds = to_rebuild.len(), + deferred = n_deferred, + skipped = n_skipped, + sweep_id, + "auto-update: startup sweep" + ); + + for name in to_rebuild { coord.rebuild_queue.enqueue( crate::rebuild_queue::QueueKind::Rebuild, name, diff --git a/hive-c0re/src/rebuild_queue.rs b/hive-c0re/src/rebuild_queue.rs index 8482a838..b6853698 100644 --- a/hive-c0re/src/rebuild_queue.rs +++ b/hive-c0re/src/rebuild_queue.rs @@ -1034,11 +1034,33 @@ async fn rebuild_for_entry( /// Uses the cold-start fallback (stop + kill + start retry) so the /// deferred start-after-rebuild keeps the same activation-error recovery /// it had when it ran inline on the build lane. +/// +/// If the hyperhive flake rev has changed since the container was last built +/// (i.e. the rev marker is stale or missing), the start is upgraded to a full +/// rebuild so the container runs current nix derivations. This is the +/// "deferred stopped container" path from `auto_update::run`. async fn run_start( coord: &std::sync::Arc, entry: &QueueEntry, ) -> anyhow::Result<()> { let name = &entry.agent; + // Upgrade to rebuild+start if the rev marker is stale. + let current_rev = crate::auto_update::current_flake_rev(&coord.hyperhive_flake); + if let Some(ref rev) = current_rev { + let stored = std::fs::read_to_string(crate::auto_update::rev_marker_path(name)).ok(); + if stored.as_deref() != Some(rev.as_str()) { + tracing::info!(%name, "start: rev stale — upgrading to rebuild+start"); + return crate::auto_update::rebuild_agent( + coord, + name, + rev, + Some(entry.id), + true, + Some(entry.source), + ) + .await; + } + } let _guard = coord.transient_guard(name, crate::coordinator::TransientKind::Starting); coord.set_queue_step(Some(entry.id), "nixos-container start"); crate::lifecycle::start_with_fallback(name).await?; diff --git a/hive-c0re/src/socket_server.rs b/hive-c0re/src/socket_server.rs index d44b69d5..1d5f2d44 100644 --- a/hive-c0re/src/socket_server.rs +++ b/hive-c0re/src/socket_server.rs @@ -803,6 +803,25 @@ async fn handle_start(coord: &Arc, agent: &str, name: &str) -> Agen return err; } tracing::info!(%agent, %name, "start container"); + // If the hyperhive rev is stale, route through the rebuild queue so the + // container runs current nix derivations before it starts. Same logic as + // `run_start`; this covers the MCP `start` tool path. + let current_rev = crate::auto_update::current_flake_rev(&coord.hyperhive_flake); + if let Some(ref rev) = current_rev { + let stored = std::fs::read_to_string(crate::auto_update::rev_marker_path(name)).ok(); + if stored.as_deref() != Some(rev.as_str()) { + tracing::info!(%agent, %name, "start: rev stale — enqueuing rebuild"); + coord.rebuild_queue.enqueue( + crate::rebuild_queue::QueueKind::Rebuild, + name.to_owned(), + crate::rebuild_queue::QueueSource::Manual, + format!("start {name}: rev stale — rebuilding first"), + None, + ); + coord.emit_rebuild_queue_snapshot(); + return AgentResponse::Ok; + } + } match crate::lifecycle::start(name).await { Ok(()) => { coord.kick_agent(name, "container started");