diff --git a/CLAUDE.md b/CLAUDE.md index af5643cd..3adcb3d1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -102,7 +102,8 @@ hive-c0re/ host daemon + sibling operator CLI (lib + 2 bins) ManagerRequest::GetLooseEnds (the get_loose_ends MCP tool). src/rebuild_queue.rs global serialised queue for long-running ops - (rebuild / meta_update / spawn / destroy). + (rebuild / meta_update / spawn / destroy / + startup_sweep). Single background worker drains FIFO; dedup collapses re-enqueued still-queued entries. `QueueEntry` carries id, agent, kind, state, diff --git a/docs/coordinator.md b/docs/coordinator.md index 421e0312..10e865a9 100644 --- a/docs/coordinator.md +++ b/docs/coordinator.md @@ -54,14 +54,15 @@ since the current run started). | Source | Meaning | |--------|---------| | `Manual` | Operator clicked rebuild / update-all / meta-update on the dashboard, or any other direct human action (CLI, manager tool). | -| `AutoUpdate` | Fired by the startup sweep or a meta-update cascade. | +| `AutoUpdate` | Legacy startup-sweep source (flat, no parent). Replaced by `StartupSweep` for new boots. | +| `StartupSweep` | Child of a `StartupSweep` parent entry; boot-time per-agent rebuild with the sweep as the visual group header. | | `Approval` | Triggered by an operator-approved `ApprovalKind::{Spawn, ApplyCommit}`. | ### Cascade parent tracking -`MetaUpdate` entries fan out `Rebuild` children, each carrying -`parent_id = `. The dashboard groups children under their parent -in the queue panel so the operator sees the whole meta-update cascade as a tree, +`MetaUpdate` and `StartupSweep` entries fan out `Rebuild` children, each carrying +`parent_id = `. The dashboard groups children under their parent +in the queue panel so the operator sees the whole cascade as a tree, not a flat list. ### Step labels @@ -89,8 +90,17 @@ render. On startup, `auto_update.rs` rebuilds every known container unconditionally. `nixos-container update` is a no-op at the nix level when nothing changed (same store path), so the cost is low and avoids rev-marker staleness — all agents always -need an update pass when any meta commit lands. Each rebuild is enqueued as a -`Rebuild` entry with `source = AutoUpdate` and drains through the global queue. +need an update pass when any meta commit lands. + +`auto_update::run` enqueues a single `StartupSweep` parent entry (`kind = +startup_sweep`, `agent = "hyperhive"`) followed by per-agent `Rebuild` children +(`source = startup_sweep`, `parent_id = sweep_id`). The worker processes the parent +by bumping the meta `hyperhive` input lock, then transitions it to Done. The child +rebuilds drain sequentially through the queue; the dashboard renders them nested +under the parent so the operator can see the whole boot-time sweep in one group. + +Before this change, each boot enqueued flat `Rebuild` entries with +`source = AutoUpdate` and no parent — visible but ungrouped. ## Meta flake diff --git a/hive-c0re/src/auto_update.rs b/hive-c0re/src/auto_update.rs index 36d89e92..4291809f 100644 --- a/hive-c0re/src/auto_update.rs +++ b/hive-c0re/src/auto_update.rs @@ -200,15 +200,12 @@ pub async fn ensure_manager(coord: &Arc) -> Result<()> { Ok(()) } -/// Rebuild every container on startup. Sequential to avoid nix-store sqlite -/// races and keep logs readable. Returns Ok even if some rebuilds failed. +/// Rebuild every container on startup. Enqueues a `StartupSweep` parent +/// entry (agent = `"hyperhive"`) followed by per-agent `Rebuild` children +/// linked via `parent_id`. The dashboard renders them nested so the operator +/// can see at a glance "boot N agents, here is each rebuild's status". +/// Returns Ok even if some rebuilds failed. pub async fn run(coord: Arc) -> Result<()> { - // Bump meta's hyperhive input up-front so per-agent rebuilds build - // against the latest base. Non-fatal on failure. - if let Err(e) = crate::meta::lock_update_hyperhive().await { - tracing::warn!(error = ?e, "auto-update: meta lock_update_hyperhive failed"); - } - let containers = match lifecycle::list().await { Ok(c) => c, Err(e) => { @@ -217,13 +214,24 @@ pub async fn run(coord: Arc) -> Result<()> { } }; - let _current_rev = current_flake_rev(&coord.hyperhive_flake).unwrap_or_default(); + // Enqueue the parent sweep entry. The worker processes it trivially + // (no-op dispatch) so it completes quickly; its purpose is to give the + // dashboard a "why" header for the per-agent child rebuilds below. + let sweep_id = coord.rebuild_queue.enqueue( + crate::rebuild_queue::QueueKind::StartupSweep, + "hyperhive".to_owned(), + crate::rebuild_queue::QueueSource::Manual, + format!("startup sweep ({} containers)", containers.len()), + None, + ); tracing::info!( agents = containers.len(), + sweep_id, "auto-update: queueing all on startup" ); - for container in containers { + + for container in &containers { let logical = if container == MANAGER_NAME { Some(MANAGER_NAME.to_owned()) } else { @@ -233,9 +241,9 @@ pub async fn run(coord: Arc) -> Result<()> { coord.rebuild_queue.enqueue( crate::rebuild_queue::QueueKind::Rebuild, name, - crate::rebuild_queue::QueueSource::AutoUpdate, + crate::rebuild_queue::QueueSource::StartupSweep, "startup sweep".to_owned(), - None, + Some(sweep_id), ); } coord.emit_rebuild_queue_snapshot(); diff --git a/hive-c0re/src/rebuild_queue.rs b/hive-c0re/src/rebuild_queue.rs index eeecaf2c..85ee18ca 100644 --- a/hive-c0re/src/rebuild_queue.rs +++ b/hive-c0re/src/rebuild_queue.rs @@ -27,6 +27,11 @@ pub enum QueueKind { /// variant exists so the wire shape doesn't need to change later. #[allow(dead_code, reason = "wire shape — routed by a future PR")] Destroy, + /// hive-c0re boot-time sweep: bumps the meta hyperhive lock then + /// enqueues a `Rebuild` child for every managed container. Completes + /// after the lock bump; children run as independent queue entries + /// grouped under this parent's `id`. `agent` = `"hyperhive"`. + StartupSweep, } impl QueueKind { @@ -36,6 +41,7 @@ impl QueueKind { QueueKind::MetaUpdate => "meta_update", QueueKind::Spawn => "spawn", QueueKind::Destroy => "destroy", + QueueKind::StartupSweep => "startup_sweep", } } } @@ -54,8 +60,13 @@ pub enum QueueSource { /// the originating meta-update. MetaUpdate, /// `auto_update::run` startup sweep — rebuild every container on - /// hive-c0re boot. + /// hive-c0re boot. Child rebuilds of a `StartupSweep` entry use + /// this source; the `parent_id` links them to the parent. AutoUpdate, + /// Direct child of a `StartupSweep` queue entry. Same semantics + /// as `AutoUpdate` but carries the `parent_id` back-link so the + /// dashboard can render the sweep's per-agent rebuilds nested. + StartupSweep, /// Crash recovery path (future use — currently no auto-rebuild on /// crash, but the variant exists for the imminent feature). #[allow(dead_code, reason = "wire shape — used by a future feature")] @@ -73,6 +84,7 @@ impl QueueSource { QueueSource::Manual => "manual", QueueSource::MetaUpdate => "meta_update", QueueSource::AutoUpdate => "auto_update", + QueueSource::StartupSweep => "startup_sweep", QueueSource::CrashRecover => "crash_recover", QueueSource::Approval => "approval", } @@ -552,6 +564,19 @@ async fn dispatch( // Reserved for future `destroy --purge` integration. anyhow::bail!("Destroy kind not yet implemented in rebuild_queue worker"); } + (QueueKind::StartupSweep, _) => { + // Bump meta's hyperhive input before per-agent child rebuilds + // run so they build against the latest base. Non-fatal on + // failure — child rebuilds proceed regardless. After the bump + // (or failure) this entry transitions to Done and the worker + // drains the pre-enqueued child Rebuild entries. + coord.set_queue_step(Some(entry.id), "nix flake update hyperhive"); + if let Err(e) = crate::meta::lock_update_hyperhive().await { + tracing::warn!(error = ?e, "startup_sweep: meta lock_update_hyperhive failed"); + } + // `finish` clears the step label; no explicit clear needed here. + Ok(()) + } } }