Compare commits
4 changed files with 20 additions and 68 deletions
|
|
@ -102,8 +102,7 @@ hive-c0re/ host daemon + sibling operator CLI (lib + 2 bins)
|
||||||
ManagerRequest::GetLooseEnds (the
|
ManagerRequest::GetLooseEnds (the
|
||||||
get_loose_ends MCP tool).
|
get_loose_ends MCP tool).
|
||||||
src/rebuild_queue.rs global serialised queue for long-running ops
|
src/rebuild_queue.rs global serialised queue for long-running ops
|
||||||
(rebuild / meta_update / spawn / destroy /
|
(rebuild / meta_update / spawn / destroy).
|
||||||
startup_sweep).
|
|
||||||
Single background worker drains FIFO; dedup
|
Single background worker drains FIFO; dedup
|
||||||
collapses re-enqueued still-queued entries.
|
collapses re-enqueued still-queued entries.
|
||||||
`QueueEntry` carries id, agent, kind, state,
|
`QueueEntry` carries id, agent, kind, state,
|
||||||
|
|
|
||||||
|
|
@ -54,15 +54,14 @@ since the current run started).
|
||||||
| Source | Meaning |
|
| Source | Meaning |
|
||||||
|--------|---------|
|
|--------|---------|
|
||||||
| `Manual` | Operator clicked rebuild / update-all / meta-update on the dashboard, or any other direct human action (CLI, manager tool). |
|
| `Manual` | Operator clicked rebuild / update-all / meta-update on the dashboard, or any other direct human action (CLI, manager tool). |
|
||||||
| `AutoUpdate` | Legacy startup-sweep source (flat, no parent). Replaced by `StartupSweep` for new boots. |
|
| `AutoUpdate` | Fired by the startup sweep or a meta-update cascade. |
|
||||||
| `StartupSweep` | Child of a `StartupSweep` parent entry; boot-time per-agent rebuild with the sweep as the visual group header. |
|
|
||||||
| `Approval` | Triggered by an operator-approved `ApprovalKind::{Spawn, ApplyCommit}`. |
|
| `Approval` | Triggered by an operator-approved `ApprovalKind::{Spawn, ApplyCommit}`. |
|
||||||
|
|
||||||
### Cascade parent tracking
|
### Cascade parent tracking
|
||||||
|
|
||||||
`MetaUpdate` and `StartupSweep` entries fan out `Rebuild` children, each carrying
|
`MetaUpdate` entries fan out `Rebuild` children, each carrying
|
||||||
`parent_id = <parent_id>`. The dashboard groups children under their parent
|
`parent_id = <meta_update_id>`. The dashboard groups children under their parent
|
||||||
in the queue panel so the operator sees the whole cascade as a tree,
|
in the queue panel so the operator sees the whole meta-update cascade as a tree,
|
||||||
not a flat list.
|
not a flat list.
|
||||||
|
|
||||||
### Step labels
|
### Step labels
|
||||||
|
|
@ -90,17 +89,8 @@ render.
|
||||||
On startup, `auto_update.rs` rebuilds every known container unconditionally.
|
On startup, `auto_update.rs` rebuilds every known container unconditionally.
|
||||||
`nixos-container update` is a no-op at the nix level when nothing changed (same
|
`nixos-container update` is a no-op at the nix level when nothing changed (same
|
||||||
store path), so the cost is low and avoids rev-marker staleness — all agents always
|
store path), so the cost is low and avoids rev-marker staleness — all agents always
|
||||||
need an update pass when any meta commit lands.
|
need an update pass when any meta commit lands. Each rebuild is enqueued as a
|
||||||
|
`Rebuild` entry with `source = AutoUpdate` and drains through the global queue.
|
||||||
`auto_update::run` enqueues a single `StartupSweep` parent entry (`kind =
|
|
||||||
startup_sweep`, `agent = "hyperhive"`) followed by per-agent `Rebuild` children
|
|
||||||
(`source = startup_sweep`, `parent_id = sweep_id`). The worker processes the parent
|
|
||||||
by bumping the meta `hyperhive` input lock, then transitions it to Done. The child
|
|
||||||
rebuilds drain sequentially through the queue; the dashboard renders them nested
|
|
||||||
under the parent so the operator can see the whole boot-time sweep in one group.
|
|
||||||
|
|
||||||
Before this change, each boot enqueued flat `Rebuild` entries with
|
|
||||||
`source = AutoUpdate` and no parent — visible but ungrouped.
|
|
||||||
|
|
||||||
## Meta flake
|
## Meta flake
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -200,12 +200,15 @@ pub async fn ensure_manager(coord: &Arc<Coordinator>) -> Result<()> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Rebuild every container on startup. Enqueues a `StartupSweep` parent
|
/// Rebuild every container on startup. Sequential to avoid nix-store sqlite
|
||||||
/// entry (agent = `"hyperhive"`) followed by per-agent `Rebuild` children
|
/// races and keep logs readable. Returns Ok even if some rebuilds failed.
|
||||||
/// linked via `parent_id`. The dashboard renders them nested so the operator
|
|
||||||
/// can see at a glance "boot N agents, here is each rebuild's status".
|
|
||||||
/// Returns Ok even if some rebuilds failed.
|
|
||||||
pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
|
pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
|
||||||
|
// Bump meta's hyperhive input up-front so per-agent rebuilds build
|
||||||
|
// against the latest base. Non-fatal on failure.
|
||||||
|
if let Err(e) = crate::meta::lock_update_hyperhive().await {
|
||||||
|
tracing::warn!(error = ?e, "auto-update: meta lock_update_hyperhive failed");
|
||||||
|
}
|
||||||
|
|
||||||
let containers = match lifecycle::list().await {
|
let containers = match lifecycle::list().await {
|
||||||
Ok(c) => c,
|
Ok(c) => c,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|
@ -214,24 +217,13 @@ pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Enqueue the parent sweep entry. The worker processes it trivially
|
let _current_rev = current_flake_rev(&coord.hyperhive_flake).unwrap_or_default();
|
||||||
// (no-op dispatch) so it completes quickly; its purpose is to give the
|
|
||||||
// dashboard a "why" header for the per-agent child rebuilds below.
|
|
||||||
let sweep_id = coord.rebuild_queue.enqueue(
|
|
||||||
crate::rebuild_queue::QueueKind::StartupSweep,
|
|
||||||
"hyperhive".to_owned(),
|
|
||||||
crate::rebuild_queue::QueueSource::AutoUpdate,
|
|
||||||
format!("startup sweep ({} containers)", containers.len()),
|
|
||||||
None,
|
|
||||||
);
|
|
||||||
|
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
agents = containers.len(),
|
agents = containers.len(),
|
||||||
sweep_id,
|
|
||||||
"auto-update: queueing all on startup"
|
"auto-update: queueing all on startup"
|
||||||
);
|
);
|
||||||
|
for container in containers {
|
||||||
for container in &containers {
|
|
||||||
let logical = if container == MANAGER_NAME {
|
let logical = if container == MANAGER_NAME {
|
||||||
Some(MANAGER_NAME.to_owned())
|
Some(MANAGER_NAME.to_owned())
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -241,9 +233,9 @@ pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
|
||||||
coord.rebuild_queue.enqueue(
|
coord.rebuild_queue.enqueue(
|
||||||
crate::rebuild_queue::QueueKind::Rebuild,
|
crate::rebuild_queue::QueueKind::Rebuild,
|
||||||
name,
|
name,
|
||||||
crate::rebuild_queue::QueueSource::StartupSweep,
|
crate::rebuild_queue::QueueSource::AutoUpdate,
|
||||||
"startup sweep".to_owned(),
|
"startup sweep".to_owned(),
|
||||||
Some(sweep_id),
|
None,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
coord.emit_rebuild_queue_snapshot();
|
coord.emit_rebuild_queue_snapshot();
|
||||||
|
|
|
||||||
|
|
@ -27,11 +27,6 @@ pub enum QueueKind {
|
||||||
/// variant exists so the wire shape doesn't need to change later.
|
/// variant exists so the wire shape doesn't need to change later.
|
||||||
#[allow(dead_code, reason = "wire shape — routed by a future PR")]
|
#[allow(dead_code, reason = "wire shape — routed by a future PR")]
|
||||||
Destroy,
|
Destroy,
|
||||||
/// hive-c0re boot-time sweep: bumps the meta hyperhive lock then
|
|
||||||
/// enqueues a `Rebuild` child for every managed container. Completes
|
|
||||||
/// after the lock bump; children run as independent queue entries
|
|
||||||
/// grouped under this parent's `id`. `agent` = `"hyperhive"`.
|
|
||||||
StartupSweep,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl QueueKind {
|
impl QueueKind {
|
||||||
|
|
@ -41,7 +36,6 @@ impl QueueKind {
|
||||||
QueueKind::MetaUpdate => "meta_update",
|
QueueKind::MetaUpdate => "meta_update",
|
||||||
QueueKind::Spawn => "spawn",
|
QueueKind::Spawn => "spawn",
|
||||||
QueueKind::Destroy => "destroy",
|
QueueKind::Destroy => "destroy",
|
||||||
QueueKind::StartupSweep => "startup_sweep",
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -60,17 +54,8 @@ pub enum QueueSource {
|
||||||
/// the originating meta-update.
|
/// the originating meta-update.
|
||||||
MetaUpdate,
|
MetaUpdate,
|
||||||
/// `auto_update::run` startup sweep — rebuild every container on
|
/// `auto_update::run` startup sweep — rebuild every container on
|
||||||
/// hive-c0re boot. Legacy flat source (no parent); replaced by
|
/// hive-c0re boot.
|
||||||
/// `StartupSweep` for the parent entry and child rebuilds once the
|
|
||||||
/// queue introduced `parent_id` grouping. Kept for wire compatibility
|
|
||||||
/// with entries logged before the migration.
|
|
||||||
AutoUpdate,
|
AutoUpdate,
|
||||||
/// Direct child of a `StartupSweep` queue entry — one per agent in
|
|
||||||
/// the boot-time rebuild sweep. Carries `parent_id` back-link so
|
|
||||||
/// the dashboard renders the sweep's per-agent rebuilds nested under
|
|
||||||
/// the parent header. The parent entry itself uses `QueueSource::AutoUpdate`
|
|
||||||
/// (automated boot action, not operator-driven).
|
|
||||||
StartupSweep,
|
|
||||||
/// Crash recovery path (future use — currently no auto-rebuild on
|
/// Crash recovery path (future use — currently no auto-rebuild on
|
||||||
/// crash, but the variant exists for the imminent feature).
|
/// crash, but the variant exists for the imminent feature).
|
||||||
#[allow(dead_code, reason = "wire shape — used by a future feature")]
|
#[allow(dead_code, reason = "wire shape — used by a future feature")]
|
||||||
|
|
@ -88,7 +73,6 @@ impl QueueSource {
|
||||||
QueueSource::Manual => "manual",
|
QueueSource::Manual => "manual",
|
||||||
QueueSource::MetaUpdate => "meta_update",
|
QueueSource::MetaUpdate => "meta_update",
|
||||||
QueueSource::AutoUpdate => "auto_update",
|
QueueSource::AutoUpdate => "auto_update",
|
||||||
QueueSource::StartupSweep => "startup_sweep",
|
|
||||||
QueueSource::CrashRecover => "crash_recover",
|
QueueSource::CrashRecover => "crash_recover",
|
||||||
QueueSource::Approval => "approval",
|
QueueSource::Approval => "approval",
|
||||||
}
|
}
|
||||||
|
|
@ -568,19 +552,6 @@ async fn dispatch(
|
||||||
// Reserved for future `destroy --purge` integration.
|
// Reserved for future `destroy --purge` integration.
|
||||||
anyhow::bail!("Destroy kind not yet implemented in rebuild_queue worker");
|
anyhow::bail!("Destroy kind not yet implemented in rebuild_queue worker");
|
||||||
}
|
}
|
||||||
(QueueKind::StartupSweep, _) => {
|
|
||||||
// Bump meta's hyperhive input before per-agent child rebuilds
|
|
||||||
// run so they build against the latest base. Non-fatal on
|
|
||||||
// failure — child rebuilds proceed regardless. After the bump
|
|
||||||
// (or failure) this entry transitions to Done and the worker
|
|
||||||
// drains the pre-enqueued child Rebuild entries.
|
|
||||||
coord.set_queue_step(Some(entry.id), "nix flake update hyperhive");
|
|
||||||
if let Err(e) = crate::meta::lock_update_hyperhive().await {
|
|
||||||
tracing::warn!(error = ?e, "startup_sweep: meta lock_update_hyperhive failed");
|
|
||||||
}
|
|
||||||
// `finish` clears the step label; no explicit clear needed here.
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue