feat(#2392): group boot sweep + reconciles under one boot dag

This commit is contained in:
damocles 2026-07-13 11:34:42 +02:00 committed by mara
commit 16f69ca890
6 changed files with 111 additions and 23 deletions

View file

@ -215,11 +215,21 @@ function renderRebuildQueue(s) {
}
orderedLis.push(li);
}
for (const top of tops) {
addEntry(top, false);
for (const child of childrenOf.get(top.id) || []) addEntry(child, true);
// Render each root followed by its whole subtree, depth-first. Recursion
// (rather than a single level of children) handles nested fan-outs such as
// the boot DAG: a `boot` root anchors the `startup_sweep` child, which itself
// fans out `rebuild` grandchildren — all of which must appear under the one
// boot tree. `visited` guards against re-rendering a node reached twice and
// any malformed parent cycle.
const visited = new Set();
function addTree(entry, isChild) {
if (visited.has(entry.id)) return;
visited.add(entry.id);
addEntry(entry, isChild);
for (const child of childrenOf.get(entry.id) || []) addTree(child, true);
}
for (const o of orphans) addEntry(o, true);
for (const top of tops) addTree(top, false);
for (const o of orphans) addTree(o, true);
const liveIds = new Set(queue.map((e) => e.id));
for (const [id, entry] of rebuildQueueRowCache) {

View file

@ -94,6 +94,9 @@ pub(super) async fn run_node(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
NodeKind::WriteDropin => run_write_dropin(coord, claim).await,
NodeKind::WritePermFile => run_write_perm_file(coord, claim, &ctx).await,
NodeKind::ApprovalDeploy => run_approval_deploy(coord, claim).await,
// A pure grouping anchor (boot root): no work, completes immediately so
// its child DAGs settle it and the boot tree resolves.
NodeKind::Noop => Ok(NodeOutput::default()),
}
}

View file

@ -125,6 +125,12 @@ pub enum NodeKind {
/// deploy stays inside `actions.rs` in v1 — deliberately not
/// modeled as scheduler nodes (see the design doc §9).
ApprovalDeploy,
/// No-op anchor that completes immediately with no work. It exists so a
/// boot's `StartupSweep` + per-agent `Reconcile` child DAGs can hang off
/// one root (`parent_id`) and render as a single boot tree on the
/// dashboard. Holds no lease and no build slot; the child DAGs it anchors
/// still run concurrently — the grouping is a display link, not a dep edge.
Noop,
}
impl NodeKind {
@ -144,6 +150,7 @@ impl NodeKind {
NodeKind::WriteDropin => "write_dropin",
NodeKind::WritePermFile => "write_perm_file",
NodeKind::ApprovalDeploy => "approval_deploy",
NodeKind::Noop => "noop",
}
}

View file

@ -281,6 +281,30 @@ pub fn meta_update(
/// Boot-time sweep parent: bump meta's hyperhive input (non-fatal),
/// then fan out `Rebuild` children for the precomputed stale agent
/// list (topology-sorted by the caller).
/// Boot-time root anchor DAG: a single [`NodeKind::Noop`] node that groups
/// this boot's `StartupSweep` + per-agent `Reconcile` child DAGs (linked via
/// `parent_id`) into one tree so the dashboard renders the boot as one entry.
/// Holds no lease and does no work — the children it anchors still run
/// concurrently. `auto_update::run` submits it first (when there's any boot
/// work), then parents the sweep + reconciles onto its id.
pub fn boot_root(reason: String) -> DagSpec {
DagSpec {
template: Template::Boot,
agent: "hyperhive".to_owned(),
source: Source::AutoUpdate,
reason,
parent_id: None,
approval_id: None,
inputs: Vec::new(),
perm_payload: None,
transient: None,
nodes: vec![NodeSpec {
kind: NodeKind::Noop,
deps: Vec::new(),
}],
}
}
pub fn startup_sweep(reason: String, stale_agents: Vec<String>) -> DagSpec {
DagSpec {
template: Template::StartupSweep,

View file

@ -273,10 +273,52 @@ pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
"boot reconcile"
);
// Sweep parent whenever ANY marker is stale — even when every
// stale agent is wanted-offline: the hyperhive lock bump must land
// now so their later start-upgrade rebuilds build against it.
// No stale agents ⇒ no sweep ⇒ no meta commit on a no-change boot.
submit_boot_tree(&coord, any_stale, fanout, drifted, n_deferred, n_skipped);
Ok(())
}
/// Submit this boot's DAGs under one `Boot` root: a `Noop` anchor with the
/// startup sweep + per-agent reconciles hung off it via `parent_id`, so the
/// dashboard renders the boot as a single tree instead of N+1 rows. No-op when
/// there's nothing to do. The `parent_id` link is a display grouping, not a
/// dependency edge — the children run concurrently, so the reconciles never
/// wait behind the lock bump.
fn submit_boot_tree(
coord: &Arc<Coordinator>,
any_stale: bool,
fanout: Vec<String>,
drifted: Vec<String>,
n_deferred: usize,
n_skipped: usize,
) {
// Only emit a boot root when there's actually boot work — a fully-quiet
// boot (nothing stale, nothing drifted) submits nothing, exactly as before.
let boot_root_id = if any_stale || !drifted.is_empty() {
let reason = format!(
"boot: {} rebuild(s), {} reconcile(s), {} deferred (offline), {} up-to-date",
fanout.len(),
drifted.len(),
n_deferred,
n_skipped,
);
match coord
.job_queue
.submit(crate::job_queue::templates::boot_root(reason))
{
Ok(id) => Some(id),
Err(e) => {
tracing::warn!(error = ?e, "boot reconcile: boot-root submit failed");
None
}
}
} else {
None
};
// Sweep whenever ANY marker is stale — even when every stale agent is
// wanted-offline: the hyperhive lock bump must land now so their later
// start-upgrade rebuilds build against it. No stale agents ⇒ no sweep ⇒ no
// meta commit on a no-change boot.
if any_stale {
let reason = format!(
"startup sweep: {} rebuild(s), {} deferred (offline), {} up-to-date",
@ -284,27 +326,24 @@ pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
n_deferred,
n_skipped,
);
if let Err(e) = coord
.job_queue
.submit(crate::job_queue::templates::startup_sweep(reason, fanout))
{
let mut spec = crate::job_queue::templates::startup_sweep(reason, fanout);
spec.parent_id = boot_root_id;
if let Err(e) = coord.job_queue.submit(spec) {
tracing::warn!(error = ?e, "boot reconcile: sweep submit failed");
}
}
for name in drifted {
if let Err(e) = coord
.job_queue
.submit(crate::job_queue::templates::reconcile_only(
crate::job_queue::Template::Reconcile,
&name,
crate::job_queue::Source::AutoUpdate,
"boot reconcile".to_owned(),
None,
))
{
let mut spec = crate::job_queue::templates::reconcile_only(
crate::job_queue::Template::Reconcile,
&name,
crate::job_queue::Source::AutoUpdate,
"boot reconcile".to_owned(),
None,
);
spec.parent_id = boot_root_id;
if let Err(e) = coord.job_queue.submit(spec) {
tracing::warn!(%name, error = ?e, "boot reconcile: submit failed");
}
}
coord.emit_rebuild_queue_snapshot();
Ok(())
}

View file

@ -40,6 +40,10 @@ pub enum Template {
/// Bare converge of observed power state to the persisted intent
/// (boot reconcile).
Reconcile,
/// Boot-time root anchor: a single no-op node that groups this boot's
/// `StartupSweep` + per-agent `Reconcile` child DAGs under one tree, so the
/// dashboard renders the whole boot as one entry instead of N+1 rows.
Boot,
}
impl Template {
@ -57,6 +61,7 @@ impl Template {
Template::Start => "start",
Template::Stop => "stop",
Template::Reconcile => "reconcile",
Template::Boot => "boot",
}
}
}