feat(#2392): group boot sweep + reconciles under one boot dag
This commit is contained in:
parent
6e0f6893cf
commit
16f69ca890
6 changed files with 111 additions and 23 deletions
|
|
@ -215,11 +215,21 @@ function renderRebuildQueue(s) {
|
|||
}
|
||||
orderedLis.push(li);
|
||||
}
|
||||
for (const top of tops) {
|
||||
addEntry(top, false);
|
||||
for (const child of childrenOf.get(top.id) || []) addEntry(child, true);
|
||||
// Render each root followed by its whole subtree, depth-first. Recursion
|
||||
// (rather than a single level of children) handles nested fan-outs such as
|
||||
// the boot DAG: a `boot` root anchors the `startup_sweep` child, which itself
|
||||
// fans out `rebuild` grandchildren — all of which must appear under the one
|
||||
// boot tree. `visited` guards against re-rendering a node reached twice and
|
||||
// any malformed parent cycle.
|
||||
const visited = new Set();
|
||||
function addTree(entry, isChild) {
|
||||
if (visited.has(entry.id)) return;
|
||||
visited.add(entry.id);
|
||||
addEntry(entry, isChild);
|
||||
for (const child of childrenOf.get(entry.id) || []) addTree(child, true);
|
||||
}
|
||||
for (const o of orphans) addEntry(o, true);
|
||||
for (const top of tops) addTree(top, false);
|
||||
for (const o of orphans) addTree(o, true);
|
||||
|
||||
const liveIds = new Set(queue.map((e) => e.id));
|
||||
for (const [id, entry] of rebuildQueueRowCache) {
|
||||
|
|
|
|||
|
|
@ -94,6 +94,9 @@ pub(super) async fn run_node(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
|
|||
NodeKind::WriteDropin => run_write_dropin(coord, claim).await,
|
||||
NodeKind::WritePermFile => run_write_perm_file(coord, claim, &ctx).await,
|
||||
NodeKind::ApprovalDeploy => run_approval_deploy(coord, claim).await,
|
||||
// A pure grouping anchor (boot root): no work, completes immediately so
|
||||
// its child DAGs settle it and the boot tree resolves.
|
||||
NodeKind::Noop => Ok(NodeOutput::default()),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -125,6 +125,12 @@ pub enum NodeKind {
|
|||
/// deploy stays inside `actions.rs` in v1 — deliberately not
|
||||
/// modeled as scheduler nodes (see the design doc §9).
|
||||
ApprovalDeploy,
|
||||
/// No-op anchor that completes immediately with no work. It exists so a
|
||||
/// boot's `StartupSweep` + per-agent `Reconcile` child DAGs can hang off
|
||||
/// one root (`parent_id`) and render as a single boot tree on the
|
||||
/// dashboard. Holds no lease and no build slot; the child DAGs it anchors
|
||||
/// still run concurrently — the grouping is a display link, not a dep edge.
|
||||
Noop,
|
||||
}
|
||||
|
||||
impl NodeKind {
|
||||
|
|
@ -144,6 +150,7 @@ impl NodeKind {
|
|||
NodeKind::WriteDropin => "write_dropin",
|
||||
NodeKind::WritePermFile => "write_perm_file",
|
||||
NodeKind::ApprovalDeploy => "approval_deploy",
|
||||
NodeKind::Noop => "noop",
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -281,6 +281,30 @@ pub fn meta_update(
|
|||
/// Boot-time sweep parent: bump meta's hyperhive input (non-fatal),
|
||||
/// then fan out `Rebuild` children for the precomputed stale agent
|
||||
/// list (topology-sorted by the caller).
|
||||
/// Boot-time root anchor DAG: a single [`NodeKind::Noop`] node that groups
|
||||
/// this boot's `StartupSweep` + per-agent `Reconcile` child DAGs (linked via
|
||||
/// `parent_id`) into one tree so the dashboard renders the boot as one entry.
|
||||
/// Holds no lease and does no work — the children it anchors still run
|
||||
/// concurrently. `auto_update::run` submits it first (when there's any boot
|
||||
/// work), then parents the sweep + reconciles onto its id.
|
||||
pub fn boot_root(reason: String) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::Boot,
|
||||
agent: "hyperhive".to_owned(),
|
||||
source: Source::AutoUpdate,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: None,
|
||||
nodes: vec![NodeSpec {
|
||||
kind: NodeKind::Noop,
|
||||
deps: Vec::new(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
pub fn startup_sweep(reason: String, stale_agents: Vec<String>) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::StartupSweep,
|
||||
|
|
|
|||
|
|
@ -273,10 +273,52 @@ pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
|
|||
"boot reconcile"
|
||||
);
|
||||
|
||||
// Sweep parent whenever ANY marker is stale — even when every
|
||||
// stale agent is wanted-offline: the hyperhive lock bump must land
|
||||
// now so their later start-upgrade rebuilds build against it.
|
||||
// No stale agents ⇒ no sweep ⇒ no meta commit on a no-change boot.
|
||||
submit_boot_tree(&coord, any_stale, fanout, drifted, n_deferred, n_skipped);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Submit this boot's DAGs under one `Boot` root: a `Noop` anchor with the
|
||||
/// startup sweep + per-agent reconciles hung off it via `parent_id`, so the
|
||||
/// dashboard renders the boot as a single tree instead of N+1 rows. No-op when
|
||||
/// there's nothing to do. The `parent_id` link is a display grouping, not a
|
||||
/// dependency edge — the children run concurrently, so the reconciles never
|
||||
/// wait behind the lock bump.
|
||||
fn submit_boot_tree(
|
||||
coord: &Arc<Coordinator>,
|
||||
any_stale: bool,
|
||||
fanout: Vec<String>,
|
||||
drifted: Vec<String>,
|
||||
n_deferred: usize,
|
||||
n_skipped: usize,
|
||||
) {
|
||||
// Only emit a boot root when there's actually boot work — a fully-quiet
|
||||
// boot (nothing stale, nothing drifted) submits nothing, exactly as before.
|
||||
let boot_root_id = if any_stale || !drifted.is_empty() {
|
||||
let reason = format!(
|
||||
"boot: {} rebuild(s), {} reconcile(s), {} deferred (offline), {} up-to-date",
|
||||
fanout.len(),
|
||||
drifted.len(),
|
||||
n_deferred,
|
||||
n_skipped,
|
||||
);
|
||||
match coord
|
||||
.job_queue
|
||||
.submit(crate::job_queue::templates::boot_root(reason))
|
||||
{
|
||||
Ok(id) => Some(id),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "boot reconcile: boot-root submit failed");
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Sweep whenever ANY marker is stale — even when every stale agent is
|
||||
// wanted-offline: the hyperhive lock bump must land now so their later
|
||||
// start-upgrade rebuilds build against it. No stale agents ⇒ no sweep ⇒ no
|
||||
// meta commit on a no-change boot.
|
||||
if any_stale {
|
||||
let reason = format!(
|
||||
"startup sweep: {} rebuild(s), {} deferred (offline), {} up-to-date",
|
||||
|
|
@ -284,27 +326,24 @@ pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
|
|||
n_deferred,
|
||||
n_skipped,
|
||||
);
|
||||
if let Err(e) = coord
|
||||
.job_queue
|
||||
.submit(crate::job_queue::templates::startup_sweep(reason, fanout))
|
||||
{
|
||||
let mut spec = crate::job_queue::templates::startup_sweep(reason, fanout);
|
||||
spec.parent_id = boot_root_id;
|
||||
if let Err(e) = coord.job_queue.submit(spec) {
|
||||
tracing::warn!(error = ?e, "boot reconcile: sweep submit failed");
|
||||
}
|
||||
}
|
||||
for name in drifted {
|
||||
if let Err(e) = coord
|
||||
.job_queue
|
||||
.submit(crate::job_queue::templates::reconcile_only(
|
||||
crate::job_queue::Template::Reconcile,
|
||||
&name,
|
||||
crate::job_queue::Source::AutoUpdate,
|
||||
"boot reconcile".to_owned(),
|
||||
None,
|
||||
))
|
||||
{
|
||||
let mut spec = crate::job_queue::templates::reconcile_only(
|
||||
crate::job_queue::Template::Reconcile,
|
||||
&name,
|
||||
crate::job_queue::Source::AutoUpdate,
|
||||
"boot reconcile".to_owned(),
|
||||
None,
|
||||
);
|
||||
spec.parent_id = boot_root_id;
|
||||
if let Err(e) = coord.job_queue.submit(spec) {
|
||||
tracing::warn!(%name, error = ?e, "boot reconcile: submit failed");
|
||||
}
|
||||
}
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,6 +40,10 @@ pub enum Template {
|
|||
/// Bare converge of observed power state to the persisted intent
|
||||
/// (boot reconcile).
|
||||
Reconcile,
|
||||
/// Boot-time root anchor: a single no-op node that groups this boot's
|
||||
/// `StartupSweep` + per-agent `Reconcile` child DAGs under one tree, so the
|
||||
/// dashboard renders the whole boot as one entry instead of N+1 rows.
|
||||
Boot,
|
||||
}
|
||||
|
||||
impl Template {
|
||||
|
|
@ -57,6 +61,7 @@ impl Template {
|
|||
Template::Start => "start",
|
||||
Template::Stop => "stop",
|
||||
Template::Reconcile => "reconcile",
|
||||
Template::Boot => "boot",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue