feat(#2392): group boot sweep + reconciles under one boot dag
This commit is contained in:
parent
6e0f6893cf
commit
16f69ca890
6 changed files with 111 additions and 23 deletions
|
|
@ -215,11 +215,21 @@ function renderRebuildQueue(s) {
|
||||||
}
|
}
|
||||||
orderedLis.push(li);
|
orderedLis.push(li);
|
||||||
}
|
}
|
||||||
for (const top of tops) {
|
// Render each root followed by its whole subtree, depth-first. Recursion
|
||||||
addEntry(top, false);
|
// (rather than a single level of children) handles nested fan-outs such as
|
||||||
for (const child of childrenOf.get(top.id) || []) addEntry(child, true);
|
// the boot DAG: a `boot` root anchors the `startup_sweep` child, which itself
|
||||||
|
// fans out `rebuild` grandchildren — all of which must appear under the one
|
||||||
|
// boot tree. `visited` guards against re-rendering a node reached twice and
|
||||||
|
// any malformed parent cycle.
|
||||||
|
const visited = new Set();
|
||||||
|
function addTree(entry, isChild) {
|
||||||
|
if (visited.has(entry.id)) return;
|
||||||
|
visited.add(entry.id);
|
||||||
|
addEntry(entry, isChild);
|
||||||
|
for (const child of childrenOf.get(entry.id) || []) addTree(child, true);
|
||||||
}
|
}
|
||||||
for (const o of orphans) addEntry(o, true);
|
for (const top of tops) addTree(top, false);
|
||||||
|
for (const o of orphans) addTree(o, true);
|
||||||
|
|
||||||
const liveIds = new Set(queue.map((e) => e.id));
|
const liveIds = new Set(queue.map((e) => e.id));
|
||||||
for (const [id, entry] of rebuildQueueRowCache) {
|
for (const [id, entry] of rebuildQueueRowCache) {
|
||||||
|
|
|
||||||
|
|
@ -94,6 +94,9 @@ pub(super) async fn run_node(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
|
||||||
NodeKind::WriteDropin => run_write_dropin(coord, claim).await,
|
NodeKind::WriteDropin => run_write_dropin(coord, claim).await,
|
||||||
NodeKind::WritePermFile => run_write_perm_file(coord, claim, &ctx).await,
|
NodeKind::WritePermFile => run_write_perm_file(coord, claim, &ctx).await,
|
||||||
NodeKind::ApprovalDeploy => run_approval_deploy(coord, claim).await,
|
NodeKind::ApprovalDeploy => run_approval_deploy(coord, claim).await,
|
||||||
|
// A pure grouping anchor (boot root): no work, completes immediately so
|
||||||
|
// its child DAGs settle it and the boot tree resolves.
|
||||||
|
NodeKind::Noop => Ok(NodeOutput::default()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -125,6 +125,12 @@ pub enum NodeKind {
|
||||||
/// deploy stays inside `actions.rs` in v1 — deliberately not
|
/// deploy stays inside `actions.rs` in v1 — deliberately not
|
||||||
/// modeled as scheduler nodes (see the design doc §9).
|
/// modeled as scheduler nodes (see the design doc §9).
|
||||||
ApprovalDeploy,
|
ApprovalDeploy,
|
||||||
|
/// No-op anchor that completes immediately with no work. It exists so a
|
||||||
|
/// boot's `StartupSweep` + per-agent `Reconcile` child DAGs can hang off
|
||||||
|
/// one root (`parent_id`) and render as a single boot tree on the
|
||||||
|
/// dashboard. Holds no lease and no build slot; the child DAGs it anchors
|
||||||
|
/// still run concurrently — the grouping is a display link, not a dep edge.
|
||||||
|
Noop,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl NodeKind {
|
impl NodeKind {
|
||||||
|
|
@ -144,6 +150,7 @@ impl NodeKind {
|
||||||
NodeKind::WriteDropin => "write_dropin",
|
NodeKind::WriteDropin => "write_dropin",
|
||||||
NodeKind::WritePermFile => "write_perm_file",
|
NodeKind::WritePermFile => "write_perm_file",
|
||||||
NodeKind::ApprovalDeploy => "approval_deploy",
|
NodeKind::ApprovalDeploy => "approval_deploy",
|
||||||
|
NodeKind::Noop => "noop",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -281,6 +281,30 @@ pub fn meta_update(
|
||||||
/// Boot-time sweep parent: bump meta's hyperhive input (non-fatal),
|
/// Boot-time sweep parent: bump meta's hyperhive input (non-fatal),
|
||||||
/// then fan out `Rebuild` children for the precomputed stale agent
|
/// then fan out `Rebuild` children for the precomputed stale agent
|
||||||
/// list (topology-sorted by the caller).
|
/// list (topology-sorted by the caller).
|
||||||
|
/// Boot-time root anchor DAG: a single [`NodeKind::Noop`] node that groups
|
||||||
|
/// this boot's `StartupSweep` + per-agent `Reconcile` child DAGs (linked via
|
||||||
|
/// `parent_id`) into one tree so the dashboard renders the boot as one entry.
|
||||||
|
/// Holds no lease and does no work — the children it anchors still run
|
||||||
|
/// concurrently. `auto_update::run` submits it first (when there's any boot
|
||||||
|
/// work), then parents the sweep + reconciles onto its id.
|
||||||
|
pub fn boot_root(reason: String) -> DagSpec {
|
||||||
|
DagSpec {
|
||||||
|
template: Template::Boot,
|
||||||
|
agent: "hyperhive".to_owned(),
|
||||||
|
source: Source::AutoUpdate,
|
||||||
|
reason,
|
||||||
|
parent_id: None,
|
||||||
|
approval_id: None,
|
||||||
|
inputs: Vec::new(),
|
||||||
|
perm_payload: None,
|
||||||
|
transient: None,
|
||||||
|
nodes: vec![NodeSpec {
|
||||||
|
kind: NodeKind::Noop,
|
||||||
|
deps: Vec::new(),
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn startup_sweep(reason: String, stale_agents: Vec<String>) -> DagSpec {
|
pub fn startup_sweep(reason: String, stale_agents: Vec<String>) -> DagSpec {
|
||||||
DagSpec {
|
DagSpec {
|
||||||
template: Template::StartupSweep,
|
template: Template::StartupSweep,
|
||||||
|
|
|
||||||
|
|
@ -273,10 +273,52 @@ pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
|
||||||
"boot reconcile"
|
"boot reconcile"
|
||||||
);
|
);
|
||||||
|
|
||||||
// Sweep parent whenever ANY marker is stale — even when every
|
submit_boot_tree(&coord, any_stale, fanout, drifted, n_deferred, n_skipped);
|
||||||
// stale agent is wanted-offline: the hyperhive lock bump must land
|
Ok(())
|
||||||
// now so their later start-upgrade rebuilds build against it.
|
}
|
||||||
// No stale agents ⇒ no sweep ⇒ no meta commit on a no-change boot.
|
|
||||||
|
/// Submit this boot's DAGs under one `Boot` root: a `Noop` anchor with the
|
||||||
|
/// startup sweep + per-agent reconciles hung off it via `parent_id`, so the
|
||||||
|
/// dashboard renders the boot as a single tree instead of N+1 rows. No-op when
|
||||||
|
/// there's nothing to do. The `parent_id` link is a display grouping, not a
|
||||||
|
/// dependency edge — the children run concurrently, so the reconciles never
|
||||||
|
/// wait behind the lock bump.
|
||||||
|
fn submit_boot_tree(
|
||||||
|
coord: &Arc<Coordinator>,
|
||||||
|
any_stale: bool,
|
||||||
|
fanout: Vec<String>,
|
||||||
|
drifted: Vec<String>,
|
||||||
|
n_deferred: usize,
|
||||||
|
n_skipped: usize,
|
||||||
|
) {
|
||||||
|
// Only emit a boot root when there's actually boot work — a fully-quiet
|
||||||
|
// boot (nothing stale, nothing drifted) submits nothing, exactly as before.
|
||||||
|
let boot_root_id = if any_stale || !drifted.is_empty() {
|
||||||
|
let reason = format!(
|
||||||
|
"boot: {} rebuild(s), {} reconcile(s), {} deferred (offline), {} up-to-date",
|
||||||
|
fanout.len(),
|
||||||
|
drifted.len(),
|
||||||
|
n_deferred,
|
||||||
|
n_skipped,
|
||||||
|
);
|
||||||
|
match coord
|
||||||
|
.job_queue
|
||||||
|
.submit(crate::job_queue::templates::boot_root(reason))
|
||||||
|
{
|
||||||
|
Ok(id) => Some(id),
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(error = ?e, "boot reconcile: boot-root submit failed");
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
// Sweep whenever ANY marker is stale — even when every stale agent is
|
||||||
|
// wanted-offline: the hyperhive lock bump must land now so their later
|
||||||
|
// start-upgrade rebuilds build against it. No stale agents ⇒ no sweep ⇒ no
|
||||||
|
// meta commit on a no-change boot.
|
||||||
if any_stale {
|
if any_stale {
|
||||||
let reason = format!(
|
let reason = format!(
|
||||||
"startup sweep: {} rebuild(s), {} deferred (offline), {} up-to-date",
|
"startup sweep: {} rebuild(s), {} deferred (offline), {} up-to-date",
|
||||||
|
|
@ -284,27 +326,24 @@ pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
|
||||||
n_deferred,
|
n_deferred,
|
||||||
n_skipped,
|
n_skipped,
|
||||||
);
|
);
|
||||||
if let Err(e) = coord
|
let mut spec = crate::job_queue::templates::startup_sweep(reason, fanout);
|
||||||
.job_queue
|
spec.parent_id = boot_root_id;
|
||||||
.submit(crate::job_queue::templates::startup_sweep(reason, fanout))
|
if let Err(e) = coord.job_queue.submit(spec) {
|
||||||
{
|
|
||||||
tracing::warn!(error = ?e, "boot reconcile: sweep submit failed");
|
tracing::warn!(error = ?e, "boot reconcile: sweep submit failed");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for name in drifted {
|
for name in drifted {
|
||||||
if let Err(e) = coord
|
let mut spec = crate::job_queue::templates::reconcile_only(
|
||||||
.job_queue
|
crate::job_queue::Template::Reconcile,
|
||||||
.submit(crate::job_queue::templates::reconcile_only(
|
&name,
|
||||||
crate::job_queue::Template::Reconcile,
|
crate::job_queue::Source::AutoUpdate,
|
||||||
&name,
|
"boot reconcile".to_owned(),
|
||||||
crate::job_queue::Source::AutoUpdate,
|
None,
|
||||||
"boot reconcile".to_owned(),
|
);
|
||||||
None,
|
spec.parent_id = boot_root_id;
|
||||||
))
|
if let Err(e) = coord.job_queue.submit(spec) {
|
||||||
{
|
|
||||||
tracing::warn!(%name, error = ?e, "boot reconcile: submit failed");
|
tracing::warn!(%name, error = ?e, "boot reconcile: submit failed");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
coord.emit_rebuild_queue_snapshot();
|
coord.emit_rebuild_queue_snapshot();
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -40,6 +40,10 @@ pub enum Template {
|
||||||
/// Bare converge of observed power state to the persisted intent
|
/// Bare converge of observed power state to the persisted intent
|
||||||
/// (boot reconcile).
|
/// (boot reconcile).
|
||||||
Reconcile,
|
Reconcile,
|
||||||
|
/// Boot-time root anchor: a single no-op node that groups this boot's
|
||||||
|
/// `StartupSweep` + per-agent `Reconcile` child DAGs under one tree, so the
|
||||||
|
/// dashboard renders the whole boot as one entry instead of N+1 rows.
|
||||||
|
Boot,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Template {
|
impl Template {
|
||||||
|
|
@ -57,6 +61,7 @@ impl Template {
|
||||||
Template::Start => "start",
|
Template::Start => "start",
|
||||||
Template::Stop => "stop",
|
Template::Stop => "stop",
|
||||||
Template::Reconcile => "reconcile",
|
Template::Reconcile => "reconcile",
|
||||||
|
Template::Boot => "boot",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue