job_queue: add NodeKind::Reparent (topology moves as a queue node, #2719)

This commit is contained in:
damocles 2026-07-26 18:46:38 +02:00 committed by mara
commit 05b373474a
6 changed files with 177 additions and 5 deletions

View file

@ -54,6 +54,7 @@ Cheap — no build slot:
| `Drain` | await the harness clearing the fence, bounded by the 3-min graceful-stop timeout; resolves ok either way |
| `WriteDropin` | `set_nspawn_flags` + `set_resource_limits` + daemon-reload |
| `WritePermFile` | commit `tool-groups.json` / `capabilities.json` (single git commit under `META_LOCK`) + emit the P3RM1SS10NS snapshots |
| `Reparent` | `set-parent` / `set-parent-bulk`: apply every `(child, new_parent)` move under one `META_LOCK` commit (`meta::bulk_commit_topology`), send the per-agent move notifications, rescan + diff-emit. Agentless like `MetaLock` — a bulk move can span multiple agents, and a reparent touches the meta repo, not any one container. `moves` is `(Ident, Option<Ident>)` pairs, not raw strings — mara: "use Ident type instead of string" (#2719, issuecomment 42691). Rides `Template::MetaUpdate` rather than a dedicated `Template` variant — that enum is on its way out (see `#2665`, still open/blocked on a scope question) and is already internal-only (not on `DagView`'s wire shape), so the stand-in only affects `terminal_hook` dispatch (resolves to no hook either way) and history-retention bucketing |
There is deliberately **no `GitCommit` node**: `meta.rs` fuses each mutation
with its commit under its internal `META_LOCK` mutex, so a standalone commit
@ -66,7 +67,7 @@ container build:
- **The deploy window** (`Resource::MetaWindow`): a global, capacity-1 queue
resource declared by every node kind that mutates the meta repo — `MetaSync`,
`MetaLock`, `WritePermFile`, `Provision`'s agent registration, and
`MetaLock`, `WritePermFile`, `Reparent`, `Provision`'s agent registration, and
`DeployWindow` — the deploy subtree's root, which holds it across every
phase below it (`NodeKind::needs_meta_window`). Two meta
mutations can therefore never interleave, so no commit lands inside another
@ -122,6 +123,7 @@ perm-change(a): WritePermFile(a) → «rebuild subgraph»
meta-update(inp): MetaLock(inp) →(in-DAG) «rebuild subgraph» per affected agent
boot: (if any rev marker stale) MetaLock(hyperhive) →(in-DAG) «rebuild subgraph» per stale agent;
plus Reconcile(a) for every drifted agent (all ONE DAG)
reparent(moves): Reparent(moves) (no rebuild — topology.json is read live)
```
Notable collapses:
@ -190,7 +192,7 @@ resources are free. Resources:
write, not a container op, but takes the lease anyway so a power-op DAG's
intent write + reconcile is atomic — two racing ops can't clobber intent
before either reconciles.) **Lease-exempt**: `MetaSync`, `Prebuild`,
`MetaLock`, `WritePermFile`
`MetaLock`, `WritePermFile`, `Reparent`
they touch the store / meta, not the running container, which is exactly
why a stop can land while another DAG's prebuild is still building.

View file

@ -89,6 +89,7 @@ pub(super) async fn run_node(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
NodeKind::Drain { .. } => run_drain(coord, claim).await,
NodeKind::WriteDropin { .. } => run_write_dropin(coord, claim).await,
NodeKind::WritePermFile { .. } => run_write_perm_file(coord, claim).await,
NodeKind::Reparent { .. } => run_reparent(coord, claim).await,
NodeKind::DeployWindow { .. } => run_deploy_window(claim),
NodeKind::MergeVerify { .. } => run_merge_verify(coord, claim).await,
NodeKind::DeployApply { .. } => run_deploy_apply(coord, claim).await,
@ -530,6 +531,33 @@ async fn run_write_perm_file(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
Ok(NodeOutput::default())
}
/// Apply the node's `(child, new_parent)` moves as one `META_LOCK`-fused
/// commit (`Coordinator::reparent_bulk_with_notify`, which already handles
/// both the single- and bulk-move case, sends the per-agent move
/// notifications, and rescans + diff-emits the container tree). Runs under
/// the deploy window (`NodeKind::needs_meta_window`), same reasoning as
/// `run_write_perm_file`: a topology commit landing inside another node's
/// staged deploy window would sweep the staged lock into its commit.
async fn run_reparent(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
let NodeKind::Reparent { moves } = &claim.kind else {
anyhow::bail!("run_reparent on a non-Reparent node");
};
let refs: Vec<(&str, Option<&str>)> = moves
.iter()
.map(|(child, parent)| {
(
child.as_str(),
parent.as_ref().map(hive_types::Ident::as_str),
)
})
.collect();
coord
.reparent_bulk_with_notify(&refs)
.await
.map_err(|e| anyhow::anyhow!(e))?;
Ok(NodeOutput::default())
}
/// The approval id every deploy phase re-reads its approval row by. Fails the
/// node when the DAG carries none, which would mean a `MergeConfigPr` DAG was
/// built without going through `templates::approval_deploy`.

View file

@ -186,6 +186,25 @@ pub enum NodeKind {
/// (commit fused under `META_LOCK`). The payload rides this node — the only
/// consumer — rather than the generic DAG container.
WritePermFile { agent: String, payload: PermPayload },
/// Topology move(s) — `set-parent` (len 1) or `set-parent-bulk` (len N) —
/// as a single queue node. Agentless like [`NodeKind::MetaLock`]: a
/// reparent touches the meta repo, not any one container, and a bulk
/// move spans multiple agents anyway. `needs_meta_window() = true`, same
/// precedent as [`NodeKind::WritePermFile`] (also a small
/// git-commit-under-`META_LOCK` op) — a reparent's commit must not land
/// inside another node's staged deploy `prepare_deploy`→`finalize_deploy`
/// window. `(child, new_parent)` pairs, applied in order under one
/// `META_LOCK` acquisition / one git commit (`meta::bulk_commit_topology`
/// handles both the single- and multi-move case uniformly).
#[allow(
dead_code,
reason = "constructed by templates::reparent(), landed ahead of the call-site swap \
(pending an answer on whether that swap should be synchronous or \
fire-and-forget) exercised today by job_queue::tests only"
)]
Reparent {
moves: Vec<(hive_types::Ident, Option<hive_types::Ident>)>,
},
/// Group root of the approval-deploy (`MergeConfigPr`) subtree, and the
/// node that **owns the deploy window**. It performs no work of its own —
/// it exists so the resources it declares (the global
@ -316,6 +335,7 @@ impl NodeKind {
NodeKind::Drain { .. } => "drain",
NodeKind::WriteDropin { .. } => "write_dropin",
NodeKind::WritePermFile { .. } => "write_perm_file",
NodeKind::Reparent { .. } => "reparent",
NodeKind::DeployWindow { .. } => "deploy_window",
NodeKind::MergeVerify { .. } => "merge_verify",
NodeKind::DeployApply { .. } => "deploy_apply",
@ -327,7 +347,8 @@ impl NodeKind {
}
/// The agent this node targets, or `""` for agentless kinds
/// ([`NodeKind::MetaLock`] on the `hyperhive` pseudo-agent, and the
/// ([`NodeKind::MetaLock`] on the `hyperhive` pseudo-agent,
/// [`NodeKind::Reparent`] which can span multiple agents, and the
/// [`NodeKind::Dag`] container).
#[must_use]
pub fn agent(&self) -> &str {
@ -352,7 +373,7 @@ impl NodeKind {
| NodeKind::FinalizeDeploy { agent }
| NodeKind::DeployTail { agent }
| NodeKind::SetWanted { agent, .. } => agent,
NodeKind::MetaLock { .. } | NodeKind::Dag { .. } => "",
NodeKind::MetaLock { .. } | NodeKind::Reparent { .. } | NodeKind::Dag { .. } => "",
}
}
@ -372,7 +393,7 @@ impl NodeKind {
/// Container-affecting kinds require the DAG to hold the agent's
/// lifecycle lease (acquired at the first such node, held until the
/// DAG is terminal). Lease-exempt kinds (`MetaSync`, `Prebuild`,
/// `Provision`, `MetaLock`, `WritePermFile`) touch the store / meta repo, not the
/// `Provision`, `MetaLock`, `WritePermFile`, `Reparent`) touch the store / meta repo, not the
/// running container — which is exactly why a `Prebuild` can overlap
/// another DAG's work on the same agent. `Provision` precedes the
/// container's existence entirely, so the lease is first taken at the
@ -425,6 +446,7 @@ impl NodeKind {
| NodeKind::Provision { .. }
| NodeKind::MetaLock { .. }
| NodeKind::WritePermFile { .. }
| NodeKind::Reparent { .. }
| NodeKind::DeployWindow { .. }
| NodeKind::FinalizeDeploy { .. }
)

View file

@ -420,3 +420,28 @@ pub fn meta_update(
) -> u64 {
submit_and_emit(coord, templates::meta_update(inputs, source, reason, None))
}
/// Topology move(s) as a queue DAG. `moves` is `(child, new_parent)` pairs —
/// one entry for `set-parent`, N for `set-parent-bulk`. Not yet wired to the
/// `set-parent`/`set-parent-bulk` HTTP handlers or the `hivectl`/MCP
/// `SetParent` surface — those still call `Coordinator::reparent*_with_notify`
/// directly, which blocks until the commit lands and returns a synchronous
/// `ok`/`err`, unlike every other queue-backed op. Whether that call-site
/// swap should keep the synchronous contract or go fire-and-forget (submit
/// returns a DAG id immediately, like everything else in this module) is an
/// open question — exists so the `NodeKind::Reparent` shape is exercised
/// end-to-end (tests, and any future caller) ahead of that call-site swap.
#[allow(
dead_code,
reason = "landed ahead of the server.rs/dashboard::topology call-site swap, pending an \
answer on whether that swap should be synchronous or fire-and-forget \
exercised today by job_queue::tests"
)]
pub fn reparent(
coord: &Arc<Coordinator>,
moves: Vec<(hive_types::Ident, Option<hive_types::Ident>)>,
source: Source,
reason: String,
) -> u64 {
submit_and_emit(coord, templates::reparent(moves, source, reason))
}

View file

@ -22,6 +22,7 @@
//! spawn(a): Provision(a) → Create(a) → WriteDropin(a) → Reconcile(a) [wanted=Up at approve]
//! perm-change(a): WritePermFile(a) → «rebuild subgraph»
//! meta-update(inp): MetaLock(inp) →«in-DAG rebuild subgraph per affected a»
//! reparent(moves): Reparent(moves) [no rebuild — topology.json is read live]
//! ```
//!
//! For the dynamic power-op shapes (`stop` / `start` / `restart`, built from
@ -341,6 +342,47 @@ pub fn meta_update(
}
}
/// Topology move(s) as a single-node DAG. `moves` is `(child, new_parent)`
/// pairs — len 1 for `set-parent`, len N for `set-parent-bulk`, applied
/// uniformly by the one [`NodeKind::Reparent`] node (which holds the global
/// meta window for its duration, same precedent as [`NodeKind::WritePermFile`]).
/// No rebuild subgraph: `topology.json` is read live by every consumer
/// (dashboard tree, `<parent>`/`<children>` sentinel routing, permission
/// checks), so a parent move needs no container rebuild to take effect.
/// No transient pill either — the node is agentless (no lease to hang one
/// off of) and near-instant.
///
/// Rides `Template::MetaUpdate` rather than a dedicated variant because
/// `Template` is being removed and nothing should dispatch on it — a fresh
/// variant would just be more surface to delete later. `Template` is already
/// internal-only (not on `DagView`'s wire shape — the dashboard derives its
/// label from `nodes`), so the choice of stand-in variant only affects
/// `terminal_hook` dispatch (`MetaUpdate` resolves to `None`, same as a
/// dedicated variant would) and the per-template history-retention bucket —
/// both cosmetic. Swap this to whatever the eventual node-kind-derived
/// dispatch lands with, whenever it lands.
#[allow(
dead_code,
reason = "landed ahead of the server.rs/dashboard::topology call-site swap, pending an \
answer on whether that swap should be synchronous or fire-and-forget \
exercised today by job_queue::tests and submit::reparent"
)]
pub fn reparent(
moves: Vec<(hive_types::Ident, Option<hive_types::Ident>)>,
source: Source,
reason: String,
) -> DagSpec {
DagSpec {
template: Template::MetaUpdate,
source,
reason,
approval_id: None,
inputs: Vec::new(),
transient: None,
nodes: vec![node(NodeKind::Reparent { moves }, Vec::new())],
}
}
// The boot is assembled inline in `workers/auto_update.rs::submit_boot_tree`
// as ONE `Boot` DAG (a sweep `MetaLock` root that grows rebuild subgraphs
// in-DAG, plus a `Reconcile` root per drifted agent) — no anchor node and no

View file

@ -13,6 +13,10 @@ fn submit(q: &JobQueue, spec: DagSpec) -> u64 {
q.submit(spec).expect("valid spec")
}
fn ident(s: &str) -> hive_types::Ident {
hive_types::Ident::parse(s).expect("valid test ident")
}
fn rebuild(agent: &str, reason: &str) -> DagSpec {
templates::rebuild(agent, Source::Manual, reason.to_owned(), true)
}
@ -1357,3 +1361,52 @@ fn perm_change_shape_prefixes_rebuild_chain() {
}
assert_eq!(state_of(&q, id), State::Done);
}
#[test]
fn reparent_shape_is_a_lone_agentless_meta_window_node() {
// Single-move `set-parent` shape: one node, no rebuild subgraph (no
// container rebuild needed for a parent move), agentless like
// `MetaLock`, and it must declare the meta window — a topology commit
// must not land inside another node's staged deploy window.
let q = JobQueue::new(1);
let id = submit(
&q,
templates::reparent(
vec![(ident("alice"), Some(ident("bob")))],
Source::Manual,
"set-parent".to_owned(),
),
);
let c = claim_one(&q);
assert_eq!(c.kind.as_str(), "reparent");
assert_eq!(c.agent, "", "Reparent is agentless — no per-agent lease");
assert!(
c.kind.needs_meta_window(),
"a topology commit must hold the same MetaWindow as WritePermFile"
);
assert!(!c.kind.needs_lease());
assert!(!c.kind.needs_build_slot());
q.complete_node(id, c.node_id, Ok(()));
assert_eq!(state_of(&q, id), State::Done);
}
#[test]
fn reparent_bulk_shape_carries_every_move_on_one_node() {
// `set-parent-bulk`: still ONE node (one git commit, `moves.len() > 1`),
// not one node per move — bulk atomicity across every move in the
// request is the reason a single node was chosen in the first place.
let moves = vec![(ident("alice"), Some(ident("bob"))), (ident("carol"), None)];
let q = JobQueue::new(1);
let id = submit(
&q,
templates::reparent(moves.clone(), Source::Manual, "set-parent-bulk".to_owned()),
);
let c = claim_one(&q);
assert_eq!(c.kind.as_str(), "reparent");
let NodeKind::Reparent { moves: got } = &c.kind else {
panic!("expected a Reparent node, got {:?}", c.kind);
};
assert_eq!(got, &moves);
q.complete_node(id, c.node_id, Ok(()));
assert_eq!(state_of(&q, id), State::Done);
}