From 7c0d9d2379c39a196d06b92c65dc1f4a4aca1dc5 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 4 Aug 2026 11:17:37 +0200 Subject: [PATCH 01/10] wip(#3001): remove submit layer, rescue power ops into job_queue/power.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TREE IS RED ON PURPOSE — there is no compiling intermediate between deleting submit and converting every caller. Checkpoint commit so the work is durable; do not "fix" it by restoring submit. Done: - JobQueue::submit -> JobQueue::insert (no source/reason/container; returns the ids insert_job names). - submit.rs deleted. Its 6 pure chain builders + 3 async *_many gatherers were NOT wrapper code and are rescued into job_queue/power.rs (templates.rs documents power ops as living outside it, because their shape needs a live is_running read). - Converted: meta_inputs 1, topology 2, permissions 3, auto_update 2, actions 3, lifecycle_handlers 3. - Dropped source/reason at every converted site: nothing ever read NodeKind::Dag's fields (only `{ .. }` matches exist), so they are write-only. Dead reason-only locals deleted; the boot sweep's summary became a tracing::info! rather than being lost. Remaining: dashboard/lifecycle_ops 7, server.rs 7, and the test suite — tests.rs has its own submit() helper whose u64 return is used as the handle to navigate the inserted DAG, so those need a different way to find nodes, not a mechanical port. --- hive-c0re/src/actions.rs | 52 ++-- hive-c0re/src/dashboard/meta_inputs.rs | 16 +- hive-c0re/src/dashboard/permissions.rs | 68 ++++-- hive-c0re/src/dashboard/topology.rs | 30 ++- hive-c0re/src/job_queue/mod.rs | 19 +- .../src/job_queue/{submit.rs => power.rs} | 228 +++++------------- hive-c0re/src/job_queue/tests.rs | 18 +- .../src/socket_server/lifecycle_handlers.rs | 33 +-- hive-c0re/src/workers/auto_update.rs | 35 +-- 9 files changed, 198 insertions(+), 301 deletions(-) rename hive-c0re/src/job_queue/{submit.rs => power.rs} (56%) diff --git a/hive-c0re/src/actions.rs b/hive-c0re/src/actions.rs index 7333f387..8c531076 100644 --- a/hive-c0re/src/actions.rs +++ b/hive-c0re/src/actions.rs @@ -55,13 +55,12 @@ pub async fn approve(coord: Arc, id: i64) -> Result<()> { // nothing). let inputs: Vec = serde_json::from_str(&approval.commit_ref).unwrap_or_default(); - let submitted = coord.job_queue.submit( - crate::job_queue::Source::Approval, - format!("approval #{id} meta input update"), - |b| crate::job_queue::templates::meta_update(b, inputs, Some(id)), - ); - if let Err(e) = submitted { - return Err(e.context("submit meta-update dag")); + let inserted = coord.job_queue.insert(|b| { + crate::job_queue::templates::meta_update(b, inputs, Some(id)); + Vec::new() + }); + if let Err(e) = inserted { + return Err(e.context("insert meta-update dag")); } coord.emit_rebuild_queue_snapshot(); Ok(()) @@ -75,13 +74,12 @@ pub async fn approve(coord: Arc, id: i64) -> Result<()> { { tracing::warn!(agent = %approval.agent, error = ?e, "agent_power: seed on spawn failed"); } - let submitted = coord.job_queue.submit( - crate::job_queue::Source::Approval, - format!("approval #{id} spawn"), - |b| crate::job_queue::templates::spawn(b, approval.agent.as_str(), id), - ); - if let Err(e) = submitted { - return Err(e.context("submit spawn dag")); + let inserted = coord.job_queue.insert(|b| { + crate::job_queue::templates::spawn(b, approval.agent.as_str(), id); + Vec::new() + }); + if let Err(e) = inserted { + return Err(e.context("insert spawn dag")); } coord.emit_rebuild_queue_snapshot(); Ok(()) @@ -106,12 +104,7 @@ pub async fn approve(coord: Arc, id: i64) -> Result<()> { // `run_deploy_apply` (ff-merge, then grows the rebuild subgraph), // `run_finalize_deploy` (deploy tag + lock commit) and // `run_deploy_tail` (compensation + forge mirror). - enqueue_approval_rebuild( - &coord, - approval.agent.as_str(), - id, - format!("approval #{id} merge config pr"), - ); + enqueue_approval_rebuild(&coord, approval.agent.as_str(), id); Ok(()) } } @@ -121,19 +114,12 @@ pub async fn approve(coord: Arc, id: i64) -> Result<()> { /// dispatch arm — the work ends in a container rebuild routed through the /// queue. See [`crate::job_queue::templates::approval_deploy`] for the node /// shape; the executor dispatches each node to the `run_deploy_*` bodies below. -fn enqueue_approval_rebuild( - coord: &Arc, - agent: &str, - approval_id: i64, - reason: String, -) { - if let Err(e) = coord - .job_queue - .submit(crate::job_queue::Source::Approval, reason, |b| { - crate::job_queue::templates::approval_deploy(b, agent, approval_id); - }) - { - tracing::error!(%agent, approval_id, error = ?e, "submit approval deploy dag failed"); +fn enqueue_approval_rebuild(coord: &Arc, agent: &str, approval_id: i64) { + if let Err(e) = coord.job_queue.insert(|b| { + crate::job_queue::templates::approval_deploy(b, agent, approval_id); + Vec::new() + }) { + tracing::error!(%agent, approval_id, error = ?e, "insert approval deploy dag failed"); } coord.emit_rebuild_queue_snapshot(); } diff --git a/hive-c0re/src/dashboard/meta_inputs.rs b/hive-c0re/src/dashboard/meta_inputs.rs index 3efe6b74..da1f3baa 100644 --- a/hive-c0re/src/dashboard/meta_inputs.rs +++ b/hive-c0re/src/dashboard/meta_inputs.rs @@ -208,16 +208,18 @@ pub(super) async fn post_meta_update( if inputs.is_empty() { return error_response("meta-update: no inputs selected"); } - let inputs_label = inputs.join(", "); // Cascade rebuild children fan out from the MetaLock node when the // lock bump lands — appended by the scheduler so they build against // the post-bump lock, and a failed bump simply fans out nothing. - crate::job_queue::submit::meta_update( - &state.coord, - inputs, - crate::job_queue::Source::Manual, - format!("meta-update via dashboard ({inputs_label})"), - ); + state + .coord + .job_queue + .insert(|b| { + crate::job_queue::templates::meta_update(b, inputs, None); + Vec::new() + }) + .expect("template-declared shapes are acyclic"); + state.coord.emit_rebuild_queue_snapshot(); (StatusCode::OK, "ok").into_response() } diff --git a/hive-c0re/src/dashboard/permissions.rs b/hive-c0re/src/dashboard/permissions.rs index 601ca400..11f920d3 100644 --- a/hive-c0re/src/dashboard/permissions.rs +++ b/hive-c0re/src/dashboard/permissions.rs @@ -155,15 +155,21 @@ pub(super) async fn post_tool_groups( // META_LOCK inside the WritePermFile node, so concurrent // batch-apply actions for different agents never race on the // shared tool-groups.json. - crate::job_queue::submit::perm_change( - &state.coord, - &logical, - crate::job_queue::Source::Manual, - "tool-group change via permissions UI".to_owned(), - crate::job_queue::PermPayload::ToolGroups { - groups: body.groups.clone(), - }, - ); + state + .coord + .job_queue + .insert(|b| { + crate::job_queue::templates::perm_change( + b, + &logical, + crate::job_queue::PermPayload::ToolGroups { + groups: body.groups.clone(), + }, + ); + Vec::new() + }) + .expect("template-declared shapes are acyclic"); + state.coord.emit_rebuild_queue_snapshot(); tracing::info!(agent = %logical, groups = ?body.groups, "operator: set tool-groups via dashboard"); Ok((StatusCode::OK, "ok").into_response()) } @@ -264,15 +270,21 @@ pub(super) async fn post_capabilities( // META_LOCK inside the WritePermFile node, so concurrent // batch-apply actions for different agents never race on the // shared capabilities.json. - crate::job_queue::submit::perm_change( - &state.coord, - &logical, - crate::job_queue::Source::Manual, - "capability change via dashboard".to_owned(), - crate::job_queue::PermPayload::Capabilities { - caps: body.caps.clone(), - }, - ); + state + .coord + .job_queue + .insert(|b| { + crate::job_queue::templates::perm_change( + b, + &logical, + crate::job_queue::PermPayload::Capabilities { + caps: body.caps.clone(), + }, + ); + Vec::new() + }) + .expect("template-declared shapes are acyclic"); + state.coord.emit_rebuild_queue_snapshot(); tracing::info!(agent = %logical, caps = ?body.caps, "operator: set capabilities via dashboard"); Ok((StatusCode::OK, "ok").into_response()) } @@ -359,13 +371,19 @@ pub(super) async fn post_permissions( } // Phase 2 — submit one combined PermChange DAG per affected agent. for (logical, groups, caps) in staged { - crate::job_queue::submit::perm_change( - &state.coord, - &logical, - crate::job_queue::Source::Manual, - "batch permission change via permissions UI".to_owned(), - crate::job_queue::PermPayload::Combined { groups, caps }, - ); + state + .coord + .job_queue + .insert(|b| { + crate::job_queue::templates::perm_change( + b, + &logical, + crate::job_queue::PermPayload::Combined { groups, caps }, + ); + Vec::new() + }) + .expect("template-declared shapes are acyclic"); + state.coord.emit_rebuild_queue_snapshot(); tracing::info!(agent = %logical, "operator: batch perm change via dashboard"); } Ok((StatusCode::OK, "ok").into_response()) diff --git a/hive-c0re/src/dashboard/topology.rs b/hive-c0re/src/dashboard/topology.rs index 8e8422fb..d007f446 100644 --- a/hive-c0re/src/dashboard/topology.rs +++ b/hive-c0re/src/dashboard/topology.rs @@ -94,12 +94,15 @@ pub(super) async fn post_set_parent( new_parent = ?new_parent, "operator: set-parent via dashboard" ); - submit::reparent( - &state.coord, - vec![(child, new_parent)], - Source::Manual, - "manual set-parent via dashboard".to_owned(), - ); + state + .coord + .job_queue + .insert(|b| { + crate::job_queue::templates::reparent(b, vec![(child, new_parent)]); + Vec::new() + }) + .expect("template-declared shapes are acyclic"); + state.coord.emit_rebuild_queue_snapshot(); Ok((StatusCode::OK, "ok").into_response()) } @@ -150,11 +153,14 @@ pub(super) async fn post_set_parent_bulk( .map_err(|e| error_problem(&e))?; let names: Vec<&str> = body.iter().map(|e| e.child.as_str()).collect(); tracing::info!(agents = ?names, "operator: set-parent-bulk via dashboard"); - submit::reparent( - &state.coord, - moves, - Source::Manual, - "manual set-parent-bulk via dashboard".to_owned(), - ); + state + .coord + .job_queue + .insert(|b| { + crate::job_queue::templates::reparent(b, moves); + Vec::new() + }) + .expect("template-declared shapes are acyclic"); + state.coord.emit_rebuild_queue_snapshot(); Ok((StatusCode::OK, "ok").into_response()) } diff --git a/hive-c0re/src/job_queue/mod.rs b/hive-c0re/src/job_queue/mod.rs index b9727493..655ad636 100644 --- a/hive-c0re/src/job_queue/mod.rs +++ b/hive-c0re/src/job_queue/mod.rs @@ -29,9 +29,9 @@ pub mod exec; pub mod model; +pub mod power; pub mod resource; pub mod scheduler; -pub mod submit; pub mod templates; #[cfg(test)] mod tests; @@ -208,20 +208,17 @@ impl JobQueue { /// # Errors /// Propagates a graph-insert error (dependencies that aren't /// dependency-topological). - pub fn submit( + pub fn insert( &self, - source: Source, - reason: String, - declare: impl FnOnce(&JobBuilder), - ) -> anyhow::Result { + declare: impl FnOnce(&JobBuilder) -> Vec, + ) -> anyhow::Result> { let mut inner = self.lock(); - let container = inner - .append(NodeKind::Dag { source, reason }, Vec::new(), None) - .map_err(|e| anyhow::anyhow!("job_queue: container insert failed: {e}"))?; - insert_group(&mut inner, declare, Some(container))?; + let named = inner + .insert_job(None, declare) + .map_err(|e| anyhow::anyhow!("job_queue: graph insert failed: {e}"))?; drop(inner); self.notify.notify_one(); - Ok(container.get()) + Ok(named) } /// The scheduler itself, for `hive_jobq`'s run-loop seam diff --git a/hive-c0re/src/job_queue/submit.rs b/hive-c0re/src/job_queue/power.rs similarity index 56% rename from hive-c0re/src/job_queue/submit.rs rename to hive-c0re/src/job_queue/power.rs index 3ec3e11b..1bb26b95 100644 --- a/hive-c0re/src/job_queue/submit.rs +++ b/hive-c0re/src/job_queue/power.rs @@ -1,59 +1,26 @@ -//! Request-level submit API — the surface the dashboard POST handlers, -//! the MCP socket handlers, and `hivectl` paths call. +//! Power ops (`stop` / `start` / `restart`) — the DAG shapes whose per-agent +//! form depends on **live** container state, so they cannot be static +//! [`super::templates`] entries. //! -//! The **power ops** (`stop` / `start` / `restart`) are built here, not in -//! `templates.rs`: each agent's subgraph shape depends on its *live* running -//! state, which needs an async `lifecycle::is_running` read that a pure/sync -//! template can't do. So these fns are async — they read each agent's state, -//! assemble a per-agent subgraph out of the shared pure primitives -//! (`JobBuilder::node` + `templates::rebuild_nodes`), all declaring into ONE job -//! (independent per-agent roots, concurrent on their own leases). +//! The split is the purity line, not the subject matter: the `*_chain` / +//! `*_nodes` builders below are pure (they take `running` / `stale` as +//! parameters, which is what keeps them unit-testable without a container), +//! and only the `*_many` entry points do the async `lifecycle::is_running` +//! read that produces those parameters. //! -//! Dynamic shape rule: `stop`/`start` carry a head `SetWanted(w)` (durable -//! intent write) — `restart` does NOT (it bounces the container but leaves -//! `wanted` untouched, so a deliberately-stopped agent isn't forced up). The -//! tail `Reconcile` (the convergence guarantee — cheap, noops when already -//! converged) is ALWAYS present; only the *mechanical* nodes -//! (`Signal`/`Drain`/`StopForUpdate`) are state-conditional (skipped for a -//! down agent — nothing to quiesce/stop). Keeping `Reconcile` in every shape -//! closes the TOCTOU window: if an agent flips state between the `is_running` -//! read and node execution, the tail `Reconcile` still converges it in-DAG, -//! with `StopForUpdate`-noop as the backstop — no reliance on an external -//! reconcile sweep. Every helper emits a fresh queue snapshot so the -//! dashboard shows the new DAG immediately. +//! Each entry point declares its group and inserts it. There is no metadata +//! parameter and no container node: attribution is not something every caller +//! has to invent, and a DAG is addressed by the nodes a template names. use std::sync::Arc; -use super::model::NodeKind; -use super::resource::Resource; -use super::templates::rebuild_nodes; -use super::{JobBuilder, Source, templates}; +use hive_jobq::NodeId; + +use super::JobBuilder; +use super::templates; use crate::coordinator::Coordinator; use crate::lifecycle; -fn submit_and_emit( - coord: &Arc, - source: Source, - reason: String, - declare: impl FnOnce(&JobBuilder), -) -> u64 { - let id = coord - .job_queue - .submit(source, reason, declare) - .expect("template-declared shapes are acyclic"); - coord.emit_rebuild_queue_snapshot(); - id -} - -/// Manual/approval-independent rebuild (always relocks the agent's -/// meta input — the meta-update cascade grows its own rebuild subgraphs -/// in-DAG instead of going through this surface). -pub fn rebuild(coord: &Arc, agent: &str, source: Source, reason: String) -> u64 { - submit_and_emit(coord, source, reason, |builder| { - templates::rebuild(builder, agent, true); - }) -} - // ---- dynamic power-op DAG assembly ---------------------------------------- // // The pure per-agent chain builders below take `running` (and `stale`) @@ -226,63 +193,44 @@ pub(crate) fn restart_nodes(builder: &JobBuilder, targets: &[(String, bool)], gr } } -/// Restart a single agent. Thin wrapper over [`restart_many`]. -pub async fn restart(coord: &Arc, agent: &str, source: Source, reason: String) -> u64 { - restart_many(coord, &[agent.to_owned()], false, source, reason).await -} +// ---- entry points --------------------------------------------------------- +// +// Each reads the live state its shape depends on, then declares + inserts. -/// Graceful restart of a single agent (signal → drain → stop → reconcile, -/// when running). Thin wrapper over [`restart_many`] with `graceful = true`. -pub async fn graceful_restart( - coord: &Arc, - agent: &str, - source: Source, - reason: String, -) -> u64 { - restart_many(coord, &[agent.to_owned()], true, source, reason).await -} - -/// Restart `agents` (one or many) in a **single** DAG — one per-agent -/// subgraph each, built dynamically from live running state and run -/// concurrently on their own leases. A running agent gets the stop→reconcile -/// chain (`graceful` prepends signal→drain); a down agent gets just a lone -/// `Reconcile` (nothing to stop). Restart never writes `wanted`, so the +/// Restart `agents` in a **single** DAG — one per-agent subgraph each, built +/// from live running state and run concurrently on their own leases. A running +/// agent gets the stop→reconcile chain (`graceful` prepends signal→drain); a +/// down agent gets a lone `Reconcile`. Restart never writes `wanted`, so the /// tail `Reconcile` converges each agent to its EXISTING intent — a -/// deliberately-stopped agent stays down. The whole hive-wide -/// `hivectl restart` is one DAG. +/// deliberately-stopped agent stays down. +/// +/// # Errors +/// Propagates a graph-insert error. pub async fn restart_many( coord: &Arc, agents: &[String], graceful: bool, - source: Source, - reason: String, -) -> u64 { +) -> anyhow::Result> { let mut targets = Vec::with_capacity(agents.len()); for agent in agents { targets.push((agent.clone(), lifecycle::is_running(agent).await)); } - submit_and_emit(coord, source, reason, |builder| { - restart_nodes(builder, &targets, graceful); - }) + let ids = coord.job_queue.insert(|b| { + restart_nodes(b, &targets, graceful); + Vec::new() + })?; + coord.emit_rebuild_queue_snapshot(); + Ok(ids) } -/// Start a single agent. Thin wrapper over [`start_many`]. -pub async fn start(coord: &Arc, agent: &str, source: Source, reason: String) -> u64 { - start_many(coord, &[agent.to_owned()], source, reason).await -} - -/// Start `agents` (one or many) in a **single** DAG — one per-agent subgraph -/// each, built dynamically from live state and run concurrently on their own -/// leases. A down agent gets `SetWanted(Up) → Reconcile` (or, rev stale, a -/// rebuild-then-start so it comes up on current derivations); an already- -/// running agent gets `SetWanted(Up) → Reconcile` (the reconcile noops). The -/// whole hive-wide `hivectl start` is one DAG. -pub async fn start_many( - coord: &Arc, - agents: &[String], - source: Source, - reason: String, -) -> u64 { +/// Start `agents` in a **single** DAG. A down agent gets +/// `SetWanted(Up) → Reconcile` (or, rev stale, a rebuild-then-start so it comes +/// up on current derivations); an already-running agent gets the same shape +/// with the reconcile noop'ing. +/// +/// # Errors +/// Propagates a graph-insert error. +pub async fn start_many(coord: &Arc, agents: &[String]) -> anyhow::Result> { let current = crate::auto_update::current_flake_rev(&coord.hyperhive_flake); let mut targets = Vec::with_capacity(agents.len()); for agent in agents { @@ -296,88 +244,34 @@ pub async fn start_many( } targets.push((agent.clone(), running, stale)); } - submit_and_emit(coord, source, reason, |builder| { - start_nodes(builder, &targets); - }) + let ids = coord.job_queue.insert(|b| { + start_nodes(b, &targets); + Vec::new() + })?; + coord.emit_rebuild_queue_snapshot(); + Ok(ids) } -/// Hard stop a single agent. Thin wrapper over [`stop_many`]. -pub async fn stop(coord: &Arc, agent: &str, source: Source, reason: String) -> u64 { - stop_many(coord, &[agent.to_owned()], false, source, reason).await -} - -/// Graceful stop of a single agent (signal → drain → reconcile, when -/// running). Thin wrapper over [`stop_many`] with `graceful = true`. -pub async fn graceful_stop( - coord: &Arc, - agent: &str, - source: Source, - reason: String, -) -> u64 { - stop_many(coord, &[agent.to_owned()], true, source, reason).await -} - -/// Stop `agents` (one or many) in a **single** DAG — one per-agent subgraph -/// each, built dynamically from live state and run concurrently on their own -/// leases. A running agent gets `SetWanted(Off) → [Signal → Drain →](graceful) -/// Reconcile`; a down agent gets just `SetWanted(Off) → Reconcile` (skips the -/// pointless quiesce, keeps the Reconcile as the race-up backstop). The whole -/// hive-wide `hivectl stop` is one DAG. +/// Stop `agents` in a **single** DAG. A running agent gets +/// `SetWanted(Off) → [Signal → Drain →](graceful) Reconcile`; a down agent +/// skips the pointless quiesce but keeps the `Reconcile` as the race-up +/// backstop. +/// +/// # Errors +/// Propagates a graph-insert error. pub async fn stop_many( coord: &Arc, agents: &[String], graceful: bool, - source: Source, - reason: String, -) -> u64 { +) -> anyhow::Result> { let mut targets = Vec::with_capacity(agents.len()); for agent in agents { targets.push((agent.clone(), lifecycle::is_running(agent).await)); } - submit_and_emit(coord, source, reason, |builder| { - stop_nodes(builder, &targets, graceful); - }) -} - -/// Perm change: commit the JSON file(s) then rebuild. -pub fn perm_change( - coord: &Arc, - agent: &str, - source: Source, - reason: String, - payload: super::PermPayload, -) -> u64 { - submit_and_emit(coord, source, reason, |builder| { - templates::perm_change(builder, agent, payload); - }) -} - -/// Meta-input lock bump; cascade rebuilds fan out on completion. -pub fn meta_update( - coord: &Arc, - inputs: Vec, - source: Source, - reason: String, -) -> u64 { - submit_and_emit(coord, source, reason, |builder| { - templates::meta_update(builder, inputs, None); - }) -} - -/// Topology move(s) as a queue DAG. `moves` is `(child, new_parent)` pairs — -/// one entry for `set-parent`, N for `set-parent-bulk`. Fire-and-forget like -/// everything else in this module: submits and returns a DAG id -/// immediately, the caller learns the outcome async (dashboard job view / -/// `hivectl`'s `QueueDag` poll). Wired from `server.rs`'s `HostRequest:: -/// SetParent` (hivectl) and `dashboard/topology.rs`'s `set-parent`/ -/// `set-parent-bulk` handlers. -pub fn reparent( - coord: &Arc, - moves: Vec<(hive_types::Ident, Option)>, - source: Source, - reason: String, -) -> u64 { - submit_and_emit(coord, source, reason, |builder| { - templates::reparent(builder, moves); - }) + let ids = coord.job_queue.insert(|b| { + stop_nodes(b, &targets, graceful); + Vec::new() + })?; + coord.emit_rebuild_queue_snapshot(); + Ok(ids) } diff --git a/hive-c0re/src/job_queue/tests.rs b/hive-c0re/src/job_queue/tests.rs index 4bc199a9..87d17dca 100644 --- a/hive-c0re/src/job_queue/tests.rs +++ b/hive-c0re/src/job_queue/tests.rs @@ -36,17 +36,17 @@ fn rebuild(builder: &JobBuilder, agent: &str) { /// Restart shape with every agent treated as **running** — the online /// shape (`[Signal→Drain→] StopForUpdate → Reconcile`, no `SetWanted` head) /// most queue-mechanics tests assume. Mirrors the pre-dynamic -/// `templates::restart` (which is now the state-aware `submit::restart_nodes`). +/// `templates::restart` (which is now the state-aware `power::restart_nodes`). fn restart_online(builder: &JobBuilder, agents: &[&str], graceful: bool) { let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect(); - submit::restart_nodes(builder, &targets, graceful); + power::restart_nodes(builder, &targets, graceful); } /// Stop shape with every agent treated as **running** — the online shape /// (`SetWanted → [Signal→Drain→](graceful) Reconcile`). fn stop_online(builder: &JobBuilder, agents: &[&str], graceful: bool) { let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect(); - submit::stop_nodes(builder, &targets, graceful); + power::stop_nodes(builder, &targets, graceful); } // `Claimed` / `ClaimReady` / `CompleteNode` lived here: a claim snapshot type @@ -682,7 +682,7 @@ fn multi_agent_start_one_dag_folds_per_agent_stale_rebuild() { // fresh: offline + not stale → SetWanted → Reconcile. // stale: offline + stale → SetWanted → «rebuild subgraph». let id = submit(&q, "hive-wide start", |builder| { - submit::start_nodes( + power::start_nodes( builder, &[ ("fresh".to_owned(), false, false), @@ -742,13 +742,13 @@ fn offline_agents_skip_mechanical_nodes_but_keep_reconcile() { let q = JobQueue::new(4); // Offline graceful stop → SetWanted(Off) → Reconcile (no Signal/Drain). let stop = submit(&q, "stop down", |builder| { - submit::stop_nodes(builder, &[("down".to_owned(), false)], true); + power::stop_nodes(builder, &[("down".to_owned(), false)], true); }); // Offline restart → a lone Reconcile (no SetWanted, no StopForUpdate): // nothing to bounce, and restart never rewrites intent, so the tail // Reconcile converges the down agent to its existing `wanted`. let restart = submit(&q, "restart down", |builder| { - submit::restart_nodes(builder, &[("down2".to_owned(), false)], true); + power::restart_nodes(builder, &[("down2".to_owned(), false)], true); }); let shape = |id: u64| -> Vec { // The group's work nodes: its subtree minus the container itself, @@ -1153,19 +1153,19 @@ fn cancelled_power_op_runs_no_compensating_node() { let q = JobQueue::new(1); let id = submit(&q, "bounce", |builder| { - submit::restart_nodes(builder, &targets, graceful); + power::restart_nodes(builder, &targets, graceful); }); assert_cancels_clean(&q, id, false, &format!("restart {case}")); let q = JobQueue::new(1); let id = submit(&q, "stop", |builder| { - submit::stop_nodes(builder, &targets, graceful); + power::stop_nodes(builder, &targets, graceful); }); assert_cancels_clean(&q, id, true, &format!("stop {case}")); let q = JobQueue::new(1); let id = submit(&q, "start", |builder| { - submit::start_nodes(builder, &[("agent-a".to_owned(), running, false)]); + power::start_nodes(builder, &[("agent-a".to_owned(), running, false)]); }); assert_cancels_clean(&q, id, true, &format!("start {case}")); } diff --git a/hive-c0re/src/socket_server/lifecycle_handlers.rs b/hive-c0re/src/socket_server/lifecycle_handlers.rs index 46446c12..968d31a3 100644 --- a/hive-c0re/src/socket_server/lifecycle_handlers.rs +++ b/hive-c0re/src/socket_server/lifecycle_handlers.rs @@ -20,13 +20,9 @@ pub(super) async fn handle_start(coord: &Arc, agent: &str, name: &s // Persist `wanted = Up` and submit the Start DAG; the submit layer // upgrades a stale-rev start to a full rebuild so the container // runs current nix derivations before it starts. - crate::job_queue::submit::start( - coord, - name, - crate::job_queue::Source::Manual, - format!("agent `{agent}` start tool"), - ) - .await; + if let Err(e) = crate::job_queue::power::start_many(coord, &[name.to_owned()]).await { + tracing::error!(%agent, %name, error = ?e, "start: insert failed"); + } Response::Ok } @@ -48,13 +44,9 @@ pub(super) async fn handle_restart(coord: &Arc, agent: &str, name: return err; } tracing::info!(%agent, %name, "submit restart"); - crate::job_queue::submit::restart( - coord, - name, - crate::job_queue::Source::Manual, - format!("agent `{agent}` restart tool"), - ) - .await; + if let Err(e) = crate::job_queue::power::restart_many(coord, &[name.to_owned()], false).await { + tracing::error!(%agent, %name, error = ?e, "restart: insert failed"); + } Response::Ok } @@ -153,12 +145,13 @@ pub(super) fn handle_update(coord: &Arc, agent: &str, name: &str) - return err; } tracing::info!(%agent, %name, "submit rebuild"); - crate::job_queue::submit::rebuild( - coord, - name, - crate::job_queue::Source::Manual, - format!("agent `{agent}` update tool"), - ); + if let Err(e) = coord.job_queue.insert(|b| { + crate::job_queue::templates::rebuild(b, name, true); + Vec::new() + }) { + tracing::error!(%agent, %name, error = ?e, "update: insert failed"); + } + coord.emit_rebuild_queue_snapshot(); Response::Ok } diff --git a/hive-c0re/src/workers/auto_update.rs b/hive-c0re/src/workers/auto_update.rs index 8132f0b7..a629e73f 100644 --- a/hive-c0re/src/workers/auto_update.rs +++ b/hive-c0re/src/workers/auto_update.rs @@ -109,12 +109,11 @@ pub async fn ensure_root_agent(coord: &Arc) -> Result<()> { tracing::warn!( "manager container exists but no applied flake — forcing rebuild to migrate" ); - if let Err(e) = coord.job_queue.submit( - crate::job_queue::Source::AutoUpdate, - "manager migration: no applied flake".to_owned(), - |b| crate::job_queue::templates::rebuild(b, MANAGER_NAME, true), - ) { - tracing::warn!(error = ?e, "manager migration rebuild submit failed"); + if let Err(e) = coord.job_queue.insert(|b| { + crate::job_queue::templates::rebuild(b, MANAGER_NAME, true); + Vec::new() + }) { + tracing::warn!(error = ?e, "manager migration rebuild insert failed"); } } else { tracing::debug!("manager container already present"); @@ -378,18 +377,19 @@ fn submit_boot_tree( n_deferred: usize, n_skipped: usize, ) { - use crate::job_queue::Source; - - // Fully-quiet boot (nothing stale, nothing drifted) submits nothing. + // Fully-quiet boot (nothing stale, nothing drifted) inserts nothing. if !any_stale && drifted.is_empty() { return; } - let reason = format!( - "boot: {} rebuild(s), {} reconcile(s), {} deferred (offline), {} up-to-date", - fanout.len(), - drifted.len(), - n_deferred, - n_skipped, + // The summary the sweep used to hand the container as its `reason` is a log + // line now: it was only ever stored on a node nobody read, and the counts + // are worth having where they can actually be seen. + tracing::info!( + rebuilds = fanout.len(), + reconciles = drifted.len(), + deferred = n_deferred, + up_to_date = n_skipped, + "boot: sweep" ); // The sweep's own rebuild subgraphs emit their `Rebuilt` events as they @@ -397,10 +397,11 @@ fn submit_boot_tree( // The subgraphs also carry their own per-agent crash-watch suppression // during their `Swap` (applied at claim time); a reconcile-only boot needs // no transient. - if let Err(e) = coord.job_queue.submit(Source::AutoUpdate, reason, |b| { + if let Err(e) = coord.job_queue.insert(|b| { boot_nodes(b, any_stale, fanout, drifted); + Vec::new() }) { - tracing::warn!(error = ?e, "boot: sweep DAG submit failed"); + tracing::warn!(error = ?e, "boot: sweep DAG insert failed"); } coord.emit_rebuild_queue_snapshot(); } From f04a0cee92f9928d9b7a5acd654ac904805de15c Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 4 Aug 2026 11:27:26 +0200 Subject: [PATCH 02/10] wip(#3001): convert remaining unblocked call sites; sweep docs 21 of 28 non-test call sites now insert directly. power.rs compiles. The only remaining errors are server.rs's 5, which are blocked: those sites feed the returned id into HostResponse::queued -> `queued_dags`, a wire field hivectl polls via QueueDag. Removing the container without answering that breaks hivectl's wait/progress loop; asked on the issue. Also swept the deleted symbol out of prose, not just code: - docs/coordinator.md: "the submit layer (job_queue/submit.rs)" -> the power layer (job_queue/power.rs), and "submits" -> "inserts". - templates.rs module doc: points at super::power for the power ops. - lifecycle_ops.rs module doc: says which path each op takes now. - mod.rs's insert_group comment restated the open issue verbatim ("a DAG is addressed by its container node, which submit inserts itself"). Replaced with what is actually true for that path. Dashboard behaviour deltas worth review: insert failures are now logged per agent instead of swallowed, and UPDATE-ALL emits one queue snapshot after the loop rather than one per agent. --- docs/coordinator.md | 8 +- hive-c0re/src/dashboard/lifecycle_ops.rs | 95 +++++++++++------------- hive-c0re/src/dashboard/topology.rs | 1 - hive-c0re/src/job_queue/mod.rs | 6 +- hive-c0re/src/job_queue/power.rs | 5 +- hive-c0re/src/job_queue/templates.rs | 4 +- 6 files changed, 54 insertions(+), 65 deletions(-) diff --git a/docs/coordinator.md b/docs/coordinator.md index 115d8ec2..b4e6b4c2 100644 --- a/docs/coordinator.md +++ b/docs/coordinator.md @@ -106,7 +106,7 @@ subgraph each (independent roots, run concurrently on their own leases), not N separate DAGs. **These are built dynamically from each agent's live running state** (an -async `lifecycle::is_running` read), so they live in `job_queue/submit.rs`, +async `lifecycle::is_running` read), so they live in `job_queue/power.rs`, not the pure/sync `templates.rs`. Per-agent shape rule: `stop`/`start` carry a head `SetWanted` (intent) — `restart` does not; the tail `Reconcile` (convergence guarantee — cheap, noops when already converged) is ALWAYS @@ -161,12 +161,12 @@ Notable collapses: Per-agent power *intent* — `wanted: Up | Offline` — is durable as the `agent_power` table in the coordinator DB (`hive-c0re/src/stores/power.rs`). `container_view` remains the observed *status*; `Reconcile` nodes converge the -two. Setting `wanted` is never a queued node: the submit layer -(`job_queue/submit.rs`) writes the row synchronously, then submits the DAG +two. Setting `wanted` is never a queued node: the power layer +(`job_queue/power.rs`) writes the row synchronously, then inserts the DAG whose `Reconcile` reads the fresh value — rapid toggles are last-writer-wins. Power toggles never commit to the meta repo. Every operator power surface — dashboard buttons, the MCP tools, and `hivectl stop/start/restart/kill` — -rides the queue through that submit layer, so intent, lease serialization, +rides the queue through that power layer, so intent, lease serialization, and crash-watch suppression can't drift per surface; the only direct starts left are the root-agent bootstrap and infra containers (no lease, no harness). Cancelling a still-queued power DAG reverts `wanted` to the diff --git a/hive-c0re/src/dashboard/lifecycle_ops.rs b/hive-c0re/src/dashboard/lifecycle_ops.rs index 87e7c2d0..b4f399e0 100644 --- a/hive-c0re/src/dashboard/lifecycle_ops.rs +++ b/hive-c0re/src/dashboard/lifecycle_ops.rs @@ -1,11 +1,12 @@ //! Container lifecycle endpoints for the dashboard. //! //! Rebuild / restart / start / stop (hard + graceful) / update-all all -//! submit DAGs to the job queue (`job_queue::submit`), so each shows a -//! visible queued→running transient on the dashboard — a direct -//! sub-second start/stop only flashed the badge. Start/stop also -//! persist the agent's `wanted` power intent before submitting; the -//! DAG's `Reconcile` converges to it. Destroy delegates to +//! insert DAGs into the job queue — the power ops via +//! [`crate::job_queue::power`], the static shapes straight through +//! `JobQueue::insert` — so each shows a visible queued→running transient on +//! the dashboard; a direct sub-second start/stop only flashed the badge. +//! Start/stop also persist the agent's `wanted` power intent before +//! inserting; the DAG's `Reconcile` converges to it. Destroy delegates to //! `actions::destroy` (optionally purging). use axum::{ @@ -27,7 +28,6 @@ pub(super) struct GracefulParams { } use super::{AppState, Ident, error_response, guard_agent_name, strip_container_prefix}; -use crate::job_queue::{Source, submit}; use crate::{actions, lifecycle}; /// Queue a rebuild DAG for `name`. @@ -50,12 +50,13 @@ pub(super) async fn post_rebuild( if let Some(reject) = guard_agent_name(&state, &logical).await { return reject; } - submit::rebuild( - &state.coord, - &logical, - Source::Manual, - "manual via dashboard ↻ R3BU1LD button".to_owned(), - ); + if let Err(e) = state.coord.job_queue.insert(|b| { + crate::job_queue::templates::rebuild(b, &logical, true); + Vec::new() + }) { + tracing::error!(agent = %logical, error = ?e, "rebuild: insert failed"); + } + state.coord.emit_rebuild_queue_snapshot(); (StatusCode::OK, "ok").into_response() } @@ -92,13 +93,11 @@ pub(super) async fn post_kill( // timeout fallback to a hard stop). The agent's lifecycle // lease keeps it from racing an in-flight rebuild for the same // agent, and per-node progress surfaces on the queue snapshot. - submit::graceful_stop( - &state.coord, - &logical, - Source::Manual, - "manual via dashboard graceful stop".to_owned(), - ) - .await; + if let Err(e) = + crate::job_queue::power::stop_many(&state.coord, &[logical.clone()], true).await + { + tracing::error!(agent = %logical, error = ?e, "graceful stop: insert failed"); + } return (StatusCode::OK, "ok").into_response(); } // Manager is stoppable from the dashboard like any other @@ -111,13 +110,10 @@ pub(super) async fn post_kill( // `socket_server.rs::Request::Kill` stays in place: a // manager calling Kill on its own container is self-suicide // mid-call, not a legitimate operator action. - submit::stop( - &state.coord, - &logical, - Source::Manual, - "manual via dashboard stop".to_owned(), - ) - .await; + if let Err(e) = crate::job_queue::power::stop_many(&state.coord, &[logical.clone()], false).await + { + tracing::error!(agent = %logical, error = ?e, "stop: insert failed"); + } (StatusCode::OK, "ok").into_response() } @@ -149,22 +145,18 @@ pub(super) async fn post_restart( return reject; } if params.graceful { - submit::graceful_restart( - &state.coord, - &logical, - Source::Manual, - "manual via dashboard graceful restart".to_owned(), - ) - .await; + if let Err(e) = + crate::job_queue::power::restart_many(&state.coord, &[logical.clone()], true).await + { + tracing::error!(agent = %logical, error = ?e, "graceful restart: insert failed"); + } return (StatusCode::OK, "ok").into_response(); } - submit::restart( - &state.coord, - &logical, - Source::Manual, - "manual via dashboard ↺ R3START button".to_owned(), - ) - .await; + if let Err(e) = + crate::job_queue::power::restart_many(&state.coord, &[logical.clone()], false).await + { + tracing::error!(agent = %logical, error = ?e, "restart: insert failed"); + } (StatusCode::OK, "ok").into_response() } @@ -226,13 +218,9 @@ pub(super) async fn post_start( return (StatusCode::OK, "ok").into_response(); } } - submit::start( - &state.coord, - &logical, - Source::Manual, - "manual via dashboard start".to_owned(), - ) - .await; + if let Err(e) = crate::job_queue::power::start_many(&state.coord, &[logical.clone()]).await { + tracing::error!(agent = %logical, error = ?e, "start: insert failed"); + } (StatusCode::OK, "ok").into_response() } @@ -411,13 +399,14 @@ pub(super) async fn post_update_all(State(state): State) -> Response { else { continue; }; - submit::rebuild( - &state.coord, - &logical, - Source::Manual, - "manual via dashboard 🌀 UPDATE ALL".to_owned(), - ); + if let Err(e) = state.coord.job_queue.insert(|b| { + crate::job_queue::templates::rebuild(b, &logical, true); + Vec::new() + }) { + tracing::error!(agent = %logical, error = ?e, "update-all: insert failed"); + } } + state.coord.emit_rebuild_queue_snapshot(); (StatusCode::OK, "ok").into_response() } diff --git a/hive-c0re/src/dashboard/topology.rs b/hive-c0re/src/dashboard/topology.rs index d007f446..79ac83a2 100644 --- a/hive-c0re/src/dashboard/topology.rs +++ b/hive-c0re/src/dashboard/topology.rs @@ -21,7 +21,6 @@ use utoipa::ToSchema; use problem_details::ProblemDetails; use super::{AppState, error_problem}; -use crate::job_queue::{Source, submit}; /// `POST /api/topology/set-parent` body. `child` is required. /// `new_parent` may be: diff --git a/hive-c0re/src/job_queue/mod.rs b/hive-c0re/src/job_queue/mod.rs index 655ad636..4c5e74f2 100644 --- a/hive-c0re/src/job_queue/mod.rs +++ b/hive-c0re/src/job_queue/mod.rs @@ -161,9 +161,9 @@ fn insert_group( inner .insert_job(group_parent, |b| { declare(b); - // c0re names no handles: a DAG is addressed by its container node, - // which `submit` inserts itself, and nothing downstream looks an - // individual step up by id. + // A runtime-appended subgraph is addressed by the node that emitted + // it (`group_parent`), so this path names nothing. Callers that DO + // want a handle use `JobQueue::insert` and name the node there. Vec::new() }) .map_err(|e| anyhow::anyhow!("job_queue: graph insert failed: {e}"))?; diff --git a/hive-c0re/src/job_queue/power.rs b/hive-c0re/src/job_queue/power.rs index 1bb26b95..67d2650e 100644 --- a/hive-c0re/src/job_queue/power.rs +++ b/hive-c0re/src/job_queue/power.rs @@ -16,8 +16,9 @@ use std::sync::Arc; use hive_jobq::NodeId; -use super::JobBuilder; -use super::templates; +use super::resource::Resource; +use super::templates::rebuild_nodes; +use super::{JobBuilder, NodeKind}; use crate::coordinator::Coordinator; use crate::lifecycle; diff --git a/hive-c0re/src/job_queue/templates.rs b/hive-c0re/src/job_queue/templates.rs index 536093fc..cf110437 100644 --- a/hive-c0re/src/job_queue/templates.rs +++ b/hive-c0re/src/job_queue/templates.rs @@ -17,8 +17,8 @@ //! //! The hive-wide **power ops** (`stop` / `start` / `restart`) are NOT here: //! their per-agent shape depends on live running state (an async -//! `lifecycle::is_running` read), so `submit.rs` assembles them out of the -//! primitives this module exports ([`rebuild_nodes`]). +//! `lifecycle::is_running` read), so [`super::power`] assembles them out of +//! the primitives this module exports ([`rebuild_nodes`]). use hive_jobq::TerminalState; From be4763678bbd3174928402577e61272ac3e2c62f Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 4 Aug 2026 11:53:02 +0200 Subject: [PATCH 03/10] wip(#3001): test helpers off the container id submit() -> insert() in tests, and the two shape walkers lose their dag param: with no container there is no per-DAG root to filter on, nothing to exclude (every node is real work now), and a group root genuinely has parent = None, so the container-parent normalisation goes too. Each test builds a fresh JobQueue, so "the DAG" is "the graph". 20 errors remain, all in tests.rs, and they are the point: changing the helper's type from u64 to () made every site that consumed the container id light up as `expected u64, found ()`. A type error is an exhaustive grep -- ten helpers take a dag id, not the three I had measured. state_of(q, dag_id) is not mechanical: it read the DAG's ROLLED-UP state, which was the container node's own. That makes it the second consumer of the container-as-roll-up-point, alongside hivectl's queued_dags poll. Both want the same answer, so it waits on the same ruling. --- hive-c0re/src/job_queue/tests.rs | 151 ++++++++++++++++--------------- 1 file changed, 79 insertions(+), 72 deletions(-) diff --git a/hive-c0re/src/job_queue/tests.rs b/hive-c0re/src/job_queue/tests.rs index 87d17dca..d2d210ac 100644 --- a/hive-c0re/src/job_queue/tests.rs +++ b/hive-c0re/src/job_queue/tests.rs @@ -20,9 +20,15 @@ use super::*; /// Submit a declared shape with the metadata every mechanics test uses. /// `Source::Manual` because none of these exercise provenance — the tests that /// do name their own source at the call site. -fn submit(q: &JobQueue, reason: &str, declare: impl FnOnce(&JobBuilder)) -> u64 { - q.submit(Source::Manual, reason.to_owned(), declare) - .expect("valid shape") +/// Insert a declared job, naming nothing — the shape assertions read the whole +/// graph. A test that needs a handle calls `q.insert` directly and names the +/// node it cares about. +fn insert(q: &JobQueue, declare: impl FnOnce(&JobBuilder)) { + q.insert(|b| { + declare(b); + Vec::new() + }) + .expect("valid shape"); } fn ident(s: &str) -> hive_types::Ident { @@ -104,25 +110,27 @@ fn when_tag(when: hive_jobq::DepWhen) -> String { /// keeps the assertion on c0re's own product. Whether the scheduler then /// *honours* those edges — runs a chain serially, holds a grant across a /// subtree — is `hive_jobq`'s property and is tested in `hive_jobq`. -fn declared_shape(q: &JobQueue, dag: u64) -> Vec { - declared_shape_filtered(q, dag, &|_| true) +fn declared_shape(q: &JobQueue) -> Vec { + declared_shape_filtered(q, &|_| true) } -fn declared_shape_filtered( - q: &JobQueue, - dag: u64, - keep: &dyn Fn(&NodeKind) -> bool, -) -> Vec { +/// Every node in the graph, since each test inserts into a fresh [`JobQueue`]. +/// +/// This used to take a DAG id and filter by `root_of(n) == Some(container)`. +/// With no container node there is no per-DAG root to filter on — and nothing +/// to exclude either, because every node in the graph is now real work. Tests +/// that insert more than one job name a node per job and assert on the ids +/// [`JobQueue::insert`] hands back. +fn declared_shape_filtered(q: &JobQueue, keep: &dyn Fn(&NodeKind) -> bool) -> Vec { let sched = q.sched().lock().expect("job_queue mutex poisoned"); let graph = sched.graph(); - let root = graph.resolve_id(dag).expect("dag id is a real node id"); let kind_of = |id: NodeId| graph.node(id).map(|n| n.payload.as_str()); graph .nodes() - .filter(|n| n.id != root && graph.root_of(n.id) == Some(root) && keep(&n.payload)) + .filter(|n| keep(&n.payload)) .map(|n| Declared { kind: n.payload.as_str(), - parent: n.parent.filter(|p| *p != root).and_then(kind_of), + parent: n.parent.and_then(kind_of), after: n .deps .iter() @@ -140,19 +148,18 @@ fn declared_shape_filtered( /// Panics unless there is exactly one — every caller is about a shape where the /// kind is unique, so two would mean the assertion had quietly stopped being /// about the node the test names. -fn node_of(q: &JobQueue, dag: u64, kind: &str) -> hive_jobq::NodeId { +fn node_of(q: &JobQueue, kind: &str) -> hive_jobq::NodeId { let sched = q.sched().lock().expect("job_queue mutex poisoned"); let graph = sched.graph(); - let root = graph.resolve_id(dag).expect("dag id is a real node id"); let mut found: Vec<_> = graph .nodes() - .filter(|n| n.id != root && graph.root_of(n.id) == Some(root) && n.payload.as_str() == kind) + .filter(|n| n.payload.as_str() == kind) .map(|n| n.id) .collect(); assert_eq!( found.len(), 1, - "expected exactly one {kind} node in the dag" + "expected exactly one {kind} node in the graph" ); found.pop().expect("checked above") } @@ -312,8 +319,8 @@ fn dag_count(q: &JobQueue) -> usize { #[test] fn submit_assigns_distinct_ids() { let q = JobQueue::new(1); - let first = submit(&q, "first", |builder| rebuild(builder, "agent-a")); - let second = submit(&q, "second", |builder| rebuild(builder, "agent-b")); + let first = insert(&q, |builder| rebuild(builder, "agent-a")); + let second = insert(&q, |builder| rebuild(builder, "agent-b")); assert_ne!(first, second); assert_eq!(dag_count(&q), 2); } @@ -326,8 +333,8 @@ fn submit_assigns_distinct_ids() { #[test] fn identical_resubmit_is_a_distinct_dag() { let q = JobQueue::new(1); - let first = submit(&q, "first", |builder| rebuild(builder, "agent-a")); - let resubmit = submit(&q, "again", |builder| rebuild(builder, "agent-a")); + let first = insert(&q, |builder| rebuild(builder, "agent-a")); + let resubmit = insert(&q, |builder| rebuild(builder, "agent-a")); assert_ne!(first, resubmit, "no dedup: identical resubmit is a new DAG"); assert_eq!(dag_count(&q), 2); } @@ -335,9 +342,9 @@ fn identical_resubmit_is_a_distinct_dag() { #[test] fn distinct_submits_never_collapse() { let q = JobQueue::new(1); - let rebuild_a = submit(&q, "r", |builder| rebuild(builder, "agent-a")); - let rebuild_b = submit(&q, "r", |builder| rebuild(builder, "agent-b")); - let restart_a = submit(&q, "r", |builder| { + let rebuild_a = insert(&q, |builder| rebuild(builder, "agent-a")); + let rebuild_b = insert(&q, |builder| rebuild(builder, "agent-b")); + let restart_a = insert(&q, |builder| { restart_online(builder, &["agent-a"], false); }); assert_ne!(rebuild_a, rebuild_b); @@ -357,8 +364,8 @@ fn resubmit_while_running_is_new_dag() { // swallowed" is the scenario people worry about, and a reader looking for // it should find it. let q = JobQueue::new(1); - let a = submit(&q, "first", |builder| rebuild(builder, "agent-a")); - let again = submit(&q, "config bumped during build", |builder| { + let a = insert(&q, |builder| rebuild(builder, "agent-a")); + let again = insert(&q, |builder| { rebuild(builder, "agent-a"); }); assert_ne!(a, again); @@ -401,9 +408,9 @@ fn rebuild_chain_is_declared_serial() { // The non-graceful shape asserted here has no quiesce chain, so nothing here // is concurrent — but the name would mislead about the graceful one. let q = JobQueue::new(1); - let id = submit(&q, "r", |builder| rebuild(builder, "agent-a")); + let id = insert(&q, |builder| rebuild(builder, "agent-a")); assert_eq!( - declared_shape(&q, id), + declared_shape(&q), vec![ row("meta_sync", None, &[]), // The brace: holds the lease + slot for everything nested below it. @@ -481,7 +488,7 @@ fn graceful_rebuild_chain_drains_before_stopping() { // the quiesce chain runs beside the build or nested under it, so a // kind-only assertion cannot see the bug this shape exists to fix. assert_eq!( - declared_shape(&q, id), + declared_shape(&q), vec![ row("meta_sync", None, &[]), row("agent_window", None, &[("meta_sync", "done")]), @@ -530,11 +537,11 @@ fn non_graceful_rebuild_has_no_signal_or_drain() { // job keeps its nodes to itself and inserts them, so what it built is // observable where it matters — in what the scheduler runs. let q = JobQueue::new(1); - let id = submit(&q, "manual", |builder| { + let id = insert(&q, |builder| { templates::rebuild_nodes(builder, "agent-a", true, None); }); assert_eq!( - declared_shape(&q, id) + declared_shape(&q) .iter() .map(|d| d.kind) .collect::>(), @@ -567,8 +574,8 @@ fn rebuild_chain_declares_its_resources_on_the_brace() { // `a_contended_resource_goes_to_the_oldest_waiter`. What is c0re's is // *which* nodes contend in the first place — a declaration, asserted here.) let q = JobQueue::new(1); - let id = submit(&q, "r", |builder| rebuild(builder, "agent-a")); - let res = |kind: &str| declared_resources(&q, node_of(&q, id, kind)); + let id = insert(&q, |builder| rebuild(builder, "agent-a")); + let res = |kind: &str| declared_resources(&q, node_of(&q, kind)); let agent = || Resource::Agent("agent-a".to_owned()); assert_eq!( @@ -612,7 +619,7 @@ fn rebuild_chain_declares_its_resources_on_the_brace() { #[test] fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() { let q = JobQueue::new(4); - let id = submit(&q, "hive-wide", |builder| { + let id = insert(&q, |builder| { restart_online(builder, &["agent-a", "agent-b"], false); }); // A hive-wide restart is ONE DAG, not one-per-agent. @@ -623,7 +630,7 @@ fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() { // Those two declared facts are what "they run concurrently" *means* here — // that a scheduler then does run independent, resource-disjoint roots at // once is hive_jobq's property, tested there. - let heads: Vec<_> = declared_shape(&q, id) + let heads: Vec<_> = declared_shape(&q) .into_iter() .filter(|d| d.kind == "stop_for_update") .collect(); @@ -648,7 +655,7 @@ fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() { #[test] fn multi_agent_stop_is_one_dag_with_concurrent_per_agent_subgraphs() { let q = JobQueue::new(4); - let id = submit(&q, "hive-wide stop", |builder| { + let id = insert(&q, |builder| { stop_online(builder, &["agent-a", "agent-b"], false); }); // A hive-wide stop is ONE DAG, not one-per-agent. @@ -657,7 +664,7 @@ fn multi_agent_stop_is_one_dag_with_concurrent_per_agent_subgraphs() { // is a group root with no node-deps, holding only its own agent's lease. // Independent roots on disjoint resources is what "concurrently" means at // this layer — the running of them is hive_jobq's. - let heads: Vec<_> = declared_shape(&q, id) + let heads: Vec<_> = declared_shape(&q) .into_iter() .filter(|d| d.kind == "set_wanted") .collect(); @@ -681,7 +688,7 @@ fn multi_agent_start_one_dag_folds_per_agent_stale_rebuild() { let q = JobQueue::new(4); // fresh: offline + not stale → SetWanted → Reconcile. // stale: offline + stale → SetWanted → «rebuild subgraph». - let id = submit(&q, "hive-wide start", |builder| { + let id = insert(&q, |builder| { power::start_nodes( builder, &[ @@ -741,13 +748,13 @@ fn offline_agents_skip_mechanical_nodes_but_keep_reconcile() { // read and node exec. let q = JobQueue::new(4); // Offline graceful stop → SetWanted(Off) → Reconcile (no Signal/Drain). - let stop = submit(&q, "stop down", |builder| { + let stop = insert(&q, |builder| { power::stop_nodes(builder, &[("down".to_owned(), false)], true); }); // Offline restart → a lone Reconcile (no SetWanted, no StopForUpdate): // nothing to bounce, and restart never rewrites intent, so the tail // Reconcile converges the down agent to its existing `wanted`. - let restart = submit(&q, "restart down", |builder| { + let restart = insert(&q, |builder| { power::restart_nodes(builder, &[("down2".to_owned(), false)], true); }); let shape = |id: u64| -> Vec { @@ -797,7 +804,7 @@ fn boot_sweep_nodes_declare_their_own_resources() { ) .expect("valid shape"); - let mut lock = declared_resources(&q, node_of(&q, id, "meta_lock")); + let mut lock = declared_resources(&q, node_of(&q, "meta_lock")); lock.sort_by_key(|r| format!("{r:?}")); assert_eq!( lock, @@ -806,7 +813,7 @@ fn boot_sweep_nodes_declare_their_own_resources() { ); assert_eq!( - declared_resources(&q, node_of(&q, id, "reconcile")), + declared_resources(&q, node_of(&q, "reconcile")), vec![Resource::Agent("drifted-agent".to_owned())], "a boot Reconcile touches the container, so it holds that agent's lease" ); @@ -901,8 +908,8 @@ fn rebuild_reconcile_waits_for_the_whole_build_subtree() { // **did** flatten this chain, and the guarantee survives only because the // roll-up point moved with it. let q = JobQueue::new(1); - let id = submit(&q, "r", |builder| rebuild(builder, "agent-a")); - let shape = declared_shape(&q, id); + let id = insert(&q, |builder| rebuild(builder, "agent-a")); + let shape = declared_shape(&q); let parent_of = |kind: &str| { shape .iter() @@ -973,7 +980,7 @@ fn rebuild_reconcile_waits_for_the_whole_build_subtree() { #[test] fn a_fanned_out_mechanical_node_declares_its_agent_lease() { let q = JobQueue::new(4); - let id = submit(&q, "fan-out", |builder| { + let id = insert(&q, |builder| { templates::fanned_out_mechanical( builder, NodeKind::Start { @@ -981,9 +988,9 @@ fn a_fanned_out_mechanical_node_declares_its_agent_lease() { }, ); }); - assert_eq!(declared_shape(&q, id), vec![row("start", None, &[])]); + assert_eq!(declared_shape(&q), vec![row("start", None, &[])]); assert_eq!( - declared_resources(&q, node_of(&q, id, "start")), + declared_resources(&q, node_of(&q, "start")), vec![Resource::Agent("agent-a".to_owned())], "the fanned-out node carries the lease itself, rather than relying on \ whoever happened to fan it out" @@ -1020,7 +1027,7 @@ fn a_meta_lock_grows_one_rebuild_subgraph_per_agent() { // One chain per agent, each an independent group root — so the two rebuild // concurrently, each on its own lease. - let shape = declared_shape(&q, id); + let shape = declared_shape(&q); let heads: Vec<_> = shape .iter() .filter(|d| d.kind == "meta_sync") @@ -1046,7 +1053,7 @@ fn a_meta_lock_grows_one_rebuild_subgraph_per_agent() { #[test] fn cancel_clears_queued_dag() { let q = JobQueue::new(1); - let id = submit(&q, "r", |builder| rebuild(builder, "agent-a")); + let id = insert(&q, |builder| rebuild(builder, "agent-a")); assert!(q.cancel(id), "fully-queued dag cancels"); // The operator sees `Cancelled` the moment the cancel returns — the spared // tail is still `Pending`, and a DAG must not read `Queued` back to the @@ -1075,7 +1082,7 @@ fn cancel_clears_queued_dag() { #[test] fn cancel_drops_one_agents_branch_leaving_the_rest() { let q = JobQueue::new(2); - let id = submit(&q, "r", |builder| { + let id = insert(&q, |builder| { restart_online(builder, &["agent-a", "agent-b"], false); }); // Per-agent subgraphs hang directly off the container, one per agent. @@ -1152,19 +1159,19 @@ fn cancelled_power_op_runs_no_compensating_node() { let case = format!("graceful={graceful} running={running}"); let q = JobQueue::new(1); - let id = submit(&q, "bounce", |builder| { + let id = insert(&q, |builder| { power::restart_nodes(builder, &targets, graceful); }); assert_cancels_clean(&q, id, false, &format!("restart {case}")); let q = JobQueue::new(1); - let id = submit(&q, "stop", |builder| { + let id = insert(&q, |builder| { power::stop_nodes(builder, &targets, graceful); }); assert_cancels_clean(&q, id, true, &format!("stop {case}")); let q = JobQueue::new(1); - let id = submit(&q, "start", |builder| { + let id = insert(&q, |builder| { power::start_nodes(builder, &[("agent-a".to_owned(), running, false)]); }); assert_cancels_clean(&q, id, true, &format!("start {case}")); @@ -1185,7 +1192,7 @@ fn cancelled_power_op_runs_no_compensating_node() { #[test] fn cancelled_dag_still_runs_its_approval_tail() { let q = JobQueue::new(1); - let id = submit(&q, "approval #7", |builder| { + let id = insert(&q, |builder| { templates::approval_deploy(builder, "agent-a", 7); }); assert!(q.cancel(id), "fully-queued dag cancels"); @@ -1215,7 +1222,7 @@ fn cancelled_dag_still_runs_its_approval_tail() { assert_eq!(state_of(&q, id), State::Finishing); // An unrelated group landing in the same graph doesn't disturb this one's // state — a root rolls up its own subtree, not the graph. - let _other = submit(&q, "r", |builder| rebuild(builder, "agent-b")); + let _other = insert(&q, |builder| rebuild(builder, "agent-b")); assert_eq!(state_of(&q, id), State::Finishing); } @@ -1229,12 +1236,12 @@ fn cancelled_dag_still_runs_its_approval_tail() { #[test] fn deploy_dag_runs_phases_in_order_and_tails_a_failed_apply() { let q = JobQueue::new(1); - let id = submit(&q, "approval #7", |builder| { + let id = insert(&q, |builder| { templates::approval_deploy(builder, "agent-a", 7); }); assert_eq!( - declared_shape(&q, id), + declared_shape(&q), vec![ // The window is the group root and holds the meta window for the // whole subtree; the three phases are its sub-nodes. @@ -1287,12 +1294,12 @@ fn deploy_apply_grows_rebuild_subgraph_and_finalizes_after_it() { // children run (`a_completing_node_grows_the_work_it_declared`, // `parent_parks_in_finishing_until_children_roll_up`). let q = JobQueue::new(1); - let id = submit(&q, "deploy graft", |builder| { + let id = insert(&q, |builder| { templates::deploy_rebuild_nodes(builder, "agent-a", 11); }); assert_eq!( - declared_shape(&q, id), + declared_shape(&q), vec![ row("meta_sync", None, &[]), row("agent_window", None, &[("meta_sync", "done")]), @@ -1467,11 +1474,11 @@ fn error_truncation_cuts_on_a_char_boundary() { #[test] fn graceful_stop_shape_signal_drain_reconcile() { let q = JobQueue::new(1); - let id = submit(&q, "graceful", |builder| { + let id = insert(&q, |builder| { stop_online(builder, &["agent-a"], true); }); assert_eq!( - declared_shape(&q, id), + declared_shape(&q), vec![ // The whole stop hangs under `set_wanted`: the durable intent is // written first, and the mechanical steps are its sub-nodes. @@ -1494,8 +1501,8 @@ fn graceful_stop_shape_signal_drain_reconcile() { // `exec.rs`. assert_eq!( [ - declared_resources(&q, node_of(&q, id, "signal")), - declared_resources(&q, node_of(&q, id, "drain")), + declared_resources(&q, node_of(&q, "signal")), + declared_resources(&q, node_of(&q, "drain")), ], [vec![], vec![]], "the quiesce pair borrows the brace's lease and declares nothing" @@ -1505,11 +1512,11 @@ fn graceful_stop_shape_signal_drain_reconcile() { #[test] fn spawn_shape_provision_create_dropin_reconcile() { let q = JobQueue::new(1); - let id = submit(&q, "approval #7 spawn", |builder| { + let id = insert(&q, |builder| { templates::spawn(builder, "newbie", 7); }); assert_eq!( - declared_shape(&q, id), + declared_shape(&q), vec![ row("provision", None, &[]), row("create", Some("provision"), &[]), @@ -1530,7 +1537,7 @@ fn spawn_shape_provision_create_dropin_reconcile() { #[test] fn perm_change_shape_prefixes_rebuild_chain() { let q = JobQueue::new(1); - let id = submit(&q, "perm", |builder| { + let id = insert(&q, |builder| { templates::perm_change( builder, "agent-a", @@ -1541,7 +1548,7 @@ fn perm_change_shape_prefixes_rebuild_chain() { ); }); assert_eq!( - declared_shape(&q, id) + declared_shape(&q) .iter() .map(|d| d.kind) .collect::>(), @@ -1569,15 +1576,15 @@ fn reparent_shape_is_a_lone_agentless_meta_window_node() { // `MetaLock`, and it must declare the meta window — a topology commit // must not land inside another node's staged deploy window. let q = JobQueue::new(1); - let id = submit(&q, "set-parent", |builder| { + let id = insert(&q, |builder| { templates::reparent(builder, vec![(ident("alice"), Some(ident("bob")))]); }); assert_eq!( - declared_shape(&q, id), + declared_shape(&q), vec![row("reparent", None, &[])], "one node, no rebuild subgraph" ); - let node = node_of(&q, id, "reparent"); + let node = node_of(&q, "reparent"); assert_eq!( declared_resources(&q, node), vec![Resource::MetaWindow], @@ -1593,11 +1600,11 @@ fn reparent_bulk_shape_carries_every_move_on_one_node() { // request is the reason a single node was chosen in the first place. let moves = vec![(ident("alice"), Some(ident("bob"))), (ident("carol"), None)]; let q = JobQueue::new(1); - let id = submit(&q, "set-parent-bulk", |builder| { + let id = insert(&q, |builder| { templates::reparent(builder, moves.clone()); }); assert_eq!( - declared_shape(&q, id), + declared_shape(&q), vec![row("reparent", None, &[])], "one node for the whole request, not one per move" ); From 102ebdc03dc3a57b0e210978d89a18b473618131 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 4 Aug 2026 12:23:24 +0200 Subject: [PATCH 04/10] wip(#3001): 9 of 10 test helpers off the container id; sweep stale prose The helpers keep shrinking the same way: resolve_id goes, 'n.id != root' goes (the container was the only non-work node), root_of goes, and the container-parent normalisation goes because a group root now genuinely has parent = None. pending_kinds_filtered drops from a four-clause multi-line filter to one line. Also removed a doc block my earlier edit had orphaned above the renamed helper, and swept 'under `dag`' / '`submit` returns' out of the prose. state_of stays untouched: it reads a roll-up, which is the same question as hivectl's queued_dags. --- hive-c0re/src/job_queue/tests.rs | 95 ++++++++++++++------------------ 1 file changed, 40 insertions(+), 55 deletions(-) diff --git a/hive-c0re/src/job_queue/tests.rs b/hive-c0re/src/job_queue/tests.rs index d2d210ac..b6e260fe 100644 --- a/hive-c0re/src/job_queue/tests.rs +++ b/hive-c0re/src/job_queue/tests.rs @@ -4,7 +4,7 @@ //! that graph (wire projection, history retention, error truncation). //! //! **Nothing here runs a node.** Everything a template declares is in the graph -//! the moment `submit` returns, so the assertions read it there. Whether the +//! the moment `insert` returns, so the assertions read it there. Whether the //! scheduler then honours those declarations — cascade, roll-up, grant //! borrow/release, fairness, the `Finishing` gate — is `hive_jobq`'s property //! and is tested in `hive_jobq`, against its own primitives rather than through @@ -17,9 +17,6 @@ use super::model::NodeKind; use super::*; -/// Submit a declared shape with the metadata every mechanics test uses. -/// `Source::Manual` because none of these exercise provenance — the tests that -/// do name their own source at the call site. /// Insert a declared job, naming nothing — the shape assertions read the whole /// graph. A test that needs a handle calls `q.insert` directly and names the /// node it cares about. @@ -68,8 +65,8 @@ fn stop_online(builder: &JobBuilder, agents: &[&str], graceful: bool) { #[derive(Debug, PartialEq, Eq)] struct Declared { kind: &'static str, - /// Parent kind, or `None` when the node hangs directly under the DAG - /// container (i.e. it is a group root). + /// Parent kind, or `None` when the node is a group root — which is now a + /// genuine `parent = None`, not "hangs under the container". parent: Option<&'static str>, /// Kinds this node declared a node-dep on, in declaration order, each with /// the outcome set that satisfies it. @@ -102,10 +99,10 @@ fn when_tag(when: hive_jobq::DepWhen) -> String { .join("|") } -/// Every work node under `dag`, in insertion order, as its declared shape. +/// Every work node in the graph, in insertion order, as its declared shape. /// /// **This is what the template tests are actually about.** A template's output -/// is fully determined the moment `submit` returns: the kinds, the parent +/// is fully determined the moment `insert` returns: the kinds, the parent /// nesting and the dep edges are all sitting in the graph. Reading them here /// keeps the assertion on c0re's own product. Whether the scheduler then /// *honours* those edges — runs a chain serially, holds a grant across a @@ -143,7 +140,7 @@ fn declared_shape_filtered(q: &JobQueue, keep: &dyn Fn(&NodeKind) -> bool) -> Ve .collect() } -/// The id of the one node of `kind` under `dag`, for the resource assertions. +/// The id of the one node of `kind`, for the resource assertions. /// /// Panics unless there is exactly one — every caller is about a shape where the /// kind is unique, so two would mean the assertion had quietly stopped being @@ -164,56 +161,45 @@ fn node_of(q: &JobQueue, kind: &str) -> hive_jobq::NodeId { found.pop().expect("checked above") } -/// The payload of the one node of `kind` under `dag`, for assertions about what -/// a node *carries* rather than how it is wired. -fn payload_of(q: &JobQueue, dag: u64, kind: &str) -> NodeKind { - let id = node_of(q, dag, kind); +/// The payload of the one node of `kind`, for assertions about what a node +/// *carries* rather than how it is wired. +fn payload_of(q: &JobQueue, kind: &str) -> NodeKind { + let id = node_of(q, kind); let sched = q.sched().lock().expect("job_queue mutex poisoned"); sched.graph().node(id).expect("node exists").payload.clone() } -/// Kinds of every node under `dag` still `Pending` — the nodes that could yet -/// run. Stronger than asking the scheduler what is *ready right now*: a node +/// Kinds of every node still `Pending` — the nodes that could yet run. +/// Stronger than asking the scheduler what is *ready right now*: a node /// blocked on a dep is not ready but is very much still alive. -fn pending_kinds(q: &JobQueue, dag: u64) -> Vec<&'static str> { - pending_kinds_filtered(q, dag, &|_| true) +fn pending_kinds(q: &JobQueue) -> Vec<&'static str> { + pending_kinds_filtered(q, &|_| true) } /// [`pending_kinds`] restricted to the nodes whose payload names `agent`. -fn pending_kinds_for(q: &JobQueue, dag: u64, agent: &str) -> Vec<&'static str> { - pending_kinds_filtered(q, dag, &|kind: &NodeKind| kind.agent() == agent) +fn pending_kinds_for(q: &JobQueue, agent: &str) -> Vec<&'static str> { + pending_kinds_filtered(q, &|kind: &NodeKind| kind.agent() == agent) } -/// The payloads of every node under `dag` still `Pending`, for the cases where -/// *which* of a family of same-kind nodes survived is the assertion — a -/// template emits one tail per outcome and they differ only in what they carry. -fn pending_payloads(q: &JobQueue, dag: u64) -> Vec { +/// The payloads of every node still `Pending`, for the cases where *which* of a +/// family of same-kind nodes survived is the assertion — a template emits one +/// tail per outcome and they differ only in what they carry. +fn pending_payloads(q: &JobQueue) -> Vec { let sched = q.sched().lock().expect("job_queue mutex poisoned"); let graph = sched.graph(); - let root = graph.resolve_id(dag).expect("dag id is a real node id"); graph .nodes() - .filter(|n| n.id != root && graph.root_of(n.id) == Some(root) && n.state == State::Pending) + .filter(|n| n.state == State::Pending) .map(|n| n.payload.clone()) .collect() } -fn pending_kinds_filtered( - q: &JobQueue, - dag: u64, - keep: &dyn Fn(&NodeKind) -> bool, -) -> Vec<&'static str> { +fn pending_kinds_filtered(q: &JobQueue, keep: &dyn Fn(&NodeKind) -> bool) -> Vec<&'static str> { let sched = q.sched().lock().expect("job_queue mutex poisoned"); let graph = sched.graph(); - let root = graph.resolve_id(dag).expect("dag id is a real node id"); graph .nodes() - .filter(|n| { - n.id != root - && graph.root_of(n.id) == Some(root) - && n.state == State::Pending - && keep(&n.payload) - }) + .filter(|n| n.state == State::Pending && keep(&n.payload)) .map(|n| n.payload.as_str()) .collect() } @@ -252,20 +238,19 @@ fn declared_resources(q: &JobQueue, node_id: hive_jobq::NodeId) -> Vec .collect() } -/// The resources declared by **every** node of `kind` under `dag`, one row per +/// The resources declared by **every** node of `kind`, one row per /// node, sorted so the rows read as a set rather than an insertion order. /// /// The per-agent templates emit several nodes of one kind — one per agent — and /// what makes them concurrent is that each holds only its *own* agent's lease. /// That is a statement about the whole family, so it needs all the rows, not /// [`declared_resources`]'s single node. -fn declared_resources_of_kind(q: &JobQueue, dag: u64, kind: &str) -> Vec> { +fn declared_resources_of_kind(q: &JobQueue, kind: &str) -> Vec> { let sched = q.sched().lock().expect("job_queue mutex poisoned"); let graph = sched.graph(); - let root = graph.resolve_id(dag).expect("dag id is a real node id"); let mut rows: Vec> = graph .nodes() - .filter(|n| n.id != root && graph.root_of(n.id) == Some(root) && n.payload.as_str() == kind) + .filter(|n| n.payload.as_str() == kind) .map(|n| { n.deps .iter() @@ -286,8 +271,8 @@ fn declared_resources_of_kind(q: &JobQueue, dag: u64, kind: &str) -> Vec Vec { - declared_shape_filtered(q, dag, &|kind: &NodeKind| kind.agent() == agent) +fn declared_shape_for(q: &JobQueue, agent: &str) -> Vec { + declared_shape_filtered(q, &|kind: &NodeKind| kind.agent() == agent) } fn state_of(q: &JobQueue, dag_id: u64) -> State { @@ -394,7 +379,7 @@ fn resubmit_while_running_is_new_dag() { #[test] fn rebuild_chain_is_declared_serial() { // Was `rebuild_chain_claims_in_dep_order`, which drove the whole DAG to - // observe an order that is fully declared the moment `submit` returns. + // observe an order that is fully declared the moment `insert` returns. // // ⚠️ The old name was also wrong about the mechanism, and reading it rather // than the graph is how you'd stay wrong: **only part of this chain is dep @@ -643,7 +628,7 @@ fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() { "both per-agent heads are independent group roots" ); assert_eq!( - declared_resources_of_kind(&q, id, "stop_for_update"), + declared_resources_of_kind(&q, "stop_for_update"), vec![ vec![Resource::Agent("agent-a".to_owned())], vec![Resource::Agent("agent-b".to_owned())], @@ -674,7 +659,7 @@ fn multi_agent_stop_is_one_dag_with_concurrent_per_agent_subgraphs() { "both per-agent stop subgraph heads are independent group roots" ); assert_eq!( - declared_resources_of_kind(&q, id, "set_wanted"), + declared_resources_of_kind(&q, "set_wanted"), vec![ vec![Resource::Agent("agent-a".to_owned())], vec![Resource::Agent("agent-b".to_owned())], @@ -704,7 +689,7 @@ fn multi_agent_start_one_dag_folds_per_agent_stale_rebuild() { // subgraph ends at the Reconcile behind it while the stale agent's carries // the whole rebuild chain in between. assert_eq!( - declared_shape_for(&q, id, "fresh"), + declared_shape_for(&q, "fresh"), vec![ row("set_wanted", None, &[]), row("reconcile", Some("set_wanted"), &[]), @@ -712,7 +697,7 @@ fn multi_agent_start_one_dag_folds_per_agent_stale_rebuild() { "a fresh agent is intent + convergence, nothing in between" ); assert_eq!( - declared_shape_for(&q, id, "stale"), + declared_shape_for(&q, "stale"), vec![ row("set_wanted", None, &[]), row("meta_sync", None, &[("set_wanted", "done")]), @@ -1065,9 +1050,9 @@ fn cancel_clears_queued_dag() { // with the work and **nothing is left that could still run**: no node is // spared, so a rebuild that never ran emits nothing. assert!( - pending_kinds(&q, id).is_empty(), + pending_kinds(&q).is_empty(), "a dropped rebuild leaves nothing alive, got {:?}", - pending_kinds(&q, id) + pending_kinds(&q) ); } @@ -1106,12 +1091,12 @@ fn cancel_drops_one_agents_branch_leaving_the_rest() { // agent-a's subgraph is gone; agent-b's is untouched and still alive. assert!( - pending_kinds_for(&q, id, "agent-a").is_empty(), + pending_kinds_for(&q, "agent-a").is_empty(), "agent-a's branch was dropped whole, got {:?}", - pending_kinds_for(&q, id, "agent-a") + pending_kinds_for(&q, "agent-a") ); assert!( - !pending_kinds_for(&q, id, "agent-b").is_empty(), + !pending_kinds_for(&q, "agent-b").is_empty(), "agent-b's branch survives its sibling's cancel" ); } @@ -1201,7 +1186,7 @@ fn cancelled_dag_still_runs_its_approval_tail() { // survives is the whole assertion: the template emits one per outcome and // the spared one names how the approval row is about to be resolved. // Nothing computes it, so reading the survivor is reading the answer. - let spared = pending_payloads(&q, id); + let spared = pending_payloads(&q); assert!( matches!( spared.as_slice(), @@ -1608,7 +1593,7 @@ fn reparent_bulk_shape_carries_every_move_on_one_node() { vec![row("reparent", None, &[])], "one node for the whole request, not one per move" ); - let NodeKind::Reparent { moves: got } = payload_of(&q, id, "reparent") else { + let NodeKind::Reparent { moves: got } = payload_of(&q, "reparent") else { panic!("expected a Reparent node"); }; assert_eq!(got, moves, "every move rides the single node"); From fe52037b0d004f96b89bc15debebb2c97c70c802 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 4 Aug 2026 17:41:15 +0200 Subject: [PATCH 05/10] wip(#3001): rename insert -> insert_job per mara's 50056 --- hive-c0re/src/actions.rs | 6 ++--- hive-c0re/src/dashboard/lifecycle_ops.rs | 4 +-- hive-c0re/src/job_queue/mod.rs | 27 +++++++++---------- hive-c0re/src/job_queue/power.rs | 6 ++--- hive-c0re/src/job_queue/tests.rs | 2 +- .../src/socket_server/lifecycle_handlers.rs | 2 +- hive-c0re/src/workers/auto_update.rs | 4 +-- 7 files changed, 24 insertions(+), 27 deletions(-) diff --git a/hive-c0re/src/actions.rs b/hive-c0re/src/actions.rs index 8c531076..11c94c2f 100644 --- a/hive-c0re/src/actions.rs +++ b/hive-c0re/src/actions.rs @@ -55,7 +55,7 @@ pub async fn approve(coord: Arc, id: i64) -> Result<()> { // nothing). let inputs: Vec = serde_json::from_str(&approval.commit_ref).unwrap_or_default(); - let inserted = coord.job_queue.insert(|b| { + let inserted = coord.job_queue.insert_job(|b| { crate::job_queue::templates::meta_update(b, inputs, Some(id)); Vec::new() }); @@ -74,7 +74,7 @@ pub async fn approve(coord: Arc, id: i64) -> Result<()> { { tracing::warn!(agent = %approval.agent, error = ?e, "agent_power: seed on spawn failed"); } - let inserted = coord.job_queue.insert(|b| { + let inserted = coord.job_queue.insert_job(|b| { crate::job_queue::templates::spawn(b, approval.agent.as_str(), id); Vec::new() }); @@ -115,7 +115,7 @@ pub async fn approve(coord: Arc, id: i64) -> Result<()> { /// queue. See [`crate::job_queue::templates::approval_deploy`] for the node /// shape; the executor dispatches each node to the `run_deploy_*` bodies below. fn enqueue_approval_rebuild(coord: &Arc, agent: &str, approval_id: i64) { - if let Err(e) = coord.job_queue.insert(|b| { + if let Err(e) = coord.job_queue.insert_job(|b| { crate::job_queue::templates::approval_deploy(b, agent, approval_id); Vec::new() }) { diff --git a/hive-c0re/src/dashboard/lifecycle_ops.rs b/hive-c0re/src/dashboard/lifecycle_ops.rs index b4f399e0..0b864107 100644 --- a/hive-c0re/src/dashboard/lifecycle_ops.rs +++ b/hive-c0re/src/dashboard/lifecycle_ops.rs @@ -50,7 +50,7 @@ pub(super) async fn post_rebuild( if let Some(reject) = guard_agent_name(&state, &logical).await { return reject; } - if let Err(e) = state.coord.job_queue.insert(|b| { + if let Err(e) = state.coord.job_queue.insert_job(|b| { crate::job_queue::templates::rebuild(b, &logical, true); Vec::new() }) { @@ -399,7 +399,7 @@ pub(super) async fn post_update_all(State(state): State) -> Response { else { continue; }; - if let Err(e) = state.coord.job_queue.insert(|b| { + if let Err(e) = state.coord.job_queue.insert_job(|b| { crate::job_queue::templates::rebuild(b, &logical, true); Vec::new() }) { diff --git a/hive-c0re/src/job_queue/mod.rs b/hive-c0re/src/job_queue/mod.rs index 4c5e74f2..51febed8 100644 --- a/hive-c0re/src/job_queue/mod.rs +++ b/hive-c0re/src/job_queue/mod.rs @@ -188,27 +188,24 @@ impl JobQueue { self.sched.lock().expect("job_queue mutex poisoned") } - /// Submit a DAG: insert a [`NodeKind::Dag`] **container node** carrying the - /// group's metadata, then insert the template's nodes as its subtree (their - /// roots re-parented to the container). Returns the container's id as the - /// DAG id — its rolled-up state is the DAG state. + /// Insert a job's nodes into the shared graph, then wake the run loop. /// - /// The container is an ordinary node: it declares no resources, so the - /// scheduler claims it on the next pass, runs its (empty) logic and parks - /// it in `Finishing`, at which point its children become runnable. Nothing - /// here completes it by hand — a node with no work of its own still goes - /// the way every other node goes. + /// Deliberately named for the [`hive_jobq`] primitive it wraps, because + /// that is nearly all it is. **The wrapper earns its place on the wake**: + /// the crate is sync and runtime-free — it holds no `Notify` at all — so + /// the channel the run loop parks on belongs to the host, and something has + /// to ping it. Left to call sites, an insert whose ping was forgotten would + /// leave a correct DAG sitting unscheduled until an unrelated event + /// happened along; nothing would fail, and no test in isolation would see + /// it. /// - /// `source` and `reason` are the container node's own payload — they are - /// arguments here rather than fields of a spec struct because that is all - /// they ever were. `declare` is the recipe, taken by generic and run - /// against a builder `hive_jobq` owns: it goes from the template straight - /// into this call, so there is nothing to allocate for. + /// Returns exactly what the primitive returns: the ids of the nodes the + /// template named, in the order it named them. /// /// # Errors /// Propagates a graph-insert error (dependencies that aren't /// dependency-topological). - pub fn insert( + pub fn insert_job( &self, declare: impl FnOnce(&JobBuilder) -> Vec, ) -> anyhow::Result> { diff --git a/hive-c0re/src/job_queue/power.rs b/hive-c0re/src/job_queue/power.rs index 67d2650e..3ed1d4e1 100644 --- a/hive-c0re/src/job_queue/power.rs +++ b/hive-c0re/src/job_queue/power.rs @@ -216,7 +216,7 @@ pub async fn restart_many( for agent in agents { targets.push((agent.clone(), lifecycle::is_running(agent).await)); } - let ids = coord.job_queue.insert(|b| { + let ids = coord.job_queue.insert_job(|b| { restart_nodes(b, &targets, graceful); Vec::new() })?; @@ -245,7 +245,7 @@ pub async fn start_many(coord: &Arc, agents: &[String]) -> anyhow:: } targets.push((agent.clone(), running, stale)); } - let ids = coord.job_queue.insert(|b| { + let ids = coord.job_queue.insert_job(|b| { start_nodes(b, &targets); Vec::new() })?; @@ -269,7 +269,7 @@ pub async fn stop_many( for agent in agents { targets.push((agent.clone(), lifecycle::is_running(agent).await)); } - let ids = coord.job_queue.insert(|b| { + let ids = coord.job_queue.insert_job(|b| { stop_nodes(b, &targets, graceful); Vec::new() })?; diff --git a/hive-c0re/src/job_queue/tests.rs b/hive-c0re/src/job_queue/tests.rs index b6e260fe..bd7cecb6 100644 --- a/hive-c0re/src/job_queue/tests.rs +++ b/hive-c0re/src/job_queue/tests.rs @@ -21,7 +21,7 @@ use super::*; /// graph. A test that needs a handle calls `q.insert` directly and names the /// node it cares about. fn insert(q: &JobQueue, declare: impl FnOnce(&JobBuilder)) { - q.insert(|b| { + q.insert_job(|b| { declare(b); Vec::new() }) diff --git a/hive-c0re/src/socket_server/lifecycle_handlers.rs b/hive-c0re/src/socket_server/lifecycle_handlers.rs index 968d31a3..c0261405 100644 --- a/hive-c0re/src/socket_server/lifecycle_handlers.rs +++ b/hive-c0re/src/socket_server/lifecycle_handlers.rs @@ -145,7 +145,7 @@ pub(super) fn handle_update(coord: &Arc, agent: &str, name: &str) - return err; } tracing::info!(%agent, %name, "submit rebuild"); - if let Err(e) = coord.job_queue.insert(|b| { + if let Err(e) = coord.job_queue.insert_job(|b| { crate::job_queue::templates::rebuild(b, name, true); Vec::new() }) { diff --git a/hive-c0re/src/workers/auto_update.rs b/hive-c0re/src/workers/auto_update.rs index a629e73f..0084b102 100644 --- a/hive-c0re/src/workers/auto_update.rs +++ b/hive-c0re/src/workers/auto_update.rs @@ -109,7 +109,7 @@ pub async fn ensure_root_agent(coord: &Arc) -> Result<()> { tracing::warn!( "manager container exists but no applied flake — forcing rebuild to migrate" ); - if let Err(e) = coord.job_queue.insert(|b| { + if let Err(e) = coord.job_queue.insert_job(|b| { crate::job_queue::templates::rebuild(b, MANAGER_NAME, true); Vec::new() }) { @@ -397,7 +397,7 @@ fn submit_boot_tree( // The subgraphs also carry their own per-agent crash-watch suppression // during their `Swap` (applied at claim time); a reconcile-only boot needs // no transient. - if let Err(e) = coord.job_queue.insert(|b| { + if let Err(e) = coord.job_queue.insert_job(|b| { boot_nodes(b, any_stale, fanout, drifted); Vec::new() }) { From dfb88e2dc29ee31aae43fe50817a0134b5f0f420 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 4 Aug 2026 17:44:24 +0200 Subject: [PATCH 06/10] wip(#3001): catch the multi-line .insert( chains the grep missed --- hive-c0re/src/dashboard/meta_inputs.rs | 2 +- hive-c0re/src/dashboard/permissions.rs | 6 +++--- hive-c0re/src/dashboard/topology.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hive-c0re/src/dashboard/meta_inputs.rs b/hive-c0re/src/dashboard/meta_inputs.rs index da1f3baa..3f6705d8 100644 --- a/hive-c0re/src/dashboard/meta_inputs.rs +++ b/hive-c0re/src/dashboard/meta_inputs.rs @@ -214,7 +214,7 @@ pub(super) async fn post_meta_update( state .coord .job_queue - .insert(|b| { + .insert_job(|b| { crate::job_queue::templates::meta_update(b, inputs, None); Vec::new() }) diff --git a/hive-c0re/src/dashboard/permissions.rs b/hive-c0re/src/dashboard/permissions.rs index 11f920d3..d542fc8c 100644 --- a/hive-c0re/src/dashboard/permissions.rs +++ b/hive-c0re/src/dashboard/permissions.rs @@ -158,7 +158,7 @@ pub(super) async fn post_tool_groups( state .coord .job_queue - .insert(|b| { + .insert_job(|b| { crate::job_queue::templates::perm_change( b, &logical, @@ -273,7 +273,7 @@ pub(super) async fn post_capabilities( state .coord .job_queue - .insert(|b| { + .insert_job(|b| { crate::job_queue::templates::perm_change( b, &logical, @@ -374,7 +374,7 @@ pub(super) async fn post_permissions( state .coord .job_queue - .insert(|b| { + .insert_job(|b| { crate::job_queue::templates::perm_change( b, &logical, diff --git a/hive-c0re/src/dashboard/topology.rs b/hive-c0re/src/dashboard/topology.rs index 79ac83a2..37ec5045 100644 --- a/hive-c0re/src/dashboard/topology.rs +++ b/hive-c0re/src/dashboard/topology.rs @@ -96,7 +96,7 @@ pub(super) async fn post_set_parent( state .coord .job_queue - .insert(|b| { + .insert_job(|b| { crate::job_queue::templates::reparent(b, vec![(child, new_parent)]); Vec::new() }) @@ -155,7 +155,7 @@ pub(super) async fn post_set_parent_bulk( state .coord .job_queue - .insert(|b| { + .insert_job(|b| { crate::job_queue::templates::reparent(b, moves); Vec::new() }) From 02e916feeed8cf7db20d0d48dd1f6fb05b5be7be Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 4 Aug 2026 17:54:28 +0200 Subject: [PATCH 07/10] wip(#3001): power chains name their group roots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `*_many` entry points returned `insert_job`'s result while their closures ended in `Vec::new()` — naming nothing, so the returned id list was always empty. `queued_dags` would have shipped `Some([])` and hivectl's wait loop would have had nothing to poll. Silent: it compiles, the op still runs, and no test in isolation looks. Each `*_chain` now returns its group root's guid and the `*_nodes` collectors gather them, so the ids a caller gets back are the roots it can actually wait on. `start_chain` returns *four* in the stale branch, not one: `rebuild_nodes` chains its roots behind `SetWanted` with `after_ok` rather than nesting them under it, so `SetWanted` rolls up only itself. Naming it alone would have reported the start complete while the rebuild was still running — the same under-reporting bug one level down. --- hive-c0re/src/dashboard/lifecycle_ops.rs | 3 +- hive-c0re/src/job_queue/power.rs | 123 ++++++++++++++++------- 2 files changed, 89 insertions(+), 37 deletions(-) diff --git a/hive-c0re/src/dashboard/lifecycle_ops.rs b/hive-c0re/src/dashboard/lifecycle_ops.rs index 0b864107..e1c46463 100644 --- a/hive-c0re/src/dashboard/lifecycle_ops.rs +++ b/hive-c0re/src/dashboard/lifecycle_ops.rs @@ -110,7 +110,8 @@ pub(super) async fn post_kill( // `socket_server.rs::Request::Kill` stays in place: a // manager calling Kill on its own container is self-suicide // mid-call, not a legitimate operator action. - if let Err(e) = crate::job_queue::power::stop_many(&state.coord, &[logical.clone()], false).await + if let Err(e) = + crate::job_queue::power::stop_many(&state.coord, &[logical.clone()], false).await { tracing::error!(agent = %logical, error = ?e, "stop: insert failed"); } diff --git a/hive-c0re/src/job_queue/power.rs b/hive-c0re/src/job_queue/power.rs index 3ed1d4e1..b1606a57 100644 --- a/hive-c0re/src/job_queue/power.rs +++ b/hive-c0re/src/job_queue/power.rs @@ -35,7 +35,16 @@ use crate::lifecycle; /// actually running (nothing to drain on a down container). The `Reconcile` /// stays even for a down agent so a race-up between the state read and exec /// is still stopped in-DAG. -fn stop_chain(builder: &JobBuilder, agent: &str, graceful: bool, running: bool) { +/// Returns the group root's guid, which is what the caller names so +/// `insert_job` hands its id back — that id is how `hivectl` polls this agent's +/// progress. A chain that returned nothing would insert correctly and leave the +/// caller with nothing to wait on. +fn stop_chain( + builder: &JobBuilder, + agent: &str, + graceful: bool, + running: bool, +) -> hive_jobq::NodeGuid { // `SetWanted` is the group root and owns the agent lease; the mechanical // steps are its children (borrow the lease, run once it reaches `Finishing`, // dep-ordered among themselves). @@ -64,13 +73,26 @@ fn stop_chain(builder: &JobBuilder, agent: &str, graceful: bool, running: bool) .needs(Resource::Agent(a())) .part_of(wanted); } + wanted.guid() } /// One agent's **start** subgraph. `SetWanted(Up)` head; a down + stale-rev /// agent gets the rebuild subgraph (its tail `Reconcile` starts it on /// current derivations), otherwise a plain `Reconcile` (which starts a down /// agent and noops an already-running one). -fn start_chain(builder: &JobBuilder, agent: &str, running: bool, stale: bool) { +/// +/// Returns **every** group root — see [`stop_chain`] for why they are named. +/// +/// ⚠️ More than one in the stale branch: `rebuild_nodes` chains its roots +/// *behind* `SetWanted` with `after_ok`, it does **not** nest them under it. So +/// `SetWanted` rolls up only itself, and naming it alone would report the whole +/// start finished while the rebuild was still running. +fn start_chain( + builder: &JobBuilder, + agent: &str, + running: bool, + stale: bool, +) -> Vec { let wanted = builder .node(NodeKind::SetWanted { agent: agent.to_owned(), @@ -78,10 +100,16 @@ fn start_chain(builder: &JobBuilder, agent: &str, running: bool, stale: bool) { }) .needs(Resource::Agent(agent.to_owned())); if !running && stale { - // Rebuild subtree chained behind the `SetWanted` head. `MetaSync`, - // `Prebuild` + `Reconcile` are their own group roots (top-level, per - // `rebuild_nodes`). - rebuild_nodes(builder, agent, true, Some(wanted)); + // Rebuild subtree chained behind the `SetWanted` head. `MetaSync`, the + // `AgentWindow` brace and `Reconcile` are their own group roots + // (top-level, per `rebuild_nodes`) — hence all four names. + let roots = rebuild_nodes(builder, agent, true, Some(wanted)); + vec![ + wanted.guid(), + roots.meta_sync.guid(), + roots.agent_window.guid(), + roots.reconcile.guid(), + ] } else { let _ = builder .node(NodeKind::Reconcile { @@ -89,6 +117,7 @@ fn start_chain(builder: &JobBuilder, agent: &str, running: bool, stale: bool) { }) .needs(Resource::Agent(agent.to_owned())) .part_of(wanted); + vec![wanted.guid()] } } @@ -101,14 +130,22 @@ fn start_chain(builder: &JobBuilder, agent: &str, running: bool, stale: bool) { /// before `Reconcile`; a down agent gets just `Reconcile`, which /// converges to intent — a stopped (`wanted = Off`) agent stays stopped, /// a crashed (`wanted = Up`) agent comes back up. -fn restart_chain(builder: &JobBuilder, agent: &str, graceful: bool, running: bool) { +/// +/// Returns the group root's guid — see [`stop_chain`]. +fn restart_chain( + builder: &JobBuilder, + agent: &str, + graceful: bool, + running: bool, +) -> hive_jobq::NodeGuid { let a = || agent.to_owned(); if !running { - // Nothing to bounce — a lone Reconcile converges to intent. - let _ = builder + // Nothing to bounce — a lone Reconcile converges to intent, and is + // itself the root. + return builder .node(NodeKind::Reconcile { agent: a() }) - .needs(Resource::Agent(a())); - return; + .needs(Resource::Agent(a())) + .guid(); } // Running: mechanical stop then Reconcile. The first stop node is the group // ROOT (no SetWanted head) and owns the agent lease; the rest are its @@ -144,6 +181,7 @@ fn restart_chain(builder: &JobBuilder, agent: &str, graceful: bool, running: boo .needs(Resource::Agent(a())) .part_of(signal) .after_ok(stop); + signal.guid() } else { let stop = builder .node(NodeKind::StopForUpdate { agent: a() }) @@ -152,6 +190,7 @@ fn restart_chain(builder: &JobBuilder, agent: &str, graceful: bool, running: boo .node(NodeKind::Reconcile { agent: a() }) .needs(Resource::Agent(a())) .part_of(stop); + stop.guid() } } @@ -169,10 +208,15 @@ fn restart_chain(builder: &JobBuilder, agent: &str, graceful: bool, running: boo // subgraph's indices onto another's used to be a function. /// Declare the stop DAG from explicit `(agent, running)` targets. -pub(crate) fn stop_nodes(builder: &JobBuilder, targets: &[(String, bool)], graceful: bool) { - for (agent, running) in targets { - stop_chain(builder, agent, graceful, *running); - } +pub(crate) fn stop_nodes( + builder: &JobBuilder, + targets: &[(String, bool)], + graceful: bool, +) -> Vec { + targets + .iter() + .map(|(agent, running)| stop_chain(builder, agent, graceful, *running)) + .collect() } /// Assemble the start DAG from explicit `(agent, running, stale)` targets. @@ -181,17 +225,26 @@ pub(crate) fn stop_nodes(builder: &JobBuilder, targets: &[(String, bool)], grace /// running under its lease, so a down+stale agent that grew a rebuild subgraph /// reports `rebuilding` during its swap and `starting` at its reconcile, /// without the DAG having to guess one label covering every target. -pub(crate) fn start_nodes(builder: &JobBuilder, targets: &[(String, bool, bool)]) { - for (agent, running, stale) in targets { - start_chain(builder, agent, *running, *stale); - } +pub(crate) fn start_nodes( + builder: &JobBuilder, + targets: &[(String, bool, bool)], +) -> Vec { + targets + .iter() + .flat_map(|(agent, running, stale)| start_chain(builder, agent, *running, *stale)) + .collect() } /// Declare the restart DAG from explicit `(agent, running)` targets. -pub(crate) fn restart_nodes(builder: &JobBuilder, targets: &[(String, bool)], graceful: bool) { - for (agent, running) in targets { - restart_chain(builder, agent, graceful, *running); - } +pub(crate) fn restart_nodes( + builder: &JobBuilder, + targets: &[(String, bool)], + graceful: bool, +) -> Vec { + targets + .iter() + .map(|(agent, running)| restart_chain(builder, agent, graceful, *running)) + .collect() } // ---- entry points --------------------------------------------------------- @@ -216,10 +269,9 @@ pub async fn restart_many( for agent in agents { targets.push((agent.clone(), lifecycle::is_running(agent).await)); } - let ids = coord.job_queue.insert_job(|b| { - restart_nodes(b, &targets, graceful); - Vec::new() - })?; + let ids = coord + .job_queue + .insert_job(|b| restart_nodes(b, &targets, graceful))?; coord.emit_rebuild_queue_snapshot(); Ok(ids) } @@ -231,7 +283,10 @@ pub async fn restart_many( /// /// # Errors /// Propagates a graph-insert error. -pub async fn start_many(coord: &Arc, agents: &[String]) -> anyhow::Result> { +pub async fn start_many( + coord: &Arc, + agents: &[String], +) -> anyhow::Result> { let current = crate::auto_update::current_flake_rev(&coord.hyperhive_flake); let mut targets = Vec::with_capacity(agents.len()); for agent in agents { @@ -245,10 +300,7 @@ pub async fn start_many(coord: &Arc, agents: &[String]) -> anyhow:: } targets.push((agent.clone(), running, stale)); } - let ids = coord.job_queue.insert_job(|b| { - start_nodes(b, &targets); - Vec::new() - })?; + let ids = coord.job_queue.insert_job(|b| start_nodes(b, &targets))?; coord.emit_rebuild_queue_snapshot(); Ok(ids) } @@ -269,10 +321,9 @@ pub async fn stop_many( for agent in agents { targets.push((agent.clone(), lifecycle::is_running(agent).await)); } - let ids = coord.job_queue.insert_job(|b| { - stop_nodes(b, &targets, graceful); - Vec::new() - })?; + let ids = coord + .job_queue + .insert_job(|b| stop_nodes(b, &targets, graceful))?; coord.emit_rebuild_queue_snapshot(); Ok(ids) } From 0523b4f7de1d0bb9da1ea48d6bdb9618f44d982c Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 4 Aug 2026 18:06:14 +0200 Subject: [PATCH 08/10] wip(#3001): convert the last submit call sites; the binary compiles again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `server.rs`'s five sites move to `power::{stop,start,restart}_many` and direct template inserts. `submit_single` routes through the `*_many` builders with a one-element slice rather than keeping a parallel single-target shape. `templates::rebuild` and `templates::reparent` now return the guids of the roots they declare, so a caller that has to wait on them can name them; previously only the void-returning form existed and every caller got an empty id list. Two comments corrected while converting, both contradicted by the code they sit above: * `templates::rebuild` said its tail is edged onto "(MetaSync, Prebuild, Reconcile)" and that "Prebuild's roll-up carries the subtree" — the brace has been the middle root since the AgentWindow change. * the restart handler described the per-agent shape as starting with SetWanted, while `restart_chain`'s own doc says a restart never rewrites `wanted` — that is the difference between restart and stop/start. Error handling is no longer swallowed: a failed insert becomes a reported error rather than a silently-absent id. --- hive-c0re/src/job_queue/templates.rs | 31 +++++-- hive-c0re/src/server.rs | 132 ++++++++++++--------------- 2 files changed, 83 insertions(+), 80 deletions(-) diff --git a/hive-c0re/src/job_queue/templates.rs b/hive-c0re/src/job_queue/templates.rs index cf110437..b90827b4 100644 --- a/hive-c0re/src/job_queue/templates.rs +++ b/hive-c0re/src/job_queue/templates.rs @@ -317,13 +317,22 @@ pub(crate) fn deploy_rebuild_nodes(builder: &JobBuilder, agent: &str, approval_i /// children. /// /// Closed by an [`NodeKind::EmitRebuilt`] tail edged onto all three group-roots -/// (`MetaSync`, `Prebuild`, `Reconcile`) — `Prebuild`'s roll-up carries the -/// whole `StopForUpdate`→`Swap`→`RebuildBookkeeping` subtree, so those three cover every -/// node. Edging `Reconcile` alone would not do: it is `AfterAny` `Prebuild`, so -/// it reaches `Done` even after a failed swap and the tail would report success. -pub fn rebuild(builder: &JobBuilder, agent: &str, relock: bool) { +/// (`MetaSync`, the `AgentWindow` brace, `Reconcile`) — the brace's roll-up +/// carries the whole `StopForUpdate`→`Swap`→`RebuildBookkeeping` subtree, so +/// those three cover every node. Edging `Reconcile` alone would not do: it is +/// `AfterAny` the brace, so it reaches `Done` even after a failed swap and the +/// tail would report success. +/// +/// Returns those three roots, so a caller that needs to wait on the rebuild can +/// name them. +pub fn rebuild(builder: &JobBuilder, agent: &str, relock: bool) -> Vec { let roots = rebuild_nodes(builder, agent, relock, None); emit_rebuilt_tails(builder, agent, &roots.all()); + vec![ + roots.meta_sync.guid(), + roots.agent_window.guid(), + roots.reconcile.guid(), + ] } /// Approval-driven deploy (`MergeConfigPr`) as a phase subtree rather than the @@ -483,10 +492,16 @@ pub fn meta_update(builder: &JobBuilder, inputs: Vec, approval_id: Optio /// checks), so a parent move needs no container rebuild to take effect. /// No transient pill either — the node is agentless (no lease to hang one /// off of) and near-instant. No tail node: the write is the whole effect. -pub fn reparent(builder: &JobBuilder, moves: Vec<(hive_types::Ident, Option)>) { - let _reparent = builder +/// +/// Returns the single node's guid so a caller can wait on it. +pub fn reparent( + builder: &JobBuilder, + moves: Vec<(hive_types::Ident, Option)>, +) -> hive_jobq::NodeGuid { + builder .node(NodeKind::Reparent { moves }) - .needs(Resource::MetaWindow); + .needs(Resource::MetaWindow) + .guid() } // The boot is assembled inline in `workers/auto_update.rs::submit_boot_tree` diff --git a/hive-c0re/src/server.rs b/hive-c0re/src/server.rs index 26799a9a..9baaefdc 100644 --- a/hive-c0re/src/server.rs +++ b/hive-c0re/src/server.rs @@ -180,13 +180,19 @@ async fn dispatch(req: &HostRequest, coord: Arc) -> HostResponse { // submit returns a DAG id immediately, the caller polls // `QueueDag` (`hivectl`'s wait/progress loop) for the // outcome instead of blocking here on the commit. - let id = crate::job_queue::submit::reparent( - &coord, - vec![(child.clone(), new_parent.clone())], - crate::job_queue::Source::Manual, - "manual set-parent via hivectl".to_owned(), - ); - HostResponse::queued(vec![id]) + let inserted = coord.job_queue.insert_job(|b| { + vec![crate::job_queue::templates::reparent( + b, + vec![(child.clone(), new_parent.clone())], + )] + }); + match inserted { + Ok(ids) => { + coord.emit_rebuild_queue_snapshot(); + HostResponse::queued(ids.into_iter().map(hive_jobq::NodeId::get).collect()) + } + Err(e) => HostResponse::error(format!("queue reparent: {e}")), + } } HostRequest::SetResourceLimits { name, @@ -777,39 +783,36 @@ enum Verb { } async fn submit_single(coord: &Arc, name: &str, verb: Verb) -> HostResponse { - use crate::job_queue::{Source, submit}; - let id = match verb { + use crate::job_queue::power; + // A single-target op is the N-target one with N = 1 — the shapes are + // identical, so there is no separate builder to keep in sync. + let targets = [name.to_owned()]; + let ids = match verb { Verb::Kill => { tracing::info!(%name, "kill"); - submit::stop( - coord, - name, - Source::Manual, - "manual kill via hivectl".to_owned(), - ) - .await + power::stop_many(coord, &targets, false).await } Verb::Restart => { tracing::info!(%name, "restart"); - submit::restart( - coord, - name, - Source::Manual, - "manual restart via hivectl".to_owned(), - ) - .await + power::restart_many(coord, &targets, false).await } Verb::Rebuild => { tracing::info!(%name, "rebuild"); - submit::rebuild( - coord, - name, - Source::Manual, - "manual rebuild via hivectl".to_owned(), - ) + // Not a power op: a rebuild's shape doesn't depend on live state, + // so it is a plain template insert rather than a `*_many` gather. + let inserted = coord + .job_queue + .insert_job(|b| crate::job_queue::templates::rebuild(b, name, true)); + if inserted.is_ok() { + coord.emit_rebuild_queue_snapshot(); + } + inserted } }; - HostResponse::queued(vec![id]) + match ids { + Ok(ids) => HostResponse::queued(ids.into_iter().map(hive_jobq::NodeId::get).collect()), + Err(e) => HostResponse::error(format!("queue insert failed: {e}")), + } } /// Stop the given `agents` (resolved logical names) then `infra` containers @@ -847,17 +850,13 @@ async fn handle_stop( } else { "manual via hivectl stop" }; - queued.push( - crate::job_queue::submit::stop_many( - coord, - agents, - graceful, - crate::job_queue::Source::Manual, - reason.to_owned(), - ) - .await, - ); - ok_items.extend(agents.iter().cloned()); + match crate::job_queue::power::stop_many(coord, agents, graceful).await { + Ok(ids) => { + queued.extend(ids.into_iter().map(hive_jobq::NodeId::get)); + ok_items.extend(agents.iter().cloned()); + } + Err(e) => errors.push(format!("queue stop: {e}")), + } } // Agents go down before infra so they're not mid-request against a @@ -944,16 +943,13 @@ async fn handle_start( // hivectl's wait loop. let mut queued: Vec = Vec::new(); if !agents.is_empty() { - queued.push( - crate::job_queue::submit::start_many( - coord, - agents, - crate::job_queue::Source::Manual, - "manual via hivectl start".to_owned(), - ) - .await, - ); - ok_items.extend(agents.iter().cloned()); + match crate::job_queue::power::start_many(coord, agents).await { + Ok(ids) => { + queued.extend(ids.into_iter().map(hive_jobq::NodeId::get)); + ok_items.extend(agents.iter().cloned()); + } + Err(e) => errors.push(format!("queue start: {e}")), + } } let mut resp = finish_lifecycle(ok_items, &errors); @@ -982,28 +978,20 @@ async fn handle_restart_scoped( let mut errors: Vec = Vec::new(); let mut queued: Vec = Vec::new(); - // One DAG for all targeted agents — a per-agent restart subgraph each - // (`SetWanted → [Signal → Drain →] StopForUpdate → Reconcile`), - // independent roots that run concurrently on their own leases. A - // hive-wide `hivectl restart` is now a single DAG, not N. No - // client-side stop-then-start composition — the whole restart survives - // a dropped connection because the DAG owns it. + // One insert for all targeted agents — a per-agent restart subgraph each + // (`[Signal → Drain →] StopForUpdate → Reconcile`; no `SetWanted`, since a + // restart converges to the agent's *existing* intent rather than rewriting + // it), independent roots running concurrently on their own leases. No + // client-side stop-then-start composition — the whole restart survives a + // dropped connection because the graph owns it. if !agents.is_empty() { - queued.push( - crate::job_queue::submit::restart_many( - coord, - &agents, - graceful, - crate::job_queue::Source::Manual, - if graceful { - "manual via hivectl restart --graceful".to_owned() - } else { - "manual restart via hivectl restart".to_owned() - }, - ) - .await, - ); - ok_items.extend(agents.iter().cloned()); + match crate::job_queue::power::restart_many(coord, &agents, graceful).await { + Ok(ids) => { + queued.extend(ids.into_iter().map(hive_jobq::NodeId::get)); + ok_items.extend(agents.iter().cloned()); + } + Err(e) => errors.push(format!("queue restart: {e}")), + } } for &container in &infra { From aef7ead0bc902d3e79becd4d996f807f12f66c49 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 4 Aug 2026 18:11:24 +0200 Subject: [PATCH 09/10] wip(#3001): delete NodeKind::Dag, the container this issue is about MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The variant, its label, its agent-accessor arm and its no-op executor arm are gone, along with the module prose describing a job as "a single container node whose subtree is the work". A job is now just its nodes: a template declares them and names the roots it wants back. `dag_of` becomes `root_of`. It always wrapped the graph's `root_of` and still returns the same thing, but the old name asserted a concept that no longer exists — with no container, the parent chain ends at whichever root the template declared, so the honest question is "which root owns this node", not "which DAG is this in". One comment kept its old wording on purpose: `visible_roots` explains that the projection it replaced keyed on the container kind rather than selecting structurally. That is a statement about the past and stays true; it now says "the since-removed container kind" rather than naming a type that is not there to look up. --- hive-c0re/src/job_queue/exec.rs | 21 +++++------- hive-c0re/src/job_queue/mod.rs | 48 +++++++++++++++------------- hive-c0re/src/job_queue/model.rs | 20 ++---------- hive-c0re/src/job_queue/scheduler.rs | 2 +- 4 files changed, 37 insertions(+), 54 deletions(-) diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index fe0c4074..d8866327 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -53,7 +53,7 @@ pub(super) async fn run_node( kind: &NodeKind, ) -> (super::JobBuilder, Result<()>) { // The agent this node targets rides the payload — empty for the agentless - // container kinds (`MetaLock`, `Dag`), which never read it. + // kinds (`MetaLock`, `Reparent`), which never read it. let agent = kind.agent(); // Every arm is `Result<()>`; the three that grow work declare into `builder` // *synchronously*, after their own awaits have finished. Borrowing `&builder` @@ -115,26 +115,21 @@ pub(super) async fn run_node( run_finalize_deploy(coord, *approval_id).await } NodeKind::DeployTail { approval_id, .. } => { - run_deploy_tail(coord, coord.job_queue.dag_of(id), agent, *approval_id).await + run_deploy_tail(coord, coord.job_queue.root_of(id), agent, *approval_id).await } NodeKind::ResolveApproval { approval_id, outcome, - } => run_resolve_approval(coord, coord.job_queue.dag_of(id), *approval_id, *outcome).await, + } => run_resolve_approval(coord, coord.job_queue.root_of(id), *approval_id, *outcome).await, NodeKind::EmitRebuilt { ok, .. } => { - run_emit_rebuilt(coord, agent, coord.job_queue.dag_of(id), *ok).await; + run_emit_rebuilt(coord, agent, coord.job_queue.root_of(id), *ok).await; Ok(()) } NodeKind::SetWanted { up, .. } => run_set_wanted(coord, agent, *up), - // The nodes that carry no work of their own; completing one lets it - // reach `Finishing` so the nodes under it start. - // - `Dag`: pure grouping container. The DAG's terminal side effect, if - // any, is its own tail node in the graph. - // - `DeployWindow` / `AgentWindow`: pure resource holders (braces) — - // what they declare stays held until their subtree settles. - NodeKind::Dag { .. } | NodeKind::DeployWindow { .. } | NodeKind::AgentWindow { .. } => { - Ok(()) - } + // Braces carry no work of their own; completing one lets it reach + // `Finishing` so the nodes under it start. What they declare stays held + // until their whole subtree settles. + NodeKind::DeployWindow { .. } | NodeKind::AgentWindow { .. } => Ok(()), }; (builder, result) } diff --git a/hive-c0re/src/job_queue/mod.rs b/hive-c0re/src/job_queue/mod.rs index 51febed8..24f58559 100644 --- a/hive-c0re/src/job_queue/mod.rs +++ b/hive-c0re/src/job_queue/mod.rs @@ -10,17 +10,16 @@ //! carries the agent it targets ([`NodeKind::agent`]); the two resource //! classes are [`resource::Resource`] (`BuildSlot` node-held, `Agent` lease //! subtree-held), declared per node at its construction site; -//! - a **DAG is a single container node** ([`NodeKind::Dag`], `parent = None`) -//! carrying the group's metadata, with the work nodes hung under it as -//! its subtree (the **parent axis** groups; `deps` order). So the container's -//! `NodeId` is the DAG id, its rolled-up state is the DAG state, and membership -//! is a graph walk — there are no host grouping side-tables. The lease is owned -//! by a subtree root and borrowed by its descendants (continuity); -//! - per-DAG terminal work is an ordinary **tail node** +//! - **a job has no container node.** A template declares its nodes and names +//! the roots it wants back; `insert_job` returns those ids. Grouping is the +//! parent axis (a root's rolled-up state *is* its subtree's), so membership is +//! a graph walk with no host-side side-tables. The lease is owned by a subtree +//! root and borrowed by its descendants (continuity); +//! - terminal work is an ordinary **tail node** //! ([`NodeKind::ResolveApproval`] / [`NodeKind::EmitRebuilt`]) that the builder -//! appends in [`templates`], edged onto the DAG's other group roots by the -//! outcome it reports. Templates emit one tail per outcome and the graph runs -//! exactly one, so nothing branches at runtime. +//! appends in [`templates`], edged onto the job's group roots by the outcome it +//! reports. Templates emit one tail per outcome and the graph runs exactly one, +//! so nothing branches at runtime. //! //! The queue is runtime-only (no persistence): an empty graph on boot; desired //! state is re-derived by the reconcile sweep. A single scheduler task @@ -90,12 +89,10 @@ pub struct RunningTransient { /// The crate scheduler, specialised to this host's node + resource types. /// -/// A **DAG is a single container node** ([`NodeKind::Dag`], `parent = None`) -/// whose subtree is the DAG's work — so the container's `NodeId` is the DAG id, -/// its rolled-up state is the DAG state, and there are no grouping side-tables: -/// membership + meta are graph queries ([`container`] + the `hive_jobq::Graph` -/// accessors, with the meta read straight off the container's payload). One -/// shared crate [`Graph`] holds every DAG. +/// A job is **just its nodes** — no container, no grouping side-tables. A +/// root's rolled-up state is its subtree's, so membership is a graph walk and +/// "which job is this node in" is [`JobQueue::root_of`]. One shared crate +/// [`Graph`] holds every job's nodes. /// /// There is deliberately **no wrapper struct and no per-node side map**. The /// last map held the `build_logs` row id; that link now lives on the log row @@ -229,11 +226,16 @@ impl JobQueue { &self.sched } - /// The DAG container id owning `node`, for log lines and the dashboard. - /// Derived from the graph rather than carried alongside the node — the - /// parent axis already knows it. + /// The id of the **group root** `node` belongs to, for log lines and the + /// dashboard. Derived from the graph rather than carried alongside the node + /// — the parent axis already knows it. + /// + /// Was `dag_of`, when a job's nodes hung under a container node that *was* + /// the group. Without it the parent chain ends at whichever root the + /// template declared, so this answers "which root owns this node", not + /// "which DAG is this in" — there is no longer such a thing. #[must_use] - pub fn dag_of(&self, node: NodeId) -> Option { + pub fn root_of(&self, node: NodeId) -> Option { self.lock().graph().root_of(node).map(NodeId::get) } @@ -418,9 +420,9 @@ fn find_node(sched: &Sched, id: u64) -> Option { /// root, plus the newest [`MAX_HISTORY_DAGS`] settled ones. /// /// Selected *structurally* — a root is a node with no parent. The typed -/// projection this replaced keyed on `NodeKind::Dag` instead, which made the -/// visible set depend on one host node kind; nothing here knows what a node -/// means. +/// projection this replaced keyed on the since-removed container kind instead, +/// which made the visible set depend on one host node kind; nothing here knows +/// what a node means. /// /// **This bound is load-bearing, not tidiness.** Nothing ever removes a node /// from the graph (bounded pruning is a Stage-C follow-up), so serving diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs index f1f7936d..fff79617 100644 --- a/hive-c0re/src/job_queue/model.rs +++ b/hive-c0re/src/job_queue/model.rs @@ -280,18 +280,6 @@ pub enum NodeKind { /// `Prebuild`, but that's a no-op there — the agent is down, so prebuild /// is skipped.) SetWanted { agent: String, up: bool }, - /// The **DAG container** node: one per submitted DAG, carrying the group's - /// domain metadata. Every node hangs *under* it (its subtree), so - /// the container's `NodeId` **is** the DAG id and its rolled-up state **is** - /// the DAG state. Pure grouping — lease- and - /// build-slot-exempt; the executor instant-completes it (`Done`) so it - /// reaches `Finishing` and its children start. - /// - /// No `created_at` here: the graph stamps [`hive_jobq::Node::created_at`] on - /// every node at insert, so the container already has one. A second copy in - /// the payload would be the same instant recorded twice, with only this - /// variant's version reachable to a generic viewer. - Dag { source: Source, reason: String }, } /// How a hive-c0re node describes itself to a generic graph viewer. @@ -362,14 +350,13 @@ impl NodeKind { NodeKind::ResolveApproval { .. } => "resolve_approval", NodeKind::EmitRebuilt { .. } => "emit_rebuilt", NodeKind::SetWanted { .. } => "set_wanted", - NodeKind::Dag { .. } => "dag", } } /// The agent this node targets, or `""` for agentless kinds /// ([`NodeKind::MetaLock`] on the `hyperhive` pseudo-agent, - /// [`NodeKind::Reparent`] which can span multiple agents, and the - /// [`NodeKind::Dag`] container). + /// [`NodeKind::Reparent`] which can span multiple agents, and + /// [`NodeKind::ResolveApproval`] which acts on an approval row). #[must_use] pub fn agent(&self) -> &str { match self { @@ -397,8 +384,7 @@ impl NodeKind { | NodeKind::SetWanted { agent, .. } => agent, NodeKind::MetaLock { .. } | NodeKind::Reparent { .. } - | NodeKind::ResolveApproval { .. } - | NodeKind::Dag { .. } => "", + | NodeKind::ResolveApproval { .. } => "", } } diff --git a/hive-c0re/src/job_queue/scheduler.rs b/hive-c0re/src/job_queue/scheduler.rs index 3c19c529..f60f2028 100644 --- a/hive-c0re/src/job_queue/scheduler.rs +++ b/hive-c0re/src/job_queue/scheduler.rs @@ -93,7 +93,7 @@ pub async fn run_worker(coord: Arc) { let coord = node_coord; async move { tracing::info!( - dag = coord.job_queue.dag_of(id).unwrap_or_default(), + dag = coord.job_queue.root_of(id).unwrap_or_default(), node = id.get(), kind = kind.as_str(), agent = %kind.agent(), From ddc017f01b230a171268778553913765e9561e23 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 4 Aug 2026 19:31:33 +0200 Subject: [PATCH 10/10] wip(#3001): convert tests off the container id; drop Source + insert_group The last of the DAG-container removal. `tests.rs` navigated by the id `submit` returned, so removing the container removed the tests' way of finding what they inserted; they name the roots they assert on now, which is the same handle production uses. Three findings the port surfaced, each a behaviour change rather than a test fix: - Cancelling a rebuild's head no longer drops the job. `Reconcile`'s edge accepts a skipped brace, and a cancel-cascade skips rather than cancels, so the tail stays claimable. Dropping a job means cancelling every id the insert returned. - A directly-cancelled group root reads terminal while a spared tail still runs; the cancel used to land on a node above it, which rolled up Finishing instead. - "One DAG per hive-wide op" is not expressible without a container. The three tests asserting it now assert that every named root is top-level, which is what makes the per-agent subgraphs concurrent. Deletes two tests: one asserted only that two containers get distinct ids, the other re-ran an existing case under a second name. `Source`, `insert_group` and the stop path's `reason` string went dead with the container and are removed with it. --- hive-c0re/src/dashboard/lifecycle_ops.rs | 21 +- hive-c0re/src/job_queue/mod.rs | 37 +- hive-c0re/src/job_queue/model.rs | 26 +- hive-c0re/src/job_queue/tests.rs | 410 +++++++++++++---------- hive-c0re/src/server.rs | 10 +- hive-host-sock/src/jobs.rs | 42 +-- 6 files changed, 272 insertions(+), 274 deletions(-) diff --git a/hive-c0re/src/dashboard/lifecycle_ops.rs b/hive-c0re/src/dashboard/lifecycle_ops.rs index e1c46463..e12723fa 100644 --- a/hive-c0re/src/dashboard/lifecycle_ops.rs +++ b/hive-c0re/src/dashboard/lifecycle_ops.rs @@ -94,7 +94,8 @@ pub(super) async fn post_kill( // lease keeps it from racing an in-flight rebuild for the same // agent, and per-node progress surfaces on the queue snapshot. if let Err(e) = - crate::job_queue::power::stop_many(&state.coord, &[logical.clone()], true).await + crate::job_queue::power::stop_many(&state.coord, std::slice::from_ref(&logical), true) + .await { tracing::error!(agent = %logical, error = ?e, "graceful stop: insert failed"); } @@ -111,7 +112,8 @@ pub(super) async fn post_kill( // manager calling Kill on its own container is self-suicide // mid-call, not a legitimate operator action. if let Err(e) = - crate::job_queue::power::stop_many(&state.coord, &[logical.clone()], false).await + crate::job_queue::power::stop_many(&state.coord, std::slice::from_ref(&logical), false) + .await { tracing::error!(agent = %logical, error = ?e, "stop: insert failed"); } @@ -146,15 +148,20 @@ pub(super) async fn post_restart( return reject; } if params.graceful { - if let Err(e) = - crate::job_queue::power::restart_many(&state.coord, &[logical.clone()], true).await + if let Err(e) = crate::job_queue::power::restart_many( + &state.coord, + std::slice::from_ref(&logical), + true, + ) + .await { tracing::error!(agent = %logical, error = ?e, "graceful restart: insert failed"); } return (StatusCode::OK, "ok").into_response(); } if let Err(e) = - crate::job_queue::power::restart_many(&state.coord, &[logical.clone()], false).await + crate::job_queue::power::restart_many(&state.coord, std::slice::from_ref(&logical), false) + .await { tracing::error!(agent = %logical, error = ?e, "restart: insert failed"); } @@ -219,7 +226,9 @@ pub(super) async fn post_start( return (StatusCode::OK, "ok").into_response(); } } - if let Err(e) = crate::job_queue::power::start_many(&state.coord, &[logical.clone()]).await { + if let Err(e) = + crate::job_queue::power::start_many(&state.coord, std::slice::from_ref(&logical)).await + { tracing::error!(agent = %logical, error = ?e, "start: insert failed"); } (StatusCode::OK, "ok").into_response() diff --git a/hive-c0re/src/job_queue/mod.rs b/hive-c0re/src/job_queue/mod.rs index 24f58559..04ed00af 100644 --- a/hive-c0re/src/job_queue/mod.rs +++ b/hive-c0re/src/job_queue/mod.rs @@ -45,7 +45,7 @@ use hive_jobq_wire::{GraphNode, GraphWire}; use tokio::sync::Notify; pub use hive_jobq::TerminalState; -pub use model::{NodeKind, PermPayload, Source, State}; +pub use model::{NodeKind, PermPayload, State}; use resource::Resource; /// A job under construction: `hive_jobq`'s builder over this queue's payload @@ -137,35 +137,12 @@ fn outcome_of(result: Result<(), String>) -> Outcome { } } -/// Insert a declared `job` into the shared graph, returning the inserted ids. -/// -/// A node that declared no parent hangs under `group_parent` — the DAG -/// container for a template, the emitting node for a runtime-appended -/// subgraph. Templates declare the parent axis + sibling ordering directly, so -/// there is no dep-on-root to drop and no lease to hoist: each node declares -/// its own resources, and the crate's borrow model keeps a resource continuous -/// across a subtree (a root owns it, descendants borrow it). Independent group -/// roots carry no cross-links, so a multi-agent DAG's per-agent subgraphs run -/// concurrently, each on its own lease. -/// -/// # Errors -/// Propagates a crate graph-insert error (malformed dep/parent / dep-scope). -fn insert_group( - inner: &mut Sched, - declare: impl FnOnce(&JobBuilder), - group_parent: Option, -) -> anyhow::Result<()> { - inner - .insert_job(group_parent, |b| { - declare(b); - // A runtime-appended subgraph is addressed by the node that emitted - // it (`group_parent`), so this path names nothing. Callers that DO - // want a handle use `JobQueue::insert` and name the node there. - Vec::new() - }) - .map_err(|e| anyhow::anyhow!("job_queue: graph insert failed: {e}"))?; - Ok(()) -} +// `insert_group` lived here: a `group_parent`-taking insert whose only +// remaining caller was the DAG container, everything under it. Runtime growth +// never went through it — an executor declares into the builder `hive_jobq` +// hands it, which parents the new work under the emitting node by +// construction. With no container to be the other kind of parent, the +// distinction it existed to express is gone. impl JobQueue { #[must_use] diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs index fff79617..57cf0dca 100644 --- a/hive-c0re/src/job_queue/model.rs +++ b/hive-c0re/src/job_queue/model.rs @@ -1,18 +1,18 @@ -//! Data model for the generic job-DAG queue: node kinds (the primitive -//! operations), dependency edges, and the runtime `Dag` / `Node` store. -//! The `Source` / `State` / `PermPayload` wire enums live in -//! `hive_host_sock::jobs` (they travel on the host admin socket) and are -//! re-exported here for the queue's internal use. The graph itself is -//! served through `hive_jobq_wire`'s generic projection — there is no -//! second, typed view of it any more. +//! Data model for the generic job-DAG queue: the node kinds — the primitive +//! operations — and what each one carries. The `State` / `PermPayload` wire +//! enums live in `hive_host_sock::jobs` (they travel on the host admin +//! socket) and are re-exported here for the queue's internal use. The graph +//! itself is served through `hive_jobq_wire`'s generic projection — there is +//! no second, typed view of it any more. //! -//! Two levels: the **DAG** is the unit of cancel / approval-resolution -//! and the dashboard group; the **node** is the unit of scheduling / -//! execution / build-log, and carries its own `agent` (a -//! DAG can span agents). See `docs/coordinator.md::Job queue` for the -//! full design. +//! **One level, not two.** The node is the unit of everything: scheduling, +//! execution, build-log, cancel, and the dashboard group (a group root's +//! subtree *is* the group). A DAG used to be a second level above it, with +//! its own store and its own id; there is no container node any more, so a +//! job is exactly the nodes it declared. See `docs/coordinator.md::Job queue` +//! for the full design. -pub use hive_host_sock::jobs::{PermPayload, Source, State}; +pub use hive_host_sock::jobs::{PermPayload, State}; use serde::Serialize; use hive_jobq::TerminalState; diff --git a/hive-c0re/src/job_queue/tests.rs b/hive-c0re/src/job_queue/tests.rs index bd7cecb6..41e45018 100644 --- a/hive-c0re/src/job_queue/tests.rs +++ b/hive-c0re/src/job_queue/tests.rs @@ -28,28 +28,53 @@ fn insert(q: &JobQueue, declare: impl FnOnce(&JobBuilder)) { .expect("valid shape"); } +/// Insert a declared job and hand back the ids of the nodes it **named**, in +/// the order it named them. +/// +/// This is the handle that replaced the DAG id: there is no container to point +/// at any more, so a test that needs to cancel a job or read its state names +/// the roots it cares about — exactly what production does with the ids +/// [`JobQueue::insert_job`] returns. +/// Handed back as raw `u64`, the same form production passes to `cancel` and +/// `node_subtrees` — a `NodeId` cannot be fabricated, so the read surface takes +/// raw ids and searches for them. +fn insert_named( + q: &JobQueue, + declare: impl FnOnce(&JobBuilder) -> Vec, +) -> Vec { + q.insert_job(declare) + .expect("valid shape") + .into_iter() + .map(NodeId::get) + .collect() +} + fn ident(s: &str) -> hive_types::Ident { hive_types::Ident::parse(s).expect("valid test ident") } -fn rebuild(builder: &JobBuilder, agent: &str) { - templates::rebuild(builder, agent, true); +fn rebuild(builder: &JobBuilder, agent: &str) -> Vec { + templates::rebuild(builder, agent, true) } /// Restart shape with every agent treated as **running** — the online /// shape (`[Signal→Drain→] StopForUpdate → Reconcile`, no `SetWanted` head) /// most queue-mechanics tests assume. Mirrors the pre-dynamic /// `templates::restart` (which is now the state-aware `power::restart_nodes`). -fn restart_online(builder: &JobBuilder, agents: &[&str], graceful: bool) { +fn restart_online( + builder: &JobBuilder, + agents: &[&str], + graceful: bool, +) -> Vec { let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect(); - power::restart_nodes(builder, &targets, graceful); + power::restart_nodes(builder, &targets, graceful) } /// Stop shape with every agent treated as **running** — the online shape /// (`SetWanted → [Signal→Drain→](graceful) Reconcile`). -fn stop_online(builder: &JobBuilder, agents: &[&str], graceful: bool) { +fn stop_online(builder: &JobBuilder, agents: &[&str], graceful: bool) -> Vec { let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect(); - power::stop_nodes(builder, &targets, graceful); + power::stop_nodes(builder, &targets, graceful) } // `Claimed` / `ClaimReady` / `CompleteNode` lived here: a claim snapshot type @@ -299,63 +324,68 @@ fn dag_count(q: &JobQueue) -> usize { .count() } -// ---- submit (dedup removed — every submit is a fresh DAG) ---- +// ---- insert (dedup removed — every insert is a fresh job) ---- +// +// `submit_assigns_distinct_ids` lived here and is gone. Its whole body was +// "two inserts get different container ids" — an assertion about the id +// allocation of a node type this issue deleted, and in any case `hive_jobq`'s +// property rather than c0re's. What the tests below keep is the part that was +// about c0re: **no dedup**, now read off the group count instead of off an id. -#[test] -fn submit_assigns_distinct_ids() { - let q = JobQueue::new(1); - let first = insert(&q, |builder| rebuild(builder, "agent-a")); - let second = insert(&q, |builder| rebuild(builder, "agent-b")); - assert_ne!(first, second); - assert_eq!(dag_count(&q), 2); -} - -/// Submit-time dedup was removed with the agent-per-node refactor (a -/// multi-agent DAG has no single agent to key a dedup on), so an identical -/// resubmit — same template + agent, still queued — now enqueues a distinct -/// DAG instead of collapsing into the pending one. Whether any dedup needs +/// Insert-time dedup was removed with the agent-per-node refactor (a +/// multi-agent job has no single agent to key a dedup on), so an identical +/// re-insert — same template + agent, still queued — now enqueues a distinct +/// group instead of collapsing into the pending one. Whether any dedup needs /// reintroducing is tracked as a follow-up. #[test] fn identical_resubmit_is_a_distinct_dag() { let q = JobQueue::new(1); - let first = insert(&q, |builder| rebuild(builder, "agent-a")); - let resubmit = insert(&q, |builder| rebuild(builder, "agent-a")); - assert_ne!(first, resubmit, "no dedup: identical resubmit is a new DAG"); - assert_eq!(dag_count(&q), 2); + let first = insert_named(&q, |builder| rebuild(builder, "agent-a")); + let after_first = dag_count(&q); + let resubmit = insert_named(&q, |builder| rebuild(builder, "agent-a")); + assert_ne!( + first, resubmit, + "no dedup: an identical re-insert declares its own nodes" + ); + // Counted as a doubling rather than against a literal: one rebuild is + // several group roots now, and pinning the number here would make this + // test fail on any shape change while saying nothing about dedup. + assert_eq!( + dag_count(&q), + after_first * 2, + "the re-insert added its own roots instead of collapsing into the pending ones" + ); } #[test] fn distinct_submits_never_collapse() { let q = JobQueue::new(1); - let rebuild_a = insert(&q, |builder| rebuild(builder, "agent-a")); - let rebuild_b = insert(&q, |builder| rebuild(builder, "agent-b")); - let restart_a = insert(&q, |builder| { - restart_online(builder, &["agent-a"], false); - }); + let rebuild_a = insert_named(&q, |builder| rebuild(builder, "agent-a")); + let one_rebuild = dag_count(&q); + let rebuild_b = insert_named(&q, |builder| rebuild(builder, "agent-b")); + let two_rebuilds = dag_count(&q); + let restart_a = insert_named(&q, |builder| restart_online(builder, &["agent-a"], false)); assert_ne!(rebuild_a, rebuild_b); assert_ne!(rebuild_a, restart_a); - assert_eq!(dag_count(&q), 3); + assert_eq!( + two_rebuilds, + one_rebuild * 2, + "two rebuilds, nothing merged" + ); + assert_eq!( + dag_count(&q), + two_rebuilds + restart_a.len(), + "a restart of an agent that already has a queued rebuild is still its own group" + ); } -#[test] -fn resubmit_while_running_is_new_dag() { - // The "while running" is not load-bearing and used to be staged by claiming - // a node first. `submit` appends a container and inserts the declared - // group; it never consults the state of any existing node, so whether an - // earlier DAG is running cannot change the outcome. What is actually being - // asserted — no dedup, ever — is `identical_resubmit_is_a_distinct_dag`. - // - // Kept as the *named* case because "a config bump mid-build must not be - // swallowed" is the scenario people worry about, and a reader looking for - // it should find it. - let q = JobQueue::new(1); - let a = insert(&q, |builder| rebuild(builder, "agent-a")); - let again = insert(&q, |builder| { - rebuild(builder, "agent-a"); - }); - assert_ne!(a, again); - assert_eq!(dag_count(&q), 2); -} +// `resubmit_while_running_is_new_dag` lived here: the same two inserts as +// above, kept under a second name so a reader looking for "a config bump +// mid-build must not be swallowed" would find it. It asserted nothing the +// test above doesn't — `insert_job` never consults the state of an existing +// node, so "while running" could not change the outcome and was never staged. +// The scenario is named in that test's doc instead; a duplicate test is a +// second place for the same fact to rot. // ---- malformed specs: no longer expressible ---- // @@ -393,7 +423,9 @@ fn rebuild_chain_is_declared_serial() { // The non-graceful shape asserted here has no quiesce chain, so nothing here // is concurrent — but the name would mislead about the graceful one. let q = JobQueue::new(1); - let id = insert(&q, |builder| rebuild(builder, "agent-a")); + insert(&q, |builder| { + rebuild(builder, "agent-a"); + }); assert_eq!( declared_shape(&q), vec![ @@ -460,15 +492,9 @@ fn rebuild_chain_is_declared_serial() { #[test] fn graceful_rebuild_chain_drains_before_stopping() { let q = JobQueue::new(1); - let id = q - .submit( - Source::AutoUpdate, - "sweep".to_owned(), - |builder: &JobBuilder| { - templates::graceful_rebuild_nodes(builder, "agent-a", true, None); - }, - ) - .expect("valid shape"); + insert(&q, |builder| { + templates::graceful_rebuild_nodes(builder, "agent-a", true, None); + }); // Asserted as full rows, not just kinds: the kind list is identical whether // the quiesce chain runs beside the build or nested under it, so a // kind-only assertion cannot see the bug this shape exists to fix. @@ -522,7 +548,7 @@ fn non_graceful_rebuild_has_no_signal_or_drain() { // job keeps its nodes to itself and inserts them, so what it built is // observable where it matters — in what the scheduler runs. let q = JobQueue::new(1); - let id = insert(&q, |builder| { + insert(&q, |builder| { templates::rebuild_nodes(builder, "agent-a", true, None); }); assert_eq!( @@ -559,7 +585,9 @@ fn rebuild_chain_declares_its_resources_on_the_brace() { // `a_contended_resource_goes_to_the_oldest_waiter`. What is c0re's is // *which* nodes contend in the first place — a declaration, asserted here.) let q = JobQueue::new(1); - let id = insert(&q, |builder| rebuild(builder, "agent-a")); + insert(&q, |builder| { + rebuild(builder, "agent-a"); + }); let res = |kind: &str| declared_resources(&q, node_of(&q, kind)); let agent = || Resource::Agent("agent-a".to_owned()); @@ -602,13 +630,18 @@ fn rebuild_chain_declares_its_resources_on_the_brace() { // ---- per-agent lease ---- #[test] -fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() { +fn multi_agent_restart_declares_concurrent_per_agent_subgraphs() { let q = JobQueue::new(4); - let id = insert(&q, |builder| { - restart_online(builder, &["agent-a", "agent-b"], false); + let roots = insert_named(&q, |builder| { + restart_online(builder, &["agent-a", "agent-b"], false) }); - // A hive-wide restart is ONE DAG, not one-per-agent. - assert_eq!(dag_count(&q), 1); + // Was "a hive-wide restart is ONE DAG": one container over both agents. + // With the container gone it is one *insert* over N independent groups — + // which is the same claim about the operator's action and a better one + // about the graph, since independence is what lets them run at once. + // Asserted against the named count rather than a literal: the point is + // that every root the job named is top-level, with nothing above it. + assert_eq!(dag_count(&q), roots.len(), "every named root is top-level"); // Each agent's subgraph head (StopForUpdate, since both are running) is a // group root with no deps, so nothing orders them against each other; and // each declares only its OWN agent's lease, so nothing makes them contend. @@ -638,13 +671,13 @@ fn multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs() { } #[test] -fn multi_agent_stop_is_one_dag_with_concurrent_per_agent_subgraphs() { +fn multi_agent_stop_declares_concurrent_per_agent_subgraphs() { let q = JobQueue::new(4); - let id = insert(&q, |builder| { - stop_online(builder, &["agent-a", "agent-b"], false); + let roots = insert_named(&q, |builder| { + stop_online(builder, &["agent-a", "agent-b"], false) }); - // A hive-wide stop is ONE DAG, not one-per-agent. - assert_eq!(dag_count(&q), 1); + // See the restart case above for why this is a named-root count now. + assert_eq!(dag_count(&q), roots.len(), "every named root is top-level"); // Same declared story as the restart case above: each agent's subgraph head // is a group root with no node-deps, holding only its own agent's lease. // Independent roots on disjoint resources is what "concurrently" means at @@ -669,21 +702,24 @@ fn multi_agent_stop_is_one_dag_with_concurrent_per_agent_subgraphs() { } #[test] -fn multi_agent_start_one_dag_folds_per_agent_stale_rebuild() { +fn multi_agent_start_folds_per_agent_stale_rebuild() { let q = JobQueue::new(4); // fresh: offline + not stale → SetWanted → Reconcile. // stale: offline + stale → SetWanted → «rebuild subgraph». - let id = insert(&q, |builder| { + let roots = insert_named(&q, |builder| { power::start_nodes( builder, &[ ("fresh".to_owned(), false, false), ("stale".to_owned(), false, true), ], - ); + ) }); - // One DAG spanning both agents. - assert_eq!(dag_count(&q), 1); + // One insert spanning both agents — and an *uneven* number of roots, which + // is the shape this test is about: the fresh agent names one, the stale one + // names four (its rebuild chains behind `SetWanted` rather than nesting + // under it, so the head alone would report the start done mid-rebuild). + assert_eq!(dag_count(&q), roots.len(), "every named root is top-level"); // The fold is a *declared* difference, readable the moment submit returns: // both agents get a `SetWanted(Up)` group root, but the fresh agent's // subgraph ends at the Reconcile behind it while the stale agent's carries @@ -733,31 +769,32 @@ fn offline_agents_skip_mechanical_nodes_but_keep_reconcile() { // read and node exec. let q = JobQueue::new(4); // Offline graceful stop → SetWanted(Off) → Reconcile (no Signal/Drain). - let stop = insert(&q, |builder| { - power::stop_nodes(builder, &[("down".to_owned(), false)], true); + let stop = insert_named(&q, |builder| { + power::stop_nodes(builder, &[("down".to_owned(), false)], true) }); // Offline restart → a lone Reconcile (no SetWanted, no StopForUpdate): // nothing to bounce, and restart never rewrites intent, so the tail // Reconcile converges the down agent to its existing `wanted`. - let restart = insert(&q, |builder| { - power::restart_nodes(builder, &[("down2".to_owned(), false)], true); + let restart = insert_named(&q, |builder| { + power::restart_nodes(builder, &[("down2".to_owned(), false)], true) }); - let shape = |id: u64| -> Vec { - // The group's work nodes: its subtree minus the container itself, - // which the generic view carries as an ordinary node. - q.node_subtrees(&[id]) + // The whole group, **root included** — the root is a work node now + // (`SetWanted` for the stop, the lone `Reconcile` for the restart), not a + // container to be filtered out. One id per agent, which is what the chain + // named. + let shape = |roots: &[u64]| -> Vec { + q.node_subtrees(roots) .iter() - .filter(|n| n.id != id) .map(|n| n.payload.label.clone()) .collect() }; assert_eq!( - shape(stop), + shape(&stop), vec!["set_wanted".to_owned(), "reconcile".to_owned()], "offline graceful stop skips the signal/drain quiesce, keeps Reconcile" ); assert_eq!( - shape(restart), + shape(&restart), vec!["reconcile".to_owned()], "offline restart is a lone Reconcile (no SetWanted head, nothing to stop)" ); @@ -774,20 +811,14 @@ fn boot_sweep_nodes_declare_their_own_resources() { // meta commit inside another node's staged deploy window. Nothing failed to // compile; only an exhaustive caller list would have caught it. let q = JobQueue::new(4); - let id = q - .submit( - Source::AutoUpdate, - "boot".to_owned(), - |builder: &JobBuilder| { - crate::workers::auto_update::boot_nodes( - builder, - true, - vec!["stale-agent".to_owned()], - vec!["drifted-agent".to_owned()], - ); - }, - ) - .expect("valid shape"); + insert(&q, |builder| { + crate::workers::auto_update::boot_nodes( + builder, + true, + vec!["stale-agent".to_owned()], + vec!["drifted-agent".to_owned()], + ); + }); let mut lock = declared_resources(&q, node_of(&q, "meta_lock")); lock.sort_by_key(|r| format!("{r:?}")); @@ -893,7 +924,9 @@ fn rebuild_reconcile_waits_for_the_whole_build_subtree() { // **did** flatten this chain, and the guarantee survives only because the // roll-up point moved with it. let q = JobQueue::new(1); - let id = insert(&q, |builder| rebuild(builder, "agent-a")); + insert(&q, |builder| { + rebuild(builder, "agent-a"); + }); let shape = declared_shape(&q); let parent_of = |kind: &str| { shape @@ -965,7 +998,7 @@ fn rebuild_reconcile_waits_for_the_whole_build_subtree() { #[test] fn a_fanned_out_mechanical_node_declares_its_agent_lease() { let q = JobQueue::new(4); - let id = insert(&q, |builder| { + insert(&q, |builder| { templates::fanned_out_mechanical( builder, NodeKind::Start { @@ -1000,15 +1033,9 @@ fn a_fanned_out_mechanical_node_declares_its_agent_lease() { fn a_meta_lock_grows_one_rebuild_subgraph_per_agent() { let q = JobQueue::new(4); let agents = vec!["alice".to_owned(), "bob".to_owned()]; - let id = q - .submit( - Source::AutoUpdate, - "sweep".to_owned(), - |builder: &JobBuilder| { - templates::grown_graceful_rebuilds(builder, &agents, true); - }, - ) - .expect("valid shape"); + insert(&q, |builder| { + templates::grown_graceful_rebuilds(builder, &agents, true); + }); // One chain per agent, each an independent group root — so the two rebuild // concurrently, each on its own lease. @@ -1038,20 +1065,39 @@ fn a_meta_lock_grows_one_rebuild_subgraph_per_agent() { #[test] fn cancel_clears_queued_dag() { let q = JobQueue::new(1); - let id = insert(&q, |builder| rebuild(builder, "agent-a")); - assert!(q.cancel(id), "fully-queued dag cancels"); - // The operator sees `Cancelled` the moment the cancel returns — the spared - // tail is still `Pending`, and a DAG must not read `Queued` back to the - // operator who just cancelled it (the dashboard renders this roll-up from - // the snapshot `post_rebuild_queue_cancel` emits synchronously). - assert_eq!(state_of(&q, id), State::Cancelled, "no stale Queued gap"); - // Neither `EmitRebuilt` tail accepts a *dropped* dependency — the ok one is - // `AFTER_OK`, the failure one keys on elimination — so both are cancelled - // with the work and **nothing is left that could still run**: no node is - // spared, so a rebuild that never ran emits nothing. + let roots = insert_named(&q, |builder| rebuild(builder, "agent-a")); + let [head, brace, tail] = roots.as_slice() else { + panic!("a rebuild names three roots, got {roots:?}") + }; + assert!(q.cancel(*head), "fully-queued dag cancels"); + // The operator sees `Cancelled` the moment the cancel returns — a group must + // not read `Queued` back to the operator who just cancelled it (the + // dashboard renders this roll-up from the snapshot + // `post_rebuild_queue_cancel` emits synchronously). + assert_eq!(state_of(&q, *head), State::Cancelled, "no stale Queued gap"); + // Both `EmitRebuilt` tails go with it — the ok one is `AFTER_OK`, the + // failure one keys on elimination — so a rebuild that never ran emits + // nothing. + // + // 🎯 **But `Reconcile` survives, and that is the finding.** Its edge onto + // the brace accepts `done|failed|skipped`, and a cancel-cascade *skips* the + // brace rather than cancelling it — so the edge is satisfied and the tail + // stays claimable. With a container above them all, one cancel took the + // whole group; a job is its roots now, and dropping it means dropping every + // id the insert returned. That is what the ids are for. + assert_eq!( + pending_kinds(&q), + vec!["reconcile"], + "the convergence tail outlives its head's cancel" + ); + assert!( + !q.cancel(*brace), + "the brace was eliminated with the head — there is nothing left to cancel" + ); + assert!(q.cancel(*tail), "the surviving tail cancels on its own id"); assert!( pending_kinds(&q).is_empty(), - "a dropped rebuild leaves nothing alive, got {:?}", + "cancelling every named root leaves nothing alive, got {:?}", pending_kinds(&q) ); } @@ -1067,27 +1113,17 @@ fn cancel_clears_queued_dag() { #[test] fn cancel_drops_one_agents_branch_leaving_the_rest() { let q = JobQueue::new(2); - let id = insert(&q, |builder| { - restart_online(builder, &["agent-a", "agent-b"], false); + let roots = insert_named(&q, |builder| { + restart_online(builder, &["agent-a", "agent-b"], false) }); - // Per-agent subgraphs hang directly off the container, one per agent. - // ⚠️ `parent` is the *graph* parent here, not a DAG-relative one: what - // the typed view called a parentless group root is a direct child of the - // container node in the generic view. - let snap = q.node_subtrees(&[id]); - let a_root = snap - .iter() - .find(|n| { - n.parent == Some(id) - && n.payload - .data - .get("agent") - .and_then(serde_json::Value::as_str) - == Some("agent-a") - }) - .expect("agent-a has a group root"); + // One root per agent, **in the order the chain named them** — that ordering + // is `insert_job`'s contract, and it is what replaced digging the right + // subgraph out of a snapshot by matching on its payload's agent field. + let [a_root, _b_root] = roots.as_slice() else { + panic!("a two-agent restart names one root per agent, got {roots:?}") + }; - assert!(q.cancel(a_root.id), "an interior/group root cancels alone"); + assert!(q.cancel(*a_root), "an interior/group root cancels alone"); // agent-a's subgraph is gone; agent-b's is untouched and still alive. assert!( @@ -1116,23 +1152,26 @@ fn cancel_drops_one_agents_branch_leaving_the_rest() { /// deliberately-stopped as far as reconcile and crash-watch are concerned. #[test] fn cancelled_power_op_runs_no_compensating_node() { - /// Submit-cancel-assert for one power op. Taking the already-submitted DAG - /// id is what removes the need to put three differently-typed recipes in - /// one array: each caller submits its own spec, so no closure type has to - /// be erased to a boxed one. - fn assert_cancels_clean(q: &JobQueue, id: u64, writes_intent: bool, case: &str) { - // Read the intent head off the submitted DAG rather than out of the - // spec: a declared job holds its own nodes and inserts them. - let has_intent = declared_shape(q, id).iter().any(|d| d.kind == "set_wanted"); + /// Insert-cancel-assert for one power op. Taking the roots the job already + /// named is what removes the need to put three differently-typed recipes in + /// one array: each caller inserts its own, so no closure type has to be + /// erased to a boxed one. + fn assert_cancels_clean(q: &JobQueue, roots: &[u64], writes_intent: bool, case: &str) { + // Read the intent head off the inserted nodes rather than out of a + // spec: a declared job holds its own nodes and inserts them. The queue + // is fresh per case, so the whole graph is this one op. + let has_intent = declared_shape(q).iter().any(|d| d.kind == "set_wanted"); assert_eq!(has_intent, writes_intent, "{case}: intent head"); - assert!(q.cancel(id), "{case}: cancelled while queued"); - assert_eq!(state_of(q, id), State::Cancelled); + for root in roots { + assert!(q.cancel(*root), "{case}: cancelled while queued"); + assert_eq!(state_of(q, *root), State::Cancelled); + } // Nothing is left that *could* run. Asserting on the pending set rather // than on "what is ready this instant" also covers a node that is alive // but blocked — which is exactly what a leftover compensating node // would look like. assert_eq!( - pending_kinds(q, id), + pending_kinds(q), Vec::<&str>::new(), "{case}: a power op emits no tail node, so a cancelled one leaves nothing" ); @@ -1144,22 +1183,20 @@ fn cancelled_power_op_runs_no_compensating_node() { let case = format!("graceful={graceful} running={running}"); let q = JobQueue::new(1); - let id = insert(&q, |builder| { - power::restart_nodes(builder, &targets, graceful); + let roots = insert_named(&q, |builder| { + power::restart_nodes(builder, &targets, graceful) }); - assert_cancels_clean(&q, id, false, &format!("restart {case}")); + assert_cancels_clean(&q, &roots, false, &format!("restart {case}")); let q = JobQueue::new(1); - let id = insert(&q, |builder| { - power::stop_nodes(builder, &targets, graceful); - }); - assert_cancels_clean(&q, id, true, &format!("stop {case}")); + let roots = insert_named(&q, |builder| power::stop_nodes(builder, &targets, graceful)); + assert_cancels_clean(&q, &roots, true, &format!("stop {case}")); let q = JobQueue::new(1); - let id = insert(&q, |builder| { - power::start_nodes(builder, &[("agent-a".to_owned(), running, false)]); + let roots = insert_named(&q, |builder| { + power::start_nodes(builder, &[("agent-a".to_owned(), running, false)]) }); - assert_cancels_clean(&q, id, true, &format!("start {case}")); + assert_cancels_clean(&q, &roots, true, &format!("start {case}")); } } } @@ -1177,10 +1214,15 @@ fn cancelled_power_op_runs_no_compensating_node() { #[test] fn cancelled_dag_still_runs_its_approval_tail() { let q = JobQueue::new(1); - let id = insert(&q, |builder| { + insert(&q, |builder| { templates::approval_deploy(builder, "agent-a", 7); }); - assert!(q.cancel(id), "fully-queued dag cancels"); + // Found by kind, not returned: `approval_deploy` deliberately names + // nothing, because nothing polls it — the approval row is how an operator + // follows a deploy, so the template is fire-and-forget in production and a + // test must not make it return an id it would otherwise have no use for. + let window = node_of(&q, "deploy_window").get(); + assert!(q.cancel(window), "fully-queued dag cancels"); // The `Cancelled` tail is the only node whose edge accepts a dropped // dependency, so it is the only one `cancel` spares — and *which* tail // survives is the whole assertion: the template emits one per outcome and @@ -1197,18 +1239,20 @@ fn cancelled_dag_still_runs_its_approval_tail() { ), "only the cancelled-outcome tail is spared, got {spared:?}" ); - // ⚠️ `Finishing`, not `Cancelled` — and the change is a **fix**, not a - // regression. This used to read the host-side `DagView::rollup_state`, - // which flattened the spared tail away and reported the group settled - // while a node of it was still pending. The root's own state is the - // scheduler's answer: `Finishing` means "own logic done, children still - // running", and the tail this test exists to protect *is* such a child. - // A group that still has work to do does not read terminal. - assert_eq!(state_of(&q, id), State::Finishing); + // ⚠️ `Cancelled`, and it reads terminal **while the spared tail is still + // pending** — the one place this differs from the container era, where the + // cancel landed on a node *above* the window and the window rolled up + // `Finishing`. Here the operator cancels the window itself, so its own + // state is `Cancelled` however its subtree is doing. Deliberately asserted + // rather than routed around: the group's card goes terminal while a + // bookkeeping node runs on. That is acceptable for the tail this test + // protects (it resolves the approval row and nothing waits on it), and it + // would not be for work an operator expects to still be watching. + assert_eq!(state_of(&q, window), State::Cancelled); // An unrelated group landing in the same graph doesn't disturb this one's // state — a root rolls up its own subtree, not the graph. - let _other = insert(&q, |builder| rebuild(builder, "agent-b")); - assert_eq!(state_of(&q, id), State::Finishing); + let _other = insert_named(&q, |builder| rebuild(builder, "agent-b")); + assert_eq!(state_of(&q, window), State::Cancelled); } // ---- approval deploy subtree ---- @@ -1221,7 +1265,7 @@ fn cancelled_dag_still_runs_its_approval_tail() { #[test] fn deploy_dag_runs_phases_in_order_and_tails_a_failed_apply() { let q = JobQueue::new(1); - let id = insert(&q, |builder| { + insert(&q, |builder| { templates::approval_deploy(builder, "agent-a", 7); }); @@ -1279,7 +1323,7 @@ fn deploy_apply_grows_rebuild_subgraph_and_finalizes_after_it() { // children run (`a_completing_node_grows_the_work_it_declared`, // `parent_parks_in_finishing_until_children_roll_up`). let q = JobQueue::new(1); - let id = insert(&q, |builder| { + insert(&q, |builder| { templates::deploy_rebuild_nodes(builder, "agent-a", 11); }); @@ -1459,7 +1503,7 @@ fn error_truncation_cuts_on_a_char_boundary() { #[test] fn graceful_stop_shape_signal_drain_reconcile() { let q = JobQueue::new(1); - let id = insert(&q, |builder| { + insert(&q, |builder| { stop_online(builder, &["agent-a"], true); }); assert_eq!( @@ -1497,7 +1541,7 @@ fn graceful_stop_shape_signal_drain_reconcile() { #[test] fn spawn_shape_provision_create_dropin_reconcile() { let q = JobQueue::new(1); - let id = insert(&q, |builder| { + insert(&q, |builder| { templates::spawn(builder, "newbie", 7); }); assert_eq!( @@ -1522,7 +1566,7 @@ fn spawn_shape_provision_create_dropin_reconcile() { #[test] fn perm_change_shape_prefixes_rebuild_chain() { let q = JobQueue::new(1); - let id = insert(&q, |builder| { + insert(&q, |builder| { templates::perm_change( builder, "agent-a", @@ -1561,7 +1605,7 @@ fn reparent_shape_is_a_lone_agentless_meta_window_node() { // `MetaLock`, and it must declare the meta window — a topology commit // must not land inside another node's staged deploy window. let q = JobQueue::new(1); - let id = insert(&q, |builder| { + insert(&q, |builder| { templates::reparent(builder, vec![(ident("alice"), Some(ident("bob")))]); }); assert_eq!( @@ -1585,7 +1629,7 @@ fn reparent_bulk_shape_carries_every_move_on_one_node() { // request is the reason a single node was chosen in the first place. let moves = vec![(ident("alice"), Some(ident("bob"))), (ident("carol"), None)]; let q = JobQueue::new(1); - let id = insert(&q, |builder| { + insert(&q, |builder| { templates::reparent(builder, moves.clone()); }); assert_eq!( diff --git a/hive-c0re/src/server.rs b/hive-c0re/src/server.rs index 9baaefdc..8e534ba1 100644 --- a/hive-c0re/src/server.rs +++ b/hive-c0re/src/server.rs @@ -840,16 +840,10 @@ async fn handle_stop( let mut errors: Vec = Vec::new(); let mut queued: Vec = Vec::new(); - // One DAG for all targeted agents — a per-agent stop subgraph each + // One insert for all targeted agents — a per-agent stop subgraph each // (`SetWanted(Offline) → [Signal → Drain →] Reconcile`), independent - // roots that run concurrently on their own leases. A hive-wide - // `hivectl stop` is now a single DAG, not N. + // roots that run concurrently on their own leases. if !agents.is_empty() { - let reason = if graceful { - "manual via hivectl graceful stop" - } else { - "manual via hivectl stop" - }; match crate::job_queue::power::stop_many(coord, agents, graceful).await { Ok(ids) => { queued.extend(ids.into_iter().map(hive_jobq::NodeId::get)); diff --git a/hive-host-sock/src/jobs.rs b/hive-host-sock/src/jobs.rs index 7502277b..7930807e 100644 --- a/hive-host-sock/src/jobs.rs +++ b/hive-host-sock/src/jobs.rs @@ -1,6 +1,11 @@ -//! Vocabulary hive-c0re's job queue shares with its clients: where a job -//! came from ([`Source`]), what a permission change carries -//! ([`PermPayload`]), and the scheduler's lifecycle [`State`]. +//! Vocabulary hive-c0re's job queue shares with its clients: what a +//! permission change carries ([`PermPayload`]) and the scheduler's +//! lifecycle [`State`]. +//! +//! A `Source` enum lived here too — where a job came from, rendered as the +//! "why" chip. It was a field on the DAG container, and it went with it: a +//! job is its nodes now, and a node says what it does rather than who asked +//! for it. //! //! **The typed `DagView`/`NodeView` projection that used to live here is //! gone.** One graph is served one way now — `hive_jobq_wire`'s generic @@ -12,37 +17,6 @@ use serde::{Deserialize, Serialize}; -/// Where the submit request originated — drives the "why" chip on the -/// dashboard. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum Source { - /// Operator action (dashboard button, CLI, manager tool). - Manual, - /// Meta-update cascade rebuild (grown into the meta-update DAG). - MetaUpdate, - /// Boot-time submission (the boot sweep DAG + boot reconciles). - AutoUpdate, - /// Crash recovery path (future use). - CrashRecover, - /// Operator approved a pending `Approval` row; `approval_id` on - /// the DAG points back at the source row. - Approval, -} - -impl Source { - #[must_use] - pub fn as_str(self) -> &'static str { - match self { - Source::Manual => "manual", - Source::MetaUpdate => "meta_update", - Source::AutoUpdate => "auto_update", - Source::CrashRecover => "crash_recover", - Source::Approval => "approval", - } - } -} - pub use hive_jobq::State; /// Kind-specific payload for `Template::PermChange` DAGs.