wip(#3001): remove submit layer, rescue power ops into job_queue/power.rs
TREE IS RED ON PURPOSE — there is no compiling intermediate between
deleting submit and converting every caller. Checkpoint commit so the
work is durable; do not "fix" it by restoring submit.
Done:
- JobQueue::submit -> JobQueue::insert (no source/reason/container;
returns the ids insert_job names).
- submit.rs deleted. Its 6 pure chain builders + 3 async *_many
gatherers were NOT wrapper code and are rescued into
job_queue/power.rs (templates.rs documents power ops as living
outside it, because their shape needs a live is_running read).
- Converted: meta_inputs 1, topology 2, permissions 3, auto_update 2,
actions 3, lifecycle_handlers 3.
- Dropped source/reason at every converted site: nothing ever read
NodeKind::Dag's fields (only `{ .. }` matches exist), so they are
write-only. Dead reason-only locals deleted; the boot sweep's summary
became a tracing::info! rather than being lost.
Remaining: dashboard/lifecycle_ops 7, server.rs 7, and the test suite —
tests.rs has its own submit() helper whose u64 return is used as the
handle to navigate the inserted DAG, so those need a different way to
find nodes, not a mechanical port.
This commit is contained in:
parent
6f87821110
commit
7c0d9d2379
9 changed files with 198 additions and 301 deletions
|
|
@ -55,13 +55,12 @@ pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
|
||||||
// nothing).
|
// nothing).
|
||||||
let inputs: Vec<String> =
|
let inputs: Vec<String> =
|
||||||
serde_json::from_str(&approval.commit_ref).unwrap_or_default();
|
serde_json::from_str(&approval.commit_ref).unwrap_or_default();
|
||||||
let submitted = coord.job_queue.submit(
|
let inserted = coord.job_queue.insert(|b| {
|
||||||
crate::job_queue::Source::Approval,
|
crate::job_queue::templates::meta_update(b, inputs, Some(id));
|
||||||
format!("approval #{id} meta input update"),
|
Vec::new()
|
||||||
|b| crate::job_queue::templates::meta_update(b, inputs, Some(id)),
|
});
|
||||||
);
|
if let Err(e) = inserted {
|
||||||
if let Err(e) = submitted {
|
return Err(e.context("insert meta-update dag"));
|
||||||
return Err(e.context("submit meta-update dag"));
|
|
||||||
}
|
}
|
||||||
coord.emit_rebuild_queue_snapshot();
|
coord.emit_rebuild_queue_snapshot();
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|
@ -75,13 +74,12 @@ pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
|
||||||
{
|
{
|
||||||
tracing::warn!(agent = %approval.agent, error = ?e, "agent_power: seed on spawn failed");
|
tracing::warn!(agent = %approval.agent, error = ?e, "agent_power: seed on spawn failed");
|
||||||
}
|
}
|
||||||
let submitted = coord.job_queue.submit(
|
let inserted = coord.job_queue.insert(|b| {
|
||||||
crate::job_queue::Source::Approval,
|
crate::job_queue::templates::spawn(b, approval.agent.as_str(), id);
|
||||||
format!("approval #{id} spawn"),
|
Vec::new()
|
||||||
|b| crate::job_queue::templates::spawn(b, approval.agent.as_str(), id),
|
});
|
||||||
);
|
if let Err(e) = inserted {
|
||||||
if let Err(e) = submitted {
|
return Err(e.context("insert spawn dag"));
|
||||||
return Err(e.context("submit spawn dag"));
|
|
||||||
}
|
}
|
||||||
coord.emit_rebuild_queue_snapshot();
|
coord.emit_rebuild_queue_snapshot();
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|
@ -106,12 +104,7 @@ pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
|
||||||
// `run_deploy_apply` (ff-merge, then grows the rebuild subgraph),
|
// `run_deploy_apply` (ff-merge, then grows the rebuild subgraph),
|
||||||
// `run_finalize_deploy` (deploy tag + lock commit) and
|
// `run_finalize_deploy` (deploy tag + lock commit) and
|
||||||
// `run_deploy_tail` (compensation + forge mirror).
|
// `run_deploy_tail` (compensation + forge mirror).
|
||||||
enqueue_approval_rebuild(
|
enqueue_approval_rebuild(&coord, approval.agent.as_str(), id);
|
||||||
&coord,
|
|
||||||
approval.agent.as_str(),
|
|
||||||
id,
|
|
||||||
format!("approval #{id} merge config pr"),
|
|
||||||
);
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -121,19 +114,12 @@ pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
|
||||||
/// dispatch arm — the work ends in a container rebuild routed through the
|
/// dispatch arm — the work ends in a container rebuild routed through the
|
||||||
/// queue. See [`crate::job_queue::templates::approval_deploy`] for the node
|
/// queue. See [`crate::job_queue::templates::approval_deploy`] for the node
|
||||||
/// shape; the executor dispatches each node to the `run_deploy_*` bodies below.
|
/// shape; the executor dispatches each node to the `run_deploy_*` bodies below.
|
||||||
fn enqueue_approval_rebuild(
|
fn enqueue_approval_rebuild(coord: &Arc<Coordinator>, agent: &str, approval_id: i64) {
|
||||||
coord: &Arc<Coordinator>,
|
if let Err(e) = coord.job_queue.insert(|b| {
|
||||||
agent: &str,
|
crate::job_queue::templates::approval_deploy(b, agent, approval_id);
|
||||||
approval_id: i64,
|
Vec::new()
|
||||||
reason: String,
|
}) {
|
||||||
) {
|
tracing::error!(%agent, approval_id, error = ?e, "insert approval deploy dag failed");
|
||||||
if let Err(e) = coord
|
|
||||||
.job_queue
|
|
||||||
.submit(crate::job_queue::Source::Approval, reason, |b| {
|
|
||||||
crate::job_queue::templates::approval_deploy(b, agent, approval_id);
|
|
||||||
})
|
|
||||||
{
|
|
||||||
tracing::error!(%agent, approval_id, error = ?e, "submit approval deploy dag failed");
|
|
||||||
}
|
}
|
||||||
coord.emit_rebuild_queue_snapshot();
|
coord.emit_rebuild_queue_snapshot();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -208,16 +208,18 @@ pub(super) async fn post_meta_update(
|
||||||
if inputs.is_empty() {
|
if inputs.is_empty() {
|
||||||
return error_response("meta-update: no inputs selected");
|
return error_response("meta-update: no inputs selected");
|
||||||
}
|
}
|
||||||
let inputs_label = inputs.join(", ");
|
|
||||||
// Cascade rebuild children fan out from the MetaLock node when the
|
// Cascade rebuild children fan out from the MetaLock node when the
|
||||||
// lock bump lands — appended by the scheduler so they build against
|
// lock bump lands — appended by the scheduler so they build against
|
||||||
// the post-bump lock, and a failed bump simply fans out nothing.
|
// the post-bump lock, and a failed bump simply fans out nothing.
|
||||||
crate::job_queue::submit::meta_update(
|
state
|
||||||
&state.coord,
|
.coord
|
||||||
inputs,
|
.job_queue
|
||||||
crate::job_queue::Source::Manual,
|
.insert(|b| {
|
||||||
format!("meta-update via dashboard ({inputs_label})"),
|
crate::job_queue::templates::meta_update(b, inputs, None);
|
||||||
);
|
Vec::new()
|
||||||
|
})
|
||||||
|
.expect("template-declared shapes are acyclic");
|
||||||
|
state.coord.emit_rebuild_queue_snapshot();
|
||||||
(StatusCode::OK, "ok").into_response()
|
(StatusCode::OK, "ok").into_response()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -155,15 +155,21 @@ pub(super) async fn post_tool_groups(
|
||||||
// META_LOCK inside the WritePermFile node, so concurrent
|
// META_LOCK inside the WritePermFile node, so concurrent
|
||||||
// batch-apply actions for different agents never race on the
|
// batch-apply actions for different agents never race on the
|
||||||
// shared tool-groups.json.
|
// shared tool-groups.json.
|
||||||
crate::job_queue::submit::perm_change(
|
state
|
||||||
&state.coord,
|
.coord
|
||||||
&logical,
|
.job_queue
|
||||||
crate::job_queue::Source::Manual,
|
.insert(|b| {
|
||||||
"tool-group change via permissions UI".to_owned(),
|
crate::job_queue::templates::perm_change(
|
||||||
crate::job_queue::PermPayload::ToolGroups {
|
b,
|
||||||
groups: body.groups.clone(),
|
&logical,
|
||||||
},
|
crate::job_queue::PermPayload::ToolGroups {
|
||||||
);
|
groups: body.groups.clone(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
Vec::new()
|
||||||
|
})
|
||||||
|
.expect("template-declared shapes are acyclic");
|
||||||
|
state.coord.emit_rebuild_queue_snapshot();
|
||||||
tracing::info!(agent = %logical, groups = ?body.groups, "operator: set tool-groups via dashboard");
|
tracing::info!(agent = %logical, groups = ?body.groups, "operator: set tool-groups via dashboard");
|
||||||
Ok((StatusCode::OK, "ok").into_response())
|
Ok((StatusCode::OK, "ok").into_response())
|
||||||
}
|
}
|
||||||
|
|
@ -264,15 +270,21 @@ pub(super) async fn post_capabilities(
|
||||||
// META_LOCK inside the WritePermFile node, so concurrent
|
// META_LOCK inside the WritePermFile node, so concurrent
|
||||||
// batch-apply actions for different agents never race on the
|
// batch-apply actions for different agents never race on the
|
||||||
// shared capabilities.json.
|
// shared capabilities.json.
|
||||||
crate::job_queue::submit::perm_change(
|
state
|
||||||
&state.coord,
|
.coord
|
||||||
&logical,
|
.job_queue
|
||||||
crate::job_queue::Source::Manual,
|
.insert(|b| {
|
||||||
"capability change via dashboard".to_owned(),
|
crate::job_queue::templates::perm_change(
|
||||||
crate::job_queue::PermPayload::Capabilities {
|
b,
|
||||||
caps: body.caps.clone(),
|
&logical,
|
||||||
},
|
crate::job_queue::PermPayload::Capabilities {
|
||||||
);
|
caps: body.caps.clone(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
Vec::new()
|
||||||
|
})
|
||||||
|
.expect("template-declared shapes are acyclic");
|
||||||
|
state.coord.emit_rebuild_queue_snapshot();
|
||||||
tracing::info!(agent = %logical, caps = ?body.caps, "operator: set capabilities via dashboard");
|
tracing::info!(agent = %logical, caps = ?body.caps, "operator: set capabilities via dashboard");
|
||||||
Ok((StatusCode::OK, "ok").into_response())
|
Ok((StatusCode::OK, "ok").into_response())
|
||||||
}
|
}
|
||||||
|
|
@ -359,13 +371,19 @@ pub(super) async fn post_permissions(
|
||||||
}
|
}
|
||||||
// Phase 2 — submit one combined PermChange DAG per affected agent.
|
// Phase 2 — submit one combined PermChange DAG per affected agent.
|
||||||
for (logical, groups, caps) in staged {
|
for (logical, groups, caps) in staged {
|
||||||
crate::job_queue::submit::perm_change(
|
state
|
||||||
&state.coord,
|
.coord
|
||||||
&logical,
|
.job_queue
|
||||||
crate::job_queue::Source::Manual,
|
.insert(|b| {
|
||||||
"batch permission change via permissions UI".to_owned(),
|
crate::job_queue::templates::perm_change(
|
||||||
crate::job_queue::PermPayload::Combined { groups, caps },
|
b,
|
||||||
);
|
&logical,
|
||||||
|
crate::job_queue::PermPayload::Combined { groups, caps },
|
||||||
|
);
|
||||||
|
Vec::new()
|
||||||
|
})
|
||||||
|
.expect("template-declared shapes are acyclic");
|
||||||
|
state.coord.emit_rebuild_queue_snapshot();
|
||||||
tracing::info!(agent = %logical, "operator: batch perm change via dashboard");
|
tracing::info!(agent = %logical, "operator: batch perm change via dashboard");
|
||||||
}
|
}
|
||||||
Ok((StatusCode::OK, "ok").into_response())
|
Ok((StatusCode::OK, "ok").into_response())
|
||||||
|
|
|
||||||
|
|
@ -94,12 +94,15 @@ pub(super) async fn post_set_parent(
|
||||||
new_parent = ?new_parent,
|
new_parent = ?new_parent,
|
||||||
"operator: set-parent via dashboard"
|
"operator: set-parent via dashboard"
|
||||||
);
|
);
|
||||||
submit::reparent(
|
state
|
||||||
&state.coord,
|
.coord
|
||||||
vec![(child, new_parent)],
|
.job_queue
|
||||||
Source::Manual,
|
.insert(|b| {
|
||||||
"manual set-parent via dashboard".to_owned(),
|
crate::job_queue::templates::reparent(b, vec![(child, new_parent)]);
|
||||||
);
|
Vec::new()
|
||||||
|
})
|
||||||
|
.expect("template-declared shapes are acyclic");
|
||||||
|
state.coord.emit_rebuild_queue_snapshot();
|
||||||
Ok((StatusCode::OK, "ok").into_response())
|
Ok((StatusCode::OK, "ok").into_response())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -150,11 +153,14 @@ pub(super) async fn post_set_parent_bulk(
|
||||||
.map_err(|e| error_problem(&e))?;
|
.map_err(|e| error_problem(&e))?;
|
||||||
let names: Vec<&str> = body.iter().map(|e| e.child.as_str()).collect();
|
let names: Vec<&str> = body.iter().map(|e| e.child.as_str()).collect();
|
||||||
tracing::info!(agents = ?names, "operator: set-parent-bulk via dashboard");
|
tracing::info!(agents = ?names, "operator: set-parent-bulk via dashboard");
|
||||||
submit::reparent(
|
state
|
||||||
&state.coord,
|
.coord
|
||||||
moves,
|
.job_queue
|
||||||
Source::Manual,
|
.insert(|b| {
|
||||||
"manual set-parent-bulk via dashboard".to_owned(),
|
crate::job_queue::templates::reparent(b, moves);
|
||||||
);
|
Vec::new()
|
||||||
|
})
|
||||||
|
.expect("template-declared shapes are acyclic");
|
||||||
|
state.coord.emit_rebuild_queue_snapshot();
|
||||||
Ok((StatusCode::OK, "ok").into_response())
|
Ok((StatusCode::OK, "ok").into_response())
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -29,9 +29,9 @@
|
||||||
|
|
||||||
pub mod exec;
|
pub mod exec;
|
||||||
pub mod model;
|
pub mod model;
|
||||||
|
pub mod power;
|
||||||
pub mod resource;
|
pub mod resource;
|
||||||
pub mod scheduler;
|
pub mod scheduler;
|
||||||
pub mod submit;
|
|
||||||
pub mod templates;
|
pub mod templates;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests;
|
mod tests;
|
||||||
|
|
@ -208,20 +208,17 @@ impl JobQueue {
|
||||||
/// # Errors
|
/// # Errors
|
||||||
/// Propagates a graph-insert error (dependencies that aren't
|
/// Propagates a graph-insert error (dependencies that aren't
|
||||||
/// dependency-topological).
|
/// dependency-topological).
|
||||||
pub fn submit(
|
pub fn insert(
|
||||||
&self,
|
&self,
|
||||||
source: Source,
|
declare: impl FnOnce(&JobBuilder) -> Vec<hive_jobq::NodeGuid>,
|
||||||
reason: String,
|
) -> anyhow::Result<Vec<NodeId>> {
|
||||||
declare: impl FnOnce(&JobBuilder),
|
|
||||||
) -> anyhow::Result<u64> {
|
|
||||||
let mut inner = self.lock();
|
let mut inner = self.lock();
|
||||||
let container = inner
|
let named = inner
|
||||||
.append(NodeKind::Dag { source, reason }, Vec::new(), None)
|
.insert_job(None, declare)
|
||||||
.map_err(|e| anyhow::anyhow!("job_queue: container insert failed: {e}"))?;
|
.map_err(|e| anyhow::anyhow!("job_queue: graph insert failed: {e}"))?;
|
||||||
insert_group(&mut inner, declare, Some(container))?;
|
|
||||||
drop(inner);
|
drop(inner);
|
||||||
self.notify.notify_one();
|
self.notify.notify_one();
|
||||||
Ok(container.get())
|
Ok(named)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The scheduler itself, for `hive_jobq`'s run-loop seam
|
/// The scheduler itself, for `hive_jobq`'s run-loop seam
|
||||||
|
|
|
||||||
277
hive-c0re/src/job_queue/power.rs
Normal file
277
hive-c0re/src/job_queue/power.rs
Normal file
|
|
@ -0,0 +1,277 @@
|
||||||
|
//! Power ops (`stop` / `start` / `restart`) — the DAG shapes whose per-agent
|
||||||
|
//! form depends on **live** container state, so they cannot be static
|
||||||
|
//! [`super::templates`] entries.
|
||||||
|
//!
|
||||||
|
//! The split is the purity line, not the subject matter: the `*_chain` /
|
||||||
|
//! `*_nodes` builders below are pure (they take `running` / `stale` as
|
||||||
|
//! parameters, which is what keeps them unit-testable without a container),
|
||||||
|
//! and only the `*_many` entry points do the async `lifecycle::is_running`
|
||||||
|
//! read that produces those parameters.
|
||||||
|
//!
|
||||||
|
//! Each entry point declares its group and inserts it. There is no metadata
|
||||||
|
//! parameter and no container node: attribution is not something every caller
|
||||||
|
//! has to invent, and a DAG is addressed by the nodes a template names.
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use hive_jobq::NodeId;
|
||||||
|
|
||||||
|
use super::JobBuilder;
|
||||||
|
use super::templates;
|
||||||
|
use crate::coordinator::Coordinator;
|
||||||
|
use crate::lifecycle;
|
||||||
|
|
||||||
|
// ---- dynamic power-op DAG assembly ----------------------------------------
|
||||||
|
//
|
||||||
|
// The pure per-agent chain builders below take `running` (and `stale`)
|
||||||
|
// explicitly so they stay pure + unit-testable without a live container;
|
||||||
|
// the async `*_many` fns read the real state via `lifecycle::is_running`
|
||||||
|
// then hand it in. Each chain declares into the shared job it is handed, and
|
||||||
|
// names the nodes it depends on — so there is nothing to rebase.
|
||||||
|
|
||||||
|
/// One agent's **stop** subgraph. `SetWanted(Off)` head + `Reconcile` tail
|
||||||
|
/// always; the graceful `Signal → Drain` quiesce only when the agent is
|
||||||
|
/// actually running (nothing to drain on a down container). The `Reconcile`
|
||||||
|
/// stays even for a down agent so a race-up between the state read and exec
|
||||||
|
/// is still stopped in-DAG.
|
||||||
|
fn stop_chain(builder: &JobBuilder, agent: &str, graceful: bool, running: bool) {
|
||||||
|
// `SetWanted` is the group root and owns the agent lease; the mechanical
|
||||||
|
// steps are its children (borrow the lease, run once it reaches `Finishing`,
|
||||||
|
// dep-ordered among themselves).
|
||||||
|
let a = || agent.to_owned();
|
||||||
|
let wanted = builder
|
||||||
|
.node(NodeKind::SetWanted {
|
||||||
|
agent: a(),
|
||||||
|
up: false,
|
||||||
|
})
|
||||||
|
.needs(Resource::Agent(a()));
|
||||||
|
// Declaration order is dependency order: the quiesce steps come first so
|
||||||
|
// the `Reconcile` that waits on them can name them.
|
||||||
|
if graceful && running {
|
||||||
|
// `SetWanted` is the brace here, so the quiesce pair borrows its grant
|
||||||
|
// rather than declaring the lease itself — same shape the rebuild
|
||||||
|
// template uses, one definition.
|
||||||
|
let drain = super::templates::quiesce(builder, agent, wanted);
|
||||||
|
let _ = builder
|
||||||
|
.node(NodeKind::Reconcile { agent: a() })
|
||||||
|
.needs(Resource::Agent(a()))
|
||||||
|
.part_of(wanted)
|
||||||
|
.after_ok(drain);
|
||||||
|
} else {
|
||||||
|
let _ = builder
|
||||||
|
.node(NodeKind::Reconcile { agent: a() })
|
||||||
|
.needs(Resource::Agent(a()))
|
||||||
|
.part_of(wanted);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One agent's **start** subgraph. `SetWanted(Up)` head; a down + stale-rev
|
||||||
|
/// agent gets the rebuild subgraph (its tail `Reconcile` starts it on
|
||||||
|
/// current derivations), otherwise a plain `Reconcile` (which starts a down
|
||||||
|
/// agent and noops an already-running one).
|
||||||
|
fn start_chain(builder: &JobBuilder, agent: &str, running: bool, stale: bool) {
|
||||||
|
let wanted = builder
|
||||||
|
.node(NodeKind::SetWanted {
|
||||||
|
agent: agent.to_owned(),
|
||||||
|
up: true,
|
||||||
|
})
|
||||||
|
.needs(Resource::Agent(agent.to_owned()));
|
||||||
|
if !running && stale {
|
||||||
|
// Rebuild subtree chained behind the `SetWanted` head. `MetaSync`,
|
||||||
|
// `Prebuild` + `Reconcile` are their own group roots (top-level, per
|
||||||
|
// `rebuild_nodes`).
|
||||||
|
rebuild_nodes(builder, agent, true, Some(wanted));
|
||||||
|
} else {
|
||||||
|
let _ = builder
|
||||||
|
.node(NodeKind::Reconcile {
|
||||||
|
agent: agent.to_owned(),
|
||||||
|
})
|
||||||
|
.needs(Resource::Agent(agent.to_owned()))
|
||||||
|
.part_of(wanted);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One agent's **restart** subgraph. Restart NEVER rewrites `wanted`
|
||||||
|
/// intent (no `SetWanted` head, unlike stop/start): it bounces the
|
||||||
|
/// container and lets the tail `Reconcile` converge to the agent's
|
||||||
|
/// EXISTING intent, so a deliberately-stopped (`wanted = Off`) agent is
|
||||||
|
/// not forced back up by a hive-wide restart. A running agent gets the
|
||||||
|
/// mechanical stop (`Signal → Drain` when graceful, then `StopForUpdate`)
|
||||||
|
/// before `Reconcile`; a down agent gets just `Reconcile`, which
|
||||||
|
/// converges to intent — a stopped (`wanted = Off`) agent stays stopped,
|
||||||
|
/// a crashed (`wanted = Up`) agent comes back up.
|
||||||
|
fn restart_chain(builder: &JobBuilder, agent: &str, graceful: bool, running: bool) {
|
||||||
|
let a = || agent.to_owned();
|
||||||
|
if !running {
|
||||||
|
// Nothing to bounce — a lone Reconcile converges to intent.
|
||||||
|
let _ = builder
|
||||||
|
.node(NodeKind::Reconcile { agent: a() })
|
||||||
|
.needs(Resource::Agent(a()));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Running: mechanical stop then Reconcile. The first stop node is the group
|
||||||
|
// ROOT (no SetWanted head) and owns the agent lease; the rest are its
|
||||||
|
// children (borrow the lease, dep-ordered), so the bounce holds one
|
||||||
|
// continuous lease and `Reconcile` cancel-cascades if a stop step fails.
|
||||||
|
//
|
||||||
|
// `Reconcile` gates on the last mechanical step. For a non-graceful bounce
|
||||||
|
// that step *is* the root, and the parent gate already orders it — a child
|
||||||
|
// must NOT dep on its own parent (dep-scope), so it takes no sibling edge.
|
||||||
|
//
|
||||||
|
// ⚠️ This is the one quiesce site that does NOT use `templates::quiesce`.
|
||||||
|
// The helper needs a brace holding the lease above the pair; here `Signal`
|
||||||
|
// *is* the holder, and the nesting under it is what keeps the grant
|
||||||
|
// continuous across the bounce. Giving this chain its own brace would
|
||||||
|
// unify all three sites — at the cost of one extra no-op node on every
|
||||||
|
// graceful restart, which an operator would see. Deliberately not done as
|
||||||
|
// a side effect of a rebuild-shape change.
|
||||||
|
if graceful {
|
||||||
|
let signal = builder
|
||||||
|
.node(NodeKind::Signal { agent: a() })
|
||||||
|
.needs(Resource::Agent(a()));
|
||||||
|
let drain = builder
|
||||||
|
.node(NodeKind::Drain { agent: a() })
|
||||||
|
.needs(Resource::Agent(a()))
|
||||||
|
.part_of(signal);
|
||||||
|
let stop = builder
|
||||||
|
.node(NodeKind::StopForUpdate { agent: a() })
|
||||||
|
.needs(Resource::Agent(a()))
|
||||||
|
.part_of(signal)
|
||||||
|
.after_ok(drain);
|
||||||
|
let _ = builder
|
||||||
|
.node(NodeKind::Reconcile { agent: a() })
|
||||||
|
.needs(Resource::Agent(a()))
|
||||||
|
.part_of(signal)
|
||||||
|
.after_ok(stop);
|
||||||
|
} else {
|
||||||
|
let stop = builder
|
||||||
|
.node(NodeKind::StopForUpdate { agent: a() })
|
||||||
|
.needs(Resource::Agent(a()));
|
||||||
|
let _ = builder
|
||||||
|
.node(NodeKind::Reconcile { agent: a() })
|
||||||
|
.needs(Resource::Agent(a()))
|
||||||
|
.part_of(stop);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The `*_nodes` declarers below are the PURE core the async `*_many` fns call
|
||||||
|
// after reading live state — they take the per-agent running (and stale)
|
||||||
|
// flags explicitly, so unit tests exercise the online/offline shapes without
|
||||||
|
// a live container. `*_many` = gather state + declare + submit.
|
||||||
|
//
|
||||||
|
// A power op has no tail node: its effect is its nodes (`SetWanted` +
|
||||||
|
// `Reconcile`), with nothing left to do once they settle.
|
||||||
|
//
|
||||||
|
// There is no concatenation step either: every chain declares into the same
|
||||||
|
// builder and each keeps its own root, so the per-agent subgraphs are
|
||||||
|
// independent and run concurrently, each on its own lease. Rebasing one
|
||||||
|
// subgraph's indices onto another's used to be a function.
|
||||||
|
|
||||||
|
/// Declare the stop DAG from explicit `(agent, running)` targets.
|
||||||
|
pub(crate) fn stop_nodes(builder: &JobBuilder, targets: &[(String, bool)], graceful: bool) {
|
||||||
|
for (agent, running) in targets {
|
||||||
|
stop_chain(builder, agent, graceful, *running);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Assemble the start DAG from explicit `(agent, running, stale)` targets.
|
||||||
|
///
|
||||||
|
/// No DAG-level pill: each agent's dashboard label is derived from the node
|
||||||
|
/// running under its lease, so a down+stale agent that grew a rebuild subgraph
|
||||||
|
/// reports `rebuilding` during its swap and `starting` at its reconcile,
|
||||||
|
/// without the DAG having to guess one label covering every target.
|
||||||
|
pub(crate) fn start_nodes(builder: &JobBuilder, targets: &[(String, bool, bool)]) {
|
||||||
|
for (agent, running, stale) in targets {
|
||||||
|
start_chain(builder, agent, *running, *stale);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Declare the restart DAG from explicit `(agent, running)` targets.
|
||||||
|
pub(crate) fn restart_nodes(builder: &JobBuilder, targets: &[(String, bool)], graceful: bool) {
|
||||||
|
for (agent, running) in targets {
|
||||||
|
restart_chain(builder, agent, graceful, *running);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- entry points ---------------------------------------------------------
|
||||||
|
//
|
||||||
|
// Each reads the live state its shape depends on, then declares + inserts.
|
||||||
|
|
||||||
|
/// Restart `agents` in a **single** DAG — one per-agent subgraph each, built
|
||||||
|
/// from live running state and run concurrently on their own leases. A running
|
||||||
|
/// agent gets the stop→reconcile chain (`graceful` prepends signal→drain); a
|
||||||
|
/// down agent gets a lone `Reconcile`. Restart never writes `wanted`, so the
|
||||||
|
/// tail `Reconcile` converges each agent to its EXISTING intent — a
|
||||||
|
/// deliberately-stopped agent stays down.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Propagates a graph-insert error.
|
||||||
|
pub async fn restart_many(
|
||||||
|
coord: &Arc<Coordinator>,
|
||||||
|
agents: &[String],
|
||||||
|
graceful: bool,
|
||||||
|
) -> anyhow::Result<Vec<NodeId>> {
|
||||||
|
let mut targets = Vec::with_capacity(agents.len());
|
||||||
|
for agent in agents {
|
||||||
|
targets.push((agent.clone(), lifecycle::is_running(agent).await));
|
||||||
|
}
|
||||||
|
let ids = coord.job_queue.insert(|b| {
|
||||||
|
restart_nodes(b, &targets, graceful);
|
||||||
|
Vec::new()
|
||||||
|
})?;
|
||||||
|
coord.emit_rebuild_queue_snapshot();
|
||||||
|
Ok(ids)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Start `agents` in a **single** DAG. A down agent gets
|
||||||
|
/// `SetWanted(Up) → Reconcile` (or, rev stale, a rebuild-then-start so it comes
|
||||||
|
/// up on current derivations); an already-running agent gets the same shape
|
||||||
|
/// with the reconcile noop'ing.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Propagates a graph-insert error.
|
||||||
|
pub async fn start_many(coord: &Arc<Coordinator>, agents: &[String]) -> anyhow::Result<Vec<NodeId>> {
|
||||||
|
let current = crate::auto_update::current_flake_rev(&coord.hyperhive_flake);
|
||||||
|
let mut targets = Vec::with_capacity(agents.len());
|
||||||
|
for agent in agents {
|
||||||
|
let running = lifecycle::is_running(agent).await;
|
||||||
|
let stored = std::fs::read_to_string(crate::paths::applied_rev_marker(agent)).ok();
|
||||||
|
let stale = current
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|rev| stored.as_deref() != Some(rev.as_str()));
|
||||||
|
if !running && stale {
|
||||||
|
tracing::info!(%agent, "start: rev stale + agent down — rebuild-then-start");
|
||||||
|
}
|
||||||
|
targets.push((agent.clone(), running, stale));
|
||||||
|
}
|
||||||
|
let ids = coord.job_queue.insert(|b| {
|
||||||
|
start_nodes(b, &targets);
|
||||||
|
Vec::new()
|
||||||
|
})?;
|
||||||
|
coord.emit_rebuild_queue_snapshot();
|
||||||
|
Ok(ids)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stop `agents` in a **single** DAG. A running agent gets
|
||||||
|
/// `SetWanted(Off) → [Signal → Drain →](graceful) Reconcile`; a down agent
|
||||||
|
/// skips the pointless quiesce but keeps the `Reconcile` as the race-up
|
||||||
|
/// backstop.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Propagates a graph-insert error.
|
||||||
|
pub async fn stop_many(
|
||||||
|
coord: &Arc<Coordinator>,
|
||||||
|
agents: &[String],
|
||||||
|
graceful: bool,
|
||||||
|
) -> anyhow::Result<Vec<NodeId>> {
|
||||||
|
let mut targets = Vec::with_capacity(agents.len());
|
||||||
|
for agent in agents {
|
||||||
|
targets.push((agent.clone(), lifecycle::is_running(agent).await));
|
||||||
|
}
|
||||||
|
let ids = coord.job_queue.insert(|b| {
|
||||||
|
stop_nodes(b, &targets, graceful);
|
||||||
|
Vec::new()
|
||||||
|
})?;
|
||||||
|
coord.emit_rebuild_queue_snapshot();
|
||||||
|
Ok(ids)
|
||||||
|
}
|
||||||
|
|
@ -36,17 +36,17 @@ fn rebuild(builder: &JobBuilder, agent: &str) {
|
||||||
/// Restart shape with every agent treated as **running** — the online
|
/// Restart shape with every agent treated as **running** — the online
|
||||||
/// shape (`[Signal→Drain→] StopForUpdate → Reconcile`, no `SetWanted` head)
|
/// shape (`[Signal→Drain→] StopForUpdate → Reconcile`, no `SetWanted` head)
|
||||||
/// most queue-mechanics tests assume. Mirrors the pre-dynamic
|
/// most queue-mechanics tests assume. Mirrors the pre-dynamic
|
||||||
/// `templates::restart` (which is now the state-aware `submit::restart_nodes`).
|
/// `templates::restart` (which is now the state-aware `power::restart_nodes`).
|
||||||
fn restart_online(builder: &JobBuilder, agents: &[&str], graceful: bool) {
|
fn restart_online(builder: &JobBuilder, agents: &[&str], graceful: bool) {
|
||||||
let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
|
let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
|
||||||
submit::restart_nodes(builder, &targets, graceful);
|
power::restart_nodes(builder, &targets, graceful);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Stop shape with every agent treated as **running** — the online shape
|
/// Stop shape with every agent treated as **running** — the online shape
|
||||||
/// (`SetWanted → [Signal→Drain→](graceful) Reconcile`).
|
/// (`SetWanted → [Signal→Drain→](graceful) Reconcile`).
|
||||||
fn stop_online(builder: &JobBuilder, agents: &[&str], graceful: bool) {
|
fn stop_online(builder: &JobBuilder, agents: &[&str], graceful: bool) {
|
||||||
let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
|
let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
|
||||||
submit::stop_nodes(builder, &targets, graceful);
|
power::stop_nodes(builder, &targets, graceful);
|
||||||
}
|
}
|
||||||
|
|
||||||
// `Claimed` / `ClaimReady` / `CompleteNode` lived here: a claim snapshot type
|
// `Claimed` / `ClaimReady` / `CompleteNode` lived here: a claim snapshot type
|
||||||
|
|
@ -682,7 +682,7 @@ fn multi_agent_start_one_dag_folds_per_agent_stale_rebuild() {
|
||||||
// fresh: offline + not stale → SetWanted → Reconcile.
|
// fresh: offline + not stale → SetWanted → Reconcile.
|
||||||
// stale: offline + stale → SetWanted → «rebuild subgraph».
|
// stale: offline + stale → SetWanted → «rebuild subgraph».
|
||||||
let id = submit(&q, "hive-wide start", |builder| {
|
let id = submit(&q, "hive-wide start", |builder| {
|
||||||
submit::start_nodes(
|
power::start_nodes(
|
||||||
builder,
|
builder,
|
||||||
&[
|
&[
|
||||||
("fresh".to_owned(), false, false),
|
("fresh".to_owned(), false, false),
|
||||||
|
|
@ -742,13 +742,13 @@ fn offline_agents_skip_mechanical_nodes_but_keep_reconcile() {
|
||||||
let q = JobQueue::new(4);
|
let q = JobQueue::new(4);
|
||||||
// Offline graceful stop → SetWanted(Off) → Reconcile (no Signal/Drain).
|
// Offline graceful stop → SetWanted(Off) → Reconcile (no Signal/Drain).
|
||||||
let stop = submit(&q, "stop down", |builder| {
|
let stop = submit(&q, "stop down", |builder| {
|
||||||
submit::stop_nodes(builder, &[("down".to_owned(), false)], true);
|
power::stop_nodes(builder, &[("down".to_owned(), false)], true);
|
||||||
});
|
});
|
||||||
// Offline restart → a lone Reconcile (no SetWanted, no StopForUpdate):
|
// Offline restart → a lone Reconcile (no SetWanted, no StopForUpdate):
|
||||||
// nothing to bounce, and restart never rewrites intent, so the tail
|
// nothing to bounce, and restart never rewrites intent, so the tail
|
||||||
// Reconcile converges the down agent to its existing `wanted`.
|
// Reconcile converges the down agent to its existing `wanted`.
|
||||||
let restart = submit(&q, "restart down", |builder| {
|
let restart = submit(&q, "restart down", |builder| {
|
||||||
submit::restart_nodes(builder, &[("down2".to_owned(), false)], true);
|
power::restart_nodes(builder, &[("down2".to_owned(), false)], true);
|
||||||
});
|
});
|
||||||
let shape = |id: u64| -> Vec<String> {
|
let shape = |id: u64| -> Vec<String> {
|
||||||
// The group's work nodes: its subtree minus the container itself,
|
// The group's work nodes: its subtree minus the container itself,
|
||||||
|
|
@ -1153,19 +1153,19 @@ fn cancelled_power_op_runs_no_compensating_node() {
|
||||||
|
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "bounce", |builder| {
|
let id = submit(&q, "bounce", |builder| {
|
||||||
submit::restart_nodes(builder, &targets, graceful);
|
power::restart_nodes(builder, &targets, graceful);
|
||||||
});
|
});
|
||||||
assert_cancels_clean(&q, id, false, &format!("restart {case}"));
|
assert_cancels_clean(&q, id, false, &format!("restart {case}"));
|
||||||
|
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "stop", |builder| {
|
let id = submit(&q, "stop", |builder| {
|
||||||
submit::stop_nodes(builder, &targets, graceful);
|
power::stop_nodes(builder, &targets, graceful);
|
||||||
});
|
});
|
||||||
assert_cancels_clean(&q, id, true, &format!("stop {case}"));
|
assert_cancels_clean(&q, id, true, &format!("stop {case}"));
|
||||||
|
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(&q, "start", |builder| {
|
let id = submit(&q, "start", |builder| {
|
||||||
submit::start_nodes(builder, &[("agent-a".to_owned(), running, false)]);
|
power::start_nodes(builder, &[("agent-a".to_owned(), running, false)]);
|
||||||
});
|
});
|
||||||
assert_cancels_clean(&q, id, true, &format!("start {case}"));
|
assert_cancels_clean(&q, id, true, &format!("start {case}"));
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -20,13 +20,9 @@ pub(super) async fn handle_start(coord: &Arc<Coordinator>, agent: &str, name: &s
|
||||||
// Persist `wanted = Up` and submit the Start DAG; the submit layer
|
// Persist `wanted = Up` and submit the Start DAG; the submit layer
|
||||||
// upgrades a stale-rev start to a full rebuild so the container
|
// upgrades a stale-rev start to a full rebuild so the container
|
||||||
// runs current nix derivations before it starts.
|
// runs current nix derivations before it starts.
|
||||||
crate::job_queue::submit::start(
|
if let Err(e) = crate::job_queue::power::start_many(coord, &[name.to_owned()]).await {
|
||||||
coord,
|
tracing::error!(%agent, %name, error = ?e, "start: insert failed");
|
||||||
name,
|
}
|
||||||
crate::job_queue::Source::Manual,
|
|
||||||
format!("agent `{agent}` start tool"),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
Response::Ok
|
Response::Ok
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -48,13 +44,9 @@ pub(super) async fn handle_restart(coord: &Arc<Coordinator>, agent: &str, name:
|
||||||
return err;
|
return err;
|
||||||
}
|
}
|
||||||
tracing::info!(%agent, %name, "submit restart");
|
tracing::info!(%agent, %name, "submit restart");
|
||||||
crate::job_queue::submit::restart(
|
if let Err(e) = crate::job_queue::power::restart_many(coord, &[name.to_owned()], false).await {
|
||||||
coord,
|
tracing::error!(%agent, %name, error = ?e, "restart: insert failed");
|
||||||
name,
|
}
|
||||||
crate::job_queue::Source::Manual,
|
|
||||||
format!("agent `{agent}` restart tool"),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
Response::Ok
|
Response::Ok
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -153,12 +145,13 @@ pub(super) fn handle_update(coord: &Arc<Coordinator>, agent: &str, name: &str) -
|
||||||
return err;
|
return err;
|
||||||
}
|
}
|
||||||
tracing::info!(%agent, %name, "submit rebuild");
|
tracing::info!(%agent, %name, "submit rebuild");
|
||||||
crate::job_queue::submit::rebuild(
|
if let Err(e) = coord.job_queue.insert(|b| {
|
||||||
coord,
|
crate::job_queue::templates::rebuild(b, name, true);
|
||||||
name,
|
Vec::new()
|
||||||
crate::job_queue::Source::Manual,
|
}) {
|
||||||
format!("agent `{agent}` update tool"),
|
tracing::error!(%agent, %name, error = ?e, "update: insert failed");
|
||||||
);
|
}
|
||||||
|
coord.emit_rebuild_queue_snapshot();
|
||||||
Response::Ok
|
Response::Ok
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -109,12 +109,11 @@ pub async fn ensure_root_agent(coord: &Arc<Coordinator>) -> Result<()> {
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
"manager container exists but no applied flake — forcing rebuild to migrate"
|
"manager container exists but no applied flake — forcing rebuild to migrate"
|
||||||
);
|
);
|
||||||
if let Err(e) = coord.job_queue.submit(
|
if let Err(e) = coord.job_queue.insert(|b| {
|
||||||
crate::job_queue::Source::AutoUpdate,
|
crate::job_queue::templates::rebuild(b, MANAGER_NAME, true);
|
||||||
"manager migration: no applied flake".to_owned(),
|
Vec::new()
|
||||||
|b| crate::job_queue::templates::rebuild(b, MANAGER_NAME, true),
|
}) {
|
||||||
) {
|
tracing::warn!(error = ?e, "manager migration rebuild insert failed");
|
||||||
tracing::warn!(error = ?e, "manager migration rebuild submit failed");
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
tracing::debug!("manager container already present");
|
tracing::debug!("manager container already present");
|
||||||
|
|
@ -378,18 +377,19 @@ fn submit_boot_tree(
|
||||||
n_deferred: usize,
|
n_deferred: usize,
|
||||||
n_skipped: usize,
|
n_skipped: usize,
|
||||||
) {
|
) {
|
||||||
use crate::job_queue::Source;
|
// Fully-quiet boot (nothing stale, nothing drifted) inserts nothing.
|
||||||
|
|
||||||
// Fully-quiet boot (nothing stale, nothing drifted) submits nothing.
|
|
||||||
if !any_stale && drifted.is_empty() {
|
if !any_stale && drifted.is_empty() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let reason = format!(
|
// The summary the sweep used to hand the container as its `reason` is a log
|
||||||
"boot: {} rebuild(s), {} reconcile(s), {} deferred (offline), {} up-to-date",
|
// line now: it was only ever stored on a node nobody read, and the counts
|
||||||
fanout.len(),
|
// are worth having where they can actually be seen.
|
||||||
drifted.len(),
|
tracing::info!(
|
||||||
n_deferred,
|
rebuilds = fanout.len(),
|
||||||
n_skipped,
|
reconciles = drifted.len(),
|
||||||
|
deferred = n_deferred,
|
||||||
|
up_to_date = n_skipped,
|
||||||
|
"boot: sweep"
|
||||||
);
|
);
|
||||||
|
|
||||||
// The sweep's own rebuild subgraphs emit their `Rebuilt` events as they
|
// The sweep's own rebuild subgraphs emit their `Rebuilt` events as they
|
||||||
|
|
@ -397,10 +397,11 @@ fn submit_boot_tree(
|
||||||
// The subgraphs also carry their own per-agent crash-watch suppression
|
// The subgraphs also carry their own per-agent crash-watch suppression
|
||||||
// during their `Swap` (applied at claim time); a reconcile-only boot needs
|
// during their `Swap` (applied at claim time); a reconcile-only boot needs
|
||||||
// no transient.
|
// no transient.
|
||||||
if let Err(e) = coord.job_queue.submit(Source::AutoUpdate, reason, |b| {
|
if let Err(e) = coord.job_queue.insert(|b| {
|
||||||
boot_nodes(b, any_stale, fanout, drifted);
|
boot_nodes(b, any_stale, fanout, drifted);
|
||||||
|
Vec::new()
|
||||||
}) {
|
}) {
|
||||||
tracing::warn!(error = ?e, "boot: sweep DAG submit failed");
|
tracing::warn!(error = ?e, "boot: sweep DAG insert failed");
|
||||||
}
|
}
|
||||||
coord.emit_rebuild_queue_snapshot();
|
coord.emit_rebuild_queue_snapshot();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue