refactor(#2449): write power intent via a SetWanted DAG node, not a pre-submit side effect
The durable 'wanted' power intent was written by submit::{start,stop,
restart,graceful_restart,graceful_stop} as a synchronous pre-submit side
effect, then read by the DAG's tail Reconcile. That's not crash-safe
(a crash between the write and the enqueue loses it) and, with agent now
per-node, can't be per-agent in a DAG that spans agents.
Move it into the DAG as a head SetWanted node:
- NodeKind::SetWanted { up } + run_set_wanted executor (fails the node on
a write error, unlike the old warn-and-continue, so a stale intent
never reaches Reconcile).
- LEASE-NEEDING, not lease-exempt: it takes the agent lease so a power-op
DAG's intent-write + reconcile is atomic per-agent. If it were exempt,
two racing ops (restart vs stop) would run both intent-writes up front
and clobber each other before either reconciled — defeating the point
of moving the write into the DAG. (In stale_start the lease is thus held
across the head Prebuild, but that's a no-op there: the agent is down so
prebuild is skipped.)
- templates: explicit SetWanted node 0 on restart/graceful_restart/
graceful_stop, plus dedicated start/stop templates (SetWanted -> Reconcile)
and stale_start (SetWanted(Up) -> rebuild subgraph, reusing rebuild_nodes).
No compose helper / rebuild variant. reconcile_only is now boot-only.
- submit.rs: drop the set_wanted side effect; the stale-rev shape decision
(start vs stale_start) stays submit-side.
All 33 job_queue tests pass (shape/lease tests updated for the head node).
This commit is contained in:
parent
6c654921a0
commit
5fe8008cce
5 changed files with 189 additions and 100 deletions
|
|
@ -98,9 +98,31 @@ pub(super) async fn run_node(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
|
||||||
// A pure grouping anchor (boot root): no work, completes immediately so
|
// A pure grouping anchor (boot root): no work, completes immediately so
|
||||||
// its child DAGs settle it and the boot tree resolves.
|
// its child DAGs settle it and the boot tree resolves.
|
||||||
NodeKind::Noop => Ok(NodeOutput::default()),
|
NodeKind::Noop => Ok(NodeOutput::default()),
|
||||||
|
NodeKind::SetWanted { up } => run_set_wanted(coord, claim, *up),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Write the agent's durable power intent — the DAG-node form of the old
|
||||||
|
/// pre-submit `set_wanted` side effect. Store-only (no container touch), so
|
||||||
|
/// build-slot-exempt; but it takes the agent's lifecycle lease (see
|
||||||
|
/// `NodeKind::needs_lease`) so the whole power-op DAG is atomic per-agent.
|
||||||
|
/// The downstream `Reconcile` reads the intent this writes. Unlike the old
|
||||||
|
/// warn-and-continue write, a failed write fails the node (cancel-downstream
|
||||||
|
/// cancels the `Reconcile`) rather than letting it converge to a stale
|
||||||
|
/// intent — that atomicity is the point of moving it into the DAG.
|
||||||
|
fn run_set_wanted(coord: &Arc<Coordinator>, claim: &Claim, up: bool) -> Result<NodeOutput> {
|
||||||
|
let wanted = if up {
|
||||||
|
crate::power::Wanted::Up
|
||||||
|
} else {
|
||||||
|
crate::power::Wanted::Offline
|
||||||
|
};
|
||||||
|
coord
|
||||||
|
.power
|
||||||
|
.set(&claim.agent, wanted)
|
||||||
|
.with_context(|| format!("set wanted={} for agent {}", wanted.as_str(), claim.agent))?;
|
||||||
|
Ok(NodeOutput::default())
|
||||||
|
}
|
||||||
|
|
||||||
/// Out-of-band toplevel build while the container keeps serving: meta
|
/// Out-of-band toplevel build while the container keeps serving: meta
|
||||||
/// sync + optional per-agent relock, then warm
|
/// sync + optional per-agent relock, then warm
|
||||||
/// `system.build.toplevel` so the later `Swap` hits cache and skips
|
/// `system.build.toplevel` so the later `Swap` hits cache and skips
|
||||||
|
|
|
||||||
|
|
@ -121,6 +121,20 @@ pub enum NodeKind {
|
||||||
/// dashboard. Holds no lease and no build slot; the child DAGs it anchors
|
/// dashboard. Holds no lease and no build slot; the child DAGs it anchors
|
||||||
/// still run concurrently — the grouping is a display link, not a dep edge.
|
/// still run concurrently — the grouping is a display link, not a dep edge.
|
||||||
Noop,
|
Noop,
|
||||||
|
/// Write the agent's durable power intent (`wanted = Up` when `up`, else
|
||||||
|
/// `Offline`) as a first-class DAG node, at the head of a power-op
|
||||||
|
/// template so the downstream `Reconcile` reads it. Replaces the old
|
||||||
|
/// pre-submit `set_wanted` side effect: the intent write is now part of
|
||||||
|
/// the atomic DAG (crash-safe, per-agent — a multi-agent DAG carries one
|
||||||
|
/// `SetWanted` per agent). Build-slot-exempt (a store write), but
|
||||||
|
/// **lease-needing**: it takes the agent's lifecycle lease so the whole
|
||||||
|
/// power-op DAG (intent write → reconcile) is atomic per-agent — two
|
||||||
|
/// racing ops (e.g. restart vs stop) can't clobber each other's intent
|
||||||
|
/// before either reconciles, which is the point of moving the write into
|
||||||
|
/// the DAG. (In `stale_start` the lease is thus held across the head
|
||||||
|
/// `Prebuild`, but that's a no-op there — the agent is down, so prebuild
|
||||||
|
/// is skipped.)
|
||||||
|
SetWanted { up: bool },
|
||||||
}
|
}
|
||||||
|
|
||||||
impl NodeKind {
|
impl NodeKind {
|
||||||
|
|
@ -142,6 +156,7 @@ impl NodeKind {
|
||||||
NodeKind::WritePermFile => "write_perm_file",
|
NodeKind::WritePermFile => "write_perm_file",
|
||||||
NodeKind::ApprovalDeploy => "approval_deploy",
|
NodeKind::ApprovalDeploy => "approval_deploy",
|
||||||
NodeKind::Noop => "noop",
|
NodeKind::Noop => "noop",
|
||||||
|
NodeKind::SetWanted { .. } => "set_wanted",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -177,6 +192,7 @@ impl NodeKind {
|
||||||
| NodeKind::Drain
|
| NodeKind::Drain
|
||||||
| NodeKind::WriteDropin
|
| NodeKind::WriteDropin
|
||||||
| NodeKind::ApprovalDeploy
|
| NodeKind::ApprovalDeploy
|
||||||
|
| NodeKind::SetWanted { .. }
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,16 +1,15 @@
|
||||||
//! Request-level submit API — the surface the dashboard POST handlers,
|
//! Request-level submit API — the surface the dashboard POST handlers,
|
||||||
//! the MCP socket handlers, and `hivectl` paths call. Owns the
|
//! the MCP socket handlers, and `hivectl` paths call. The durable
|
||||||
//! submit-time side effects the DAG templates deliberately don't:
|
//! `wanted` power intent is now written by a `SetWanted` DAG node at the
|
||||||
//! writing the durable `wanted` power intent (synchronously,
|
//! head of each power-op template (not a pre-submit side effect); the
|
||||||
//! last-writer-wins) before the DAG whose `Reconcile` reads it, and
|
//! only submit-time logic left is the stale-start *shape* decision
|
||||||
//! upgrading a stale start to a full rebuild. Every helper emits a
|
//! (`start` vs `stale_start`). Every helper emits a fresh queue snapshot
|
||||||
//! fresh queue snapshot so the dashboard shows the new DAG immediately.
|
//! so the dashboard shows the new DAG immediately.
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use super::{Source, Template, templates};
|
use super::{Source, templates};
|
||||||
use crate::coordinator::Coordinator;
|
use crate::coordinator::Coordinator;
|
||||||
use crate::power::Wanted;
|
|
||||||
|
|
||||||
fn submit_and_emit(coord: &Arc<Coordinator>, spec: super::DagSpec) -> u64 {
|
fn submit_and_emit(coord: &Arc<Coordinator>, spec: super::DagSpec) -> u64 {
|
||||||
let id = coord
|
let id = coord
|
||||||
|
|
@ -21,12 +20,6 @@ fn submit_and_emit(coord: &Arc<Coordinator>, spec: super::DagSpec) -> u64 {
|
||||||
id
|
id
|
||||||
}
|
}
|
||||||
|
|
||||||
fn set_wanted(coord: &Arc<Coordinator>, agent: &str, wanted: Wanted) {
|
|
||||||
if let Err(e) = coord.power.set(agent, wanted) {
|
|
||||||
tracing::warn!(%agent, wanted = wanted.as_str(), error = ?e, "agent_power: set failed");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Manual/approval-independent rebuild (always relocks the agent's
|
/// Manual/approval-independent rebuild (always relocks the agent's
|
||||||
/// meta input — cascade children are built by the scheduler's fan-out
|
/// meta input — cascade children are built by the scheduler's fan-out
|
||||||
/// instead of this surface).
|
/// instead of this surface).
|
||||||
|
|
@ -35,20 +28,20 @@ pub fn rebuild(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: St
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Restart: mechanical stop + converge to `wanted = Up`. The intent
|
/// Restart: mechanical stop + converge to `wanted = Up`. The intent
|
||||||
/// write matters when `wanted` drifted `Offline` under a running
|
/// write is the template's head `SetWanted(Up)` node — it matters when
|
||||||
/// agent — the old `kill + start` always ended up, and an operator
|
/// `wanted` drifted `Offline` under a running agent (an operator asking
|
||||||
/// asking for a restart plainly wants it running, not a stop.
|
/// for a restart plainly wants it running, not a stop).
|
||||||
pub fn restart(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
pub fn restart(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||||
set_wanted(coord, agent, Wanted::Up);
|
|
||||||
submit_and_emit(coord, templates::restart(agent, source, reason))
|
submit_and_emit(coord, templates::restart(agent, source, reason))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Start: persist `wanted = Up`, then reconcile. A stale rev marker
|
/// Start: `SetWanted(Up)` (a DAG node now) then reconcile. A stale rev
|
||||||
/// upgrades the start to a full rebuild (whose tail `Reconcile` does
|
/// marker upgrades the start to a rebuild-then-start (`stale_start`, whose
|
||||||
/// the start) so the container always comes up on current derivations
|
/// tail `Reconcile` does the start) so the container always comes up on
|
||||||
/// — the old fast-lane `run_start` upgrade, moved to submit time.
|
/// current derivations — the old fast-lane `run_start` upgrade, still a
|
||||||
|
/// submit-time *shape* decision (which template), while the intent write
|
||||||
|
/// itself is now the template's head node.
|
||||||
pub fn start(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
pub fn start(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||||
set_wanted(coord, agent, Wanted::Up);
|
|
||||||
let stored = std::fs::read_to_string(crate::paths::applied_rev_marker(agent)).ok();
|
let stored = std::fs::read_to_string(crate::paths::applied_rev_marker(agent)).ok();
|
||||||
let stale = crate::auto_update::current_flake_rev(&coord.hyperhive_flake)
|
let stale = crate::auto_update::current_flake_rev(&coord.hyperhive_flake)
|
||||||
.is_some_and(|rev| stored.as_deref() != Some(rev.as_str()));
|
.is_some_and(|rev| stored.as_deref() != Some(rev.as_str()));
|
||||||
|
|
@ -56,60 +49,34 @@ pub fn start(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: Stri
|
||||||
tracing::info!(%agent, "start: rev stale — upgrading to rebuild+start");
|
tracing::info!(%agent, "start: rev stale — upgrading to rebuild+start");
|
||||||
return submit_and_emit(
|
return submit_and_emit(
|
||||||
coord,
|
coord,
|
||||||
templates::rebuild(
|
templates::stale_start(agent, source, format!("{reason} (stale — rebuild+start)")),
|
||||||
agent,
|
|
||||||
source,
|
|
||||||
format!("{reason} (stale — rebuild+start)"),
|
|
||||||
None,
|
|
||||||
true,
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
submit_and_emit(
|
submit_and_emit(coord, templates::start(agent, source, reason))
|
||||||
coord,
|
|
||||||
templates::reconcile_only(
|
|
||||||
Template::Start,
|
|
||||||
agent,
|
|
||||||
source,
|
|
||||||
reason,
|
|
||||||
Some(crate::coordinator::TransientKind::Starting),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Hard stop: persist `wanted = Offline`, then reconcile (kill +
|
/// Hard stop: `SetWanted(Offline)` (a DAG node now) then reconcile (kill +
|
||||||
/// unregister + `Killed` event).
|
/// unregister + `Killed` event).
|
||||||
pub fn stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
pub fn stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||||
set_wanted(coord, agent, Wanted::Offline);
|
submit_and_emit(coord, templates::stop(agent, source, reason))
|
||||||
submit_and_emit(
|
|
||||||
coord,
|
|
||||||
templates::reconcile_only(
|
|
||||||
Template::Stop,
|
|
||||||
agent,
|
|
||||||
source,
|
|
||||||
reason,
|
|
||||||
Some(crate::coordinator::TransientKind::Stopping),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Graceful stop: persist `wanted = Offline`, then signal → drain →
|
/// Graceful stop: signal → drain → reconcile (the actual stop). The head
|
||||||
/// reconcile (the actual stop).
|
/// `SetWanted(Offline)` node writes the intent as part of the DAG.
|
||||||
pub fn graceful_stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
pub fn graceful_stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||||
set_wanted(coord, agent, Wanted::Offline);
|
|
||||||
submit_and_emit(coord, templates::graceful_stop(agent, source, reason))
|
submit_and_emit(coord, templates::graceful_stop(agent, source, reason))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Graceful restart: persist `wanted = Up`, then signal → drain →
|
/// Graceful restart: signal → drain → mechanical stop → reconcile (starts
|
||||||
/// mechanical stop → reconcile (starts it back up) — one atomic DAG,
|
/// it back up) — one atomic DAG, no client-side "await the stop DAG then
|
||||||
/// no client-side "await the stop DAG then submit a start DAG" split.
|
/// submit a start DAG" split. The head `SetWanted(Up)` node writes the
|
||||||
|
/// intent as part of the DAG.
|
||||||
pub fn graceful_restart(
|
pub fn graceful_restart(
|
||||||
coord: &Arc<Coordinator>,
|
coord: &Arc<Coordinator>,
|
||||||
agent: &str,
|
agent: &str,
|
||||||
source: Source,
|
source: Source,
|
||||||
reason: String,
|
reason: String,
|
||||||
) -> u64 {
|
) -> u64 {
|
||||||
set_wanted(coord, agent, Wanted::Up);
|
|
||||||
submit_and_emit(coord, templates::graceful_restart(agent, source, reason))
|
submit_and_emit(coord, templates::graceful_restart(agent, source, reason))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -8,14 +8,19 @@
|
||||||
//! are single-agent (every node shares one agent); a future multi-agent
|
//! are single-agent (every node shares one agent); a future multi-agent
|
||||||
//! template would stamp different agents per subgraph.
|
//! template would stamp different agents per subgraph.
|
||||||
//!
|
//!
|
||||||
|
//! The power ops write the durable `wanted` intent via a head
|
||||||
|
//! `SetWanted(w)` node (not a pre-submit side effect); it holds the agent
|
||||||
|
//! lease so intent+reconcile is atomic per-agent.
|
||||||
|
//!
|
||||||
//! ```text
|
//! ```text
|
||||||
//! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a)
|
//! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a)
|
||||||
//! graceful-stop(a): [wanted=Offline] Signal(a) → Drain(a) → Reconcile(a)
|
//! graceful-stop(a): SetWanted(a,Off) → Signal(a) → Drain(a) → Reconcile(a)
|
||||||
//! restart(a): [wanted=Up] StopForUpdate(a) → Reconcile(a)
|
//! restart(a): SetWanted(a,Up) → StopForUpdate(a) → Reconcile(a)
|
||||||
//! graceful-restart(a): [wanted=Up] Signal(a) → Drain(a) → StopForUpdate(a) → Reconcile(a)
|
//! graceful-restart(a): SetWanted(a,Up) → Signal(a) → Drain(a) → StopForUpdate(a) → Reconcile(a)
|
||||||
//! start(a): [wanted=Up] Reconcile(a)
|
//! start(a): SetWanted(a,Up) → Reconcile(a)
|
||||||
//! stop(a): [wanted=Offline] Reconcile(a)
|
//! stop(a): SetWanted(a,Off) → Reconcile(a)
|
||||||
//! spawn(a): [wanted=Up] Provision(a) → Create(a) → WriteDropin(a) → Reconcile(a)
|
//! stale-start(a): SetWanted(a,Up) → «rebuild subgraph» (rev stale; prebuild noops, agent down)
|
||||||
|
//! spawn(a): Provision(a) → Create(a) → WriteDropin(a) → Reconcile(a) [wanted=Up at approve]
|
||||||
//! perm-change(a): WritePermFile(a) → «rebuild subgraph»
|
//! perm-change(a): WritePermFile(a) → «rebuild subgraph»
|
||||||
//! meta-update(inp): MetaLock(inp) → «fan-out rebuild(a) per affected a»
|
//! meta-update(inp): MetaLock(inp) → «fan-out rebuild(a) per affected a»
|
||||||
//! startup sweep: MetaLock(hyperhive, non-fatal) → «fan-out rebuild(stale a)»
|
//! startup sweep: MetaLock(hyperhive, non-fatal) → «fan-out rebuild(stale a)»
|
||||||
|
|
@ -117,10 +122,11 @@ pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec
|
||||||
|
|
||||||
/// Graceful stop: cheap `Signal` fires immediately (no build slot), the
|
/// Graceful stop: cheap `Signal` fires immediately (no build slot), the
|
||||||
/// `Drain` awaits the harness checkpoint (bounded), and the tail
|
/// `Drain` awaits the harness checkpoint (bounded), and the tail
|
||||||
/// `Reconcile` performs the actual container stop — the caller sets
|
/// `Reconcile` performs the actual container stop. The head `SetWanted`
|
||||||
/// `wanted = Offline` at submit time. A whole-hive graceful stop
|
/// node writes `wanted = Offline` as part of the DAG (was a pre-submit
|
||||||
/// therefore signals every agent up front and overlaps every drain,
|
/// side effect). A whole-hive graceful stop therefore signals every agent
|
||||||
/// replacing the old detached-watcher thread structurally.
|
/// up front and overlaps every drain, replacing the old detached-watcher
|
||||||
|
/// thread structurally.
|
||||||
pub fn graceful_stop(agent: &str, source: Source, reason: String) -> DagSpec {
|
pub fn graceful_stop(agent: &str, source: Source, reason: String) -> DagSpec {
|
||||||
DagSpec {
|
DagSpec {
|
||||||
template: Template::GracefulStop,
|
template: Template::GracefulStop,
|
||||||
|
|
@ -132,16 +138,17 @@ pub fn graceful_stop(agent: &str, source: Source, reason: String) -> DagSpec {
|
||||||
perm_payload: None,
|
perm_payload: None,
|
||||||
transient: Some(TransientKind::Stopping),
|
transient: Some(TransientKind::Stopping),
|
||||||
nodes: vec![
|
nodes: vec![
|
||||||
node(agent, NodeKind::Signal, Vec::new()),
|
node(agent, NodeKind::SetWanted { up: false }, Vec::new()),
|
||||||
node(agent, NodeKind::Drain, after_ok(0)),
|
node(agent, NodeKind::Signal, after_ok(0)),
|
||||||
node(agent, NodeKind::Reconcile, after_ok(1)),
|
node(agent, NodeKind::Drain, after_ok(1)),
|
||||||
|
node(agent, NodeKind::Reconcile, after_ok(2)),
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Restart: mechanical stop, then converge to `wanted` — the submit
|
/// Restart: write `wanted = Up` (head `SetWanted` node), mechanical stop,
|
||||||
/// layer writes `wanted = Up` first, so this is a stop + start like
|
/// then converge — a stop + start like the old `lifecycle::restart`
|
||||||
/// the old `lifecycle::restart` regardless of prior intent drift.
|
/// regardless of prior intent drift.
|
||||||
pub fn restart(agent: &str, source: Source, reason: String) -> DagSpec {
|
pub fn restart(agent: &str, source: Source, reason: String) -> DagSpec {
|
||||||
DagSpec {
|
DagSpec {
|
||||||
template: Template::Restart,
|
template: Template::Restart,
|
||||||
|
|
@ -153,20 +160,20 @@ pub fn restart(agent: &str, source: Source, reason: String) -> DagSpec {
|
||||||
perm_payload: None,
|
perm_payload: None,
|
||||||
transient: Some(TransientKind::Restarting),
|
transient: Some(TransientKind::Restarting),
|
||||||
nodes: vec![
|
nodes: vec![
|
||||||
node(agent, NodeKind::StopForUpdate, Vec::new()),
|
node(agent, NodeKind::SetWanted { up: true }, Vec::new()),
|
||||||
node(agent, NodeKind::Reconcile, after_ok(0)),
|
node(agent, NodeKind::StopForUpdate, after_ok(0)),
|
||||||
|
node(agent, NodeKind::Reconcile, after_ok(1)),
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Graceful restart: signal → drain → mechanical stop → converge to
|
/// Graceful restart: write `wanted = Up` (head `SetWanted`), signal →
|
||||||
/// `wanted` — the caller writes `wanted = Up` first, same as `restart`.
|
/// drain → mechanical stop → converge. One atomic DAG start to finish
|
||||||
/// One atomic DAG start to finish (no client- or server-side "submit
|
/// (no client- or server-side "submit one DAG, await it, submit the next"
|
||||||
/// one DAG, await it, submit the next" composition): the `Drain` node
|
/// composition): the `Drain` node is the same bounded harness-checkpoint
|
||||||
/// is the same bounded harness-checkpoint wait `graceful_stop` uses,
|
/// wait `graceful_stop` uses, then `StopForUpdate` (mechanical, ignores
|
||||||
/// then `StopForUpdate` (mechanical, ignores `wanted`) and the tail
|
/// `wanted`) and the tail `Reconcile` (converges to `wanted = Up`, i.e.
|
||||||
/// `Reconcile` (converges to `wanted = Up`, i.e. starts it back up)
|
/// starts it back up) chain exactly like `restart`'s tail.
|
||||||
/// chain exactly like `restart`'s tail.
|
|
||||||
pub fn graceful_restart(agent: &str, source: Source, reason: String) -> DagSpec {
|
pub fn graceful_restart(agent: &str, source: Source, reason: String) -> DagSpec {
|
||||||
DagSpec {
|
DagSpec {
|
||||||
template: Template::GracefulRestart,
|
template: Template::GracefulRestart,
|
||||||
|
|
@ -178,16 +185,19 @@ pub fn graceful_restart(agent: &str, source: Source, reason: String) -> DagSpec
|
||||||
perm_payload: None,
|
perm_payload: None,
|
||||||
transient: Some(TransientKind::Restarting),
|
transient: Some(TransientKind::Restarting),
|
||||||
nodes: vec![
|
nodes: vec![
|
||||||
node(agent, NodeKind::Signal, Vec::new()),
|
node(agent, NodeKind::SetWanted { up: true }, Vec::new()),
|
||||||
node(agent, NodeKind::Drain, after_ok(0)),
|
node(agent, NodeKind::Signal, after_ok(0)),
|
||||||
node(agent, NodeKind::StopForUpdate, after_ok(1)),
|
node(agent, NodeKind::Drain, after_ok(1)),
|
||||||
node(agent, NodeKind::Reconcile, after_ok(2)),
|
node(agent, NodeKind::StopForUpdate, after_ok(2)),
|
||||||
|
node(agent, NodeKind::Reconcile, after_ok(3)),
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Single-`Reconcile` DAG: `Start` / `Stop` (caller writes `wanted`
|
/// Boot-time reconcile: a single `Reconcile` node that converges observed
|
||||||
/// first) and the boot-time `Reconcile` converge (wanted untouched).
|
/// power state to the persisted intent — `wanted` is untouched (no
|
||||||
|
/// `SetWanted`), unlike the operator `start`/`stop` templates. Used only
|
||||||
|
/// by the boot sweep now.
|
||||||
pub fn reconcile_only(
|
pub fn reconcile_only(
|
||||||
template: Template,
|
template: Template,
|
||||||
agent: &str,
|
agent: &str,
|
||||||
|
|
@ -208,6 +218,68 @@ pub fn reconcile_only(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Start: write `wanted = Up` (head `SetWanted`), then reconcile (which
|
||||||
|
/// starts the container). The intent write is a DAG node now, not a
|
||||||
|
/// pre-submit side effect.
|
||||||
|
pub fn start(agent: &str, source: Source, reason: String) -> DagSpec {
|
||||||
|
DagSpec {
|
||||||
|
template: Template::Start,
|
||||||
|
source,
|
||||||
|
reason,
|
||||||
|
parent_id: None,
|
||||||
|
approval_id: None,
|
||||||
|
inputs: Vec::new(),
|
||||||
|
perm_payload: None,
|
||||||
|
transient: Some(TransientKind::Starting),
|
||||||
|
nodes: vec![
|
||||||
|
node(agent, NodeKind::SetWanted { up: true }, Vec::new()),
|
||||||
|
node(agent, NodeKind::Reconcile, after_ok(0)),
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stop: write `wanted = Offline` (head `SetWanted`), then reconcile
|
||||||
|
/// (kill + unregister + `Killed` event).
|
||||||
|
pub fn stop(agent: &str, source: Source, reason: String) -> DagSpec {
|
||||||
|
DagSpec {
|
||||||
|
template: Template::Stop,
|
||||||
|
source,
|
||||||
|
reason,
|
||||||
|
parent_id: None,
|
||||||
|
approval_id: None,
|
||||||
|
inputs: Vec::new(),
|
||||||
|
perm_payload: None,
|
||||||
|
transient: Some(TransientKind::Stopping),
|
||||||
|
nodes: vec![
|
||||||
|
node(agent, NodeKind::SetWanted { up: false }, Vec::new()),
|
||||||
|
node(agent, NodeKind::Reconcile, after_ok(0)),
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stale start: a `start` whose rev marker is stale, so it rebuilds
|
||||||
|
/// before coming up — `SetWanted(Up)` → «rebuild subgraph» → `Reconcile`
|
||||||
|
/// (the rebuild's tail `Reconcile` starts it, since `wanted = Up`). The
|
||||||
|
/// rebuild nodes are the same `rebuild_nodes` chain a manual rebuild uses
|
||||||
|
/// (reused, not a variant); only the leading `SetWanted(Up)` intent
|
||||||
|
/// differs. Shows as a `Rebuild` on the dashboard like the old
|
||||||
|
/// submit-time upgrade did.
|
||||||
|
pub fn stale_start(agent: &str, source: Source, reason: String) -> DagSpec {
|
||||||
|
let mut nodes = vec![node(agent, NodeKind::SetWanted { up: true }, Vec::new())];
|
||||||
|
nodes.extend(rebuild_nodes(agent, true, 1));
|
||||||
|
DagSpec {
|
||||||
|
template: Template::Rebuild,
|
||||||
|
source,
|
||||||
|
reason,
|
||||||
|
parent_id: None,
|
||||||
|
approval_id: None,
|
||||||
|
inputs: Vec::new(),
|
||||||
|
perm_payload: None,
|
||||||
|
transient: Some(TransientKind::Rebuilding),
|
||||||
|
nodes,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// First-deploy spawn (approval-driven): `Provision` (proposed/applied
|
/// First-deploy spawn (approval-driven): `Provision` (proposed/applied
|
||||||
/// repos, state subvolume, meta registration) then `Create`
|
/// repos, state subvolume, meta registration) then `Create`
|
||||||
/// (`nixos-container create`), drop-in write, then `Reconcile` starts
|
/// (`nixos-container create`), drop-in write, then `Reconcile` starts
|
||||||
|
|
|
||||||
|
|
@ -212,21 +212,27 @@ fn lease_serializes_two_lifecycle_dags_for_same_agent() {
|
||||||
None,
|
None,
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
// Restart's StopForUpdate acquires the lease; stop's Reconcile
|
// Restart's head SetWanted takes the lease; stop's Reconcile must
|
||||||
// must wait even though slots are free.
|
// wait even though slots are free.
|
||||||
let first = claim_one(&q);
|
let first = claim_one(&q);
|
||||||
assert_eq!(first.dag_id, restart);
|
assert_eq!(first.dag_id, restart);
|
||||||
|
assert_eq!(first.kind.as_str(), "set_wanted");
|
||||||
assert!(first.lease_acquired);
|
assert!(first.lease_acquired);
|
||||||
q.complete_node(restart, first.node_id, Ok(()));
|
q.complete_node(restart, first.node_id, Ok(()));
|
||||||
// Same DAG keeps the lease for its Reconcile.
|
// Same DAG keeps the lease through StopForUpdate then Reconcile.
|
||||||
let second = claim_one(&q);
|
let second = claim_one(&q);
|
||||||
assert_eq!(second.dag_id, restart);
|
assert_eq!(second.dag_id, restart);
|
||||||
|
assert_eq!(second.kind.as_str(), "stop_for_update");
|
||||||
assert!(!second.lease_acquired, "lease already held by this DAG");
|
assert!(!second.lease_acquired, "lease already held by this DAG");
|
||||||
q.complete_node(restart, second.node_id, Ok(()));
|
q.complete_node(restart, second.node_id, Ok(()));
|
||||||
// Restart terminal → lease released → stop's Reconcile runs.
|
|
||||||
let third = claim_one(&q);
|
let third = claim_one(&q);
|
||||||
assert_eq!(third.dag_id, stop);
|
assert_eq!(third.dag_id, restart);
|
||||||
q.complete_node(stop, third.node_id, Ok(()));
|
assert_eq!(third.kind.as_str(), "reconcile");
|
||||||
|
q.complete_node(restart, third.node_id, Ok(()));
|
||||||
|
// Restart terminal → lease released → stop's Reconcile runs.
|
||||||
|
let fourth = claim_one(&q);
|
||||||
|
assert_eq!(fourth.dag_id, stop);
|
||||||
|
q.complete_node(stop, fourth.node_id, Ok(()));
|
||||||
assert_eq!(state_of(&q, restart), State::Done);
|
assert_eq!(state_of(&q, restart), State::Done);
|
||||||
assert_eq!(state_of(&q, stop), State::Done);
|
assert_eq!(state_of(&q, stop), State::Done);
|
||||||
}
|
}
|
||||||
|
|
@ -510,6 +516,11 @@ fn terminal_dag_reported_exactly_once_and_lease_released() {
|
||||||
&q,
|
&q,
|
||||||
templates::restart("agent-a", Source::Manual, "r".to_owned()),
|
templates::restart("agent-a", Source::Manual, "r".to_owned()),
|
||||||
);
|
);
|
||||||
|
// restart = SetWanted → StopForUpdate → Reconcile; not terminal until
|
||||||
|
// the last node completes.
|
||||||
|
let set_wanted = claim_one(&q);
|
||||||
|
q.complete_node(id, set_wanted.node_id, Ok(()));
|
||||||
|
assert!(q.drain_terminal().is_empty(), "dag not terminal yet");
|
||||||
let stop = claim_one(&q);
|
let stop = claim_one(&q);
|
||||||
q.complete_node(id, stop.node_id, Ok(()));
|
q.complete_node(id, stop.node_id, Ok(()));
|
||||||
assert!(q.drain_terminal().is_empty(), "dag not terminal yet");
|
assert!(q.drain_terminal().is_empty(), "dag not terminal yet");
|
||||||
|
|
@ -727,7 +738,7 @@ fn graceful_stop_shape_signal_drain_reconcile() {
|
||||||
&q,
|
&q,
|
||||||
templates::graceful_stop("agent-a", Source::Manual, "graceful".to_owned()),
|
templates::graceful_stop("agent-a", Source::Manual, "graceful".to_owned()),
|
||||||
);
|
);
|
||||||
for expected in ["signal", "drain", "reconcile"] {
|
for expected in ["set_wanted", "signal", "drain", "reconcile"] {
|
||||||
let c = claim_one(&q);
|
let c = claim_one(&q);
|
||||||
assert_eq!(c.kind.as_str(), expected);
|
assert_eq!(c.kind.as_str(), expected);
|
||||||
q.complete_node(id, c.node_id, Ok(()));
|
q.complete_node(id, c.node_id, Ok(()));
|
||||||
|
|
@ -753,8 +764,9 @@ fn graceful_signal_and_drain_hold_no_build_slot() {
|
||||||
let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect();
|
let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
kinds,
|
kinds,
|
||||||
vec!["prebuild", "signal", "signal"],
|
vec!["prebuild", "set_wanted", "set_wanted"],
|
||||||
"both agents' signals fire while the slot is held"
|
"both agents' graceful-stop heads (SetWanted, build-slot-exempt) run \
|
||||||
|
while the slot is held; their signals follow"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue