Compare commits

..
7 changed files with 111 additions and 211 deletions

View file

@ -74,19 +74,14 @@ container build:
### Every operation as a DAG ### Every operation as a DAG
The power ops write the durable `wanted` intent via a head `SetWanted`
node (not a pre-submit side effect) — it holds the agent lease, so
intent-write + reconcile is atomic per-agent.
```text ```text
rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(after-any) Reconcile(a) rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(after-any) Reconcile(a)
graceful-stop(a): SetWanted(a,Off) → Signal(a) → Drain(a) → Reconcile(a) graceful-stop(a): [wanted=Offline] Signal(a) → Drain(a) → Reconcile(a)
restart(a): SetWanted(a,Up) → StopForUpdate(a) → Reconcile(a) restart(a): [wanted=Up] StopForUpdate(a) → Reconcile(a)
graceful-restart(a): SetWanted(a,Up) → Signal(a) → Drain(a) → StopForUpdate(a) → Reconcile(a) graceful-restart(a): [wanted=Up] Signal(a) → Drain(a) → StopForUpdate(a) → Reconcile(a)
start(a): SetWanted(a,Up) → Reconcile(a) (stale rev ⇒ stale-start below) start(a): [wanted=Up] Reconcile(a) (stale rev ⇒ upgraded to rebuild)
stale-start(a): SetWanted(a,Up) → «rebuild subgraph» (prebuild noops — agent is down) stop(a): [wanted=Offline] Reconcile(a)
stop(a): SetWanted(a,Off) → Reconcile(a) spawn(a): [wanted=Up] Create(a) → WriteDropin(a) → Reconcile(a)
spawn(a): [wanted=Up at approve] Create(a) → WriteDropin(a) → Reconcile(a)
perm-change(a): WritePermFile(a) → «rebuild subgraph» perm-change(a): WritePermFile(a) → «rebuild subgraph»
meta-update(inp): MetaLock(inp) → «fan-out rebuild(a) per affected agent» meta-update(inp): MetaLock(inp) → «fan-out rebuild(a) per affected agent»
boot: (if any rev marker stale) MetaLock(hyperhive) → «fan-out rebuild»; boot: (if any rev marker stale) MetaLock(hyperhive) → «fan-out rebuild»;
@ -146,15 +141,11 @@ resources are free. Resources:
held by nix-heavy nodes for the node's duration. held by nix-heavy nodes for the node's duration.
2. **Per-agent lifecycle lease** — keyed on the **node's** agent (agent is 2. **Per-agent lifecycle lease** — keyed on the **node's** agent (agent is
per-node; a DAG can span agents) and globally exclusive per agent across per-node; a DAG can span agents) and globally exclusive per agent across
all DAGs: acquired at a container-affecting node (`SetWanted`, all DAGs: acquired at a container-affecting node (`StopForUpdate`, `Swap`,
`StopForUpdate`, `Swap`, `Signal`, `Drain`, `Reconcile`, `WriteDropin`, `Signal`, `Drain`, `Reconcile`, `WriteDropin`, `Create`, `ApprovalDeploy`),
`Create`, `ApprovalDeploy`), held by the owning DAG until it's terminal, held by the owning DAG until it's terminal, so two DAGs never interleave
so two DAGs never interleave container ops on the same agent. A DAG container ops on the same agent. A DAG touching several agents holds one
touching several agents holds one lease per agent. (`SetWanted` is a store lease per agent. **Lease-exempt**: `Prebuild`, `MetaLock`, `WritePermFile`
write, not a container op, but takes the lease anyway so a power-op DAG's
intent write + reconcile is atomic — two racing ops can't clobber intent
before either reconciles.) **Lease-exempt**: `Prebuild`, `MetaLock`,
`WritePermFile`
they touch the store / meta, not the running container, which is exactly they touch the store / meta, not the running container, which is exactly
why a stop can land while another DAG's prebuild is still building. why a stop can land while another DAG's prebuild is still building.

View file

@ -154,8 +154,6 @@ const NODE_KIND_LABEL = {
write_dropin: 'dropin', write_dropin: 'dropin',
write_perm_file: 'perm file', write_perm_file: 'perm file',
approval_deploy: 'deploy', approval_deploy: 'deploy',
set_wanted: 'set wanted',
noop: 'noop',
}; };
// The currently-running node of a DAG (per-node `step` labels + build // The currently-running node of a DAG (per-node `step` labels + build

View file

@ -98,31 +98,9 @@ pub(super) async fn run_node(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
// A pure grouping anchor (boot root): no work, completes immediately so // A pure grouping anchor (boot root): no work, completes immediately so
// its child DAGs settle it and the boot tree resolves. // its child DAGs settle it and the boot tree resolves.
NodeKind::Noop => Ok(NodeOutput::default()), NodeKind::Noop => Ok(NodeOutput::default()),
NodeKind::SetWanted { up } => run_set_wanted(coord, claim, *up),
} }
} }
/// Write the agent's durable power intent — the DAG-node form of the old
/// pre-submit `set_wanted` side effect. Store-only (no container touch), so
/// build-slot-exempt; but it takes the agent's lifecycle lease (see
/// `NodeKind::needs_lease`) so the whole power-op DAG is atomic per-agent.
/// The downstream `Reconcile` reads the intent this writes. Unlike the old
/// warn-and-continue write, a failed write fails the node (cancel-downstream
/// cancels the `Reconcile`) rather than letting it converge to a stale
/// intent — that atomicity is the point of moving it into the DAG.
fn run_set_wanted(coord: &Arc<Coordinator>, claim: &Claim, up: bool) -> Result<NodeOutput> {
let wanted = if up {
crate::power::Wanted::Up
} else {
crate::power::Wanted::Offline
};
coord
.power
.set(&claim.agent, wanted)
.with_context(|| format!("set wanted={} for agent {}", wanted.as_str(), claim.agent))?;
Ok(NodeOutput::default())
}
/// Out-of-band toplevel build while the container keeps serving: meta /// Out-of-band toplevel build while the container keeps serving: meta
/// sync + optional per-agent relock, then warm /// sync + optional per-agent relock, then warm
/// `system.build.toplevel` so the later `Swap` hits cache and skips /// `system.build.toplevel` so the later `Swap` hits cache and skips

View file

@ -121,20 +121,6 @@ pub enum NodeKind {
/// dashboard. Holds no lease and no build slot; the child DAGs it anchors /// dashboard. Holds no lease and no build slot; the child DAGs it anchors
/// still run concurrently — the grouping is a display link, not a dep edge. /// still run concurrently — the grouping is a display link, not a dep edge.
Noop, Noop,
/// Write the agent's durable power intent (`wanted = Up` when `up`, else
/// `Offline`) as a first-class DAG node, at the head of a power-op
/// template so the downstream `Reconcile` reads it. Replaces the old
/// pre-submit `set_wanted` side effect: the intent write is now part of
/// the atomic DAG (crash-safe, per-agent — a multi-agent DAG carries one
/// `SetWanted` per agent). Build-slot-exempt (a store write), but
/// **lease-needing**: it takes the agent's lifecycle lease so the whole
/// power-op DAG (intent write → reconcile) is atomic per-agent — two
/// racing ops (e.g. restart vs stop) can't clobber each other's intent
/// before either reconciles, which is the point of moving the write into
/// the DAG. (In `stale_start` the lease is thus held across the head
/// `Prebuild`, but that's a no-op there — the agent is down, so prebuild
/// is skipped.)
SetWanted { up: bool },
} }
impl NodeKind { impl NodeKind {
@ -156,7 +142,6 @@ impl NodeKind {
NodeKind::WritePermFile => "write_perm_file", NodeKind::WritePermFile => "write_perm_file",
NodeKind::ApprovalDeploy => "approval_deploy", NodeKind::ApprovalDeploy => "approval_deploy",
NodeKind::Noop => "noop", NodeKind::Noop => "noop",
NodeKind::SetWanted { .. } => "set_wanted",
} }
} }
@ -192,7 +177,6 @@ impl NodeKind {
| NodeKind::Drain | NodeKind::Drain
| NodeKind::WriteDropin | NodeKind::WriteDropin
| NodeKind::ApprovalDeploy | NodeKind::ApprovalDeploy
| NodeKind::SetWanted { .. }
) )
} }
} }

View file

@ -1,15 +1,16 @@
//! Request-level submit API — the surface the dashboard POST handlers, //! Request-level submit API — the surface the dashboard POST handlers,
//! the MCP socket handlers, and `hivectl` paths call. The durable //! the MCP socket handlers, and `hivectl` paths call. Owns the
//! `wanted` power intent is now written by a `SetWanted` DAG node at the //! submit-time side effects the DAG templates deliberately don't:
//! head of each power-op template (not a pre-submit side effect); the //! writing the durable `wanted` power intent (synchronously,
//! only submit-time logic left is the stale-start *shape* decision //! last-writer-wins) before the DAG whose `Reconcile` reads it, and
//! (`start` vs `stale_start`). Every helper emits a fresh queue snapshot //! upgrading a stale start to a full rebuild. Every helper emits a
//! so the dashboard shows the new DAG immediately. //! fresh queue snapshot so the dashboard shows the new DAG immediately.
use std::sync::Arc; use std::sync::Arc;
use super::{Source, templates}; use super::{Source, Template, templates};
use crate::coordinator::Coordinator; use crate::coordinator::Coordinator;
use crate::power::Wanted;
fn submit_and_emit(coord: &Arc<Coordinator>, spec: super::DagSpec) -> u64 { fn submit_and_emit(coord: &Arc<Coordinator>, spec: super::DagSpec) -> u64 {
let id = coord let id = coord
@ -20,6 +21,12 @@ fn submit_and_emit(coord: &Arc<Coordinator>, spec: super::DagSpec) -> u64 {
id id
} }
fn set_wanted(coord: &Arc<Coordinator>, agent: &str, wanted: Wanted) {
if let Err(e) = coord.power.set(agent, wanted) {
tracing::warn!(%agent, wanted = wanted.as_str(), error = ?e, "agent_power: set failed");
}
}
/// Manual/approval-independent rebuild (always relocks the agent's /// Manual/approval-independent rebuild (always relocks the agent's
/// meta input — cascade children are built by the scheduler's fan-out /// meta input — cascade children are built by the scheduler's fan-out
/// instead of this surface). /// instead of this surface).
@ -28,20 +35,20 @@ pub fn rebuild(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: St
} }
/// Restart: mechanical stop + converge to `wanted = Up`. The intent /// Restart: mechanical stop + converge to `wanted = Up`. The intent
/// write is the template's head `SetWanted(Up)` node — it matters when /// write matters when `wanted` drifted `Offline` under a running
/// `wanted` drifted `Offline` under a running agent (an operator asking /// agent — the old `kill + start` always ended up, and an operator
/// for a restart plainly wants it running, not a stop). /// asking for a restart plainly wants it running, not a stop.
pub fn restart(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 { pub fn restart(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
set_wanted(coord, agent, Wanted::Up);
submit_and_emit(coord, templates::restart(agent, source, reason)) submit_and_emit(coord, templates::restart(agent, source, reason))
} }
/// Start: `SetWanted(Up)` (a DAG node now) then reconcile. A stale rev /// Start: persist `wanted = Up`, then reconcile. A stale rev marker
/// marker upgrades the start to a rebuild-then-start (`stale_start`, whose /// upgrades the start to a full rebuild (whose tail `Reconcile` does
/// tail `Reconcile` does the start) so the container always comes up on /// the start) so the container always comes up on current derivations
/// current derivations — the old fast-lane `run_start` upgrade, still a /// — the old fast-lane `run_start` upgrade, moved to submit time.
/// submit-time *shape* decision (which template), while the intent write
/// itself is now the template's head node.
pub fn start(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 { pub fn start(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
set_wanted(coord, agent, Wanted::Up);
let stored = std::fs::read_to_string(crate::paths::applied_rev_marker(agent)).ok(); let stored = std::fs::read_to_string(crate::paths::applied_rev_marker(agent)).ok();
let stale = crate::auto_update::current_flake_rev(&coord.hyperhive_flake) let stale = crate::auto_update::current_flake_rev(&coord.hyperhive_flake)
.is_some_and(|rev| stored.as_deref() != Some(rev.as_str())); .is_some_and(|rev| stored.as_deref() != Some(rev.as_str()));
@ -49,34 +56,60 @@ pub fn start(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: Stri
tracing::info!(%agent, "start: rev stale — upgrading to rebuild+start"); tracing::info!(%agent, "start: rev stale — upgrading to rebuild+start");
return submit_and_emit( return submit_and_emit(
coord, coord,
templates::stale_start(agent, source, format!("{reason} (stale — rebuild+start)")), templates::rebuild(
agent,
source,
format!("{reason} (stale — rebuild+start)"),
None,
true,
),
); );
} }
submit_and_emit(coord, templates::start(agent, source, reason)) submit_and_emit(
coord,
templates::reconcile_only(
Template::Start,
agent,
source,
reason,
Some(crate::coordinator::TransientKind::Starting),
),
)
} }
/// Hard stop: `SetWanted(Offline)` (a DAG node now) then reconcile (kill + /// Hard stop: persist `wanted = Offline`, then reconcile (kill +
/// unregister + `Killed` event). /// unregister + `Killed` event).
pub fn stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 { pub fn stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
submit_and_emit(coord, templates::stop(agent, source, reason)) set_wanted(coord, agent, Wanted::Offline);
submit_and_emit(
coord,
templates::reconcile_only(
Template::Stop,
agent,
source,
reason,
Some(crate::coordinator::TransientKind::Stopping),
),
)
} }
/// Graceful stop: signal → drain → reconcile (the actual stop). The head /// Graceful stop: persist `wanted = Offline`, then signal → drain →
/// `SetWanted(Offline)` node writes the intent as part of the DAG. /// reconcile (the actual stop).
pub fn graceful_stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 { pub fn graceful_stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
set_wanted(coord, agent, Wanted::Offline);
submit_and_emit(coord, templates::graceful_stop(agent, source, reason)) submit_and_emit(coord, templates::graceful_stop(agent, source, reason))
} }
/// Graceful restart: signal → drain → mechanical stop → reconcile (starts /// Graceful restart: persist `wanted = Up`, then signal → drain →
/// it back up) — one atomic DAG, no client-side "await the stop DAG then /// mechanical stop → reconcile (starts it back up) — one atomic DAG,
/// submit a start DAG" split. The head `SetWanted(Up)` node writes the /// no client-side "await the stop DAG then submit a start DAG" split.
/// intent as part of the DAG.
pub fn graceful_restart( pub fn graceful_restart(
coord: &Arc<Coordinator>, coord: &Arc<Coordinator>,
agent: &str, agent: &str,
source: Source, source: Source,
reason: String, reason: String,
) -> u64 { ) -> u64 {
set_wanted(coord, agent, Wanted::Up);
submit_and_emit(coord, templates::graceful_restart(agent, source, reason)) submit_and_emit(coord, templates::graceful_restart(agent, source, reason))
} }

View file

@ -8,19 +8,14 @@
//! are single-agent (every node shares one agent); a future multi-agent //! are single-agent (every node shares one agent); a future multi-agent
//! template would stamp different agents per subgraph. //! template would stamp different agents per subgraph.
//! //!
//! The power ops write the durable `wanted` intent via a head
//! `SetWanted(w)` node (not a pre-submit side effect); it holds the agent
//! lease so intent+reconcile is atomic per-agent.
//!
//! ```text //! ```text
//! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a) //! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a)
//! graceful-stop(a): SetWanted(a,Off) → Signal(a) → Drain(a) → Reconcile(a) //! graceful-stop(a): [wanted=Offline] Signal(a) → Drain(a) → Reconcile(a)
//! restart(a): SetWanted(a,Up) → StopForUpdate(a) → Reconcile(a) //! restart(a): [wanted=Up] StopForUpdate(a) → Reconcile(a)
//! graceful-restart(a): SetWanted(a,Up) → Signal(a) → Drain(a) → StopForUpdate(a) → Reconcile(a) //! graceful-restart(a): [wanted=Up] Signal(a) → Drain(a) → StopForUpdate(a) → Reconcile(a)
//! start(a): SetWanted(a,Up) → Reconcile(a) //! start(a): [wanted=Up] Reconcile(a)
//! stop(a): SetWanted(a,Off) → Reconcile(a) //! stop(a): [wanted=Offline] Reconcile(a)
//! stale-start(a): SetWanted(a,Up) → «rebuild subgraph» (rev stale; prebuild noops, agent down) //! spawn(a): [wanted=Up] Provision(a) → Create(a) → WriteDropin(a) → Reconcile(a)
//! spawn(a): Provision(a) → Create(a) → WriteDropin(a) → Reconcile(a) [wanted=Up at approve]
//! perm-change(a): WritePermFile(a) → «rebuild subgraph» //! perm-change(a): WritePermFile(a) → «rebuild subgraph»
//! meta-update(inp): MetaLock(inp) → «fan-out rebuild(a) per affected a» //! meta-update(inp): MetaLock(inp) → «fan-out rebuild(a) per affected a»
//! startup sweep: MetaLock(hyperhive, non-fatal) → «fan-out rebuild(stale a)» //! startup sweep: MetaLock(hyperhive, non-fatal) → «fan-out rebuild(stale a)»
@ -122,11 +117,10 @@ pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec
/// Graceful stop: cheap `Signal` fires immediately (no build slot), the /// Graceful stop: cheap `Signal` fires immediately (no build slot), the
/// `Drain` awaits the harness checkpoint (bounded), and the tail /// `Drain` awaits the harness checkpoint (bounded), and the tail
/// `Reconcile` performs the actual container stop. The head `SetWanted` /// `Reconcile` performs the actual container stop — the caller sets
/// node writes `wanted = Offline` as part of the DAG (was a pre-submit /// `wanted = Offline` at submit time. A whole-hive graceful stop
/// side effect). A whole-hive graceful stop therefore signals every agent /// therefore signals every agent up front and overlaps every drain,
/// up front and overlaps every drain, replacing the old detached-watcher /// replacing the old detached-watcher thread structurally.
/// thread structurally.
pub fn graceful_stop(agent: &str, source: Source, reason: String) -> DagSpec { pub fn graceful_stop(agent: &str, source: Source, reason: String) -> DagSpec {
DagSpec { DagSpec {
template: Template::GracefulStop, template: Template::GracefulStop,
@ -138,17 +132,16 @@ pub fn graceful_stop(agent: &str, source: Source, reason: String) -> DagSpec {
perm_payload: None, perm_payload: None,
transient: Some(TransientKind::Stopping), transient: Some(TransientKind::Stopping),
nodes: vec![ nodes: vec![
node(agent, NodeKind::SetWanted { up: false }, Vec::new()), node(agent, NodeKind::Signal, Vec::new()),
node(agent, NodeKind::Signal, after_ok(0)), node(agent, NodeKind::Drain, after_ok(0)),
node(agent, NodeKind::Drain, after_ok(1)), node(agent, NodeKind::Reconcile, after_ok(1)),
node(agent, NodeKind::Reconcile, after_ok(2)),
], ],
} }
} }
/// Restart: write `wanted = Up` (head `SetWanted` node), mechanical stop, /// Restart: mechanical stop, then converge to `wanted` — the submit
/// then converge — a stop + start like the old `lifecycle::restart` /// layer writes `wanted = Up` first, so this is a stop + start like
/// regardless of prior intent drift. /// the old `lifecycle::restart` regardless of prior intent drift.
pub fn restart(agent: &str, source: Source, reason: String) -> DagSpec { pub fn restart(agent: &str, source: Source, reason: String) -> DagSpec {
DagSpec { DagSpec {
template: Template::Restart, template: Template::Restart,
@ -160,20 +153,20 @@ pub fn restart(agent: &str, source: Source, reason: String) -> DagSpec {
perm_payload: None, perm_payload: None,
transient: Some(TransientKind::Restarting), transient: Some(TransientKind::Restarting),
nodes: vec![ nodes: vec![
node(agent, NodeKind::SetWanted { up: true }, Vec::new()), node(agent, NodeKind::StopForUpdate, Vec::new()),
node(agent, NodeKind::StopForUpdate, after_ok(0)), node(agent, NodeKind::Reconcile, after_ok(0)),
node(agent, NodeKind::Reconcile, after_ok(1)),
], ],
} }
} }
/// Graceful restart: write `wanted = Up` (head `SetWanted`), signal → /// Graceful restart: signal → drain → mechanical stop → converge to
/// drain → mechanical stop → converge. One atomic DAG start to finish /// `wanted` — the caller writes `wanted = Up` first, same as `restart`.
/// (no client- or server-side "submit one DAG, await it, submit the next" /// One atomic DAG start to finish (no client- or server-side "submit
/// composition): the `Drain` node is the same bounded harness-checkpoint /// one DAG, await it, submit the next" composition): the `Drain` node
/// wait `graceful_stop` uses, then `StopForUpdate` (mechanical, ignores /// is the same bounded harness-checkpoint wait `graceful_stop` uses,
/// `wanted`) and the tail `Reconcile` (converges to `wanted = Up`, i.e. /// then `StopForUpdate` (mechanical, ignores `wanted`) and the tail
/// starts it back up) chain exactly like `restart`'s tail. /// `Reconcile` (converges to `wanted = Up`, i.e. starts it back up)
/// chain exactly like `restart`'s tail.
pub fn graceful_restart(agent: &str, source: Source, reason: String) -> DagSpec { pub fn graceful_restart(agent: &str, source: Source, reason: String) -> DagSpec {
DagSpec { DagSpec {
template: Template::GracefulRestart, template: Template::GracefulRestart,
@ -185,19 +178,16 @@ pub fn graceful_restart(agent: &str, source: Source, reason: String) -> DagSpec
perm_payload: None, perm_payload: None,
transient: Some(TransientKind::Restarting), transient: Some(TransientKind::Restarting),
nodes: vec![ nodes: vec![
node(agent, NodeKind::SetWanted { up: true }, Vec::new()), node(agent, NodeKind::Signal, Vec::new()),
node(agent, NodeKind::Signal, after_ok(0)), node(agent, NodeKind::Drain, after_ok(0)),
node(agent, NodeKind::Drain, after_ok(1)), node(agent, NodeKind::StopForUpdate, after_ok(1)),
node(agent, NodeKind::StopForUpdate, after_ok(2)), node(agent, NodeKind::Reconcile, after_ok(2)),
node(agent, NodeKind::Reconcile, after_ok(3)),
], ],
} }
} }
/// Boot-time reconcile: a single `Reconcile` node that converges observed /// Single-`Reconcile` DAG: `Start` / `Stop` (caller writes `wanted`
/// power state to the persisted intent — `wanted` is untouched (no /// first) and the boot-time `Reconcile` converge (wanted untouched).
/// `SetWanted`), unlike the operator `start`/`stop` templates. Used only
/// by the boot sweep now.
pub fn reconcile_only( pub fn reconcile_only(
template: Template, template: Template,
agent: &str, agent: &str,
@ -218,68 +208,6 @@ pub fn reconcile_only(
} }
} }
/// Start: write `wanted = Up` (head `SetWanted`), then reconcile (which
/// starts the container). The intent write is a DAG node now, not a
/// pre-submit side effect.
pub fn start(agent: &str, source: Source, reason: String) -> DagSpec {
DagSpec {
template: Template::Start,
source,
reason,
parent_id: None,
approval_id: None,
inputs: Vec::new(),
perm_payload: None,
transient: Some(TransientKind::Starting),
nodes: vec![
node(agent, NodeKind::SetWanted { up: true }, Vec::new()),
node(agent, NodeKind::Reconcile, after_ok(0)),
],
}
}
/// Stop: write `wanted = Offline` (head `SetWanted`), then reconcile
/// (kill + unregister + `Killed` event).
pub fn stop(agent: &str, source: Source, reason: String) -> DagSpec {
DagSpec {
template: Template::Stop,
source,
reason,
parent_id: None,
approval_id: None,
inputs: Vec::new(),
perm_payload: None,
transient: Some(TransientKind::Stopping),
nodes: vec![
node(agent, NodeKind::SetWanted { up: false }, Vec::new()),
node(agent, NodeKind::Reconcile, after_ok(0)),
],
}
}
/// Stale start: a `start` whose rev marker is stale, so it rebuilds
/// before coming up — `SetWanted(Up)` → «rebuild subgraph» → `Reconcile`
/// (the rebuild's tail `Reconcile` starts it, since `wanted = Up`). The
/// rebuild nodes are the same `rebuild_nodes` chain a manual rebuild uses
/// (reused, not a variant); only the leading `SetWanted(Up)` intent
/// differs. Shows as a `Rebuild` on the dashboard like the old
/// submit-time upgrade did.
pub fn stale_start(agent: &str, source: Source, reason: String) -> DagSpec {
let mut nodes = vec![node(agent, NodeKind::SetWanted { up: true }, Vec::new())];
nodes.extend(rebuild_nodes(agent, true, 1));
DagSpec {
template: Template::Rebuild,
source,
reason,
parent_id: None,
approval_id: None,
inputs: Vec::new(),
perm_payload: None,
transient: Some(TransientKind::Rebuilding),
nodes,
}
}
/// First-deploy spawn (approval-driven): `Provision` (proposed/applied /// First-deploy spawn (approval-driven): `Provision` (proposed/applied
/// repos, state subvolume, meta registration) then `Create` /// repos, state subvolume, meta registration) then `Create`
/// (`nixos-container create`), drop-in write, then `Reconcile` starts /// (`nixos-container create`), drop-in write, then `Reconcile` starts

View file

@ -212,27 +212,21 @@ fn lease_serializes_two_lifecycle_dags_for_same_agent() {
None, None,
), ),
); );
// Restart's head SetWanted takes the lease; stop's Reconcile must // Restart's StopForUpdate acquires the lease; stop's Reconcile
// wait even though slots are free. // must wait even though slots are free.
let first = claim_one(&q); let first = claim_one(&q);
assert_eq!(first.dag_id, restart); assert_eq!(first.dag_id, restart);
assert_eq!(first.kind.as_str(), "set_wanted");
assert!(first.lease_acquired); assert!(first.lease_acquired);
q.complete_node(restart, first.node_id, Ok(())); q.complete_node(restart, first.node_id, Ok(()));
// Same DAG keeps the lease through StopForUpdate then Reconcile. // Same DAG keeps the lease for its Reconcile.
let second = claim_one(&q); let second = claim_one(&q);
assert_eq!(second.dag_id, restart); assert_eq!(second.dag_id, restart);
assert_eq!(second.kind.as_str(), "stop_for_update");
assert!(!second.lease_acquired, "lease already held by this DAG"); assert!(!second.lease_acquired, "lease already held by this DAG");
q.complete_node(restart, second.node_id, Ok(())); q.complete_node(restart, second.node_id, Ok(()));
let third = claim_one(&q);
assert_eq!(third.dag_id, restart);
assert_eq!(third.kind.as_str(), "reconcile");
q.complete_node(restart, third.node_id, Ok(()));
// Restart terminal → lease released → stop's Reconcile runs. // Restart terminal → lease released → stop's Reconcile runs.
let fourth = claim_one(&q); let third = claim_one(&q);
assert_eq!(fourth.dag_id, stop); assert_eq!(third.dag_id, stop);
q.complete_node(stop, fourth.node_id, Ok(())); q.complete_node(stop, third.node_id, Ok(()));
assert_eq!(state_of(&q, restart), State::Done); assert_eq!(state_of(&q, restart), State::Done);
assert_eq!(state_of(&q, stop), State::Done); assert_eq!(state_of(&q, stop), State::Done);
} }
@ -516,11 +510,6 @@ fn terminal_dag_reported_exactly_once_and_lease_released() {
&q, &q,
templates::restart("agent-a", Source::Manual, "r".to_owned()), templates::restart("agent-a", Source::Manual, "r".to_owned()),
); );
// restart = SetWanted → StopForUpdate → Reconcile; not terminal until
// the last node completes.
let set_wanted = claim_one(&q);
q.complete_node(id, set_wanted.node_id, Ok(()));
assert!(q.drain_terminal().is_empty(), "dag not terminal yet");
let stop = claim_one(&q); let stop = claim_one(&q);
q.complete_node(id, stop.node_id, Ok(())); q.complete_node(id, stop.node_id, Ok(()));
assert!(q.drain_terminal().is_empty(), "dag not terminal yet"); assert!(q.drain_terminal().is_empty(), "dag not terminal yet");
@ -738,7 +727,7 @@ fn graceful_stop_shape_signal_drain_reconcile() {
&q, &q,
templates::graceful_stop("agent-a", Source::Manual, "graceful".to_owned()), templates::graceful_stop("agent-a", Source::Manual, "graceful".to_owned()),
); );
for expected in ["set_wanted", "signal", "drain", "reconcile"] { for expected in ["signal", "drain", "reconcile"] {
let c = claim_one(&q); let c = claim_one(&q);
assert_eq!(c.kind.as_str(), expected); assert_eq!(c.kind.as_str(), expected);
q.complete_node(id, c.node_id, Ok(())); q.complete_node(id, c.node_id, Ok(()));
@ -764,9 +753,8 @@ fn graceful_signal_and_drain_hold_no_build_slot() {
let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect(); let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect();
assert_eq!( assert_eq!(
kinds, kinds,
vec!["prebuild", "set_wanted", "set_wanted"], vec!["prebuild", "signal", "signal"],
"both agents' graceful-stop heads (SetWanted, build-slot-exempt) run \ "both agents' signals fire while the slot is held"
while the slot is held; their signals follow"
); );
} }