refactor(#2390): split provision out of the create node in the spawn dag
This commit is contained in:
parent
96c748475e
commit
436adf6fd0
6 changed files with 79 additions and 35 deletions
|
|
@ -145,6 +145,7 @@ const NODE_KIND_LABEL = {
|
||||||
prebuild: 'prebuild',
|
prebuild: 'prebuild',
|
||||||
stop_for_update: 'stop',
|
stop_for_update: 'stop',
|
||||||
swap: 'swap',
|
swap: 'swap',
|
||||||
|
provision: 'provision',
|
||||||
create: 'create',
|
create: 'create',
|
||||||
meta_lock: 'meta lock',
|
meta_lock: 'meta lock',
|
||||||
reconcile: 'reconcile',
|
reconcile: 'reconcile',
|
||||||
|
|
|
||||||
|
|
@ -81,7 +81,8 @@ pub(super) async fn run_node(coord: &Arc<Coordinator>, claim: &Claim) -> Result<
|
||||||
match &claim.kind {
|
match &claim.kind {
|
||||||
NodeKind::Prebuild { relock } => run_prebuild(coord, claim, &ctx, *relock).await,
|
NodeKind::Prebuild { relock } => run_prebuild(coord, claim, &ctx, *relock).await,
|
||||||
NodeKind::Swap => run_swap(coord, claim, &ctx).await,
|
NodeKind::Swap => run_swap(coord, claim, &ctx).await,
|
||||||
NodeKind::Create => run_create(coord, claim, &ctx).await,
|
NodeKind::Provision => run_provision(coord, claim, &ctx).await,
|
||||||
|
NodeKind::Create => run_create(claim, &ctx).await,
|
||||||
NodeKind::MetaLock { sweep, fanout } => {
|
NodeKind::MetaLock { sweep, fanout } => {
|
||||||
run_meta_lock(coord, claim, &ctx, *sweep, fanout.clone()).await
|
run_meta_lock(coord, claim, &ctx, *sweep, fanout.clone()).await
|
||||||
}
|
}
|
||||||
|
|
@ -191,23 +192,35 @@ async fn run_swap(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Res
|
||||||
result.map(|()| NodeOutput::default())
|
result.map(|()| NodeOutput::default())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// First-spawn provisioning + `nixos-container create` (atomic
|
/// First-spawn pre-create provisioning: proposed/applied repos, state
|
||||||
/// build+create — no prebuild needed).
|
/// subvolume, and the meta `sync_agents` registration. Holds the
|
||||||
async fn run_create(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
|
/// deploy-window gate for the commit so it can't land inside another
|
||||||
|
/// node's staged deploy window — same discipline as `Prebuild`, which
|
||||||
|
/// commits under the gate then drops it before the (store-only) build.
|
||||||
|
async fn run_provision(
|
||||||
|
coord: &Arc<Coordinator>,
|
||||||
|
claim: &Claim,
|
||||||
|
ctx: &Ctx<'_>,
|
||||||
|
) -> Result<NodeOutput> {
|
||||||
let name = &claim.agent;
|
let name = &claim.agent;
|
||||||
let agent_dir = crate::paths::agent_runtime_dir(name);
|
let agent_dir = crate::paths::agent_runtime_dir(name);
|
||||||
let hive = coord.hive_env();
|
let hive = coord.hive_env();
|
||||||
let paths = Coordinator::agent_paths(name, agent_dir);
|
let paths = Coordinator::agent_paths(name, agent_dir);
|
||||||
ctx.step("nixos-container create");
|
ctx.step("provisioning");
|
||||||
// create_container registers the new agent in the meta flake
|
|
||||||
// (sync_agents commit) before `nixos-container create` — hold the
|
|
||||||
// deploy-window gate so that commit can't land inside another
|
|
||||||
// node's staged deploy window.
|
|
||||||
// Runtime dir creation and MCP listener registration are deferred to
|
|
||||||
// the tail Reconcile (converge_start_preamble + register_agent) so this
|
|
||||||
// node stays purely "provision + create", not "create + start".
|
|
||||||
let _window = crate::meta::exclusive().await;
|
let _window = crate::meta::exclusive().await;
|
||||||
crate::lifecycle::create_container(name, &hive, &paths).await?;
|
crate::lifecycle::provision_container(name, &hive, &paths).await?;
|
||||||
|
Ok(NodeOutput::default())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `nixos-container create` proper — the upstream `Provision` node
|
||||||
|
/// already registered the agent in meta, so this only reads the store
|
||||||
|
/// (no deploy-window gate needed, mirroring `Prebuild`'s build). Runtime
|
||||||
|
/// dir creation and MCP listener registration are deferred to the tail
|
||||||
|
/// `Reconcile` (`converge_start_preamble` + `register_agent`) so this
|
||||||
|
/// node stays purely "create", not "create + start".
|
||||||
|
async fn run_create(claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
|
||||||
|
ctx.step("nixos-container create");
|
||||||
|
crate::lifecycle::create_only(&claim.agent).await?;
|
||||||
Ok(NodeOutput::default())
|
Ok(NodeOutput::default())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -75,9 +75,14 @@ pub enum NodeKind {
|
||||||
/// rebuild is the reconcile verb — and carries the post-rebuild
|
/// rebuild is the reconcile verb — and carries the post-rebuild
|
||||||
/// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan).
|
/// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan).
|
||||||
Swap,
|
Swap,
|
||||||
/// First-spawn `nixos-container create` plus the pre-create
|
/// First-spawn pre-create provisioning: proposed/applied repos,
|
||||||
/// provisioning (proposed/applied repos, state subvolume, meta
|
/// state subvolume, and meta registration (`sync_agents`). Runs
|
||||||
/// registration).
|
/// ahead of `Create` so the `nixos-container create --flake
|
||||||
|
/// meta#<name>` ref resolves. Store/meta-only — no container yet —
|
||||||
|
/// so it's lease- and build-slot-exempt like `Prebuild`.
|
||||||
|
Provision,
|
||||||
|
/// First-spawn `nixos-container create` proper. Assumes the
|
||||||
|
/// upstream `Provision` node already registered the agent in meta.
|
||||||
Create,
|
Create,
|
||||||
/// Meta flake lock bump. `sweep = false`: `meta::lock_update`
|
/// Meta flake lock bump. `sweep = false`: `meta::lock_update`
|
||||||
/// (commit fused, under `META_LOCK`) with the DAG's `inputs`;
|
/// (commit fused, under `META_LOCK`) with the DAG's `inputs`;
|
||||||
|
|
@ -139,6 +144,7 @@ impl NodeKind {
|
||||||
match self {
|
match self {
|
||||||
NodeKind::Prebuild { .. } => "prebuild",
|
NodeKind::Prebuild { .. } => "prebuild",
|
||||||
NodeKind::Swap => "swap",
|
NodeKind::Swap => "swap",
|
||||||
|
NodeKind::Provision => "provision",
|
||||||
NodeKind::Create => "create",
|
NodeKind::Create => "create",
|
||||||
NodeKind::MetaLock { .. } => "meta_lock",
|
NodeKind::MetaLock { .. } => "meta_lock",
|
||||||
NodeKind::Reconcile => "reconcile",
|
NodeKind::Reconcile => "reconcile",
|
||||||
|
|
@ -169,10 +175,12 @@ impl NodeKind {
|
||||||
|
|
||||||
/// Container-affecting kinds require the DAG to hold the agent's
|
/// Container-affecting kinds require the DAG to hold the agent's
|
||||||
/// lifecycle lease (acquired at the first such node, held until the
|
/// lifecycle lease (acquired at the first such node, held until the
|
||||||
/// DAG is terminal). Lease-exempt kinds (`Prebuild`, `MetaLock`,
|
/// DAG is terminal). Lease-exempt kinds (`Prebuild`, `Provision`,
|
||||||
/// `WritePermFile`) touch the store / meta repo, not the running
|
/// `MetaLock`, `WritePermFile`) touch the store / meta repo, not the
|
||||||
/// container — which is exactly why a `Prebuild` can overlap
|
/// running container — which is exactly why a `Prebuild` can overlap
|
||||||
/// another DAG's work on the same agent.
|
/// another DAG's work on the same agent. `Provision` precedes the
|
||||||
|
/// container's existence entirely, so the lease is first taken at the
|
||||||
|
/// `Create` node it feeds.
|
||||||
pub fn needs_lease(&self) -> bool {
|
pub fn needs_lease(&self) -> bool {
|
||||||
matches!(
|
matches!(
|
||||||
self,
|
self,
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@
|
||||||
//! restart(a): [wanted=Up] StopForUpdate(a) → Reconcile(a)
|
//! restart(a): [wanted=Up] StopForUpdate(a) → Reconcile(a)
|
||||||
//! start(a): [wanted=Up] Reconcile(a)
|
//! start(a): [wanted=Up] Reconcile(a)
|
||||||
//! stop(a): [wanted=Offline] Reconcile(a)
|
//! stop(a): [wanted=Offline] Reconcile(a)
|
||||||
//! spawn(a): [wanted=Up] Create(a) → WriteDropin(a) → Reconcile(a)
|
//! spawn(a): [wanted=Up] Provision(a) → Create(a) → WriteDropin(a) → Reconcile(a)
|
||||||
//! perm-change(a): WritePermFile(a) → «rebuild subgraph»
|
//! perm-change(a): WritePermFile(a) → «rebuild subgraph»
|
||||||
//! meta-update(inp): MetaLock(inp) → «fan-out rebuild(a) per affected a»
|
//! meta-update(inp): MetaLock(inp) → «fan-out rebuild(a) per affected a»
|
||||||
//! startup sweep: MetaLock(hyperhive, non-fatal) → «fan-out rebuild(stale a)»
|
//! startup sweep: MetaLock(hyperhive, non-fatal) → «fan-out rebuild(stale a)»
|
||||||
|
|
@ -194,9 +194,10 @@ pub fn reconcile_only(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// First-deploy spawn (approval-driven): pre-start provisioning +
|
/// First-deploy spawn (approval-driven): `Provision` (proposed/applied
|
||||||
/// `nixos-container create`, drop-in write, then `Reconcile` starts the
|
/// repos, state subvolume, meta registration) then `Create`
|
||||||
/// container (`wanted = Up` written at approve time).
|
/// (`nixos-container create`), drop-in write, then `Reconcile` starts
|
||||||
|
/// the container (`wanted = Up` written at approve time).
|
||||||
pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
|
pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
|
||||||
DagSpec {
|
DagSpec {
|
||||||
template: Template::Spawn,
|
template: Template::Spawn,
|
||||||
|
|
@ -210,17 +211,21 @@ pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
|
||||||
transient: Some(TransientKind::Spawning),
|
transient: Some(TransientKind::Spawning),
|
||||||
nodes: vec![
|
nodes: vec![
|
||||||
NodeSpec {
|
NodeSpec {
|
||||||
kind: NodeKind::Create,
|
kind: NodeKind::Provision,
|
||||||
deps: Vec::new(),
|
deps: Vec::new(),
|
||||||
},
|
},
|
||||||
NodeSpec {
|
NodeSpec {
|
||||||
kind: NodeKind::WriteDropin,
|
kind: NodeKind::Create,
|
||||||
deps: after_ok(0),
|
deps: after_ok(0),
|
||||||
},
|
},
|
||||||
NodeSpec {
|
NodeSpec {
|
||||||
kind: NodeKind::Reconcile,
|
kind: NodeKind::WriteDropin,
|
||||||
deps: after_ok(1),
|
deps: after_ok(1),
|
||||||
},
|
},
|
||||||
|
NodeSpec {
|
||||||
|
kind: NodeKind::Reconcile,
|
||||||
|
deps: after_ok(2),
|
||||||
|
},
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -874,13 +874,13 @@ fn graceful_signal_and_drain_hold_no_build_slot() {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn spawn_shape_create_dropin_reconcile() {
|
fn spawn_shape_provision_create_dropin_reconcile() {
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
let id = submit(
|
let id = submit(
|
||||||
&q,
|
&q,
|
||||||
templates::spawn("newbie", 7, "approval #7 spawn".to_owned()),
|
templates::spawn("newbie", 7, "approval #7 spawn".to_owned()),
|
||||||
);
|
);
|
||||||
for expected in ["create", "write_dropin", "reconcile"] {
|
for expected in ["provision", "create", "write_dropin", "reconcile"] {
|
||||||
let c = claim_one(&q);
|
let c = claim_one(&q);
|
||||||
assert_eq!(c.kind.as_str(), expected);
|
assert_eq!(c.kind.as_str(), expected);
|
||||||
assert_eq!(c.approval_id, Some(7));
|
assert_eq!(c.approval_id, Some(7));
|
||||||
|
|
|
||||||
|
|
@ -226,11 +226,13 @@ pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()>
|
||||||
priv_run("start", name).await
|
priv_run("start", name).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// First-spawn provisioning + `nixos-container create`, without the
|
/// First-spawn pre-create provisioning — the job queue's `Provision`
|
||||||
/// drop-in write or the start — the job queue's `Create` node.
|
/// node. Fail fast on a port collision, set up the proposed/applied
|
||||||
/// `spawn` composes this with `write_dropins` + start for direct
|
/// repos + state subvolume + claude/notes dirs, then register the new
|
||||||
/// callers (root-agent bootstrap).
|
/// agent in the meta flake (`sync_agents`) so the later
|
||||||
pub async fn create_container(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
|
/// `nixos-container create --flake meta#<name>` ref resolves. Does NOT
|
||||||
|
/// create the container — that's `create_only` / the `Create` node.
|
||||||
|
pub async fn provision_container(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
|
||||||
validate(name)?;
|
validate(name)?;
|
||||||
if let Some(other) = port_collision(name).await {
|
if let Some(other) = port_collision(name).await {
|
||||||
bail!(
|
bail!(
|
||||||
|
|
@ -247,10 +249,25 @@ pub async fn create_container(name: &str, hive: &HiveEnv, paths: &AgentPaths) ->
|
||||||
// before `nixos-container create` so the `--flake meta#<name>`
|
// before `nixos-container create` so the `--flake meta#<name>`
|
||||||
// ref resolves.
|
// ref resolves.
|
||||||
let agents = agents_after_spawn(name).await?;
|
let agents = agents_after_spawn(name).await?;
|
||||||
crate::meta::sync_agents(hive, &agents).await?;
|
crate::meta::sync_agents(hive, &agents).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The `nixos-container create` proper — the job queue's `Create` node.
|
||||||
|
/// Assumes `provision_container` already registered the agent in meta.
|
||||||
|
pub async fn create_only(name: &str) -> Result<()> {
|
||||||
priv_run("create", name).await
|
priv_run("create", name).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// First-spawn provisioning + `nixos-container create`, without the
|
||||||
|
/// drop-in write or the start. `spawn` composes this with
|
||||||
|
/// `write_dropins` + start for direct callers (root-agent bootstrap);
|
||||||
|
/// the job queue instead runs `provision_container` (`Provision` node)
|
||||||
|
/// and `create_only` (`Create` node) as separate DAG steps.
|
||||||
|
pub async fn create_container(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
|
||||||
|
provision_container(name, hive, paths).await?;
|
||||||
|
create_only(name).await
|
||||||
|
}
|
||||||
|
|
||||||
/// Rebuild-path preamble shared by the job queue's `Prebuild` node and
|
/// Rebuild-path preamble shared by the job queue's `Prebuild` node and
|
||||||
/// `rebuild_no_meta`: fail fast on a port collision, then make sure
|
/// `rebuild_no_meta`: fail fast on a port collision, then make sure
|
||||||
/// the applied repo + state dirs exist. Container untouched.
|
/// the applied repo + state dirs exist. Container untouched.
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue