From bfd019fe8d59c83583490f070617e808e97ebdab Mon Sep 17 00:00:00 2001 From: iris Date: Fri, 18 Sep 2026 17:48:48 +0200 Subject: [PATCH 1/5] swarm-controller: declare a new agent paused at creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Creating an agent queued its identity, forge repo and deploy, but never wrote a wanted-state declaration for it — so the agent showed up in swarm-ui as "no declaration", and the hive brought it up with nothing saying whether it should be driving turns. Add a `SetAgentWanted` job node that declares the agent `Paused` in its hive's wanted-state bucket, using the same `WantedWriter::set` primitive the per-agent state HTTP handler already uses. A fresh agent therefore sits paused until the operator explicitly flips it to `Up`. The node is a root — it needs only the hive and agent names known at request time — but the deploy trigger now waits on it, so the pause is in the store before the hive brings the container up rather than landing some time after a freshly deployed agent has already started taking turns. --- swarm-controller/src/main.rs | 210 ++++++++++++++++++++++++++++++++++- 1 file changed, 206 insertions(+), 4 deletions(-) diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 5594654e..609fd7ab 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -105,6 +105,12 @@ enum SwarmNodeKind { /// its hive's shared queue credential, so the document cannot be rendered /// without knowing which hive the agent belongs to. MintAgentIdentity { hive: String, agent: String }, + /// Declare `agent` on `hive` as `Paused` in the swarm's wanted-state + /// store, so a freshly created agent does not start driving turns the + /// moment it's deployed — the operator has to explicitly flip it to `Up`. + /// Carries the hive for the same reason `TriggerDeploy`/`MintAgentIdentity` + /// do: the wanted-state bucket is keyed per hive. + SetAgentWanted { hive: String, agent: String }, /// Tell `hive` to rebuild `agent`, by publishing on the swarm's deploy /// subject. The one node kind whose effect leaves this host. /// @@ -124,6 +130,7 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind { SwarmNodeKind::AddRepoMember { .. } => "add_repo_member".to_owned(), SwarmNodeKind::InitAgentConfigRepo { .. } => "init_agent_config_repo".to_owned(), SwarmNodeKind::MintAgentIdentity { .. } => "mint_agent_identity".to_owned(), + SwarmNodeKind::SetAgentWanted { .. } => "set_agent_wanted".to_owned(), SwarmNodeKind::TriggerDeploy { .. } => "trigger_deploy".to_owned(), } } @@ -144,7 +151,8 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind { serde_json::json!({ "agent": agent }) } SwarmNodeKind::TriggerDeploy { hive, agent } - | SwarmNodeKind::MintAgentIdentity { hive, agent } => { + | SwarmNodeKind::MintAgentIdentity { hive, agent } + | SwarmNodeKind::SetAgentWanted { hive, agent } => { serde_json::json!({ "agent": agent, "hive": hive }) } } @@ -186,8 +194,22 @@ struct WorkerDeps { /// a per-node chance to read one. `None` on a host the operator has not /// given an authority — see `agent_identity::Authority::from_env`. agent_ca: Option>, + /// The wanted-state writer, for nodes that declare what a hive should + /// converge an agent to. Shares the status reader's queue connection — + /// see `wanted_writer`. `None` exactly when no swarm queue is configured + /// on this host, same as `queue`. + wanted: Option>, } +/// What [`SwarmNodeKind::SetAgentWanted`] declares a brand-new agent to be. +/// +/// A named constant rather than the literal inline, because this *is* the +/// decision the node exists to carry — writing a declaration at all is the +/// mechanism, `Paused` is the policy — and a test can pin a name where it +/// cannot pin a value written into a KV bucket no unit test has a server for. +const NEW_AGENT_WANTED_STATE: swarm_queue_client::wanted::AgentState = + swarm_queue_client::wanted::AgentState::Paused; + /// Run a claimed node's actual work. Mirrors `hive-c0re/src/job_queue/ /// exec.rs::run_node`'s role exactly — the one place a `SwarmNodeKind` /// variant turns into a real effect. @@ -286,6 +308,14 @@ async fn run_swarm_node( } } }, + SwarmNodeKind::SetAgentWanted { hive, agent } => match deps.wanted { + None => Outcome::Failed( + "no swarm queue is configured on this host, so no wanted-state \ + declaration can be written" + .to_owned(), + ), + Some(writer) => declare_new_agent(&writer, &hive, &agent).await, + }, SwarmNodeKind::TriggerDeploy { hive, agent } => match deps.queue { None => Outcome::Failed( "no swarm queue is configured on this host, so no hive can be told to deploy" @@ -297,6 +327,25 @@ async fn run_swarm_node( (builder, outcome) } +/// Declare a brand-new agent at [`NEW_AGENT_WANTED_STATE`]. +/// +/// A named function beside [`publish_deploy`] rather than the two lines +/// inline, for the same reason that one is: `run_swarm_node`'s match is a +/// per-variant index, and every arm that grows a body past a call pushes the +/// next reader further from the variant they came to read. +async fn declare_new_agent( + writer: &wanted::WantedWriter, + hive: &str, + agent: &str, +) -> hive_jobq::scheduler::Outcome { + use hive_jobq::scheduler::Outcome; + + match writer.set(hive, agent, NEW_AGENT_WANTED_STATE).await { + Ok(_) => Outcome::Done, + Err(e) => Outcome::Failed(format!("{e:#}")), + } +} + /// Publish one deploy request and report whether it left this process. /// /// The flush is not belt-and-braces: `publish` hands the message to the @@ -1316,6 +1365,15 @@ fn declare_agent_job( agent: agent.to_owned(), }) .after_ok(create_identity); + // Declared before the deploy trigger so the pause is visible in the + // wanted-state store before the hive brings the container up — see the + // node's own doc comment for why "before", not just "eventually". Needs + // no parent of its own: the declaration is written from the hive/agent + // names known at request time, same as `create_forge_user`. + let set_wanted = b.node(SwarmNodeKind::SetAgentWanted { + hive: hive.to_owned(), + agent: agent.to_owned(), + }); // Last, and specifically after the config repo is seeded: the hive // deploys by reading that repo, so a deploy asked for any earlier would // find nothing to build. This is the edge that makes creating an agent at @@ -1327,13 +1385,21 @@ fn declare_agent_job( // overtake the mint — but a host with no authority configured must still // create agents exactly as it does today. `after_ok` there would turn an // unconfigured option into an agent nobody runs. + // + // `after_ok` on the declaration, though: an agent deployed without its + // pause landing first is the race this node exists to close, and the + // only way the declaration fails without a live queue is a host that has + // no queue at all — on which `TriggerDeploy` has nothing to publish to + // either, so the strict edge cancels a node that could not have + // succeeded. let _trigger_deploy = b .node(SwarmNodeKind::TriggerDeploy { hive: hive.to_owned(), agent: agent.to_owned(), }) .after_ok(init_config) - .after_any(mint_identity); + .after_any(mint_identity) + .after_ok(set_wanted); vec![create_identity.guid()] } @@ -1664,6 +1730,7 @@ async fn main() -> Result<()> { forge: forge_client.clone(), queue: status.as_ref().map(|s| s.queue_client()), agent_ca: load_agent_authority(), + wanted: wanted_writer(status.as_ref()), }; let jobq = Arc::new(Mutex::new(hive_jobq::scheduler::Scheduler::new( @@ -1882,8 +1949,9 @@ mod tests { }]), links: std::sync::Arc::new(Vec::new()), status: None, - // No queue, for the same reason as `status`: these tests drive - // agent creation, which publishes no declaration. + // No queue, for the same reason as `status`: these tests build + // the creation graph and read its shape, and never run the node + // that would write a declaration. wanted: None, agent_status: None, jobq: std::sync::Arc::clone(&sched), @@ -2031,6 +2099,7 @@ mod tests { forge: None, queue: None, agent_ca: None, + wanted: None, }; let runner = hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { @@ -2084,6 +2153,7 @@ mod tests { forge: None, queue: None, agent_ca: None, + wanted: None, }; let runner = hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { @@ -2137,6 +2207,7 @@ mod tests { forge: None, queue: None, agent_ca: None, + wanted: None, }; let runner = hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { @@ -2159,6 +2230,137 @@ mod tests { ); } + /// Fourth sibling, same shape and the same caveat once more: with no + /// queue configured this reaches only the graceful-absence-is-failure + /// branch. Writing the declaration for real needs a `JetStream` server, + /// which no test in this crate has — `wanted::apply`'s own unit tests + /// cover what gets written, and [`a_new_agent_is_declared_paused`] below + /// pins the state this node asks for. + #[tokio::test] + async fn set_agent_wanted_node_runs_end_to_end_and_fails_without_a_queue() { + let mut sched = hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ); + let id = sched + .append( + SwarmNodeKind::SetAgentWanted { + hive: "pr1ma".to_owned(), + agent: "atlas".to_owned(), + }, + Vec::new(), + None, + ) + .expect("insert"); + let sched = std::sync::Arc::new(std::sync::Mutex::new(sched)); + + let deps = WorkerDeps { + auth: None, + forge: None, + queue: None, + agent_ca: None, + wanted: None, + }; + let runner = + hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { + run_swarm_node(id, kind, builder, deps) + }) + .expect("the node just inserted is runnable"); + runner + .await + .1 + .expect("no growth declared, nothing to reject"); + + let guard = sched.lock().unwrap(); + let node = guard.graph().node(id).expect("node still present"); + assert_eq!(node.state, hive_jobq::State::Failed); + assert!( + node.error.as_deref().unwrap_or_default().contains("queue"), + "expected a no-queue-configured error, got {:?}", + node.error + ); + } + + /// The whole point of the node: a freshly created agent is declared + /// `Paused`, so it does not start driving turns the moment its hive + /// brings it up. Any other state here is the bug this fixes. + #[test] + fn a_new_agent_is_declared_paused() { + assert_eq!( + crate::NEW_AGENT_WANTED_STATE, + swarm_queue_client::wanted::AgentState::Paused + ); + } + + /// Same reasoning as `a_mint_node_renders_both_the_agent_and_the_hive`: + /// this variant carries a hive too, and has to have joined `data`'s + /// two-field or-pattern rather than the agent-only one. + #[test] + fn a_set_wanted_node_renders_both_the_agent_and_the_hive() { + use hive_jobq_wire::WireNode as _; + + let kind = SwarmNodeKind::SetAgentWanted { + hive: "pr1ma".to_owned(), + agent: "atlas".to_owned(), + }; + assert_eq!(kind.label(), "set_agent_wanted"); + let data = kind.data(1); + assert_eq!(data["agent"], "atlas"); + assert_eq!(data["hive"], "pr1ma"); + } + + /// The ordering the fix depends on: the pause has to be in the + /// wanted-state store *before* the hive is told to deploy, or a freshly + /// created agent runs undeclared for the window between the two. + /// + /// Asserted on the graph `create_agent` builds, same as the mint edge's + /// test above and for the same reason — the edge is one line in a + /// builder closure whose absence changes nothing observable until a real + /// agent boots and starts taking turns nobody asked for. + #[tokio::test] + async fn the_deploy_waits_for_the_pause_to_be_declared() { + use hive_jobq_wire::WireNode as _; + + let (state, sched) = state_with_roster(); + let _queued = super::create_agent( + axum::extract::State(state), + axum::Json(super::CreateAgentRequest { + name: "atlas".to_owned(), + hive: "pr1ma".to_owned(), + }), + ) + .await + .expect("a hive in the roster must be accepted"); + + let guard = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let graph = guard.graph(); + let id_of = |label: &str| { + graph + .nodes() + .find(|n| n.payload.label() == label) + .unwrap_or_else(|| panic!("the graph holds a {label} node")) + .id + }; + let set_wanted = id_of("set_agent_wanted"); + let deploy = graph.node(id_of("trigger_deploy")).expect("just found"); + + let when = deploy + .deps + .iter() + .find_map(|d| match d { + hive_jobq::Dep::Node { id, when } if *id == set_wanted => Some(*when), + _ => None, + }) + .expect("the deploy waits for the declaration"); + assert!( + !when.accepts(hive_jobq::TerminalState::Failed), + "a declaration that never landed must not be deployed past; this \ + edge has to be `after_ok`, not `after_any`" + ); + } + /// The node carries the hive, and the viewer has to see it. The `data` /// match is an or-pattern on purpose (see its own comment), and this is /// the assertion that the new variant joined the two-field arm rather From 48aa7a1e7922f1fc2ae79bcd7143a59f31447736 Mon Sep 17 00:00:00 2001 From: iris Date: Fri, 18 Sep 2026 18:09:47 +0200 Subject: [PATCH 2/5] swarm-controller: let creating an agent reuse a destroyed name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review caught a path the new declaration node breaks: an operator may already declare an agent `Destroyed` over the per-agent state endpoint, and `apply` then refuses any transition off that state. Before the wanted-state node existed the refusal was inert at creation time, but now creating an agent under a previously-destroyed name builds its identity, repo and config, fails the declaration, and silently cancels the deploy — while the caller sees a 200 and a job id. `AgentState::Destroyed`'s own doc comment already sanctions this case ("no state that brings a destroyed agent back short of a fresh deploy"); nothing implemented it. Give `apply` an `Intent`, keep the refusal for redeclares, and add `WantedWriter::create` for the one caller that is a fresh deploy. A separate method rather than a parameter on `set`, so no other caller can reach the override by passing an argument wrong. --- swarm-controller/src/main.rs | 17 ++-- swarm-controller/src/wanted.rs | 160 +++++++++++++++++++++++++++++---- 2 files changed, 152 insertions(+), 25 deletions(-) diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 609fd7ab..f394f011 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -340,7 +340,12 @@ async fn declare_new_agent( ) -> hive_jobq::scheduler::Outcome { use hive_jobq::scheduler::Outcome; - match writer.set(hive, agent, NEW_AGENT_WANTED_STATE).await { + // `create`, not `set`: a name that was destroyed earlier still carries a + // terminal entry, and recreating the agent is the fresh deploy that + // entry's own doc comment names as the way back. `set` would refuse it, + // cancelling the deploy of an agent whose identity, repo and config the + // swarm has just built. + match writer.create(hive, agent, NEW_AGENT_WANTED_STATE).await { Ok(_) => Outcome::Done, Err(e) => Outcome::Failed(format!("{e:#}")), } @@ -1387,11 +1392,11 @@ fn declare_agent_job( // unconfigured option into an agent nobody runs. // // `after_ok` on the declaration, though: an agent deployed without its - // pause landing first is the race this node exists to close, and the - // only way the declaration fails without a live queue is a host that has - // no queue at all — on which `TriggerDeploy` has nothing to publish to - // either, so the strict edge cancels a node that could not have - // succeeded. + // pause landing first is the race this node exists to close, so a + // declaration that did not land must not be deployed past. A host with no + // queue at all fails this node — but `TriggerDeploy` has nothing to + // publish to on such a host either, so the strict edge cancels a node + // that could not have succeeded anyway. let _trigger_deploy = b .node(SwarmNodeKind::TriggerDeploy { hive: hive.to_owned(), diff --git a/swarm-controller/src/wanted.rs b/swarm-controller/src/wanted.rs index 6a4d0863..4d5def90 100644 --- a/swarm-controller/src/wanted.rs +++ b/swarm-controller/src/wanted.rs @@ -57,6 +57,28 @@ enum Wrote { Failed(anyhow::Error), } +/// Why a caller is declaring a state, which is what decides whether a +/// `Destroyed` entry stands in its way. +/// +/// The distinction is the one [`AgentState::Destroyed`]'s own doc comment +/// already promises — "no state that brings a destroyed agent back **short of +/// a fresh deploy**". Redeclaring is not a fresh deploy and must keep being +/// refused; creating the agent again *is* one, and the whole of its identity, +/// repo and config has just been built anew alongside this declaration. +/// +/// An enum rather than a bool because the two call sites read as opposites +/// only if the argument says which one it is — `apply(.., true)` at the create +/// path would be a flag nobody can check at a glance. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Intent { + /// An operator (or anything else) changing what an existing agent should + /// be doing. Cannot move off `Destroyed`. + Redeclare, + /// The agent is being created from nothing — a fresh deploy. Replaces a + /// `Destroyed` entry left behind by an agent that had this name before. + Create, +} + /// How many times a losing writer re-reads and re-applies before giving up. /// /// A conflict means another writer changed a *different* agent between this @@ -75,7 +97,15 @@ const MAX_ATTEMPTS: usize = 5; /// value is an **error**, never treated as absent: overwriting a document /// nobody can read discards the declarations of every other agent on that /// hive, which is exactly what a fresh-start fallback would do quietly. -fn apply(current: Option<&[u8]>, agent: &str, state: AgentState) -> Result<(HiveWanted, Vec)> { +/// +/// `intent` decides whether an existing `Destroyed` entry for this agent is a +/// wall or something to write over — see [`Intent`]. +fn apply( + current: Option<&[u8]>, + agent: &str, + state: AgentState, + intent: Intent, +) -> Result<(HiveWanted, Vec)> { let mut declaration = match current { Some(raw) => serde_json::from_slice::(raw) .context("the hive's current declaration is not decodable")?, @@ -84,8 +114,11 @@ fn apply(current: Option<&[u8]>, agent: &str, state: AgentState) -> Result<(Hive // Redeclaring the same terminal state is a no-op, not a transition — a // client that already sees the agent as destroyed and asks again should // not be punished for it. Anything else moving off `Destroyed` is the - // one transition this module exists to refuse. - if let Some(existing) = declaration.agents.get(agent) + // one transition this module exists to refuse — unless the caller is + // creating the agent from nothing, which is the "fresh deploy" carve-out + // `Intent` documents. + if intent == Intent::Redeclare + && let Some(existing) = declaration.agents.get(agent) && existing.state == AgentState::Destroyed && state != AgentState::Destroyed { @@ -149,15 +182,47 @@ impl WantedWriter { .with_context(|| format!("the declaration for {hive} is not decodable")) } - /// Declare `agent` on `hive` to be in `state`, and return the whole + /// Redeclare `agent` on `hive` to be in `state`, and return the whole /// declaration as published. /// + /// Refuses to move an agent off `Destroyed` — see + /// [`create`](Self::create) for the one caller that may. + pub async fn set(&self, hive: &str, agent: &str, state: AgentState) -> Result { + self.write(hive, agent, state, Intent::Redeclare).await + } + + /// Declare a *newly created* `agent` on `hive`, and return the whole + /// declaration as published. + /// + /// Same write as [`set`](Self::set) but with [`Intent::Create`], so a + /// `Destroyed` entry left by an earlier agent of this name does not + /// refuse it. Recreating a destroyed name is precisely the "fresh deploy" + /// [`AgentState::Destroyed`]'s doc comment names as the way back, and by + /// the time this runs the swarm has already built that agent's identity, + /// repo and config from nothing. + /// + /// A separate method rather than an `Intent` parameter on `set`: the + /// caller that may overwrite a terminal state is the agent-creation job + /// node and nothing else, and an argument every other caller has to pass + /// correctly is an argument one of them eventually passes wrong. + pub async fn create(&self, hive: &str, agent: &str, state: AgentState) -> Result { + self.write(hive, agent, state, Intent::Create).await + } + + /// The write both of the above are. + /// /// Read-modify-write against the entry's revision rather than a plain /// `put`: the value is the hive's whole agent map, so a blind write /// would drop a concurrent change to a different agent. The bucket keeps /// one revision of history, so a lost write is not recoverable after the /// fact — the conflict has to be caught here. - pub async fn set(&self, hive: &str, agent: &str, state: AgentState) -> Result { + async fn write( + &self, + hive: &str, + agent: &str, + state: AgentState, + intent: Intent, + ) -> Result { swarm_queue_client::ensure_connected(&self.client)?; let store = self.store(hive).await?; @@ -167,8 +232,12 @@ impl WantedWriter { .await .with_context(|| format!("reading the declaration for {hive}"))?; let revision = entry.as_ref().map(|e| e.revision); - let (declaration, encoded) = - apply(entry.as_ref().map(|e| e.value.as_ref()), agent, state)?; + let (declaration, encoded) = apply( + entry.as_ref().map(|e| e.value.as_ref()), + agent, + state, + intent, + )?; // `update` and `create` have separate error types, and only one // variant of each means "someone else got there first". Every @@ -209,13 +278,24 @@ impl WantedWriter { #[cfg(test)] mod tests { - use super::apply; - use swarm_queue_client::wanted::AgentState; + use super::{Intent, apply}; + use swarm_queue_client::wanted::{AgentState, HiveWanted}; + + /// Every case below that predates the create carve-out is a redeclare; + /// spelling the intent out in each of them would say nothing the test + /// name doesn't. The carve-out's own tests call `apply` directly. + fn redeclare( + current: Option<&[u8]>, + agent: &str, + state: AgentState, + ) -> anyhow::Result<(HiveWanted, Vec)> { + apply(current, agent, state, Intent::Redeclare) + } #[test] fn declaring_one_agent_preserves_every_other() { let current = br#"{"agents":{"iris":{"state":"up"},"argus":{"state":"offline"}}}"#; - let (declaration, _) = apply(Some(current), "atlas", AgentState::Up).unwrap(); + let (declaration, _) = redeclare(Some(current), "atlas", AgentState::Up).unwrap(); assert_eq!(declaration.agents.len(), 3); assert_eq!(declaration.agents["iris"].state, AgentState::Up); assert_eq!(declaration.agents["argus"].state, AgentState::Offline); @@ -225,7 +305,7 @@ mod tests { #[test] fn redeclaring_an_agent_replaces_only_its_own_state() { let current = br#"{"agents":{"iris":{"state":"up"},"atlas":{"state":"up"}}}"#; - let (declaration, _) = apply(Some(current), "atlas", AgentState::Offline).unwrap(); + let (declaration, _) = redeclare(Some(current), "atlas", AgentState::Offline).unwrap(); assert_eq!(declaration.agents.len(), 2); assert_eq!(declaration.agents["iris"].state, AgentState::Up); assert_eq!(declaration.agents["atlas"].state, AgentState::Offline); @@ -233,7 +313,7 @@ mod tests { #[test] fn a_hive_with_no_declaration_yet_gets_a_one_agent_one() { - let (declaration, _) = apply(None, "atlas", AgentState::Up).unwrap(); + let (declaration, _) = redeclare(None, "atlas", AgentState::Up).unwrap(); assert_eq!(declaration.agents.len(), 1); assert_eq!(declaration.agents["atlas"].state, AgentState::Up); } @@ -242,7 +322,7 @@ mod tests { // here would publish a one-agent document over a hive's whole set. #[test] fn an_undecodable_declaration_is_an_error_not_a_fresh_start() { - let err = apply(Some(b"{not json"), "atlas", AgentState::Up).unwrap_err(); + let err = redeclare(Some(b"{not json"), "atlas", AgentState::Up).unwrap_err(); assert!( err.to_string().contains("not decodable"), "unexpected error: {err}" @@ -252,13 +332,13 @@ mod tests { #[test] fn an_unknown_state_in_the_current_value_is_also_an_error() { let current = br#"{"agents":{"iris":{"state":"sideways"}}}"#; - assert!(apply(Some(current), "atlas", AgentState::Up).is_err()); + assert!(redeclare(Some(current), "atlas", AgentState::Up).is_err()); } #[test] fn the_encoded_form_round_trips() { - let (_, encoded) = apply(None, "atlas", AgentState::Offline).unwrap(); - let (again, _) = apply(Some(&encoded), "iris", AgentState::Up).unwrap(); + let (_, encoded) = redeclare(None, "atlas", AgentState::Offline).unwrap(); + let (again, _) = redeclare(Some(&encoded), "iris", AgentState::Up).unwrap(); assert_eq!(again.agents["atlas"].state, AgentState::Offline); assert_eq!(again.agents["iris"].state, AgentState::Up); } @@ -270,7 +350,7 @@ mod tests { fn moving_a_destroyed_agent_to_any_other_state_is_refused() { let current = br#"{"agents":{"atlas":{"state":"destroyed"}}}"#; for state in [AgentState::Up, AgentState::Offline, AgentState::Paused] { - let err = apply(Some(current), "atlas", state).unwrap_err(); + let err = redeclare(Some(current), "atlas", state).unwrap_err(); assert!( err.downcast_ref::().is_some(), "expected a TerminalStateError for {state:?}, got: {err}" @@ -284,7 +364,7 @@ mod tests { #[test] fn redeclaring_destroyed_as_destroyed_is_not_a_transition() { let current = br#"{"agents":{"atlas":{"state":"destroyed"}}}"#; - let (declaration, _) = apply(Some(current), "atlas", AgentState::Destroyed).unwrap(); + let (declaration, _) = redeclare(Some(current), "atlas", AgentState::Destroyed).unwrap(); assert_eq!(declaration.agents["atlas"].state, AgentState::Destroyed); } @@ -293,8 +373,50 @@ mod tests { #[test] fn a_destroyed_agent_does_not_block_declaring_a_different_one() { let current = br#"{"agents":{"atlas":{"state":"destroyed"},"iris":{"state":"up"}}}"#; - let (declaration, _) = apply(Some(current), "iris", AgentState::Offline).unwrap(); + let (declaration, _) = redeclare(Some(current), "iris", AgentState::Offline).unwrap(); assert_eq!(declaration.agents["atlas"].state, AgentState::Destroyed); assert_eq!(declaration.agents["iris"].state, AgentState::Offline); } + + /// The carve-out `AgentState::Destroyed`'s doc comment already promised + /// and nothing implemented: a *fresh deploy* does bring a destroyed name + /// back. Without this, creating an agent under a name that was destroyed + /// earlier builds its identity, repo and config and then fails the + /// declaration — cancelling the deploy while the operator sees a 200. + #[test] + fn creating_an_agent_may_reuse_a_destroyed_name() { + let current = br#"{"agents":{"atlas":{"state":"destroyed"}}}"#; + let (declaration, _) = + apply(Some(current), "atlas", AgentState::Paused, Intent::Create).unwrap(); + assert_eq!(declaration.agents["atlas"].state, AgentState::Paused); + } + + /// The carve-out is scoped to the agent being created — recreating one + /// name must not disturb another destroyed agent on the same hive. + #[test] + fn creating_an_agent_leaves_other_destroyed_agents_destroyed() { + let current = br#"{"agents":{"atlas":{"state":"destroyed"},"iris":{"state":"destroyed"}}}"#; + let (declaration, _) = + apply(Some(current), "atlas", AgentState::Paused, Intent::Create).unwrap(); + assert_eq!(declaration.agents["atlas"].state, AgentState::Paused); + assert_eq!(declaration.agents["iris"].state, AgentState::Destroyed); + } + + /// `Intent::Create` relaxes exactly one rule. An undecodable current + /// value is still an error, for the same reason it is on a redeclare: + /// writing over it discards every other agent on the hive. + #[test] + fn creating_an_agent_still_refuses_an_undecodable_declaration() { + let err = apply( + Some(b"{not json"), + "atlas", + AgentState::Paused, + Intent::Create, + ) + .unwrap_err(); + assert!( + err.to_string().contains("not decodable"), + "unexpected error: {err}" + ); + } } From 75af27abe093523fa43f497e66169cae64e72551 Mon Sep 17 00:00:00 2001 From: iris Date: Fri, 18 Sep 2026 21:39:20 +0200 Subject: [PATCH 3/5] swarm-controller: make agent-creation's wanted-state write idempotent Intent::Create now no-ops when the agent already has a non-Destroyed declaration, instead of unconditionally overwriting it to the create path's state. Without this, re-running create against a name that already has a wanted-state entry (an operator migrating a pre-existing agent into this bookkeeping, or a retried request) would silently pause an agent already running under some other state. write() skips the network round-trip entirely when apply returns the declaration unchanged. mara: this does not match the expectation that agent creation is idempotent so pre existing agents can be migrated --- swarm-controller/src/wanted.rs | 58 ++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/swarm-controller/src/wanted.rs b/swarm-controller/src/wanted.rs index 4d5def90..a80eaf47 100644 --- a/swarm-controller/src/wanted.rs +++ b/swarm-controller/src/wanted.rs @@ -127,6 +127,25 @@ fn apply( } .into()); } + // `Create` against a name that already has a *non-terminal* declaration + // is not a creation — it's the create-agent endpoint being asked to + // declare a name that turns out to already exist (a retried request, or + // an operator migrating a pre-existing agent into this bookkeeping for + // the first time). Creation is expected to be idempotent, so this is a + // no-op that leaves the existing declaration exactly as it stood rather + // than stomping it back to whatever state the create path passes — + // otherwise calling create against a name that is already `Up` would + // silently pause a running agent. `Destroyed` is excluded on purpose: + // that's the one case a fresh create *does* need to write over (see + // `Intent::Create`'s own doc comment). + if intent == Intent::Create + && let Some(existing) = declaration.agents.get(agent) + && existing.state != AgentState::Destroyed + { + let encoded = + serde_json::to_vec(&declaration).context("encoding the current declaration")?; + return Ok((declaration, encoded)); + } declaration .agents .insert(agent.to_owned(), AgentWanted { state }); @@ -201,6 +220,12 @@ impl WantedWriter { /// the time this runs the swarm has already built that agent's identity, /// repo and config from nothing. /// + /// Idempotent against any other existing declaration: an agent already + /// declared in a non-terminal state (an operator migrating a pre-existing + /// agent into this bookkeeping, or a retried request) is left exactly as + /// it stood — creation must not silently pause an agent already running + /// under some other wanted state. + /// /// A separate method rather than an `Intent` parameter on `set`: the /// caller that may overwrite a terminal state is the agent-creation job /// node and nothing else, and an argument every other caller has to pass @@ -239,6 +264,15 @@ impl WantedWriter { intent, )?; + // `apply`'s `Intent::Create` no-op (an already-declared, + // non-terminal agent) re-encodes the declaration unchanged — + // skip the network round-trip for it entirely rather than + // writing identical bytes back. + if entry.as_ref().map(|e| e.value.as_ref()) == Some(encoded.as_slice()) { + tracing::debug!(hive, agent, "declaration already current, no write needed"); + return Ok(declaration); + } + // `update` and `create` have separate error types, and only one // variant of each means "someone else got there first". Every // other failure returns immediately: retrying a disconnect or a @@ -402,6 +436,30 @@ mod tests { assert_eq!(declaration.agents["iris"].state, AgentState::Destroyed); } + /// mara: "this does not match the expectation that agent creation is + /// idempotent so pre existing agents can be migrated" — an agent already + /// declared in some other, non-terminal state (say an operator migrating + /// a pre-existing agent into this bookkeeping) must not be stomped back + /// to whatever state the create path passes. + #[test] + fn creating_an_agent_does_not_disturb_an_existing_non_terminal_declaration() { + let current = br#"{"agents":{"atlas":{"state":"up"}}}"#; + let (declaration, _) = + apply(Some(current), "atlas", AgentState::Paused, Intent::Create).unwrap(); + assert_eq!(declaration.agents["atlas"].state, AgentState::Up); + } + + /// The idempotent no-op only inspects the agent being created — it must + /// not touch any other agent's declaration on the same hive. + #[test] + fn creating_an_agent_that_already_exists_leaves_every_other_agent_alone() { + let current = br#"{"agents":{"atlas":{"state":"up"},"iris":{"state":"offline"}}}"#; + let (declaration, _) = + apply(Some(current), "atlas", AgentState::Paused, Intent::Create).unwrap(); + assert_eq!(declaration.agents["atlas"].state, AgentState::Up); + assert_eq!(declaration.agents["iris"].state, AgentState::Offline); + } + /// `Intent::Create` relaxes exactly one rule. An undecodable current /// value is still an error, for the same reason it is on a redeclare: /// writing over it discards every other agent on the hive. From c908a71f3ecd4cce20f4d6384244c300764c5be8 Mon Sep 17 00:00:00 2001 From: iris Date: Fri, 18 Sep 2026 22:34:15 +0200 Subject: [PATCH 4/5] swarm-controller: drop the Intent split, one unconditional wanted-state write mara: "i dont want any logic differene between the two cases" and "do not refuse to recreate an agent". wanted.rs goes back to a single write path (set), with no terminal-state refusal at all, used identically by the pause/resume endpoint and the agent-creation job node. The agent-creation node's own idempotency requirement (re-running create against a name that already has a declaration must not silently pause it) now lives entirely in declare_new_agent: it reads the current declaration first and only writes Paused when the agent has no entry, or its entry is Destroyed (recreating a previously- destroyed name is the fresh deploy that state's own doc comment names as the way back). --- swarm-controller/src/main.rs | 49 ++++-- swarm-controller/src/wanted.rs | 306 ++++++--------------------------- 2 files changed, 86 insertions(+), 269 deletions(-) diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index f394f011..98139a21 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -327,25 +327,51 @@ async fn run_swarm_node( (builder, outcome) } -/// Declare a brand-new agent at [`NEW_AGENT_WANTED_STATE`]. +/// Declare a brand-new agent at [`NEW_AGENT_WANTED_STATE`] — unless it turns +/// out not to be new. /// /// A named function beside [`publish_deploy`] rather than the two lines /// inline, for the same reason that one is: `run_swarm_node`'s match is a /// per-variant index, and every arm that grows a body past a call pushes the /// next reader further from the variant they came to read. +/// +/// Reads the current declaration first and only calls +/// [`WantedWriter::set`](wanted::WantedWriter::set) when this agent has no +/// entry yet — `wanted.rs` itself no longer distinguishes creation from any +/// other write (mara: "i dont want any logic difference between the two +/// cases"), so the idempotency this node needs — calling create against a +/// name that already has a declaration (an operator migrating a pre-existing +/// agent into this bookkeeping, or a retried request) must not silently +/// pause an agent already running under some other state — lives entirely +/// here, the one caller it applies to. An existing `Destroyed` entry counts +/// as "no entry" for this check and gets written over: recreating a +/// previously-destroyed name is exactly the fresh deploy that state's own +/// doc comment names as the way back, and mara separately ruled "do not +/// refuse to recreate an agent". async fn declare_new_agent( writer: &wanted::WantedWriter, hive: &str, agent: &str, ) -> hive_jobq::scheduler::Outcome { use hive_jobq::scheduler::Outcome; + use swarm_queue_client::wanted::AgentState; - // `create`, not `set`: a name that was destroyed earlier still carries a - // terminal entry, and recreating the agent is the fresh deploy that - // entry's own doc comment names as the way back. `set` would refuse it, - // cancelling the deploy of an agent whose identity, repo and config the - // swarm has just built. - match writer.create(hive, agent, NEW_AGENT_WANTED_STATE).await { + let existing = match writer.view(hive).await { + Ok(declaration) => declaration.and_then(|d| d.agents.get(agent).map(|a| a.state)), + Err(e) => return Outcome::Failed(format!("{e:#}")), + }; + if let Some(state) = existing + && state != AgentState::Destroyed + { + tracing::debug!( + hive, + agent, + ?state, + "agent already declared, leaving its wanted state alone" + ); + return Outcome::Done; + } + match writer.set(hive, agent, NEW_AGENT_WANTED_STATE).await { Ok(_) => Outcome::Done, Err(e) => Outcome::Failed(format!("{e:#}")), } @@ -894,7 +920,6 @@ fn declaration_target( responses( (status = 200, description = "the declaration as now published", body = Vec), (status = 400, description = "a name is not an identifier, the hive is not in this swarm, or the state is unknown (problem+json)", body = String), - (status = 409, description = "the agent is already declared destroyed, a terminal state the request tries to move it off of (problem+json)", body = String), (status = 503, description = "no swarm queue is wired up (problem+json)", body = String), (status = 500, description = "the declaration could not be published (problem+json)", body = String), ), @@ -912,14 +937,6 @@ async fn set_agent_state( .into_string(); let declaration = writer.set(&hive, &agent, req.state).await.map_err(|e| { - // A `TerminalStateError` is the caller's mistake, but not a malformed - // request — the request is well-formed, it just conflicts with the - // target resource's current (terminal) state, which per RFC 9110 is - // what 409 exists for, not 400. Everything else here is the - // pre-existing catch-all. - if let Some(terminal) = e.downcast_ref::() { - return error_problem(axum::http::StatusCode::CONFLICT, &terminal.to_string()); - } tracing::warn!(hive = %hive, agent = %agent, error = %format!("{e:#}"), "declaring agent state failed"); error_problem( axum::http::StatusCode::INTERNAL_SERVER_ERROR, diff --git a/swarm-controller/src/wanted.rs b/swarm-controller/src/wanted.rs index a80eaf47..4f0d4cd7 100644 --- a/swarm-controller/src/wanted.rs +++ b/swarm-controller/src/wanted.rs @@ -12,42 +12,25 @@ //! The bucket is the record. Nothing here keeps a second copy of the //! declaration to reconcile against, because the current value can be read //! back from the queue whenever it is needed. - -use std::fmt; +//! +//! **One write path, no per-caller behaviour.** An earlier revision of this +//! module carried an `Intent` enum that let the agent-creation job node +//! write over a `Destroyed` entry while the ordinary `PUT .../state` +//! endpoint could not — mara's ruling on that: "i dont want any logic +//! difference between the two cases" (plus "do not refuse to recreate an +//! agent", which the old refusal made impossible for *either* caller). So +//! [`WantedWriter::set`] just writes whatever it is told, unconditionally, +//! for every caller alike. A caller that wants "don't touch an agent that +//! already has a declaration" (the agent-creation node's own idempotency +//! requirement — recreating an already-`Up` agent must not silently pause +//! it) makes that decision itself, by reading the current declaration via +//! [`WantedWriter::view`] first — that policy lives with the one caller it +//! applies to, not inside this module's shared write. use anyhow::{Context, Result}; use async_nats::jetstream::kv::{CreateErrorKind, UpdateErrorKind}; use swarm_queue_client::wanted::{AgentState, AgentWanted, HiveWanted}; -/// The one failure out of [`apply`] that is the caller's mistake, not a -/// server fault — worth a distinct type so `set_agent_state` can tell it -/// apart from every other error this module raises (an undecodable -/// declaration, a lost write) and answer 400 instead of 500. -/// -/// Carried through as a plain `anyhow::Error` like every other error here -/// (this crate has no typed-error convention to join), and recovered at the -/// HTTP boundary with `downcast_ref` — the shape every other "kind of error -/// that came from underneath" check in this module already uses -/// ([`Wrote`]'s `UpdateErrorKind`/`CreateErrorKind` matching), just inspecting -/// an `anyhow::Error` instead of a client library's own error enum. -#[derive(Debug)] -pub struct TerminalStateError { - agent: String, -} - -impl fmt::Display for TerminalStateError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "{} is already declared destroyed — terminal, per AgentState::Destroyed's own doc \ - comment: no transition brings it back short of a fresh deploy", - self.agent - ) - } -} - -impl std::error::Error for TerminalStateError {} - /// The three outcomes of one write attempt, which the two KV verbs report /// through separate error types. enum Wrote { @@ -57,28 +40,6 @@ enum Wrote { Failed(anyhow::Error), } -/// Why a caller is declaring a state, which is what decides whether a -/// `Destroyed` entry stands in its way. -/// -/// The distinction is the one [`AgentState::Destroyed`]'s own doc comment -/// already promises — "no state that brings a destroyed agent back **short of -/// a fresh deploy**". Redeclaring is not a fresh deploy and must keep being -/// refused; creating the agent again *is* one, and the whole of its identity, -/// repo and config has just been built anew alongside this declaration. -/// -/// An enum rather than a bool because the two call sites read as opposites -/// only if the argument says which one it is — `apply(.., true)` at the create -/// path would be a flag nobody can check at a glance. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Intent { - /// An operator (or anything else) changing what an existing agent should - /// be doing. Cannot move off `Destroyed`. - Redeclare, - /// The agent is being created from nothing — a fresh deploy. Replaces a - /// `Destroyed` entry left behind by an agent that had this name before. - Create, -} - /// How many times a losing writer re-reads and re-applies before giving up. /// /// A conflict means another writer changed a *different* agent between this @@ -98,54 +59,14 @@ const MAX_ATTEMPTS: usize = 5; /// nobody can read discards the declarations of every other agent on that /// hive, which is exactly what a fresh-start fallback would do quietly. /// -/// `intent` decides whether an existing `Destroyed` entry for this agent is a -/// wall or something to write over — see [`Intent`]. -fn apply( - current: Option<&[u8]>, - agent: &str, - state: AgentState, - intent: Intent, -) -> Result<(HiveWanted, Vec)> { +/// Unconditional otherwise — see the module doc for why there is no +/// terminal-state refusal here any more. +fn apply(current: Option<&[u8]>, agent: &str, state: AgentState) -> Result<(HiveWanted, Vec)> { let mut declaration = match current { Some(raw) => serde_json::from_slice::(raw) .context("the hive's current declaration is not decodable")?, None => HiveWanted::default(), }; - // Redeclaring the same terminal state is a no-op, not a transition — a - // client that already sees the agent as destroyed and asks again should - // not be punished for it. Anything else moving off `Destroyed` is the - // one transition this module exists to refuse — unless the caller is - // creating the agent from nothing, which is the "fresh deploy" carve-out - // `Intent` documents. - if intent == Intent::Redeclare - && let Some(existing) = declaration.agents.get(agent) - && existing.state == AgentState::Destroyed - && state != AgentState::Destroyed - { - return Err(TerminalStateError { - agent: agent.to_owned(), - } - .into()); - } - // `Create` against a name that already has a *non-terminal* declaration - // is not a creation — it's the create-agent endpoint being asked to - // declare a name that turns out to already exist (a retried request, or - // an operator migrating a pre-existing agent into this bookkeeping for - // the first time). Creation is expected to be idempotent, so this is a - // no-op that leaves the existing declaration exactly as it stood rather - // than stomping it back to whatever state the create path passes — - // otherwise calling create against a name that is already `Up` would - // silently pause a running agent. `Destroyed` is excluded on purpose: - // that's the one case a fresh create *does* need to write over (see - // `Intent::Create`'s own doc comment). - if intent == Intent::Create - && let Some(existing) = declaration.agents.get(agent) - && existing.state != AgentState::Destroyed - { - let encoded = - serde_json::to_vec(&declaration).context("encoding the current declaration")?; - return Ok((declaration, encoded)); - } declaration .agents .insert(agent.to_owned(), AgentWanted { state }); @@ -201,53 +122,21 @@ impl WantedWriter { .with_context(|| format!("the declaration for {hive} is not decodable")) } - /// Redeclare `agent` on `hive` to be in `state`, and return the whole + /// Declare `agent` on `hive` to be in `state`, and return the whole /// declaration as published. /// - /// Refuses to move an agent off `Destroyed` — see - /// [`create`](Self::create) for the one caller that may. - pub async fn set(&self, hive: &str, agent: &str, state: AgentState) -> Result { - self.write(hive, agent, state, Intent::Redeclare).await - } - - /// Declare a *newly created* `agent` on `hive`, and return the whole - /// declaration as published. - /// - /// Same write as [`set`](Self::set) but with [`Intent::Create`], so a - /// `Destroyed` entry left by an earlier agent of this name does not - /// refuse it. Recreating a destroyed name is precisely the "fresh deploy" - /// [`AgentState::Destroyed`]'s doc comment names as the way back, and by - /// the time this runs the swarm has already built that agent's identity, - /// repo and config from nothing. - /// - /// Idempotent against any other existing declaration: an agent already - /// declared in a non-terminal state (an operator migrating a pre-existing - /// agent into this bookkeeping, or a retried request) is left exactly as - /// it stood — creation must not silently pause an agent already running - /// under some other wanted state. - /// - /// A separate method rather than an `Intent` parameter on `set`: the - /// caller that may overwrite a terminal state is the agent-creation job - /// node and nothing else, and an argument every other caller has to pass - /// correctly is an argument one of them eventually passes wrong. - pub async fn create(&self, hive: &str, agent: &str, state: AgentState) -> Result { - self.write(hive, agent, state, Intent::Create).await - } - - /// The write both of the above are. + /// Unconditional — see the module doc. Every caller (the pause/resume + /// endpoint, the agent-creation job node, anything else) gets the exact + /// same write; a caller that needs to decide *whether* to call this at + /// all (idempotency, terminal-state handling, …) makes that call itself + /// by reading [`view`](Self::view) first. /// /// Read-modify-write against the entry's revision rather than a plain /// `put`: the value is the hive's whole agent map, so a blind write /// would drop a concurrent change to a different agent. The bucket keeps /// one revision of history, so a lost write is not recoverable after the /// fact — the conflict has to be caught here. - async fn write( - &self, - hive: &str, - agent: &str, - state: AgentState, - intent: Intent, - ) -> Result { + pub async fn set(&self, hive: &str, agent: &str, state: AgentState) -> Result { swarm_queue_client::ensure_connected(&self.client)?; let store = self.store(hive).await?; @@ -257,17 +146,16 @@ impl WantedWriter { .await .with_context(|| format!("reading the declaration for {hive}"))?; let revision = entry.as_ref().map(|e| e.revision); - let (declaration, encoded) = apply( - entry.as_ref().map(|e| e.value.as_ref()), - agent, - state, - intent, - )?; + let (declaration, encoded) = + apply(entry.as_ref().map(|e| e.value.as_ref()), agent, state)?; - // `apply`'s `Intent::Create` no-op (an already-declared, - // non-terminal agent) re-encodes the declaration unchanged — - // skip the network round-trip for it entirely rather than - // writing identical bytes back. + // A redeclare of the identical state re-encodes the declaration + // unchanged (a caller that already sees this state and asks + // again is not doing anything) — skip the network round-trip for + // it entirely rather than writing identical bytes back. + // `HiveWanted::agents` is a `BTreeMap`, so this byte comparison + // is deterministic, not vulnerable to a map re-ordering itself + // between reads. if entry.as_ref().map(|e| e.value.as_ref()) == Some(encoded.as_slice()) { tracing::debug!(hive, agent, "declaration already current, no write needed"); return Ok(declaration); @@ -312,24 +200,13 @@ impl WantedWriter { #[cfg(test)] mod tests { - use super::{Intent, apply}; - use swarm_queue_client::wanted::{AgentState, HiveWanted}; - - /// Every case below that predates the create carve-out is a redeclare; - /// spelling the intent out in each of them would say nothing the test - /// name doesn't. The carve-out's own tests call `apply` directly. - fn redeclare( - current: Option<&[u8]>, - agent: &str, - state: AgentState, - ) -> anyhow::Result<(HiveWanted, Vec)> { - apply(current, agent, state, Intent::Redeclare) - } + use super::apply; + use swarm_queue_client::wanted::AgentState; #[test] fn declaring_one_agent_preserves_every_other() { let current = br#"{"agents":{"iris":{"state":"up"},"argus":{"state":"offline"}}}"#; - let (declaration, _) = redeclare(Some(current), "atlas", AgentState::Up).unwrap(); + let (declaration, _) = apply(Some(current), "atlas", AgentState::Up).unwrap(); assert_eq!(declaration.agents.len(), 3); assert_eq!(declaration.agents["iris"].state, AgentState::Up); assert_eq!(declaration.agents["argus"].state, AgentState::Offline); @@ -339,7 +216,7 @@ mod tests { #[test] fn redeclaring_an_agent_replaces_only_its_own_state() { let current = br#"{"agents":{"iris":{"state":"up"},"atlas":{"state":"up"}}}"#; - let (declaration, _) = redeclare(Some(current), "atlas", AgentState::Offline).unwrap(); + let (declaration, _) = apply(Some(current), "atlas", AgentState::Offline).unwrap(); assert_eq!(declaration.agents.len(), 2); assert_eq!(declaration.agents["iris"].state, AgentState::Up); assert_eq!(declaration.agents["atlas"].state, AgentState::Offline); @@ -347,7 +224,7 @@ mod tests { #[test] fn a_hive_with_no_declaration_yet_gets_a_one_agent_one() { - let (declaration, _) = redeclare(None, "atlas", AgentState::Up).unwrap(); + let (declaration, _) = apply(None, "atlas", AgentState::Up).unwrap(); assert_eq!(declaration.agents.len(), 1); assert_eq!(declaration.agents["atlas"].state, AgentState::Up); } @@ -356,7 +233,7 @@ mod tests { // here would publish a one-agent document over a hive's whole set. #[test] fn an_undecodable_declaration_is_an_error_not_a_fresh_start() { - let err = redeclare(Some(b"{not json"), "atlas", AgentState::Up).unwrap_err(); + let err = apply(Some(b"{not json"), "atlas", AgentState::Up).unwrap_err(); assert!( err.to_string().contains("not decodable"), "unexpected error: {err}" @@ -366,115 +243,38 @@ mod tests { #[test] fn an_unknown_state_in_the_current_value_is_also_an_error() { let current = br#"{"agents":{"iris":{"state":"sideways"}}}"#; - assert!(redeclare(Some(current), "atlas", AgentState::Up).is_err()); + assert!(apply(Some(current), "atlas", AgentState::Up).is_err()); } #[test] fn the_encoded_form_round_trips() { - let (_, encoded) = redeclare(None, "atlas", AgentState::Offline).unwrap(); - let (again, _) = redeclare(Some(&encoded), "iris", AgentState::Up).unwrap(); + let (_, encoded) = apply(None, "atlas", AgentState::Offline).unwrap(); + let (again, _) = apply(Some(&encoded), "iris", AgentState::Up).unwrap(); assert_eq!(again.agents["atlas"].state, AgentState::Offline); assert_eq!(again.agents["iris"].state, AgentState::Up); } - /// The bug this module exists to fix: nothing used to stop a caller from - /// declaring a destroyed agent back to `Up`/`Offline`, silently - /// un-terminaling a state whose own doc comment says that is impossible. + /// No terminal-state refusal any more (mara: "do not refuse to recreate + /// an agent") — `apply` writes whatever it is asked to, including moving + /// a `Destroyed` agent to any other state. Whether that write should + /// happen at all is now the caller's own decision (see the module doc). #[test] - fn moving_a_destroyed_agent_to_any_other_state_is_refused() { + fn a_destroyed_agent_can_be_declared_into_any_other_state() { let current = br#"{"agents":{"atlas":{"state":"destroyed"}}}"#; for state in [AgentState::Up, AgentState::Offline, AgentState::Paused] { - let err = redeclare(Some(current), "atlas", state).unwrap_err(); - assert!( - err.downcast_ref::().is_some(), - "expected a TerminalStateError for {state:?}, got: {err}" - ); + let (declaration, _) = apply(Some(current), "atlas", state).unwrap(); + assert_eq!(declaration.agents["atlas"].state, state); } } - /// Redeclaring `Destroyed` while already `Destroyed` is a no-op, not a - /// transition — a client that already sees the terminal state and asks - /// again should get success back, not an error for stating a fact. - #[test] - fn redeclaring_destroyed_as_destroyed_is_not_a_transition() { - let current = br#"{"agents":{"atlas":{"state":"destroyed"}}}"#; - let (declaration, _) = redeclare(Some(current), "atlas", AgentState::Destroyed).unwrap(); - assert_eq!(declaration.agents["atlas"].state, AgentState::Destroyed); - } - - /// The refusal is per-agent — a destroyed agent on the same hive as a - /// live one must not block declaring the live one. + /// The write is per-agent — a destroyed agent on the same hive as a live + /// one must not block declaring the live one, and declaring one must not + /// disturb the other. #[test] fn a_destroyed_agent_does_not_block_declaring_a_different_one() { let current = br#"{"agents":{"atlas":{"state":"destroyed"},"iris":{"state":"up"}}}"#; - let (declaration, _) = redeclare(Some(current), "iris", AgentState::Offline).unwrap(); + let (declaration, _) = apply(Some(current), "iris", AgentState::Offline).unwrap(); assert_eq!(declaration.agents["atlas"].state, AgentState::Destroyed); assert_eq!(declaration.agents["iris"].state, AgentState::Offline); } - - /// The carve-out `AgentState::Destroyed`'s doc comment already promised - /// and nothing implemented: a *fresh deploy* does bring a destroyed name - /// back. Without this, creating an agent under a name that was destroyed - /// earlier builds its identity, repo and config and then fails the - /// declaration — cancelling the deploy while the operator sees a 200. - #[test] - fn creating_an_agent_may_reuse_a_destroyed_name() { - let current = br#"{"agents":{"atlas":{"state":"destroyed"}}}"#; - let (declaration, _) = - apply(Some(current), "atlas", AgentState::Paused, Intent::Create).unwrap(); - assert_eq!(declaration.agents["atlas"].state, AgentState::Paused); - } - - /// The carve-out is scoped to the agent being created — recreating one - /// name must not disturb another destroyed agent on the same hive. - #[test] - fn creating_an_agent_leaves_other_destroyed_agents_destroyed() { - let current = br#"{"agents":{"atlas":{"state":"destroyed"},"iris":{"state":"destroyed"}}}"#; - let (declaration, _) = - apply(Some(current), "atlas", AgentState::Paused, Intent::Create).unwrap(); - assert_eq!(declaration.agents["atlas"].state, AgentState::Paused); - assert_eq!(declaration.agents["iris"].state, AgentState::Destroyed); - } - - /// mara: "this does not match the expectation that agent creation is - /// idempotent so pre existing agents can be migrated" — an agent already - /// declared in some other, non-terminal state (say an operator migrating - /// a pre-existing agent into this bookkeeping) must not be stomped back - /// to whatever state the create path passes. - #[test] - fn creating_an_agent_does_not_disturb_an_existing_non_terminal_declaration() { - let current = br#"{"agents":{"atlas":{"state":"up"}}}"#; - let (declaration, _) = - apply(Some(current), "atlas", AgentState::Paused, Intent::Create).unwrap(); - assert_eq!(declaration.agents["atlas"].state, AgentState::Up); - } - - /// The idempotent no-op only inspects the agent being created — it must - /// not touch any other agent's declaration on the same hive. - #[test] - fn creating_an_agent_that_already_exists_leaves_every_other_agent_alone() { - let current = br#"{"agents":{"atlas":{"state":"up"},"iris":{"state":"offline"}}}"#; - let (declaration, _) = - apply(Some(current), "atlas", AgentState::Paused, Intent::Create).unwrap(); - assert_eq!(declaration.agents["atlas"].state, AgentState::Up); - assert_eq!(declaration.agents["iris"].state, AgentState::Offline); - } - - /// `Intent::Create` relaxes exactly one rule. An undecodable current - /// value is still an error, for the same reason it is on a redeclare: - /// writing over it discards every other agent on the hive. - #[test] - fn creating_an_agent_still_refuses_an_undecodable_declaration() { - let err = apply( - Some(b"{not json"), - "atlas", - AgentState::Paused, - Intent::Create, - ) - .unwrap_err(); - assert!( - err.to_string().contains("not decodable"), - "unexpected error: {err}" - ); - } } From 8aafe4eaee2347e68054b79aed63984f5888c4dc Mon Sep 17 00:00:00 2001 From: iris Date: Fri, 18 Sep 2026 22:42:05 +0200 Subject: [PATCH 5/5] swarm-controller: trim oversized comments, drop a trivial wrapper fn mara: "pls dont make comments longer than functions or fns that just call a single other fn". Trimmed wanted.rs's module doc, apply()'s doc, and declare_new_agent's doc down to what's non-obvious; deleted WantedWriter::store (a one-line call to open_or_create with a 10-line doc comment above it) and inlined its body into its two callers. --- swarm-controller/src/main.rs | 25 +++-------- swarm-controller/src/wanted.rs | 76 ++++++++-------------------------- 2 files changed, 23 insertions(+), 78 deletions(-) diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 98139a21..8f673991 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -328,26 +328,11 @@ async fn run_swarm_node( } /// Declare a brand-new agent at [`NEW_AGENT_WANTED_STATE`] — unless it turns -/// out not to be new. -/// -/// A named function beside [`publish_deploy`] rather than the two lines -/// inline, for the same reason that one is: `run_swarm_node`'s match is a -/// per-variant index, and every arm that grows a body past a call pushes the -/// next reader further from the variant they came to read. -/// -/// Reads the current declaration first and only calls -/// [`WantedWriter::set`](wanted::WantedWriter::set) when this agent has no -/// entry yet — `wanted.rs` itself no longer distinguishes creation from any -/// other write (mara: "i dont want any logic difference between the two -/// cases"), so the idempotency this node needs — calling create against a -/// name that already has a declaration (an operator migrating a pre-existing -/// agent into this bookkeeping, or a retried request) must not silently -/// pause an agent already running under some other state — lives entirely -/// here, the one caller it applies to. An existing `Destroyed` entry counts -/// as "no entry" for this check and gets written over: recreating a -/// previously-destroyed name is exactly the fresh deploy that state's own -/// doc comment names as the way back, and mara separately ruled "do not -/// refuse to recreate an agent". +/// out not to be new: an agent that already has a declaration (other than +/// `Destroyed`, which this treats as reusable) is left alone, so a retried +/// or migrating create is idempotent rather than pausing an already-running +/// agent. `wanted.rs` itself has no such carve-out, so this node makes the +/// call. async fn declare_new_agent( writer: &wanted::WantedWriter, hive: &str, diff --git a/swarm-controller/src/wanted.rs b/swarm-controller/src/wanted.rs index 4f0d4cd7..254ba5cd 100644 --- a/swarm-controller/src/wanted.rs +++ b/swarm-controller/src/wanted.rs @@ -13,19 +13,11 @@ //! declaration to reconcile against, because the current value can be read //! back from the queue whenever it is needed. //! -//! **One write path, no per-caller behaviour.** An earlier revision of this -//! module carried an `Intent` enum that let the agent-creation job node -//! write over a `Destroyed` entry while the ordinary `PUT .../state` -//! endpoint could not — mara's ruling on that: "i dont want any logic -//! difference between the two cases" (plus "do not refuse to recreate an -//! agent", which the old refusal made impossible for *either* caller). So -//! [`WantedWriter::set`] just writes whatever it is told, unconditionally, -//! for every caller alike. A caller that wants "don't touch an agent that -//! already has a declaration" (the agent-creation node's own idempotency -//! requirement — recreating an already-`Up` agent must not silently pause -//! it) makes that decision itself, by reading the current declaration via -//! [`WantedWriter::view`] first — that policy lives with the one caller it -//! applies to, not inside this module's shared write. +//! [`WantedWriter::set`] writes unconditionally, for every caller alike — no +//! per-caller `Intent`, no terminal-state refusal. A caller that needs to +//! decide *whether* to write at all (the agent-creation node's own +//! idempotency check) reads [`WantedWriter::view`] first and makes that call +//! itself. use anyhow::{Context, Result}; use async_nats::jetstream::kv::{CreateErrorKind, UpdateErrorKind}; @@ -50,17 +42,9 @@ const MAX_ATTEMPTS: usize = 5; /// Apply one agent's declared state to a hive's current declaration. /// -/// Split out from the write loop because it holds the invariant that matters: -/// the value under a hive's key is the map of **every** agent on that hive, -/// so declaring one agent must preserve the rest. -/// -/// `current` is `None` when the hive has no declaration yet. An undecodable -/// value is an **error**, never treated as absent: overwriting a document -/// nobody can read discards the declarations of every other agent on that -/// hive, which is exactly what a fresh-start fallback would do quietly. -/// -/// Unconditional otherwise — see the module doc for why there is no -/// terminal-state refusal here any more. +/// `current` is the hive's whole agent map — an undecodable value is an +/// **error**, not treated as absent, so a bad read never discards every +/// other agent's declaration under a fresh-start fallback. fn apply(current: Option<&[u8]>, agent: &str, state: AgentState) -> Result<(HiveWanted, Vec)> { let mut declaration = match current { Some(raw) => serde_json::from_slice::(raw) @@ -85,31 +69,15 @@ impl WantedWriter { Self { client } } - /// One hive's bucket handle, created on first use if nothing has made it - /// yet. - /// - /// Creation lives in [`swarm_queue_client::wanted`] because a bucket is - /// described identically by everyone who may create it. Only the - /// controller creates these; a hive opens its own read-only. - /// - /// Resolved per call rather than cached: there is one bucket per hive, so - /// a single cached handle cannot serve them, and the declarations this - /// writes change on operator action rather than on a loop — the extra - /// lookup is per *declaration*, not per tick. A cache here would be a map - /// whose invalidation nobody needs yet. - async fn store( - &self, - hive: &str, - ) -> std::result::Result { - swarm_queue_client::wanted::open_or_create(&self.client, hive).await - } - /// The declaration currently published for `hive`, or `None`. pub async fn view(&self, hive: &str) -> Result> { // An unconnected client does not fail a JetStream request, it hangs // on it — see `swarm_queue_client::ensure_connected`. swarm_queue_client::ensure_connected(&self.client)?; - let store = self.store(hive).await?; + // One bucket per hive, resolved per call, created on first use — see + // `swarm_queue_client::wanted` for why (only the controller creates + // these; a hive opens its own read-only). + let store = swarm_queue_client::wanted::open_or_create(&self.client, hive).await?; let Some(entry) = store .entry(hive) .await @@ -122,23 +90,15 @@ impl WantedWriter { .with_context(|| format!("the declaration for {hive} is not decodable")) } - /// Declare `agent` on `hive` to be in `state`, and return the whole - /// declaration as published. + /// Declare `agent` on `hive` to be in `state`, unconditionally (see the + /// module doc), and return the whole declaration as published. /// - /// Unconditional — see the module doc. Every caller (the pause/resume - /// endpoint, the agent-creation job node, anything else) gets the exact - /// same write; a caller that needs to decide *whether* to call this at - /// all (idempotency, terminal-state handling, …) makes that call itself - /// by reading [`view`](Self::view) first. - /// - /// Read-modify-write against the entry's revision rather than a plain - /// `put`: the value is the hive's whole agent map, so a blind write - /// would drop a concurrent change to a different agent. The bucket keeps - /// one revision of history, so a lost write is not recoverable after the - /// fact — the conflict has to be caught here. + /// Read-modify-write against the entry's revision, not a plain `put`: + /// the value is the hive's whole agent map, so a blind write would drop + /// a concurrent change to a different agent. pub async fn set(&self, hive: &str, agent: &str, state: AgentState) -> Result { swarm_queue_client::ensure_connected(&self.client)?; - let store = self.store(hive).await?; + let store = swarm_queue_client::wanted::open_or_create(&self.client, hive).await?; for _ in 0..MAX_ATTEMPTS { let entry = store