a hive-side pause holds against the swarm's declared state until an operator resumes

the harness writes the reason into the pause marker when it pauses itself and reports it as paused_by_hive in its turn-state header. swarm-controller declares such an agent paused (only if declared up) and shows the reason on the agents status row and the swarm ui card.

swarm-controller/src/wanted.rs

@@ -40,17 +40,23 @@ enum Wrote {
 /// and a caller that gets an error can ask again with fresh intent.
 const MAX_ATTEMPTS: usize = 5;
 
+/// A hive's current declaration; none yet reads as an empty one, an undecodable
+/// value as an error.
+fn decode(current: Option<&[u8]>) -> Result<HiveWanted> {
+    match current {
+        Some(raw) => serde_json::from_slice::<HiveWanted>(raw)
+            .context("the hive's current declaration is not decodable"),
+        None => Ok(HiveWanted::default()),
+    }
+}
+
 /// Apply one agent's declared state to a hive's current declaration.
 ///
 /// `current` is the hive's whole agent map — an undecodable value is an
 /// **error**, not treated as absent, so a bad read never discards every
 /// other agent's declaration under a fresh-start fallback.
 fn apply(current: Option<&[u8]>, agent: &str, state: AgentState) -> Result<(HiveWanted, Vec<u8>)> {
-    let mut declaration = match current {
-        Some(raw) => serde_json::from_slice::<HiveWanted>(raw)
-            .context("the hive's current declaration is not decodable")?,
-        None => HiveWanted::default(),
-    };
+    let mut declaration = decode(current)?;
     declaration
         .agents
         .insert(agent.to_owned(), AgentWanted { state });
@@ -98,11 +104,37 @@ impl WantedWriter {
 
     /// Declare `agent` on `hive` to be in `state`, unconditionally (see the
     /// module doc), and return the whole declaration as published.
+    pub async fn set(&self, hive: &str, agent: &str, state: AgentState) -> Result<HiveWanted> {
+        self.write(hive, agent, state, None)
+            .await
+            .map(|(declaration, _)| declaration)
+    }
+
+    /// Declare `agent` on `hive` `Paused` if, and only if, the declaration at
+    /// the moment of the write says `Up`: an operator who has since set the
+    /// agent offline, destroyed or already paused is not overruled. `true`
+    /// when the declaration now says `Paused` because of this call.
+    pub async fn pause_if_up(&self, hive: &str, agent: &str) -> Result<bool> {
+        self.write(hive, agent, AgentState::Paused, Some(AgentState::Up))
+            .await
+            .map(|(_, applied)| applied)
+    }
+
+    /// The write behind [`Self::set`] and [`Self::pause_if_up`]: the
+    /// declaration as published and whether `state` was written. With
+    /// `only_if`, the write happens only while the agent is declared in that
+    /// state.
     ///
     /// Read-modify-write against the entry's revision, not a plain `put`:
     /// the value is the hive's whole agent map, so a blind write would drop
     /// a concurrent change to a different agent.
-    pub async fn set(&self, hive: &str, agent: &str, state: AgentState) -> Result<HiveWanted> {
+    async fn write(
+        &self,
+        hive: &str,
+        agent: &str,
+        state: AgentState,
+        only_if: Option<AgentState>,
+    ) -> Result<(HiveWanted, bool)> {
         swarm_queue_client::ensure_connected(&self.client)?;
         let store = swarm_queue_client::wanted::open_or_create(&self.client, hive).await?;
 
@@ -112,8 +144,14 @@ impl WantedWriter {
                 .await
                 .with_context(|| format!("reading the declaration for {hive}"))?;
             let revision = entry.as_ref().map(|e| e.revision);
-            let (declaration, encoded) =
-                apply(entry.as_ref().map(|e| e.value.as_ref()), agent, state)?;
+            let current = entry.as_ref().map(|e| e.value.as_ref());
+            if let Some(from) = only_if {
+                let declaration = decode(current)?;
+                if declaration.agents.get(agent).map(|a| a.state) != Some(from) {
+                    return Ok((declaration, false));
+                }
+            }
+            let (declaration, encoded) = apply(current, agent, state)?;
 
             // A redeclare of the identical state re-encodes the declaration
             // unchanged (a caller that already sees this state and asks
@@ -124,7 +162,7 @@ impl WantedWriter {
             // between reads.
             if entry.as_ref().map(|e| e.value.as_ref()) == Some(encoded.as_slice()) {
                 tracing::debug!(hive, agent, "declaration already current, no write needed");
-                return Ok(declaration);
+                return Ok((declaration, true));
             }
 
             // `update` and `create` have separate error types, and only one
@@ -149,7 +187,7 @@ impl WantedWriter {
             match written {
                 Wrote::Ok => {
                     tracing::info!(hive, agent, ?state, "declared agent state");
-                    return Ok(declaration);
+                    return Ok((declaration, true));
                 }
                 // Re-read and re-apply onto the winner's value, not over it.
                 Wrote::LostRace => {