controller stores: recover from a record they can't decode

After a rollback, a record a newer build wrote (a new enum variant, a
changed required field) stopped decoding, and the strict
`schedules::read` turned that into an error on every read and write of
the key until someone deleted it by hand.

- agent_limits / inference (one document per agent): an undecodable
  document reads as unset, with a WARN naming the bucket, key and error.
  The PUT's "equal to stored, skip the write" shortcut no longer applies
  to an undecodable document, so a PUT of the defaults replaces it too.
- schedule definitions (one document, every schedule): salvaged per
  definition, as the review-classifier run list is. An undecodable
  definition is kept as its raw JSON, hidden from the list, edits and
  projections, written back by every commit after the readable ones, and
  its id is never minted again. A document with no list of schedules
  stays an error: reading it as empty would empty every agent's key.
- per-agent agent-schedules: an undecodable key reads as no schedules
  and is rewritten on the next projection, even when the agent has none.

`schedules::read` is replaced by `read_or_default`; `wanted` keeps its
own strict read.

Closes #5193

swarm-controller/src/agent_limits.rs

@@ -19,7 +19,7 @@ use serde::{Deserialize, Serialize};
 use swarm_queue_client::agent_limits::{self, AgentCaps};
 use utoipa::ToSchema;
 
-use crate::schedules::{live, queue, read, refused, write};
+use crate::schedules::{live, queue, read_or_default, refused, write};
 use crate::{AppState, error_problem, wanted_error_status};
 
 /// How many times a losing writer re-reads and re-applies, as in
@@ -209,7 +209,7 @@ pub(crate) async fn get_agent_limits(
     else {
         return Ok(Json(Caps::default()));
     };
-    let (current, _) = read::<AgentCaps>(&store, &agent)
+    let (current, _, _) = read_or_default::<AgentCaps>(&store, &agent)
         .await
         .map_err(|e| queue_problem(&e))?;
     Ok(Json(current.into()))
@@ -255,10 +255,10 @@ pub(crate) async fn put_agent_limits(
     }
     let store = open_or_create(&client).await?;
     for _ in 0..MAX_ATTEMPTS {
-        let (current, revision) = read::<AgentCaps>(&store, &agent)
+        let (current, revision, undecodable) = read_or_default::<AgentCaps>(&store, &agent)
             .await
             .map_err(|e| queue_problem(&e))?;
-        if current == value
+        if (current == value && !undecodable)
             || write(&store, &agent, &value, revision)
                 .await
                 .map_err(|e| queue_problem(&e))?
@@ -302,7 +302,7 @@ pub(crate) async fn delete_agent_limits(
 }
 
 #[cfg(test)]
-mod tests {
+pub(crate) mod tests {
     use super::{
         Caps, cpu_quota, delete_agent_limits, get_agent_limits, memory_max, put_agent_limits,
     };
@@ -464,7 +464,7 @@ mod tests {
     }
 
     /// A state on a real server whose `pr1ma` declares `atlas`.
-    async fn declared(client: &async_nats::Client) -> crate::AppState {
+    pub(crate) async fn declared(client: &async_nats::Client) -> crate::AppState {
         WantedWriter::new(client.clone())
             .set("pr1ma", "atlas", AgentState::Up)
             .await
@@ -698,4 +698,30 @@ mod tests {
             );
         }
     }
+
+    /// A stored value this build cannot decode, such as one a newer build
+    /// wrote, reads as no caps, and a `PUT`, even of no caps, replaces it.
+    #[tokio::test]
+    async fn an_undecodable_value_reads_as_no_caps_until_a_put() {
+        let nats = TestNats::start();
+        let client = nats.client().await;
+        let state = declared(&client).await;
+        let store = swarm_queue_client::agent_limits::open_or_create(&client)
+            .await
+            .expect("bucket");
+        store
+            .put(
+                "atlas",
+                br#"{"cpu_quota":{"later":"400%"}}"#.to_vec().into(),
+            )
+            .await
+            .expect("written");
+
+        assert_eq!(get(&state, "atlas").await, Caps::default());
+        let answer = put(&state, "atlas", Caps::default())
+            .await
+            .expect("caps are set");
+        assert_eq!(answer.0, Caps::default());
+        assert_eq!(published(&client, "atlas").await.as_deref(), Some("{}"));
+    }
 }