controller stores: recover from a record they can't decode

After a rollback, a record a newer build wrote (a new enum variant, a
changed required field) stopped decoding, and the strict
`schedules::read` turned that into an error on every read and write of
the key until someone deleted it by hand.

- agent_limits / inference (one document per agent): an undecodable
  document reads as unset, with a WARN naming the bucket, key and error.
  The PUT's "equal to stored, skip the write" shortcut no longer applies
  to an undecodable document, so a PUT of the defaults replaces it too.
- schedule definitions (one document, every schedule): salvaged per
  definition, as the review-classifier run list is. An undecodable
  definition is kept as its raw JSON, hidden from the list, edits and
  projections, written back by every commit after the readable ones, and
  its id is never minted again. A document with no list of schedules
  stays an error: reading it as empty would empty every agent's key.
- per-agent agent-schedules: an undecodable key reads as no schedules
  and is rewritten on the next projection, even when the agent has none.

`schedules::read` is replaced by `read_or_default`; `wanted` keeps its
own strict read.

Closes #5193

swarm-controller/src/inference.rs

@@ -25,7 +25,7 @@ use serde::{Deserialize, Serialize};
 use swarm_queue_client::agent_inference::{self, AgentInference, Effective, Effort, Source};
 use utoipa::ToSchema;
 
-use crate::schedules::{live, queue, read, refused, write};
+use crate::schedules::{live, queue, read_or_default, refused, write};
 use crate::{AppState, error_problem, wanted_error_status};
 
 /// How many times a losing writer re-reads and re-applies, as in
@@ -242,9 +242,9 @@ pub(crate) async fn get_agent_inference(
     let agent = agent_name(&name).map_err(refused)?;
     let client = queue(&state).map_err(refused)?;
     let store = open(&client).await?;
-    // `Option`'s default is the `None` an absent or deleted key reads as; a
-    // stored `{}` decodes to `Some`.
-    let (current, _) = read::<Option<AgentInference>>(&store, &agent)
+    // `Option`'s default is the `None` an absent, deleted or undecodable key
+    // reads as; a stored `{}` decodes to `Some`.
+    let (current, _, _) = read_or_default::<Option<AgentInference>>(&store, &agent)
         .await
         .map_err(|e| queue_problem(&e))?;
     Ok(Json(AgentInferenceView {
@@ -296,10 +296,10 @@ pub(crate) async fn put_agent_inference(
     }
     let store = open(&client).await?;
     for _ in 0..MAX_ATTEMPTS {
-        let (current, revision) = read::<AgentInference>(&store, &agent)
+        let (current, revision, undecodable) = read_or_default::<AgentInference>(&store, &agent)
             .await
             .map_err(|e| queue_problem(&e))?;
-        if current == value
+        if (current == value && !undecodable)
             || write(&store, &agent, &value, revision)
                 .await
                 .map_err(|e| queue_problem(&e))?
@@ -506,4 +506,36 @@ mod tests {
             Some(AgentInference::default())
         );
     }
+
+    /// A stored value this build cannot decode, such as an effort a newer
+    /// build added, reads as no settings, and a `PUT`, even of no fields,
+    /// replaces it.
+    #[tokio::test]
+    async fn an_undecodable_value_reads_as_unset_until_a_put() {
+        let nats = crate::test_nats::TestNats::start();
+        let client = nats.client().await;
+        let state = crate::agent_limits::tests::declared(&client).await;
+        let store = agent_inference::open_or_create(&client)
+            .await
+            .expect("bucket");
+        store
+            .put("atlas", br#"{"effort":"later"}"#.to_vec().into())
+            .await
+            .expect("written");
+
+        let view = get_agent_inference(State(state.clone()), Path("atlas".to_owned()))
+            .await
+            .expect("read");
+        assert_eq!(view.0.configured, None);
+        let stored = put_agent_inference(
+            State(state),
+            Path("atlas".to_owned()),
+            Json(InferenceSettings::default()),
+        )
+        .await
+        .expect("set");
+        assert_eq!(stored.0, InferenceSettings::default());
+        let entry = store.get("atlas").await.expect("read").expect("a value");
+        assert_eq!(&entry[..], b"{}");
+    }
 }