review classifier: recover from a stored record it can't decode

The controller keeps the classifier's newest runs as one JSON array and
its settings as one JSON object, each under one KV key. A document this
build could not decode, as after a rollback past a build that added an
outcome or a settings field, failed every read and every write until
someone deleted the key by hand. A classifier newer than its controller
refused every run request for the missing `model`.

The run store now decodes the array one record at a time: a record that
does not decode is dropped with a WARN naming the bucket, key, index and
error, and a document that is not an array reads as no runs. Listing
keeps working, and the next recorded run writes back only the records
that decoded.

The stored settings ignore a field this build does not know; one that
still does not decode reads as the defaults with a WARN. The PUT body
keeps `deny_unknown_fields`. A PUT now always writes, so a PUT of the
defaults also replaces an undecodable document.

A run request without `model` runs on haiku, the swarm-wide setting's
default, now one constant in swarm-queue-client.

`schedules::read` gains a sibling, `read_with`, taking the decoder.

Closes #5162

swarm-controller/src/review_classifier/runs.rs

@@ -23,7 +23,7 @@ use serde::{Deserialize, Serialize};
 use swarm_queue_client::review_classifier::{Overall, Reply, Row};
 use utoipa::ToSchema;
 
-use crate::schedules::{read, write};
+use crate::schedules::{read_with, write};
 use crate::{AppState, error_problem};
 
 /// The bucket holding the runs. Agents hold no grant on it.
@@ -133,7 +133,7 @@ pub(crate) trait RunDoc {
 
 impl RunDoc for Store {
     async fn read(&self) -> anyhow::Result<(Vec<RunRecord>, Option<u64>)> {
-        read(self, KEY).await
+        read_with(self, KEY, |bytes| Ok(decode(bytes))).await
     }
 
     async fn write(&self, runs: Vec<RunRecord>, revision: Option<u64>) -> anyhow::Result<bool> {
@@ -141,6 +141,37 @@ impl RunDoc for Store {
     }
 }
 
+/// The records in `bytes`, the stored document. A record this build cannot
+/// decode, such as one a newer build wrote, is dropped with a warning, and a
+/// document that is not a JSON array reads as no runs; either way the next
+/// write replaces the document with one holding only decodable records.
+fn decode(bytes: &[u8]) -> Vec<RunRecord> {
+    let values: Vec<serde_json::Value> = match serde_json::from_slice(bytes) {
+        Ok(values) => values,
+        Err(e) => {
+            tracing::warn!(
+                bucket = BUCKET, key = KEY, error = %e,
+                "the review classifier runs are not decodable; reading them as none"
+            );
+            return Vec::new();
+        }
+    };
+    values
+        .into_iter()
+        .enumerate()
+        .filter_map(|(index, value)| {
+            serde_json::from_value(value)
+                .inspect_err(|e| {
+                    tracing::warn!(
+                        bucket = BUCKET, key = KEY, index, error = %e,
+                        "dropping a review classifier run that is not decodable"
+                    );
+                })
+                .ok()
+        })
+        .collect()
+}
+
 /// Put `run` first in `doc`, keeping the newest [`MAX_RUNS`], re-reading and
 /// re-applying when another write landed in between.
 async fn push(doc: &impl RunDoc, run: &RunRecord) -> anyhow::Result<()> {
@@ -417,6 +448,42 @@ pub(crate) mod tests {
         assert_eq!(ids(&listed), ["b", "a"]);
     }
 
+    /// A record from a newer build among readable ones, and a document that
+    /// is no list at all: listing and recording both keep working, and the
+    /// next write leaves a document this build decodes whole.
+    #[tokio::test]
+    async fn an_undecodable_record_or_document_is_dropped_not_fatal() {
+        let nats = crate::test_nats::TestNats::start();
+        let client = nats.client().await;
+        let Ok(store) = open(Some(&client)).await else {
+            panic!("the bucket did not open");
+        };
+        let runs = Runs::Live(Some(client));
+        let stored = || async {
+            let entry = store.get(KEY).await.expect("readable").expect("a document");
+            serde_json::from_slice::<Vec<RunRecord>>(&entry).expect("decodes whole")
+        };
+
+        let mut newer = serde_json::to_value(record("newer")).expect("encodes");
+        newer["outcome"] = "a-later-outcome".into();
+        let doc = serde_json::json!([record("b"), newer, record("a")]);
+        store
+            .put(KEY, doc.to_string().into())
+            .await
+            .expect("written");
+        assert_eq!(ids(&runs.list().await.expect("listed")), ["b", "a"]);
+        runs.record(record("c")).await;
+        assert_eq!(ids(&stored().await), ["c", "b", "a"]);
+
+        store
+            .put(KEY, br#"{"not":"a list"}"#.to_vec().into())
+            .await
+            .expect("written");
+        assert!(runs.list().await.expect("listed").is_empty());
+        runs.record(record("d")).await;
+        assert_eq!(ids(&stored().await), ["d"]);
+    }
+
     #[tokio::test]
     async fn the_route_lists_the_runs_newest_first_and_a_down_store_is_an_error() {
         let doc = Arc::new(FakeDoc::default());