//! Reading the model's answer to one classification. //! //! An answer that does not parse, names an unknown verdict or carries a //! certainty outside 0..=100 is an error, and the row it belongs to is //! excluded rather than guessed at. A reason over the cap is not an error: it //! is cut to the cap, so the verdict and certainty survive. use serde::Deserialize; use swarm_wire::review_classifier::Verdict; /// The longest reason, in characters, an answer carries; a longer one is /// truncated to this, ending in `…`. pub const REASON_MAX_CHARS: usize = 500; /// The length the prompt asks a reason to stay under. Below `REASON_MAX_CHARS` /// so a model that overshoots the ask still lands inside the cap, uncut. pub const REASON_ASKED_CHARS: usize = 350; const _: () = assert!(REASON_ASKED_CHARS < REASON_MAX_CHARS); /// One validated answer. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Answer { pub verdict: Verdict, pub certainty: u8, pub reason: String, } /// The answer's fields as the model wrote them, before validation. #[derive(Deserialize)] struct Raw { verdict: String, certainty: serde_json::Number, reason: String, } /// Parse `text`, the model's final message. A code fence around the object is /// tolerated, with prose around the fence; prose around a bare object is not. /// /// # Errors /// A message describing what is wrong with the answer, without quoting it. pub fn parse(text: &str) -> Result { let body = unfence(text.trim()); let raw: Raw = serde_json::from_str(body) .map_err(|e| format!("the answer is not the JSON asked for: {e}"))?; let verdict = match raw.verdict.as_str() { "match" => Verdict::Match, "no-match" => Verdict::NoMatch, "not-applicable" => Verdict::NotApplicable, other => { return Err(format!( "the answer's verdict {other:?} is not one of the three" )); } }; let certainty = raw .certainty .as_u64() .and_then(|c| u8::try_from(c).ok()) .filter(|c| *c <= 100) .ok_or_else(|| { format!( "the answer's certainty {} is not a whole 0-100", raw.certainty ) })?; Ok(Answer { verdict, certainty, reason: cap_reason(raw.reason), }) } /// `reason` cut to `REASON_MAX_CHARS` characters, the last being `…`, if longer. fn cap_reason(reason: String) -> String { if reason.chars().count() <= REASON_MAX_CHARS { return reason; } let mut cut: String = reason.chars().take(REASON_MAX_CHARS - 1).collect(); cut.push('…'); cut } /// `text` without its code fence, with or without a language tag. A fence /// around the whole message is stripped; otherwise the first fenced block in /// the message is taken, so prose before or after it is dropped. fn unfence(text: &str) -> &str { let inner = match text.strip_prefix("```").and_then(|t| t.strip_suffix("```")) { Some(inner) => inner, None => match text .split_once("```") .and_then(|(_, t)| t.split_once("```")) { Some((block, _)) => block, None => return text, }, }; let inner = inner.strip_prefix("json").unwrap_or(inner); inner.trim() } #[cfg(test)] mod tests { use super::*; #[test] fn a_valid_answer_parses() { let a = parse(r#"{"verdict": "no-match", "certainty": 92, "reason": "Naive datetimes."}"#) .expect("valid"); assert_eq!( a, Answer { verdict: Verdict::NoMatch, certainty: 92, reason: "Naive datetimes.".to_owned(), } ); let fenced = "```json\n{\"verdict\":\"not-applicable\",\"certainty\":0,\"reason\":\"\"}\n```"; assert_eq!( parse(fenced).expect("fenced").verdict, Verdict::NotApplicable ); } #[test] fn a_fenced_answer_inside_prose_parses() { let a = parse( "Here is my answer:\n```json\n{\"verdict\":\"match\",\"certainty\":80,\"reason\":\"x\"}\n```\nHope that helps.", ) .expect("fence in prose"); assert_eq!((a.verdict, a.certainty), (Verdict::Match, 80)); let untagged = "Answer: ```{\"verdict\":\"no-match\",\"certainty\":70,\"reason\":\"\"}``` done"; assert_eq!(parse(untagged).expect("untagged").verdict, Verdict::NoMatch); assert!(parse("Here is my answer:\n```json\nnot json\n```").is_err()); assert!(parse("an unclosed ```json\n{\"verdict\":\"match\"").is_err()); } #[test] fn a_malformed_answer_is_an_error() { for bad in [ "", "yes", "Sure! {\"verdict\":\"match\",\"certainty\":90,\"reason\":\"x\"}", r#"{"verdict":"maybe","certainty":50,"reason":"x"}"#, r#"{"verdict":"match","certainty":101,"reason":"x"}"#, r#"{"verdict":"match","certainty":-1,"reason":"x"}"#, r#"{"verdict":"match","certainty":80.5,"reason":"x"}"#, r#"{"verdict":"match","reason":"x"}"#, ] { assert!(parse(bad).is_err(), "{bad:?} parsed"); } } #[test] fn an_oversized_reason_is_truncated_and_one_at_the_cap_is_kept() { let answer = |n: usize| { format!( r#"{{"verdict":"match","certainty":90,"reason":"{}"}}"#, "é".repeat(n) ) }; // Characters, not bytes: `é` is two bytes. let at_cap = parse(&answer(REASON_MAX_CHARS)).expect("at the cap"); assert_eq!(at_cap.reason, "é".repeat(REASON_MAX_CHARS)); let long = parse(&answer(600)).expect("over the cap still parses"); assert_eq!(long.verdict, Verdict::Match); assert_eq!(long.certainty, 90); assert_eq!(long.reason.chars().count(), REASON_MAX_CHARS); assert_eq!( long.reason, format!("{}…", "é".repeat(REASON_MAX_CHARS - 1)) ); } }