Compare commits
3 changed files with 19 additions and 34 deletions
|
|
@ -235,8 +235,7 @@ The model answers each classification in JSON. The answer holds:
|
|||
|
||||
- a **verdict**, one of _match_, _no match_ or _not applicable to this PR_
|
||||
- a **certainty** from 0 to 100, how sure the model is of that verdict
|
||||
- a **reason**; the classifier truncates one over 500 characters to 499 plus
|
||||
`…`, and keeps the verdict and certainty
|
||||
- a **reason**, at most 500 characters
|
||||
|
||||
An answer:
|
||||
|
||||
|
|
|
|||
|
|
@ -18,9 +18,9 @@ forge and holds no forge credential.
|
|||
- One `claude --print` per classification, with no session, no tools
|
||||
(`--tools ""`), no MCP server, its own empty directory and
|
||||
`ANTHROPIC_API_KEY` in its environment only.
|
||||
- An answer is a JSON object with a verdict, a certainty and a reason; the
|
||||
classifier truncates a reason over 500 characters to 499 plus `…`. A failed run, a
|
||||
timeout or an answer that doesn't validate leaves that row without a verdict, and it doesn't count.
|
||||
- An answer is a JSON object with a verdict, a certainty and a reason of at
|
||||
most 500 characters. A failed run, a timeout or an answer that doesn't
|
||||
validate leaves that row without a verdict, and it doesn't count.
|
||||
- A run with no API key, or with no classification answered, fails as a
|
||||
whole: the caller sees an error, never a verdict.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,15 +1,13 @@
|
|||
//! Reading the model's answer to one classification.
|
||||
//!
|
||||
//! An answer that does not parse, names an unknown verdict or carries a
|
||||
//! certainty outside 0..=100 is an error, and the row it belongs to is
|
||||
//! excluded rather than guessed at. A reason over the cap is not an error: it
|
||||
//! is cut to the cap, so the verdict and certainty survive.
|
||||
//! An answer that does not parse, names an unknown verdict, carries a
|
||||
//! certainty outside 0..=100 or a reason over the cap is an error, and the
|
||||
//! row it belongs to is excluded rather than guessed at.
|
||||
|
||||
use serde::Deserialize;
|
||||
use swarm_queue_client::review_classifier::Verdict;
|
||||
|
||||
/// The longest reason, in characters, an answer carries; a longer one is
|
||||
/// truncated to this, ending in `…`.
|
||||
/// The longest reason, in characters, an answer may carry.
|
||||
pub const REASON_MAX_CHARS: usize = 500;
|
||||
|
||||
/// One validated answer.
|
||||
|
|
@ -58,23 +56,19 @@ pub fn parse(text: &str) -> Result<Answer, String> {
|
|||
raw.certainty
|
||||
)
|
||||
})?;
|
||||
let length = raw.reason.chars().count();
|
||||
if length > REASON_MAX_CHARS {
|
||||
return Err(format!(
|
||||
"the answer's reason is {length} characters, over the {REASON_MAX_CHARS}-character cap"
|
||||
));
|
||||
}
|
||||
Ok(Answer {
|
||||
verdict,
|
||||
certainty,
|
||||
reason: cap_reason(raw.reason),
|
||||
reason: raw.reason,
|
||||
})
|
||||
}
|
||||
|
||||
/// `reason` cut to `REASON_MAX_CHARS` characters, the last being `…`, if longer.
|
||||
fn cap_reason(reason: String) -> String {
|
||||
if reason.chars().count() <= REASON_MAX_CHARS {
|
||||
return reason;
|
||||
}
|
||||
let mut cut: String = reason.chars().take(REASON_MAX_CHARS - 1).collect();
|
||||
cut.push('…');
|
||||
cut
|
||||
}
|
||||
|
||||
/// `text` without a surrounding code fence, with or without a language tag.
|
||||
fn unfence(text: &str) -> &str {
|
||||
let Some(inner) = text.strip_prefix("```").and_then(|t| t.strip_suffix("```")) else {
|
||||
|
|
@ -125,24 +119,16 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn an_oversized_reason_is_truncated_and_one_at_the_cap_is_kept() {
|
||||
fn an_oversized_reason_is_an_error_and_one_at_the_cap_is_not() {
|
||||
let answer = |n: usize| {
|
||||
format!(
|
||||
r#"{{"verdict":"match","certainty":90,"reason":"{}"}}"#,
|
||||
"é".repeat(n)
|
||||
)
|
||||
};
|
||||
let e = parse(&answer(REASON_MAX_CHARS + 1)).expect_err("over the cap");
|
||||
assert!(e.contains("501 characters"), "{e}");
|
||||
// Characters, not bytes: `é` is two bytes.
|
||||
let at_cap = parse(&answer(REASON_MAX_CHARS)).expect("at the cap");
|
||||
assert_eq!(at_cap.reason, "é".repeat(REASON_MAX_CHARS));
|
||||
|
||||
let long = parse(&answer(600)).expect("over the cap still parses");
|
||||
assert_eq!(long.verdict, Verdict::Match);
|
||||
assert_eq!(long.certainty, 90);
|
||||
assert_eq!(long.reason.chars().count(), REASON_MAX_CHARS);
|
||||
assert_eq!(
|
||||
long.reason,
|
||||
format!("{}…", "é".repeat(REASON_MAX_CHARS - 1))
|
||||
);
|
||||
assert!(parse(&answer(REASON_MAX_CHARS)).is_ok());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue