Compare commits
3 changed files with 19 additions and 34 deletions
|
|
@ -235,8 +235,7 @@ The model answers each classification in JSON. The answer holds:
|
||||||
|
|
||||||
- a **verdict**, one of _match_, _no match_ or _not applicable to this PR_
|
- a **verdict**, one of _match_, _no match_ or _not applicable to this PR_
|
||||||
- a **certainty** from 0 to 100, how sure the model is of that verdict
|
- a **certainty** from 0 to 100, how sure the model is of that verdict
|
||||||
- a **reason**; the classifier truncates one over 500 characters to 499 plus
|
- a **reason**, at most 500 characters
|
||||||
`…`, and keeps the verdict and certainty
|
|
||||||
|
|
||||||
An answer:
|
An answer:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -18,9 +18,9 @@ forge and holds no forge credential.
|
||||||
- One `claude --print` per classification, with no session, no tools
|
- One `claude --print` per classification, with no session, no tools
|
||||||
(`--tools ""`), no MCP server, its own empty directory and
|
(`--tools ""`), no MCP server, its own empty directory and
|
||||||
`ANTHROPIC_API_KEY` in its environment only.
|
`ANTHROPIC_API_KEY` in its environment only.
|
||||||
- An answer is a JSON object with a verdict, a certainty and a reason; the
|
- An answer is a JSON object with a verdict, a certainty and a reason of at
|
||||||
classifier truncates a reason over 500 characters to 499 plus `…`. A failed run, a
|
most 500 characters. A failed run, a timeout or an answer that doesn't
|
||||||
timeout or an answer that doesn't validate leaves that row without a verdict, and it doesn't count.
|
validate leaves that row without a verdict, and it doesn't count.
|
||||||
- A run with no API key, or with no classification answered, fails as a
|
- A run with no API key, or with no classification answered, fails as a
|
||||||
whole: the caller sees an error, never a verdict.
|
whole: the caller sees an error, never a verdict.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,13 @@
|
||||||
//! Reading the model's answer to one classification.
|
//! Reading the model's answer to one classification.
|
||||||
//!
|
//!
|
||||||
//! An answer that does not parse, names an unknown verdict or carries a
|
//! An answer that does not parse, names an unknown verdict, carries a
|
||||||
//! certainty outside 0..=100 is an error, and the row it belongs to is
|
//! certainty outside 0..=100 or a reason over the cap is an error, and the
|
||||||
//! excluded rather than guessed at. A reason over the cap is not an error: it
|
//! row it belongs to is excluded rather than guessed at.
|
||||||
//! is cut to the cap, so the verdict and certainty survive.
|
|
||||||
|
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use swarm_queue_client::review_classifier::Verdict;
|
use swarm_queue_client::review_classifier::Verdict;
|
||||||
|
|
||||||
/// The longest reason, in characters, an answer carries; a longer one is
|
/// The longest reason, in characters, an answer may carry.
|
||||||
/// truncated to this, ending in `…`.
|
|
||||||
pub const REASON_MAX_CHARS: usize = 500;
|
pub const REASON_MAX_CHARS: usize = 500;
|
||||||
|
|
||||||
/// One validated answer.
|
/// One validated answer.
|
||||||
|
|
@ -58,23 +56,19 @@ pub fn parse(text: &str) -> Result<Answer, String> {
|
||||||
raw.certainty
|
raw.certainty
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
|
let length = raw.reason.chars().count();
|
||||||
|
if length > REASON_MAX_CHARS {
|
||||||
|
return Err(format!(
|
||||||
|
"the answer's reason is {length} characters, over the {REASON_MAX_CHARS}-character cap"
|
||||||
|
));
|
||||||
|
}
|
||||||
Ok(Answer {
|
Ok(Answer {
|
||||||
verdict,
|
verdict,
|
||||||
certainty,
|
certainty,
|
||||||
reason: cap_reason(raw.reason),
|
reason: raw.reason,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `reason` cut to `REASON_MAX_CHARS` characters, the last being `…`, if longer.
|
|
||||||
fn cap_reason(reason: String) -> String {
|
|
||||||
if reason.chars().count() <= REASON_MAX_CHARS {
|
|
||||||
return reason;
|
|
||||||
}
|
|
||||||
let mut cut: String = reason.chars().take(REASON_MAX_CHARS - 1).collect();
|
|
||||||
cut.push('…');
|
|
||||||
cut
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `text` without a surrounding code fence, with or without a language tag.
|
/// `text` without a surrounding code fence, with or without a language tag.
|
||||||
fn unfence(text: &str) -> &str {
|
fn unfence(text: &str) -> &str {
|
||||||
let Some(inner) = text.strip_prefix("```").and_then(|t| t.strip_suffix("```")) else {
|
let Some(inner) = text.strip_prefix("```").and_then(|t| t.strip_suffix("```")) else {
|
||||||
|
|
@ -125,24 +119,16 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn an_oversized_reason_is_truncated_and_one_at_the_cap_is_kept() {
|
fn an_oversized_reason_is_an_error_and_one_at_the_cap_is_not() {
|
||||||
let answer = |n: usize| {
|
let answer = |n: usize| {
|
||||||
format!(
|
format!(
|
||||||
r#"{{"verdict":"match","certainty":90,"reason":"{}"}}"#,
|
r#"{{"verdict":"match","certainty":90,"reason":"{}"}}"#,
|
||||||
"é".repeat(n)
|
"é".repeat(n)
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
|
let e = parse(&answer(REASON_MAX_CHARS + 1)).expect_err("over the cap");
|
||||||
|
assert!(e.contains("501 characters"), "{e}");
|
||||||
// Characters, not bytes: `é` is two bytes.
|
// Characters, not bytes: `é` is two bytes.
|
||||||
let at_cap = parse(&answer(REASON_MAX_CHARS)).expect("at the cap");
|
assert!(parse(&answer(REASON_MAX_CHARS)).is_ok());
|
||||||
assert_eq!(at_cap.reason, "é".repeat(REASON_MAX_CHARS));
|
|
||||||
|
|
||||||
let long = parse(&answer(600)).expect("over the cap still parses");
|
|
||||||
assert_eq!(long.verdict, Verdict::Match);
|
|
||||||
assert_eq!(long.certainty, 90);
|
|
||||||
assert_eq!(long.reason.chars().count(), REASON_MAX_CHARS);
|
|
||||||
assert_eq!(
|
|
||||||
long.reason,
|
|
||||||
format!("{}…", "é".repeat(REASON_MAX_CHARS - 1))
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue