vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

review-classifier: ask a check once more when its answer isn't valid JSON

Browse source 
An answer that fails to validate is asked for a second time, for that
classification only, with a reminder to reply with the JSON object alone.
The second ask comes out of the classification's CLASSIFICATION_DEADLINE:
it gets what the first ask left, and is skipped when less than 20 seconds
remain, so RUN_DEADLINE's bound still holds. A failed ask (runtime error,
timeout) is not repeated. If the second answer does not validate either,
the row has no answer, as before.

Model::ask takes the deadline for the ask; the ACP runtime no longer holds
its own.

The parser also takes the first fenced block of a reply that has prose
around the fence; prose around a bare object is still rejected.
This commit is contained in:
atlas 2026-10-16 15:32:07 +02:00 • committed by mara
commit ce7b3e055f
8 changed files with 345 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

69
swarm-review-classifier/src/acp.rs
View file

@ -65,8 +65,6 @@ pub struct Acp<S> {
/// This binary, which execs [`Self::program`] when started as the
/// trampoline.
pub trampoline: PathBuf,
/// How long one classification may take before it is cancelled.
pub deadline: Duration,
/// [`CANCEL_WAIT`] outside the tests.
pub cancel_wait: Duration,
pub cost: S,
@ -83,12 +81,15 @@ impl<S: CostSink> crate::run::Model for Acp<S> {
prompt: &crate::prompt::Prompt<'_>,
key: &str,
run: &TryOut,
deadline: Duration,
) -> Result<String, String> {
let Runtime::Acp(provider) = &run.runtime else {
return Err("not an ACP run".to_owned());
};
let dir = crate::run::scratch_dir();
let outcome = self.ask_in(&dir, &prompt.text(), key, run, provider).await;
let outcome = self
.ask_in(&dir, &prompt.text(), key, run, provider, deadline)
.await;
if let Err(e) = tokio::fs::remove_dir_all(&dir).await {
tracing::warn!(dir = %dir.display(), error = %e, "removing a run's directory failed");
}
@ -104,6 +105,7 @@ impl<S: CostSink> Acp<S> {
key: &str,
run: &TryOut,
provider: &Provider,
deadline: Duration,
) -> Result<String, String> {
let system = dir.join(SYSTEM_FILE);
let config_file = dir.join("opencode.json");
@ -132,7 +134,7 @@ impl<S: CostSink> Acp<S> {
let sink = FinalText::default();
let turn = runtime.run(&config, prompt, &sink);
tokio::pin!(turn);
match tokio::time::timeout(self.deadline, &mut turn).await {
match tokio::time::timeout(deadline, &mut turn).await {
Ok(Ok(progress)) => {
self.record_cost(run, provider, &progress.telemetry.cost);
sink.take()
@ -141,10 +143,7 @@ impl<S: CostSink> Acp<S> {
Err(_) => {
let _ = canceller.cancel();
let _ = tokio::time::timeout(self.cancel_wait, turn).await;
Err(format!(
"no answer within {} seconds",
self.deadline.as_secs()
))
Err(format!("no answer within {} seconds", deadline.as_secs()))
}
}
}
@ -283,6 +282,11 @@ mod tests {
use crate::prompt::Prompt;
use crate::run::Model as _;
/// A deadline no test reaches.
const LONG: Duration = Duration::from_mins(1);
/// A deadline the stalled agents run into.
const SHORT: Duration = Duration::from_millis(500);
/// An ACP agent in plain `sh`, logging to `<log>.*`. It writes its
/// environment and copies of its config and the system prompt file
/// beside it and appends its `$HOME` to `<log>.homes` at start, appends
@ -340,7 +344,7 @@ done
}
/// An [`Acp`] whose trampoline is [`AGENT`], logging under `dir`.
fn acp(dir: &Path, deadline: Duration) -> (Acp<Arc<Costs>>, Arc<Costs>) {
fn acp(dir: &Path) -> (Acp<Arc<Costs>>, Arc<Costs>) {
let script = dir.join("agent.sh");
let log = dir.join("log");
std::fs::write(
@ -353,7 +357,6 @@ done
let acp = Acp {
program: PathBuf::from("/nix/store/x-opencode/bin/opencode"),
trampoline: script,
deadline,
cancel_wait: Duration::from_millis(500),
cost: Arc::clone(&costs),
cwd: dir.join("cwd"),
@ -392,9 +395,9 @@ done
#[tokio::test]
async fn the_answer_is_the_final_text_and_its_cost_is_labelled_with_the_pr() {
let dir = tempfile::tempdir().expect("a dir");
let (acp, costs) = acp(dir.path(), Duration::from_mins(1));
let (acp, costs) = acp(dir.path());
let answer = acp.ask(&Prompt::of("classify"), "p", &run()).await;
let answer = acp.ask(&Prompt::of("classify"), "p", &run(), LONG).await;
assert_eq!(
answer.as_deref(),
Ok(r#"{"verdict":"match","certainty":90,"reason":"ok"}"#)
@ -416,9 +419,9 @@ done
#[tokio::test]
async fn opencode_gets_the_key_no_tools_and_the_runs_provider() {
let dir = tempfile::tempdir().expect("a dir");
let (acp, _) = acp(dir.path(), Duration::from_mins(1));
let (acp, _) = acp(dir.path());
acp.ask(&Prompt::of("classify"), "p", &run())
acp.ask(&Prompt::of("classify"), "p", &run(), LONG)
.await
.expect("an answer");
let env = read(dir.path(), "env");
@ -448,9 +451,9 @@ done
#[tokio::test]
async fn the_system_prompt_is_the_agents_and_the_prompt_is_the_pr_alone() {
let dir = tempfile::tempdir().expect("a dir");
let (acp, _) = acp(dir.path(), Duration::from_mins(1));
let (acp, _) = acp(dir.path());
acp.ask(&Prompt::of("classify"), "p", &run())
acp.ask(&Prompt::of("classify"), "p", &run(), LONG)
.await
.expect("an answer");
let config: Value = serde_json::from_str(&read(dir.path(), "config")).expect("JSON");
@ -470,9 +473,9 @@ done
#[tokio::test]
async fn a_permission_request_is_refused() {
let dir = tempfile::tempdir().expect("a dir");
let (acp, _) = acp(dir.path(), Duration::from_mins(1));
let (acp, _) = acp(dir.path());
acp.ask(&Prompt::of("ASK-READ"), "p", &run())
acp.ask(&Prompt::of("ASK-READ"), "p", &run(), LONG)
.await
.expect("an answer");
let reply: Value =
@ -483,9 +486,9 @@ done
#[tokio::test]
async fn the_deadline_cancels_the_turn() {
let dir = tempfile::tempdir().expect("a dir");
let (acp, costs) = acp(dir.path(), Duration::from_millis(500));
let (acp, costs) = acp(dir.path());
let answer = acp.ask(&Prompt::of("STALL"), "p", &run()).await;
let answer = acp.ask(&Prompt::of("STALL"), "p", &run(), SHORT).await;
assert_eq!(answer, Err("no answer within 0 seconds".to_owned()));
assert!(
read(dir.path(), "methods")
@ -498,7 +501,7 @@ done
#[tokio::test]
async fn an_agent_stuck_before_its_prompt_is_dropped_at_the_deadline() {
let dir = tempfile::tempdir().expect("a dir");
let (acp, _) = acp(dir.path(), Duration::from_millis(500));
let (acp, _) = acp(dir.path());
let script = dir.path().join("agent.sh");
let unanswered = std::fs::read_to_string(&script).expect("written").replacen(
" initialize)",
@ -509,7 +512,7 @@ done
let answer = tokio::time::timeout(
Duration::from_secs(10),
acp.ask(&Prompt::of("one"), "p", &run()),
acp.ask(&Prompt::of("one"), "p", &run(), SHORT),
)
.await
.expect("the deadline ends the ask");
@ -519,9 +522,9 @@ done
#[tokio::test]
async fn no_token_counts_record_no_cost() {
let dir = tempfile::tempdir().expect("a dir");
let (acp, costs) = acp(dir.path(), Duration::from_mins(1));
let (acp, costs) = acp(dir.path());
acp.ask(&Prompt::of("MUTE"), "p", &run())
acp.ask(&Prompt::of("MUTE"), "p", &run(), LONG)
.await
.expect("an answer");
assert!(costs.0.lock().expect("unpoisoned").is_empty());
@ -530,12 +533,12 @@ done
#[tokio::test]
async fn a_claude_run_is_refused() {
let dir = tempfile::tempdir().expect("a dir");
let (acp, _) = acp(dir.path(), Duration::from_mins(1));
let (acp, _) = acp(dir.path());
let mut claude = run();
claude.runtime = Runtime::ApiKey;
assert!(
acp.ask(&Prompt::of("classify"), "p", &claude)
acp.ask(&Prompt::of("classify"), "p", &claude, LONG)
.await
.is_err()
);
@ -556,17 +559,17 @@ done
#[tokio::test]
async fn every_run_gets_the_same_cwd_even_at_once() {
let dir = tempfile::tempdir().expect("a dir");
let (acp, _) = acp(dir.path(), Duration::from_mins(1));
let (acp, _) = acp(dir.path());
let (run_one, run_two) = (run(), run());
let (prompt_one, prompt_two) = (Prompt::of("one"), Prompt::of("two"));
let (one, two) = tokio::join!(
acp.ask(&prompt_one, "p", &run_one),
acp.ask(&prompt_two, "p", &run_two)
acp.ask(&prompt_one, "p", &run_one, LONG),
acp.ask(&prompt_two, "p", &run_two, LONG)
);
one.expect("an answer");
two.expect("an answer");
acp.ask(&Prompt::of("three"), "p", &run())
acp.ask(&Prompt::of("three"), "p", &run(), LONG)
.await
.expect("an answer");
let cwd = dir.path().join("cwd").display().to_string();
@ -576,12 +579,12 @@ done
#[tokio::test]
async fn each_run_has_a_home_of_its_own_removed_after_it() {
let dir = tempfile::tempdir().expect("a dir");
let (acp, _) = acp(dir.path(), Duration::from_mins(1));
let (acp, _) = acp(dir.path());
acp.ask(&Prompt::of("one"), "p", &run())
acp.ask(&Prompt::of("one"), "p", &run(), LONG)
.await
.expect("an answer");
acp.ask(&Prompt::of("two"), "p", &run())
acp.ask(&Prompt::of("two"), "p", &run(), LONG)
.await
.expect("an answer");
let homes = read(dir.path(), "homes");