vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

review classifier: model, effort and review concurrency as swarm-wide settings

Browse source 
The classifier ran every classification on a hard-coded `haiku` with no
`--effort`, and the controller ran at most two requested reviews at once,
also hard-coded, so changing any of them took a code change and a
redeploy.

swarm-controller now keeps one settings document,
`{model, effort, max_concurrent_reviews}`, in its own
`review-classifier-settings` KV bucket under `all`, the way it keeps the
schedule definitions. No document means today's values: haiku, no
`--effort`, two reviews. `GET /api/review-classifier/settings` reads it;
`PUT` replaces it whole, needs the gateway's `Remote-User` like the
try-out POST, and refuses a bad model (the inference route's check), an
effort outside the five levels or a limit outside 1..=8 with
problem+json 400.

Every run request (try-out and requested review) now carries the model
and effort, and the classifier passes them to claude instead of its
constant, which is gone. A try-out reads the settings when it is posted;
a review reads them for the limit before it waits for a place and again
for the model and effort once it has one. The review semaphore is
replaced by a running count under a resizable limit, so a PUT, or the
next review's read, changes how many run without a restart, and a lower
limit lets the runs already going finish. MAX_WAITING_REVIEWS is
unchanged.

Closes #5151, part of #5096.
This commit is contained in:
atlas 2026-10-12 19:21:33 +02:00 • committed by mara
commit 2c5eef73cd
10 changed files with 868 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

83
swarm-controller/src/review_classifier.rs
View file

@ -35,11 +35,13 @@ use swarm_queue_client::review_classifier::{
};
use utoipa::{IntoParams, ToSchema};
use self::settings::{Settings, SettingsSource};
use crate::forge::{self, DefaultBranchFiles, PullForReview};
use crate::{AppState, error_problem};
pub(crate) mod review;
pub(crate) mod runs;
pub(crate) mod settings;
/// How long a finished run's result stays readable.
pub(crate) const RESULT_TTL: Duration = Duration::from_hours(1);
@ -321,6 +323,8 @@ impl RunSink for async_nats::Client {
(status = 503, description = "this host has no forge token yet, no swarm queue \
configured, or a queue that is not connected; or the forge failed or did not \
answer within two minutes (problem+json)", body = String),
(status = 500, description = "the review classifier's settings could not be read \
(problem+json)", body = String),
),
tag = "review-classifier"
)]
@ -340,6 +344,7 @@ pub(crate) async fn post_try_out(
let run = Run::checked(req).map_err(|d| error_problem(StatusCode::BAD_REQUEST, &d))?;
#[cfg(test)]
if let Some(fake) = state.try_outs.fake.clone() {
let settings = fake.settings().await.map_err(|e| settings_problem(&e))?;
return Box::pin(start(
&state.try_outs,
&*fake,
@ -347,6 +352,7 @@ pub(crate) async fn post_try_out(
state.runs.clone(),
operator,
run,
settings,
))
.await;
}
@ -370,6 +376,7 @@ pub(crate) async fn post_try_out(
)
})?;
let runs = state.runs.clone();
let settings = queue.settings().await.map_err(|e| settings_problem(&e))?;
Box::pin(start(
&state.try_outs,
forge.as_ref(),
@ -377,6 +384,7 @@ pub(crate) async fn post_try_out(
runs,
operator,
run,
settings,
))
.await
}
@ -617,8 +625,9 @@ fn operator(headers: &HeaderMap) -> Option<String> {
.map(str::to_owned)
}
/// Read the PR, record the run and send it; the reply settles the run when
/// it arrives, and the settled run goes into `runs`.
/// Read the PR, record the run and send it on `settings`' model and effort;
/// the reply settles the run when it arrives, and the settled run goes into
/// `runs`.
async fn start(
try_outs: &TryOuts,
pulls: &impl PullSource,
@ -626,6 +635,7 @@ async fn start(
runs: runs::Runs,
operator: String,
run: Run,
settings: Settings,
) -> Result<(StatusCode, Json<TryOutAccepted>), ProblemDetails> {
let started_unix = crate::agent_renewal::unix_now();
let repo = format!("{}/{}", run.owner, run.name);
@ -647,6 +657,8 @@ async fn start(
commits: pull.commits,
diff: pull.diff,
classifications: run.classifications,
model: settings.model,
effort: settings.effort,
};
let payload = serde_json::to_vec(&request).map_err(|e| {
error_problem(
@ -692,6 +704,14 @@ async fn start(
Ok((StatusCode::ACCEPTED, Json(TryOutAccepted { id })))
}
/// The settings could not be read: 503 when the queue is not connected, 500
/// otherwise.
fn settings_problem(e: &anyhow::Error) -> ProblemDetails {
let detail = format!("{e:#}");
tracing::warn!(error = %detail, "reading the review classifier's settings for a try-out failed");
error_problem(crate::wanted_error_status(e), &detail)
}
/// 404 when the forge knows no such repo or PR, 503 for anything else.
fn pull_problem(e: &anyhow::Error, repo: &str, pr: i64) -> ProblemDetails {
match e.downcast_ref::<forgejo_api::ForgejoError>() {
@ -743,18 +763,31 @@ pub(crate) mod tests {
use super::*;
/// A forge holding one PR, `o/r` 7, with `files` on `o/r`'s default
/// branch, and a classifier answering every run with `reply`. `o/empty`
/// has no commit, `o/down` fails with something other than a 404, and
/// `o/hang` never answers.
/// branch, `settings` stored, and a classifier answering every run with
/// `reply` and keeping its request. `o/empty` has no commit, `o/down`
/// fails with something other than a 404, and `o/hang` never answers.
#[derive(Clone)]
pub(crate) struct Fake {
reply: Reply,
files: Vec<(&'static str, String)>,
pub(crate) settings: Settings,
pub(crate) sent: Arc<std::sync::Mutex<Vec<TryOut>>>,
}
impl Fake {
pub(crate) fn new(reply: Reply, files: Vec<(&'static str, String)>) -> Self {
Self { reply, files }
Self {
reply,
files,
settings: Settings::default(),
sent: Arc::default(),
}
}
}
impl SettingsSource for Fake {
async fn settings(&self) -> anyhow::Result<Settings> {
Ok(self.settings.clone())
}
}
@ -805,7 +838,10 @@ pub(crate) mod tests {
}
impl RunSink for Fake {
async fn run(&self, _request: Vec<u8>) -> Reply {
async fn run(&self, request: Vec<u8>) -> Reply {
if let Ok(sent) = serde_json::from_slice(&request) {
self.sent.lock().expect("unpoisoned").push(sent);
}
self.reply.clone()
}
}
@ -833,7 +869,7 @@ pub(crate) mod tests {
fn app_with_files(reply: Reply, files: Vec<(&'static str, String)>) -> axum::Router {
let (mut state, _) = crate::tests::state_with_roster();
state.try_outs.fake = Some(Arc::new(Fake { reply, files }));
state.try_outs.fake = Some(Arc::new(Fake::new(reply, files)));
crate::build_app(state)
}
@ -903,6 +939,32 @@ pub(crate) mod tests {
assert!(run.get("error").is_none(), "{run}");
}
#[tokio::test]
async fn a_try_out_runs_on_the_stored_model_and_effort() {
use swarm_queue_client::agent_inference::Effort;
let (mut state, _) = crate::tests::state_with_roster();
let fake = Arc::new(Fake {
settings: Settings {
model: "opus".to_owned(),
effort: Some(Effort::High),
max_concurrent_reviews: 2,
},
..Fake::new(done(), Vec::new())
});
state.try_outs.fake = Some(Arc::clone(&fake));
let app = crate::build_app(state);
let (status, accepted) = post(&app, body("o/r", 7)).await;
assert_eq!(status, StatusCode::ACCEPTED, "{accepted}");
settled(&app, accepted["id"].as_str().expect("an id")).await;
let sent = fake.sent.lock().expect("unpoisoned");
assert_eq!(sent.len(), 1);
assert_eq!(
(sent[0].model.as_str(), sent[0].effort),
("opus", Some(Effort::High))
);
}
#[tokio::test]
async fn a_failed_run_reports_its_error_and_no_verdict() {
let app = app(Reply::Failed {
@ -929,10 +991,7 @@ pub(crate) mod tests {
] {
let doc = Arc::new(runs::tests::FakeDoc::default());
let (mut state, _) = crate::tests::state_with_roster();
state.try_outs.fake = Some(Arc::new(Fake {
reply: reply.clone(),
files: Vec::new(),
}));
state.try_outs.fake = Some(Arc::new(Fake::new(reply.clone(), Vec::new())));
state.runs = runs::Runs::Fake(Arc::clone(&doc));
let app = crate::build_app(state);
let (_, accepted) = post(&app, body("o/r", 7)).await;