mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

review classifier: the most classifications a run carries is a setting

Browse source 
The limit was a constant of ten in swarm-wire, so a repo with eleven rules
could not be reviewed and nothing short of a rebuild changed it. It is now
`max_classifications` in the classifier settings, beside
`max_concurrent_reviews`: 1 to 1000, ten until set, and a document stored
before the field decodes with the default. The classifier settings form in
the swarm UI gets a "rules per review" field, and the try-out's add button
stops at the saved limit instead of a hard-coded ten.

The controller checks a try-out, and a requested review's rules file, against
the setting where the run starts. The rules file itself may hold up to the
1000 the setting allows, so raising the limit needs no change to a repo.
A review whose rules exceed the limit posts a no-verdict comment saying so.

A run's reply deadline was a constant that covered ten classifications, a
minute plus three each. It is now computed from the run's own size, so a
bigger run waits as long as it needs and a default-sized one still waits 31
minutes. The classifier's answer window in the queue permissions covers the
biggest run the setting allows.
This commit is contained in:
iris 2026-11-07 10:37:52 +01:00 • committed by mara
commit cffe8895df
3 changed files with 400 additions and 101 deletions
Download patch file Download diff file Expand all files Collapse all files

15
swarm-controller/src/review_classifier/open_pr.rs
View file

@ -26,7 +26,7 @@ use axum::http::{HeaderMap, StatusCode};
use forgejo_api::{ApiErrorKind, ForgejoError};
use problem_details::ProblemDetails;
use serde::{Deserialize, Serialize};
use swarm_wire::review_classifier::{Classification, Polarity};
use swarm_wire::review_classifier::{CLASSIFICATION_LIMITS, Classification, Polarity};
use utoipa::ToSchema;
use super::{FileSource, RULES_PATHS, check_classifications, operator, parse_rules, split_repo};
@ -109,7 +109,7 @@ impl BranchSink for forge::Client {
commit was made (problem+json)", body = String),
(status = 422, description = "the default branch holds both rules files, or one the \
`rules` GET refuses; the rules already hold every classification as it is; or \
the merged rules fail the try-out POST's checks, such as more than ten \
the merged rules fail the try-out POST's checks, such as more than a thousand \
(problem+json)", body = String),
(status = 503, description = "this host has no forge token yet, or the classifier's \
account has none in the store; or the forge failed or did not answer within two \
@ -132,7 +132,7 @@ pub(crate) async fn post_open_pr(
let Json(req) = body.map_err(|e| error_problem(StatusCode::BAD_REQUEST, &e.body_text()))?;
let (owner, name) =
split_repo(&req.repo).map_err(|d| error_problem(StatusCode::BAD_REQUEST, &d))?;
check_classifications(&req.classifications)
check_classifications(&req.classifications, *CLASSIFICATION_LIMITS.end())
.map_err(|d| error_problem(StatusCode::BAD_REQUEST, &d))?;
#[cfg(test)]
if let Some(fake) = state.try_outs.fake.clone() {
@ -355,7 +355,8 @@ fn merge(existing: Option<&BranchFile>, rows: Vec<Classification>) -> Result<Mer
"{path} already holds every classification as it is"
));
}
check_classifications(&merged).map_err(|e| format!("{path} with these merged: {e}"))?;
check_classifications(&merged, *CLASSIFICATION_LIMITS.end())
.map_err(|e| format!("{path} with these merged: {e}"))?;
let content = if std::path::Path::new(&path)
.extension()
.is_some_and(|e| e == "json")
@ -659,11 +660,11 @@ classifications:
let before = parse_rules(yml, OWN.as_bytes()).expect("this repo's rules parse");
let err = merge(Some(&file(yml, OWN)), before.clone()).expect_err("nothing to change");
assert!(err.contains("already holds"), "{err}");
let many: Vec<_> = (0..=10 - before.len())
let many: Vec<_> = (0..=1000 - before.len())
.map(|i| rule(&format!("extra-{i}"), "d"))
.collect();
let err = merge(Some(&file(yml, OWN)), many).expect_err("eleven");
assert!(err.contains("11 classifications"), "{err}");
let err = merge(Some(&file(yml, OWN)), many).expect_err("1001");
assert!(err.contains("1001 classifications"), "{err}");
let err = merge(Some(&file(yml, "nope: 1")), vec![rule("a", "b")]).expect_err("bad");
assert!(err.contains(yml), "{err}");
}

41
swarm-controller/src/review_classifier/review.rs
View file

@ -36,6 +36,7 @@ use futures_util::StreamExt as _;
use serde::{Deserialize, Serialize};
use swarm_wire::review_classifier::{
Overall, REQUEST_CONSUMER, REQUEST_STREAM, REQUEST_SUBJECT_PREFIX, Reply, Row, TryOut, Verdict,
run_deadline,
};
use tokio::sync::Notify;
use tokio::task::JoinSet;
@ -821,7 +822,7 @@ impl ReviewPoster for forge::Client {
struct Queue(Option<async_nats::Client>);
impl RunSink for Queue {
async fn run(&self, request: Vec<u8>) -> Reply {
async fn run(&self, request: Vec<u8>, deadline: Duration) -> Reply {
let Some(client) = &self.0 else {
return Reply::Failed {
error: "no swarm queue is configured on this host".to_owned(),
@ -832,7 +833,7 @@ impl RunSink for Queue {
error: swarm_queue_client::chain(&e),
};
}
client.run(request).await
client.run(request, deadline).await
}
}
@ -1087,6 +1088,14 @@ async fn classify(
);
}
};
if let Err(why) = super::check_count(rules.classifications.len(), settings.max_classifications)
{
return (
head,
narrowed,
Outcome::NotRun(format!("{repo}'s rules do not fit a run: {why}")),
);
}
let runtime = match settings.runtime() {
Ok(runtime) => runtime,
Err(e) => {
@ -1113,7 +1122,7 @@ async fn classify(
let outcome = match serde_json::to_vec(&request) {
Ok(payload) => Outcome::Ran {
sent,
reply: sink.run(payload).await,
reply: sink.run(payload, run_deadline(sent)).await,
},
Err(e) => Outcome::NotRun(format!("encoding the run failed: {e}")),
};
@ -1602,7 +1611,7 @@ mod tests {
}
impl RunSink for Gate {
async fn run(&self, _request: Vec<u8>) -> Reply {
async fn run(&self, _request: Vec<u8>, _deadline: Duration) -> Reply {
self.reached.fetch_add(1, Ordering::SeqCst);
let _permit = self.permits.acquire().await.expect("never closed");
Reply::Done {
@ -2717,6 +2726,7 @@ mod tests {
model: "sonnet".to_owned(),
effort: Some(Effort::Medium),
max_concurrent_reviews: 2,
max_classifications: 10,
runtime: super::super::settings::RuntimeKind::Acp,
acp: Some(acp.clone()),
};
@ -2734,6 +2744,29 @@ mod tests {
);
}
#[tokio::test]
async fn a_repo_with_more_rules_than_the_settings_allow_runs_nothing_and_posts_a_comment() {
let mut fake = Fake::new(
Reply::Failed {
error: "must not run".to_owned(),
},
vec![crate::review_classifier::tests::rules_json_of(3)],
);
fake.settings.max_classifications = 2;
let poster = Posted::default();
run_read(&fake, &fake, &poster, &request("r")).await;
let (_, _, review) = poster.0.into_inner().expect("unpoisoned").remove(0);
assert_eq!(review.event, "COMMENT");
assert!(
review.body.contains("rules do not fit a run")
&& review.body.contains("3 classifications")
&& review.body.contains("1 to 2"),
"{}",
review.body
);
assert!(fake.sent.lock().expect("unpoisoned").is_empty());
}
#[tokio::test]
async fn unreadable_settings_run_nothing_and_post_a_comment() {
let fake = Fake::new(

86
swarm-controller/src/review_classifier/settings.rs
View file

@ -1,6 +1,6 @@
//! `GET` and `PUT /api/review-classifier/settings`: the runtime, model and
//! effort every run uses, and how many requested reviews run at once, for the
//! whole swarm.
//! effort every run uses, how many requested reviews run at once and how many
//! classifications one run carries, for the whole swarm.
//!
//! The settings are one document under one key ([`KEY`] in [`BUCKET`]),
//! which only this daemon reads or writes, as the schedule definitions are.
@ -21,7 +21,9 @@ use axum::http::{HeaderMap, StatusCode};
use problem_details::ProblemDetails;
use serde::{Deserialize, Serialize};
use swarm_wire::agent_inference::Effort;
use swarm_wire::review_classifier::{Acp, Runtime};
use swarm_wire::review_classifier::{
Acp, CLASSIFICATION_LIMITS, DEFAULT_MAX_CLASSIFICATIONS, Runtime,
};
use utoipa::ToSchema;
use crate::routes::{AppState, error_problem, wanted_error_status};
@ -56,6 +58,12 @@ pub(crate) struct Settings {
pub(crate) effort: Option<Effort>,
/// How many requested reviews run at once, 1 to 8. Try-outs don't count.
pub(crate) max_concurrent_reviews: usize,
/// The most classifications one run carries, 1 to 1000: a try-out, or a
/// review of a repo whose rules file holds more, is refused above it. The
/// classifications of a run go one after another, each for up to three
/// minutes. A document stored before this field has the default.
#[serde(default = "default_max_classifications")]
pub(crate) max_classifications: usize,
/// What every classification runs on.
#[serde(default)]
pub(crate) runtime: RuntimeKind,
@ -82,12 +90,17 @@ impl Default for Settings {
model: swarm_wire::review_classifier::DEFAULT_MODEL.to_owned(),
effort: None,
max_concurrent_reviews: 2,
max_classifications: DEFAULT_MAX_CLASSIFICATIONS,
runtime: RuntimeKind::ApiKey,
acp: None,
}
}
}
fn default_max_classifications() -> usize {
DEFAULT_MAX_CLASSIFICATIONS
}
impl Settings {
/// What is wrong with these settings, if anything.
fn check(&self) -> Result<(), String> {
@ -100,6 +113,14 @@ impl Settings {
CONCURRENT_REVIEWS.end()
));
}
if !CLASSIFICATION_LIMITS.contains(&self.max_classifications) {
return Err(format!(
"max_classifications {} is not {} to {}",
self.max_classifications,
CLASSIFICATION_LIMITS.start(),
CLASSIFICATION_LIMITS.end()
));
}
if let Some(acp) = &self.acp {
check_acp(acp)?;
}
@ -167,6 +188,9 @@ pub(crate) struct SettingsUpdate {
#[schema(value_type = Option<usize>)]
max_concurrent_reviews: Option<usize>,
#[serde(default, deserialize_with = "present")]
#[schema(value_type = Option<usize>)]
max_classifications: Option<usize>,
#[serde(default, deserialize_with = "present")]
#[schema(value_type = Option<RuntimeKind>)]
runtime: Option<RuntimeKind>,
#[serde(default)]
@ -217,6 +241,9 @@ impl SettingsUpdate {
max_concurrent_reviews: self
.max_concurrent_reviews
.unwrap_or(stored.max_concurrent_reviews),
max_classifications: self
.max_classifications
.unwrap_or(stored.max_classifications),
runtime: self.runtime.unwrap_or(stored.runtime),
acp: self.acp.or(stored.acp),
}
@ -249,6 +276,7 @@ const FIELDS: &[&str] = &[
"model",
"effort",
"max_concurrent_reviews",
"max_classifications",
"runtime",
"acp",
];
@ -301,7 +329,7 @@ fn queue_problem(e: &anyhow::Error) -> ProblemDetails {
responses(
(status = 200, description = "the settings in force; with none stored, \
`{\"model\":\"haiku\",\"effort\":null,\"max_concurrent_reviews\":2,\
\"runtime\":\"api-key\",\"acp\":null}`",
\"max_classifications\":10,\"runtime\":\"api-key\",\"acp\":null}`",
body = Settings),
(status = 503, description = "no swarm queue is configured on this host, or it is \
not connected (problem+json)", body = String),
@ -331,10 +359,12 @@ pub(crate) async fn get_settings(
stored ones, which apply from the next run", body = Settings),
(status = 400, description = "not JSON or not this shape: an unknown field, a \
`null` other than for `effort` or `acp`, an effort other than the five \
levels, a negative or non-integer `max_concurrent_reviews`, a runtime other \
levels, a negative or non-integer `max_concurrent_reviews` or \
`max_classifications`, a runtime other \
than `api-key` or `acp`, an `acp` without `base_url`, `model` or `prices`; or, \
once applied, a model that is empty, has whitespace or starts with `-`, \
`max_concurrent_reviews` outside 1 to 8, an `acp.base_url` that is not \
`max_concurrent_reviews` outside 1 to 8, `max_classifications` outside 1 to \
1000, an `acp.base_url` that is not \
https://, an empty `acp.model` or one with whitespace, a negative price, or \
the `acp` runtime with no `acp` provider (problem+json)", body = String),
(status = 401, description = "the request carries no `Remote-User`, so there is no \
@ -419,7 +449,7 @@ mod tests {
serde_json::to_value(Settings::default()).unwrap(),
serde_json::json!({
"model": "haiku", "effort": null, "max_concurrent_reviews": 2,
"runtime": "api-key", "acp": null,
"max_classifications": 10, "runtime": "api-key", "acp": null,
})
);
assert_eq!(Settings::default().runtime(), Ok(Runtime::ApiKey));
@ -433,6 +463,7 @@ mod tests {
model: "sonnet".to_owned(),
effort: Some(Effort::High),
max_concurrent_reviews: 5,
max_classifications: 40,
runtime: RuntimeKind::Acp,
acp: Some(provider()),
};
@ -465,6 +496,40 @@ mod tests {
}
}
#[test]
fn the_rules_per_run_limit_is_one_to_a_thousand_and_ten_until_set() {
for count in [0, 1001, 5000] {
let bad = Settings {
max_classifications: count,
..Settings::default()
};
assert!(bad.check().is_err(), "{count}");
}
for count in [1, 10, 1000] {
let ok = Settings {
max_classifications: count,
..Settings::default()
};
assert!(ok.check().is_ok(), "{count}");
}
// A document stored before the field existed decodes with the default.
let old = r#"{"model":"haiku","effort":null,"max_concurrent_reviews":2}"#;
assert_eq!(decode(old.as_bytes()).max_classifications, 10);
// An update changes it alone.
let raised = update(r#"{"max_classifications":300}"#)
.unwrap()
.apply(Settings::default());
assert_eq!(
raised,
Settings {
max_classifications: 300,
..Settings::default()
}
);
assert!(update(r#"{"max_classifications":null}"#).is_err());
assert!(update(r#"{"max_classifications":-1}"#).is_err());
}
#[test]
fn a_bad_acp_provider_is_refused() {
let with = |change: fn(&mut Acp)| {
@ -571,6 +636,7 @@ mod tests {
model: "sonnet".to_owned(),
effort: Some(Effort::High),
max_concurrent_reviews: 3,
max_classifications: 25,
runtime: RuntimeKind::Acp,
acp: Some(provider()),
};
@ -666,7 +732,7 @@ mod tests {
read,
serde_json::json!({
"model": "sonnet", "effort": "high", "max_concurrent_reviews": 3,
"runtime": "api-key", "acp": null,
"max_classifications": 10, "runtime": "api-key", "acp": null,
})
);
@ -704,7 +770,7 @@ mod tests {
let wanted = serde_json::json!({
"model": "sonnet", "effort": "high", "max_concurrent_reviews": 5,
"runtime": "api-key", "acp": null,
"max_classifications": 10, "runtime": "api-key", "acp": null,
});
let (status, stored, _) = send(&app, put(&wanted.to_string())).await;
assert_eq!(status, StatusCode::OK, "{stored}");
@ -745,7 +811,7 @@ mod tests {
after,
serde_json::json!({
"model": "sonnet", "effort": null, "max_concurrent_reviews": 4,
"runtime": "acp", "acp": acp,
"max_classifications": 10, "runtime": "acp", "acp": acp,
})
);
let read = nats.client().await.settings().await.expect("readable");