swarm-controller: the review classifier reads a diff with 200 lines of context

The classifier read the forge's PR `.diff`, whose context is fixed at 3
lines. A block moved into a glue module lost its enclosing gate (a
`config = lib.mkIf …` 13 to 191 lines up in the misfired moves #5199,
#5200, #5201), so the glue restating that gate read as a new condition.

The controller now builds the diff itself: it lists the PR's changed files,
reads each at the merge base and at the head through the git tree and blob
API with the token it already holds, and diffs them with 200 lines of
context. It sends the forge's `.diff` instead when the PR changes more than
100 files, the forge names no merge base, or the wide diff is over 256 KiB,
and the posted review then says so with the reason.

Fixes #5227

swarm-controller/src/forge.rs

@@ -29,7 +29,8 @@ use forgejo_api::structs::{
     CreateRepoOption, CreateStatusOption, CreateUserOption, EditUserOption, GetTreeQuery, GitEntry,
     IssueListIssuesQuery, IssueListIssuesQueryState, IssueListIssuesQueryType,
     RepoDownloadPullDiffOrPatchQuery, RepoGetContentsQuery, RepoGetPullRequestCommitsQuery,
-    RepoListPullRequestsQuery, RepoListPullRequestsQueryState, RepoSearchQuery, StateType,
+    RepoGetPullRequestFilesQuery, RepoListPullRequestsQuery, RepoListPullRequestsQueryState,
+    RepoSearchQuery, StateType,
 };
 use forgejo_api::{ApiErrorKind, ForgejoError};
 use futures_util::{StreamExt as _, TryStreamExt as _};
@@ -51,6 +52,7 @@ pub mod docs_grant;
 pub mod hive_token;
 pub mod legacy_tokens;
 pub mod objects;
+pub mod pull_diff;
 pub mod runner_token;
 pub mod site_admin;
 
@@ -114,6 +116,9 @@ pub struct PullForReview {
     pub head_sha: String,
     pub commits: Vec<Commit>,
     pub diff: String,
+    /// Why `diff` is the forge's `.diff`, with 3 lines of context, rather
+    /// than [`pull_diff::wide`]'s. `None` when it is the wide one.
+    pub narrowed: Option<String>,
 }
 
 /// Files on a repo's default branch, as [`Client::default_branch_files`] reads
@@ -1242,7 +1247,8 @@ impl Client {
 
     /// What the review classifier reads of `owner/repo`'s PR `number`: its
     /// head commit, its commits oldest first with their full messages, and
-    /// its unified diff.
+    /// its unified diff from the merge base: [`pull_diff::wide`]'s, or the
+    /// forge's `.diff` when the wide one is not read or is too large.
     ///
     /// # Errors
     /// A forge failure, with the [`ForgejoError`] underneath; a 404 on the
@@ -1259,6 +1265,7 @@ impl Client {
             .call(|api| api.repo_get_pull_request(owner, repo, number).into_future())
             .await
             .with_context(|| format!("read PR {number} of {owner}/{repo}"))?;
+        let (merge_base, changed_files) = (pull.merge_base, pull.changed_files);
         let head_sha = pull
             .head
             .and_then(|h| h.sha)
@@ -1287,8 +1294,40 @@ impl Client {
                 message: c.commit.and_then(|m| m.message).unwrap_or_default(),
             })
             .collect();
-        let diff = self
-            .api
+        let count = changed_files.map_or(0, |n| usize::try_from(n).unwrap_or(usize::MAX));
+        let wide = match (pull_diff::too_many_files(count), merge_base) {
+            (Some(reason), _) => Err(reason),
+            (None, None) => Err("the forge names no merge base".to_owned()),
+            (None, Some(base)) => {
+                let files = self
+                    .changed_files(owner, repo, number, &base, &head_sha)
+                    .await?;
+                tokio::task::spawn_blocking(move || pull_diff::wide(&files))
+                    .await
+                    .context("building the PR's wide diff")?
+            }
+        };
+        let (diff, narrowed) = match wide {
+            Ok(diff) => (diff, None),
+            Err(reason) => {
+                tracing::info!(
+                    %owner, %repo, pr = number, %reason,
+                    "the review classifier reads the forge's 3-line diff"
+                );
+                (self.forge_diff(owner, repo, number).await?, Some(reason))
+            }
+        };
+        Ok(PullForReview {
+            head_sha,
+            commits,
+            diff,
+            narrowed,
+        })
+    }
+
+    /// The forge's own `.diff` of PR `number`, with 3 lines of context.
+    async fn forge_diff(&self, owner: &str, repo: &str, number: i64) -> Result<String> {
+        self.api
             .call(|api| {
                 api.repo_download_pull_diff_or_patch(
                     owner,
@@ -1300,12 +1339,55 @@ impl Client {
                 .into_future()
             })
             .await
-            .with_context(|| format!("read the diff of PR {number} of {owner}/{repo}"))?;
-        Ok(PullForReview {
-            head_sha,
-            commits,
-            diff,
-        })
+            .with_context(|| format!("read the diff of PR {number} of {owner}/{repo}"))
+    }
+
+    /// Each file PR `number` changes, with its bytes at `base` and at `head`.
+    /// A renamed file is read at its old path on the `base` side.
+    async fn changed_files(
+        &self,
+        owner: &str,
+        repo: &str,
+        number: i64,
+        base: &str,
+        head: &str,
+    ) -> Result<Vec<pull_diff::FileChange>> {
+        let listed = self
+            .api
+            .call(|api| {
+                Box::pin(
+                    api.repo_get_pull_request_files(
+                        owner,
+                        repo,
+                        number,
+                        RepoGetPullRequestFilesQuery::default(),
+                    )
+                    .all(),
+                )
+            })
+            .await
+            .with_context(|| format!("list the files of PR {number} of {owner}/{repo}"))?;
+        let mut trees = HashMap::new();
+        let mut files = Vec::with_capacity(listed.len());
+        for file in listed {
+            let new_path = file
+                .filename
+                .with_context(|| format!("a file of PR {number} of {owner}/{repo} has no name"))?;
+            let old_path = file.previous_filename.unwrap_or_else(|| new_path.clone());
+            let old = self
+                .file_at(owner, repo, base, &old_path, &mut trees)
+                .await?;
+            let new = self
+                .file_at(owner, repo, head, &new_path, &mut trees)
+                .await?;
+            files.push(pull_diff::FileChange {
+                old_path,
+                new_path,
+                old: old.map(|(_, bytes)| bytes),
+                new: new.map(|(_, bytes)| bytes),
+            });
+        }
+        Ok(files)
     }
 
     /// The `paths` of `owner/repo` that exist on its default branch, all
@@ -2437,6 +2519,111 @@ mod tests {
         (Client { api, url }, seen)
     }
 
+    /// A stand-in forge holding PR 7 of `o/r`, head `h1`, which changes
+    /// `m.nix` from `old` at merge base `mb` to `new`. `pull` is the PR's own
+    /// JSON. Records every request's path.
+    async fn pull_forge(
+        pull: &'static str,
+        old: &'static str,
+        new: &'static str,
+    ) -> (Client, Arc<std::sync::Mutex<Vec<String>>>) {
+        use axum::http::Uri;
+        use base64::Engine as _;
+
+        let blob = |content: &str| {
+            format!(
+                r#"{{"encoding":"base64","content":"{}"}}"#,
+                base64::engine::general_purpose::STANDARD.encode(content)
+            )
+        };
+        let tree = |sha: &str, blob: &str| {
+            format!(
+                r#"{{"sha":"{sha}","truncated":false,"tree":[
+                    {{"path":"m.nix","type":"blob","sha":"{blob}"}}]}}"#
+            )
+        };
+        let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let recorded = Arc::clone(&seen);
+        let app = axum::Router::new().fallback(move |uri: Uri| {
+            let recorded = Arc::clone(&recorded);
+            let body = match uri.path() {
+                "/api/v1/repos/o/r/pulls/7" => Some(pull.to_owned()),
+                "/api/v1/repos/o/r/pulls/7/commits" => Some("[]".to_owned()),
+                "/api/v1/repos/o/r/pulls/7/files" => {
+                    Some(r#"[{"filename":"m.nix","status":"modified"}]"#.to_owned())
+                }
+                "/api/v1/repos/o/r/pulls/7.diff" => Some("forge diff\n".to_owned()),
+                "/api/v1/repos/o/r/git/trees/mb" => Some(tree("mb", "bo")),
+                "/api/v1/repos/o/r/git/trees/h1" => Some(tree("h1", "bn")),
+                "/api/v1/repos/o/r/git/blobs/bo" => Some(blob(old)),
+                "/api/v1/repos/o/r/git/blobs/bn" => Some(blob(new)),
+                _ => None,
+            };
+            async move {
+                recorded.lock().unwrap().push(uri.path().to_owned());
+                let builder = axum::http::Response::builder();
+                let (builder, body) = match body {
+                    Some(body) => {
+                        let count = if body == "[]" { "0" } else { "1" };
+                        (builder.status(200).header("x-total-count", count), body)
+                    }
+                    None => (builder.status(404), r#"{"message":"not found"}"#.to_owned()),
+                };
+                builder.body(axum::body::Body::from(body)).unwrap()
+            }
+        });
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let url = url::Url::parse(&format!("http://{}/", listener.local_addr().unwrap())).unwrap();
+        tokio::spawn(async move { axum::serve(listener, app).await });
+        let api = crate::forge::api::tests::fixed(url.clone(), "stub-token");
+        (Client { api, url }, seen)
+    }
+
+    #[tokio::test]
+    async fn a_review_reads_the_wide_diff_from_the_merge_base_or_says_why_not() {
+        let old = "{\n  config = lib.mkIf gate {\n    a = 1;\n    b = 2;\n    c = 3;\n    d = 4;\n    e = 5;\n  };\n}\n";
+        let new = "{\n  config = lib.mkIf gate {\n    a = 1;\n    b = 2;\n    c = 3;\n    d = 4;\n  };\n}\n";
+        let wide = r#"{"number":7,"requested_reviewers":[],"head":{"sha":"h1"},"merge_base":"mb","changed_files":1}"#;
+        let (client, _) = pull_forge(wide, old, new).await;
+        let read = client.pull_for_review("o", "r", 7).await.unwrap();
+        assert_eq!(read.narrowed, None);
+        assert!(
+            read.diff
+                .starts_with("diff --git a/m.nix b/m.nix\n--- a/m.nix\n+++ b/m.nix\n"),
+            "{}",
+            read.diff
+        );
+        assert!(
+            read.diff.contains("\n   config = lib.mkIf gate {\n"),
+            "{}",
+            read.diff
+        );
+        assert!(read.diff.contains("\n-    e = 5;\n"), "{}", read.diff);
+
+        let many = r#"{"number":7,"requested_reviewers":[],"head":{"sha":"h1"},"merge_base":"mb","changed_files":101}"#;
+        let (client, seen) = pull_forge(many, old, new).await;
+        let read = client.pull_for_review("o", "r", 7).await.unwrap();
+        assert_eq!(read.diff, "forge diff\n");
+        assert_eq!(
+            read.narrowed.as_deref(),
+            Some("the PR changes 101 files, more than the 100 a wide diff is read for")
+        );
+        assert!(
+            !seen.lock().unwrap().iter().any(|p| p.ends_with("/files")),
+            "no file is read for a PR past the limit"
+        );
+
+        let no_base =
+            r#"{"number":7,"requested_reviewers":[],"head":{"sha":"h1"},"changed_files":1}"#;
+        let (client, _) = pull_forge(no_base, old, new).await;
+        let read = client.pull_for_review("o", "r", 7).await.unwrap();
+        assert_eq!(read.diff, "forge diff\n");
+        assert_eq!(
+            read.narrowed.as_deref(),
+            Some("the forge names no merge base")
+        );
+    }
+
     /// A root tree with the `.hyperhive` tree and a blob beside it.
     const ROOT: &str = r#"{"sha":"def456","truncated":false,"tree":[
         {"path":"README.md","type":"blob","sha":"b0"},