swarm-controller: the review classifier reads a diff with 200 lines of context

The classifier read the forge's PR `.diff`, whose context is fixed at 3
lines. A block moved into a glue module lost its enclosing gate (a
`config = lib.mkIf …` 13 to 191 lines up in the misfired moves #5199,
#5200, #5201), so the glue restating that gate read as a new condition.

The controller now builds the diff itself: it lists the PR's changed files,
reads each at the merge base and at the head through the git tree and blob
API with the token it already holds, and diffs them with 200 lines of
context. It sends the forge's `.diff` instead when the PR changes more than
100 files, the forge names no merge base, or the wide diff is over 256 KiB,
and the posted review then says so with the reason.

Fixes #5227

swarm-controller/src/forge/pull_diff.rs

@@ -0,0 +1,235 @@
+//! The diff the review classifier reads, built from both sides of each
+//! changed file rather than taken from the forge's `.diff`, whose context is
+//! fixed at 3 lines. With [`CONTEXT_LINES`] of context a moved block keeps
+//! its enclosing gate (a `config = lib.mkIf …`, an option's `default`) in
+//! view, so a move that restates that gate does not read as adding one.
+
+use std::fmt::Write as _;
+
+use std::time::Duration;
+
+use similar::TextDiff;
+
+/// Lines of context around each change. In this repo's Nix modules a block's
+/// enclosing `config = lib.mkIf …` sits up to 191 lines above it.
+pub const CONTEXT_LINES: usize = 200;
+
+/// The largest wide diff sent, in bytes, about 64k tokens of code. A larger
+/// one gives way to the forge's `.diff`.
+pub const MAX_BYTES: usize = 256 * 1024;
+
+/// The most changed files a wide diff is read for. Each costs a few forge
+/// reads per side, all within the one deadline the PR read runs under.
+pub const MAX_FILES: usize = 100;
+
+/// How long one file's diff may search for the smallest edit before settling
+/// for a larger one. Without it a rewritten file of tens of thousands of
+/// lines takes minutes.
+const DIFF_TIMEOUT: Duration = Duration::from_secs(1);
+
+/// One changed file: its path and bytes at the merge base and at the head,
+/// `None` on a side where it does not exist.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct FileChange {
+    pub old_path: String,
+    pub new_path: String,
+    pub old: Option<Vec<u8>>,
+    pub new: Option<Vec<u8>>,
+}
+
+/// Why a PR of `count` changed files gets the forge's `.diff`, if it does.
+pub fn too_many_files(count: usize) -> Option<String> {
+    (count > MAX_FILES).then(|| {
+        format!("the PR changes {count} files, more than the {MAX_FILES} a wide diff is read for")
+    })
+}
+
+/// `files` as one diff with [`CONTEXT_LINES`] of context, or why the forge's
+/// `.diff` stands in: the result is over [`MAX_BYTES`].
+pub fn wide(files: &[FileChange]) -> Result<String, String> {
+    let diff = render(files, CONTEXT_LINES);
+    if diff.len() > MAX_BYTES {
+        return Err(format!(
+            "the diff with {CONTEXT_LINES} lines of context is {} bytes, over the {MAX_BYTES} \
+             a review sends",
+            diff.len()
+        ));
+    }
+    Ok(diff)
+}
+
+/// `files` as one unified diff with `context` lines around each change, each
+/// file in `git diff`'s shape: a `diff --git` line, `---`/`+++`, then hunks.
+/// A file with a non-UTF-8 or NUL-bearing side is one `Binary files` line.
+pub fn render(files: &[FileChange], context: usize) -> String {
+    let mut out = String::new();
+    for file in files {
+        section(&mut out, file, context);
+    }
+    out
+}
+
+fn section(out: &mut String, file: &FileChange, context: usize) {
+    let (old_path, new_path) = (&file.old_path, &file.new_path);
+    let _ = writeln!(out, "diff --git a/{old_path} b/{new_path}");
+    if file.old.is_some() && file.new.is_some() && old_path != new_path {
+        let _ = writeln!(out, "rename from {old_path}\nrename to {new_path}");
+    }
+    let a = file
+        .old
+        .as_ref()
+        .map_or_else(|| "/dev/null".to_owned(), |_| format!("a/{old_path}"));
+    let b = file
+        .new
+        .as_ref()
+        .map_or_else(|| "/dev/null".to_owned(), |_| format!("b/{new_path}"));
+    let (Some(old), Some(new)) = (text(file.old.as_deref()), text(file.new.as_deref())) else {
+        let _ = writeln!(out, "Binary files {a} and {b} differ");
+        return;
+    };
+    let diff = TextDiff::configure()
+        .timeout(DIFF_TIMEOUT)
+        .diff_lines(old, new);
+    let _ = write!(
+        out,
+        "{}",
+        diff.unified_diff().context_radius(context).header(&a, &b)
+    );
+}
+
+/// A side's text: empty when the file is absent, `None` when it is binary.
+fn text(bytes: Option<&[u8]>) -> Option<&str> {
+    let bytes = bytes.unwrap_or_default();
+    if bytes.contains(&0) {
+        return None;
+    }
+    std::str::from_utf8(bytes).ok()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn change(path: &str, old: Option<&str>, new: Option<&str>) -> FileChange {
+        FileChange {
+            old_path: path.to_owned(),
+            new_path: path.to_owned(),
+            old: old.map(|s| s.as_bytes().to_vec()),
+            new: new.map(|s| s.as_bytes().to_vec()),
+        }
+    }
+
+    /// A Nix module whose `config = lib.mkIf …` block opens `gap` lines above
+    /// a firewall rule on the wireguard interface.
+    fn module(gap: usize, with_rule: bool) -> String {
+        let mut s =
+            String::from("{ lib, config, ... }:\n{\n  config = lib.mkIf deployCfg.nats.enable {\n");
+        for i in 0..gap {
+            let _ = writeln!(s, "    services.nats.settings.k{i} = {i};");
+        }
+        if with_rule {
+            s.push_str(
+                "    networking.firewall.interfaces.wg0 = {\n      allowedTCPPorts = lib.mkIf \
+                 deployCfg.wireguard.enable [ 4222 ];\n    };\n",
+            );
+        }
+        for i in 0..5 {
+            let _ = writeln!(s, "    systemd.services.nats.environment.E{i} = \"{i}\";");
+        }
+        s.push_str("  };\n}\n");
+        s
+    }
+
+    /// The rule moves out of the module into a glue module that restates
+    /// both conditions.
+    fn move_into_glue() -> Vec<FileChange> {
+        let glue = "{ lib, config, ... }:\n{\n  config = lib.mkIf (deployCfg.nats.enable && \
+                    deployCfg.wireguard.enable) {\n    networking.firewall.interfaces.wg0.\
+                    allowedTCPPorts = [ 4222 ];\n  };\n}\n";
+        vec![
+            change("nix/host-modules/glue-nats-wireguard.nix", None, Some(glue)),
+            change(
+                "nix/host-modules/swarm-nats.nix",
+                Some(&module(47, true)),
+                Some(&module(47, false)),
+            ),
+        ]
+    }
+
+    /// `count` lines, line `i` reading `line(i)`.
+    fn numbered(count: usize, line: impl Fn(usize) -> String) -> String {
+        (0..count).fold(String::new(), |mut s, i| {
+            let _ = writeln!(s, "{}", line(i));
+            s
+        })
+    }
+
+    const GATE: &str = "config = lib.mkIf deployCfg.nats.enable {";
+
+    #[test]
+    fn a_moved_block_keeps_its_enclosing_gate_in_view() {
+        let diff = wide(&move_into_glue()).expect("fits");
+        assert!(diff.contains(&format!("\n   {GATE}\n")), "{diff}");
+        assert!(diff.contains("\n-      allowedTCPPorts = lib.mkIf deployCfg.wireguard.enable"));
+    }
+
+    #[test]
+    fn three_lines_of_context_leave_that_gate_out() {
+        let diff = render(&move_into_glue(), 3);
+        assert!(!diff.contains(GATE), "{diff}");
+    }
+
+    #[test]
+    fn context_is_as_wide_as_asked_and_no_wider() {
+        let old = numbered(30, |i| format!("line {i}"));
+        let new = old.replace("line 15\n", "line fifteen\n");
+        let at = |n| render(&[change("f", Some(&old), Some(&new))], n);
+        let diff = at(10);
+        assert!(diff.contains("@@ -6,21 +6,21 @@"), "{diff}");
+        assert!(diff.contains(" line 5\n") && !diff.contains("line 4\n"));
+        assert!(at(CONTEXT_LINES).contains("@@ -1,30 +1,30 @@"));
+    }
+
+    #[test]
+    fn file_headers_follow_git_diff() {
+        let diff = render(
+            &[
+                change("added", None, Some("a\n")),
+                change("gone", Some("a\n"), None),
+                FileChange {
+                    old_path: "old".to_owned(),
+                    new_path: "new".to_owned(),
+                    old: Some(b"a\n".to_vec()),
+                    new: Some(b"b\n".to_vec()),
+                },
+                change("img", Some("a\n"), Some("\0")),
+            ],
+            3,
+        );
+        assert_eq!(
+            diff,
+            "diff --git a/added b/added\n--- /dev/null\n+++ b/added\n@@ -0,0 +1 @@\n+a\n\
+             diff --git a/gone b/gone\n--- a/gone\n+++ /dev/null\n@@ -1 +0,0 @@\n-a\n\
+             diff --git a/old b/new\nrename from old\nrename to new\n--- a/old\n+++ b/new\n\
+             @@ -1 +1 @@\n-a\n+b\n\
+             diff --git a/img b/img\nBinary files a/img and b/img differ\n"
+        );
+    }
+
+    #[test]
+    fn a_wide_diff_over_the_limit_gives_way() {
+        let big = numbered(MAX_BYTES / 8, |i| format!("{i:07}"));
+        let edited = numbered(MAX_BYTES / 8, |i| format!("{i:07}x"));
+        let files = [change("big", Some(&big), Some(&edited))];
+        let reason = wide(&files).expect_err("over the limit");
+        assert!(reason.contains("over the"), "{reason}");
+        let small = [change("small", Some("a\n"), Some("b\n"))];
+        assert!(wide(&small).is_ok());
+    }
+
+    #[test]
+    fn more_files_than_the_limit_give_way() {
+        assert!(too_many_files(MAX_FILES).is_none());
+        assert!(too_many_files(MAX_FILES + 1).is_some());
+    }
+}