hyperhive/swarm-builder/src/nix_build.rs

//! One build: `nix build` of the check flake with its three inputs pointed at
//! this host's hyperhive and nixpkgs and at the agent's config
//! commit. One meta build: `nix build` of one agent's toplevel in
//! `internal/meta` at a commit, every input at the flake's own lock. One lock:
//! `nix flake lock` of the swarm meta flake in a scratch directory.
//!
//! The config repo is private, so `git` fetches it with the builder's forge
//! token. The token reaches `git` through a credential helper that reads it
//! from the child's environment: it is in no argument, no URL and no lock, so
//! no error line `nix` prints can carry it into the reply.
//!
//! ⚠️ A config's own flake inputs are fetched by the same `git`, from hosts the
//! config names. The helper is configured for the forge's URL only, and itself
//! answers only an `https` request for the forge's host, so the token never
//! goes anywhere else.

use std::path::PathBuf;
use std::process::Stdio;
use std::time::Duration;

use swarm_wire::builder::{
    BuildAgent, BuildFromMeta, FromMetaReply, LockMeta, LockReply, META_REPO_PATH, Reply,
};
use tokio::io::AsyncReadExt as _;

/// The most of `nix`'s stderr a failed build's reply carries, from its end.
const TAIL_LINES: usize = 30;
const TAIL_BYTES: usize = 4000;

/// How much of each stream is held while `nix` runs, from its end. Output
/// under this is read whole; past it, only the end is kept, so a build's log
/// cannot grow the builder's memory.
const KEEP_BYTES: usize = 64 * 1024;

/// The variable the credential helper reads the token from, in the child only.
const TOKEN_VAR: &str = "SWARM_BUILDER_GIT_TOKEN";

/// The variable holding the one host the helper answers for, in the child
/// only.
const HOST_VAR: &str = "SWARM_BUILDER_GIT_HOST";

/// What every build is run with, fixed at start.
pub struct Builder {
    /// The `nix` binary.
    pub nix: PathBuf,
    /// The check flake's directory in the store.
    pub check_flake: String,
    /// The hyperhive source this host was deployed from.
    pub hyperhive: String,
    /// The nixpkgs source this host was deployed from.
    pub nixpkgs: String,
    /// `https://<forge>`, with no trailing slash and no path.
    pub forge_url: String,
    /// The org the config repos are in.
    pub config_org: String,
    /// How long one build may run.
    pub deadline: Duration,
    /// How long one lock may run.
    pub lock_deadline: Duration,
    /// The platform this host builds for, reported with each meta build.
    pub system: String,
    /// Where a meta build's closure is pushed after it succeeds; `None` pushes
    /// nothing.
    pub cache: Option<crate::push::Remote>,
}

impl Builder {
    /// Build `job` with `token` as the forge credential.
    pub async fn build(&self, job: &BuildAgent, token: &str) -> Reply {
        if let Err(error) = validate(job) {
            return Reply::Failed { error_tail: error };
        }
        let mut cmd = self.nix_command(token);
        cmd.args(BUILD_FLAGS).args(self.args(job));
        run(cmd, self.deadline).await
    }

    /// Build `job`'s agent from `internal/meta` with `token` as the forge
    /// credential.
    pub async fn build_from_meta(&self, job: &BuildFromMeta, token: &str) -> FromMetaReply {
        if let Err(error_tail) = validate_meta(job) {
            return FromMetaReply::Failed { error_tail };
        }
        let mut cmd = self.nix_command(token);
        cmd.args(BUILD_FLAGS).arg(self.meta_attr(job));
        // Any build that doesn't succeed is `Failed`, a timeout included; the
        // controller builds it again on its next round.
        match run_raw(cmd, self.deadline).await {
            Err(error_tail) => FromMetaReply::Failed { error_tail },
            Ok((status, out, err)) => match verdict(status, &out, &err) {
                Reply::Built { out_path } => FromMetaReply::Built {
                    out_path,
                    system: self.system.clone(),
                    cached: false,
                },
                Reply::Failed { error_tail } => FromMetaReply::Failed { error_tail },
            },
        }
    }

    /// Lock `job`'s flake with `token` as the forge credential, in a scratch
    /// directory removed afterwards, and answer with the lock `nix` wrote.
    pub async fn lock(&self, job: &LockMeta, token: &str) -> LockReply {
        let scratch = match Scratch::new() {
            Ok(s) => s,
            Err(e) => {
                return LockReply::Failed {
                    error_tail: format!("creating the scratch directory: {e}"),
                };
            }
        };
        if let Err(e) = std::fs::write(scratch.0.join("flake.nix"), &job.flake_nix) {
            return LockReply::Failed {
                error_tail: format!("writing flake.nix: {e}"),
            };
        }
        let mut cmd = self.nix_command(token);
        cmd.args(LOCK_ARGS).current_dir(&scratch.0);
        match run_raw(cmd, self.lock_deadline).await {
            Err(error_tail) => LockReply::Failed { error_tail },
            Ok((status, _, err)) if !status.success() => LockReply::Failed {
                error_tail: tail(&String::from_utf8_lossy(&err), status),
            },
            Ok(_) => match std::fs::read_to_string(scratch.0.join("flake.lock")) {
                Ok(flake_lock) => LockReply::Locked { flake_lock },
                Err(e) => LockReply::Failed {
                    error_tail: format!("nix exited 0 but flake.lock is unreadable: {e}"),
                },
            },
        }
    }

    /// `nix` with the forge credential reachable by `git` alone, and its
    /// output captured.
    fn nix_command(&self, token: &str) -> tokio::process::Command {
        let mut cmd = tokio::process::Command::new(&self.nix);
        cmd.env(TOKEN_VAR, token)
            .env("GIT_TERMINAL_PROMPT", "0")
            .env("GIT_CONFIG_COUNT", "1")
            .env(HOST_VAR, forge_host(&self.forge_url))
            .env(
                "GIT_CONFIG_KEY_0",
                format!("credential.{}.helper", self.forge_url),
            )
            .env("GIT_CONFIG_VALUE_0", credential_helper())
            .stdin(Stdio::null())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .kill_on_drop(true);
        cmd
    }

    /// The `nix build` arguments for `job`, after [`BUILD_FLAGS`].
    fn args(&self, job: &BuildAgent) -> Vec<String> {
        vec![
            format!("{}#check", self.check_flake),
            "--override-input".to_owned(),
            "hyperhive".to_owned(),
            self.hyperhive.clone(),
            "--override-input".to_owned(),
            "nixpkgs".to_owned(),
            self.nixpkgs.clone(),
            "--override-input".to_owned(),
            "agent".to_owned(),
            self.agent_url(job),
        ]
    }

    /// The agent's toplevel in `internal/meta` at the commit, the attribute
    /// the hives build. No input is overridden: the committed `flake.lock`
    /// decides all of them.
    fn meta_attr(&self, job: &BuildFromMeta) -> String {
        format!(
            "git+{}/{META_REPO_PATH}.git?rev={}#nixosConfigurations.{}.config.system.build.toplevel",
            self.forge_url, job.meta_rev, job.agent
        )
    }

    /// The config commit as a flake reference, fetched through the PR's head
    /// ref.
    fn agent_url(&self, job: &BuildAgent) -> String {
        format!(
            "git+{}/{}/{}?ref=refs/pull/{}/head&rev={}",
            self.forge_url, self.config_org, job.agent, job.pr, job.rev
        )
    }
}

/// What every `nix build` starts with: nothing linked, nothing written to the
/// flake's lock, the flake's own `nix.conf` settings refused.
const BUILD_FLAGS: [&str; 8] = [
    "build",
    "--no-link",
    "--print-out-paths",
    "--no-write-lock-file",
    "--option",
    "accept-flake-config",
    "false",
    "--print-build-logs",
];

/// `nix flake lock` of the flake in the working directory. The flake's config
/// is not trusted, and nothing is built.
const LOCK_ARGS: &[&str] = &["flake", "lock", "--option", "accept-flake-config", "false"];

/// A fresh directory under the temp dir, removed with everything in it on drop.
struct Scratch(PathBuf);

impl Scratch {
    fn new() -> std::io::Result<Self> {
        use std::sync::atomic::{AtomicU64, Ordering};
        static NEXT: AtomicU64 = AtomicU64::new(0);
        let dir = std::env::temp_dir().join(format!(
            "lock-meta-{}-{}",
            std::process::id(),
            NEXT.fetch_add(1, Ordering::Relaxed)
        ));
        std::fs::create_dir(&dir)?;
        Ok(Self(dir))
    }
}

impl Drop for Scratch {
    fn drop(&mut self) {
        let _ = std::fs::remove_dir_all(&self.0);
    }
}

/// The host part of `forge_url` (`https://<host>[:port]`), as `git` names it
/// to a credential helper.
fn forge_host(forge_url: &str) -> &str {
    let rest = forge_url.strip_prefix("https://").unwrap_or(forge_url);
    rest.split('/').next().unwrap_or(rest)
}

/// A `git` credential helper that answers a `get` with the builder's login and
/// the token from [`TOKEN_VAR`] only when the request is `https` to the host in
/// [`HOST_VAR`], and with nothing otherwise.
fn credential_helper() -> String {
    format!(
        "!f() {{ test \"$1\" = get || return 0; p=; h=; \
         while IFS='=' read -r k v; do case \"$k\" in '') break ;; protocol) p=$v ;; host) h=$v ;; esac; done; \
         test -n \"${HOST_VAR}\" && test \"$p\" = https && test \"$h\" = \"${HOST_VAR}\" \
         && printf 'username=builder\\npassword=%s\\n' \"${TOKEN_VAR}\"; return 0; }}; f"
    )
}

/// Refuse a job whose fields would change the reference's meaning: an agent
/// is one `[a-z0-9-]` segment, and a rev is a full commit id.
fn validate(job: &BuildAgent) -> Result<(), String> {
    check_agent_and_rev(&job.agent, &job.rev)?;
    if job.pr == 0 {
        return Err("PR 0 does not exist".to_owned());
    }
    Ok(())
}

/// [`validate`] for a meta build: the same agent and commit rules.
fn validate_meta(job: &BuildFromMeta) -> Result<(), String> {
    check_agent_and_rev(&job.agent, &job.meta_rev)
}

fn check_agent_and_rev(agent: &str, rev: &str) -> Result<(), String> {
    let agent_ok = !agent.is_empty()
        && agent
            .bytes()
            .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-');
    if !agent_ok {
        return Err(format!("{agent:?} is not an agent name"));
    }
    let rev_ok = rev.len() == 40
        && rev
            .bytes()
            .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b));
    if !rev_ok {
        return Err(format!("{rev:?} is not a full commit id"));
    }
    Ok(())
}

/// Run `cmd` to its end or `deadline`, and read the store path it printed or
/// the tail of what it said.
async fn run(cmd: tokio::process::Command, deadline: Duration) -> Reply {
    match run_raw(cmd, deadline).await {
        Err(error_tail) => Reply::Failed { error_tail },
        Ok((status, out, err)) => verdict(status, &out, &err),
    }
}

/// What a `nix build` that exited said: the store path it printed, or the tail
/// of what it said.
fn verdict(status: std::process::ExitStatus, out: &[u8], err: &[u8]) -> Reply {
    let out_path = String::from_utf8_lossy(out)
        .lines()
        .rfind(|l| !l.trim().is_empty())
        .map(|l| l.trim().to_owned());
    match out_path {
        Some(out_path) if status.success() => Reply::Built { out_path },
        _ => Reply::Failed {
            error_tail: tail(&String::from_utf8_lossy(err), status),
        },
    }
}

/// Run `cmd` to its end or `deadline`: its status and the kept end of its
/// stdout and stderr, or why it did not finish.
async fn run_raw(
    mut cmd: tokio::process::Command,
    deadline: Duration,
) -> Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>), String> {
    let mut child = match cmd.spawn() {
        Ok(child) => child,
        Err(e) => return Err(format!("starting nix: {e}")),
    };
    let stdout = child.stdout.take().expect("piped");
    let stderr = child.stderr.take().expect("piped");
    let work = async {
        let (o, e, status) = tokio::join!(
            read_tail(stdout, KEEP_BYTES),
            read_tail(stderr, KEEP_BYTES),
            child.wait()
        );
        let (out, err) = (o?, e?);
        status.map(|s| (s, out, err))
    };
    match tokio::time::timeout(deadline, work).await {
        Err(_) => Err(format!(
            "nix timed out: it did not finish within {} minutes",
            deadline.as_secs() / 60
        )),
        Ok(Err(e)) => Err(format!("waiting for nix: {e}")),
        Ok(Ok(done)) => Ok(done),
    }
}

/// The last `keep` bytes of a stream, held while it is read: at most
/// `2 * keep` bytes after each push.
struct Tail {
    buf: Vec<u8>,
    keep: usize,
    cut: bool,
}

impl Tail {
    fn new(keep: usize) -> Self {
        Self {
            buf: Vec::new(),
            keep,
            cut: false,
        }
    }

    fn push(&mut self, chunk: &[u8]) {
        self.buf.extend_from_slice(chunk);
        // Trim at twice the cap, so a copy happens once per `keep` bytes.
        if self.buf.len() > self.keep * 2 {
            self.trim();
        }
    }

    fn trim(&mut self) {
        self.buf.drain(..self.buf.len() - self.keep);
        self.cut = true;
    }

    /// The kept bytes. When the stream was longer than `keep`, the cut line
    /// at the front is dropped, so the result starts on a line boundary,
    /// unless that line is all the window holds.
    fn finish(mut self) -> Vec<u8> {
        if self.buf.len() > self.keep {
            self.trim();
        }
        if self.cut
            && let Some(nl) = self.buf.iter().position(|&b| b == b'\n')
            && nl + 1 < self.buf.len()
        {
            self.buf.drain(..=nl);
        }
        self.buf
    }
}

/// Read `reader` to its end, keeping only its last `keep` bytes (see [`Tail`]).
async fn read_tail(
    mut reader: impl tokio::io::AsyncRead + Unpin,
    keep: usize,
) -> std::io::Result<Vec<u8>> {
    let mut tail = Tail::new(keep);
    let mut chunk = vec![0u8; 8192];
    loop {
        let n = reader.read(&mut chunk).await?;
        if n == 0 {
            return Ok(tail.finish());
        }
        tail.push(&chunk[..n]);
    }
}

/// The last [`TAIL_LINES`] lines of `stderr`, at most [`TAIL_BYTES`] bytes,
/// or the exit status when it said nothing.
pub(crate) fn tail(stderr: &str, status: std::process::ExitStatus) -> String {
    last_lines(stderr).unwrap_or_else(|| format!("nix exited with {status} and printed nothing"))
}

/// The last [`TAIL_LINES`] non-blank lines of `stderr`, at most [`TAIL_BYTES`]
/// bytes; `None` when it has none.
pub(crate) fn last_lines(stderr: &str) -> Option<String> {
    let lines: Vec<&str> = stderr.lines().filter(|l| !l.trim().is_empty()).collect();
    if lines.is_empty() {
        return None;
    }
    let joined = lines[lines.len().saturating_sub(TAIL_LINES)..].join("\n");
    let mut start = joined.len().saturating_sub(TAIL_BYTES);
    while !joined.is_char_boundary(start) {
        start += 1;
    }
    Some(joined[start..].to_owned())
}

#[cfg(test)]
mod tests {
    use super::*;
    use swarm_wire::builder::{BUILD_DEADLINE, LOCK_DEADLINE};

    fn job() -> BuildAgent {
        BuildAgent {
            agent: "atlas".to_owned(),
            rev: "0123456789abcdef0123456789abcdef01234567".to_owned(),
            pr: 7,
        }
    }

    fn builder() -> Builder {
        Builder {
            check_flake: "/nix/store/c-check".to_owned(),
            hyperhive: "/nix/store/h-source".to_owned(),
            nixpkgs: "/nix/store/n-source".to_owned(),
            forge_url: "https://forge.example".to_owned(),
            config_org: "agent-configs".to_owned(),
            nix: PathBuf::from("nix"),
            deadline: BUILD_DEADLINE,
            lock_deadline: LOCK_DEADLINE,
            system: "x86_64-linux".to_owned(),
            cache: None,
        }
    }

    #[test]
    fn the_agent_is_fetched_at_its_commit_through_the_prs_ref() {
        assert_eq!(
            builder().agent_url(&job()),
            "git+https://forge.example/agent-configs/atlas?ref=refs/pull/7/head\
             &rev=0123456789abcdef0123456789abcdef01234567"
        );
    }

    #[test]
    fn every_input_is_overridden_and_no_lock_is_written() {
        let args = builder().args(&job());
        let overridden: Vec<&str> = args
            .windows(2)
            .filter(|w| w[0] == "--override-input")
            .map(|w| w[1].as_str())
            .collect();
        assert_eq!(overridden, ["hyperhive", "nixpkgs", "agent"]);
        assert!(args.iter().any(|a| a == "/nix/store/c-check#check"));
        assert!(BUILD_FLAGS.contains(&"--no-write-lock-file"));
    }

    fn meta_job() -> BuildFromMeta {
        BuildFromMeta {
            agent: "atlas".to_owned(),
            meta_rev: "0123456789abcdef0123456789abcdef01234567".to_owned(),
        }
    }

    #[test]
    fn a_meta_build_is_the_agents_toplevel_in_internal_meta_at_the_commit() {
        assert_eq!(
            builder().meta_attr(&meta_job()),
            "git+https://forge.example/internal/meta.git\
             ?rev=0123456789abcdef0123456789abcdef01234567\
             #nixosConfigurations.atlas.config.system.build.toplevel"
        );
    }

    #[test]
    fn a_meta_job_that_would_bend_the_reference_is_refused() {
        for bad in [
            BuildFromMeta {
                agent: "a#b".to_owned(),
                ..meta_job()
            },
            BuildFromMeta {
                agent: String::new(),
                ..meta_job()
            },
            BuildFromMeta {
                meta_rev: "main".to_owned(),
                ..meta_job()
            },
        ] {
            assert!(validate_meta(&bad).is_err(), "{bad:?}");
        }
        assert!(validate_meta(&meta_job()).is_ok());
    }

    #[tokio::test]
    async fn a_meta_build_reports_the_path_and_the_hosts_system() {
        // `echo` stands in for `nix`: it prints the arguments it was given, which
        // are taken for the store path, and exits 0.
        let b = Builder {
            nix: PathBuf::from("echo"),
            ..builder()
        };
        let reply = b.build_from_meta(&meta_job(), "t").await;
        assert_eq!(
            reply,
            FromMetaReply::Built {
                out_path: format!("{} {}", BUILD_FLAGS.join(" "), b.meta_attr(&meta_job())),
                system: "x86_64-linux".to_owned(),
                cached: false,
            }
        );
    }

    #[tokio::test]
    async fn a_failed_meta_build_is_a_failed_reply_and_a_refused_job_is_failed_without_nix() {
        let b = Builder {
            nix: PathBuf::from("false"),
            ..builder()
        };
        let failed = b.build_from_meta(&meta_job(), "t").await;
        assert!(matches!(failed, FromMetaReply::Failed { .. }), "{failed:?}");
        // Control: `a_meta_build_reports_the_path_and_the_hosts_system` builds this
        // same job, so the refusal below is the validation, not the stub.
        let refused = b
            .build_from_meta(
                &BuildFromMeta {
                    meta_rev: "main".to_owned(),
                    ..meta_job()
                },
                "t",
            )
            .await;
        assert!(
            matches!(&refused, FromMetaReply::Failed { error_tail } if error_tail.contains("commit id")),
            "{refused:?}"
        );
    }

    /// What the helper prints for `op` and a request with `protocol` and
    /// `host`, run the way `git` runs a `!` helper.
    fn helper_answer(op: &str, protocol: &str, host: &str) -> String {
        use std::io::Write as _;

        let helper = credential_helper();
        let script = format!("{} {op}", helper.trim_start_matches('!'));
        let mut child = std::process::Command::new("sh")
            .args(["-c", &script])
            .env(TOKEN_VAR, "s3cret")
            .env(HOST_VAR, "forge.example:3000")
            .stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .spawn()
            .expect("sh runs");
        // The helper exits without reading the request for any op but `get`,
        // so the write can hit EPIPE. `git` ignores EPIPE here too.
        let written = write!(
            child.stdin.take().expect("piped"),
            "protocol={protocol}\nhost={host}\npath=agent-configs/atlas\n\n"
        );
        if let Err(e) = written {
            assert_eq!(e.kind(), std::io::ErrorKind::BrokenPipe, "{e}");
        }
        let out = child.wait_with_output().expect("the helper exits");
        assert!(out.status.success());
        String::from_utf8(out.stdout).expect("utf-8")
    }

    #[test]
    fn the_helper_gives_the_token_to_the_forge_alone() {
        assert_eq!(
            helper_answer("get", "https", "forge.example:3000"),
            "username=builder\npassword=s3cret\n"
        );
        for (op, protocol, host) in [
            ("get", "https", "github.com"),
            ("get", "https", "forge.example"),
            ("get", "https", "forge.example:3000.evil.example"),
            ("get", "http", "forge.example:3000"),
            ("store", "https", "forge.example:3000"),
        ] {
            assert_eq!(
                helper_answer(op, protocol, host),
                "",
                "{op} {protocol}://{host}"
            );
        }
    }

    #[test]
    fn the_helper_is_configured_for_the_forge_url_only() {
        assert_eq!(
            forge_host("https://forge.example:3000"),
            "forge.example:3000"
        );
        assert_eq!(forge_host("https://forge.example/sub"), "forge.example");
    }

    #[test]
    fn the_token_is_in_no_argument() {
        let args = builder().args(&job()).join(" ");
        assert!(!args.contains("password"));
        assert!(credential_helper().contains(&format!("${TOKEN_VAR}")));
    }

    #[test]
    fn a_job_that_would_bend_the_reference_is_refused() {
        for bad in [
            BuildAgent {
                agent: "a?rev=x".to_owned(),
                ..job()
            },
            BuildAgent {
                agent: String::new(),
                ..job()
            },
            BuildAgent {
                rev: "main".to_owned(),
                ..job()
            },
            BuildAgent { pr: 0, ..job() },
        ] {
            assert!(validate(&bad).is_err(), "{bad:?}");
        }
        assert!(validate(&job()).is_ok());
    }

    #[test]
    fn the_tail_keeps_the_end_of_a_long_log() {
        use std::os::unix::process::ExitStatusExt as _;

        let log = (0..100)
            .map(|i| format!("line {i}"))
            .collect::<Vec<_>>()
            .join("\n");
        let t = tail(&log, std::process::ExitStatus::from_raw(256));
        assert!(t.ends_with("line 99"));
        assert!(!t.contains("line 69\n"));
        assert!(t.starts_with("line 70"));
        let quiet = tail("", std::process::ExitStatus::from_raw(256));
        assert!(quiet.contains("printed nothing"));
    }

    fn sh(script: &str, deadline: Duration) -> impl Future<Output = Reply> {
        let mut cmd = tokio::process::Command::new("sh");
        cmd.args(["-c", script])
            .stdin(Stdio::null())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .kill_on_drop(true);
        run(cmd, deadline)
    }

    #[tokio::test]
    async fn a_success_replies_with_the_last_path_printed() {
        let reply = sh(
            "echo warming >&2; echo /nix/store/abc-nixos-system",
            Duration::from_secs(30),
        )
        .await;
        assert_eq!(
            reply,
            Reply::Built {
                out_path: "/nix/store/abc-nixos-system".to_owned()
            }
        );
    }

    #[tokio::test]
    async fn a_failure_replies_with_what_nix_said_last() {
        let reply = sh(
            "echo first >&2; echo 'error: attribute missing' >&2; exit 1",
            Duration::from_secs(30),
        )
        .await;
        let Reply::Failed { error_tail } = reply else {
            panic!("{reply:?}")
        };
        assert!(error_tail.ends_with("error: attribute missing"));
    }

    /// Write the shell script `body` to `path` as an executable, from a child
    /// process rather than from this one: a write fd on it held here could be
    /// inherited by a child another test thread forks, and exec'ing a file
    /// some process still holds open for writing fails with `ETXTBSY`.
    fn write_executable(path: &std::path::Path, body: &str) {
        let status = std::process::Command::new("/bin/sh")
            .args([
                "-c",
                r#"printf '%s' "$1" > "$2" && chmod 755 "$2""#,
                "sh",
                &format!("#!/bin/sh\n{body}\n"),
            ])
            .arg(path)
            .status()
            .expect("run /bin/sh");
        assert!(status.success(), "writing {} failed", path.display());
    }

    /// A `nix` stand-in: runs `body` as a shell script, ignoring its arguments.
    fn fake_nix(tag: &str, body: &str) -> PathBuf {
        let dir = std::env::temp_dir().join(format!(
            "swarm-builder-fake-nix-{tag}-{}",
            std::process::id()
        ));
        std::fs::create_dir_all(&dir).expect("scratch dir");
        let path = dir.join("nix");
        write_executable(&path, body);
        path
    }

    /// Builds with `nix`.
    async fn build_with(nix: PathBuf) -> Reply {
        let builder = Builder { nix, ..builder() };
        builder.build(&job(), "token").await
    }

    #[tokio::test]
    async fn eight_megabytes_of_log_leave_only_the_tail_in_the_reply() {
        let nix = fake_nix(
            "big",
            "i=0; while [ $i -lt 8 ]; do \
               head -c 1048576 /dev/zero | tr '\\0' 'x' >&2; echo >&2; i=$((i+1)); done\n\
             printf 'noise\\n' >&2\n\
             for i in 1 2 3; do echo \"final line $i\" >&2; done\n\
             exit 1",
        );
        let Reply::Failed { error_tail } = build_with(nix).await else {
            panic!("expected a failure")
        };
        assert_eq!(
            error_tail,
            "noise\nfinal line 1\nfinal line 2\nfinal line 3"
        );
    }

    #[tokio::test]
    async fn eight_megabytes_on_stdout_keep_the_last_path() {
        let nix = fake_nix(
            "bigout",
            "head -c 8388608 /dev/zero | tr '\\0' 'y'; echo\n\
             echo /nix/store/abc-nixos-system",
        );
        assert_eq!(
            build_with(nix).await,
            Reply::Built {
                out_path: "/nix/store/abc-nixos-system".to_owned()
            }
        );
    }

    #[tokio::test]
    async fn output_under_the_cap_reaches_the_reply_whole() {
        let nix = fake_nix(
            "small",
            "echo 'first line' >&2; echo >&2; echo '  second line' >&2; \
             echo 'a long stdout' ; echo /nix/store/abc-nixos-system; echo",
        );
        assert_eq!(
            build_with(nix).await,
            Reply::Built {
                out_path: "/nix/store/abc-nixos-system".to_owned()
            }
        );
        let failing = fake_nix(
            "smallfail",
            "echo 'first line' >&2; echo >&2; echo '  second line' >&2; exit 1",
        );
        let Reply::Failed { error_tail } = build_with(failing).await else {
            panic!("expected a failure")
        };
        assert_eq!(error_tail, "first line\n  second line");
    }

    #[tokio::test]
    async fn read_tail_keeps_the_end_and_starts_on_a_line() {
        let data = "aaaa\nbbbb\ncccc\ndddd\n";
        let whole = read_tail(data.as_bytes(), 100).await.expect("read");
        assert_eq!(whole, data.as_bytes());
        let cut = read_tail(data.as_bytes(), 12).await.expect("read");
        assert_eq!(String::from_utf8(cut).expect("utf-8"), "cccc\ndddd\n");
        let one_long_line = read_tail(&b"x".repeat(50)[..], 12).await.expect("read");
        assert_eq!(one_long_line, b"x".repeat(12));
        let long_last_line = read_tail(format!("{}\n", "x".repeat(50)).as_bytes(), 12)
            .await
            .expect("read");
        assert_eq!(long_last_line, format!("{}\n", "x".repeat(11)).as_bytes());
    }

    #[test]
    fn the_tail_never_holds_more_than_twice_the_cap_while_reading() {
        let mut tail = Tail::new(KEEP_BYTES);
        let mut peak = 0;
        for _ in 0..1024 {
            tail.push(&[b'x'; 8192]);
            peak = peak.max(tail.buf.len());
        }
        assert!(peak > KEEP_BYTES, "control: the buffer does grow: {peak}");
        assert!(peak <= 2 * KEEP_BYTES, "{peak} bytes held");
    }

    #[tokio::test]
    async fn read_tail_holds_at_most_the_cap_of_an_eight_megabyte_stream() {
        let mut stream = b"line\n".repeat(8 * 1024 * 1024 / 5);
        stream.extend_from_slice(b"the end\n");
        let kept = read_tail(&stream[..], KEEP_BYTES).await.expect("read");
        assert!(kept.len() <= KEEP_BYTES, "{} bytes kept", kept.len());
        assert!(kept.ends_with(b"line\nthe end\n"));
        assert!(kept.starts_with(b"line\n"));
    }

    /// A builder whose `nix` is the shell script `body`, in a directory the
    /// returned guard removes.
    fn scripted(body: &str) -> (Builder, Scratch) {
        let dir = Scratch::new().expect("a temp dir");
        let nix = dir.0.join("nix");
        write_executable(&nix, body);
        (Builder { nix, ..builder() }, dir)
    }

    /// A build past its deadline and a `nix` that cannot start are `Failed`
    /// with the reason, the same as a `nix` that exits non-zero.
    #[tokio::test]
    async fn a_meta_build_that_times_out_or_cannot_start_is_failed_like_one_that_exits() {
        let (slow, _guard) = scripted("sleep 5");
        let slow = Builder {
            deadline: Duration::from_millis(200),
            ..slow
        };
        let reply = slow.build_from_meta(&meta_job(), "t").await;
        assert!(
            matches!(&reply, FromMetaReply::Failed { error_tail } if error_tail.contains("timed out")),
            "{reply:?}"
        );

        let missing = Builder {
            nix: PathBuf::from("/nonexistent/nix"),
            ..builder()
        };
        let reply = missing.build_from_meta(&meta_job(), "t").await;
        assert!(
            matches!(&reply, FromMetaReply::Failed { error_tail } if error_tail.contains("starting nix")),
            "{reply:?}"
        );

        let (failing, _guard) = scripted("echo 'error: attribute missing' >&2; exit 1");
        let reply = failing.build_from_meta(&meta_job(), "t").await;
        assert!(
            matches!(&reply, FromMetaReply::Failed { error_tail } if error_tail.ends_with("attribute missing")),
            "{reply:?}"
        );
    }

    fn lock_job() -> LockMeta {
        LockMeta {
            flake_nix: "{ outputs = _: { }; }".to_owned(),
        }
    }

    #[tokio::test]
    async fn a_lock_answers_with_the_lock_nix_wrote_and_leaves_no_scratch() {
        // The lock records where it ran, so the test can look for it after.
        let (b, _guard) = scripted(r#"test "$1 $2" = "flake lock" && pwd > flake.lock"#);
        let reply = b.lock(&lock_job(), "t").await;
        let LockReply::Locked { flake_lock } = reply else {
            panic!("{reply:?}")
        };
        let ran_in = PathBuf::from(flake_lock.trim());
        assert!(ran_in.starts_with(std::env::temp_dir()), "{ran_in:?}");
        assert!(
            !ran_in.exists(),
            "the scratch dir {ran_in:?} was left behind"
        );
    }

    #[tokio::test]
    async fn a_lock_sees_the_flake_it_was_sent() {
        let (b, _guard) = scripted("cp flake.nix flake.lock");
        assert_eq!(
            b.lock(&lock_job(), "t").await,
            LockReply::Locked {
                flake_lock: lock_job().flake_nix
            }
        );
    }

    #[tokio::test]
    async fn a_failed_lock_answers_with_what_nix_said_last() {
        let (b, _guard) = scripted("echo 'error: cannot fetch' >&2; exit 1");
        let reply = b.lock(&lock_job(), "t").await;
        assert!(
            matches!(&reply, LockReply::Failed { error_tail } if error_tail.ends_with("error: cannot fetch")),
            "{reply:?}"
        );
    }

    #[tokio::test]
    async fn a_lock_that_wrote_no_lock_file_failed() {
        let (b, _guard) = scripted("exit 0");
        let reply = b.lock(&lock_job(), "t").await;
        assert!(
            matches!(&reply, LockReply::Failed { error_tail } if error_tail.contains("flake.lock")),
            "{reply:?}"
        );
    }

    #[tokio::test]
    async fn a_build_past_its_deadline_is_stopped() {
        let reply = sh("sleep 30", Duration::from_millis(200)).await;
        assert!(
            matches!(&reply, Reply::Failed { error_tail } if error_tail.contains("did not finish")),
            "{reply:?}"
        );
    }
}