hyperhive/swarm-builder/src/nix_build.rs

//! One build: `nix build` of the check flake with its three inputs pointed at
//! this host's hyperhive and nixpkgs and at the agent's config commit.
//!
//! The config repo is private, so `git` fetches it with the builder's forge
//! token. The token reaches `git` through a credential helper that reads it
//! from the child's environment: it is in no argument, no URL and no lock, so
//! no error line `nix` prints can carry it into the reply.

use std::path::PathBuf;
use std::process::Stdio;
use std::time::Duration;

use swarm_queue_client::builder::{BuildAgent, Reply};
use tokio::io::AsyncReadExt as _;

/// The most of `nix`'s stderr a failed build's reply carries, from its end.
const TAIL_LINES: usize = 30;
const TAIL_BYTES: usize = 4000;

/// The variable the credential helper reads the token from, in the child only.
const TOKEN_VAR: &str = "SWARM_BUILDER_GIT_TOKEN";

/// What every build is run with, fixed at start.
pub struct Builder {
    /// The `nix` binary.
    pub nix: PathBuf,
    /// The check flake's directory in the store.
    pub check_flake: String,
    /// The hyperhive source this host was deployed from.
    pub hyperhive: String,
    /// The nixpkgs source this host was deployed from.
    pub nixpkgs: String,
    /// `https://<forge>`, with no trailing slash.
    pub forge_url: String,
    /// The org the config repos are in.
    pub config_org: String,
    /// How long one build may run.
    pub deadline: Duration,
}

impl Builder {
    /// Build `job` with `token` as the forge credential.
    pub async fn build(&self, job: &BuildAgent, token: &str) -> Reply {
        if let Err(error) = validate(job) {
            return Reply::Failed { error_tail: error };
        }
        let mut cmd = tokio::process::Command::new(&self.nix);
        cmd.args(self.args(job))
            .env(TOKEN_VAR, token)
            .env("GIT_TERMINAL_PROMPT", "0")
            .env("GIT_CONFIG_COUNT", "1")
            .env("GIT_CONFIG_KEY_0", "credential.helper")
            .env("GIT_CONFIG_VALUE_0", credential_helper())
            .stdin(Stdio::null())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .kill_on_drop(true);
        run(cmd, self.deadline).await
    }

    /// The `nix build` arguments for `job`.
    fn args(&self, job: &BuildAgent) -> Vec<String> {
        [
            "build",
            "--no-link",
            "--print-out-paths",
            "--no-write-lock-file",
            "--option",
            "accept-flake-config",
            "false",
            "--print-build-logs",
        ]
        .into_iter()
        .map(str::to_owned)
        .chain([
            format!("{}#check", self.check_flake),
            "--override-input".to_owned(),
            "hyperhive".to_owned(),
            self.hyperhive.clone(),
            "--override-input".to_owned(),
            "nixpkgs".to_owned(),
            self.nixpkgs.clone(),
            "--override-input".to_owned(),
            "agent".to_owned(),
            self.agent_url(job),
        ])
        .collect()
    }

    /// The config commit as a flake reference, fetched through the PR's head
    /// ref.
    fn agent_url(&self, job: &BuildAgent) -> String {
        format!(
            "git+{}/{}/{}?ref=refs/pull/{}/head&rev={}",
            self.forge_url, self.config_org, job.agent, job.pr, job.rev
        )
    }
}

/// A `git` credential helper answering every `get` with the builder's login
/// and the token from [`TOKEN_VAR`].
fn credential_helper() -> String {
    format!(
        "!f() {{ test \"$1\" = get && printf 'username=builder\\npassword=%s\\n' \"${TOKEN_VAR}\"; }}; f"
    )
}

/// Refuse a job whose fields would change the reference's meaning: an agent
/// is one `[a-z0-9-]` segment, and a rev is a full commit id.
fn validate(job: &BuildAgent) -> Result<(), String> {
    let agent_ok = !job.agent.is_empty()
        && job
            .agent
            .bytes()
            .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-');
    if !agent_ok {
        return Err(format!("{:?} is not an agent name", job.agent));
    }
    let rev_ok = job.rev.len() == 40
        && job
            .rev
            .bytes()
            .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b));
    if !rev_ok {
        return Err(format!("{:?} is not a full commit id", job.rev));
    }
    if job.pr == 0 {
        return Err("PR 0 does not exist".to_owned());
    }
    Ok(())
}

/// Run `cmd` to its end or `deadline`, and read the store path it printed or
/// the tail of what it said.
async fn run(mut cmd: tokio::process::Command, deadline: Duration) -> Reply {
    let mut child = match cmd.spawn() {
        Ok(child) => child,
        Err(e) => {
            return Reply::Failed {
                error_tail: format!("starting nix: {e}"),
            };
        }
    };
    let mut stdout = child.stdout.take().expect("piped");
    let mut stderr = child.stderr.take().expect("piped");
    let work = async {
        let (mut out, mut err) = (Vec::new(), Vec::new());
        let (o, e, status) = tokio::join!(
            stdout.read_to_end(&mut out),
            stderr.read_to_end(&mut err),
            child.wait()
        );
        o.and(e).map(|_| ())?;
        status.map(|s| (s, out, err))
    };
    match tokio::time::timeout(deadline, work).await {
        Err(_) => Reply::Failed {
            error_tail: format!(
                "the build did not finish within {} minutes",
                deadline.as_secs() / 60
            ),
        },
        Ok(Err(e)) => Reply::Failed {
            error_tail: format!("waiting for nix: {e}"),
        },
        Ok(Ok((status, out, err))) => {
            let out_path = String::from_utf8_lossy(&out)
                .lines()
                .rfind(|l| !l.trim().is_empty())
                .map(|l| l.trim().to_owned());
            match out_path {
                Some(out_path) if status.success() => Reply::Built { out_path },
                _ => Reply::Failed {
                    error_tail: tail(&String::from_utf8_lossy(&err), status),
                },
            }
        }
    }
}

/// The last [`TAIL_LINES`] lines of `stderr`, at most [`TAIL_BYTES`] bytes,
/// or the exit status when it said nothing.
fn tail(stderr: &str, status: std::process::ExitStatus) -> String {
    let lines: Vec<&str> = stderr.lines().filter(|l| !l.trim().is_empty()).collect();
    if lines.is_empty() {
        return format!("nix exited with {status} and printed nothing");
    }
    let joined = lines[lines.len().saturating_sub(TAIL_LINES)..].join("\n");
    let mut start = joined.len().saturating_sub(TAIL_BYTES);
    while !joined.is_char_boundary(start) {
        start += 1;
    }
    joined[start..].to_owned()
}

#[cfg(test)]
mod tests {
    use super::*;
    use swarm_queue_client::builder::BUILD_DEADLINE;

    fn job() -> BuildAgent {
        BuildAgent {
            agent: "atlas".to_owned(),
            rev: "0123456789abcdef0123456789abcdef01234567".to_owned(),
            pr: 7,
        }
    }

    fn builder() -> Builder {
        Builder {
            check_flake: "/nix/store/c-check".to_owned(),
            hyperhive: "/nix/store/h-source".to_owned(),
            nixpkgs: "/nix/store/n-source".to_owned(),
            forge_url: "https://forge.example".to_owned(),
            config_org: "agent-configs".to_owned(),
            nix: PathBuf::from("nix"),
            deadline: BUILD_DEADLINE,
        }
    }

    #[test]
    fn the_agent_is_fetched_at_its_commit_through_the_prs_ref() {
        assert_eq!(
            builder().agent_url(&job()),
            "git+https://forge.example/agent-configs/atlas?ref=refs/pull/7/head\
             &rev=0123456789abcdef0123456789abcdef01234567"
        );
    }

    #[test]
    fn every_input_is_overridden_and_no_lock_is_written() {
        let args = builder().args(&job());
        let overridden: Vec<&str> = args
            .windows(2)
            .filter(|w| w[0] == "--override-input")
            .map(|w| w[1].as_str())
            .collect();
        assert_eq!(overridden, ["hyperhive", "nixpkgs", "agent"]);
        assert!(args.iter().any(|a| a == "/nix/store/c-check#check"));
        assert!(args.iter().any(|a| a == "--no-write-lock-file"));
    }

    #[test]
    fn the_token_is_in_no_argument() {
        let args = builder().args(&job()).join(" ");
        assert!(!args.contains("password"));
        assert!(credential_helper().contains(&format!("${TOKEN_VAR}")));
    }

    #[test]
    fn a_job_that_would_bend_the_reference_is_refused() {
        for bad in [
            BuildAgent {
                agent: "a?rev=x".to_owned(),
                ..job()
            },
            BuildAgent {
                agent: String::new(),
                ..job()
            },
            BuildAgent {
                rev: "main".to_owned(),
                ..job()
            },
            BuildAgent { pr: 0, ..job() },
        ] {
            assert!(validate(&bad).is_err(), "{bad:?}");
        }
        assert!(validate(&job()).is_ok());
    }

    #[test]
    fn the_tail_keeps_the_end_of_a_long_log() {
        use std::os::unix::process::ExitStatusExt as _;

        let log = (0..100)
            .map(|i| format!("line {i}"))
            .collect::<Vec<_>>()
            .join("\n");
        let t = tail(&log, std::process::ExitStatus::from_raw(256));
        assert!(t.ends_with("line 99"));
        assert!(!t.contains("line 69\n"));
        assert!(t.starts_with("line 70"));
        let quiet = tail("", std::process::ExitStatus::from_raw(256));
        assert!(quiet.contains("printed nothing"));
    }

    fn sh(script: &str, deadline: Duration) -> impl Future<Output = Reply> {
        let mut cmd = tokio::process::Command::new("sh");
        cmd.args(["-c", script])
            .stdin(Stdio::null())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .kill_on_drop(true);
        run(cmd, deadline)
    }

    #[tokio::test]
    async fn a_success_replies_with_the_last_path_printed() {
        let reply = sh(
            "echo warming >&2; echo /nix/store/abc-nixos-system",
            Duration::from_secs(30),
        )
        .await;
        assert_eq!(
            reply,
            Reply::Built {
                out_path: "/nix/store/abc-nixos-system".to_owned()
            }
        );
    }

    #[tokio::test]
    async fn a_failure_replies_with_what_nix_said_last() {
        let reply = sh(
            "echo first >&2; echo 'error: attribute missing' >&2; exit 1",
            Duration::from_secs(30),
        )
        .await;
        let Reply::Failed { error_tail } = reply else {
            panic!("{reply:?}")
        };
        assert!(error_tail.ends_with("error: attribute missing"));
    }

    #[tokio::test]
    async fn a_build_past_its_deadline_is_stopped() {
        let reply = sh("sleep 30", Duration::from_millis(200)).await;
        assert!(
            matches!(&reply, Reply::Failed { error_tail } if error_tail.contains("did not finish")),
            "{reply:?}"
        );
    }
}