swarm-builder: build each agent config PR head and post a commit status

A new `swarm-builder` service, in its own container on the host that sets
`deploy.builder.enable` (defaulted on by `allSwarmServices` where the host
has the builder's store leaf, like the review classifier).

When an `agent-configs/<agent>` PR is opened, reopened or synchronized,
swarm-controller queues a `build_agent` jobq node. The node sets a
`swarm-builder` commit status to pending on the PR head, sends
`{agent, rev, pr}` to the builder over NATS request-reply
(`$SWARM.builder.build`), and writes success (store path), failure (last
line nix printed) or error (no builder / no reply) on the head. The node
fails when the build does, so the Jobs page matches the PR.

The builder builds nix/builder/check-flake's `check` with every input
overridden: hyperhive and nixpkgs to the host's own
`c0re.hyperhiveFlake`/`c0re.nixpkgsFlake` (what a hive there renders into
its meta flake), and the agent to the PR head on the forge. Evaluation is
pure and runs in the container; builds go to the host nix daemon as an
untrusted client. The private config repo is fetched with the token of a
new read-only `builder` forge account (in the `hives` team), which the
controller mints into `swarm/services/builder/forge/token`; git gets it
from a credential helper reading the child's environment, so it is in no
argument or URL.

Credentials: a `builder` store leaf minted by glue-bao-tls on the store
host, a `swarm-builder` reader policy (forge token + its own OIDC client
secret), an authelia machine client, and a `--builder-client` grant in the
queue's auth-callout responder. The controller gains a grant on the token
path and the secret publisher a deny on it.

This is the agent-config build check atlas scoped on #5081 (slice 3):
nothing is pushed to a cache, nothing is written to KV, and a red status
gates no deploy. The check composes agent-base + the config without the
values a hive adds (CA, docs, claude-code path, ports, input follows), so
it can disagree with a hive's own build.

swarm-builder/src/nix_build.rs

@@ -0,0 +1,334 @@
+//! One build: `nix build` of the check flake with its three inputs pointed at
+//! this host's hyperhive and nixpkgs and at the agent's config commit.
+//!
+//! The config repo is private, so `git` fetches it with the builder's forge
+//! token. The token reaches `git` through a credential helper that reads it
+//! from the child's environment: it is in no argument, no URL and no lock, so
+//! no error line `nix` prints can carry it into the reply.
+
+use std::path::PathBuf;
+use std::process::Stdio;
+use std::time::Duration;
+
+use swarm_queue_client::builder::{BuildAgent, Reply};
+use tokio::io::AsyncReadExt as _;
+
+/// The most of `nix`'s stderr a failed build's reply carries, from its end.
+const TAIL_LINES: usize = 30;
+const TAIL_BYTES: usize = 4000;
+
+/// The variable the credential helper reads the token from, in the child only.
+const TOKEN_VAR: &str = "SWARM_BUILDER_GIT_TOKEN";
+
+/// What every build is run with, fixed at start.
+pub struct Builder {
+    /// The `nix` binary.
+    pub nix: PathBuf,
+    /// The check flake's directory in the store.
+    pub check_flake: String,
+    /// The hyperhive source this host was deployed from.
+    pub hyperhive: String,
+    /// The nixpkgs source this host was deployed from.
+    pub nixpkgs: String,
+    /// `https://<forge>`, with no trailing slash.
+    pub forge_url: String,
+    /// The org the config repos are in.
+    pub config_org: String,
+    /// How long one build may run.
+    pub deadline: Duration,
+}
+
+impl Builder {
+    /// Build `job` with `token` as the forge credential.
+    pub async fn build(&self, job: &BuildAgent, token: &str) -> Reply {
+        if let Err(error) = validate(job) {
+            return Reply::Failed { error_tail: error };
+        }
+        let mut cmd = tokio::process::Command::new(&self.nix);
+        cmd.args(self.args(job))
+            .env(TOKEN_VAR, token)
+            .env("GIT_TERMINAL_PROMPT", "0")
+            .env("GIT_CONFIG_COUNT", "1")
+            .env("GIT_CONFIG_KEY_0", "credential.helper")
+            .env("GIT_CONFIG_VALUE_0", credential_helper())
+            .stdin(Stdio::null())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .kill_on_drop(true);
+        run(cmd, self.deadline).await
+    }
+
+    /// The `nix build` arguments for `job`.
+    fn args(&self, job: &BuildAgent) -> Vec<String> {
+        [
+            "build",
+            "--no-link",
+            "--print-out-paths",
+            "--no-write-lock-file",
+            "--option",
+            "accept-flake-config",
+            "false",
+            "--print-build-logs",
+        ]
+        .into_iter()
+        .map(str::to_owned)
+        .chain([
+            format!("{}#check", self.check_flake),
+            "--override-input".to_owned(),
+            "hyperhive".to_owned(),
+            self.hyperhive.clone(),
+            "--override-input".to_owned(),
+            "nixpkgs".to_owned(),
+            self.nixpkgs.clone(),
+            "--override-input".to_owned(),
+            "agent".to_owned(),
+            self.agent_url(job),
+        ])
+        .collect()
+    }
+
+    /// The config commit as a flake reference, fetched through the PR's head
+    /// ref.
+    fn agent_url(&self, job: &BuildAgent) -> String {
+        format!(
+            "git+{}/{}/{}?ref=refs/pull/{}/head&rev={}",
+            self.forge_url, self.config_org, job.agent, job.pr, job.rev
+        )
+    }
+}
+
+/// A `git` credential helper answering every `get` with the builder's login
+/// and the token from [`TOKEN_VAR`].
+fn credential_helper() -> String {
+    format!(
+        "!f() {{ test \"$1\" = get && printf 'username=builder\\npassword=%s\\n' \"${TOKEN_VAR}\"; }}; f"
+    )
+}
+
+/// Refuse a job whose fields would change the reference's meaning: an agent
+/// is one `[a-z0-9-]` segment, and a rev is a full commit id.
+fn validate(job: &BuildAgent) -> Result<(), String> {
+    let agent_ok = !job.agent.is_empty()
+        && job
+            .agent
+            .bytes()
+            .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-');
+    if !agent_ok {
+        return Err(format!("{:?} is not an agent name", job.agent));
+    }
+    let rev_ok = job.rev.len() == 40
+        && job
+            .rev
+            .bytes()
+            .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b));
+    if !rev_ok {
+        return Err(format!("{:?} is not a full commit id", job.rev));
+    }
+    if job.pr == 0 {
+        return Err("PR 0 does not exist".to_owned());
+    }
+    Ok(())
+}
+
+/// Run `cmd` to its end or `deadline`, and read the store path it printed or
+/// the tail of what it said.
+async fn run(mut cmd: tokio::process::Command, deadline: Duration) -> Reply {
+    let mut child = match cmd.spawn() {
+        Ok(child) => child,
+        Err(e) => {
+            return Reply::Failed {
+                error_tail: format!("starting nix: {e}"),
+            };
+        }
+    };
+    let mut stdout = child.stdout.take().expect("piped");
+    let mut stderr = child.stderr.take().expect("piped");
+    let work = async {
+        let (mut out, mut err) = (Vec::new(), Vec::new());
+        let (o, e, status) = tokio::join!(
+            stdout.read_to_end(&mut out),
+            stderr.read_to_end(&mut err),
+            child.wait()
+        );
+        o.and(e).map(|_| ())?;
+        status.map(|s| (s, out, err))
+    };
+    match tokio::time::timeout(deadline, work).await {
+        Err(_) => Reply::Failed {
+            error_tail: format!(
+                "the build did not finish within {} minutes",
+                deadline.as_secs() / 60
+            ),
+        },
+        Ok(Err(e)) => Reply::Failed {
+            error_tail: format!("waiting for nix: {e}"),
+        },
+        Ok(Ok((status, out, err))) => {
+            let out_path = String::from_utf8_lossy(&out)
+                .lines()
+                .rfind(|l| !l.trim().is_empty())
+                .map(|l| l.trim().to_owned());
+            match out_path {
+                Some(out_path) if status.success() => Reply::Built { out_path },
+                _ => Reply::Failed {
+                    error_tail: tail(&String::from_utf8_lossy(&err), status),
+                },
+            }
+        }
+    }
+}
+
+/// The last [`TAIL_LINES`] lines of `stderr`, at most [`TAIL_BYTES`] bytes,
+/// or the exit status when it said nothing.
+fn tail(stderr: &str, status: std::process::ExitStatus) -> String {
+    let lines: Vec<&str> = stderr.lines().filter(|l| !l.trim().is_empty()).collect();
+    if lines.is_empty() {
+        return format!("nix exited with {status} and printed nothing");
+    }
+    let joined = lines[lines.len().saturating_sub(TAIL_LINES)..].join("\n");
+    let mut start = joined.len().saturating_sub(TAIL_BYTES);
+    while !joined.is_char_boundary(start) {
+        start += 1;
+    }
+    joined[start..].to_owned()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use swarm_queue_client::builder::BUILD_DEADLINE;
+
+    fn job() -> BuildAgent {
+        BuildAgent {
+            agent: "atlas".to_owned(),
+            rev: "0123456789abcdef0123456789abcdef01234567".to_owned(),
+            pr: 7,
+        }
+    }
+
+    fn builder() -> Builder {
+        Builder {
+            check_flake: "/nix/store/c-check".to_owned(),
+            hyperhive: "/nix/store/h-source".to_owned(),
+            nixpkgs: "/nix/store/n-source".to_owned(),
+            forge_url: "https://forge.example".to_owned(),
+            config_org: "agent-configs".to_owned(),
+            nix: PathBuf::from("nix"),
+            deadline: BUILD_DEADLINE,
+        }
+    }
+
+    #[test]
+    fn the_agent_is_fetched_at_its_commit_through_the_prs_ref() {
+        assert_eq!(
+            builder().agent_url(&job()),
+            "git+https://forge.example/agent-configs/atlas?ref=refs/pull/7/head\
+             &rev=0123456789abcdef0123456789abcdef01234567"
+        );
+    }
+
+    #[test]
+    fn every_input_is_overridden_and_no_lock_is_written() {
+        let args = builder().args(&job());
+        let overridden: Vec<&str> = args
+            .windows(2)
+            .filter(|w| w[0] == "--override-input")
+            .map(|w| w[1].as_str())
+            .collect();
+        assert_eq!(overridden, ["hyperhive", "nixpkgs", "agent"]);
+        assert!(args.iter().any(|a| a == "/nix/store/c-check#check"));
+        assert!(args.iter().any(|a| a == "--no-write-lock-file"));
+    }
+
+    #[test]
+    fn the_token_is_in_no_argument() {
+        let args = builder().args(&job()).join(" ");
+        assert!(!args.contains("password"));
+        assert!(credential_helper().contains(&format!("${TOKEN_VAR}")));
+    }
+
+    #[test]
+    fn a_job_that_would_bend_the_reference_is_refused() {
+        for bad in [
+            BuildAgent {
+                agent: "a?rev=x".to_owned(),
+                ..job()
+            },
+            BuildAgent {
+                agent: String::new(),
+                ..job()
+            },
+            BuildAgent {
+                rev: "main".to_owned(),
+                ..job()
+            },
+            BuildAgent { pr: 0, ..job() },
+        ] {
+            assert!(validate(&bad).is_err(), "{bad:?}");
+        }
+        assert!(validate(&job()).is_ok());
+    }
+
+    #[test]
+    fn the_tail_keeps_the_end_of_a_long_log() {
+        use std::os::unix::process::ExitStatusExt as _;
+
+        let log = (0..100)
+            .map(|i| format!("line {i}"))
+            .collect::<Vec<_>>()
+            .join("\n");
+        let t = tail(&log, std::process::ExitStatus::from_raw(256));
+        assert!(t.ends_with("line 99"));
+        assert!(!t.contains("line 69\n"));
+        assert!(t.starts_with("line 70"));
+        let quiet = tail("", std::process::ExitStatus::from_raw(256));
+        assert!(quiet.contains("printed nothing"));
+    }
+
+    fn sh(script: &str, deadline: Duration) -> impl Future<Output = Reply> {
+        let mut cmd = tokio::process::Command::new("sh");
+        cmd.args(["-c", script])
+            .stdin(Stdio::null())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .kill_on_drop(true);
+        run(cmd, deadline)
+    }
+
+    #[tokio::test]
+    async fn a_success_replies_with_the_last_path_printed() {
+        let reply = sh(
+            "echo warming >&2; echo /nix/store/abc-nixos-system",
+            Duration::from_secs(30),
+        )
+        .await;
+        assert_eq!(
+            reply,
+            Reply::Built {
+                out_path: "/nix/store/abc-nixos-system".to_owned()
+            }
+        );
+    }
+
+    #[tokio::test]
+    async fn a_failure_replies_with_what_nix_said_last() {
+        let reply = sh(
+            "echo first >&2; echo 'error: attribute missing' >&2; exit 1",
+            Duration::from_secs(30),
+        )
+        .await;
+        let Reply::Failed { error_tail } = reply else {
+            panic!("{reply:?}")
+        };
+        assert!(error_tail.ends_with("error: attribute missing"));
+    }
+
+    #[tokio::test]
+    async fn a_build_past_its_deadline_is_stopped() {
+        let reply = sh("sleep 30", Duration::from_millis(200)).await;
+        assert!(
+            matches!(&reply, Reply::Failed { error_tail } if error_tail.contains("did not finish")),
+            "{reply:?}"
+        );
+    }
+}