vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-builder: lock the swarm meta flake on request

Browse source 
A request on $SWARM.builder.lock-meta carries a rendered flake.nix. The
builder writes it into a scratch directory, runs `nix flake lock` there
with the same forge credential helper its builds use, answers with the
flake.lock nix wrote, and removes the directory. It commits nothing; the
controller commits the lock beside the flake.

Locks are answered in their own loop, like archetype evaluations, so one
does not wait behind an hour-long build. The auth callout grants the
subject to the builder (subscribe) and the controller (publish) only.

Ported from c574a1694 onto the swarm-wire crate.

Refs #4857
This commit is contained in:
atlas 2026-10-22 01:23:02 +02:00
commit de323a72dd
4 changed files with 320 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

67
swarm-builder/src/main.rs
View file

@ -8,6 +8,10 @@
//! [`swarm_wire::builder::ARCHETYPES_SUBJECT`] with the agent
//! archetypes its hyperhive source declares, which the controller stores.
//!
//! Lock requests on [`swarm_wire::builder::LOCK_SUBJECT`] carry the swarm meta
//! flake's `flake.nix`, and the answer is the `flake.lock` `nix` wrote for it in
//! a scratch directory. The controller commits it.
//!
//! The store identity comes from the `BAO_*` environment. The queue client
//! secret is read at start and again when authelia refuses it, the forge token
//! once per build, both from the store under that identity.
@ -23,8 +27,8 @@ use swarm_queue_client::{ClientSecret, QueueConfig};
use swarm_secret_client::forge::Credential;
use swarm_secret_client::stored_secret::StoredSecret;
use swarm_wire::builder::{
ARCHETYPES_DEADLINE, ARCHETYPES_SUBJECT, BUILD_DEADLINE, BuildAgent, QUEUE_GROUP,
REPLY_DEADLINE, Reply, SUBJECT,
ARCHETYPES_DEADLINE, ARCHETYPES_SUBJECT, BUILD_DEADLINE, BuildAgent, LOCK_DEADLINE,
LOCK_SUBJECT, LockMeta, LockReply, QUEUE_GROUP, REPLY_DEADLINE, Reply, SUBJECT,
};
mod archetypes;
@ -71,6 +75,7 @@ async fn main() -> Result<()> {
forge_url: forge_url()?,
config_org: var("CONFIG_ORG")?,
deadline: BUILD_DEADLINE,
lock_deadline: LOCK_DEADLINE,
});
let secret =
swarm_secret_client::read_with_startup_retry("reading the queue client secret", || {
@ -109,6 +114,13 @@ async fn main() -> Result<()> {
.with_context(|| format!("subscribing to {ARCHETYPES_SUBJECT}"))?;
// Outside the build slot below: a list refresh does not wait on a build.
tokio::spawn(answer_evaluations(evaluations, queue.clone(), evaluator));
let locks = queue
.queue_subscribe(LOCK_SUBJECT, QUEUE_GROUP.to_owned())
.await
.with_context(|| format!("subscribing to {LOCK_SUBJECT}"))?;
// Outside the build slot too: a lock builds nothing, so it does not wait
// behind an hour-long build.
tokio::spawn(answer_locks(locks, queue.clone(), Arc::clone(&builder)));
// One build at a time. swarm-controller already sends one at a time; this
// holds the bound for any request it did not serialise, and `in_slot`
@ -186,6 +198,57 @@ async fn answer_evaluations(
tracing::error!("the subscription to {ARCHETYPES_SUBJECT} ended");
}
/// Answer every meta-flake lock request, one at a time, with the lock `nix`
/// wrote or why it did not.
async fn answer_locks(
mut requests: async_nats::Subscriber,
queue: async_nats::Client,
builder: Arc<nix_build::Builder>,
) {
while let Some(msg) = requests.next().await {
let Some(reply_to) = msg.reply.clone() else {
tracing::warn!("a meta-flake lock arrived with no reply subject; dropping it");
continue;
};
let reply = handle_lock(&msg.payload, &builder).await;
match serde_json::to_vec(&reply) {
Ok(body) => {
if let Err(e) = queue.publish(reply_to, body.into()).await {
tracing::warn!(error = %e, "publishing a lock's reply failed");
}
}
Err(e) => tracing::warn!(error = %e, "encoding a lock's reply failed"),
}
}
tracing::error!("the subscription to {LOCK_SUBJECT} ended");
}
/// Decode one lock request, lock it and log the lock.
async fn handle_lock(payload: &[u8], builder: &nix_build::Builder) -> LockReply {
let job: LockMeta = match serde_json::from_slice(payload) {
Ok(job) => job,
Err(e) => {
tracing::warn!(error = %e, "an undecodable lock; answering with the error");
return LockReply::Failed {
error_tail: format!("the builder could not decode the request: {e}"),
};
}
};
let reply = match within(TOKEN_DEADLINE, forge_token()).await {
Ok(token) => builder.lock(&job, &token).await,
Err(e) => LockReply::Failed {
error_tail: format!("{e:#}"),
},
};
match &reply {
LockReply::Locked { .. } => tracing::info!("swarm meta flake locked"),
LockReply::Failed { error_tail } => {
tracing::warn!(%error_tail, "swarm meta flake lock failed");
}
}
reply
}
/// `build`'s reply once `job` holds the build slot, or a refusal, without
/// running `build`, once it has waited [`MAX_SLOT_WAIT`] since `received`.
/// `None` only when the slot is closed.

206
swarm-builder/src/nix_build.rs
View file

@ -1,6 +1,7 @@
//! One build: `nix build` of the check flake with its three inputs pointed at
//! this host's hyperhive and nixpkgs and at the agent's config
//! commit.
//! commit. One lock: `nix flake lock` of the swarm meta flake in a scratch
//! directory.
//!
//! The config repo is private, so `git` fetches it with the builder's forge
//! token. The token reaches `git` through a credential helper that reads it
@ -16,7 +17,7 @@ use std::path::PathBuf;
use std::process::Stdio;
use std::time::Duration;
use swarm_wire::builder::{BuildAgent, Reply};
use swarm_wire::builder::{BuildAgent, LockMeta, LockReply, Reply};
use tokio::io::AsyncReadExt as _;
/// The most of `nix`'s stderr a failed build's reply carries, from its end.
@ -51,6 +52,8 @@ pub struct Builder {
pub config_org: String,
/// How long one build may run.
pub deadline: Duration,
/// How long one lock may run.
pub lock_deadline: Duration,
}
impl Builder {
@ -59,9 +62,48 @@ impl Builder {
if let Err(error) = validate(job) {
return Reply::Failed { error_tail: error };
}
let mut cmd = self.nix_command(token);
cmd.args(self.args(job));
run(cmd, self.deadline).await
}
/// Lock `job`'s flake with `token` as the forge credential, in a scratch
/// directory removed afterwards, and answer with the lock `nix` wrote.
pub async fn lock(&self, job: &LockMeta, token: &str) -> LockReply {
let scratch = match Scratch::new() {
Ok(s) => s,
Err(e) => {
return LockReply::Failed {
error_tail: format!("creating the scratch directory: {e}"),
};
}
};
if let Err(e) = std::fs::write(scratch.0.join("flake.nix"), &job.flake_nix) {
return LockReply::Failed {
error_tail: format!("writing flake.nix: {e}"),
};
}
let mut cmd = self.nix_command(token);
cmd.args(LOCK_ARGS).current_dir(&scratch.0);
match run_raw(cmd, self.lock_deadline).await {
Err(error_tail) => LockReply::Failed { error_tail },
Ok((status, _, err)) if !status.success() => LockReply::Failed {
error_tail: tail(&String::from_utf8_lossy(&err), status),
},
Ok(_) => match std::fs::read_to_string(scratch.0.join("flake.lock")) {
Ok(flake_lock) => LockReply::Locked { flake_lock },
Err(e) => LockReply::Failed {
error_tail: format!("nix exited 0 but flake.lock is unreadable: {e}"),
},
},
}
}
/// `nix` with the forge credential reachable by `git` alone, and its
/// output captured.
fn nix_command(&self, token: &str) -> tokio::process::Command {
let mut cmd = tokio::process::Command::new(&self.nix);
cmd.args(self.args(job))
.env(TOKEN_VAR, token)
cmd.env(TOKEN_VAR, token)
.env("GIT_TERMINAL_PROMPT", "0")
.env("GIT_CONFIG_COUNT", "1")
.env(HOST_VAR, forge_host(&self.forge_url))
@ -74,7 +116,7 @@ impl Builder {
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true);
run(cmd, self.deadline).await
cmd
}
/// The `nix build` arguments for `job`.
@ -116,6 +158,33 @@ impl Builder {
}
}
/// `nix flake lock` of the flake in the working directory. The flake's config
/// is not trusted, and nothing is built.
const LOCK_ARGS: &[&str] = &["flake", "lock", "--option", "accept-flake-config", "false"];
/// A fresh directory under the temp dir, removed with everything in it on drop.
struct Scratch(PathBuf);
impl Scratch {
fn new() -> std::io::Result<Self> {
use std::sync::atomic::{AtomicU64, Ordering};
static NEXT: AtomicU64 = AtomicU64::new(0);
let dir = std::env::temp_dir().join(format!(
"lock-meta-{}-{}",
std::process::id(),
NEXT.fetch_add(1, Ordering::Relaxed)
));
std::fs::create_dir(&dir)?;
Ok(Self(dir))
}
}
impl Drop for Scratch {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
/// The host part of `forge_url` (`https://<host>[:port]`), as `git` names it
/// to a credential helper.
fn forge_host(forge_url: &str) -> &str {
@ -162,14 +231,33 @@ fn validate(job: &BuildAgent) -> Result<(), String> {
/// Run `cmd` to its end or `deadline`, and read the store path it printed or
/// the tail of what it said.
async fn run(mut cmd: tokio::process::Command, deadline: Duration) -> Reply {
async fn run(cmd: tokio::process::Command, deadline: Duration) -> Reply {
match run_raw(cmd, deadline).await {
Err(error_tail) => Reply::Failed { error_tail },
Ok((status, out, err)) => {
let out_path = String::from_utf8_lossy(&out)
.lines()
.rfind(|l| !l.trim().is_empty())
.map(|l| l.trim().to_owned());
match out_path {
Some(out_path) if status.success() => Reply::Built { out_path },
_ => Reply::Failed {
error_tail: tail(&String::from_utf8_lossy(&err), status),
},
}
}
}
}
/// Run `cmd` to its end or `deadline`: its status and the kept end of its
/// stdout and stderr, or why it did not finish.
async fn run_raw(
mut cmd: tokio::process::Command,
deadline: Duration,
) -> Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>), String> {
let mut child = match cmd.spawn() {
Ok(child) => child,
Err(e) => {
return Reply::Failed {
error_tail: format!("starting nix: {e}"),
};
}
Err(e) => return Err(format!("starting nix: {e}")),
};
let stdout = child.stdout.take().expect("piped");
let stderr = child.stderr.take().expect("piped");
@ -183,27 +271,12 @@ async fn run(mut cmd: tokio::process::Command, deadline: Duration) -> Reply {
status.map(|s| (s, out, err))
};
match tokio::time::timeout(deadline, work).await {
Err(_) => Reply::Failed {
error_tail: format!(
"the build did not finish within {} minutes",
deadline.as_secs() / 60
),
},
Ok(Err(e)) => Reply::Failed {
error_tail: format!("waiting for nix: {e}"),
},
Ok(Ok((status, out, err))) => {
let out_path = String::from_utf8_lossy(&out)
.lines()
.rfind(|l| !l.trim().is_empty())
.map(|l| l.trim().to_owned());
match out_path {
Some(out_path) if status.success() => Reply::Built { out_path },
_ => Reply::Failed {
error_tail: tail(&String::from_utf8_lossy(&err), status),
},
}
}
Err(_) => Err(format!(
"nix did not finish within {} minutes",
deadline.as_secs() / 60
)),
Ok(Err(e)) => Err(format!("waiting for nix: {e}")),
Ok(Ok(done)) => Ok(done),
}
}
@ -288,7 +361,7 @@ pub(crate) fn tail(stderr: &str, status: std::process::ExitStatus) -> String {
#[cfg(test)]
mod tests {
use super::*;
use swarm_wire::builder::BUILD_DEADLINE;
use swarm_wire::builder::{BUILD_DEADLINE, LOCK_DEADLINE};
fn job() -> BuildAgent {
BuildAgent {
@ -307,6 +380,7 @@ mod tests {
config_org: "agent-configs".to_owned(),
nix: PathBuf::from("nix"),
deadline: BUILD_DEADLINE,
lock_deadline: LOCK_DEADLINE,
}
}
@ -600,6 +674,72 @@ mod tests {
assert!(kept.starts_with(b"line\n"));
}
/// A builder whose `nix` is the shell script `body`, in a directory the
/// returned guard removes.
fn scripted(body: &str) -> (Builder, Scratch) {
use std::os::unix::fs::PermissionsExt as _;
let dir = Scratch::new().expect("a temp dir");
let nix = dir.0.join("nix");
std::fs::write(&nix, format!("#!/bin/sh\n{body}\n")).expect("the script is written");
std::fs::set_permissions(&nix, std::fs::Permissions::from_mode(0o755))
.expect("the script is executable");
(Builder { nix, ..builder() }, dir)
}
fn lock_job() -> LockMeta {
LockMeta {
flake_nix: "{ outputs = _: { }; }".to_owned(),
}
}
#[tokio::test]
async fn a_lock_answers_with_the_lock_nix_wrote_and_leaves_no_scratch() {
// The lock records where it ran, so the test can look for it after.
let (b, _guard) = scripted(r#"test "$1 $2" = "flake lock" && pwd > flake.lock"#);
let reply = b.lock(&lock_job(), "t").await;
let LockReply::Locked { flake_lock } = reply else {
panic!("{reply:?}")
};
let ran_in = PathBuf::from(flake_lock.trim());
assert!(ran_in.starts_with(std::env::temp_dir()), "{ran_in:?}");
assert!(
!ran_in.exists(),
"the scratch dir {ran_in:?} was left behind"
);
}
#[tokio::test]
async fn a_lock_sees_the_flake_it_was_sent() {
let (b, _guard) = scripted("cp flake.nix flake.lock");
assert_eq!(
b.lock(&lock_job(), "t").await,
LockReply::Locked {
flake_lock: lock_job().flake_nix
}
);
}
#[tokio::test]
async fn a_failed_lock_answers_with_what_nix_said_last() {
let (b, _guard) = scripted("echo 'error: cannot fetch' >&2; exit 1");
let reply = b.lock(&lock_job(), "t").await;
assert!(
matches!(&reply, LockReply::Failed { error_tail } if error_tail.ends_with("error: cannot fetch")),
"{reply:?}"
);
}
#[tokio::test]
async fn a_lock_that_wrote_no_lock_file_failed() {
let (b, _guard) = scripted("exit 0");
let reply = b.lock(&lock_job(), "t").await;
assert!(
matches!(&reply, LockReply::Failed { error_tail } if error_tail.contains("flake.lock")),
"{reply:?}"
);
}
#[tokio::test]
async fn a_build_past_its_deadline_is_stopped() {
let reply = sh("sleep 30", Duration::from_millis(200)).await;