vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

gate each deploy on a swarm build of the agent from internal/meta

Browse source
This commit is contained in:
atlas 2026-10-26 16:40:06 +01:00
commit 5e64048e53
2 changed files with 1570 additions and 178 deletions
Download patch file Download diff file Expand all files Collapse all files

223
swarm-builder/src/main.rs
View file

@ -12,6 +12,11 @@
//! flake's `flake.nix`, and the answer is the `flake.lock` `nix` wrote for it in
//! a scratch directory. The controller commits it.
//!
//! Meta build requests on [`swarm_wire::builder::META_SUBJECT`] name an agent
//! and an `internal/meta` commit, and the answer is the store path of that
//! agent's toplevel, built at the flake's own lock. They take the build slot
//! PR checks take. The controller records the path; the builder writes nothing.
//!
//! The store identity comes from the `BAO_*` environment. The queue client
//! secret is read at start and again when authelia refuses it, the forge token
//! once per build, both from the store under that identity.
@ -27,8 +32,9 @@ use swarm_queue_client::{ClientSecret, QueueConfig};
use swarm_secret_client::forge::Credential;
use swarm_secret_client::stored_secret::StoredSecret;
use swarm_wire::builder::{
ARCHETYPES_DEADLINE, ARCHETYPES_SUBJECT, BUILD_DEADLINE, BuildAgent, LOCK_DEADLINE,
LOCK_SUBJECT, LockMeta, LockReply, QUEUE_GROUP, REPLY_DEADLINE, Reply, SUBJECT,
ARCHETYPES_DEADLINE, ARCHETYPES_SUBJECT, BUILD_DEADLINE, BuildAgent, BuildFromMeta,
FromMetaReply, LOCK_DEADLINE, LOCK_SUBJECT, LockMeta, LockReply, META_SUBJECT, QUEUE_GROUP,
REPLY_DEADLINE, Reply, SUBJECT,
};
mod archetypes;
@ -76,6 +82,7 @@ async fn main() -> Result<()> {
config_org: var("CONFIG_ORG")?,
deadline: BUILD_DEADLINE,
lock_deadline: LOCK_DEADLINE,
system: var("SYSTEM")?,
});
let secret =
swarm_secret_client::read_with_startup_retry("reading the queue client secret", || {
@ -124,8 +131,10 @@ async fn main() -> Result<()> {
// One build at a time. swarm-controller already sends one at a time; this
// holds the bound for any request it did not serialise, and `in_slot`
// refuses one that waited too long behind another.
// refuses one that waited too long behind another. Meta builds take the
// same slot as PR checks.
let slot = Arc::new(tokio::sync::Semaphore::new(1));
serve_meta_builds(&queue, &builder, &slot).await?;
while let Some(msg) = requests.next().await {
let received = tokio::time::Instant::now();
@ -139,7 +148,8 @@ async fn main() -> Result<()> {
tokio::spawn(async move {
let reply = match serde_json::from_slice::<BuildAgent>(&msg.payload) {
Ok(job) => {
let Some(reply) = in_slot(&slot, received, &job, handle(&job, &builder)).await
let Some(reply) =
in_slot(&slot, received, || busy(&job), handle(&job, &builder)).await
else {
return;
};
@ -249,36 +259,161 @@ async fn handle_lock(payload: &[u8], builder: &nix_build::Builder) -> LockReply
reply
}
/// `build`'s reply once `job` holds the build slot, or a refusal, without
/// Subscribe to meta build requests and answer them in the background.
async fn serve_meta_builds(
queue: &async_nats::Client,
builder: &Arc<nix_build::Builder>,
slot: &Arc<tokio::sync::Semaphore>,
) -> Result<()> {
let requests = queue
.queue_subscribe(META_SUBJECT, QUEUE_GROUP.to_owned())
.await
.with_context(|| format!("subscribing to {META_SUBJECT}"))?;
tokio::spawn(answer_meta_builds(
requests,
queue.clone(),
Arc::clone(builder),
Arc::clone(slot),
));
Ok(())
}
/// Answer every agent build from the meta flake, each in its own task, with the
/// build's result.
async fn answer_meta_builds(
mut requests: async_nats::Subscriber,
queue: async_nats::Client,
builder: Arc<nix_build::Builder>,
slot: Arc<tokio::sync::Semaphore>,
) {
while let Some(msg) = requests.next().await {
let received = tokio::time::Instant::now();
let Some(reply_to) = msg.reply.clone() else {
tracing::warn!("a meta build arrived with no reply subject; dropping it");
continue;
};
let (queue, builder, slot) = (queue.clone(), Arc::clone(&builder), Arc::clone(&slot));
tokio::spawn(async move {
let reply = match serde_json::from_slice::<BuildFromMeta>(&msg.payload) {
Ok(job) => {
let Some(reply) = in_slot(
&slot,
received,
|| busy_meta(&job),
handle_meta(&job, &builder),
)
.await
else {
return;
};
reply
}
Err(e) => {
tracing::warn!(error = %e, "an undecodable meta build; answering with the error");
FromMetaReply::Failed {
error_tail: format!("the builder could not decode the request: {e}"),
}
}
};
match serde_json::to_vec(&reply) {
Ok(body) => {
if let Err(e) = queue.publish(reply_to, body.into()).await {
tracing::warn!(error = %e, "publishing a meta build's reply failed");
}
}
Err(e) => tracing::warn!(error = %e, "encoding a meta build's reply failed"),
}
});
}
tracing::error!("the subscription to {META_SUBJECT} ended");
}
/// `build`'s reply once a request holds the build slot, or `refuse`'s, without
/// running `build`, once it has waited [`MAX_SLOT_WAIT`] since `received`.
/// `None` only when the slot is closed.
async fn in_slot(
async fn in_slot<T>(
slot: &tokio::sync::Semaphore,
received: tokio::time::Instant,
job: &BuildAgent,
build: impl Future<Output = Reply>,
) -> Option<Reply> {
refuse: impl FnOnce() -> T,
build: impl Future<Output = T>,
) -> Option<T> {
let Ok(permit) = tokio::time::timeout_at(received + MAX_SLOT_WAIT, slot.acquire()).await else {
tracing::warn!(
agent = %job.agent,
rev = %job.rev,
pr = job.pr,
waited_secs = MAX_SLOT_WAIT.as_secs(),
"build refused: another build held the slot too long for this one to finish \
before swarm-controller stops waiting"
);
return Some(Reply::Failed {
error_tail: format!(
"the swarm builder was busy with another build for {} s, too long for this one \
to finish before swarm-controller stops waiting for it",
MAX_SLOT_WAIT.as_secs()
),
});
return Some(refuse());
};
let _permit = permit.ok()?;
Some(build.await)
}
/// The refusal text of a request that waited [`MAX_SLOT_WAIT`] for the slot.
fn busy_tail() -> String {
format!(
"the swarm builder was busy with another build for {} s, too long for this one \
to finish before swarm-controller stops waiting for it",
MAX_SLOT_WAIT.as_secs()
)
}
/// Log and answer a PR check that waited too long for the slot.
fn busy(job: &BuildAgent) -> Reply {
tracing::warn!(
agent = %job.agent,
rev = %job.rev,
pr = job.pr,
waited_secs = MAX_SLOT_WAIT.as_secs(),
"build refused: another build held the slot too long for this one to finish \
before swarm-controller stops waiting"
);
Reply::Failed {
error_tail: busy_tail(),
}
}
/// Log and answer a meta build that waited too long for the slot.
fn busy_meta(job: &BuildFromMeta) -> FromMetaReply {
tracing::warn!(
agent = %job.agent,
meta_rev = %job.meta_rev,
waited_secs = MAX_SLOT_WAIT.as_secs(),
"meta build refused: another build held the slot too long for this one to finish \
before swarm-controller stops waiting"
);
FromMetaReply::Unavailable {
reason: busy_tail(),
}
}
/// Build one meta request and log the build.
async fn handle_meta(job: &BuildFromMeta, builder: &nix_build::Builder) -> FromMetaReply {
let reply = match within(TOKEN_DEADLINE, forge_token()).await {
Ok(token) => builder.build_from_meta(job, &token).await,
Err(e) => FromMetaReply::Unavailable {
reason: format!("{e:#}"),
},
};
match &reply {
FromMetaReply::Built { out_path, system } => tracing::info!(
agent = %job.agent,
meta_rev = %job.meta_rev,
%out_path,
%system,
"agent meta build built"
),
FromMetaReply::Failed { error_tail } => tracing::warn!(
agent = %job.agent,
meta_rev = %job.meta_rev,
%error_tail,
"agent meta build failed"
),
FromMetaReply::Unavailable { reason } => tracing::warn!(
agent = %job.agent,
meta_rev = %job.meta_rev,
%reason,
"agent meta build not run"
),
}
reply
}
/// Build one request and log the build.
async fn handle(job: &BuildAgent, builder: &nix_build::Builder) -> Reply {
let reply = match within(TOKEN_DEADLINE, forge_token()).await {
@ -406,7 +541,7 @@ mod tests {
out_path: "/nix/store/x".to_owned(),
}
};
let reply = in_slot(&slot, received, &job(), build).await;
let reply = in_slot(&slot, received, || busy(&job()), build).await;
(reply, ran.load(Ordering::SeqCst), received.elapsed())
}
@ -430,6 +565,44 @@ mod tests {
assert!(built);
}
/// A meta build waits on the slot a PR check holds, and is refused at the
/// same limit.
#[tokio::test(start_paused = true)]
async fn a_meta_build_shares_the_slot_with_pr_checks() {
let meta = BuildFromMeta {
agent: "atlas".to_owned(),
meta_rev: "abc".to_owned(),
};
let slot = Arc::new(Semaphore::new(1));
let check = Arc::clone(&slot).acquire_owned().await.expect("open");
let received = Instant::now();
let refused = in_slot(&slot, received, || busy_meta(&meta), async {
FromMetaReply::Built {
out_path: "/nix/store/x".to_owned(),
system: "s".to_owned(),
}
})
.await;
assert!(
matches!(&refused, Some(FromMetaReply::Unavailable { reason }) if reason.contains("busy")),
"{refused:?}"
);
assert_eq!(received.elapsed(), MAX_SLOT_WAIT);
// Control: once the check lets go, the same request builds.
drop(check);
let built = in_slot(&slot, Instant::now(), || busy_meta(&meta), async {
FromMetaReply::Built {
out_path: "/nix/store/x".to_owned(),
system: "s".to_owned(),
}
})
.await;
assert!(
matches!(built, Some(FromMetaReply::Built { .. })),
"{built:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_token_read_past_its_deadline_is_an_error_and_a_prompt_one_is_kept() {
let started = Instant::now();

186
swarm-builder/src/nix_build.rs
View file

@ -1,7 +1,8 @@
//! One build: `nix build` of the check flake with its three inputs pointed at
//! this host's hyperhive and nixpkgs and at the agent's config
//! commit. One lock: `nix flake lock` of the swarm meta flake in a scratch
//! directory.
//! commit. One meta build: `nix build` of one agent's toplevel in
//! `internal/meta` at a commit, every input at the flake's own lock. One lock:
//! `nix flake lock` of the swarm meta flake in a scratch directory.
//!
//! The config repo is private, so `git` fetches it with the builder's forge
//! token. The token reaches `git` through a credential helper that reads it
@ -17,7 +18,9 @@ use std::path::PathBuf;
use std::process::Stdio;
use std::time::Duration;
use swarm_wire::builder::{BuildAgent, LockMeta, LockReply, Reply};
use swarm_wire::builder::{
BuildAgent, BuildFromMeta, FromMetaReply, LockMeta, LockReply, META_REPO_PATH, Reply,
};
use tokio::io::AsyncReadExt as _;
/// The most of `nix`'s stderr a failed build's reply carries, from its end.
@ -54,6 +57,8 @@ pub struct Builder {
pub deadline: Duration,
/// How long one lock may run.
pub lock_deadline: Duration,
/// The platform this host builds for, reported with each meta build.
pub system: String,
}
impl Builder {
@ -63,10 +68,27 @@ impl Builder {
return Reply::Failed { error_tail: error };
}
let mut cmd = self.nix_command(token);
cmd.args(self.args(job));
cmd.args(BUILD_FLAGS).args(self.args(job));
run(cmd, self.deadline).await
}
/// Build `job`'s agent from `internal/meta` with `token` as the forge
/// credential.
pub async fn build_from_meta(&self, job: &BuildFromMeta, token: &str) -> FromMetaReply {
if let Err(error_tail) = validate_meta(job) {
return FromMetaReply::Failed { error_tail };
}
let mut cmd = self.nix_command(token);
cmd.args(BUILD_FLAGS).arg(self.meta_attr(job));
match run(cmd, self.deadline).await {
Reply::Built { out_path } => FromMetaReply::Built {
out_path,
system: self.system.clone(),
},
Reply::Failed { error_tail } => FromMetaReply::Failed { error_tail },
}
}
/// Lock `job`'s flake with `token` as the forge credential, in a scratch
/// directory removed afterwards, and answer with the lock `nix` wrote.
pub async fn lock(&self, job: &LockMeta, token: &str) -> LockReply {
@ -119,21 +141,9 @@ impl Builder {
cmd
}
/// The `nix build` arguments for `job`.
/// The `nix build` arguments for `job`, after [`BUILD_FLAGS`].
fn args(&self, job: &BuildAgent) -> Vec<String> {
[
"build",
"--no-link",
"--print-out-paths",
"--no-write-lock-file",
"--option",
"accept-flake-config",
"false",
"--print-build-logs",
]
.into_iter()
.map(str::to_owned)
.chain([
vec![
format!("{}#check", self.check_flake),
"--override-input".to_owned(),
"hyperhive".to_owned(),
@ -144,8 +154,17 @@ impl Builder {
"--override-input".to_owned(),
"agent".to_owned(),
self.agent_url(job),
])
.collect()
]
}
/// The agent's toplevel in `internal/meta` at the commit, the attribute
/// the hives build. No input is overridden: the committed `flake.lock`
/// decides all of them.
fn meta_attr(&self, job: &BuildFromMeta) -> String {
format!(
"git+{}/{META_REPO_PATH}.git?rev={}#nixosConfigurations.{}.config.system.build.toplevel",
self.forge_url, job.meta_rev, job.agent
)
}
/// The config commit as a flake reference, fetched through the PR's head
@ -158,6 +177,19 @@ impl Builder {
}
}
/// What every `nix build` starts with: nothing linked, nothing written to the
/// flake's lock, the flake's own `nix.conf` settings refused.
const BUILD_FLAGS: [&str; 8] = [
"build",
"--no-link",
"--print-out-paths",
"--no-write-lock-file",
"--option",
"accept-flake-config",
"false",
"--print-build-logs",
];
/// `nix flake lock` of the flake in the working directory. The flake's config
/// is not trusted, and nothing is built.
const LOCK_ARGS: &[&str] = &["flake", "lock", "--option", "accept-flake-config", "false"];
@ -207,24 +239,32 @@ fn credential_helper() -> String {
/// Refuse a job whose fields would change the reference's meaning: an agent
/// is one `[a-z0-9-]` segment, and a rev is a full commit id.
fn validate(job: &BuildAgent) -> Result<(), String> {
let agent_ok = !job.agent.is_empty()
&& job
.agent
check_agent_and_rev(&job.agent, &job.rev)?;
if job.pr == 0 {
return Err("PR 0 does not exist".to_owned());
}
Ok(())
}
/// [`validate`] for a meta build: the same agent and commit rules.
fn validate_meta(job: &BuildFromMeta) -> Result<(), String> {
check_agent_and_rev(&job.agent, &job.meta_rev)
}
fn check_agent_and_rev(agent: &str, rev: &str) -> Result<(), String> {
let agent_ok = !agent.is_empty()
&& agent
.bytes()
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-');
if !agent_ok {
return Err(format!("{:?} is not an agent name", job.agent));
return Err(format!("{agent:?} is not an agent name"));
}
let rev_ok = job.rev.len() == 40
&& job
.rev
let rev_ok = rev.len() == 40
&& rev
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b));
if !rev_ok {
return Err(format!("{:?} is not a full commit id", job.rev));
}
if job.pr == 0 {
return Err("PR 0 does not exist".to_owned());
return Err(format!("{rev:?} is not a full commit id"));
}
Ok(())
}
@ -387,6 +427,7 @@ mod tests {
nix: PathBuf::from("nix"),
deadline: BUILD_DEADLINE,
lock_deadline: LOCK_DEADLINE,
system: "x86_64-linux".to_owned(),
}
}
@ -409,7 +450,88 @@ mod tests {
.collect();
assert_eq!(overridden, ["hyperhive", "nixpkgs", "agent"]);
assert!(args.iter().any(|a| a == "/nix/store/c-check#check"));
assert!(args.iter().any(|a| a == "--no-write-lock-file"));
assert!(BUILD_FLAGS.contains(&"--no-write-lock-file"));
}
fn meta_job() -> BuildFromMeta {
BuildFromMeta {
agent: "atlas".to_owned(),
meta_rev: "0123456789abcdef0123456789abcdef01234567".to_owned(),
}
}
#[test]
fn a_meta_build_is_the_agents_toplevel_in_internal_meta_at_the_commit() {
assert_eq!(
builder().meta_attr(&meta_job()),
"git+https://forge.example/internal/meta.git\
?rev=0123456789abcdef0123456789abcdef01234567\
#nixosConfigurations.atlas.config.system.build.toplevel"
);
}
#[test]
fn a_meta_job_that_would_bend_the_reference_is_refused() {
for bad in [
BuildFromMeta {
agent: "a#b".to_owned(),
..meta_job()
},
BuildFromMeta {
agent: String::new(),
..meta_job()
},
BuildFromMeta {
meta_rev: "main".to_owned(),
..meta_job()
},
] {
assert!(validate_meta(&bad).is_err(), "{bad:?}");
}
assert!(validate_meta(&meta_job()).is_ok());
}
#[tokio::test]
async fn a_meta_build_reports_the_path_and_the_hosts_system() {
// `echo` stands in for `nix`: it prints the arguments it was given, which
// are taken for the store path, and exits 0.
let b = Builder {
nix: PathBuf::from("echo"),
..builder()
};
let reply = b.build_from_meta(&meta_job(), "t").await;
assert_eq!(
reply,
FromMetaReply::Built {
out_path: format!("{} {}", BUILD_FLAGS.join(" "), b.meta_attr(&meta_job())),
system: "x86_64-linux".to_owned(),
}
);
}
#[tokio::test]
async fn a_failed_meta_build_is_a_failed_reply_and_a_refused_job_is_failed_without_nix() {
let b = Builder {
nix: PathBuf::from("false"),
..builder()
};
let failed = b.build_from_meta(&meta_job(), "t").await;
assert!(matches!(failed, FromMetaReply::Failed { .. }), "{failed:?}");
// Control: `a_meta_build_reports_the_path_and_the_hosts_system` builds this
// same job, so the refusal below is the validation, not the stub.
let refused = b
.build_from_meta(
&BuildFromMeta {
meta_rev: "main".to_owned(),
..meta_job()
},
"t",
)
.await;
assert!(
matches!(&refused, FromMetaReply::Failed { error_tail } if error_tail.contains("commit id")),
"{refused:?}"
);
}
/// What the helper prints for `op` and a request with `protocol` and