mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

gate each deploy on a swarm build of the agent from internal/meta

Browse source
This commit is contained in:
atlas 2026-10-26 16:40:06 +01:00
commit 5e64048e53
1 changed files with 1570 additions and 178 deletions
Download patch file Download diff file Expand all files Collapse all files

223
swarm-builder/src/main.rs
View file

@ -12,6 +12,11 @@
//! flake's `flake.nix`, and the answer is the `flake.lock` `nix` wrote for it in
//! a scratch directory. The controller commits it.
//!
//! Meta build requests on [`swarm_wire::builder::META_SUBJECT`] name an agent
//! and an `internal/meta` commit, and the answer is the store path of that
//! agent's toplevel, built at the flake's own lock. They take the build slot
//! PR checks take. The controller records the path; the builder writes nothing.
//!
//! The store identity comes from the `BAO_*` environment. The queue client
//! secret is read at start and again when authelia refuses it, the forge token
//! once per build, both from the store under that identity.
@ -27,8 +32,9 @@ use swarm_queue_client::{ClientSecret, QueueConfig};
use swarm_secret_client::forge::Credential;
use swarm_secret_client::stored_secret::StoredSecret;
use swarm_wire::builder::{
ARCHETYPES_DEADLINE, ARCHETYPES_SUBJECT, BUILD_DEADLINE, BuildAgent, LOCK_DEADLINE,
LOCK_SUBJECT, LockMeta, LockReply, QUEUE_GROUP, REPLY_DEADLINE, Reply, SUBJECT,
ARCHETYPES_DEADLINE, ARCHETYPES_SUBJECT, BUILD_DEADLINE, BuildAgent, BuildFromMeta,
FromMetaReply, LOCK_DEADLINE, LOCK_SUBJECT, LockMeta, LockReply, META_SUBJECT, QUEUE_GROUP,
REPLY_DEADLINE, Reply, SUBJECT,
};
mod archetypes;
@ -76,6 +82,7 @@ async fn main() -> Result<()> {
config_org: var("CONFIG_ORG")?,
deadline: BUILD_DEADLINE,
lock_deadline: LOCK_DEADLINE,
system: var("SYSTEM")?,
});
let secret =
swarm_secret_client::read_with_startup_retry("reading the queue client secret", || {
@ -124,8 +131,10 @@ async fn main() -> Result<()> {
// One build at a time. swarm-controller already sends one at a time; this
// holds the bound for any request it did not serialise, and `in_slot`
// refuses one that waited too long behind another.
// refuses one that waited too long behind another. Meta builds take the
// same slot as PR checks.
let slot = Arc::new(tokio::sync::Semaphore::new(1));
serve_meta_builds(&queue, &builder, &slot).await?;
while let Some(msg) = requests.next().await {
let received = tokio::time::Instant::now();
@ -139,7 +148,8 @@ async fn main() -> Result<()> {
tokio::spawn(async move {
let reply = match serde_json::from_slice::<BuildAgent>(&msg.payload) {
Ok(job) => {
let Some(reply) = in_slot(&slot, received, &job, handle(&job, &builder)).await
let Some(reply) =
in_slot(&slot, received, || busy(&job), handle(&job, &builder)).await
else {
return;
};
@ -249,36 +259,161 @@ async fn handle_lock(payload: &[u8], builder: &nix_build::Builder) -> LockReply
reply
}
/// `build`'s reply once `job` holds the build slot, or a refusal, without
/// Subscribe to meta build requests and answer them in the background.
async fn serve_meta_builds(
queue: &async_nats::Client,
builder: &Arc<nix_build::Builder>,
slot: &Arc<tokio::sync::Semaphore>,
) -> Result<()> {
let requests = queue
.queue_subscribe(META_SUBJECT, QUEUE_GROUP.to_owned())
.await
.with_context(|| format!("subscribing to {META_SUBJECT}"))?;
tokio::spawn(answer_meta_builds(
requests,
queue.clone(),
Arc::clone(builder),
Arc::clone(slot),
));
Ok(())
}
/// Answer every agent build from the meta flake, each in its own task, with the
/// build's result.
async fn answer_meta_builds(
mut requests: async_nats::Subscriber,
queue: async_nats::Client,
builder: Arc<nix_build::Builder>,
slot: Arc<tokio::sync::Semaphore>,
) {
while let Some(msg) = requests.next().await {
let received = tokio::time::Instant::now();
let Some(reply_to) = msg.reply.clone() else {
tracing::warn!("a meta build arrived with no reply subject; dropping it");
continue;
};
let (queue, builder, slot) = (queue.clone(), Arc::clone(&builder), Arc::clone(&slot));
tokio::spawn(async move {
let reply = match serde_json::from_slice::<BuildFromMeta>(&msg.payload) {
Ok(job) => {
let Some(reply) = in_slot(
&slot,
received,
|| busy_meta(&job),
handle_meta(&job, &builder),
)
.await
else {
return;
};
reply
}
Err(e) => {
tracing::warn!(error = %e, "an undecodable meta build; answering with the error");
FromMetaReply::Failed {
error_tail: format!("the builder could not decode the request: {e}"),
}
}
};
match serde_json::to_vec(&reply) {
Ok(body) => {
if let Err(e) = queue.publish(reply_to, body.into()).await {
tracing::warn!(error = %e, "publishing a meta build's reply failed");
}
}
Err(e) => tracing::warn!(error = %e, "encoding a meta build's reply failed"),
}
});
}
tracing::error!("the subscription to {META_SUBJECT} ended");
}
/// `build`'s reply once a request holds the build slot, or `refuse`'s, without
/// running `build`, once it has waited [`MAX_SLOT_WAIT`] since `received`.
/// `None` only when the slot is closed.
async fn in_slot(
async fn in_slot<T>(
slot: &tokio::sync::Semaphore,
received: tokio::time::Instant,
job: &BuildAgent,
build: impl Future<Output = Reply>,
) -> Option<Reply> {
refuse: impl FnOnce() -> T,
build: impl Future<Output = T>,
) -> Option<T> {
let Ok(permit) = tokio::time::timeout_at(received + MAX_SLOT_WAIT, slot.acquire()).await else {
tracing::warn!(
agent = %job.agent,
rev = %job.rev,
pr = job.pr,
waited_secs = MAX_SLOT_WAIT.as_secs(),
"build refused: another build held the slot too long for this one to finish \
before swarm-controller stops waiting"
);
return Some(Reply::Failed {
error_tail: format!(
"the swarm builder was busy with another build for {} s, too long for this one \
to finish before swarm-controller stops waiting for it",
MAX_SLOT_WAIT.as_secs()
),
});
return Some(refuse());
};
let _permit = permit.ok()?;
Some(build.await)
}
/// The refusal text of a request that waited [`MAX_SLOT_WAIT`] for the slot.
fn busy_tail() -> String {
format!(
"the swarm builder was busy with another build for {} s, too long for this one \
to finish before swarm-controller stops waiting for it",
MAX_SLOT_WAIT.as_secs()
)
}
/// Log and answer a PR check that waited too long for the slot.
fn busy(job: &BuildAgent) -> Reply {
tracing::warn!(
agent = %job.agent,
rev = %job.rev,
pr = job.pr,
waited_secs = MAX_SLOT_WAIT.as_secs(),
"build refused: another build held the slot too long for this one to finish \
before swarm-controller stops waiting"
);
Reply::Failed {
error_tail: busy_tail(),
}
}
/// Log and answer a meta build that waited too long for the slot.
fn busy_meta(job: &BuildFromMeta) -> FromMetaReply {
tracing::warn!(
agent = %job.agent,
meta_rev = %job.meta_rev,
waited_secs = MAX_SLOT_WAIT.as_secs(),
"meta build refused: another build held the slot too long for this one to finish \
before swarm-controller stops waiting"
);
FromMetaReply::Unavailable {
reason: busy_tail(),
}
}
/// Build one meta request and log the build.
async fn handle_meta(job: &BuildFromMeta, builder: &nix_build::Builder) -> FromMetaReply {
let reply = match within(TOKEN_DEADLINE, forge_token()).await {
Ok(token) => builder.build_from_meta(job, &token).await,
Err(e) => FromMetaReply::Unavailable {
reason: format!("{e:#}"),
},
};
match &reply {
FromMetaReply::Built { out_path, system } => tracing::info!(
agent = %job.agent,
meta_rev = %job.meta_rev,
%out_path,
%system,
"agent meta build built"
),
FromMetaReply::Failed { error_tail } => tracing::warn!(
agent = %job.agent,
meta_rev = %job.meta_rev,
%error_tail,
"agent meta build failed"
),
FromMetaReply::Unavailable { reason } => tracing::warn!(
agent = %job.agent,
meta_rev = %job.meta_rev,
%reason,
"agent meta build not run"
),
}
reply
}
/// Build one request and log the build.
async fn handle(job: &BuildAgent, builder: &nix_build::Builder) -> Reply {
let reply = match within(TOKEN_DEADLINE, forge_token()).await {
@ -406,7 +541,7 @@ mod tests {
out_path: "/nix/store/x".to_owned(),
}
};
let reply = in_slot(&slot, received, &job(), build).await;
let reply = in_slot(&slot, received, || busy(&job()), build).await;
(reply, ran.load(Ordering::SeqCst), received.elapsed())
}
@ -430,6 +565,44 @@ mod tests {
assert!(built);
}
/// A meta build waits on the slot a PR check holds, and is refused at the
/// same limit.
#[tokio::test(start_paused = true)]
async fn a_meta_build_shares_the_slot_with_pr_checks() {
let meta = BuildFromMeta {
agent: "atlas".to_owned(),
meta_rev: "abc".to_owned(),
};
let slot = Arc::new(Semaphore::new(1));
let check = Arc::clone(&slot).acquire_owned().await.expect("open");
let received = Instant::now();
let refused = in_slot(&slot, received, || busy_meta(&meta), async {
FromMetaReply::Built {
out_path: "/nix/store/x".to_owned(),
system: "s".to_owned(),
}
})
.await;
assert!(
matches!(&refused, Some(FromMetaReply::Unavailable { reason }) if reason.contains("busy")),
"{refused:?}"
);
assert_eq!(received.elapsed(), MAX_SLOT_WAIT);
// Control: once the check lets go, the same request builds.
drop(check);
let built = in_slot(&slot, Instant::now(), || busy_meta(&meta), async {
FromMetaReply::Built {
out_path: "/nix/store/x".to_owned(),
system: "s".to_owned(),
}
})
.await;
assert!(
matches!(built, Some(FromMetaReply::Built { .. })),
"{built:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_token_read_past_its_deadline_is_an_error_and_a_prompt_one_is_kept() {
let started = Instant::now();