vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

retry every failed meta build each round, a timeout included, and drop the attempt count

Browse source
This commit is contained in:
atlas 2026-10-26 18:39:09 +01:00
commit 59456c3c58
1 changed files with 251 additions and 435 deletions
Download patch file Download diff file Expand all files Collapse all files

20
swarm-builder/src/nix_build.rs
View file

@ -80,11 +80,10 @@ impl Builder {
}
let mut cmd = self.nix_command(token);
cmd.args(BUILD_FLAGS).arg(self.meta_attr(job));
// `nix` not finishing (deadline, spawn or wait error) says nothing about
// the config, so it is `Unavailable`; only an exit of `nix` itself is a
// verdict.
// Any build that doesn't succeed is `Failed`, a timeout included; the
// controller builds it again on its next round.
match run_raw(cmd, self.deadline).await {
Err(reason) => FromMetaReply::Unavailable { reason },
Err(error_tail) => FromMetaReply::Failed { error_tail },
Ok((status, out, err)) => match verdict(status, &out, &err) {
Reply::Built { out_path } => FromMetaReply::Built {
out_path,
@ -322,7 +321,7 @@ async fn run_raw(
};
match tokio::time::timeout(deadline, work).await {
Err(_) => Err(format!(
"nix did not finish within {} minutes",
"nix timed out: it did not finish within {} minutes",
deadline.as_secs() / 60
)),
Ok(Err(e)) => Err(format!("waiting for nix: {e}")),
@ -825,11 +824,10 @@ mod tests {
(Builder { nix, ..builder() }, dir)
}
/// `nix` not finishing says nothing about the config: a build past its
/// deadline and a `nix` that cannot start are `Unavailable`. A `nix` that
/// exits non-zero is `Failed`, the control for the same builder.
/// A build past its deadline and a `nix` that cannot start are `Failed`
/// with the reason, the same as a `nix` that exits non-zero.
#[tokio::test]
async fn a_meta_build_that_does_not_finish_is_unavailable_and_one_that_exits_is_failed() {
async fn a_meta_build_that_times_out_or_cannot_start_is_failed_like_one_that_exits() {
let (slow, _guard) = scripted("sleep 5");
let slow = Builder {
deadline: Duration::from_millis(200),
@ -837,7 +835,7 @@ mod tests {
};
let reply = slow.build_from_meta(&meta_job(), "t").await;
assert!(
matches!(&reply, FromMetaReply::Unavailable { reason } if reason.contains("did not finish")),
matches!(&reply, FromMetaReply::Failed { error_tail } if error_tail.contains("timed out")),
"{reply:?}"
);
@ -847,7 +845,7 @@ mod tests {
};
let reply = missing.build_from_meta(&meta_job(), "t").await;
assert!(
matches!(&reply, FromMetaReply::Unavailable { reason } if reason.contains("starting nix")),
matches!(&reply, FromMetaReply::Failed { error_tail } if error_tail.contains("starting nix")),
"{reply:?}"
);