vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

build a failed meta build once more, and treat a timeout or missing grant as no verdict

Browse source
This commit is contained in:
atlas 2026-10-26 17:53:39 +01:00
commit 99d2ed5b9e
1 changed files with 621 additions and 111 deletions
Download patch file Download diff file Expand all files Collapse all files

180
swarm-controller/src/meta_build.rs
View file

@ -8,11 +8,17 @@
//! key itself: the builder holds no grant on the bucket. A `Built` reply and
//! nothing else writes, so a red build leaves the previous record.
//!
//! The node ends one of three ways. `Done` is a green build. A red build, the
//! builder's `Failed` reply, is `Failed` with `nix`'s tail: the deploy behind it
//! is skipped and the Jobs page shows why. Anything that stopped the build from
//! giving a verdict is `Failed` with [`RETRY_PREFIX`], which
//! `jobq::queue_meta_update` retries on the meta timer.
//! The node ends one of three ways. `Done` is a green build. Every `Failed` skips
//! the deploy behind it, and the Jobs page shows why. What decides whether the
//! timer builds again (`jobq::queue_meta_update`) is the error text:
//! - No verdict (no builder, a timeout, a forge error, no `internal/meta` head,
//! a missing read grant, a failed closure write): [`RETRY_PREFIX`]. The next
//! build has the same `attempt`.
//! - `nix` exited with an error on the first attempt: [`FIRST_FAILURE_PREFIX`]
//! and `nix`'s tail. The next build is `attempt + 1`, which catches a fetch
//! hiccup inside `nix` without reading its output.
//! - `nix` exited with an error again: `nix`'s tail alone. That is a red build:
//! the deploy stays held until a config merge, and nothing retries it.
use std::fmt::Display;
use std::future::Future;
@ -29,28 +35,54 @@ use crate::meta_flake::{META_REPO, MetaForge};
/// What a `Failed` outcome that is not a verdict on the config starts with.
pub(crate) const RETRY_PREFIX: &str = "not built, will be retried: ";
/// Whether a `Failed` node's `error` says the build gave no verdict.
/// What the `Failed` outcome of a first `nix` error starts with.
pub(crate) const FIRST_FAILURE_PREFIX: &str = "build failed, will be built once more: ";
/// How many times `nix` may exit with an error before the build is red.
const NIX_ATTEMPTS: u32 = 2;
/// Whether a `Failed` node's `error` asks for another build.
pub(crate) fn is_retried(error: &str) -> bool {
error.starts_with(RETRY_PREFIX)
error.starts_with(RETRY_PREFIX) || error.starts_with(FIRST_FAILURE_PREFIX)
}
/// The `attempt` of the build that follows a `Failed` build of `attempt`
/// with `error`: one more after an error `nix` gave, the same after none.
pub(crate) fn next_attempt(attempt: u32, error: &str) -> u32 {
if error.starts_with(FIRST_FAILURE_PREFIX) {
attempt + 1
} else {
attempt
}
}
fn not_built(why: impl Display) -> Outcome {
Outcome::Failed(format!("{RETRY_PREFIX}{why}"))
}
/// Where `internal/meta`'s head is read: the forge in production, a fake in
/// the tests.
pub(crate) trait MetaHead {
/// Where `internal/meta` is read and shared with the builder: the forge in
/// production, a fake in the tests.
pub(crate) trait MetaRepo {
/// The head commit; `None` when the repo has none.
fn head(&self) -> impl Future<Output = Result<Option<String>>> + Send;
/// Make sure the builder's forge account can read the repo.
fn grant_builder_read(&self) -> impl Future<Output = Result<()>> + Send;
}
impl MetaHead for forge::Client {
impl MetaRepo for forge::Client {
async fn head(&self) -> Result<Option<String>> {
let read = self.head_files(SHARED_ORG, META_REPO, None, &[]);
let head = crate::jobq::forge_node("build_meta", read).await?;
Ok(head.map(|h| h.commit))
}
async fn grant_builder_read(&self) -> Result<()> {
crate::jobq::forge_node(
"build_meta",
self.grant_meta_read(forge::builder_account::LOGIN),
)
.await
}
}
/// Where a build is sent and its closure recorded: the queue in production, a
@ -104,10 +136,13 @@ impl MetaBuilds for async_nats::Client {
}
/// Build `agent` from `internal/meta`'s head and record the closure.
/// `attempt` is how many times `nix` has already exited with an error for this
/// deploy.
pub(crate) async fn run(
head: Option<&impl MetaHead>,
head: Option<&impl MetaRepo>,
queue: Option<&impl MetaBuilds>,
agent: String,
attempt: u32,
) -> Outcome {
let Some(head) = head else {
return not_built(
@ -118,6 +153,12 @@ pub(crate) async fn run(
let Some(queue) = queue else {
return not_built("no swarm queue is configured on this host, so no build can be sent");
};
// The repo is private, and the grant is otherwise made only by a meta update.
if let Err(e) = head.grant_builder_read().await {
return not_built(format!(
"the builder's read on internal/meta could not be granted: {e:#}"
));
}
let meta_rev = match head.head().await {
Ok(Some(rev)) => rev,
Ok(None) => return not_built("the swarm has not published internal/meta yet"),
@ -133,6 +174,9 @@ pub(crate) async fn run(
match queue.build(request).await {
Err(why) => not_built(why),
Ok(FromMetaReply::Unavailable { reason }) => not_built(reason),
Ok(FromMetaReply::Failed { error_tail }) if attempt + 1 < NIX_ATTEMPTS => {
Outcome::Failed(format!("{FIRST_FAILURE_PREFIX}{error_tail}"))
}
Ok(FromMetaReply::Failed { error_tail }) => Outcome::Failed(error_tail),
Ok(FromMetaReply::Built { out_path, system }) => {
let closure = AgentClosure {
@ -157,15 +201,34 @@ mod tests {
const REV: &str = "0123456789abcdef0123456789abcdef01234567";
struct FakeHead(Result<Option<&'static str>, &'static str>);
/// `internal/meta` as the forge answers: the head read, and whether the
/// builder's read grant can be made.
struct FakeHead {
head: Result<Option<&'static str>, &'static str>,
grant_fails: bool,
}
impl MetaHead for FakeHead {
impl FakeHead {
fn reading(head: Result<Option<&'static str>, &'static str>) -> Self {
Self {
head,
grant_fails: false,
}
}
}
impl MetaRepo for FakeHead {
async fn head(&self) -> Result<Option<String>> {
match self.0 {
match self.head {
Ok(rev) => Ok(rev.map(str::to_owned)),
Err(e) => Err(anyhow::anyhow!(e)),
}
}
async fn grant_builder_read(&self) -> Result<()> {
anyhow::ensure!(!self.grant_fails, "the forge refused the collaborator");
Ok(())
}
}
/// A queue that answers every build with `reply`, and keeps what it was
@ -215,13 +278,17 @@ mod tests {
}
async fn run_with(head: &FakeHead, queue: &FakeQueue) -> Outcome {
run(Some(head), Some(queue), "atlas".to_owned()).await
run_at(head, queue, 0).await
}
async fn run_at(head: &FakeHead, queue: &FakeQueue, attempt: u32) -> Outcome {
run(Some(head), Some(queue), "atlas".to_owned(), attempt).await
}
#[tokio::test]
async fn a_green_build_of_the_head_is_recorded_for_the_agent() {
let queue = FakeQueue::built();
let outcome = run_with(&FakeHead(Ok(Some(REV))), &queue).await;
let outcome = run_with(&FakeHead::reading(Ok(Some(REV))), &queue).await;
assert_eq!(outcome, Outcome::Done);
assert_eq!(
*queue.sent.lock().unwrap(),
@ -241,22 +308,69 @@ mod tests {
assert!(closure.built_unix > 0);
}
/// The builder said the build is red: the node fails with `nix`'s tail,
/// which is not a retried failure, and nothing is recorded. The green test
/// above is the control that a record is written when the build is green.
#[tokio::test]
async fn a_red_build_fails_with_its_tail_and_records_nothing() {
let queue = FakeQueue::answering(Ok(FromMetaReply::Failed {
fn nix_error() -> FakeQueue {
FakeQueue::answering(Ok(FromMetaReply::Failed {
error_tail: "error: builder for foo failed".to_owned(),
}));
let outcome = run_with(&FakeHead(Ok(Some(REV))), &queue).await;
}))
}
/// `nix` exiting with an error the first time is asked for once more: the
/// failure carries the tail and the retry marker, and the next build is the
/// second attempt. Nothing is recorded.
#[tokio::test]
async fn the_first_nix_error_is_built_once_more() {
let queue = nix_error();
let Outcome::Failed(error) = run_with(&FakeHead::reading(Ok(Some(REV))), &queue).await
else {
panic!("not failed")
};
assert!(is_retried(&error), "{error}");
assert!(error.ends_with("error: builder for foo failed"), "{error}");
assert_eq!(next_attempt(0, &error), 1);
assert!(queue.recorded.lock().unwrap().is_empty());
}
/// The second `nix` error is a red build: the tail alone, nothing retries
/// it, nothing is recorded. The green test above is the control that a
/// record is written when the build is green.
#[tokio::test]
async fn the_second_nix_error_is_red_and_records_nothing() {
let queue = nix_error();
let outcome = run_at(&FakeHead::reading(Ok(Some(REV))), &queue, 1).await;
assert_eq!(
outcome,
Outcome::Failed("error: builder for foo failed".to_owned())
);
assert!(!is_retried("error: builder for foo failed"));
assert!(queue.recorded.lock().unwrap().is_empty());
}
/// A build with no verdict keeps its attempt, so a builder that is away for
/// hours does not use up the retry that a nix error is allowed.
#[test]
fn only_a_nix_error_uses_up_an_attempt() {
assert_eq!(next_attempt(0, &format!("{RETRY_PREFIX}no builder")), 0);
assert_eq!(next_attempt(1, &format!("{RETRY_PREFIX}no builder")), 1);
assert_eq!(next_attempt(0, &format!("{FIRST_FAILURE_PREFIX}tail")), 1);
}
/// A read grant the forge will not make is the controller's precondition,
/// not the config's: nothing is sent and the failure is retried.
#[tokio::test]
async fn a_read_grant_that_cannot_be_made_is_retried_before_anything_is_sent() {
let queue = FakeQueue::built();
let head = FakeHead {
grant_fails: true,
..FakeHead::reading(Ok(Some(REV)))
};
let Outcome::Failed(error) = run_with(&head, &queue).await else {
panic!("not failed")
};
assert!(is_retried(&error), "{error}");
assert!(error.contains("read on internal/meta"), "{error}");
assert!(queue.sent.lock().unwrap().is_empty());
}
#[tokio::test]
async fn a_builder_that_gave_no_verdict_is_a_retried_failure_and_records_nothing() {
for (name, reply) in [
@ -272,7 +386,7 @@ mod tests {
),
] {
let queue = FakeQueue::answering(reply);
let outcome = run_with(&FakeHead(Ok(Some(REV))), &queue).await;
let outcome = run_with(&FakeHead::reading(Ok(Some(REV))), &queue).await;
let Outcome::Failed(error) = outcome else {
panic!("{name}: not failed")
};
@ -287,7 +401,8 @@ mod tests {
record_fails: true,
..FakeQueue::built()
};
let Outcome::Failed(error) = run_with(&FakeHead(Ok(Some(REV))), &queue).await else {
let Outcome::Failed(error) = run_with(&FakeHead::reading(Ok(Some(REV))), &queue).await
else {
panic!("not failed")
};
assert!(is_retried(&error), "{error}");
@ -297,16 +412,21 @@ mod tests {
#[tokio::test]
async fn without_a_head_or_a_queue_nothing_is_sent_and_the_failure_is_retried() {
for (name, head, with_queue) in [
("no repo yet", Some(FakeHead(Ok(None))), true),
("forge error", Some(FakeHead(Err("forge down"))), true),
("no repo yet", Some(FakeHead::reading(Ok(None))), true),
(
"forge error",
Some(FakeHead::reading(Err("forge down"))),
true,
),
("no forge", None, true),
("no queue", Some(FakeHead(Ok(Some(REV)))), false),
("no queue", Some(FakeHead::reading(Ok(Some(REV)))), false),
] {
let queue = FakeQueue::built();
let outcome = run(
head.as_ref(),
with_queue.then_some(&queue),
"atlas".to_owned(),
0,
)
.await;
let Outcome::Failed(error) = outcome else {