vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

comment once on a merged config pr when its swarm build fails

Browse source
This commit is contained in:
atlas 2026-10-26 21:00:51 +01:00 • committed by mara
commit 0bc5a7e475
1 changed files with 460 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

227
swarm-controller/src/meta_build.rs
View file

@ -19,6 +19,11 @@
//!
//! Both are built again on every round of the meta timer
//! (`jobq::queue_meta_update`) until a build succeeds.
//!
//! A `nix` failure of a merged config PR's build (`pr` set) also leaves one
//! comment on that PR, deduped by a marker in the comment per agent and
//! `internal/meta` rev, so the retry rounds stay quiet. A no-verdict failure
//! says nothing about the config and leaves none.
use std::fmt::Display;
use std::future::Future;
@ -51,6 +56,15 @@ pub(crate) trait MetaRepo {
fn head(&self) -> impl Future<Output = Result<Option<String>>> + Send;
/// Make sure the builder's forge account can read the repo.
fn grant_builder_read(&self) -> impl Future<Output = Result<()>> + Send;
/// Comment `body` on `pr` of `agent`'s config repo unless a comment there
/// already contains `marker`. Whether a comment was posted.
fn comment_once(
&self,
agent: &str,
pr: u64,
marker: &str,
body: &str,
) -> impl Future<Output = Result<bool>> + Send;
}
impl MetaRepo for forge::Client {
@ -67,6 +81,14 @@ impl MetaRepo for forge::Client {
)
.await
}
async fn comment_once(&self, agent: &str, pr: u64, marker: &str, body: &str) -> Result<bool> {
crate::jobq::forge_node(
"build_meta",
forge::Client::comment_once(self, forge::CONFIG_ORG, agent, pr, marker, body),
)
.await
}
}
/// Where a build is sent and its closure recorded: the queue in production, a
@ -119,11 +141,55 @@ impl MetaBuilds for async_nats::Client {
}
}
/// The most of `nix`'s tail a PR comment carries, in bytes: the builder's own cap
/// on the tail it sends.
const COMMENT_TAIL_BYTES: usize = 4000;
/// The comment a red build leaves on its merged config PR, first line the
/// marker the dedupe looks for. One marker per agent and `internal/meta` rev.
fn failure_comment(agent: &str, meta_rev: &str, error_tail: &str) -> (String, String) {
let marker = format!("<!-- swarm-build-failed agent={agent} meta={meta_rev} -->");
let mut start = error_tail.len().saturating_sub(COMMENT_TAIL_BYTES);
while !error_tail.is_char_boundary(start) {
start += 1;
}
let tail = error_tail[start..].replace("```", "'''");
let body = format!(
"{marker}\nThe swarm could not build `{agent}` from `internal/meta` at `{meta_rev}` \
after this PR merged. The deploy is held and built again on every round until a \
build succeeds.\n\nLast lines of `nix`:\n\n```\n{}\n```",
tail.trim_end()
);
(marker, body)
}
/// Tell `pr` that its merge's build failed, once per agent and `meta_rev`. A
/// comment that cannot be posted is logged and changes nothing about the node.
async fn report_failure(
head: &impl MetaRepo,
pr: u64,
agent: &str,
meta_rev: &str,
error_tail: &str,
) {
let (marker, body) = failure_comment(agent, meta_rev, error_tail);
match head.comment_once(agent, pr, &marker, &body).await {
Ok(posted) => {
tracing::info!(%agent, pr, %meta_rev, posted, "build_meta: reported the failed build on the PR");
}
Err(e) => tracing::warn!(
%agent, pr, %meta_rev, error = %format!("{e:#}"),
"build_meta: could not comment on the PR"
),
}
}
/// Build `agent` from `internal/meta`'s head and record the closure.
pub(crate) async fn run(
head: Option<&impl MetaRepo>,
queue: Option<&impl MetaBuilds>,
agent: String,
pr: Option<u64>,
) -> Outcome {
let Some(head) = head else {
return no_verdict(
@ -155,7 +221,12 @@ pub(crate) async fn run(
match queue.build(request).await {
Err(why) => no_verdict(why),
Ok(FromMetaReply::Unavailable { reason }) => no_verdict(reason),
Ok(FromMetaReply::Failed { error_tail }) => Outcome::Failed(error_tail),
Ok(FromMetaReply::Failed { error_tail }) => {
if let Some(pr) = pr {
report_failure(head, pr, &agent, &meta_rev, &error_tail).await;
}
Outcome::Failed(error_tail)
}
Ok(FromMetaReply::Built { out_path, system }) => {
let closure = AgentClosure {
meta_rev,
@ -184,6 +255,8 @@ mod tests {
struct FakeHead {
head: Result<Option<&'static str>, &'static str>,
grant_fails: bool,
/// The forge a PR comment goes to; `None` panics on a comment.
comments: Option<std::sync::Arc<forge::Client>>,
}
impl FakeHead {
@ -191,6 +264,7 @@ mod tests {
Self {
head,
grant_fails: false,
comments: None,
}
}
}
@ -207,6 +281,19 @@ mod tests {
anyhow::ensure!(!self.grant_fails, "the forge refused the collaborator");
Ok(())
}
async fn comment_once(
&self,
agent: &str,
pr: u64,
marker: &str,
body: &str,
) -> Result<bool> {
let client = self.comments.as_ref().expect("no comment expected");
client
.comment_once(forge::CONFIG_ORG, agent, pr, marker, body)
.await
}
}
/// A queue that answers every build with `reply`, and keeps what it was
@ -256,7 +343,11 @@ mod tests {
}
async fn run_with(head: &FakeHead, queue: &FakeQueue) -> Outcome {
run(Some(head), Some(queue), "atlas".to_owned()).await
run(Some(head), Some(queue), "atlas".to_owned(), None).await
}
async fn run_for_pr(head: &FakeHead, queue: &FakeQueue, pr: u64) -> Outcome {
run(Some(head), Some(queue), "atlas".to_owned(), Some(pr)).await
}
#[tokio::test]
@ -374,6 +465,7 @@ mod tests {
head.as_ref(),
with_queue.then_some(&queue),
"atlas".to_owned(),
None,
)
.await;
let Outcome::Failed(error) = outcome else {
@ -384,6 +476,137 @@ mod tests {
}
}
const REV2: &str = "fedcba9876543210fedcba9876543210fedcba98";
fn nix_failed(tail: &str) -> FakeQueue {
FakeQueue::answering(Ok(FromMetaReply::Failed {
error_tail: tail.to_owned(),
}))
}
async fn head_with_comments(
rev: &'static str,
) -> (FakeHead, crate::forge::pr_comment::tests::Posted) {
let (client, posted) = crate::forge::pr_comment::tests::stand_in().await;
let head = FakeHead {
comments: Some(std::sync::Arc::new(client)),
..FakeHead::reading(Ok(Some(rev)))
};
(head, posted)
}
/// A `nix` failure of a merged config PR's build is one comment on that PR
/// with the agent, the `internal/meta` rev, `nix`'s tail and the retry.
#[tokio::test]
async fn a_red_build_of_a_merged_pr_comments_once_on_it() {
let (head, posted) = head_with_comments(REV).await;
let outcome = run_for_pr(&head, &nix_failed("error: builder for foo failed"), 7).await;
assert_eq!(
outcome,
Outcome::Failed("error: builder for foo failed".to_owned())
);
let posted = posted.lock().unwrap();
let [(path, body)] = posted.as_slice() else {
panic!("one comment: {posted:?}")
};
assert_eq!(
path,
&format!("/api/v1/repos/agent-configs/atlas/{}/7/comments", "issues")
);
assert!(body.contains("`atlas`"), "{body}");
assert!(body.contains(REV), "{body}");
assert!(body.contains("error: builder for foo failed"), "{body}");
assert!(body.contains("held"), "{body}");
assert!(body.contains("every round"), "{body}");
}
/// The timer builds a failed merge again on every round: the same agent and
/// `internal/meta` rev is not commented twice. A new rev is a new failure,
/// so it is.
#[tokio::test]
async fn the_retry_rounds_do_not_comment_again_but_a_new_meta_rev_does() {
let (head, posted) = head_with_comments(REV).await;
for _ in 0..3 {
run_for_pr(&head, &nix_failed("error: boom"), 7).await;
}
let count = posted.lock().unwrap().len();
assert_eq!(count, 1, "quiet retries");
let moved = FakeHead {
comments: head.comments.clone(),
..FakeHead::reading(Ok(Some(REV2)))
};
run_for_pr(&moved, &nix_failed("error: boom"), 7).await;
let posted = posted.lock().unwrap();
assert_eq!(posted.len(), 2, "{posted:?}");
assert!(posted[1].1.contains(REV2), "{:?}", posted[1]);
}
/// A build that got no verdict says nothing about the PR's config, so it
/// leaves no comment. The `nix` failure arm is the control.
#[tokio::test]
async fn a_no_verdict_failure_does_not_comment() {
for (name, reply) in [
("no responders", Err("no swarm builder".to_owned())),
(
"unavailable",
Ok(FromMetaReply::Unavailable {
reason: "busy".to_owned(),
}),
),
] {
let (head, posted) = head_with_comments(REV).await;
let outcome = run_for_pr(&head, &FakeQueue::answering(reply), 7).await;
assert!(
matches!(outcome, Outcome::Failed(e) if is_no_verdict(&e)),
"{name}"
);
assert!(posted.lock().unwrap().is_empty(), "{name}");
}
let (head, posted) = head_with_comments(REV).await;
run_for_pr(&head, &nix_failed("error: boom"), 7).await;
assert_eq!(
posted.lock().unwrap().len(),
1,
"control: a nix failure comments"
);
}
/// A build that is not a merged PR's (the pin fan-out, a created agent)
/// has nobody to comment on.
#[tokio::test]
async fn a_failed_build_that_is_not_from_a_pr_does_not_comment() {
let (head, posted) = head_with_comments(REV).await;
let outcome = run_with(&head, &nix_failed("error: boom")).await;
assert_eq!(outcome, Outcome::Failed("error: boom".to_owned()));
assert!(posted.lock().unwrap().is_empty());
}
#[tokio::test]
async fn the_comment_carries_the_end_of_a_long_tail_only() {
let tail = format!("START{}END", "x".repeat(10_000));
let (head, posted) = head_with_comments(REV).await;
run_for_pr(&head, &nix_failed(&tail), 7).await;
let posted = posted.lock().unwrap();
let body = &posted[0].1;
assert!(body.contains("END"), "the end of the tail is kept");
assert!(!body.contains("START"), "the start is cut");
assert!(body.len() < COMMENT_TAIL_BYTES + 1000, "{}", body.len());
}
/// A forge that does not take the comment leaves the node failed as it was.
#[tokio::test]
async fn a_comment_that_cannot_be_posted_does_not_change_the_outcome() {
let head = FakeHead {
comments: Some(std::sync::Arc::new(crate::forge::Client::for_test_forge(
url::Url::parse("http://127.0.0.1:1/").unwrap(),
))),
..FakeHead::reading(Ok(Some(REV)))
};
let outcome = run_for_pr(&head, &nix_failed("error: boom"), 7).await;
assert_eq!(outcome, Outcome::Failed("error: boom".to_owned()));
}
#[test]
fn the_repo_path_the_builder_fetches_is_the_one_the_controller_writes() {
assert_eq!(