vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-controller: record builder runs with nix's tail and link the status to them

Browse source
This commit is contained in:
atlas 2026-10-17 23:17:00 +02:00 • committed by mara
commit d4714072a9
11 changed files with 833 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

270
swarm-controller/src/builder.rs
View file

@ -1,21 +1,26 @@
//! The `BuildAgent` node: send a config PR's head to the swarm builder over the
//! queue, and turn its answer into the [`CONTEXT`] commit status on that head.
//! queue, turn its answer into the [`CONTEXT`] commit status on that head, and
//! record the build, `nix`'s tail with it, in [`runs`], whose
//! `GET /api/builder/runs/{id}` the status links to.
//!
//! The status is the check's only record besides the node itself: nothing is
//! pushed, and the closure the builder made stays in its host's store. A red
//! status blocks nothing; the merge deploys as before.
//! Nothing is pushed, and the closure the builder made stays in its host's
//! store. A red status blocks nothing; the merge deploys as before.
//!
//! Builds share one `SwarmResourceKind::Builder` slot, so one runs at a time,
//! and a newer push replaces its PR's build while that build is still
//! pending (`queue_build`).
pub(crate) mod runs;
use std::future::Future;
use forgejo_api::structs::CommitStatusState;
use hive_jobq::scheduler::Outcome;
use swarm_queue_client::builder::{BuildAgent, REPLY_DEADLINE, Reply, SUBJECT};
use crate::agent_renewal::unix_now;
use crate::forge::{self, CONFIG_ORG};
use runs::{BuildOutcome, BuildRecord};
/// The commit status context the check writes.
pub const CONTEXT: &str = "swarm-builder";
@ -59,6 +64,7 @@ pub(crate) trait StatusSink {
rev: &str,
state: CommitStatusState,
description: &str,
target_url: Option<&str>,
) -> impl Future<Output = anyhow::Result<()>> + Send;
}
@ -69,21 +75,36 @@ impl StatusSink for forge::Client {
rev: &str,
state: CommitStatusState,
description: &str,
target_url: Option<&str>,
) -> anyhow::Result<()> {
forge::within_forge_deadline(
"build_agent status",
self.create_commit_status(CONFIG_ORG, agent, rev, CONTEXT, state, description),
self.create_commit_status(
CONFIG_ORG,
agent,
rev,
forge::CommitStatus {
context: CONTEXT,
state,
description,
target_url,
},
),
)
.await
.unwrap_or_else(|| Err(anyhow::anyhow!("no answer from the forge")))
}
}
/// Build `job` and write its status. The node fails when the build does, so
/// the Jobs page shows red where the PR does.
/// Build `job`, record it in `builds`, and write its status, linking to the
/// recorded build under `link_base` (the swarm vhost's public URL) when there is
/// one. The node fails when the build does, so the Jobs page shows red where
/// the PR does.
pub(crate) async fn run(
queue: Option<&impl BuildSink>,
statuses: Option<&impl StatusSink>,
builds: &runs::Builds,
link_base: Option<&str>,
job: BuildAgent,
) -> Outcome {
let Some(statuses) = statuses else {
@ -93,22 +114,30 @@ pub(crate) async fn run(
.to_owned(),
);
};
let set = |state, description: String| {
let started_unix = unix_now();
let set = |state, description: String, link: Option<String>| {
let job = &job;
async move {
statuses
.set(&job.agent, &job.rev, state, &description)
.set(&job.agent, &job.rev, state, &description, link.as_deref())
.await
.map_err(|e| format!("{e:#}"))
}
};
let finish =
|outcome, text: String| record(builds, link_base, &job, started_unix, outcome, text);
let Some(queue) = queue else {
let error = "no swarm queue is configured on this host, so no build can be sent";
return settle(set(CommitStatusState::Error, error.to_owned()).await, error);
let link = finish(BuildOutcome::Error, error.to_owned()).await;
return settle(
set(CommitStatusState::Error, error.to_owned(), link).await,
error,
);
};
if let Err(e) = set(
CommitStatusState::Pending,
"building on the swarm builder".to_owned(),
None,
)
.await
{
@ -121,22 +150,74 @@ pub(crate) async fn run(
match queue.build(request).await {
Ok(Reply::Built { out_path }) => {
tracing::info!(agent = %job.agent, rev = %job.rev, %out_path, "build_agent: built");
match set(CommitStatusState::Success, out_path).await {
let link = finish(BuildOutcome::Built, out_path.clone()).await;
match set(CommitStatusState::Success, out_path, link).await {
Ok(()) => Outcome::Done,
Err(e) => Outcome::Failed(format!("built, but the status was not written: {e}")),
}
}
Ok(Reply::Failed { error_tail }) => {
let last = last_line(&error_tail);
let last = last_line(&error_tail).to_owned();
let link = finish(BuildOutcome::Failed, error_tail.clone()).await;
settle(
set(CommitStatusState::Failure, last.to_owned()).await,
set(CommitStatusState::Failure, last, link).await,
&format!("the build failed: {error_tail}"),
)
}
Err(error) => settle(set(CommitStatusState::Error, error.clone()).await, &error),
Err(error) => {
let link = finish(BuildOutcome::Error, error.clone()).await;
settle(
set(CommitStatusState::Error, error.clone(), link).await,
&error,
)
}
}
}
/// Store the build of `job` that ended as `outcome`, `text` being its store
/// path, `nix`'s tail or the reason. The link to it, when it was
/// stored and `link_base` is known.
async fn record(
builds: &runs::Builds,
link_base: Option<&str>,
job: &BuildAgent,
started_unix: i64,
outcome: BuildOutcome,
text: String,
) -> Option<String> {
let mut build = BuildRecord {
id: format!(
"{}-{}-{started_unix}",
job.agent,
job.rev.get(..12).unwrap_or(&job.rev)
),
repo: format!("{CONFIG_ORG}/{}", job.agent),
pr: job.pr,
head_sha: job.rev.clone(),
started_unix,
finished_unix: unix_now(),
outcome,
out_path: None,
error_tail: None,
detail: None,
};
*match outcome {
BuildOutcome::Built => &mut build.out_path,
BuildOutcome::Failed => &mut build.error_tail,
BuildOutcome::Error => &mut build.detail,
} = Some(text);
if !builds.record(&build).await {
return None;
}
link_base.map(|base| {
format!(
"{}/api/builder/runs/{}",
base.trim_end_matches('/'),
build.id
)
})
}
/// A failed node carrying `error`, and the status write's own failure when
/// there was one.
fn settle(written: Result<(), String>, error: &str) -> Outcome {
@ -157,10 +238,13 @@ fn last_line(tail: &str) -> &str {
#[cfg(test)]
mod tests {
use std::sync::Mutex;
use std::sync::{Arc, Mutex};
use super::runs::tests::FakeDoc;
use super::*;
const BASE: &str = "https://swarm.example";
struct FakeQueue(Result<Reply, String>);
impl BuildSink for FakeQueue {
@ -172,7 +256,7 @@ mod tests {
}
#[derive(Default)]
struct FakeStatuses(Mutex<Vec<(CommitStatusState, String)>>);
struct FakeStatuses(Mutex<Vec<(CommitStatusState, String, Option<String>)>>);
impl StatusSink for FakeStatuses {
async fn set(
@ -181,13 +265,22 @@ mod tests {
rev: &str,
state: CommitStatusState,
description: &str,
target_url: Option<&str>,
) -> anyhow::Result<()> {
assert_eq!((agent, rev), ("atlas", "abc"));
self.0.lock().unwrap().push((state, description.to_owned()));
self.0.lock().unwrap().push((
state,
description.to_owned(),
target_url.map(str::to_owned),
));
Ok(())
}
}
fn store() -> runs::Builds {
runs::Builds::Fake(Arc::new(FakeDoc::default()))
}
fn job() -> BuildAgent {
BuildAgent {
agent: "atlas".to_owned(),
@ -198,9 +291,9 @@ mod tests {
async fn statuses_after(queue: Option<FakeQueue>) -> (Outcome, Vec<CommitStatusState>) {
let statuses = FakeStatuses::default();
let outcome = run(queue.as_ref(), Some(&statuses), job()).await;
let outcome = run(queue.as_ref(), Some(&statuses), &store(), Some(BASE), job()).await;
let states = statuses.0.into_inner().unwrap();
(outcome, states.into_iter().map(|(s, _)| s).collect())
(outcome, states.into_iter().map(|(s, _, _)| s).collect())
}
#[tokio::test]
@ -216,19 +309,114 @@ mod tests {
);
}
#[tokio::test]
async fn a_failed_build_goes_red_with_its_last_line_and_fails_the_node() {
/// The build, the doc it was recorded in, and the statuses written.
async fn recorded(
reply: Result<Reply, String>,
doc: FakeDoc,
link_base: Option<&str>,
) -> (
Outcome,
Vec<runs::BuildRecord>,
Vec<(CommitStatusState, String, Option<String>)>,
) {
let statuses = FakeStatuses::default();
let queue = FakeQueue(Ok(Reply::Failed {
error_tail: "evaluating\nerror: boom\n".to_owned(),
}));
let outcome = run(Some(&queue), Some(&statuses), job()).await;
let doc = Arc::new(doc);
let outcome = run(
Some(&FakeQueue(reply)),
Some(&statuses),
&runs::Builds::Fake(Arc::clone(&doc)),
link_base,
job(),
)
.await;
(outcome, doc.builds(), statuses.0.into_inner().unwrap())
}
#[tokio::test]
async fn a_failed_build_goes_red_with_its_last_line_and_links_to_its_recorded_tail() {
let tail = "evaluating\nerror: boom\n";
let (outcome, builds, written) = recorded(
Ok(Reply::Failed {
error_tail: tail.to_owned(),
}),
FakeDoc::default(),
Some("https://swarm.example/"),
)
.await;
assert!(matches!(&outcome, Outcome::Failed(e) if e.contains("error: boom")));
let written = statuses.0.into_inner().unwrap();
let [build] = builds.as_slice() else {
panic!("one build recorded: {builds:?}")
};
assert_eq!(build.repo, "agent-configs/atlas");
assert_eq!((build.pr, build.head_sha.as_str()), (3, "abc"));
assert_eq!(build.outcome, BuildOutcome::Failed);
assert_eq!(build.error_tail.as_deref(), Some(tail));
assert_eq!((&build.out_path, &build.detail), (&None, &None));
assert!(build.id.starts_with("atlas-abc-"), "{}", build.id);
assert!(build.finished_unix >= build.started_unix);
let link = format!("https://swarm.example/api/builder/runs/{}", build.id);
assert_eq!(
written,
[
(
CommitStatusState::Pending,
"building on the swarm builder".to_owned(),
None
),
(
CommitStatusState::Failure,
"error: boom".to_owned(),
Some(link)
),
]
);
}
#[tokio::test]
async fn a_built_config_and_a_missing_builder_are_recorded_and_linked_too() {
let (_, builds, written) = recorded(
Ok(Reply::Built {
out_path: "/nix/store/x".to_owned(),
}),
FakeDoc::default(),
Some(BASE),
)
.await;
assert_eq!(builds[0].outcome, BuildOutcome::Built);
assert_eq!(builds[0].out_path.as_deref(), Some("/nix/store/x"));
assert!(written[1].2.is_some(), "{written:?}");
let (_, builds, written) = recorded(
Err("no swarm builder".to_owned()),
FakeDoc::default(),
Some(BASE),
)
.await;
assert_eq!(builds[0].outcome, BuildOutcome::Error);
assert_eq!(builds[0].detail.as_deref(), Some("no swarm builder"));
assert_eq!(written[1].0, CommitStatusState::Error);
assert!(written[1].2.is_some(), "{written:?}");
}
/// No link to a build that was not stored, nor without the swarm's URL;
/// the status is written either way.
#[tokio::test]
async fn a_status_links_only_to_a_stored_build_under_a_known_url() {
let failed = || {
Ok(Reply::Failed {
error_tail: "error: boom".to_owned(),
})
};
let (_, builds, written) = recorded(failed(), FakeDoc::failing(), Some(BASE)).await;
assert!(builds.is_empty());
assert_eq!(
written.last(),
Some(&(CommitStatusState::Failure, "error: boom".to_owned()))
Some(&(CommitStatusState::Failure, "error: boom".to_owned(), None))
);
let (_, builds, written) = recorded(failed(), FakeDoc::default(), None).await;
assert_eq!(builds.len(), 1, "recorded all the same");
assert_eq!(written.last().map(|w| &w.2), Some(&None));
}
/// A superseded build that finishes after the newer head's writes only on
@ -259,6 +447,7 @@ mod tests {
rev: &str,
state: CommitStatusState,
_: &str,
_: Option<&str>,
) -> anyhow::Result<()> {
self.0.lock().unwrap().push((rev.to_owned(), state));
Ok(())
@ -270,8 +459,22 @@ mod tests {
rev: rev.to_owned(),
..job()
};
run(Some(&ByRevQueue), Some(&statuses), at("new")).await;
run(Some(&ByRevQueue), Some(&statuses), at("old")).await;
run(
Some(&ByRevQueue),
Some(&statuses),
&store(),
None,
at("new"),
)
.await;
run(
Some(&ByRevQueue),
Some(&statuses),
&store(),
None,
at("old"),
)
.await;
let written = statuses.0.into_inner().unwrap();
let last_on = |rev: &str| {
written
@ -310,7 +513,14 @@ mod tests {
panic!("a build with nowhere to report must not be sent")
}
}
let outcome = run(Some(&Unreachable), None::<&FakeStatuses>, job()).await;
let outcome = run(
Some(&Unreachable),
None::<&FakeStatuses>,
&store(),
None,
job(),
)
.await;
assert!(matches!(&outcome, Outcome::Failed(e) if e.contains("no forge access")));
}