swarm-builder: build each agent config PR head and post a commit status
A new `swarm-builder` service, in its own container on the host that sets
`deploy.builder.enable` (defaulted on by `allSwarmServices` where the host
has the builder's store leaf, like the review classifier).
When an `agent-configs/<agent>` PR is opened, reopened or synchronized,
swarm-controller queues a `build_agent` jobq node. The node sets a
`swarm-builder` commit status to pending on the PR head, sends
`{agent, rev, pr}` to the builder over NATS request-reply
(`$SWARM.builder.build`), and writes success (store path), failure (last
line nix printed) or error (no builder / no reply) on the head. The node
fails when the build does, so the Jobs page matches the PR.
The builder builds nix/builder/check-flake's `check` with every input
overridden: hyperhive and nixpkgs to the host's own
`c0re.hyperhiveFlake`/`c0re.nixpkgsFlake` (what a hive there renders into
its meta flake), and the agent to the PR head on the forge. Evaluation is
pure and runs in the container; builds go to the host nix daemon as an
untrusted client. The private config repo is fetched with the token of a
new read-only `builder` forge account (in the `hives` team), which the
controller mints into `swarm/services/builder/forge/token`; git gets it
from a credential helper reading the child's environment, so it is in no
argument or URL.
Credentials: a `builder` store leaf minted by glue-bao-tls on the store
host, a `swarm-builder` reader policy (forge token + its own OIDC client
secret), an authelia machine client, and a `--builder-client` grant in the
queue's auth-callout responder. The controller gains a grant on the token
path and the secret publisher a deny on it.
This is the agent-config build check atlas scoped on #5081 (slice 3):
nothing is pushed to a cache, nothing is written to KV, and a red status
gates no deploy. The check composes agent-base + the config without the
values a hive adds (CA, docs, claude-code path, ports, input follows), so
it can disagree with a hive's own build.
This commit is contained in:
atlas
parent
aece1c26d0
commit
b68b74d687
43 changed files with 2452 additions and 42 deletions
190
swarm-builder/src/main.rs
Normal file
190
swarm-builder/src/main.rs
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
//! The swarm builder's build service.
|
||||
//!
|
||||
//! Connects to the swarm queue as its own OIDC client, takes build requests
|
||||
//! from swarm-controller on [`swarm_queue_client::builder::SUBJECT`], builds
|
||||
//! each agent config at its commit, answers on the request's reply subject,
|
||||
//! and logs one line per build. It writes nothing to the forge: the
|
||||
//! controller turns the reply into a commit status.
|
||||
//!
|
||||
//! The store identity comes from the `BAO_*` environment. The queue client
|
||||
//! secret is read at start and again when authelia refuses it, the forge token
|
||||
//! once per build, both from the store under that identity.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use futures_util::StreamExt as _;
|
||||
use swarm_queue_client::builder::{BUILD_DEADLINE, BuildAgent, QUEUE_GROUP, Reply, SUBJECT};
|
||||
use swarm_queue_client::stored_secret::StoredSecret;
|
||||
use swarm_queue_client::{ClientSecret, QueueConfig};
|
||||
use swarm_secret_client::forge::Credential;
|
||||
|
||||
mod nix_build;
|
||||
|
||||
/// The cert-auth role this service logs in to the store as. `swarm-bao.nix`
|
||||
/// writes it, matching the leaf's subject.
|
||||
const CERT_ROLE: &str = "swarm-builder";
|
||||
|
||||
/// The environment prefix `swarm-builder.nix` sets.
|
||||
const PREFIX: &str = "SWARM_BUILDER";
|
||||
|
||||
/// Seconds to wait after each failed read of the queue client secret. A
|
||||
/// longer outage fails the start, and the unit's `Restart=` tries again.
|
||||
const RETRY_DELAYS_S: [u64; 6] = [1, 2, 4, 8, 16, 30];
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
hive_log::init();
|
||||
swarm_queue_client::install_crypto_provider();
|
||||
let var = |name: &str| {
|
||||
let full = format!("{PREFIX}_{name}");
|
||||
std::env::var(&full).with_context(|| format!("{full} is unset"))
|
||||
};
|
||||
let client_id = var("OIDC_CLIENT_ID")?;
|
||||
let builder = Arc::new(nix_build::Builder {
|
||||
nix: PathBuf::from(var("NIX")?),
|
||||
check_flake: var("CHECK_FLAKE")?,
|
||||
hyperhive: var("HYPERHIVE_FLAKE")?,
|
||||
nixpkgs: var("NIXPKGS_FLAKE")?,
|
||||
forge_url: var("FORGE_URL")?.trim_end_matches('/').to_owned(),
|
||||
config_org: var("CONFIG_ORG")?,
|
||||
deadline: BUILD_DEADLINE,
|
||||
});
|
||||
let secret = client_secret_with_retry(&client_id).await?;
|
||||
let reread_id = client_id.clone();
|
||||
let secret = StoredSecret::new(client_id.clone(), secret, move || {
|
||||
let client_id = reread_id.clone();
|
||||
async move { client_secret(&client_id).await.map_err(Into::into) }
|
||||
});
|
||||
let queue = swarm_queue_client::connect(QueueConfig {
|
||||
url: var("NATS_URL")?,
|
||||
token_endpoint: var("OIDC_TOKEN_ENDPOINT")?,
|
||||
client_id,
|
||||
client_secret: ClientSecret::Stored(secret),
|
||||
ca_file: std::env::var(format!("{PREFIX}_OIDC_CA_FILE"))
|
||||
.ok()
|
||||
.map(PathBuf::from),
|
||||
})
|
||||
.await?;
|
||||
let mut requests = queue
|
||||
.queue_subscribe(SUBJECT, QUEUE_GROUP.to_owned())
|
||||
.await
|
||||
.with_context(|| format!("subscribing to {SUBJECT}"))?;
|
||||
tracing::info!(subject = SUBJECT, "builder: awaiting builds");
|
||||
|
||||
while let Some(msg) = requests.next().await {
|
||||
let Some(reply_to) = msg.reply.clone() else {
|
||||
tracing::warn!("a build arrived with no reply subject; dropping it");
|
||||
continue;
|
||||
};
|
||||
let queue = queue.clone();
|
||||
let builder = Arc::clone(&builder);
|
||||
tokio::spawn(async move {
|
||||
let reply = handle(&msg.payload, &builder).await;
|
||||
match serde_json::to_vec(&reply) {
|
||||
Ok(body) => {
|
||||
if let Err(e) = queue.publish(reply_to, body.into()).await {
|
||||
tracing::warn!(error = %e, "publishing a build's reply failed");
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::warn!(error = %e, "encoding a build's reply failed"),
|
||||
}
|
||||
});
|
||||
}
|
||||
bail!("the subscription to {SUBJECT} ended")
|
||||
}
|
||||
|
||||
/// Decode one request, build it and log the build.
|
||||
async fn handle(payload: &[u8], builder: &nix_build::Builder) -> Reply {
|
||||
let job: BuildAgent = match serde_json::from_slice(payload) {
|
||||
Ok(job) => job,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "an undecodable build; answering with the error");
|
||||
return Reply::Failed {
|
||||
error_tail: format!("the builder could not decode the request: {e}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
let reply = match forge_token().await {
|
||||
Ok(token) => builder.build(&job, &token).await,
|
||||
Err(e) => Reply::Failed {
|
||||
error_tail: format!("{e:#}"),
|
||||
},
|
||||
};
|
||||
log_build(&job, &reply);
|
||||
reply
|
||||
}
|
||||
|
||||
/// The one journal line a build leaves.
|
||||
fn log_build(job: &BuildAgent, reply: &Reply) {
|
||||
match reply {
|
||||
Reply::Built { out_path } => tracing::info!(
|
||||
agent = %job.agent,
|
||||
rev = %job.rev,
|
||||
pr = job.pr,
|
||||
%out_path,
|
||||
"agent config built"
|
||||
),
|
||||
Reply::Failed { error_tail } => tracing::warn!(
|
||||
agent = %job.agent,
|
||||
rev = %job.rev,
|
||||
pr = job.pr,
|
||||
%error_tail,
|
||||
"agent config build failed"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// The forge token the builder fetches config repos with, read on every
|
||||
/// build so a rotation takes effect on the next one.
|
||||
async fn forge_token() -> Result<String> {
|
||||
let path = swarm_secret_client::builder::forge_token_path()?;
|
||||
let store = swarm_secret_client::SecretStore::from_env(CERT_ROLE)
|
||||
.await
|
||||
.context("logging in to the secret store")?;
|
||||
let stored: Option<Credential> = store
|
||||
.read_optional(&path)
|
||||
.await
|
||||
.with_context(|| format!("reading {path}"))?;
|
||||
match stored {
|
||||
Some(c) if !c.value.trim().is_empty() => Ok(c.value.trim().to_owned()),
|
||||
_ => bail!("nothing at {path}: swarm-controller has not minted the builder's forge token"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The queue client secret, read from the store, retried through a store that
|
||||
/// is restarting alongside this service.
|
||||
async fn client_secret_with_retry(client_id: &str) -> Result<String> {
|
||||
for delay in RETRY_DELAYS_S.map(Duration::from_secs) {
|
||||
match client_secret(client_id).await {
|
||||
Ok(secret) => return Ok(secret),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %format!("{e:#}"), retry_in = ?delay, "reading the queue client secret failed");
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
client_secret(client_id).await
|
||||
}
|
||||
|
||||
async fn client_secret(client_id: &str) -> Result<String> {
|
||||
use swarm_secret_client::builder::{ClientSecret, client_secret_path};
|
||||
|
||||
let path = client_secret_path(client_id)?;
|
||||
let store = swarm_secret_client::SecretStore::from_env(CERT_ROLE)
|
||||
.await
|
||||
.context("logging in to the secret store")?;
|
||||
let stored: Option<ClientSecret> = store
|
||||
.read_optional(&path)
|
||||
.await
|
||||
.with_context(|| format!("reading {path}"))?;
|
||||
match stored {
|
||||
Some(s) if !s.value.trim().is_empty() => Ok(s.value.trim().to_owned()),
|
||||
_ => bail!(
|
||||
"nothing at {path}: swarm-secret-publish, on the host that runs authelia, has not \
|
||||
published this service's client secret"
|
||||
),
|
||||
}
|
||||
}
|
||||
334
swarm-builder/src/nix_build.rs
Normal file
334
swarm-builder/src/nix_build.rs
Normal file
|
|
@ -0,0 +1,334 @@
|
|||
//! One build: `nix build` of the check flake with its three inputs pointed at
|
||||
//! this host's hyperhive and nixpkgs and at the agent's config commit.
|
||||
//!
|
||||
//! The config repo is private, so `git` fetches it with the builder's forge
|
||||
//! token. The token reaches `git` through a credential helper that reads it
|
||||
//! from the child's environment: it is in no argument, no URL and no lock, so
|
||||
//! no error line `nix` prints can carry it into the reply.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::process::Stdio;
|
||||
use std::time::Duration;
|
||||
|
||||
use swarm_queue_client::builder::{BuildAgent, Reply};
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
/// The most of `nix`'s stderr a failed build's reply carries, from its end.
|
||||
const TAIL_LINES: usize = 30;
|
||||
const TAIL_BYTES: usize = 4000;
|
||||
|
||||
/// The variable the credential helper reads the token from, in the child only.
|
||||
const TOKEN_VAR: &str = "SWARM_BUILDER_GIT_TOKEN";
|
||||
|
||||
/// What every build is run with, fixed at start.
|
||||
pub struct Builder {
|
||||
/// The `nix` binary.
|
||||
pub nix: PathBuf,
|
||||
/// The check flake's directory in the store.
|
||||
pub check_flake: String,
|
||||
/// The hyperhive source this host was deployed from.
|
||||
pub hyperhive: String,
|
||||
/// The nixpkgs source this host was deployed from.
|
||||
pub nixpkgs: String,
|
||||
/// `https://<forge>`, with no trailing slash.
|
||||
pub forge_url: String,
|
||||
/// The org the config repos are in.
|
||||
pub config_org: String,
|
||||
/// How long one build may run.
|
||||
pub deadline: Duration,
|
||||
}
|
||||
|
||||
impl Builder {
|
||||
/// Build `job` with `token` as the forge credential.
|
||||
pub async fn build(&self, job: &BuildAgent, token: &str) -> Reply {
|
||||
if let Err(error) = validate(job) {
|
||||
return Reply::Failed { error_tail: error };
|
||||
}
|
||||
let mut cmd = tokio::process::Command::new(&self.nix);
|
||||
cmd.args(self.args(job))
|
||||
.env(TOKEN_VAR, token)
|
||||
.env("GIT_TERMINAL_PROMPT", "0")
|
||||
.env("GIT_CONFIG_COUNT", "1")
|
||||
.env("GIT_CONFIG_KEY_0", "credential.helper")
|
||||
.env("GIT_CONFIG_VALUE_0", credential_helper())
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
run(cmd, self.deadline).await
|
||||
}
|
||||
|
||||
/// The `nix build` arguments for `job`.
|
||||
fn args(&self, job: &BuildAgent) -> Vec<String> {
|
||||
[
|
||||
"build",
|
||||
"--no-link",
|
||||
"--print-out-paths",
|
||||
"--no-write-lock-file",
|
||||
"--option",
|
||||
"accept-flake-config",
|
||||
"false",
|
||||
"--print-build-logs",
|
||||
]
|
||||
.into_iter()
|
||||
.map(str::to_owned)
|
||||
.chain([
|
||||
format!("{}#check", self.check_flake),
|
||||
"--override-input".to_owned(),
|
||||
"hyperhive".to_owned(),
|
||||
self.hyperhive.clone(),
|
||||
"--override-input".to_owned(),
|
||||
"nixpkgs".to_owned(),
|
||||
self.nixpkgs.clone(),
|
||||
"--override-input".to_owned(),
|
||||
"agent".to_owned(),
|
||||
self.agent_url(job),
|
||||
])
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The config commit as a flake reference, fetched through the PR's head
|
||||
/// ref.
|
||||
fn agent_url(&self, job: &BuildAgent) -> String {
|
||||
format!(
|
||||
"git+{}/{}/{}?ref=refs/pull/{}/head&rev={}",
|
||||
self.forge_url, self.config_org, job.agent, job.pr, job.rev
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// A `git` credential helper answering every `get` with the builder's login
|
||||
/// and the token from [`TOKEN_VAR`].
|
||||
fn credential_helper() -> String {
|
||||
format!(
|
||||
"!f() {{ test \"$1\" = get && printf 'username=builder\\npassword=%s\\n' \"${TOKEN_VAR}\"; }}; f"
|
||||
)
|
||||
}
|
||||
|
||||
/// Refuse a job whose fields would change the reference's meaning: an agent
|
||||
/// is one `[a-z0-9-]` segment, and a rev is a full commit id.
|
||||
fn validate(job: &BuildAgent) -> Result<(), String> {
|
||||
let agent_ok = !job.agent.is_empty()
|
||||
&& job
|
||||
.agent
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-');
|
||||
if !agent_ok {
|
||||
return Err(format!("{:?} is not an agent name", job.agent));
|
||||
}
|
||||
let rev_ok = job.rev.len() == 40
|
||||
&& job
|
||||
.rev
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b));
|
||||
if !rev_ok {
|
||||
return Err(format!("{:?} is not a full commit id", job.rev));
|
||||
}
|
||||
if job.pr == 0 {
|
||||
return Err("PR 0 does not exist".to_owned());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run `cmd` to its end or `deadline`, and read the store path it printed or
|
||||
/// the tail of what it said.
|
||||
async fn run(mut cmd: tokio::process::Command, deadline: Duration) -> Reply {
|
||||
let mut child = match cmd.spawn() {
|
||||
Ok(child) => child,
|
||||
Err(e) => {
|
||||
return Reply::Failed {
|
||||
error_tail: format!("starting nix: {e}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
let mut stdout = child.stdout.take().expect("piped");
|
||||
let mut stderr = child.stderr.take().expect("piped");
|
||||
let work = async {
|
||||
let (mut out, mut err) = (Vec::new(), Vec::new());
|
||||
let (o, e, status) = tokio::join!(
|
||||
stdout.read_to_end(&mut out),
|
||||
stderr.read_to_end(&mut err),
|
||||
child.wait()
|
||||
);
|
||||
o.and(e).map(|_| ())?;
|
||||
status.map(|s| (s, out, err))
|
||||
};
|
||||
match tokio::time::timeout(deadline, work).await {
|
||||
Err(_) => Reply::Failed {
|
||||
error_tail: format!(
|
||||
"the build did not finish within {} minutes",
|
||||
deadline.as_secs() / 60
|
||||
),
|
||||
},
|
||||
Ok(Err(e)) => Reply::Failed {
|
||||
error_tail: format!("waiting for nix: {e}"),
|
||||
},
|
||||
Ok(Ok((status, out, err))) => {
|
||||
let out_path = String::from_utf8_lossy(&out)
|
||||
.lines()
|
||||
.rfind(|l| !l.trim().is_empty())
|
||||
.map(|l| l.trim().to_owned());
|
||||
match out_path {
|
||||
Some(out_path) if status.success() => Reply::Built { out_path },
|
||||
_ => Reply::Failed {
|
||||
error_tail: tail(&String::from_utf8_lossy(&err), status),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The last [`TAIL_LINES`] lines of `stderr`, at most [`TAIL_BYTES`] bytes,
|
||||
/// or the exit status when it said nothing.
|
||||
fn tail(stderr: &str, status: std::process::ExitStatus) -> String {
|
||||
let lines: Vec<&str> = stderr.lines().filter(|l| !l.trim().is_empty()).collect();
|
||||
if lines.is_empty() {
|
||||
return format!("nix exited with {status} and printed nothing");
|
||||
}
|
||||
let joined = lines[lines.len().saturating_sub(TAIL_LINES)..].join("\n");
|
||||
let mut start = joined.len().saturating_sub(TAIL_BYTES);
|
||||
while !joined.is_char_boundary(start) {
|
||||
start += 1;
|
||||
}
|
||||
joined[start..].to_owned()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use swarm_queue_client::builder::BUILD_DEADLINE;
|
||||
|
||||
fn job() -> BuildAgent {
|
||||
BuildAgent {
|
||||
agent: "atlas".to_owned(),
|
||||
rev: "0123456789abcdef0123456789abcdef01234567".to_owned(),
|
||||
pr: 7,
|
||||
}
|
||||
}
|
||||
|
||||
fn builder() -> Builder {
|
||||
Builder {
|
||||
check_flake: "/nix/store/c-check".to_owned(),
|
||||
hyperhive: "/nix/store/h-source".to_owned(),
|
||||
nixpkgs: "/nix/store/n-source".to_owned(),
|
||||
forge_url: "https://forge.example".to_owned(),
|
||||
config_org: "agent-configs".to_owned(),
|
||||
nix: PathBuf::from("nix"),
|
||||
deadline: BUILD_DEADLINE,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_agent_is_fetched_at_its_commit_through_the_prs_ref() {
|
||||
assert_eq!(
|
||||
builder().agent_url(&job()),
|
||||
"git+https://forge.example/agent-configs/atlas?ref=refs/pull/7/head\
|
||||
&rev=0123456789abcdef0123456789abcdef01234567"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_input_is_overridden_and_no_lock_is_written() {
|
||||
let args = builder().args(&job());
|
||||
let overridden: Vec<&str> = args
|
||||
.windows(2)
|
||||
.filter(|w| w[0] == "--override-input")
|
||||
.map(|w| w[1].as_str())
|
||||
.collect();
|
||||
assert_eq!(overridden, ["hyperhive", "nixpkgs", "agent"]);
|
||||
assert!(args.iter().any(|a| a == "/nix/store/c-check#check"));
|
||||
assert!(args.iter().any(|a| a == "--no-write-lock-file"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_token_is_in_no_argument() {
|
||||
let args = builder().args(&job()).join(" ");
|
||||
assert!(!args.contains("password"));
|
||||
assert!(credential_helper().contains(&format!("${TOKEN_VAR}")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_job_that_would_bend_the_reference_is_refused() {
|
||||
for bad in [
|
||||
BuildAgent {
|
||||
agent: "a?rev=x".to_owned(),
|
||||
..job()
|
||||
},
|
||||
BuildAgent {
|
||||
agent: String::new(),
|
||||
..job()
|
||||
},
|
||||
BuildAgent {
|
||||
rev: "main".to_owned(),
|
||||
..job()
|
||||
},
|
||||
BuildAgent { pr: 0, ..job() },
|
||||
] {
|
||||
assert!(validate(&bad).is_err(), "{bad:?}");
|
||||
}
|
||||
assert!(validate(&job()).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_tail_keeps_the_end_of_a_long_log() {
|
||||
use std::os::unix::process::ExitStatusExt as _;
|
||||
|
||||
let log = (0..100)
|
||||
.map(|i| format!("line {i}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
let t = tail(&log, std::process::ExitStatus::from_raw(256));
|
||||
assert!(t.ends_with("line 99"));
|
||||
assert!(!t.contains("line 69\n"));
|
||||
assert!(t.starts_with("line 70"));
|
||||
let quiet = tail("", std::process::ExitStatus::from_raw(256));
|
||||
assert!(quiet.contains("printed nothing"));
|
||||
}
|
||||
|
||||
fn sh(script: &str, deadline: Duration) -> impl Future<Output = Reply> {
|
||||
let mut cmd = tokio::process::Command::new("sh");
|
||||
cmd.args(["-c", script])
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
run(cmd, deadline)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_success_replies_with_the_last_path_printed() {
|
||||
let reply = sh(
|
||||
"echo warming >&2; echo /nix/store/abc-nixos-system",
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
reply,
|
||||
Reply::Built {
|
||||
out_path: "/nix/store/abc-nixos-system".to_owned()
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failure_replies_with_what_nix_said_last() {
|
||||
let reply = sh(
|
||||
"echo first >&2; echo 'error: attribute missing' >&2; exit 1",
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await;
|
||||
let Reply::Failed { error_tail } = reply else {
|
||||
panic!("{reply:?}")
|
||||
};
|
||||
assert!(error_tail.ends_with("error: attribute missing"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_build_past_its_deadline_is_stopped() {
|
||||
let reply = sh("sleep 30", Duration::from_millis(200)).await;
|
||||
assert!(
|
||||
matches!(&reply, Reply::Failed { error_tail } if error_tail.contains("did not finish")),
|
||||
"{reply:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue