//! The `BuildMeta` node: build one agent from `internal/meta` on the swarm //! builder before its deploy is sent, and record the closure in //! [`swarm_wire::agent_closures`]. //! //! The node reads `internal/meta`'s head when it runs, because in most DAGs it //! is declared before the `UpdateMeta` that commits that head. It sends the //! builder the agent and the head's commit, and on a `Built` reply writes the //! key itself: the builder holds no grant on the bucket. A `Built` reply and //! nothing else writes, so a red build leaves the previous record. //! //! The node ends one of three ways. `Done` is a green build. A red build, the //! builder's `Failed` reply, is `Failed` with `nix`'s tail: the deploy behind it //! is skipped and the Jobs page shows why. Anything that stopped the build from //! giving a verdict is `Failed` with [`RETRY_PREFIX`], which //! `jobq::queue_meta_update` retries on the meta timer. use std::fmt::Display; use std::future::Future; use anyhow::{Context as _, Result}; use hive_jobq::scheduler::Outcome; use swarm_wire::agent_closures::AgentClosure; use swarm_wire::builder::{BuildFromMeta, FromMetaReply, META_SUBJECT, REPLY_DEADLINE}; use crate::agent_renewal::unix_now; use crate::forge::{self, objects::SHARED_ORG}; use crate::meta_flake::{META_REPO, MetaForge}; /// What a `Failed` outcome that is not a verdict on the config starts with. pub(crate) const RETRY_PREFIX: &str = "not built, will be retried: "; /// Whether a `Failed` node's `error` says the build gave no verdict. pub(crate) fn is_retried(error: &str) -> bool { error.starts_with(RETRY_PREFIX) } fn not_built(why: impl Display) -> Outcome { Outcome::Failed(format!("{RETRY_PREFIX}{why}")) } /// Where `internal/meta`'s head is read: the forge in production, a fake in /// the tests. pub(crate) trait MetaHead { /// The head commit; `None` when the repo has none. fn head(&self) -> impl Future>> + Send; } impl MetaHead for forge::Client { async fn head(&self) -> Result> { let read = self.head_files(SHARED_ORG, META_REPO, None, &[]); let head = crate::jobq::forge_node("build_meta", read).await?; Ok(head.map(|h| h.commit)) } } /// Where a build is sent and its closure recorded: the queue in production, a /// fake in the tests. pub(crate) trait MetaBuilds { /// `Err` is a request that never reached a builder or never came back. fn build(&self, request: Vec) -> impl Future> + Send; fn record( &self, agent: &str, closure: &AgentClosure, ) -> impl Future> + Send; } impl MetaBuilds for async_nats::Client { async fn build(&self, request: Vec) -> Result { use async_nats::RequestErrorKind; let sent = async_nats::Request::new() .payload(request.into()) .timeout(Some(REPLY_DEADLINE)); match self.send_request(META_SUBJECT, sent).await { Ok(msg) => serde_json::from_slice(&msg.payload) .map_err(|e| format!("the builder's meta build reply did not decode: {e}")), Err(e) => Err(match e.kind() { RequestErrorKind::NoResponders => "no swarm builder is listening for meta \ builds: deploy.builder.enable is set on no host, or its builder predates \ the request" .to_owned(), RequestErrorKind::TimedOut => format!( "the swarm builder sent no answer within {} minutes", REPLY_DEADLINE.as_secs() / 60 ), _ => format!("sending the meta build to the swarm builder failed: {e}"), }), } } async fn record(&self, agent: &str, closure: &AgentClosure) -> Result<()> { let store = swarm_queue_client::agent_closures::open_or_create(self) .await .context("opening the agent-closures bucket")?; let body = serde_json::to_vec(closure).context("encoding the closure record")?; store .put(agent, body.into()) .await .map(|_| ()) .with_context(|| format!("writing agent-closures/{agent}")) } } /// Build `agent` from `internal/meta`'s head and record the closure. pub(crate) async fn run( head: Option<&impl MetaHead>, queue: Option<&impl MetaBuilds>, agent: String, ) -> Outcome { let Some(head) = head else { return not_built( "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token not in the \ swarm secret store), so internal/meta's head cannot be read", ); }; let Some(queue) = queue else { return not_built("no swarm queue is configured on this host, so no build can be sent"); }; let meta_rev = match head.head().await { Ok(Some(rev)) => rev, Ok(None) => return not_built("the swarm has not published internal/meta yet"), Err(e) => return not_built(format!("{e:#}")), }; let request = match serde_json::to_vec(&BuildFromMeta { agent: agent.clone(), meta_rev: meta_rev.clone(), }) { Ok(request) => request, Err(e) => return not_built(format!("encoding the build request: {e}")), }; match queue.build(request).await { Err(why) => not_built(why), Ok(FromMetaReply::Unavailable { reason }) => not_built(reason), Ok(FromMetaReply::Failed { error_tail }) => Outcome::Failed(error_tail), Ok(FromMetaReply::Built { out_path, system }) => { let closure = AgentClosure { meta_rev, store_path: out_path.clone(), system, built_unix: unix_now(), }; match queue.record(&agent, &closure).await { Ok(()) => Outcome::Done, Err(e) => not_built(format!("built {out_path}, but recording it failed: {e:#}")), } } } } #[cfg(test)] mod tests { use std::sync::Mutex; use super::*; const REV: &str = "0123456789abcdef0123456789abcdef01234567"; struct FakeHead(Result, &'static str>); impl MetaHead for FakeHead { async fn head(&self) -> Result> { match self.0 { Ok(rev) => Ok(rev.map(str::to_owned)), Err(e) => Err(anyhow::anyhow!(e)), } } } /// A queue that answers every build with `reply`, and keeps what it was /// sent and what was recorded. struct FakeQueue { reply: Result, record_fails: bool, sent: Mutex>, recorded: Mutex>, } impl FakeQueue { fn answering(reply: Result) -> Self { Self { reply, record_fails: false, sent: Mutex::new(Vec::new()), recorded: Mutex::new(Vec::new()), } } fn built() -> Self { Self::answering(Ok(FromMetaReply::Built { out_path: "/nix/store/x-toplevel".to_owned(), system: "x86_64-linux".to_owned(), })) } } impl MetaBuilds for FakeQueue { async fn build(&self, request: Vec) -> Result { self.sent .lock() .unwrap() .push(serde_json::from_slice(&request).expect("a build request")); self.reply.clone() } async fn record(&self, agent: &str, closure: &AgentClosure) -> Result<()> { anyhow::ensure!(!self.record_fails, "the bucket refused the write"); self.recorded .lock() .unwrap() .push((agent.to_owned(), closure.clone())); Ok(()) } } async fn run_with(head: &FakeHead, queue: &FakeQueue) -> Outcome { run(Some(head), Some(queue), "atlas".to_owned()).await } #[tokio::test] async fn a_green_build_of_the_head_is_recorded_for_the_agent() { let queue = FakeQueue::built(); let outcome = run_with(&FakeHead(Ok(Some(REV))), &queue).await; assert_eq!(outcome, Outcome::Done); assert_eq!( *queue.sent.lock().unwrap(), [BuildFromMeta { agent: "atlas".to_owned(), meta_rev: REV.to_owned(), }] ); let recorded = queue.recorded.lock().unwrap(); let [(agent, closure)] = recorded.as_slice() else { panic!("one record: {recorded:?}") }; assert_eq!(agent, "atlas"); assert_eq!(closure.meta_rev, REV); assert_eq!(closure.store_path, "/nix/store/x-toplevel"); assert_eq!(closure.system, "x86_64-linux"); assert!(closure.built_unix > 0); } /// The builder said the build is red: the node fails with `nix`'s tail, /// which is not a retried failure, and nothing is recorded. The green test /// above is the control that a record is written when the build is green. #[tokio::test] async fn a_red_build_fails_with_its_tail_and_records_nothing() { let queue = FakeQueue::answering(Ok(FromMetaReply::Failed { error_tail: "error: builder for foo failed".to_owned(), })); let outcome = run_with(&FakeHead(Ok(Some(REV))), &queue).await; assert_eq!( outcome, Outcome::Failed("error: builder for foo failed".to_owned()) ); assert!(queue.recorded.lock().unwrap().is_empty()); } #[tokio::test] async fn a_builder_that_gave_no_verdict_is_a_retried_failure_and_records_nothing() { for (name, reply) in [ ( "no responders", Err("no swarm builder is listening for meta builds".to_owned()), ), ( "unavailable", Ok(FromMetaReply::Unavailable { reason: "busy".to_owned(), }), ), ] { let queue = FakeQueue::answering(reply); let outcome = run_with(&FakeHead(Ok(Some(REV))), &queue).await; let Outcome::Failed(error) = outcome else { panic!("{name}: not failed") }; assert!(is_retried(&error), "{name}: {error}"); assert!(queue.recorded.lock().unwrap().is_empty(), "{name}"); } } #[tokio::test] async fn a_closure_that_cannot_be_recorded_is_a_retried_failure() { let queue = FakeQueue { record_fails: true, ..FakeQueue::built() }; let Outcome::Failed(error) = run_with(&FakeHead(Ok(Some(REV))), &queue).await else { panic!("not failed") }; assert!(is_retried(&error), "{error}"); assert!(error.contains("/nix/store/x-toplevel"), "{error}"); } #[tokio::test] async fn without_a_head_or_a_queue_nothing_is_sent_and_the_failure_is_retried() { for (name, head, with_queue) in [ ("no repo yet", Some(FakeHead(Ok(None))), true), ("forge error", Some(FakeHead(Err("forge down"))), true), ("no forge", None, true), ("no queue", Some(FakeHead(Ok(Some(REV)))), false), ] { let queue = FakeQueue::built(); let outcome = run( head.as_ref(), with_queue.then_some(&queue), "atlas".to_owned(), ) .await; let Outcome::Failed(error) = outcome else { panic!("{name}: not failed") }; assert!(is_retried(&error), "{name}: {error}"); assert!(queue.sent.lock().unwrap().is_empty(), "{name}"); } } #[test] fn the_repo_path_the_builder_fetches_is_the_one_the_controller_writes() { assert_eq!( swarm_wire::builder::META_REPO_PATH, format!("{SHARED_ORG}/{META_REPO}") ); } /// The bucket on a real server: a record written for an agent reads back /// under its key, and a second build replaces it. #[tokio::test] async fn the_closure_round_trips_through_the_bucket() { let nats = crate::test_nats::TestNats::start(); let client = nats.client().await; let mut closure = AgentClosure { meta_rev: REV.to_owned(), store_path: "/nix/store/a".to_owned(), system: "x86_64-linux".to_owned(), built_unix: 1, }; client.record("atlas", &closure).await.expect("written"); closure.store_path = "/nix/store/b".to_owned(); client.record("atlas", &closure).await.expect("replaced"); let store = swarm_queue_client::agent_closures::open_or_create(&client) .await .expect("opens"); let got = store.get("atlas").await.expect("reads").expect("present"); let got: AgentClosure = serde_json::from_slice(&got).expect("decodes"); assert_eq!(got, closure); assert!(store.get("argus").await.expect("reads").is_none()); } }