gate each deploy on a swarm build of the agent from internal/meta

swarm-controller/src/meta_build.rs

@@ -0,0 +1,351 @@
+//! The `BuildMeta` node: build one agent from `internal/meta` on the swarm
+//! builder before its deploy is sent, and record the closure in
+//! [`swarm_wire::agent_closures`].
+//!
+//! The node reads `internal/meta`'s head when it runs, because in most DAGs it
+//! is declared before the `UpdateMeta` that commits that head. It sends the
+//! builder the agent and the head's commit, and on a `Built` reply writes the
+//! key itself: the builder holds no grant on the bucket. A `Built` reply and
+//! nothing else writes, so a red build leaves the previous record.
+//!
+//! The node ends one of three ways. `Done` is a green build. A red build, the
+//! builder's `Failed` reply, is `Failed` with `nix`'s tail: the deploy behind it
+//! is skipped and the Jobs page shows why. Anything that stopped the build from
+//! giving a verdict is `Failed` with [`RETRY_PREFIX`], which
+//! `jobq::queue_meta_update` retries on the meta timer.
+
+use std::fmt::Display;
+use std::future::Future;
+
+use anyhow::{Context as _, Result};
+use hive_jobq::scheduler::Outcome;
+use swarm_wire::agent_closures::AgentClosure;
+use swarm_wire::builder::{BuildFromMeta, FromMetaReply, META_SUBJECT, REPLY_DEADLINE};
+
+use crate::agent_renewal::unix_now;
+use crate::forge::{self, objects::SHARED_ORG};
+use crate::meta_flake::{META_REPO, MetaForge};
+
+/// What a `Failed` outcome that is not a verdict on the config starts with.
+pub(crate) const RETRY_PREFIX: &str = "not built, will be retried: ";
+
+/// Whether a `Failed` node's `error` says the build gave no verdict.
+pub(crate) fn is_retried(error: &str) -> bool {
+    error.starts_with(RETRY_PREFIX)
+}
+
+fn not_built(why: impl Display) -> Outcome {
+    Outcome::Failed(format!("{RETRY_PREFIX}{why}"))
+}
+
+/// Where `internal/meta`'s head is read: the forge in production, a fake in
+/// the tests.
+pub(crate) trait MetaHead {
+    /// The head commit; `None` when the repo has none.
+    fn head(&self) -> impl Future<Output = Result<Option<String>>> + Send;
+}
+
+impl MetaHead for forge::Client {
+    async fn head(&self) -> Result<Option<String>> {
+        let read = self.head_files(SHARED_ORG, META_REPO, None, &[]);
+        let head = crate::jobq::forge_node("build_meta", read).await?;
+        Ok(head.map(|h| h.commit))
+    }
+}
+
+/// Where a build is sent and its closure recorded: the queue in production, a
+/// fake in the tests.
+pub(crate) trait MetaBuilds {
+    /// `Err` is a request that never reached a builder or never came back.
+    fn build(&self, request: Vec<u8>)
+    -> impl Future<Output = Result<FromMetaReply, String>> + Send;
+    fn record(
+        &self,
+        agent: &str,
+        closure: &AgentClosure,
+    ) -> impl Future<Output = Result<()>> + Send;
+}
+
+impl MetaBuilds for async_nats::Client {
+    async fn build(&self, request: Vec<u8>) -> Result<FromMetaReply, String> {
+        use async_nats::RequestErrorKind;
+
+        let sent = async_nats::Request::new()
+            .payload(request.into())
+            .timeout(Some(REPLY_DEADLINE));
+        match self.send_request(META_SUBJECT, sent).await {
+            Ok(msg) => serde_json::from_slice(&msg.payload)
+                .map_err(|e| format!("the builder's meta build reply did not decode: {e}")),
+            Err(e) => Err(match e.kind() {
+                RequestErrorKind::NoResponders => "no swarm builder is listening for meta \
+                     builds: deploy.builder.enable is set on no host, or its builder predates \
+                     the request"
+                    .to_owned(),
+                RequestErrorKind::TimedOut => format!(
+                    "the swarm builder sent no answer within {} minutes",
+                    REPLY_DEADLINE.as_secs() / 60
+                ),
+                _ => format!("sending the meta build to the swarm builder failed: {e}"),
+            }),
+        }
+    }
+
+    async fn record(&self, agent: &str, closure: &AgentClosure) -> Result<()> {
+        let store = swarm_queue_client::agent_closures::open_or_create(self)
+            .await
+            .context("opening the agent-closures bucket")?;
+        let body = serde_json::to_vec(closure).context("encoding the closure record")?;
+        store
+            .put(agent, body.into())
+            .await
+            .map(|_| ())
+            .with_context(|| format!("writing agent-closures/{agent}"))
+    }
+}
+
+/// Build `agent` from `internal/meta`'s head and record the closure.
+pub(crate) async fn run(
+    head: Option<&impl MetaHead>,
+    queue: Option<&impl MetaBuilds>,
+    agent: String,
+) -> Outcome {
+    let Some(head) = head else {
+        return not_built(
+            "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token not in the \
+             swarm secret store), so internal/meta's head cannot be read",
+        );
+    };
+    let Some(queue) = queue else {
+        return not_built("no swarm queue is configured on this host, so no build can be sent");
+    };
+    let meta_rev = match head.head().await {
+        Ok(Some(rev)) => rev,
+        Ok(None) => return not_built("the swarm has not published internal/meta yet"),
+        Err(e) => return not_built(format!("{e:#}")),
+    };
+    let request = match serde_json::to_vec(&BuildFromMeta {
+        agent: agent.clone(),
+        meta_rev: meta_rev.clone(),
+    }) {
+        Ok(request) => request,
+        Err(e) => return not_built(format!("encoding the build request: {e}")),
+    };
+    match queue.build(request).await {
+        Err(why) => not_built(why),
+        Ok(FromMetaReply::Unavailable { reason }) => not_built(reason),
+        Ok(FromMetaReply::Failed { error_tail }) => Outcome::Failed(error_tail),
+        Ok(FromMetaReply::Built { out_path, system }) => {
+            let closure = AgentClosure {
+                meta_rev,
+                store_path: out_path.clone(),
+                system,
+                built_unix: unix_now(),
+            };
+            match queue.record(&agent, &closure).await {
+                Ok(()) => Outcome::Done,
+                Err(e) => not_built(format!("built {out_path}, but recording it failed: {e:#}")),
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Mutex;
+
+    use super::*;
+
+    const REV: &str = "0123456789abcdef0123456789abcdef01234567";
+
+    struct FakeHead(Result<Option<&'static str>, &'static str>);
+
+    impl MetaHead for FakeHead {
+        async fn head(&self) -> Result<Option<String>> {
+            match self.0 {
+                Ok(rev) => Ok(rev.map(str::to_owned)),
+                Err(e) => Err(anyhow::anyhow!(e)),
+            }
+        }
+    }
+
+    /// A queue that answers every build with `reply`, and keeps what it was
+    /// sent and what was recorded.
+    struct FakeQueue {
+        reply: Result<FromMetaReply, String>,
+        record_fails: bool,
+        sent: Mutex<Vec<BuildFromMeta>>,
+        recorded: Mutex<Vec<(String, AgentClosure)>>,
+    }
+
+    impl FakeQueue {
+        fn answering(reply: Result<FromMetaReply, String>) -> Self {
+            Self {
+                reply,
+                record_fails: false,
+                sent: Mutex::new(Vec::new()),
+                recorded: Mutex::new(Vec::new()),
+            }
+        }
+
+        fn built() -> Self {
+            Self::answering(Ok(FromMetaReply::Built {
+                out_path: "/nix/store/x-toplevel".to_owned(),
+                system: "x86_64-linux".to_owned(),
+            }))
+        }
+    }
+
+    impl MetaBuilds for FakeQueue {
+        async fn build(&self, request: Vec<u8>) -> Result<FromMetaReply, String> {
+            self.sent
+                .lock()
+                .unwrap()
+                .push(serde_json::from_slice(&request).expect("a build request"));
+            self.reply.clone()
+        }
+
+        async fn record(&self, agent: &str, closure: &AgentClosure) -> Result<()> {
+            anyhow::ensure!(!self.record_fails, "the bucket refused the write");
+            self.recorded
+                .lock()
+                .unwrap()
+                .push((agent.to_owned(), closure.clone()));
+            Ok(())
+        }
+    }
+
+    async fn run_with(head: &FakeHead, queue: &FakeQueue) -> Outcome {
+        run(Some(head), Some(queue), "atlas".to_owned()).await
+    }
+
+    #[tokio::test]
+    async fn a_green_build_of_the_head_is_recorded_for_the_agent() {
+        let queue = FakeQueue::built();
+        let outcome = run_with(&FakeHead(Ok(Some(REV))), &queue).await;
+        assert_eq!(outcome, Outcome::Done);
+        assert_eq!(
+            *queue.sent.lock().unwrap(),
+            [BuildFromMeta {
+                agent: "atlas".to_owned(),
+                meta_rev: REV.to_owned(),
+            }]
+        );
+        let recorded = queue.recorded.lock().unwrap();
+        let [(agent, closure)] = recorded.as_slice() else {
+            panic!("one record: {recorded:?}")
+        };
+        assert_eq!(agent, "atlas");
+        assert_eq!(closure.meta_rev, REV);
+        assert_eq!(closure.store_path, "/nix/store/x-toplevel");
+        assert_eq!(closure.system, "x86_64-linux");
+        assert!(closure.built_unix > 0);
+    }
+
+    /// The builder said the build is red: the node fails with `nix`'s tail,
+    /// which is not a retried failure, and nothing is recorded. The green test
+    /// above is the control that a record is written when the build is green.
+    #[tokio::test]
+    async fn a_red_build_fails_with_its_tail_and_records_nothing() {
+        let queue = FakeQueue::answering(Ok(FromMetaReply::Failed {
+            error_tail: "error: builder for foo failed".to_owned(),
+        }));
+        let outcome = run_with(&FakeHead(Ok(Some(REV))), &queue).await;
+        assert_eq!(
+            outcome,
+            Outcome::Failed("error: builder for foo failed".to_owned())
+        );
+        assert!(queue.recorded.lock().unwrap().is_empty());
+    }
+
+    #[tokio::test]
+    async fn a_builder_that_gave_no_verdict_is_a_retried_failure_and_records_nothing() {
+        for (name, reply) in [
+            (
+                "no responders",
+                Err("no swarm builder is listening for meta builds".to_owned()),
+            ),
+            (
+                "unavailable",
+                Ok(FromMetaReply::Unavailable {
+                    reason: "busy".to_owned(),
+                }),
+            ),
+        ] {
+            let queue = FakeQueue::answering(reply);
+            let outcome = run_with(&FakeHead(Ok(Some(REV))), &queue).await;
+            let Outcome::Failed(error) = outcome else {
+                panic!("{name}: not failed")
+            };
+            assert!(is_retried(&error), "{name}: {error}");
+            assert!(queue.recorded.lock().unwrap().is_empty(), "{name}");
+        }
+    }
+
+    #[tokio::test]
+    async fn a_closure_that_cannot_be_recorded_is_a_retried_failure() {
+        let queue = FakeQueue {
+            record_fails: true,
+            ..FakeQueue::built()
+        };
+        let Outcome::Failed(error) = run_with(&FakeHead(Ok(Some(REV))), &queue).await else {
+            panic!("not failed")
+        };
+        assert!(is_retried(&error), "{error}");
+        assert!(error.contains("/nix/store/x-toplevel"), "{error}");
+    }
+
+    #[tokio::test]
+    async fn without_a_head_or_a_queue_nothing_is_sent_and_the_failure_is_retried() {
+        for (name, head, with_queue) in [
+            ("no repo yet", Some(FakeHead(Ok(None))), true),
+            ("forge error", Some(FakeHead(Err("forge down"))), true),
+            ("no forge", None, true),
+            ("no queue", Some(FakeHead(Ok(Some(REV)))), false),
+        ] {
+            let queue = FakeQueue::built();
+            let outcome = run(
+                head.as_ref(),
+                with_queue.then_some(&queue),
+                "atlas".to_owned(),
+            )
+            .await;
+            let Outcome::Failed(error) = outcome else {
+                panic!("{name}: not failed")
+            };
+            assert!(is_retried(&error), "{name}: {error}");
+            assert!(queue.sent.lock().unwrap().is_empty(), "{name}");
+        }
+    }
+
+    #[test]
+    fn the_repo_path_the_builder_fetches_is_the_one_the_controller_writes() {
+        assert_eq!(
+            swarm_wire::builder::META_REPO_PATH,
+            format!("{SHARED_ORG}/{META_REPO}")
+        );
+    }
+
+    /// The bucket on a real server: a record written for an agent reads back
+    /// under its key, and a second build replaces it.
+    #[tokio::test]
+    async fn the_closure_round_trips_through_the_bucket() {
+        let nats = crate::test_nats::TestNats::start();
+        let client = nats.client().await;
+        let mut closure = AgentClosure {
+            meta_rev: REV.to_owned(),
+            store_path: "/nix/store/a".to_owned(),
+            system: "x86_64-linux".to_owned(),
+            built_unix: 1,
+        };
+        client.record("atlas", &closure).await.expect("written");
+        closure.store_path = "/nix/store/b".to_owned();
+        client.record("atlas", &closure).await.expect("replaced");
+        let store = swarm_queue_client::agent_closures::open_or_create(&client)
+            .await
+            .expect("opens");
+        let got = store.get("atlas").await.expect("reads").expect("present");
+        let got: AgentClosure = serde_json::from_slice(&got).expect("decodes");
+        assert_eq!(got, closure);
+        assert!(store.get("argus").await.expect("reads").is_none());
+    }
+}