download an agent's cached closure in prebuild, building locally when that fails

hive-c0re/src/job_queue/exec/tests.rs

@@ -11,12 +11,12 @@ use hive_jobq::NodeId;
 use hive_jobq::scheduler::Scheduler;
 
 use super::{
-    Host, dispatch, matrix_sweep_warning, run_create, run_destroy_bookkeeping,
+    Host, dispatch, drift, matrix_sweep_warning, run_create, run_destroy_bookkeeping,
     run_destroy_container, run_drain, run_emit_rebuilt, run_forge_sweep, run_knowledge_pull,
     run_matrix_sweep, run_meta_sync, run_pause_drain, run_pause_signal, run_prebuild,
     run_provision, run_purge_state, run_rebuild_bookkeeping, run_reconcile, run_set_wanted,
     run_signal, run_start, run_stop, run_stop_for_update, run_swap, run_wanted_pull,
-    run_write_dropin,
+    run_write_dropin, substitute,
 };
 use crate::coordinator::{AgentPaths, Coordinator, HiveEnv};
 use crate::job_queue::model::NodeKind;
@@ -24,14 +24,27 @@ use crate::meta_rev::MetaBuild;
 use crate::power::Wanted;
 
 const REV: &str = "2222222222222222222222222222222222222222";
+const CLOSURE: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-alice";
 
 /// A [`Host`] that records every call as one string and fails on request.
 #[derive(Default)]
 struct FakeHost {
     running: bool,
     rev: Option<String>,
-    /// Names of the calls that fail (`kill`, `swap_update`, ...). `recorded_build` fails by
-    /// finding nothing and `matrix_ensure_all` by reporting false.
+    /// The store path the swarm's record names, when it has a pushed build of
+    /// `REV`; `None` reads as no record.
+    closure: Option<String>,
+    /// A failing `realise` says nix refused the path for want of a signature.
+    refusal: bool,
+    /// The toplevel this hive's evaluation produces; `None` is `CLOSURE`.
+    evaluates: Option<String>,
+    /// Store paths a download put in the store.
+    realised: Mutex<Vec<String>>,
+    /// Store paths a prebuild had to build because the store lacked them.
+    builds: Mutex<Vec<String>>,
+    /// Names of the calls that fail (`kill`, `realise`, `prebuild`, `swap_update`,
+    /// `set_paused`, ...). `recorded_build` fails by finding nothing and
+    /// `matrix_ensure_all` by reporting false.
     failing: Vec<&'static str>,
     /// The sweeps and pulls never finish, so only a deadline ends them.
     hang: bool,
@@ -57,6 +70,11 @@ impl FakeHost {
         }
     }
 
+    fn with_closure(mut self) -> Self {
+        self.closure = Some(CLOSURE.to_owned());
+        self
+    }
+
     fn failing(mut self, call: &'static str) -> Self {
         self.failing.push(call);
         self
@@ -82,6 +100,10 @@ impl FakeHost {
     fn calls(&self) -> Vec<String> {
         self.calls.lock().unwrap().clone()
     }
+
+    fn builds(&self) -> Vec<String> {
+        self.builds.lock().unwrap().clone()
+    }
 }
 
 impl Host for FakeHost {
@@ -105,12 +127,43 @@ impl Host for FakeHost {
             .ok_or_else(|| anyhow::anyhow!("{name}: no internal/meta commit was recorded"))
     }
 
-    async fn prebuild_toplevel(&self, name: &str, rev: &str, node_id: u64) -> anyhow::Result<()> {
+    async fn cached_closure(&self, name: &str, rev: &str) -> anyhow::Result<String> {
+        self.record(format!("cached_closure {name} {rev}"));
+        self.closure
+            .clone()
+            .ok_or_else(|| anyhow::anyhow!("no-record: no build is recorded for the agent"))
+    }
+
+    async fn realise_closure(&self, name: &str, store_path: &str) -> anyhow::Result<()> {
+        self.record(format!("realise {name} {store_path}"));
+        if self.fails("realise") {
+            if self.refusal {
+                anyhow::bail!(
+                    "not available: {}: lacks a signature",
+                    hive_priv_sock::SIGNATURE_REFUSED
+                );
+            }
+            anyhow::bail!("not available: Couldn't connect to server");
+        }
+        self.realised.lock().unwrap().push(store_path.to_owned());
+        Ok(())
+    }
+
+    async fn prebuild_toplevel(
+        &self,
+        name: &str,
+        rev: &str,
+        node_id: u64,
+    ) -> anyhow::Result<Option<String>> {
         self.record(format!("prebuild {name} {rev} node={node_id}"));
         if self.fails("prebuild") {
             anyhow::bail!("prebuild failed");
         }
-        Ok(())
+        let path = self.evaluates.clone().unwrap_or_else(|| CLOSURE.to_owned());
+        if !self.realised.lock().unwrap().contains(&path) {
+            self.builds.lock().unwrap().push(path.clone());
+        }
+        Ok(Some(path))
     }
 
     async fn swap_update(
@@ -290,11 +343,140 @@ async fn prebuild_warms_the_recorded_rev_when_the_container_is_up() {
         [
             "is_running alice".to_owned(),
             "recorded_rev alice".to_owned(),
+            format!("cached_closure alice {REV}"),
             format!("prebuild alice {REV} node={}", id.get()),
         ]
     );
 }
 
+/// The download is followed by the local build either way. When this hive
+/// evaluates the path the swarm built, the build finds it in the store.
+#[tokio::test]
+async fn a_downloaded_closure_the_hive_also_evaluates_is_found_not_built() {
+    let host = FakeHost::up().with_closure();
+    let id = node_id();
+    run_prebuild(&host, "alice", id).await.expect("prebuild ok");
+    assert_eq!(
+        host.calls(),
+        [
+            "is_running alice".to_owned(),
+            "recorded_rev alice".to_owned(),
+            format!("cached_closure alice {REV}"),
+            format!("realise alice {CLOSURE}"),
+            format!("prebuild alice {REV} node={}", id.get()),
+        ]
+    );
+    assert_eq!(host.builds(), Vec::<String>::new());
+}
+
+/// When it evaluates another path, the build happens in `Prebuild`, after the
+/// check that the agent is up and before `Swap` could stop it.
+#[tokio::test]
+async fn a_downloaded_closure_the_hive_does_not_evaluate_is_built_before_the_swap() {
+    let other = "/nix/store/zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz-nixos-system-alice";
+    let host = FakeHost {
+        evaluates: Some(other.to_owned()),
+        ..FakeHost::up().with_closure()
+    };
+    let id = node_id();
+    run_prebuild(&host, "alice", id).await.expect("prebuild ok");
+    let calls = host.calls();
+    assert_eq!(calls[0], "is_running alice", "{calls:?}");
+    assert_eq!(
+        calls.last().map(String::as_str),
+        Some(format!("prebuild alice {REV} node={}", id.get()).as_str()),
+        "{calls:?}"
+    );
+    assert_eq!(host.builds(), [other.to_owned()]);
+}
+
+#[test]
+fn only_a_different_path_is_drift() {
+    assert_eq!(drift(CLOSURE, Some(CLOSURE)), None);
+    let note = drift(CLOSURE, Some("/nix/store/other")).expect("drift");
+    assert!(
+        note.contains(CLOSURE) && note.contains("/nix/store/other"),
+        "{note}"
+    );
+    assert!(drift(CLOSURE, None).expect("no path").contains(CLOSURE));
+}
+
+#[tokio::test]
+async fn a_failed_download_builds_locally() {
+    let host = FakeHost::up().with_closure().failing("realise");
+    let id = node_id();
+    run_prebuild(&host, "alice", id)
+        .await
+        .expect("the build covers it");
+    let calls = host.calls();
+    assert_eq!(calls[3], format!("realise alice {CLOSURE}"), "{calls:?}");
+    assert_eq!(
+        calls[4],
+        format!("prebuild alice {REV} node={}", id.get()),
+        "{calls:?}"
+    );
+    assert_eq!(calls.len(), 5, "{calls:?}");
+}
+
+/// The fallback is the build, so a build that fails still fails the node.
+#[tokio::test]
+async fn a_failed_download_and_a_failed_build_fail_the_node() {
+    let host = FakeHost::up()
+        .with_closure()
+        .failing("realise")
+        .failing("prebuild");
+    let err = run_prebuild(&host, "alice", node_id())
+        .await
+        .expect_err("nothing produced the closure");
+    assert!(format!("{err:#}").contains("prebuild failed"), "{err:#}");
+}
+
+#[tokio::test]
+async fn a_stopped_container_looks_up_nothing() {
+    let host = FakeHost::down().with_closure();
+    run_prebuild(&host, "alice", node_id())
+        .await
+        .expect("skipped");
+    assert_eq!(host.calls(), ["is_running alice"]);
+}
+
+/// The error log is for a path the cache served unsigned, not for a cache that
+/// is down or a missing record.
+#[tokio::test]
+async fn only_a_signature_refusal_is_flagged() {
+    let down = substitute(
+        &FakeHost::up().with_closure().failing("realise"),
+        "alice",
+        REV,
+    )
+    .await
+    .expect_err("down");
+    assert!(!down.signature_refused, "{}", down.reason);
+    assert!(down.reason.contains("Couldn't connect"), "{}", down.reason);
+
+    let no_record = substitute(&FakeHost::up(), "alice", REV)
+        .await
+        .expect_err("no record");
+    assert!(!no_record.signature_refused);
+    assert!(
+        no_record.reason.starts_with("no-record"),
+        "{}",
+        no_record.reason
+    );
+
+    let refused = FakeHost {
+        refusal: true,
+        ..FakeHost::up().with_closure().failing("realise")
+    };
+    let refused = substitute(&refused, "alice", REV)
+        .await
+        .expect_err("refused");
+    assert!(refused.signature_refused, "{}", refused.reason);
+
+    let ok = substitute(&FakeHost::up().with_closure(), "alice", REV).await;
+    assert_eq!(ok.ok().as_deref(), Some(CLOSURE));
+}
+
 #[tokio::test]
 async fn prebuild_is_skipped_when_the_container_is_down() {
     let host = FakeHost::down();