download an agent's cached closure in prebuild, building locally when that fails

hive-c0re/src/job_queue/exec.rs

@@ -44,8 +44,21 @@ pub(super) trait Host: Sync {
     async fn kill(&self, name: &str) -> Result<()>;
     /// The `internal/meta` commit `MetaSync` recorded for the agent.
     fn recorded_rev(&self, name: &str) -> Result<String>;
+    /// The store path of the closure the swarm built for the agent at the
+    /// `internal/meta` commit `rev` and pushed to its cache. The error says why
+    /// there is none.
+    async fn cached_closure(&self, name: &str, rev: &str) -> Result<String>;
+    /// Download `store_path` from the swarm's cache into the store, never
+    /// building it.
+    async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()>;
     /// Warm the agent's toplevel build from the `internal/meta` commit `rev`.
-    async fn prebuild_toplevel(&self, name: &str, rev: &str, node_id: u64) -> Result<()>;
+    /// The toplevel it printed, when it printed one.
+    async fn prebuild_toplevel(
+        &self,
+        name: &str,
+        rev: &str,
+        node_id: u64,
+    ) -> Result<Option<String>>;
     /// Re-apply drop-ins, then `nixos-container update` at `rev`.
     async fn swap_update(
         &self,
@@ -117,7 +130,20 @@ impl Host for RealHost {
         Ok(crate::meta_rev::recorded_or_err(name)?.rev)
     }
 
-    async fn prebuild_toplevel(&self, name: &str, rev: &str, node_id: u64) -> Result<()> {
+    async fn cached_closure(&self, name: &str, rev: &str) -> Result<String> {
+        crate::attic::closure::path_for(name, rev).await
+    }
+
+    async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()> {
+        crate::priv_client::realise_closure(name, store_path).await
+    }
+
+    async fn prebuild_toplevel(
+        &self,
+        name: &str,
+        rev: &str,
+        node_id: u64,
+    ) -> Result<Option<String>> {
         let flake_ref = crate::meta_rev::flake_ref(name, rev);
         crate::lifecycle::prebuild_toplevel(name, &flake_ref, Some(node_id)).await
     }
@@ -595,6 +621,12 @@ async fn run_meta_sync(host: &impl Host, name: &str) -> Result<()> {
 /// container is already down: its only purpose is to shrink the swap's
 /// downtime window, so a stopped agent (no uptime to preserve) doesn't
 /// pay the double eval — `Swap` builds inline instead.
+///
+/// The swarm's cache is tried first: when the swarm built this agent at the
+/// recorded commit and pushed the closure, it is downloaded. The local build
+/// runs either way. After a download it is an evaluation and a no-op build when
+/// this hive evaluates the path the swarm built, and builds it, with the agent
+/// still up, when it doesn't; `Swap` never meets a build it didn't expect.
 async fn run_prebuild(host: &impl Host, name: &str, id: NodeId) -> Result<()> {
     // Warm the toplevel build only when the container is up — the whole
     // point of prebuild is to shrink the swap's downtime window. A
@@ -602,11 +634,70 @@ async fn run_prebuild(host: &impl Host, name: &str, id: NodeId) -> Result<()> {
     // eval and let the downstream `Swap` build inline.
     if host.is_running(name).await {
         let rev = host.recorded_rev(name)?;
-        host.prebuild_toplevel(name, &rev, id.get()).await?;
+        let downloaded = match substitute(host, name, &rev).await {
+            Ok(path) => Some(path),
+            Err(fallback) => {
+                let reason = &fallback.reason;
+                if fallback.signature_refused {
+                    tracing::error!(%name, %rev, "prebuild: built locally: {reason}");
+                } else {
+                    tracing::warn!(%name, %rev, "prebuild: built locally: {reason}");
+                }
+                None
+            }
+        };
+        let evaluated = host.prebuild_toplevel(name, &rev, id.get()).await?;
+        if let Some(path) = downloaded {
+            match drift(&path, evaluated.as_deref()) {
+                None => tracing::info!(%name, %rev, "prebuild: substituted {path}"),
+                Some(note) => tracing::warn!(%name, %rev, "prebuild: {note}"),
+            }
+        }
     }
     Ok(())
 }
 
+/// What to log when the closure downloaded for a rebuild isn't the one this
+/// hive's evaluation produced, which means the builder and the hive disagree
+/// about the agent's flake. `None` when they agree.
+fn drift(downloaded: &str, evaluated: Option<&str>) -> Option<String> {
+    match evaluated {
+        Some(path) if path == downloaded => None,
+        Some(path) => Some(format!(
+            "the downloaded closure {downloaded} is not the one this hive evaluates, {path}; built the latter locally"
+        )),
+        None => Some(format!(
+            "downloaded {downloaded}, but the local build printed no path to compare it with"
+        )),
+    }
+}
+
+/// Why a rebuild builds instead of downloading the swarm's closure.
+struct LocalBuild {
+    reason: String,
+    /// The cache served a path no trusted key signed.
+    signature_refused: bool,
+}
+
+/// Download the closure the swarm built for `name` at `rev`; its store path.
+async fn substitute(host: &impl Host, name: &str, rev: &str) -> Result<String, LocalBuild> {
+    let path = host
+        .cached_closure(name, rev)
+        .await
+        .map_err(|e| LocalBuild {
+            reason: format!("{e:#}"),
+            signature_refused: false,
+        })?;
+    host.realise_closure(name, &path).await.map_err(|e| {
+        let reason = format!("{e:#}");
+        LocalBuild {
+            signature_refused: reason.contains(hive_priv_sock::SIGNATURE_REFUSED),
+            reason,
+        }
+    })?;
+    Ok(path)
+}
+
 /// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb),
 /// `nixos-container update`, then the post-rebuild bookkeeping tail
 /// (rev marker, `Rebuilt` event, forge/matrix sync, kick, rescan).