hive-c0re pulls closures through the nix daemon; hive-priv verifies the swarm signature before applying

hive-c0re/src/priv_client.rs

@@ -267,9 +267,15 @@ pub async fn kill_container(name: &str) -> Result<()> {
 pub async fn update_container_streaming(
     name: &str,
     toplevel: &str,
-    on_line: impl FnMut(PrivStream, &str),
+    mut on_line: impl FnMut(PrivStream, &str),
 ) -> Result<()> {
-    ok(call_streaming(&update_container_request(name, toplevel), on_line).await?)
+    let req = update_container_request(name, toplevel);
+    let first = ok(call_streaming(&req, &mut on_line).await?);
+    if !older_hive_priv_refused(&first) {
+        return first;
+    }
+    ask_older_hive_priv_to_download(name, toplevel).await?;
+    ok(call_streaming(&req, &mut on_line).await?)
 }
 
 fn update_container_request(name: &str, toplevel: &str) -> PrivRequest {
@@ -285,9 +291,15 @@ fn update_container_request(name: &str, toplevel: &str) -> PrivRequest {
 pub async fn create_container_streaming(
     name: &str,
     toplevel: &str,
-    on_line: impl FnMut(PrivStream, &str),
+    mut on_line: impl FnMut(PrivStream, &str),
 ) -> Result<()> {
-    ok(call_streaming(&create_container_request(name, toplevel), on_line).await?)
+    let req = create_container_request(name, toplevel);
+    let first = ok(call_streaming(&req, &mut on_line).await?);
+    if !older_hive_priv_refused(&first) {
+        return first;
+    }
+    ask_older_hive_priv_to_download(name, toplevel).await?;
+    ok(call_streaming(&req, &mut on_line).await?)
 }
 
 fn create_container_request(name: &str, toplevel: &str) -> PrivRequest {
@@ -298,17 +310,49 @@ fn create_container_request(name: &str, toplevel: &str) -> PrivRequest {
     }
 }
 
-/// See [`PrivRequest::RealiseClosure`]. A path hive-priv could not download
-/// reads as an error whose message starts `not available: `.
-///
-/// # Errors
-/// The call failed, or hive-priv answered `not available`.
-pub async fn realise_closure(agent: &str, store_path: &str) -> Result<()> {
+/// What an older hive-priv, which applies only a path it downloaded itself,
+/// says to an apply of any other.
+const OLDER_PIN_REFUSAL: &str = "is not the closure realised for this agent";
+
+/// Whether `applied` is an older hive-priv refusing an apply because it did not
+/// download the path itself (hive-c0re pulled it through the nix daemon). A
+/// hive-priv of the same release never refuses that way, so a retry through
+/// [`ask_older_hive_priv_to_download`] happens only while the two restart out of
+/// step.
+fn older_hive_priv_refused(applied: &Result<()>) -> bool {
+    applied
+        .as_ref()
+        .is_err_and(|e| format!("{e:#}").contains(OLDER_PIN_REFUSAL))
+}
+
+/// Ask an older hive-priv to download `toplevel`, as it did before, which also
+/// pins the path for the apply that follows. The path is already in the store,
+/// so this is a lookup.
+async fn ask_older_hive_priv_to_download(name: &str, toplevel: &str) -> Result<()> {
+    tracing::warn!(%name, %toplevel, "an older hive-priv refused a closure it did not download; asking it to");
     ok(call(&PrivRequest::RealiseClosure {
-        agent: agent.to_owned(),
-        store_path: store_path.to_owned(),
+        agent: name.to_owned(),
+        store_path: toplevel.to_owned(),
     })
     .await?)
+    .context("asking the older hive-priv to download the closure")
+}
+
+/// How long hive-c0re waits for hive-priv's facts refresh: hive-priv bounds
+/// the unit at 60 s.
+const REFRESH_WAIT: std::time::Duration = std::time::Duration::from_secs(90);
+
+/// Run the root unit that reads this hive's cache facts from the store and
+/// writes the nix.conf fragment, and wait for it. An older hive-priv does not
+/// know the op and answers an error.
+///
+/// # Errors
+/// The call failed or did not answer in time, or the unit failed.
+pub async fn refresh_cache_facts() -> Result<()> {
+    within(REFRESH_WAIT, async {
+        ok(call(&PrivRequest::RefreshCacheFacts).await?)
+    })
+    .await
 }
 
 pub async fn destroy_container(name: &str) -> Result<()> {
@@ -684,9 +728,11 @@ mod tests {
     use super::{
         BindMount, CredentialMount, HiveFacts, NetworkIsolation, NixBuilds, PrivRequest,
         PrivResponse, PrivStream, call_at, call_streaming_at, call_with_fd_at, check,
-        create_container_request, parse_qgroup_usage, resource_limits_request,
-        send_snapshot_request, update_container_request, within, write_nspawn_flags_request,
+        create_container_request, older_hive_priv_refused, parse_qgroup_usage,
+        resource_limits_request, send_snapshot_request, update_container_request, within,
+        write_nspawn_flags_request,
     };
+    use anyhow::Context as _;
     use serde_json::{Value, json};
     use std::io::{Read as _, Write as _};
     use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd, RawFd};
@@ -697,6 +743,38 @@ mod tests {
         serde_json::to_value(req).expect("a request serialises")
     }
 
+    const REFUSAL: &str = "atlas: /nix/store/x is not the closure realised for this agent (none)";
+
+    /// An older hive-priv's refusal of a path it did not download is the
+    /// signal to ask it to download; a success and any other failure are not.
+    #[test]
+    fn only_an_older_hive_privs_pin_refusal_asks_for_a_download() {
+        assert!(older_hive_priv_refused(&Err(anyhow::anyhow!(REFUSAL))));
+        // As the streaming call words it: the error wraps hive-priv's text.
+        assert!(older_hive_priv_refused(
+            &Err(anyhow::anyhow!(REFUSAL)).context("nixos-container create")
+        ));
+        assert!(!older_hive_priv_refused(&Ok(())));
+        for other in [
+            "nixos-container create failed",
+            "atlas: refusing /nix/store/x: has no signature by the swarm cache's key",
+            "connect to hive-priv socket",
+        ] {
+            assert!(
+                !older_hive_priv_refused(&Err(anyhow::anyhow!(other))),
+                "{other}"
+            );
+        }
+    }
+
+    #[test]
+    fn the_cache_facts_refresh_names_no_unit() {
+        assert_eq!(
+            wire(&PrivRequest::RefreshCacheFacts),
+            json!({"op": "refresh_cache_facts"})
+        );
+    }
+
     #[test]
     fn create_and_update_keep_the_name_and_toplevel_apart_and_stream() {
         assert_eq!(