lifecycle: prebuild system toplevel before stop+update so container downtime shrinks (#706)

hive-c0re/src/lifecycle.rs

@@ -425,7 +425,18 @@ pub async fn rebuild_no_meta(
         // the new profile and the new EXTRA_NSPAWN_FLAGS in one go.
         // If the container was already stopped, `update` builds + sets
         // the profile and we leave it stopped.
+        //
+        // Pre-build the system toplevel **before** stopping the
+        // container so the container is only down for the
+        // profile-swap + restart, not for the full nix evaluation +
+        // fetch + build cycle (#706). Failure here aborts before we
+        // touch the running container — the agent keeps serving its
+        // previous generation while the operator looks at the eval
+        // error. `nixos-container update` then finds the toplevel
+        // already in the store and skips straight to the profile
+        // swap.
         let was_running = is_running(name).await;
+        prebuild_toplevel(name, &flake_ref).await?;
         set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?;
         set_resource_limits(&container)?;
         systemd_daemon_reload().await?;
@@ -439,8 +450,12 @@ pub async fn rebuild_no_meta(
             Ok(())
         }
     } else {
-        // First spawn: create the container first (which writes the nspawn
-        // conf file), then overwrite with our flags and start.
+        // First spawn: pre-build for parity (`nixos-container create`
+        // builds + creates the container, so warming the store first
+        // doesn't shave downtime — there's none to shave — but it
+        // surfaces eval / fetch errors before `create` half-spawns a
+        // container record that the operator then has to clean up).
+        prebuild_toplevel(name, &flake_ref).await?;
         run(&["create", &container, "--flake", &flake_ref]).await?;
         set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?;
         set_resource_limits(&container)?;
@@ -449,6 +464,89 @@ pub async fn rebuild_no_meta(
     }
 }
 
+/// Pre-build the agent's `system.build.toplevel` derivation against
+/// `meta#<name>` so the subsequent `nixos-container update` /
+/// `create` finds the result already in the store. The container
+/// itself is untouched — this is purely a store-warming pass.
+///
+/// Streams nix's stdout to INFO and stderr to WARN like the
+/// `nixos-container` shellouts so progress shows up in journald as
+/// it happens. `--no-link` keeps us from littering the working
+/// directory with `result` symlinks. Per-derivation cost: pure
+/// cache hit when nothing changed (handful of seconds for the
+/// eval), expensive only on the rebuild that actually has work.
+///
+/// Returns the same error shape as the other nixos-container
+/// helpers so callers can use `?` without translation.
+async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> {
+    use tokio::io::{AsyncBufReadExt, BufReader};
+    let attr = format!("{flake_ref}.config.system.build.toplevel");
+    let args = vec![
+        "--extra-experimental-features",
+        "nix-command flakes",
+        "build",
+        "--no-link",
+        "--print-out-paths",
+        &attr,
+    ];
+    let cmdline = format!("nix {}", args.join(" "));
+    tracing::info!(%name, %cmdline, "prebuild: warming system toplevel");
+    let mut child = Command::new("nix")
+        .args(&args)
+        .stdout(std::process::Stdio::piped())
+        .stderr(std::process::Stdio::piped())
+        .spawn()
+        .with_context(|| format!("spawn {cmdline}"))?;
+
+    let stdout = child.stdout.take().expect("piped stdout");
+    let stderr = child.stderr.take().expect("piped stderr");
+
+    let stdout_cmdline = cmdline.clone();
+    let pump_stdout = tokio::spawn(async move {
+        let mut lines = BufReader::new(stdout).lines();
+        while let Ok(Some(line)) = lines.next_line().await {
+            tracing::info!(target: "nix-prebuild", cmdline = %stdout_cmdline, "{line}");
+        }
+    });
+
+    let stderr_cmdline = cmdline.clone();
+    let stderr_tail: std::sync::Arc<std::sync::Mutex<std::collections::VecDeque<String>>> =
+        std::sync::Arc::new(std::sync::Mutex::new(
+            std::collections::VecDeque::with_capacity(32),
+        ));
+    let stderr_tail_pump = stderr_tail.clone();
+    let pump_stderr = tokio::spawn(async move {
+        let mut lines = BufReader::new(stderr).lines();
+        while let Ok(Some(line)) = lines.next_line().await {
+            tracing::warn!(target: "nix-prebuild", cmdline = %stderr_cmdline, "{line}");
+            let mut tail = stderr_tail_pump.lock().unwrap();
+            if tail.len() == 32 {
+                tail.pop_front();
+            }
+            tail.push_back(line);
+        }
+    });
+
+    let status = child
+        .wait()
+        .await
+        .with_context(|| format!("wait {cmdline}"))?;
+    let _ = pump_stdout.await;
+    let _ = pump_stderr.await;
+
+    if !status.success() {
+        let tail = stderr_tail
+            .lock()
+            .unwrap()
+            .iter()
+            .cloned()
+            .collect::<Vec<_>>()
+            .join("\n");
+        bail!("prebuild {cmdline} failed ({status}): {tail}");
+    }
+    Ok(())
+}
+
 pub async fn list() -> Result<Vec<String>> {
     let out = Command::new("nixos-container")
         .arg("list")