From f5372b6016bf273aa6945b58b7c25a97240cff51 Mon Sep 17 00:00:00 2001 From: damocles Date: Sun, 31 May 2026 11:44:53 +0200 Subject: [PATCH 1/2] lifecycle: prebuild system toplevel before stop+update so container downtime shrinks (#706) --- hive-c0re/src/lifecycle.rs | 102 ++++++++++++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 2 deletions(-) diff --git a/hive-c0re/src/lifecycle.rs b/hive-c0re/src/lifecycle.rs index e3051517..8c9040fb 100644 --- a/hive-c0re/src/lifecycle.rs +++ b/hive-c0re/src/lifecycle.rs @@ -425,7 +425,18 @@ pub async fn rebuild_no_meta( // the new profile and the new EXTRA_NSPAWN_FLAGS in one go. // If the container was already stopped, `update` builds + sets // the profile and we leave it stopped. + // + // Pre-build the system toplevel **before** stopping the + // container so the container is only down for the + // profile-swap + restart, not for the full nix evaluation + + // fetch + build cycle (#706). Failure here aborts before we + // touch the running container — the agent keeps serving its + // previous generation while the operator looks at the eval + // error. `nixos-container update` then finds the toplevel + // already in the store and skips straight to the profile + // swap. let was_running = is_running(name).await; + prebuild_toplevel(name, &flake_ref).await?; set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?; set_resource_limits(&container)?; systemd_daemon_reload().await?; @@ -439,8 +450,12 @@ pub async fn rebuild_no_meta( Ok(()) } } else { - // First spawn: create the container first (which writes the nspawn - // conf file), then overwrite with our flags and start. + // First spawn: pre-build for parity (`nixos-container create` + // builds + creates the container, so warming the store first + // doesn't shave downtime — there's none to shave — but it + // surfaces eval / fetch errors before `create` half-spawns a + // container record that the operator then has to clean up). + prebuild_toplevel(name, &flake_ref).await?; run(&["create", &container, "--flake", &flake_ref]).await?; set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?; set_resource_limits(&container)?; @@ -449,6 +464,89 @@ pub async fn rebuild_no_meta( } } +/// Pre-build the agent's `system.build.toplevel` derivation against +/// `meta#` so the subsequent `nixos-container update` / +/// `create` finds the result already in the store. The container +/// itself is untouched — this is purely a store-warming pass. +/// +/// Streams nix's stdout to INFO and stderr to WARN like the +/// `nixos-container` shellouts so progress shows up in journald as +/// it happens. `--no-link` keeps us from littering the working +/// directory with `result` symlinks. Per-derivation cost: pure +/// cache hit when nothing changed (handful of seconds for the +/// eval), expensive only on the rebuild that actually has work. +/// +/// Returns the same error shape as the other nixos-container +/// helpers so callers can use `?` without translation. +async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> { + use tokio::io::{AsyncBufReadExt, BufReader}; + let attr = format!("{flake_ref}.config.system.build.toplevel"); + let args = vec![ + "--extra-experimental-features", + "nix-command flakes", + "build", + "--no-link", + "--print-out-paths", + &attr, + ]; + let cmdline = format!("nix {}", args.join(" ")); + tracing::info!(%name, %cmdline, "prebuild: warming system toplevel"); + let mut child = Command::new("nix") + .args(&args) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .with_context(|| format!("spawn {cmdline}"))?; + + let stdout = child.stdout.take().expect("piped stdout"); + let stderr = child.stderr.take().expect("piped stderr"); + + let stdout_cmdline = cmdline.clone(); + let pump_stdout = tokio::spawn(async move { + let mut lines = BufReader::new(stdout).lines(); + while let Ok(Some(line)) = lines.next_line().await { + tracing::info!(target: "nix-prebuild", cmdline = %stdout_cmdline, "{line}"); + } + }); + + let stderr_cmdline = cmdline.clone(); + let stderr_tail: std::sync::Arc>> = + std::sync::Arc::new(std::sync::Mutex::new( + std::collections::VecDeque::with_capacity(32), + )); + let stderr_tail_pump = stderr_tail.clone(); + let pump_stderr = tokio::spawn(async move { + let mut lines = BufReader::new(stderr).lines(); + while let Ok(Some(line)) = lines.next_line().await { + tracing::warn!(target: "nix-prebuild", cmdline = %stderr_cmdline, "{line}"); + let mut tail = stderr_tail_pump.lock().unwrap(); + if tail.len() == 32 { + tail.pop_front(); + } + tail.push_back(line); + } + }); + + let status = child + .wait() + .await + .with_context(|| format!("wait {cmdline}"))?; + let _ = pump_stdout.await; + let _ = pump_stderr.await; + + if !status.success() { + let tail = stderr_tail + .lock() + .unwrap() + .iter() + .cloned() + .collect::>() + .join("\n"); + bail!("prebuild {cmdline} failed ({status}): {tail}"); + } + Ok(()) +} + pub async fn list() -> Result> { let out = Command::new("nixos-container") .arg("list") From 0a9832768500e6a4b86f90a0011e0f9ad84d63ee Mon Sep 17 00:00:00 2001 From: damocles Date: Sun, 31 May 2026 11:54:33 +0200 Subject: [PATCH 2/2] lifecycle: only prebuild when there's downtime to shave (mara on #721) --- hive-c0re/src/lifecycle.rs | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/hive-c0re/src/lifecycle.rs b/hive-c0re/src/lifecycle.rs index 8c9040fb..1f59dcc9 100644 --- a/hive-c0re/src/lifecycle.rs +++ b/hive-c0re/src/lifecycle.rs @@ -425,22 +425,25 @@ pub async fn rebuild_no_meta( // the new profile and the new EXTRA_NSPAWN_FLAGS in one go. // If the container was already stopped, `update` builds + sets // the profile and we leave it stopped. - // - // Pre-build the system toplevel **before** stopping the - // container so the container is only down for the - // profile-swap + restart, not for the full nix evaluation + - // fetch + build cycle (#706). Failure here aborts before we - // touch the running container — the agent keeps serving its - // previous generation while the operator looks at the eval - // error. `nixos-container update` then finds the toplevel - // already in the store and skips straight to the profile - // swap. let was_running = is_running(name).await; - prebuild_toplevel(name, &flake_ref).await?; set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?; set_resource_limits(&container)?; systemd_daemon_reload().await?; if was_running { + // Pre-build the system toplevel **before** stopping the + // running container so the agent keeps serving its + // previous generation while the eval + fetch + build + // happens out-of-band (#706). `nixos-container update` + // then finds the toplevel cached and skips straight to + // the profile-swap + restart — downtime collapses to that + // window only. Build failures surface here, before we + // touch the container. + // + // When the container is already stopped there's no + // downtime to shave (mara on #721#9007) — let `update` + // do the build inline rather than evaluating the flake + // twice for nothing. + prebuild_toplevel(name, &flake_ref).await?; run(&["stop", &container]).await?; } run(&["update", &container, "--flake", &flake_ref]).await?; @@ -450,12 +453,11 @@ pub async fn rebuild_no_meta( Ok(()) } } else { - // First spawn: pre-build for parity (`nixos-container create` - // builds + creates the container, so warming the store first - // doesn't shave downtime — there's none to shave — but it - // surfaces eval / fetch errors before `create` half-spawns a - // container record that the operator then has to clean up). - prebuild_toplevel(name, &flake_ref).await?; + // First spawn: no running container, no downtime to shave. + // `nixos-container create` builds + creates atomically — if + // the build fails, no container record is left around to + // clean up — so a pre-build adds nothing but a duplicate + // eval (mara on #721#9007). run(&["create", &container, "--flake", &flake_ref]).await?; set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?; set_resource_limits(&container)?;