diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index 53a4bceb..a6ac9bb7 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -623,8 +623,14 @@ async fn exec( } /// Shared body for `CreateContainer` / `UpdateContainer`: validate the -/// name, build the `nixos-container … --flake ` argv, and -/// run it (streaming line events to `writer` when `stream` is set). +/// name, build the `nixos-container …` argv, and run it (streaming +/// line events to `writer` when `stream` is set). +/// +/// `update` builds the toplevel itself first and passes `--system-path` +/// instead of `--flake` — see [`nix_build_toplevel`] for why. `create` +/// keeps the plain `--flake` form: `nixos-container`'s own `buildFlake()` +/// takes an internal `flock` for `create` (not for `update`), so the +/// hazard this works around doesn't apply there. async fn container_flake_action( verb: &str, name: &str, @@ -632,6 +638,20 @@ async fn container_flake_action( writer: &mut OwnedWriteHalf, ) -> Result<(String, String)> { validate_container_name(name)?; + if verb == "update" { + let toplevel = nix_build_toplevel(name).await?; + let args = [ + verb, + &container_system_name(name), + "--system-path", + &toplevel, + ]; + return if stream { + container_run_streaming(&args, writer).await + } else { + container_run(&args).await + }; + } let flake_ref = agent_flake_ref(name); let args = [verb, &container_system_name(name), "--flake", &flake_ref]; if stream { @@ -641,6 +661,72 @@ async fn container_flake_action( } } +/// The explicit `nixosConfigurations..config.system.build.toplevel` +/// flake attr path — same construction `hive-c0re`'s own +/// `lifecycle::prebuild_toplevel` uses, kept here as a pure function so +/// the exact string shape is unit-tested without needing to run `nix`. +fn toplevel_attr(name: &str) -> String { + format!("{META_DIR}#nixosConfigurations.{name}.config.system.build.toplevel") +} + +/// Build `nixosConfigurations..config.system.build.toplevel` and +/// return the resulting store path. +/// +/// **Why hive-priv builds this itself instead of letting `nixos-container +/// update` do it**: `nixos-container`'s own `buildFlake()` (invoked +/// whenever `--system-path` isn't passed) builds to a *hardcoded relative +/// path* — `$systemPath` is only ever assigned from the CLI flag or from +/// its own build result, so with no flag it stays `undef` and +/// `"$systemPath.tmp"` interpolates to the bare string `.tmp` in +/// whatever the caller's cwd is. Worse, the `create` action's `flock` +/// around this isn't taken for `update` at all. hive-priv never sets a +/// per-call cwd, so two concurrent `update`s (this hive runs +/// `services.hyperhive.c0re.buildSlots` > 1) share that one `.tmp`: one's +/// `readlink(".tmp")` can resolve to the *other's* build output, handing +/// an agent's container the wrong agent's closure — the "agent container +/// gets closure of other agent" mystery bug. +/// +/// Building the toplevel here and passing the resolved store path via +/// `--system-path` means `buildFlake()` — and its shared `.tmp` — never +/// runs at all. `--no-link` avoids a competing out-link race of our own; +/// we only need the store path, not a GC root (the store path is safe +/// from collection for as long as it takes `nixos-container update` to +/// register it against the container's own profile, same window every +/// other consumer of a `--print-out-paths` result already relies on). +async fn nix_build_toplevel(name: &str) -> Result { + let attr = toplevel_attr(name); + let args = [ + "--extra-experimental-features", + "nix-command flakes", + "build", + "--no-link", + "--print-out-paths", + &attr, + ]; + let out = Command::new("nix") + .args(args) + .output() + .await + .with_context(|| format!("invoke nix build {attr}"))?; + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); + for line in stderr.lines() { + tracing::warn!(target: "nix-build-toplevel", "{line}"); + } + if !out.status.success() { + bail!( + "nix build {attr} failed ({}): {}", + out.status, + stderr.trim() + ); + } + let path = stdout.trim(); + if path.is_empty() { + bail!("nix build {attr} produced no output path"); + } + Ok(path.to_owned()) +} + /// `WriteNspawnFlags` — validate the container + every bind path + every /// credential entry, then write the container's nspawn flag overrides. fn handle_write_nspawn_flags( @@ -2677,7 +2763,8 @@ mod tests { use super::{ BindMount, OwnedFd, PAUSED_MARKER_FILE, PrivRequest, check_fd_agreement, clear_runner_credentials, contains_secret_shaped_run, git_overlay_flags, - limits_dropin_body, redact_secret_line, remove_marker_in, write_state_file_nofollow, + limits_dropin_body, redact_secret_line, remove_marker_in, toplevel_attr, + write_state_file_nofollow, }; use std::path::PathBuf; use std::sync::atomic::{AtomicU32, Ordering}; @@ -2690,6 +2777,19 @@ mod tests { } } + /// Pins the exact attr path we hand to `nix build` — it has to match + /// `hive-c0re`'s own `lifecycle::prebuild_toplevel` construction, since + /// that step's whole point is warming the store for this later build. + /// A drifted attr path defeats the cache-warming silently — no error, + /// just a slower `update`. + #[test] + fn toplevel_attr_matches_prebuild_toplevels_construction() { + assert_eq!( + toplevel_attr("atlas"), + "/var/lib/hyperhive/meta#nixosConfigurations.atlas.config.system.build.toplevel" + ); + } + /// Every bound git repo gets its `.git` overlaid — the knowledge tree /// and *each* config mount, an agent's own plus every child's. ///