hive-priv: build the container toplevel ourselves instead of nixos-container's buildFlake

hive-priv/src/main.rs

@@ -623,8 +623,14 @@ async fn exec(
 }
 
 /// Shared body for `CreateContainer` / `UpdateContainer`: validate the
-/// name, build the `nixos-container <verb> … --flake <ref>` argv, and
-/// run it (streaming line events to `writer` when `stream` is set).
+/// name, build the `nixos-container <verb> …` argv, and run it (streaming
+/// line events to `writer` when `stream` is set).
+///
+/// `update` builds the toplevel itself first and passes `--system-path`
+/// instead of `--flake` — see [`nix_build_toplevel`] for why. `create`
+/// keeps the plain `--flake` form: `nixos-container`'s own `buildFlake()`
+/// takes an internal `flock` for `create` (not for `update`), so the
+/// hazard this works around doesn't apply there.
 async fn container_flake_action(
     verb: &str,
     name: &str,
@@ -632,6 +638,20 @@ async fn container_flake_action(
     writer: &mut OwnedWriteHalf,
 ) -> Result<(String, String)> {
     validate_container_name(name)?;
+    if verb == "update" {
+        let toplevel = nix_build_toplevel(name).await?;
+        let args = [
+            verb,
+            &container_system_name(name),
+            "--system-path",
+            &toplevel,
+        ];
+        return if stream {
+            container_run_streaming(&args, writer).await
+        } else {
+            container_run(&args).await
+        };
+    }
     let flake_ref = agent_flake_ref(name);
     let args = [verb, &container_system_name(name), "--flake", &flake_ref];
     if stream {
@@ -641,6 +661,72 @@ async fn container_flake_action(
     }
 }
 
+/// The explicit `nixosConfigurations.<name>.config.system.build.toplevel`
+/// flake attr path — same construction `hive-c0re`'s own
+/// `lifecycle::prebuild_toplevel` uses, kept here as a pure function so
+/// the exact string shape is unit-tested without needing to run `nix`.
+fn toplevel_attr(name: &str) -> String {
+    format!("{META_DIR}#nixosConfigurations.{name}.config.system.build.toplevel")
+}
+
+/// Build `nixosConfigurations.<name>.config.system.build.toplevel` and
+/// return the resulting store path.
+///
+/// **Why hive-priv builds this itself instead of letting `nixos-container
+/// update` do it**: `nixos-container`'s own `buildFlake()` (invoked
+/// whenever `--system-path` isn't passed) builds to a *hardcoded relative
+/// path* — `$systemPath` is only ever assigned from the CLI flag or from
+/// its own build result, so with no flag it stays `undef` and
+/// `"$systemPath.tmp"` interpolates to the bare string `.tmp` in
+/// whatever the caller's cwd is. Worse, the `create` action's `flock`
+/// around this isn't taken for `update` at all. hive-priv never sets a
+/// per-call cwd, so two concurrent `update`s (this hive runs
+/// `services.hyperhive.c0re.buildSlots` > 1) share that one `.tmp`: one's
+/// `readlink(".tmp")` can resolve to the *other's* build output, handing
+/// an agent's container the wrong agent's closure — the "agent container
+/// gets closure of other agent" mystery bug.
+///
+/// Building the toplevel here and passing the resolved store path via
+/// `--system-path` means `buildFlake()` — and its shared `.tmp` — never
+/// runs at all. `--no-link` avoids a competing out-link race of our own;
+/// we only need the store path, not a GC root (the store path is safe
+/// from collection for as long as it takes `nixos-container update` to
+/// register it against the container's own profile, same window every
+/// other consumer of a `--print-out-paths` result already relies on).
+async fn nix_build_toplevel(name: &str) -> Result<String> {
+    let attr = toplevel_attr(name);
+    let args = [
+        "--extra-experimental-features",
+        "nix-command flakes",
+        "build",
+        "--no-link",
+        "--print-out-paths",
+        &attr,
+    ];
+    let out = Command::new("nix")
+        .args(args)
+        .output()
+        .await
+        .with_context(|| format!("invoke nix build {attr}"))?;
+    let stdout = String::from_utf8_lossy(&out.stdout).into_owned();
+    let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
+    for line in stderr.lines() {
+        tracing::warn!(target: "nix-build-toplevel", "{line}");
+    }
+    if !out.status.success() {
+        bail!(
+            "nix build {attr} failed ({}): {}",
+            out.status,
+            stderr.trim()
+        );
+    }
+    let path = stdout.trim();
+    if path.is_empty() {
+        bail!("nix build {attr} produced no output path");
+    }
+    Ok(path.to_owned())
+}
+
 /// `WriteNspawnFlags` — validate the container + every bind path + every
 /// credential entry, then write the container's nspawn flag overrides.
 fn handle_write_nspawn_flags(
@@ -2677,7 +2763,8 @@ mod tests {
     use super::{
         BindMount, OwnedFd, PAUSED_MARKER_FILE, PrivRequest, check_fd_agreement,
         clear_runner_credentials, contains_secret_shaped_run, git_overlay_flags,
-        limits_dropin_body, redact_secret_line, remove_marker_in, write_state_file_nofollow,
+        limits_dropin_body, redact_secret_line, remove_marker_in, toplevel_attr,
+        write_state_file_nofollow,
     };
     use std::path::PathBuf;
     use std::sync::atomic::{AtomicU32, Ordering};
@@ -2690,6 +2777,19 @@ mod tests {
         }
     }
 
+    /// Pins the exact attr path we hand to `nix build` — it has to match
+    /// `hive-c0re`'s own `lifecycle::prebuild_toplevel` construction, since
+    /// that step's whole point is warming the store for this later build.
+    /// A drifted attr path defeats the cache-warming silently — no error,
+    /// just a slower `update`.
+    #[test]
+    fn toplevel_attr_matches_prebuild_toplevels_construction() {
+        assert_eq!(
+            toplevel_attr("atlas"),
+            "/var/lib/hyperhive/meta#nixosConfigurations.atlas.config.system.build.toplevel"
+        );
+    }
+
     /// Every bound git repo gets its `.git` overlaid — the knowledge tree
     /// and *each* config mount, an agent's own plus every child's.
     ///