run agent containers on an ephemeral root, salvaging the old one first

hive-priv copies a stopped container's root to the agent's work dir and
renames it aside before passing nspawn --ephemeral; the agent's nix state
moves to the persistent profiles bind. Adds the agent-ephemeral-rootfs vm
test and a module-eval case for the nix state redirect.

hive-priv/src/container_config.rs

@@ -27,7 +27,7 @@ pub(crate) fn handle_write_nspawn_flags(
     load_credentials: &[CredentialMount],
     hive_facts: &HiveFacts,
 ) -> Result<(String, String)> {
-    validate_container_system_name(container)?;
+    let agent = validate_container_system_name(container)?;
     for bind in binds {
         validate_bind_path(&bind.host_path)?;
         validate_bind_path(&bind.container_path)?;
@@ -40,7 +40,24 @@ pub(crate) fn handle_write_nspawn_flags(
         validate_bind_path(&cred.host_path)?;
     }
     let hive_facts = render_hive_facts(hive_facts)?;
-    write_nspawn_flags(container, binds, isolation, load_credentials, &hive_facts)?;
+    // A root that cannot be salvaged keeps its persistent start: the flag is
+    // only ever passed for a root the salvage has dealt with.
+    let ephemeral = crate::rootfs::prepare(container, agent).unwrap_or_else(|e| {
+        tracing::error!(
+            %container,
+            error = %format!("{e:#}"),
+            "root salvage failed; the container starts with its persistent root"
+        );
+        false
+    });
+    write_nspawn_flags(
+        container,
+        binds,
+        isolation,
+        load_credentials,
+        &hive_facts,
+        ephemeral,
+    )?;
     Ok((String::new(), String::new()))
 }
 
@@ -267,6 +284,7 @@ fn write_nspawn_flags(
     isolation: &NetworkIsolation,
     load_credentials: &[CredentialMount],
     hive_facts: &str,
+    ephemeral: bool,
 ) -> Result<()> {
     use std::fmt::Write as _;
     use std::os::unix::fs::MetadataExt as _;
@@ -309,25 +327,7 @@ fn write_nspawn_flags(
         out.push_str("LOCAL_ADDRESS6=\n");
         let _ = writeln!(out, "HOST_BRIDGE={}", iso.bridge);
     }
-    let mut flags: Vec<String> = binds
-        .iter()
-        .map(|b| {
-            let flag = if b.read_only { "--bind-ro" } else { "--bind" };
-            format!("{flag}={}:{}", b.host_path, b.container_path)
-        })
-        .collect();
-    flags.extend(git_overlay_flags(binds));
-    // Credential forwarding: nspawn loads each host secret into the
-    // container's credential store under `<name>`; inner units inherit it
-    // via `LoadCredential=<name>`. Validated (name charset + bind-path
-    // rules) in handle_write_nspawn_flags above.
-    for cred in load_credentials {
-        flags.push(format!(
-            "--load-credential={}:{}",
-            cred.name, cred.host_path
-        ));
-    }
-    let flags_joined = flags.join(" ");
+    let flags_joined = nspawn_flags(binds, load_credentials, ephemeral).join(" ");
     let _ = writeln!(out, "EXTRA_NSPAWN_FLAGS=\"{flags_joined}\"");
     // Keeps the owner and mode `nixos-container` created the file with; this
     // helper only rewrites its lines.
@@ -353,6 +353,38 @@ fn write_nspawn_flags(
     Ok(())
 }
 
+/// The flags `EXTRA_NSPAWN_FLAGS` carries, in the order nspawn needs them.
+///
+/// `--ephemeral` runs the container on a snapshot of its root and drops the
+/// snapshot at stop; `crate::rootfs` has made the root fit for that before
+/// the flag is asked for. The git overlays and credentials follow the binds
+/// they refer to.
+fn nspawn_flags(
+    binds: &[BindMount],
+    load_credentials: &[CredentialMount],
+    ephemeral: bool,
+) -> Vec<String> {
+    let mut flags: Vec<String> = ephemeral
+        .then(|| "--ephemeral".to_owned())
+        .into_iter()
+        .collect();
+    flags.extend(binds.iter().map(|b| {
+        let flag = if b.read_only { "--bind-ro" } else { "--bind" };
+        format!("{flag}={}:{}", b.host_path, b.container_path)
+    }));
+    flags.extend(git_overlay_flags(binds));
+    // Credential forwarding: nspawn loads each host secret into the
+    // container's credential store under `<name>`; inner units inherit it
+    // via `LoadCredential=<name>`. Validated (name charset + bind-path
+    // rules) in handle_write_nspawn_flags.
+    flags.extend(
+        load_credentials
+            .iter()
+            .map(|cred| format!("--load-credential={}:{}", cred.name, cred.host_path)),
+    );
+    flags
+}
+
 /// Filename of the in-container DNS marker, in the container's own `/etc`.
 const BRIDGE_DNS_MARKER: &str = "hyperhive-bridge-dns";
 
@@ -360,7 +392,7 @@ const BRIDGE_DNS_MARKER: &str = "hyperhive-bridge-dns";
 const HIVE_FACTS_FILE: &str = "hyperhive-hive-facts";
 
 /// The container's `/` on the host.
-fn container_rootfs(container: &str) -> PathBuf {
+pub(crate) fn container_rootfs(container: &str) -> PathBuf {
     PathBuf::from(format!("/var/lib/nixos-containers/{container}"))
 }
 
@@ -469,7 +501,7 @@ fn write_bridge_dns_marker_in(rootfs: &Path, gateway_ip: &str) -> Result<()> {
 /// materialises the rootfs on the first start), so this creates it first; the
 /// dir and the file persist through that start. Mode 0644: nothing written
 /// here is a secret.
-fn write_container_etc_file_in(rootfs: &Path, name: &str, contents: &str) -> Result<()> {
+pub(crate) fn write_container_etc_file_in(rootfs: &Path, name: &str, contents: &str) -> Result<()> {
     use std::os::unix::fs::OpenOptionsExt as _;
 
     let etc_path = rootfs.join("etc");
@@ -510,7 +542,7 @@ fn write_container_etc_file_in(rootfs: &Path, name: &str, contents: &str) -> Res
 mod tests {
     use super::{
         BindMount, HiveFacts, NetworkIsolation, git_overlay_flags, handle_write_nspawn_flags,
-        limits_dropin_body, publish_file, remove_service_dropin, render_hive_facts,
+        limits_dropin_body, nspawn_flags, publish_file, remove_service_dropin, render_hive_facts,
         validate_container_system_name, write_bridge_dns_marker_in, write_resource_limits,
     };
     use crate::shared::test_support::{entries, scratch};
@@ -909,6 +941,20 @@ mod tests {
         }
     }
 
+    /// `--ephemeral` is asked for, not assumed: present exactly when the root has
+    /// been dealt with, ahead of the binds it applies to. The binds are the
+    /// control that the lookup finds what is there.
+    #[test]
+    fn ephemeral_is_passed_only_when_asked() {
+        let binds = [bind("/agents/atlas/work")];
+        let with = nspawn_flags(&binds, &[], true);
+        assert_eq!(with[0], "--ephemeral", "{with:?}");
+        assert!(with.iter().any(|f| f.starts_with("--bind-ro=")), "{with:?}");
+        let without = nspawn_flags(&binds, &[], false);
+        assert!(!without.iter().any(|f| f == "--ephemeral"), "{without:?}");
+        assert_eq!(without.len() + 1, with.len());
+    }
+
     /// The swarm's forge and matrix containers are not the hive's: every
     /// request that names a container as a string refuses them before
     /// touching the filesystem. Control: an agent container name passes.