run agent containers on an ephemeral root, salvaging the old one first

hive-priv copies a stopped container's root to the agent's work dir and
renames it aside before passing nspawn --ephemeral; the agent's nix state
moves to the persistent profiles bind. Adds the agent-ephemeral-rootfs vm
test and a module-eval case for the nix state redirect.

nix/agent-modules/otel.nix

@@ -222,7 +222,7 @@ in
         # `StateDirectory` (nixpkgs' module already sets one, `%S`, below
         # as `WorkingDirectory` too) — container-lifetime, not the
         # host-persisted state bind mount: it indexes /var/log/journal,
-        # itself fresh on every container recreate, so the two must share
+        # itself fresh on every container start, so the two must share
         # a lifetime or the cursor outlives the journal it points into.
         # `create_directory` matters here for a build-time-vs-runtime
         # reason, not a convenience one: `validateConfigFile` above runs

nix/agent-modules/user.nix

@@ -142,6 +142,26 @@ in
         gid = config.services.hyperhive.agent.user.gid;
       };
 
+    # nix keeps a non-root user's profile under `~/.local/state/nix`, and the
+    # root is rebuilt at every start. Redirecting that directory into the
+    # per-container `/nix/var/nix/profiles` bind keeps what the agent installs
+    # (profile generations and the GC roots that point at them) across restarts.
+    # `~/.nix-profile` is the link nix makes on first use; recreating it puts
+    # the profile's `bin/` back on PATH before nix runs again.
+    systemd.tmpfiles.rules =
+      let
+        persisted = "/nix/var/nix/profiles/per-user/${userName}";
+        owner = "${userName} ${userName}";
+      in
+      [
+        "d /nix/var/nix/profiles/per-user 0755 root root - -"
+        "d ${persisted} 0755 ${owner} - -"
+        "d ${homeDir}/.local 0755 ${owner} - -"
+        "d ${homeDir}/.local/state 0755 ${owner} - -"
+        "L ${homeDir}/.local/state/nix - - - - ${persisted}"
+        "L ${homeDir}/.nix-profile - - - - ${homeDir}/.local/state/nix/profiles/profile"
+      ];
+
     # `NOPASSWD: ALL` for the agent user. Lets claude's Bash tool
     # keep working with anything that expected root (systemctl,
     # nix-env, etc.) without prompting. Flip

nix/module-eval/agent-user.nix

@@ -25,6 +25,8 @@ let
   migrate =
     (agentWith { services.hyperhive.agent.user.name = "iris"; })
     .system.activationScripts.hive-agent-user-migrate.text;
+  # The tmpfiles rules of the same named agent.
+  rules = (agentWith { services.hyperhive.agent.user.name = "iris"; }).systemd.tmpfiles.rules;
   cases = [
     {
       # hive-c0re creates the host work dir as hive-core; this activation is
@@ -63,6 +65,17 @@ let
         lib.hasInfix ''if [ -L "$socketDir" ] || { [ -e "$socketDir" ] && [ ! -d "$socketDir" ]; }; then'' migrate
         && lib.hasInfix ''elif [ -d "$socketDir" ]; then'' migrate;
     }
+    {
+      # The container root is rebuilt at every start; nix keeps a non-root
+      # user's profile in the home, so these rules are the only thing that puts
+      # it in the persistent per-container profiles bind. The `d` and `L` rules
+      # are each other's control: a misspelled path in one still finds the other.
+      name = "the agent's tmpfiles rules keep nix's per-user state in the persistent profiles bind";
+      ok =
+        lib.elem "d /nix/var/nix/profiles/per-user/iris 0755 iris iris - -" rules
+        && lib.elem "L /home/iris/.local/state/nix - - - - /nix/var/nix/profiles/per-user/iris" rules
+        && lib.elem "L /home/iris/.nix-profile - - - - /home/iris/.local/state/nix/profiles/profile" rules;
+    }
   ];
 in
 runGroup "agent-user" cases

nix/vm-tests.nix

@@ -1,7 +1,8 @@
 # NixOS VM tests: one derivation per test, exposed as `vmTests.<system>.<name>`.
 # Kept out of `checks` because each boots VMs and needs KVM; the nightly
 # workflow `.forgejo/workflows/vm-nightly.yml` builds them.
-{ pkgs }:
+{ pkgs, hivePriv }:
 {
+  agent-ephemeral-rootfs = import ./vm-tests/agent-ephemeral-rootfs.nix { inherit pkgs hivePriv; };
   hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs; };
 }

nix/vm-tests/agent-ephemeral-rootfs.nix

@@ -0,0 +1,213 @@
+# `vmTests.<system>.agent-ephemeral-rootfs`: an agent container's switch from a
+# persistent root to nspawn's `--ephemeral`, through the real hive-priv unit
+# (nix/host-modules/hive-priv.nix, sandbox included). The container is an
+# agent stand-in: the real agent user module (nix/agent-modules/user.nix) on a
+# plain NixOS container, with the binds hive-c0re passes for an agent.
+#
+# What it pins:
+#   - hive-priv's `WriteNspawnFlags` salvages the persistent root before it
+#     passes `--ephemeral`, once, and deletes nothing;
+#   - after the switch a file in /home is gone at the next start while `work/`
+#     keeps its file and the machine id holds;
+#   - the guest journal is written inside the container, where the agent's
+#     forwarder (nix/agent-modules/otel.nix) reads it, and the host can read it;
+#   - a package installed into the user's nix profile is still there, and on
+#     PATH, after a restart.
+#
+# The agent's journal forwarder itself (an OTLP push to the hive's collector) is
+# not booted here; the module-eval suite covers its wiring.
+{ pkgs, hivePriv }:
+let
+  agent = "iris";
+  machine = "h-${agent}";
+  agentDir = "/var/lib/hyperhive/agents/${agent}";
+  bridge = "hive0";
+  gateway = "10.42.0.1";
+
+  # Host side of what hive-c0re passes `WriteNspawnFlags` for an agent.
+  binds = map (dir: {
+    host_path = "${agentDir}/${dir}";
+    container_path = if dir == "claude" then "/home/${agent}/.claude" else "/agents/${agent}/${dir}";
+    read_only = false;
+  }) hostDirs;
+  hostDirs = [
+    "state"
+    "work"
+    "harness"
+    "claude"
+  ];
+
+  # The options hive-priv.nix reads, without the modules that declare them.
+  hiveStubs =
+    { lib, ... }:
+    {
+      options.services.hyperhive = {
+        c0re.package = lib.mkOption { type = lib.types.package; };
+        deploy.hive-controller.enable = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+        };
+        swarm.forge.domain = lib.mkOption {
+          type = lib.types.str;
+          default = "forge.t.local";
+        };
+      };
+      config.services.hyperhive.c0re.package = hivePriv;
+      # hive-priv.socket's group, which hive-c0re's module creates.
+      config.users.groups.hive-core = { };
+    };
+in
+pkgs.testers.runNixOSTest {
+  name = "agent-ephemeral-rootfs";
+
+  nodes.host =
+    { pkgs, ... }:
+    {
+      imports = [
+        hiveStubs
+        ../host-modules/hive-priv.nix
+      ];
+      virtualisation.memorySize = 2048;
+      # The agent bridge hive-priv names in the container's conf.
+      networking.bridges.${bridge}.interfaces = [ ];
+      networking.interfaces.${bridge}.ipv4.addresses = [
+        {
+          address = gateway;
+          prefixLength = 24;
+        }
+      ];
+      systemd.tmpfiles.rules = map (dir: "d ${agentDir}/${dir} 0755 root root - -") hostDirs;
+      environment.systemPackages = [ pkgs.socat ];
+
+      containers.${machine} = {
+        autoStart = false;
+        privateNetwork = true;
+        hostBridge = bridge;
+        config = {
+          imports = [ ../agent-modules/user.nix ];
+          system.stateVersion = "25.11";
+          services.hyperhive.agent.user.name = agent;
+          # The agent's forwarder reads /var/log/journal, which journald only
+          # writes when it stores persistently.
+          services.journald.storage = "persistent";
+        };
+      };
+    };
+
+  testScript = ''
+    import json
+
+    MACHINE = "${machine}"
+    AGENT = "${agent}"
+    AGENT_DIR = "${agentDir}"
+    WORK = AGENT_DIR + "/work"
+    ROOT = "/var/lib/nixos-containers/" + MACHINE
+    BINDS = json.loads(${builtins.toJSON (builtins.toJSON binds)})
+    HELLO = "${pkgs.hello}"
+    TREE = "${pkgs.tree}"
+
+    def in_ct(cmd):
+        return host.succeed(f"nixos-container run {MACHINE} -- {cmd}").strip()
+
+    def as_agent(script):
+        # A login shell: HOME and the profile's PATH are the agent's own.
+        return in_ct(f"runuser -l {AGENT} -c '{script}'")
+
+    def boot():
+        host.succeed(f"nixos-container start {MACHINE}")
+        host.wait_until_succeeds(
+            f"nixos-container run {MACHINE} -- systemctl is-active multi-user.target"
+        )
+
+    def stop():
+        host.succeed(f"nixos-container stop {MACHINE}")
+
+    def write_flags():
+        """What hive-c0re does before every start: one request to hive-priv."""
+        req = json.dumps({
+            "op": "write_nspawn_flags",
+            "container": MACHINE,
+            "binds": BINDS,
+            "isolation": {"bridge": "${bridge}", "gateway_ip": "${gateway}"},
+        })
+        out = host.succeed(f"echo '{req}' | socat - UNIX-CONNECT:/run/hive/priv.sock")
+        assert json.loads(out.strip().splitlines()[-1])["ok"] is True, out
+
+    def conf():
+        return host.succeed(f"cat /etc/nixos-containers/{MACHINE}.conf")
+
+    def salvage_dirs():
+        return host.succeed(f"ls {WORK}").split()
+
+    host.start()
+    host.wait_for_unit("multi-user.target")
+    host.wait_for_unit("hive-priv.socket")
+
+    with subtest("before the switch: a persistent root"):
+        assert "--ephemeral" not in conf()
+        boot()
+        uid = in_ct(f"id -u {AGENT}")
+        as_agent("echo pre-switch > /home/${agent}/pre-switch.txt")
+        in_ct("sh -c 'echo scratch > /tmp/cache-file'")
+        # A root from before the agent module redirected nix's state: the real
+        # directory, holding a package.
+        in_ct(f"rm /home/{AGENT}/.local/state/nix")
+        as_agent(f"nix-env -i {HELLO}")
+        in_ct(f"test -d /home/{AGENT}/.local/state/nix/profiles")
+        stop()
+        host.succeed(f"test -f {ROOT}/home/{AGENT}/pre-switch.txt")
+
+    with subtest("WriteNspawnFlags salvages the old root, then asks for --ephemeral"):
+        assert salvage_dirs() == []
+        write_flags()
+        assert "--ephemeral" in conf(), conf()
+        dirs = salvage_dirs()
+        assert len(dirs) == 1 and dirs[0].startswith("salvaged-rootfs-"), dirs
+        salvage = f"{WORK}/{dirs[0]}"
+        assert host.succeed(f"cat {salvage}/home/{AGENT}/pre-switch.txt").strip() == "pre-switch"
+        assert host.succeed(f"stat -c %u {salvage}/home/{AGENT}/pre-switch.txt").strip() == uid
+        assert host.succeed(f"stat -c %u {salvage}").strip() == uid
+        # The cache paths stay out; the old root is kept whole.
+        host.fail(f"test -e {salvage}/tmp/cache-file")
+        old_roots = host.succeed(f"ls -d {ROOT}.pre-ephemeral-*").split()
+        assert len(old_roots) == 1, old_roots
+        host.succeed(f"test -f {old_roots[0]}/home/{AGENT}/pre-switch.txt")
+        host.fail(f"test -e {ROOT}/home/{AGENT}/pre-switch.txt")
+        # nix's state moved to the per-container profile bind, where the next
+        # start finds it.
+        host.succeed(f"test -L /nix/var/nix/profiles/per-container/{MACHINE}/per-user/{AGENT}/profiles/profile")
+
+    with subtest("the first ephemeral start, and what survives a restart"):
+        boot()
+        machine_id = in_ct("cat /etc/machine-id")
+        in_ct("sh -c 'journalctl --sync; logger -t ephemeral-test journal-line'")
+        in_ct("sh -c 'journalctl --flush --sync'")
+        as_agent("echo gone > /home/${agent}/ephemeral.txt")
+        in_ct(f"sh -c 'echo kept > /agents/{AGENT}/work/kept.txt'")
+        # The journal is written in the guest, where the forwarder reads it, and
+        # the host reads it too.
+        assert in_ct("ls /var/log/journal") != ""
+        in_ct("journalctl -t ephemeral-test --directory=/var/log/journal | grep journal-line")
+        host.wait_until_succeeds(f"journalctl -M {MACHINE} -t ephemeral-test | grep journal-line")
+        # The package installed before the switch is still on the agent's PATH, and
+        # one installed now is kept with it.
+        as_agent("hello")
+        as_agent(f"nix-env -i {TREE}")
+
+        stop()
+        write_flags()
+        boot()
+        assert in_ct("cat /etc/machine-id") == machine_id
+        in_ct(f"sh -c 'test ! -e /home/{AGENT}/ephemeral.txt'")
+        assert in_ct(f"cat /agents/{AGENT}/work/kept.txt") == "kept"
+        assert host.succeed(f"cat {WORK}/kept.txt").strip() == "kept"
+        as_agent("hello")
+        as_agent("tree --version")
+
+    with subtest("the switch ran once"):
+        salvages = [d for d in salvage_dirs() if d.startswith("salvaged-rootfs-")]
+        assert len(salvages) == 1, salvages
+        host.succeed(f"ls -d {ROOT}.pre-ephemeral-* | wc -l | grep -x 1")
+        assert conf().count("--ephemeral") == 1, conf()
+  '';
+}