hyperhive/nix/vm-tests/agent-ephemeral-rootfs.nix

# `vmTests.<system>.agent-ephemeral-rootfs`: an agent container's switch from a
# persistent root to nspawn's `--ephemeral`, through the real hive-priv unit
# (nix/host-modules/hive-priv.nix, sandbox included). The container is an
# agent stand-in: the real agent user module (nix/agent-modules/user.nix) on a
# plain NixOS container, with the binds hive-c0re passes for an agent.
#
# What it pins:
#   - hive-priv's `WriteNspawnFlags` salvages the persistent root before it
#     passes `--ephemeral`, once, and deletes nothing;
#   - after the switch a file in /home is gone at the next start while `work/`
#     keeps its file and the machine id holds;
#   - the guest journal is written inside the container, where the agent's
#     forwarder (nix/agent-modules/otel.nix) reads it, and the host can read it;
#   - a package installed into the user's nix profile is still there, and on
#     PATH, after a restart.
#
# The agent's journal forwarder itself (an OTLP push to the hive's collector) is
# not booted here; the module-eval suite covers its wiring.
{ pkgs, hivePriv }:
let
  agent = "iris";
  machine = "h-${agent}";
  agentDir = "/var/lib/hyperhive/agents/${agent}";
  bridge = "hive0";
  gateway = "10.42.0.1";

  # Host side of what hive-c0re passes `WriteNspawnFlags` for an agent.
  binds = map (dir: {
    host_path = "${agentDir}/${dir}";
    container_path = if dir == "claude" then "/home/${agent}/.claude" else "/agents/${agent}/${dir}";
    read_only = false;
  }) hostDirs;
  hostDirs = [
    "state"
    "work"
    "harness"
    "claude"
  ];

  # The options hive-priv.nix reads, without the modules that declare them.
  hiveStubs =
    { lib, ... }:
    {
      options.services.hyperhive = {
        c0re.package = lib.mkOption { type = lib.types.package; };
        deploy.hive-controller.enable = lib.mkOption {
          type = lib.types.bool;
          default = true;
        };
        swarm.forge.domain = lib.mkOption {
          type = lib.types.str;
          default = "forge.t.local";
        };
      };
      config.services.hyperhive.c0re.package = hivePriv;
      # hive-priv.socket's group, which hive-c0re's module creates.
      config.users.groups.hive-core = { };
    };
in
pkgs.testers.runNixOSTest {
  name = "agent-ephemeral-rootfs";

  nodes.host =
    { pkgs, ... }:
    {
      imports = [
        hiveStubs
        ../host-modules/hive-priv.nix
      ];
      virtualisation.memorySize = 2048;
      # The agent bridge hive-priv names in the container's conf.
      networking.bridges.${bridge}.interfaces = [ ];
      networking.interfaces.${bridge}.ipv4.addresses = [
        {
          address = gateway;
          prefixLength = 24;
        }
      ];
      systemd.tmpfiles.rules = map (dir: "d ${agentDir}/${dir} 0755 root root - -") hostDirs;
      environment.systemPackages = [ pkgs.socat ];

      containers.${machine} = {
        autoStart = false;
        privateNetwork = true;
        hostBridge = bridge;
        config = {
          imports = [ ../agent-modules/user.nix ];
          system.stateVersion = "25.11";
          services.hyperhive.agent.user.name = agent;
          # The agent's forwarder reads /var/log/journal, which journald only
          # writes when it stores persistently.
          services.journald.storage = "persistent";
        };
      };
    };

  testScript = ''
    import json

    MACHINE = "${machine}"
    AGENT = "${agent}"
    AGENT_DIR = "${agentDir}"
    WORK = AGENT_DIR + "/work"
    ROOT = "/var/lib/nixos-containers/" + MACHINE
    BINDS = json.loads(${builtins.toJSON (builtins.toJSON binds)})
    HELLO = "${pkgs.hello}"
    TREE = "${pkgs.tree}"

    def in_ct(cmd):
        return host.succeed(f"nixos-container run {MACHINE} -- {cmd}").strip()

    def as_agent(script):
        # A login shell: HOME and the profile's PATH are the agent's own.
        return in_ct(f"runuser -l {AGENT} -c '{script}'")

    def boot():
        host.succeed(f"nixos-container start {MACHINE}")
        host.wait_until_succeeds(
            f"nixos-container run {MACHINE} -- systemctl is-active multi-user.target"
        )

    def stop():
        host.succeed(f"nixos-container stop {MACHINE}")

    def write_flags():
        """What hive-c0re does before every start: one request to hive-priv."""
        req = json.dumps({
            "op": "write_nspawn_flags",
            "container": MACHINE,
            "binds": BINDS,
            "isolation": {"bridge": "${bridge}", "gateway_ip": "${gateway}"},
        })
        out = host.succeed(f"echo '{req}' | socat - UNIX-CONNECT:/run/hive/priv.sock")
        assert json.loads(out.strip().splitlines()[-1])["ok"] is True, out

    def conf():
        return host.succeed(f"cat /etc/nixos-containers/{MACHINE}.conf")

    def salvage_dirs():
        return host.succeed(f"ls {WORK}").split()

    host.start()
    host.wait_for_unit("multi-user.target")
    host.wait_for_unit("hive-priv.socket")

    with subtest("before the switch: a persistent root"):
        assert "--ephemeral" not in conf()
        boot()
        uid = in_ct(f"id -u {AGENT}")
        as_agent("echo pre-switch > /home/${agent}/pre-switch.txt")
        in_ct("sh -c 'echo scratch > /tmp/cache-file'")
        # A root from before the agent module redirected nix's state: the real
        # directory, holding a package.
        in_ct(f"rm /home/{AGENT}/.local/state/nix")
        as_agent(f"nix-env -i {HELLO}")
        in_ct(f"test -d /home/{AGENT}/.local/state/nix/profiles")
        stop()
        host.succeed(f"test -f {ROOT}/home/{AGENT}/pre-switch.txt")

    with subtest("WriteNspawnFlags salvages the old root, then asks for --ephemeral"):
        assert salvage_dirs() == []
        write_flags()
        assert "--ephemeral" in conf(), conf()
        dirs = salvage_dirs()
        assert len(dirs) == 1 and dirs[0].startswith("salvaged-rootfs-"), dirs
        salvage = f"{WORK}/{dirs[0]}"
        assert host.succeed(f"cat {salvage}/home/{AGENT}/pre-switch.txt").strip() == "pre-switch"
        assert host.succeed(f"stat -c %u {salvage}/home/{AGENT}/pre-switch.txt").strip() == uid
        assert host.succeed(f"stat -c %u {salvage}").strip() == uid
        # The cache paths stay out; the old root is kept whole.
        host.fail(f"test -e {salvage}/tmp/cache-file")
        old_roots = host.succeed(f"ls -d {ROOT}.pre-ephemeral-*").split()
        assert len(old_roots) == 1, old_roots
        host.succeed(f"test -f {old_roots[0]}/home/{AGENT}/pre-switch.txt")
        host.fail(f"test -e {ROOT}/home/{AGENT}/pre-switch.txt")
        # nix's state moved to the per-container profile bind, where the next
        # start finds it.
        host.succeed(f"test -L /nix/var/nix/profiles/per-container/{MACHINE}/per-user/{AGENT}/profiles/profile")

    with subtest("the first ephemeral start, and what survives a restart"):
        boot()
        machine_id = in_ct("cat /etc/machine-id")
        in_ct("sh -c 'journalctl --sync; logger -t ephemeral-test journal-line'")
        in_ct("sh -c 'journalctl --flush --sync'")
        as_agent("echo gone > /home/${agent}/ephemeral.txt")
        in_ct(f"sh -c 'echo kept > /agents/{AGENT}/work/kept.txt'")
        # The journal is written in the guest, where the forwarder reads it, and
        # the host reads it too.
        assert in_ct("ls /var/log/journal") != ""
        in_ct("journalctl -t ephemeral-test --directory=/var/log/journal | grep journal-line")
        host.wait_until_succeeds(f"journalctl -M {MACHINE} -t ephemeral-test | grep journal-line")
        # The package installed before the switch is still on the agent's PATH, and
        # one installed now is kept with it.
        as_agent("hello")
        as_agent(f"nix-env -i {TREE}")

        stop()
        write_flags()
        boot()
        assert in_ct("cat /etc/machine-id") == machine_id
        in_ct(f"sh -c 'test ! -e /home/{AGENT}/ephemeral.txt'")
        assert in_ct(f"cat /agents/{AGENT}/work/kept.txt") == "kept"
        assert host.succeed(f"cat {WORK}/kept.txt").strip() == "kept"
        as_agent("hello")
        as_agent("tree --version")

    with subtest("the switch ran once"):
        salvages = [d for d in salvage_dirs() if d.startswith("salvaged-rootfs-")]
        assert len(salvages) == 1, salvages
        host.succeed(f"ls -d {ROOT}.pre-ephemeral-* | wc -l | grep -x 1")
        assert conf().count("--ephemeral") == 1, conf()
  '';
}