run agent containers on an ephemeral root, salvaging the old one first
hive-priv copies a stopped container's root to the agent's work dir and renames it aside before passing nspawn --ephemeral; the agent's nix state moves to the persistent profiles bind. Adds the agent-ephemeral-rootfs vm test and a module-eval case for the nix state redirect.
This commit is contained in:
atlas
mara
parent
22522b386a
commit
81439fa710
1 changed files with 1238 additions and 39 deletions
213
nix/vm-tests/agent-ephemeral-rootfs.nix
Normal file
213
nix/vm-tests/agent-ephemeral-rootfs.nix
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
# `vmTests.<system>.agent-ephemeral-rootfs`: an agent container's switch from a
|
||||
# persistent root to nspawn's `--ephemeral`, through the real hive-priv unit
|
||||
# (nix/host-modules/hive-priv.nix, sandbox included). The container is an
|
||||
# agent stand-in: the real agent user module (nix/agent-modules/user.nix) on a
|
||||
# plain NixOS container, with the binds hive-c0re passes for an agent.
|
||||
#
|
||||
# What it pins:
|
||||
# - hive-priv's `WriteNspawnFlags` salvages the persistent root before it
|
||||
# passes `--ephemeral`, once, and deletes nothing;
|
||||
# - after the switch a file in /home is gone at the next start while `work/`
|
||||
# keeps its file and the machine id holds;
|
||||
# - the guest journal is written inside the container, where the agent's
|
||||
# forwarder (nix/agent-modules/otel.nix) reads it, and the host can read it;
|
||||
# - a package installed into the user's nix profile is still there, and on
|
||||
# PATH, after a restart.
|
||||
#
|
||||
# The agent's journal forwarder itself (an OTLP push to the hive's collector) is
|
||||
# not booted here; the module-eval suite covers its wiring.
|
||||
{ pkgs, hivePriv }:
|
||||
let
|
||||
agent = "iris";
|
||||
machine = "h-${agent}";
|
||||
agentDir = "/var/lib/hyperhive/agents/${agent}";
|
||||
bridge = "hive0";
|
||||
gateway = "10.42.0.1";
|
||||
|
||||
# Host side of what hive-c0re passes `WriteNspawnFlags` for an agent.
|
||||
binds = map (dir: {
|
||||
host_path = "${agentDir}/${dir}";
|
||||
container_path = if dir == "claude" then "/home/${agent}/.claude" else "/agents/${agent}/${dir}";
|
||||
read_only = false;
|
||||
}) hostDirs;
|
||||
hostDirs = [
|
||||
"state"
|
||||
"work"
|
||||
"harness"
|
||||
"claude"
|
||||
];
|
||||
|
||||
# The options hive-priv.nix reads, without the modules that declare them.
|
||||
hiveStubs =
|
||||
{ lib, ... }:
|
||||
{
|
||||
options.services.hyperhive = {
|
||||
c0re.package = lib.mkOption { type = lib.types.package; };
|
||||
deploy.hive-controller.enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
};
|
||||
swarm.forge.domain = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "forge.t.local";
|
||||
};
|
||||
};
|
||||
config.services.hyperhive.c0re.package = hivePriv;
|
||||
# hive-priv.socket's group, which hive-c0re's module creates.
|
||||
config.users.groups.hive-core = { };
|
||||
};
|
||||
in
|
||||
pkgs.testers.runNixOSTest {
|
||||
name = "agent-ephemeral-rootfs";
|
||||
|
||||
nodes.host =
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
imports = [
|
||||
hiveStubs
|
||||
../host-modules/hive-priv.nix
|
||||
];
|
||||
virtualisation.memorySize = 2048;
|
||||
# The agent bridge hive-priv names in the container's conf.
|
||||
networking.bridges.${bridge}.interfaces = [ ];
|
||||
networking.interfaces.${bridge}.ipv4.addresses = [
|
||||
{
|
||||
address = gateway;
|
||||
prefixLength = 24;
|
||||
}
|
||||
];
|
||||
systemd.tmpfiles.rules = map (dir: "d ${agentDir}/${dir} 0755 root root - -") hostDirs;
|
||||
environment.systemPackages = [ pkgs.socat ];
|
||||
|
||||
containers.${machine} = {
|
||||
autoStart = false;
|
||||
privateNetwork = true;
|
||||
hostBridge = bridge;
|
||||
config = {
|
||||
imports = [ ../agent-modules/user.nix ];
|
||||
system.stateVersion = "25.11";
|
||||
services.hyperhive.agent.user.name = agent;
|
||||
# The agent's forwarder reads /var/log/journal, which journald only
|
||||
# writes when it stores persistently.
|
||||
services.journald.storage = "persistent";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
import json
|
||||
|
||||
MACHINE = "${machine}"
|
||||
AGENT = "${agent}"
|
||||
AGENT_DIR = "${agentDir}"
|
||||
WORK = AGENT_DIR + "/work"
|
||||
ROOT = "/var/lib/nixos-containers/" + MACHINE
|
||||
BINDS = json.loads(${builtins.toJSON (builtins.toJSON binds)})
|
||||
HELLO = "${pkgs.hello}"
|
||||
TREE = "${pkgs.tree}"
|
||||
|
||||
def in_ct(cmd):
|
||||
return host.succeed(f"nixos-container run {MACHINE} -- {cmd}").strip()
|
||||
|
||||
def as_agent(script):
|
||||
# A login shell: HOME and the profile's PATH are the agent's own.
|
||||
return in_ct(f"runuser -l {AGENT} -c '{script}'")
|
||||
|
||||
def boot():
|
||||
host.succeed(f"nixos-container start {MACHINE}")
|
||||
host.wait_until_succeeds(
|
||||
f"nixos-container run {MACHINE} -- systemctl is-active multi-user.target"
|
||||
)
|
||||
|
||||
def stop():
|
||||
host.succeed(f"nixos-container stop {MACHINE}")
|
||||
|
||||
def write_flags():
|
||||
"""What hive-c0re does before every start: one request to hive-priv."""
|
||||
req = json.dumps({
|
||||
"op": "write_nspawn_flags",
|
||||
"container": MACHINE,
|
||||
"binds": BINDS,
|
||||
"isolation": {"bridge": "${bridge}", "gateway_ip": "${gateway}"},
|
||||
})
|
||||
out = host.succeed(f"echo '{req}' | socat - UNIX-CONNECT:/run/hive/priv.sock")
|
||||
assert json.loads(out.strip().splitlines()[-1])["ok"] is True, out
|
||||
|
||||
def conf():
|
||||
return host.succeed(f"cat /etc/nixos-containers/{MACHINE}.conf")
|
||||
|
||||
def salvage_dirs():
|
||||
return host.succeed(f"ls {WORK}").split()
|
||||
|
||||
host.start()
|
||||
host.wait_for_unit("multi-user.target")
|
||||
host.wait_for_unit("hive-priv.socket")
|
||||
|
||||
with subtest("before the switch: a persistent root"):
|
||||
assert "--ephemeral" not in conf()
|
||||
boot()
|
||||
uid = in_ct(f"id -u {AGENT}")
|
||||
as_agent("echo pre-switch > /home/${agent}/pre-switch.txt")
|
||||
in_ct("sh -c 'echo scratch > /tmp/cache-file'")
|
||||
# A root from before the agent module redirected nix's state: the real
|
||||
# directory, holding a package.
|
||||
in_ct(f"rm /home/{AGENT}/.local/state/nix")
|
||||
as_agent(f"nix-env -i {HELLO}")
|
||||
in_ct(f"test -d /home/{AGENT}/.local/state/nix/profiles")
|
||||
stop()
|
||||
host.succeed(f"test -f {ROOT}/home/{AGENT}/pre-switch.txt")
|
||||
|
||||
with subtest("WriteNspawnFlags salvages the old root, then asks for --ephemeral"):
|
||||
assert salvage_dirs() == []
|
||||
write_flags()
|
||||
assert "--ephemeral" in conf(), conf()
|
||||
dirs = salvage_dirs()
|
||||
assert len(dirs) == 1 and dirs[0].startswith("salvaged-rootfs-"), dirs
|
||||
salvage = f"{WORK}/{dirs[0]}"
|
||||
assert host.succeed(f"cat {salvage}/home/{AGENT}/pre-switch.txt").strip() == "pre-switch"
|
||||
assert host.succeed(f"stat -c %u {salvage}/home/{AGENT}/pre-switch.txt").strip() == uid
|
||||
assert host.succeed(f"stat -c %u {salvage}").strip() == uid
|
||||
# The cache paths stay out; the old root is kept whole.
|
||||
host.fail(f"test -e {salvage}/tmp/cache-file")
|
||||
old_roots = host.succeed(f"ls -d {ROOT}.pre-ephemeral-*").split()
|
||||
assert len(old_roots) == 1, old_roots
|
||||
host.succeed(f"test -f {old_roots[0]}/home/{AGENT}/pre-switch.txt")
|
||||
host.fail(f"test -e {ROOT}/home/{AGENT}/pre-switch.txt")
|
||||
# nix's state moved to the per-container profile bind, where the next
|
||||
# start finds it.
|
||||
host.succeed(f"test -L /nix/var/nix/profiles/per-container/{MACHINE}/per-user/{AGENT}/profiles/profile")
|
||||
|
||||
with subtest("the first ephemeral start, and what survives a restart"):
|
||||
boot()
|
||||
machine_id = in_ct("cat /etc/machine-id")
|
||||
in_ct("sh -c 'journalctl --sync; logger -t ephemeral-test journal-line'")
|
||||
in_ct("sh -c 'journalctl --flush --sync'")
|
||||
as_agent("echo gone > /home/${agent}/ephemeral.txt")
|
||||
in_ct(f"sh -c 'echo kept > /agents/{AGENT}/work/kept.txt'")
|
||||
# The journal is written in the guest, where the forwarder reads it, and
|
||||
# the host reads it too.
|
||||
assert in_ct("ls /var/log/journal") != ""
|
||||
in_ct("journalctl -t ephemeral-test --directory=/var/log/journal | grep journal-line")
|
||||
host.wait_until_succeeds(f"journalctl -M {MACHINE} -t ephemeral-test | grep journal-line")
|
||||
# The package installed before the switch is still on the agent's PATH, and
|
||||
# one installed now is kept with it.
|
||||
as_agent("hello")
|
||||
as_agent(f"nix-env -i {TREE}")
|
||||
|
||||
stop()
|
||||
write_flags()
|
||||
boot()
|
||||
assert in_ct("cat /etc/machine-id") == machine_id
|
||||
in_ct(f"sh -c 'test ! -e /home/{AGENT}/ephemeral.txt'")
|
||||
assert in_ct(f"cat /agents/{AGENT}/work/kept.txt") == "kept"
|
||||
assert host.succeed(f"cat {WORK}/kept.txt").strip() == "kept"
|
||||
as_agent("hello")
|
||||
as_agent("tree --version")
|
||||
|
||||
with subtest("the switch ran once"):
|
||||
salvages = [d for d in salvage_dirs() if d.startswith("salvaged-rootfs-")]
|
||||
assert len(salvages) == 1, salvages
|
||||
host.succeed(f"ls -d {ROOT}.pre-ephemeral-* | wc -l | grep -x 1")
|
||||
assert conf().count("--ephemeral") == 1, conf()
|
||||
'';
|
||||
}
|
||||
Loading…
Reference in a new issue