260 lines
11 KiB
Nix
260 lines
11 KiB
Nix
# `vmTests.<system>.agent-ephemeral-rootfs`: an agent container's switch from a
|
|
# persistent root to nspawn's `--ephemeral`, through the real hive-priv unit
|
|
# (nix/host-modules/hive-priv.nix, sandbox included). The container is an
|
|
# agent stand-in: the real agent user module (nix/agent-modules/user.nix) on a
|
|
# plain NixOS container, with the binds hive-c0re passes for an agent.
|
|
#
|
|
# What it pins:
|
|
# - hive-priv's `WriteNspawnFlags` salvages the persistent root before it
|
|
# passes `--ephemeral`, once, and deletes nothing;
|
|
# - after the switch a file in /home is gone at the next start while `work/`
|
|
# keeps its file and the machine id holds;
|
|
# - the guest journal is written inside the container, where the agent's
|
|
# forwarder (nix/agent-modules/otel.nix) reads it, and the host can read it;
|
|
# - a package installed into the user's nix profile is still there, and on
|
|
# PATH, after a restart;
|
|
# - a link planted at `root` in the root template refuses the next start through
|
|
# the `ExecStartPre=` check in the `container@` template drop-in
|
|
# (nix/host-modules/container-rootfs-check.nix), leaves the link's target
|
|
# untouched, and the start goes through once the link is gone.
|
|
#
|
|
# The agent's journal forwarder itself (an OTLP push to the hive's collector) is
|
|
# not booted here; the module-eval suite covers its wiring.
|
|
{ pkgs, hivePriv }:
|
|
let
|
|
agent = "iris";
|
|
machine = "h-${agent}";
|
|
agentDir = "/var/lib/hyperhive/agents/${agent}";
|
|
bridge = "hive0";
|
|
gateway = "10.42.0.1";
|
|
|
|
# Host side of what hive-c0re passes `WriteNspawnFlags` for an agent.
|
|
binds = map (dir: {
|
|
host_path = "${agentDir}/${dir}";
|
|
container_path = if dir == "claude" then "/home/${agent}/.claude" else "/agents/${agent}/${dir}";
|
|
read_only = false;
|
|
}) hostDirs;
|
|
hostDirs = [
|
|
"state"
|
|
"work"
|
|
"harness"
|
|
"claude"
|
|
];
|
|
|
|
# The options hive-priv.nix reads, without the modules that declare them.
|
|
hiveStubs =
|
|
{ lib, ... }:
|
|
{
|
|
options.services.hyperhive = {
|
|
c0re.package = lib.mkOption { type = lib.types.package; };
|
|
deploy.hive-controller.enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = true;
|
|
};
|
|
swarm.forge.domain = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "forge.t.local";
|
|
};
|
|
};
|
|
config.services.hyperhive.c0re.package = hivePriv;
|
|
# hive-priv.socket's group, which hive-c0re's module creates.
|
|
config.users.groups.hive-core = { };
|
|
};
|
|
in
|
|
pkgs.testers.runNixOSTest {
|
|
name = "agent-ephemeral-rootfs";
|
|
|
|
nodes.host =
|
|
{ pkgs, ... }:
|
|
{
|
|
imports = [
|
|
hiveStubs
|
|
../host-modules/container-rootfs-check.nix
|
|
../host-modules/hive-priv.nix
|
|
];
|
|
virtualisation.memorySize = 2048;
|
|
# The script `nix-env -i`s these store paths inside the container, which
|
|
# shares the host's store; the VM has no substituter to fetch them.
|
|
virtualisation.additionalPaths = [
|
|
pkgs.hello
|
|
pkgs.tree
|
|
];
|
|
# The salvage skips when a copy would leave under 15% of the disk free, and
|
|
# this disk cannot clone files; the default 1 GiB leaves too little room.
|
|
virtualisation.diskSize = 4096;
|
|
# The agent bridge hive-priv names in the container's conf.
|
|
networking.bridges.${bridge}.interfaces = [ ];
|
|
networking.interfaces.${bridge}.ipv4.addresses = [
|
|
{
|
|
address = gateway;
|
|
prefixLength = 24;
|
|
}
|
|
];
|
|
systemd.tmpfiles.rules = map (dir: "d ${agentDir}/${dir} 0755 root root - -") hostDirs;
|
|
environment.systemPackages = [ pkgs.socat ];
|
|
|
|
containers.${machine} = {
|
|
autoStart = false;
|
|
privateNetwork = true;
|
|
hostBridge = bridge;
|
|
config = {
|
|
imports = [ ../agent-modules/user.nix ];
|
|
system.stateVersion = "25.11";
|
|
services.hyperhive.agent.user.name = agent;
|
|
# The agent's forwarder reads /var/log/journal, which journald only
|
|
# writes when it stores persistently.
|
|
services.journald.storage = "persistent";
|
|
};
|
|
};
|
|
};
|
|
|
|
testScript = ''
|
|
import json
|
|
|
|
MACHINE = "${machine}"
|
|
AGENT = "${agent}"
|
|
AGENT_DIR = "${agentDir}"
|
|
WORK = AGENT_DIR + "/work"
|
|
ROOT = "/var/lib/nixos-containers/" + MACHINE
|
|
BINDS = json.loads(${builtins.toJSON (builtins.toJSON binds)})
|
|
HELLO = "${pkgs.hello}"
|
|
TREE = "${pkgs.tree}"
|
|
|
|
def in_ct(cmd):
|
|
return host.succeed(f"nixos-container run {MACHINE} -- {cmd}").strip()
|
|
|
|
def as_agent(script):
|
|
# A login shell: HOME and the profile's PATH are the agent's own.
|
|
return in_ct(f"runuser -l {AGENT} -c '{script}'")
|
|
|
|
def boot():
|
|
host.succeed(f"nixos-container start {MACHINE}")
|
|
host.wait_until_succeeds(
|
|
f"nixos-container run {MACHINE} -- systemctl is-active multi-user.target"
|
|
)
|
|
|
|
def stop():
|
|
host.succeed(f"nixos-container stop {MACHINE}")
|
|
|
|
def priv_journal():
|
|
return host.execute("journalctl -u hive-priv --no-pager | tail -n 30")[1]
|
|
|
|
def write_flags():
|
|
"""What hive-c0re does before every start: one request to hive-priv."""
|
|
req = json.dumps({
|
|
"op": "write_nspawn_flags",
|
|
"container": MACHINE,
|
|
"binds": BINDS,
|
|
"isolation": {"bridge": "${bridge}", "gateway_ip": "${gateway}"},
|
|
})
|
|
# socat gives up 0.5s after its stdin ends unless told to wait; the
|
|
# salvage copies a whole root, which takes longer.
|
|
out = host.succeed(f"echo '{req}' | socat -t 120 - UNIX-CONNECT:/run/hive/priv.sock")
|
|
lines = out.strip().splitlines()
|
|
# The journal is read only when the assert fires.
|
|
assert lines, f"hive-priv sent no response; journal:\n{priv_journal()}"
|
|
assert json.loads(lines[-1])["ok"] is True, f"response={out!r}; journal:\n{priv_journal()}"
|
|
|
|
def conf():
|
|
return host.succeed(f"cat /etc/nixos-containers/{MACHINE}.conf")
|
|
|
|
def salvage_dirs():
|
|
return host.succeed(f"ls {WORK}").split()
|
|
|
|
host.start()
|
|
host.wait_for_unit("multi-user.target")
|
|
host.wait_for_unit("hive-priv.socket")
|
|
|
|
with subtest("before the switch: a persistent root"):
|
|
assert "--ephemeral" not in conf()
|
|
boot()
|
|
uid = in_ct(f"id -u {AGENT}")
|
|
as_agent("echo pre-switch > /home/${agent}/pre-switch.txt")
|
|
in_ct("sh -c 'echo scratch > /tmp/cache-file'")
|
|
# A root from before the agent module redirected nix's state: the real
|
|
# directory, holding a package.
|
|
in_ct(f"rm /home/{AGENT}/.local/state/nix")
|
|
as_agent(f"nix-env -i {HELLO}")
|
|
in_ct(f"test -d /home/{AGENT}/.local/state/nix/profiles")
|
|
stop()
|
|
host.succeed(f"test -f {ROOT}/home/{AGENT}/pre-switch.txt")
|
|
|
|
with subtest("WriteNspawnFlags salvages the old root, then asks for --ephemeral"):
|
|
assert salvage_dirs() == []
|
|
write_flags()
|
|
assert "--ephemeral" in conf(), conf()
|
|
dirs = salvage_dirs()
|
|
assert len(dirs) == 1 and dirs[0].startswith("salvaged-rootfs-"), dirs
|
|
salvage = f"{WORK}/{dirs[0]}"
|
|
assert host.succeed(f"cat {salvage}/home/{AGENT}/pre-switch.txt").strip() == "pre-switch"
|
|
assert host.succeed(f"stat -c %u {salvage}/home/{AGENT}/pre-switch.txt").strip() == uid
|
|
assert host.succeed(f"stat -c %u {salvage}").strip() == uid
|
|
# The cache paths stay out; the old root is kept whole.
|
|
host.fail(f"test -e {salvage}/tmp/cache-file")
|
|
old_roots = host.succeed(f"ls -d {ROOT}.pre-ephemeral-*").split()
|
|
assert len(old_roots) == 1, old_roots
|
|
host.succeed(f"test -f {old_roots[0]}/home/{AGENT}/pre-switch.txt")
|
|
host.fail(f"test -e {ROOT}/home/{AGENT}/pre-switch.txt")
|
|
# nix's state moved to the per-container profile bind, where the next
|
|
# start finds it.
|
|
host.succeed(f"test -L /nix/var/nix/profiles/per-container/{MACHINE}/per-user/{AGENT}/profiles/profile")
|
|
|
|
with subtest("the first ephemeral start, and what survives a restart"):
|
|
boot()
|
|
machine_id = in_ct("cat /etc/machine-id")
|
|
in_ct("sh -c 'journalctl --sync; logger -t ephemeral-test journal-line'")
|
|
in_ct("sh -c 'journalctl --flush --sync'")
|
|
as_agent("echo gone > /home/${agent}/ephemeral.txt")
|
|
in_ct(f"sh -c 'echo kept > /agents/{AGENT}/work/kept.txt'")
|
|
# The journal is written in the guest, where the forwarder reads it, and
|
|
# the host reads it too.
|
|
assert in_ct("ls /var/log/journal") != ""
|
|
in_ct("journalctl -t ephemeral-test --directory=/var/log/journal | grep journal-line")
|
|
host.wait_until_succeeds(f"journalctl -M {MACHINE} -t ephemeral-test | grep journal-line")
|
|
# The package installed before the switch is still on the agent's PATH, and
|
|
# one installed now is kept with it.
|
|
as_agent("hello")
|
|
as_agent(f"nix-env -i {TREE}")
|
|
|
|
stop()
|
|
write_flags()
|
|
boot()
|
|
assert in_ct("cat /etc/machine-id") == machine_id
|
|
in_ct(f"sh -c 'test ! -e /home/{AGENT}/ephemeral.txt'")
|
|
assert in_ct(f"cat /agents/{AGENT}/work/kept.txt") == "kept"
|
|
assert host.succeed(f"cat {WORK}/kept.txt").strip() == "kept"
|
|
as_agent("hello")
|
|
as_agent("tree --version")
|
|
|
|
with subtest("the switch ran once"):
|
|
salvages = [d for d in salvage_dirs() if d.startswith("salvaged-rootfs-")]
|
|
assert len(salvages) == 1, salvages
|
|
host.succeed(f"ls -d {ROOT}.pre-ephemeral-* | wc -l | grep -x 1")
|
|
assert conf().count("--ephemeral") == 1, conf()
|
|
|
|
with subtest("a link planted in the root refuses the start and nothing is touched"):
|
|
stop()
|
|
# The pre-start check is a drop-in of the `container@` template, so the
|
|
# unit carries it with no per-container drop-in written.
|
|
unit = host.succeed(f"systemctl cat container@{MACHINE}")
|
|
assert "bin/hive-priv check-rootfs %i\n" in unit, unit
|
|
|
|
# `root` is a link to a directory outside the root, which the start
|
|
# script would `chmod 0700` as host root.
|
|
victim = "/var/lib/victim"
|
|
host.succeed(f"mkdir -m 0755 {victim}")
|
|
host.succeed(f"rm -rf {ROOT}/root && ln -s {victim} {ROOT}/root")
|
|
host.fail(f"nixos-container start {MACHINE}")
|
|
host.succeed(
|
|
f"journalctl -u container@{MACHINE} | grep 'check-rootfs: refusing to start: {ROOT}/root is a symlink'"
|
|
)
|
|
assert host.succeed(f"stat -c %a {victim}").strip() == "755"
|
|
host.succeed(f"test -L {ROOT}/root")
|
|
|
|
# Control: with the link gone the same unit starts.
|
|
host.succeed(f"systemctl stop container@{MACHINE}")
|
|
host.succeed(f"systemctl reset-failed container@{MACHINE}")
|
|
host.succeed(f"rm {ROOT}/root && mkdir -m 0700 {ROOT}/root")
|
|
boot()
|
|
'';
|
|
}
|