376 lines
16 KiB
Nix
376 lines
16 KiB
Nix
# `vmTests.<system>.agent-ephemeral-rootfs`: an agent container's switch from a
|
|
# persistent root to nspawn's `--ephemeral`, through the real hive-priv unit
|
|
# (nix/host-modules/hive-priv.nix, sandbox included). The container is an
|
|
# agent stand-in: the real agent user module (nix/agent-modules/user.nix) on a
|
|
# plain NixOS container, with the binds hive-c0re passes for an agent.
|
|
#
|
|
# What it pins:
|
|
# - hive-priv's `WriteNspawnFlags` salvages the persistent root before it
|
|
# passes `--ephemeral`, once, and deletes nothing;
|
|
# - after the switch a file in /home is gone at the next start while `work/`
|
|
# keeps its file and the machine id holds;
|
|
# - the guest journal is written inside the container, where the agent's
|
|
# forwarder (nix/agent-modules/otel.nix) reads it, and the host can read it;
|
|
# - a package installed into the user's nix profile is still there, and on
|
|
# PATH, after a restart;
|
|
# - a link planted at `root` in the template refuses the next start through
|
|
# the limits drop-in's `ExecStartPre=` check (whose build has a GC root),
|
|
# leaves the link's target untouched, and the start goes through once the
|
|
# link is gone.
|
|
#
|
|
# - an agent whose config sets `services.hyperhive.agent.user.privateUsers`
|
|
# (hive-c0re sends the flag from the closure record; the test sends it)
|
|
# starts in its own
|
|
# user namespace with state it already had, owned by uid 1000 on disk: the
|
|
# data moves once to `dataOwner`, the agent user reads and writes it as
|
|
# itself, the other agent's conf (no flag sent) is untouched, and a second
|
|
# `WriteNspawnFlags` moves nothing.
|
|
#
|
|
# The agent's journal forwarder itself (an OTLP push to the hive's collector) is
|
|
# not booted here; the module-eval suite covers its wiring.
|
|
{ pkgs, hivePriv }:
|
|
let
|
|
privRequest = import ./priv-request.nix { inherit pkgs; };
|
|
agent = "iris";
|
|
machine = "h-${agent}";
|
|
agentDir = "/var/lib/hyperhive/agents/${agent}";
|
|
bridge = "hive0";
|
|
gateway = "10.42.0.1";
|
|
|
|
# An agent in its own user namespace. Its user is pinned to 1000:1000: the
|
|
# id map copies the uid map for gids, so its group has to share its uid.
|
|
nsAgent = "juno";
|
|
nsMachine = "h-${nsAgent}";
|
|
nsDir = "/var/lib/hyperhive/agents/${nsAgent}";
|
|
nsSocketDir = "/run/hive-agent/${nsAgent}";
|
|
dataOwner = "60000";
|
|
nsBinds =
|
|
map (dir: {
|
|
host_path = "${nsDir}/${dir}";
|
|
container_path =
|
|
if dir == "claude" then "/home/${nsAgent}/.claude" else "/agents/${nsAgent}/${dir}";
|
|
read_only = dir == "config";
|
|
}) (hostDirs ++ [ "config" ])
|
|
++ [
|
|
{
|
|
host_path = nsSocketDir;
|
|
container_path = nsSocketDir;
|
|
read_only = false;
|
|
}
|
|
];
|
|
|
|
# Host side of what hive-c0re passes `WriteNspawnFlags` for an agent.
|
|
binds = map (dir: {
|
|
host_path = "${agentDir}/${dir}";
|
|
container_path = if dir == "claude" then "/home/${agent}/.claude" else "/agents/${agent}/${dir}";
|
|
read_only = false;
|
|
}) hostDirs;
|
|
hostDirs = [
|
|
"state"
|
|
"work"
|
|
"harness"
|
|
"claude"
|
|
];
|
|
|
|
# The options hive-priv.nix reads, without the modules that declare them.
|
|
hiveStubs =
|
|
{ lib, ... }:
|
|
{
|
|
options.services.hyperhive = {
|
|
deploy.hive-controller.package = lib.mkOption { type = lib.types.package; };
|
|
deploy.hive-controller.enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = true;
|
|
};
|
|
swarm.forge.domain = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "forge.t.local";
|
|
};
|
|
};
|
|
config.services.hyperhive.deploy.hive-controller.package = hivePriv;
|
|
# hive-priv.socket's group, which hive-c0re's module creates.
|
|
config.users.groups.hive-core = { };
|
|
};
|
|
in
|
|
pkgs.testers.runNixOSTest {
|
|
name = "agent-ephemeral-rootfs";
|
|
|
|
nodes.host =
|
|
{ pkgs, ... }:
|
|
{
|
|
imports = [
|
|
hiveStubs
|
|
../host-modules/hive-priv.nix
|
|
];
|
|
virtualisation.memorySize = 2048;
|
|
# The script `nix-env -i`s these store paths inside the container, which
|
|
# shares the host's store; the VM has no substituter to fetch them.
|
|
virtualisation.additionalPaths = [
|
|
pkgs.hello
|
|
pkgs.tree
|
|
];
|
|
# The salvage skips when a copy would leave under 15% of the disk free, and
|
|
# this disk cannot clone files; the default 1 GiB leaves too little room.
|
|
virtualisation.diskSize = 4096;
|
|
# The agent bridge hive-priv names in the container's conf.
|
|
networking.bridges.${bridge}.interfaces = [ ];
|
|
networking.interfaces.${bridge}.ipv4.addresses = [
|
|
{
|
|
address = gateway;
|
|
prefixLength = 24;
|
|
}
|
|
];
|
|
systemd.tmpfiles.rules =
|
|
map (dir: "d ${agentDir}/${dir} 0755 root root - -") hostDirs
|
|
# What the namespaced agent has from before: dirs its user owned.
|
|
++ map (dir: "d ${nsDir}/${dir} 0755 1000 1000 - -") hostDirs
|
|
++ [
|
|
"d ${nsDir}/config 0755 root root - -"
|
|
"f ${nsDir}/config/cfg 0644 root root - from-config"
|
|
];
|
|
environment.systemPackages = [ pkgs.socat ];
|
|
|
|
containers.${machine} = {
|
|
autoStart = false;
|
|
privateNetwork = true;
|
|
hostBridge = bridge;
|
|
config = {
|
|
imports = [ ../agent-modules/user.nix ];
|
|
system.stateVersion = "25.11";
|
|
services.hyperhive.agent.user.name = agent;
|
|
# The agent's forwarder reads /var/log/journal, which journald only
|
|
# writes when it stores persistently.
|
|
services.journald.storage = "persistent";
|
|
};
|
|
};
|
|
|
|
containers.${nsMachine} = {
|
|
autoStart = false;
|
|
privateNetwork = true;
|
|
hostBridge = bridge;
|
|
config = {
|
|
imports = [ ../agent-modules/user.nix ];
|
|
system.stateVersion = "25.11";
|
|
services.hyperhive.agent.user = {
|
|
name = nsAgent;
|
|
uid = 1000;
|
|
gid = 1000;
|
|
privateUsers = true;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
testScript = ''
|
|
import json
|
|
|
|
${privRequest}
|
|
|
|
MACHINE = "${machine}"
|
|
AGENT = "${agent}"
|
|
AGENT_DIR = "${agentDir}"
|
|
WORK = AGENT_DIR + "/work"
|
|
ROOT = "/var/lib/nixos-containers/" + MACHINE
|
|
BINDS = json.loads(${builtins.toJSON (builtins.toJSON binds)})
|
|
HELLO = "${pkgs.hello}"
|
|
TREE = "${pkgs.tree}"
|
|
|
|
def in_ct(cmd):
|
|
return host.succeed(f"nixos-container run {MACHINE} -- {cmd}").strip()
|
|
|
|
def as_agent(script):
|
|
# A login shell: HOME and the profile's PATH are the agent's own.
|
|
return in_ct(f"runuser -l {AGENT} -c '{script}'")
|
|
|
|
def boot():
|
|
host.succeed(f"nixos-container start {MACHINE}")
|
|
host.wait_until_succeeds(
|
|
f"nixos-container run {MACHINE} -- systemctl is-active multi-user.target"
|
|
)
|
|
|
|
def stop():
|
|
host.succeed(f"nixos-container stop {MACHINE}")
|
|
|
|
def write_flags():
|
|
"""What hive-c0re does before every start: one request to hive-priv."""
|
|
priv_request(host, {
|
|
"op": "write_nspawn_flags",
|
|
"container": MACHINE,
|
|
"binds": BINDS,
|
|
"isolation": {"bridge": "${bridge}", "gateway_ip": "${gateway}"},
|
|
})
|
|
|
|
def conf():
|
|
return host.succeed(f"cat /etc/nixos-containers/{MACHINE}.conf")
|
|
|
|
def salvage_dirs():
|
|
return host.succeed(f"ls {WORK}").split()
|
|
|
|
host.start()
|
|
host.wait_for_unit("multi-user.target")
|
|
host.wait_for_unit("hive-priv.socket")
|
|
|
|
with subtest("before the switch: a persistent root"):
|
|
assert "--ephemeral" not in conf()
|
|
boot()
|
|
uid = in_ct(f"id -u {AGENT}")
|
|
as_agent("echo pre-switch > /home/${agent}/pre-switch.txt")
|
|
in_ct("sh -c 'echo scratch > /tmp/cache-file'")
|
|
# A root from before the agent module redirected nix's state: the real
|
|
# directory, holding a package.
|
|
in_ct(f"rm /home/{AGENT}/.local/state/nix")
|
|
as_agent(f"nix-env -i {HELLO}")
|
|
in_ct(f"test -d /home/{AGENT}/.local/state/nix/profiles")
|
|
stop()
|
|
host.succeed(f"test -f {ROOT}/home/{AGENT}/pre-switch.txt")
|
|
|
|
with subtest("WriteNspawnFlags salvages the old root, then asks for --ephemeral"):
|
|
assert salvage_dirs() == []
|
|
write_flags()
|
|
assert "--ephemeral" in conf(), conf()
|
|
dirs = salvage_dirs()
|
|
assert len(dirs) == 1 and dirs[0].startswith("salvaged-rootfs-"), dirs
|
|
salvage = f"{WORK}/{dirs[0]}"
|
|
assert host.succeed(f"cat {salvage}/home/{AGENT}/pre-switch.txt").strip() == "pre-switch"
|
|
assert host.succeed(f"stat -c %u {salvage}/home/{AGENT}/pre-switch.txt").strip() == uid
|
|
assert host.succeed(f"stat -c %u {salvage}").strip() == uid
|
|
# The cache paths stay out; the old root is kept whole.
|
|
host.fail(f"test -e {salvage}/tmp/cache-file")
|
|
old_roots = host.succeed(f"ls -d {ROOT}.pre-ephemeral-*").split()
|
|
assert len(old_roots) == 1, old_roots
|
|
host.succeed(f"test -f {old_roots[0]}/home/{AGENT}/pre-switch.txt")
|
|
host.fail(f"test -e {ROOT}/home/{AGENT}/pre-switch.txt")
|
|
# nix's state moved to the per-container profile bind, where the next
|
|
# start finds it.
|
|
host.succeed(f"test -L /nix/var/nix/profiles/per-container/{MACHINE}/per-user/{AGENT}/profiles/profile")
|
|
|
|
with subtest("the first ephemeral start, and what survives a restart"):
|
|
boot()
|
|
machine_id = in_ct("cat /etc/machine-id")
|
|
in_ct("sh -c 'journalctl --sync; logger -t ephemeral-test journal-line'")
|
|
in_ct("sh -c 'journalctl --flush --sync'")
|
|
as_agent("echo gone > /home/${agent}/ephemeral.txt")
|
|
in_ct(f"sh -c 'echo kept > /agents/{AGENT}/work/kept.txt'")
|
|
# The journal is written in the guest, where the forwarder reads it, and
|
|
# the host reads it too.
|
|
assert in_ct("ls /var/log/journal") != ""
|
|
in_ct("journalctl -t ephemeral-test --directory=/var/log/journal | grep journal-line")
|
|
host.wait_until_succeeds(f"journalctl -M {MACHINE} -t ephemeral-test | grep journal-line")
|
|
# The package installed before the switch is still on the agent's PATH, and
|
|
# one installed now is kept with it.
|
|
as_agent("hello")
|
|
as_agent(f"nix-env -i {TREE}")
|
|
|
|
stop()
|
|
write_flags()
|
|
boot()
|
|
assert in_ct("cat /etc/machine-id") == machine_id
|
|
in_ct(f"sh -c 'test ! -e /home/{AGENT}/ephemeral.txt'")
|
|
assert in_ct(f"cat /agents/{AGENT}/work/kept.txt") == "kept"
|
|
assert host.succeed(f"cat {WORK}/kept.txt").strip() == "kept"
|
|
as_agent("hello")
|
|
as_agent("tree --version")
|
|
|
|
with subtest("the switch ran once"):
|
|
salvages = [d for d in salvage_dirs() if d.startswith("salvaged-rootfs-")]
|
|
assert len(salvages) == 1, salvages
|
|
host.succeed(f"ls -d {ROOT}.pre-ephemeral-* | wc -l | grep -x 1")
|
|
assert conf().count("--ephemeral") == 1, conf()
|
|
|
|
with subtest("a link planted in the root refuses the start and nothing is touched"):
|
|
stop()
|
|
# What hive-c0re does besides WriteNspawnFlags: the limits drop-in,
|
|
# which carries the pre-start check, and the runtime dir its condition
|
|
# names.
|
|
host.succeed(f"mkdir -p /run/hyperhive/agents/{AGENT}")
|
|
for req in [
|
|
{"op": "write_resource_limits", "container": MACHINE, "memory_max": "1G", "cpu_quota": "200%"},
|
|
{"op": "daemon_reload"},
|
|
]:
|
|
priv_request(host, req)
|
|
dropin = host.succeed(f"cat /run/systemd/system/container@{MACHINE}.service.d/hyperhive-limits.conf")
|
|
assert "ExecStartPre=/nix/store/" in dropin and f"check-rootfs {MACHINE}\n" in dropin, dropin
|
|
# The build the drop-in names is rooted against garbage collection.
|
|
build = "/".join(dropin.split("ExecStartPre=")[1].split()[0].split("/")[:4])
|
|
assert host.succeed(f"readlink /nix/var/nix/gcroots/hyperhive-check-rootfs/{MACHINE}").strip() == build, build
|
|
|
|
# `root` is a link to a directory outside the root, which the start
|
|
# script would `chmod 0700` as host root.
|
|
victim = "/var/lib/victim"
|
|
host.succeed(f"mkdir -m 0755 {victim}")
|
|
host.succeed(f"rm -rf {ROOT}/root && ln -s {victim} {ROOT}/root")
|
|
host.fail(f"nixos-container start {MACHINE}")
|
|
host.succeed(
|
|
f"journalctl -u container@{MACHINE} | grep 'check-rootfs: refusing to start: {ROOT}/root is a symlink'"
|
|
)
|
|
assert host.succeed(f"stat -c %a {victim}").strip() == "755"
|
|
host.succeed(f"test -L {ROOT}/root")
|
|
|
|
# Control: with the link gone the same unit starts.
|
|
# The reset comes first: once the failed unit is stopped, systemd
|
|
# unloads it and reset-failed exits 1.
|
|
host.succeed(f"systemctl reset-failed container@{MACHINE}")
|
|
host.succeed(f"systemctl stop container@{MACHINE}")
|
|
host.succeed(f"rm {ROOT}/root && mkdir -m 0700 {ROOT}/root")
|
|
boot()
|
|
|
|
with subtest("an agent whose config sets privateUsers starts in its own user namespace with the state it had"):
|
|
NS = "${nsMachine}"
|
|
NS_AGENT = "${nsAgent}"
|
|
NS_DIR = "${nsDir}"
|
|
NS_BINDS = json.loads(${builtins.toJSON (builtins.toJSON nsBinds)})
|
|
host.succeed(f"echo before > {NS_DIR}/state/pre.txt && chown 1000:1000 {NS_DIR}/state/pre.txt")
|
|
host.succeed(f"stat -c %u:%g {NS_DIR}/state/pre.txt | grep -x 1000:1000")
|
|
priv_request(host, {"op": "ensure_agent_socket_dir", "name": NS_AGENT, "private_users": True})
|
|
write = {
|
|
"op": "write_nspawn_flags",
|
|
"container": NS,
|
|
"binds": NS_BINDS,
|
|
"isolation": {"bridge": "${bridge}", "gateway_ip": "${gateway}"},
|
|
"private_users": True,
|
|
}
|
|
priv_request(host, write)
|
|
ns_conf = host.succeed(f"cat /etc/nixos-containers/{NS}.conf")
|
|
assert "PRIVATE_USERS=pick" in ns_conf, ns_conf
|
|
assert "--private-users-ownership=map" in ns_conf, ns_conf
|
|
# Every bind, the read-only config included, is mounted owneridmap.
|
|
assert ns_conf.count(":owneridmap") == len(NS_BINDS), ns_conf
|
|
# The other agent's conf has none of it: the control that the lookups
|
|
# above find these strings where they are.
|
|
assert "owneridmap" not in conf() and "PRIVATE_USERS" not in conf(), conf()
|
|
|
|
# The data moved to the data owner, the config dir stayed root's.
|
|
for sub in ["state", "harness", "work", "claude"]:
|
|
host.succeed(f"stat -c %u:%g {NS_DIR}/{sub} | grep -x ${dataOwner}:${dataOwner}")
|
|
host.succeed(f"stat -c %u:%g {NS_DIR}/state/pre.txt | grep -x ${dataOwner}:${dataOwner}")
|
|
host.succeed(f"stat -c %u:%g {NS_DIR}/config/cfg | grep -x 0:0")
|
|
host.succeed(f"stat -c %u:%g ${nsSocketDir} | grep -x ${dataOwner}:${dataOwner}")
|
|
|
|
boot_ns = lambda: (
|
|
host.succeed(f"nixos-container start {NS}"),
|
|
host.wait_until_succeeds(f"nixos-container run {NS} -- systemctl is-active multi-user.target"),
|
|
)
|
|
boot_ns()
|
|
def as_ns(script):
|
|
return host.succeed(f"nixos-container run {NS} -- runuser -l {NS_AGENT} -c '{script}'").strip()
|
|
# Inside, the data is the agent's: read, written and listed as itself.
|
|
assert as_ns(f"cat /agents/{NS_AGENT}/state/pre.txt") == "before"
|
|
assert as_ns(f"stat -c %u:%g /agents/{NS_AGENT}/state/pre.txt") == "1000:1000"
|
|
as_ns(f"echo after > /agents/{NS_AGENT}/state/post.txt")
|
|
as_ns(f"touch /run/hive-agent/{NS_AGENT}/web.sock")
|
|
assert as_ns(f"cat /agents/{NS_AGENT}/config/cfg") == "from-config"
|
|
host.fail(f"nixos-container run {NS} -- runuser -l {NS_AGENT} -c 'touch /agents/{NS_AGENT}/config/new'")
|
|
# On disk the new file belongs to the data owner, not to a picked range.
|
|
host.succeed(f"stat -c %u:%g {NS_DIR}/state/post.txt | grep -x ${dataOwner}:${dataOwner}")
|
|
host.succeed(f"stat -c %u:%g ${nsSocketDir}/web.sock | grep -x ${dataOwner}:${dataOwner}")
|
|
host.succeed(f"nixos-container run {NS} -- cat /proc/self/uid_map | grep -v ' 4294967295$'")
|
|
|
|
# The move runs once: a file a host process leaves owned by 1000 after it
|
|
# is not chowned by the next request, and the agent restarts.
|
|
host.succeed(f"nixos-container stop {NS}")
|
|
host.succeed(f"echo late > {NS_DIR}/state/late.txt && chown 1000:1000 {NS_DIR}/state/late.txt")
|
|
priv_request(host, write)
|
|
host.succeed(f"stat -c %u:%g {NS_DIR}/state/late.txt | grep -x 1000:1000")
|
|
boot_ns()
|
|
'';
|
|
}
|