vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

run opted-in agents in a user namespace with a dedicated data owner

Browse source
This commit is contained in:
atlas 2026-11-01 13:50:53 +01:00 • committed by mara
commit bfdf603d12
1 changed files with 828 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

111
nix/vm-tests/agent-ephemeral-rootfs.nix
View file

@ -18,6 +18,12 @@
# leaves the link's target untouched, and the start goes through once the
# link is gone.
#
# - an agent listed in `services.hyperhive.userns.agents` starts in its own
# user namespace with state it already had, owned by uid 1000 on disk: the
# data moves once to `dataOwner`, the agent user reads and writes it as
# itself, the unlisted agent's conf is untouched, and a second
# `WriteNspawnFlags` moves nothing.
#
# The agent's journal forwarder itself (an OTLP push to the hive's collector) is
# not booted here; the module-eval suite covers its wiring.
{ pkgs, hivePriv }:
@ -29,6 +35,28 @@ let
bridge = "hive0";
gateway = "10.42.0.1";
# An agent in its own user namespace. Its user is pinned to 1000:1000: the
# id map copies the uid map for gids, so its group has to share its uid.
nsAgent = "juno";
nsMachine = "h-${nsAgent}";
nsDir = "/var/lib/hyperhive/agents/${nsAgent}";
nsSocketDir = "/run/hive-agent/${nsAgent}";
dataOwner = "60000";
nsBinds =
map (dir: {
host_path = "${nsDir}/${dir}";
container_path =
if dir == "claude" then "/home/${nsAgent}/.claude" else "/agents/${nsAgent}/${dir}";
read_only = dir == "config";
}) (hostDirs ++ [ "config" ])
++ [
{
host_path = nsSocketDir;
container_path = nsSocketDir;
read_only = false;
}
];
# Host side of what hive-c0re passes `WriteNspawnFlags` for an agent.
binds = map (dir: {
host_path = "${agentDir}/${dir}";
@ -90,7 +118,15 @@ pkgs.testers.runNixOSTest {
prefixLength = 24;
}
];
systemd.tmpfiles.rules = map (dir: "d ${agentDir}/${dir} 0755 root root - -") hostDirs;
services.hyperhive.userns.agents = [ nsAgent ];
systemd.tmpfiles.rules =
map (dir: "d ${agentDir}/${dir} 0755 root root - -") hostDirs
# What the namespaced agent has from before: dirs its user owned.
++ map (dir: "d ${nsDir}/${dir} 0755 1000 1000 - -") hostDirs
++ [
"d ${nsDir}/config 0755 root root - -"
"f ${nsDir}/config/cfg 0644 root root - from-config"
];
environment.systemPackages = [ pkgs.socat ];
containers.${machine} = {
@ -106,6 +142,21 @@ pkgs.testers.runNixOSTest {
services.journald.storage = "persistent";
};
};
containers.${nsMachine} = {
autoStart = false;
privateNetwork = true;
hostBridge = bridge;
config = {
imports = [ ../agent-modules/user.nix ];
system.stateVersion = "25.11";
services.hyperhive.agent.user = {
name = nsAgent;
uid = 1000;
gid = 1000;
};
};
};
};
testScript = ''
@ -260,5 +311,63 @@ pkgs.testers.runNixOSTest {
host.succeed(f"systemctl stop container@{MACHINE}")
host.succeed(f"rm {ROOT}/root && mkdir -m 0700 {ROOT}/root")
boot()
with subtest("a listed agent starts in its own user namespace with the state it had"):
NS = "${nsMachine}"
NS_AGENT = "${nsAgent}"
NS_DIR = "${nsDir}"
NS_BINDS = json.loads(${builtins.toJSON (builtins.toJSON nsBinds)})
host.succeed(f"echo before > {NS_DIR}/state/pre.txt && chown 1000:1000 {NS_DIR}/state/pre.txt")
host.succeed(f"stat -c %u:%g {NS_DIR}/state/pre.txt | grep -x 1000:1000")
priv_request(host, {"op": "ensure_agent_socket_dir", "name": NS_AGENT})
write = {
"op": "write_nspawn_flags",
"container": NS,
"binds": NS_BINDS,
"isolation": {"bridge": "${bridge}", "gateway_ip": "${gateway}"},
}
priv_request(host, write)
ns_conf = host.succeed(f"cat /etc/nixos-containers/{NS}.conf")
assert "PRIVATE_USERS=pick" in ns_conf, ns_conf
assert "--private-users-ownership=map" in ns_conf, ns_conf
# Every bind, the read-only config included, is mounted owneridmap.
assert ns_conf.count(":owneridmap") == len(NS_BINDS), ns_conf
# The unlisted agent's conf has none of it: the control that the lookups
# above find these strings where they are.
assert "owneridmap" not in conf() and "PRIVATE_USERS" not in conf(), conf()
# The data moved to the data owner, the config dir stayed root's.
for sub in ["state", "harness", "work", "claude"]:
host.succeed(f"stat -c %u:%g {NS_DIR}/{sub} | grep -x ${dataOwner}:${dataOwner}")
host.succeed(f"stat -c %u:%g {NS_DIR}/state/pre.txt | grep -x ${dataOwner}:${dataOwner}")
host.succeed(f"stat -c %u:%g {NS_DIR}/config/cfg | grep -x 0:0")
host.succeed(f"stat -c %u:%g ${nsSocketDir} | grep -x ${dataOwner}:${dataOwner}")
boot_ns = lambda: (
host.succeed(f"nixos-container start {NS}"),
host.wait_until_succeeds(f"nixos-container run {NS} -- systemctl is-active multi-user.target"),
)
boot_ns()
def as_ns(script):
return host.succeed(f"nixos-container run {NS} -- runuser -l {NS_AGENT} -c '{script}'").strip()
# Inside, the data is the agent's: read, written and listed as itself.
assert as_ns(f"cat /agents/{NS_AGENT}/state/pre.txt") == "before"
assert as_ns(f"stat -c %u:%g /agents/{NS_AGENT}/state/pre.txt") == "1000:1000"
as_ns(f"echo after > /agents/{NS_AGENT}/state/post.txt")
as_ns(f"touch /run/hive-agent/{NS_AGENT}/web.sock")
assert as_ns(f"cat /agents/{NS_AGENT}/config/cfg") == "from-config"
host.fail(f"nixos-container run {NS} -- runuser -l {NS_AGENT} -c 'touch /agents/{NS_AGENT}/config/new'")
# On disk the new file belongs to the data owner, not to a picked range.
host.succeed(f"stat -c %u:%g {NS_DIR}/state/post.txt | grep -x ${dataOwner}:${dataOwner}")
host.succeed(f"stat -c %u:%g ${nsSocketDir}/web.sock | grep -x ${dataOwner}:${dataOwner}")
host.succeed(f"nixos-container run {NS} -- cat /proc/self/uid_map | grep -v ' 4294967295$'")
# The move runs once: a file a host process leaves owned by 1000 after it
# is not chowned by the next request, and the agent restarts.
host.succeed(f"nixos-container stop {NS}")
host.succeed(f"echo late > {NS_DIR}/state/late.txt && chown 1000:1000 {NS_DIR}/state/late.txt")
priv_request(host, write)
host.succeed(f"stat -c %u:%g {NS_DIR}/state/late.txt | grep -x 1000:1000")
boot_ns()
'';
}