run opted-in agents in a user namespace with a dedicated data owner
This commit is contained in:
atlas
mara
parent
5ae0205e1d
commit
bfdf603d12
4 changed files with 828 additions and 21 deletions
|
|
@ -10,6 +10,8 @@
|
|||
let
|
||||
controllerCfg = config.services.hyperhive.deploy.hive-controller;
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
userns = config.services.hyperhive.userns;
|
||||
dataOwnerProblems = import ./lib/userns-data-owner.nix { inherit lib; };
|
||||
in
|
||||
{
|
||||
# The platform containers run their warn-only root check by this path
|
||||
|
|
@ -24,12 +26,60 @@ in
|
|||
}
|
||||
];
|
||||
|
||||
options.services.hyperhive.userns = {
|
||||
agents = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
example = [ "alice" ];
|
||||
description = ''
|
||||
Agents that run in their own user namespace (`--private-users=pick`,
|
||||
root idmapped). Every hive bind into such a container is mounted
|
||||
`owneridmap`, so its data on disk belongs to `dataOwner`, which the
|
||||
agent user is inside. Listing an agent moves its `state`, `harness`,
|
||||
`work` and `claude` dirs to `dataOwner` once, at its next start with
|
||||
the container stopped. Take the agent off the list only after chowning
|
||||
those dirs back and removing `/var/lib/hive-priv/userns/<agent>`;
|
||||
until then it will not start.
|
||||
'';
|
||||
};
|
||||
|
||||
dataOwner = lib.mkOption {
|
||||
type = lib.types.ints.u32;
|
||||
default = 60000;
|
||||
description = ''
|
||||
The host uid and gid that own a namespaced agent's data on disk. One
|
||||
number for both: nspawn copies the uid map for gids, so the gid
|
||||
cannot differ. It must be no host user or group, outside the normal and
|
||||
system users (below 30000) and the dynamic users (61184 and above), and
|
||||
below the range `pick` allocates from (524288 and above), so no
|
||||
container's range covers it. Changing it after an agent has moved is
|
||||
refused at that agent's next start.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf config.services.hyperhive.deploy.hive-controller.enable {
|
||||
# The parent of every agent socket dir. hive-priv is its only writer
|
||||
# (`EnsureAgentSocketDir`), and its unit lists it in `ReadWritePaths`,
|
||||
# which fails the unit when the path is missing.
|
||||
systemd.tmpfiles.rules = [ "d /run/hive-agent 0755 root root - -" ];
|
||||
|
||||
assertions = [
|
||||
(
|
||||
let
|
||||
problems = dataOwnerProblems {
|
||||
owner = userns.dataOwner;
|
||||
uids = lib.filter (uid: uid != null) (lib.mapAttrsToList (_: u: u.uid) config.users.users);
|
||||
gids = lib.filter (gid: gid != null) (lib.mapAttrsToList (_: g: g.gid) config.users.groups);
|
||||
};
|
||||
in
|
||||
{
|
||||
assertion = userns.agents == [ ] || problems == [ ];
|
||||
message = "services.hyperhive.userns.dataOwner: ${lib.concatStringsSep "; " problems}";
|
||||
}
|
||||
)
|
||||
];
|
||||
|
||||
# Socket unit for hive-priv — the narrow root helper that executes
|
||||
# privileged operations on behalf of hive-c0re. Systemd creates and
|
||||
# holds `/run/hive/priv.sock` before the first connection arrives.
|
||||
|
|
@ -93,6 +143,10 @@ in
|
|||
# routing so the closure `RealiseClosure` downloads is in the same
|
||||
# store the container is then pointed at.
|
||||
NIX_REMOTE = "daemon";
|
||||
# Who is namespaced, and the owner their data moves to: set by the
|
||||
# operator in the unit, so only root decides.
|
||||
HIVE_PRIV_USERNS_AGENTS = lib.concatStringsSep " " userns.agents;
|
||||
HIVE_PRIV_USERNS_DATA_OWNER = toString userns.dataOwner;
|
||||
};
|
||||
serviceConfig = {
|
||||
ExecStart = "${controllerCfg.package}/bin/hive-priv";
|
||||
|
|
|
|||
17
nix/host-modules/lib/userns-data-owner.nix
Normal file
17
nix/host-modules/lib/userns-data-owner.nix
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
# What is wrong with a uid:gid chosen as the on-disk owner of a user-namespaced
|
||||
# agent's data. Empty when it is fine.
|
||||
{ lib }:
|
||||
{
|
||||
owner,
|
||||
# Every uid and gid the host defines, from `users.users` and `users.groups`.
|
||||
uids,
|
||||
gids,
|
||||
}:
|
||||
lib.optional (
|
||||
owner < 30000
|
||||
) "${toString owner} is in the range NixOS gives normal users and system users (below 30000)"
|
||||
++
|
||||
lib.optional (owner >= 61184)
|
||||
"${toString owner} is in the range systemd allocates dynamic users from (61184 and above), or is `nobody`"
|
||||
++ lib.optional (builtins.elem owner uids) "${toString owner} is the uid of a user the host defines"
|
||||
++ lib.optional (builtins.elem owner gids) "${toString owner} is the gid of a group the host defines"
|
||||
|
|
@ -137,6 +137,58 @@ let
|
|||
remainsAfterExit = u: u.serviceConfig.RemainAfterExit or false;
|
||||
|
||||
cases = [
|
||||
{
|
||||
# The control is the default on a host that defines the ids below:
|
||||
# it gets past every arm, so each refusal is the arm's own.
|
||||
name = "the userns data owner is refused where it collides with a host id or a reserved range";
|
||||
ok =
|
||||
let
|
||||
problems =
|
||||
owner:
|
||||
import ../host-modules/lib/userns-data-owner.nix { inherit lib; } {
|
||||
inherit owner;
|
||||
uids = [
|
||||
0
|
||||
60001
|
||||
];
|
||||
gids = [
|
||||
100
|
||||
60002
|
||||
];
|
||||
};
|
||||
in
|
||||
problems 60000 == [ ]
|
||||
&&
|
||||
map (o: builtins.length (problems o)) [
|
||||
1000
|
||||
29999
|
||||
61184
|
||||
65534
|
||||
60001
|
||||
60002
|
||||
] == [
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
];
|
||||
}
|
||||
{
|
||||
# Read back from the rendered unit and assertion list: a misspelt
|
||||
# variable or a dropped assertion fails here.
|
||||
name = "hive-priv carries the userns agents and data owner, and the owner assertion passes";
|
||||
ok =
|
||||
let
|
||||
e = allLocal.systemd.services.hive-priv.environment;
|
||||
in
|
||||
e.HIVE_PRIV_USERNS_AGENTS == ""
|
||||
&& e.HIVE_PRIV_USERNS_DATA_OWNER == "60000"
|
||||
&& lib.any (
|
||||
a: lib.hasPrefix "services.hyperhive.userns.dataOwner" a.message && a.assertion
|
||||
) allLocal.assertions;
|
||||
}
|
||||
{
|
||||
# The domain is the stub's swarm domain, which both fixtures share.
|
||||
name = "the forge's publicUrl default does not consult the central toggle";
|
||||
|
|
|
|||
|
|
@ -18,6 +18,12 @@
|
|||
# leaves the link's target untouched, and the start goes through once the
|
||||
# link is gone.
|
||||
#
|
||||
# - an agent listed in `services.hyperhive.userns.agents` starts in its own
|
||||
# user namespace with state it already had, owned by uid 1000 on disk: the
|
||||
# data moves once to `dataOwner`, the agent user reads and writes it as
|
||||
# itself, the unlisted agent's conf is untouched, and a second
|
||||
# `WriteNspawnFlags` moves nothing.
|
||||
#
|
||||
# The agent's journal forwarder itself (an OTLP push to the hive's collector) is
|
||||
# not booted here; the module-eval suite covers its wiring.
|
||||
{ pkgs, hivePriv }:
|
||||
|
|
@ -29,6 +35,28 @@ let
|
|||
bridge = "hive0";
|
||||
gateway = "10.42.0.1";
|
||||
|
||||
# An agent in its own user namespace. Its user is pinned to 1000:1000: the
|
||||
# id map copies the uid map for gids, so its group has to share its uid.
|
||||
nsAgent = "juno";
|
||||
nsMachine = "h-${nsAgent}";
|
||||
nsDir = "/var/lib/hyperhive/agents/${nsAgent}";
|
||||
nsSocketDir = "/run/hive-agent/${nsAgent}";
|
||||
dataOwner = "60000";
|
||||
nsBinds =
|
||||
map (dir: {
|
||||
host_path = "${nsDir}/${dir}";
|
||||
container_path =
|
||||
if dir == "claude" then "/home/${nsAgent}/.claude" else "/agents/${nsAgent}/${dir}";
|
||||
read_only = dir == "config";
|
||||
}) (hostDirs ++ [ "config" ])
|
||||
++ [
|
||||
{
|
||||
host_path = nsSocketDir;
|
||||
container_path = nsSocketDir;
|
||||
read_only = false;
|
||||
}
|
||||
];
|
||||
|
||||
# Host side of what hive-c0re passes `WriteNspawnFlags` for an agent.
|
||||
binds = map (dir: {
|
||||
host_path = "${agentDir}/${dir}";
|
||||
|
|
@ -90,7 +118,15 @@ pkgs.testers.runNixOSTest {
|
|||
prefixLength = 24;
|
||||
}
|
||||
];
|
||||
systemd.tmpfiles.rules = map (dir: "d ${agentDir}/${dir} 0755 root root - -") hostDirs;
|
||||
services.hyperhive.userns.agents = [ nsAgent ];
|
||||
systemd.tmpfiles.rules =
|
||||
map (dir: "d ${agentDir}/${dir} 0755 root root - -") hostDirs
|
||||
# What the namespaced agent has from before: dirs its user owned.
|
||||
++ map (dir: "d ${nsDir}/${dir} 0755 1000 1000 - -") hostDirs
|
||||
++ [
|
||||
"d ${nsDir}/config 0755 root root - -"
|
||||
"f ${nsDir}/config/cfg 0644 root root - from-config"
|
||||
];
|
||||
environment.systemPackages = [ pkgs.socat ];
|
||||
|
||||
containers.${machine} = {
|
||||
|
|
@ -106,6 +142,21 @@ pkgs.testers.runNixOSTest {
|
|||
services.journald.storage = "persistent";
|
||||
};
|
||||
};
|
||||
|
||||
containers.${nsMachine} = {
|
||||
autoStart = false;
|
||||
privateNetwork = true;
|
||||
hostBridge = bridge;
|
||||
config = {
|
||||
imports = [ ../agent-modules/user.nix ];
|
||||
system.stateVersion = "25.11";
|
||||
services.hyperhive.agent.user = {
|
||||
name = nsAgent;
|
||||
uid = 1000;
|
||||
gid = 1000;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
|
|
@ -260,5 +311,63 @@ pkgs.testers.runNixOSTest {
|
|||
host.succeed(f"systemctl stop container@{MACHINE}")
|
||||
host.succeed(f"rm {ROOT}/root && mkdir -m 0700 {ROOT}/root")
|
||||
boot()
|
||||
|
||||
with subtest("a listed agent starts in its own user namespace with the state it had"):
|
||||
NS = "${nsMachine}"
|
||||
NS_AGENT = "${nsAgent}"
|
||||
NS_DIR = "${nsDir}"
|
||||
NS_BINDS = json.loads(${builtins.toJSON (builtins.toJSON nsBinds)})
|
||||
host.succeed(f"echo before > {NS_DIR}/state/pre.txt && chown 1000:1000 {NS_DIR}/state/pre.txt")
|
||||
host.succeed(f"stat -c %u:%g {NS_DIR}/state/pre.txt | grep -x 1000:1000")
|
||||
priv_request(host, {"op": "ensure_agent_socket_dir", "name": NS_AGENT})
|
||||
write = {
|
||||
"op": "write_nspawn_flags",
|
||||
"container": NS,
|
||||
"binds": NS_BINDS,
|
||||
"isolation": {"bridge": "${bridge}", "gateway_ip": "${gateway}"},
|
||||
}
|
||||
priv_request(host, write)
|
||||
ns_conf = host.succeed(f"cat /etc/nixos-containers/{NS}.conf")
|
||||
assert "PRIVATE_USERS=pick" in ns_conf, ns_conf
|
||||
assert "--private-users-ownership=map" in ns_conf, ns_conf
|
||||
# Every bind, the read-only config included, is mounted owneridmap.
|
||||
assert ns_conf.count(":owneridmap") == len(NS_BINDS), ns_conf
|
||||
# The unlisted agent's conf has none of it: the control that the lookups
|
||||
# above find these strings where they are.
|
||||
assert "owneridmap" not in conf() and "PRIVATE_USERS" not in conf(), conf()
|
||||
|
||||
# The data moved to the data owner, the config dir stayed root's.
|
||||
for sub in ["state", "harness", "work", "claude"]:
|
||||
host.succeed(f"stat -c %u:%g {NS_DIR}/{sub} | grep -x ${dataOwner}:${dataOwner}")
|
||||
host.succeed(f"stat -c %u:%g {NS_DIR}/state/pre.txt | grep -x ${dataOwner}:${dataOwner}")
|
||||
host.succeed(f"stat -c %u:%g {NS_DIR}/config/cfg | grep -x 0:0")
|
||||
host.succeed(f"stat -c %u:%g ${nsSocketDir} | grep -x ${dataOwner}:${dataOwner}")
|
||||
|
||||
boot_ns = lambda: (
|
||||
host.succeed(f"nixos-container start {NS}"),
|
||||
host.wait_until_succeeds(f"nixos-container run {NS} -- systemctl is-active multi-user.target"),
|
||||
)
|
||||
boot_ns()
|
||||
def as_ns(script):
|
||||
return host.succeed(f"nixos-container run {NS} -- runuser -l {NS_AGENT} -c '{script}'").strip()
|
||||
# Inside, the data is the agent's: read, written and listed as itself.
|
||||
assert as_ns(f"cat /agents/{NS_AGENT}/state/pre.txt") == "before"
|
||||
assert as_ns(f"stat -c %u:%g /agents/{NS_AGENT}/state/pre.txt") == "1000:1000"
|
||||
as_ns(f"echo after > /agents/{NS_AGENT}/state/post.txt")
|
||||
as_ns(f"touch /run/hive-agent/{NS_AGENT}/web.sock")
|
||||
assert as_ns(f"cat /agents/{NS_AGENT}/config/cfg") == "from-config"
|
||||
host.fail(f"nixos-container run {NS} -- runuser -l {NS_AGENT} -c 'touch /agents/{NS_AGENT}/config/new'")
|
||||
# On disk the new file belongs to the data owner, not to a picked range.
|
||||
host.succeed(f"stat -c %u:%g {NS_DIR}/state/post.txt | grep -x ${dataOwner}:${dataOwner}")
|
||||
host.succeed(f"stat -c %u:%g ${nsSocketDir}/web.sock | grep -x ${dataOwner}:${dataOwner}")
|
||||
host.succeed(f"nixos-container run {NS} -- cat /proc/self/uid_map | grep -v ' 4294967295$'")
|
||||
|
||||
# The move runs once: a file a host process leaves owned by 1000 after it
|
||||
# is not chowned by the next request, and the agent restarts.
|
||||
host.succeed(f"nixos-container stop {NS}")
|
||||
host.succeed(f"echo late > {NS_DIR}/state/late.txt && chown 1000:1000 {NS_DIR}/state/late.txt")
|
||||
priv_request(host, write)
|
||||
host.succeed(f"stat -c %u:%g {NS_DIR}/state/late.txt | grep -x 1000:1000")
|
||||
boot_ns()
|
||||
'';
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue