mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

add a per-agent nix builds mode, host or none, carried like privateUsers

Browse source
This commit is contained in:
atlas 2026-11-03 00:30:55 +01:00 • committed by mara
commit 5485151ba6
3 changed files with 618 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

30
nix/agent-modules/default.nix
View file

@ -135,6 +135,32 @@ in
'';
};
options.services.hyperhive.agent.nixBuilds = lib.mkOption {
type = lib.types.enum [
"host"
"none"
];
default = "host";
example = "none";
description = ''
Where this agent's nix builds run.
- `host` (default): builds go through the host's nix-daemon socket,
which every agent container gets bound in by default. The daemon runs
them outside this container, so the container's CPU and memory caps do
not apply to a build.
- `none`: the container cannot reach the host's nix-daemon, so
`nix build`, `nix run`, `nix shell` and the like fail for anything not
already in the store, with a "cannot connect to socket" error. The
agent's own system closure stays readable.
The swarm builder reads the option from the evaluated config into the
agent's closure record, and the hive applies it when it next starts the
container, so a running agent keeps its mode until restarted. A host
that does not know the option leaves the agent on `host`.
'';
};
config = {
assertions = [
# Guard the inputs-routed-as-output pattern: the agent flake.nix is
@ -235,6 +261,10 @@ in
# Non-root nix clients already default to the daemon socket, so this
# is a no-op for them; it only matters for root services that would
# otherwise silently use the local store.
#
# Stays set under `nixBuilds = "none"`: the missing socket then fails a
# build with an error that names it, where the local store would fail on a
# read-only filesystem.
systemd.globalEnvironment.NIX_REMOTE = "daemon";
# `claude-code` is unfree. Each per-agent container's nixosConfiguration

29
nix/module-eval/agent-user.nix
View file

@ -22,11 +22,20 @@ let
;
# A named agent, so the rendered script carries a name no default supplies.
migrate =
(agentWith { services.hyperhive.agent.user.name = "iris"; })
.system.activationScripts.hive-agent-user-migrate.text;
iris = agentWith { services.hyperhive.agent.user.name = "iris"; };
migrate = iris.system.activationScripts.hive-agent-user-migrate.text;
# The tmpfiles rules of the same named agent.
rules = (agentWith { services.hyperhive.agent.user.name = "iris"; }).systemd.tmpfiles.rules;
rules = iris.systemd.tmpfiles.rules;
# The same agent with its nix builds switched off, and one asking for a mode
# the option does not have.
noBuilds = agentWith {
services.hyperhive.agent.user.name = "iris";
services.hyperhive.agent.nixBuilds = "none";
};
unknownBuilds = agentWith {
services.hyperhive.agent.user.name = "iris";
services.hyperhive.agent.nixBuilds = "container";
};
# `privateUsers` with the given pins; the agent name is the one the cases
# above use, so only the pins differ.
withUsers =
@ -113,6 +122,18 @@ let
&& lib.elem "L /home/iris/.local/state/nix - - - - /nix/var/nix/profiles/per-user/iris" rules
&& lib.elem "L /home/iris/.nix-profile - - - - /home/iris/.local/state/nix/profiles/profile" rules;
}
{
# The option defaults to host builds, accepts `none`, and refuses a mode
# it does not declare. The default and `none` arms are each other's
# control; `none` leaves the in-container routing alone, because the
# error an agent gets then comes from the missing socket.
name = "nixBuilds defaults to host, takes none and refuses a mode it does not declare";
ok =
iris.services.hyperhive.agent.nixBuilds == "host"
&& noBuilds.services.hyperhive.agent.nixBuilds == "none"
&& noBuilds.systemd.globalEnvironment.NIX_REMOTE == "daemon"
&& !(builtins.tryEval unknownBuilds.services.hyperhive.agent.nixBuilds).success;
}
];
in
runGroup "agent-user" cases

38
nix/vm-tests/agent-ephemeral-rootfs.nix
View file

@ -16,15 +16,15 @@
# - a link planted at `root` in the template refuses the next start through
# the limits drop-in's `ExecStartPre=` check (whose build has a GC root),
# leaves the link's target untouched, and the start goes through once the
# link is gone.
#
# link is gone;
# - an agent whose config sets `services.hyperhive.agent.user.privateUsers`
# (hive-c0re sends the flag from the closure record; the test sends it)
# starts in its own
# user namespace with state it already had, owned by uid 1000 on disk: the
# data moves once to `dataOwner`, the agent user reads and writes it as
# itself, the other agent's conf (no flag sent) is untouched, and a second
# `WriteNspawnFlags` moves nothing.
# starts in its own user namespace with state it already had, owned by uid
# 1000 on disk: the data moves once to `dataOwner`, the agent user reads and
# writes it as itself, the other agent's conf (no flag sent) is untouched,
# and a second `WriteNspawnFlags` moves nothing;
# - an agent set to `services.hyperhive.agent.nixBuilds = "none"` has no
# nix-daemon socket and cannot build; a request naming no mode keeps that.
#
# The agent's journal forwarder itself (an OTLP push to the hive's collector) is
# not booted here; the module-eval suite covers its wiring.
@ -277,6 +277,9 @@ pkgs.testers.runNixOSTest {
assert host.succeed(f"cat {WORK}/kept.txt").strip() == "kept"
as_agent("hello")
as_agent("tree --version")
# The default mode: the host daemon's socket is there and answers.
assert "socket" in in_ct("ls /nix/var/nix/daemon-socket").split()
as_agent(f"nix-store --store daemon -q --hash {HELLO}")
with subtest("the switch ran once"):
salvages = [d for d in salvage_dirs() if d.startswith("salvaged-rootfs-")]
@ -335,10 +338,14 @@ pkgs.testers.runNixOSTest {
"binds": NS_BINDS,
"isolation": {"bridge": "${bridge}", "gateway_ip": "${gateway}"},
"private_users": True,
"nix_builds": "none",
}
priv_request(host, write)
ns_conf = host.succeed(f"cat /etc/nixos-containers/{NS}.conf")
assert "PRIVATE_USERS=pick" in ns_conf, ns_conf
assert "--tmpfs=/nix/var/nix/daemon-socket" in ns_conf, ns_conf
# The other agent's conf has no such flag: the control for the line above.
assert "daemon-socket" not in conf(), conf()
assert "--private-users-ownership=map" in ns_conf, ns_conf
# Every bind, the read-only config included, is mounted owneridmap.
assert ns_conf.count(":owneridmap") == len(NS_BINDS), ns_conf
@ -379,12 +386,27 @@ pkgs.testers.runNixOSTest {
host.succeed("stat -c %u:%g ${nsSocketDir}/web.sock | grep -x ${dataOwner}:${dataOwner}")
host.succeed(f"nixos-container run {NS} -- cat /proc/self/uid_map | grep -v ' 4294967295$'")
def no_daemon():
# The start script's bind is under the tmpfs: no socket in the
# directory, and a build cannot connect. The store path is still
# readable, and the first agent's directory (checked above) has the
# socket where this one has none.
host.fail(f"nixos-container run {NS} -- test -S /nix/var/nix/daemon-socket/socket")
assert as_ns("ls -A /nix/var/nix/daemon-socket") == ""
host.fail(f"nixos-container run {NS} -- runuser -l {NS_AGENT} -c 'nix-store --store daemon -q --hash {HELLO}'")
as_ns(f"test -r {HELLO}/bin/hello")
no_daemon()
# The move runs once: a file a host process leaves owned by 1000 after it
# is not chowned by the next request, and the agent restarts.
host.succeed(f"nixos-container stop {NS}")
host.succeed(f"echo late > {NS_DIR}/state/late.txt && chown 1000:1000 {NS_DIR}/state/late.txt")
priv_request(host, write)
# A request that names no nix-builds mode, as one with no readable
# record is, keeps the choice the last write made.
priv_request(host, {k: v for k, v in write.items() if k != "nix_builds"})
host.succeed(f"stat -c %u:%g {NS_DIR}/state/late.txt | grep -x 1000:1000")
assert "--tmpfs=/nix/var/nix/daemon-socket" in host.succeed(f"cat /etc/nixos-containers/{NS}.conf")
boot_ns()
no_daemon()
'';
}