move nixBuilds into its own agent module, and have the vm test's agent import it
This commit is contained in:
parent
f930571e38
commit
d5eb7711e6
4 changed files with 88 additions and 46 deletions
|
|
@ -91,6 +91,7 @@ in
|
||||||
./memory-cap.nix
|
./memory-cap.nix
|
||||||
./metrics.nix
|
./metrics.nix
|
||||||
./network.nix
|
./network.nix
|
||||||
|
./nix-builds.nix
|
||||||
./otel.nix
|
./otel.nix
|
||||||
./otel-removed.nix
|
./otel-removed.nix
|
||||||
./packages.nix
|
./packages.nix
|
||||||
|
|
@ -135,34 +136,6 @@ in
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
options.services.hyperhive.agent.nixBuilds = lib.mkOption {
|
|
||||||
type = lib.types.enum [
|
|
||||||
"host"
|
|
||||||
"none"
|
|
||||||
];
|
|
||||||
default = "host";
|
|
||||||
example = "none";
|
|
||||||
description = ''
|
|
||||||
Where this agent's nix builds run.
|
|
||||||
|
|
||||||
- `host` (default): builds go through the host's nix-daemon socket,
|
|
||||||
which every agent container gets bound in by default. The daemon runs
|
|
||||||
them outside this container, so the container's CPU and memory caps do
|
|
||||||
not apply to a build.
|
|
||||||
- `none`: the container cannot reach the host's nix-daemon, so
|
|
||||||
`nix build`, `nix run`, `nix shell` and the like fail for anything not
|
|
||||||
already in the store, with a "cannot connect to socket" error. The
|
|
||||||
agent's own system closure stays readable.
|
|
||||||
The container's own nix-daemon socket and service are off, so nothing
|
|
||||||
in the container listens on that path either.
|
|
||||||
|
|
||||||
The swarm builder reads the option from the evaluated config into the
|
|
||||||
agent's closure record, and the hive applies it when it next starts the
|
|
||||||
container, so a running agent keeps its mode until restarted. A host
|
|
||||||
that does not know the option leaves the agent on `host`.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
config = {
|
config = {
|
||||||
assertions = [
|
assertions = [
|
||||||
# Guard the inputs-routed-as-output pattern: the agent flake.nix is
|
# Guard the inputs-routed-as-output pattern: the agent flake.nix is
|
||||||
|
|
@ -269,19 +242,6 @@ in
|
||||||
# read-only filesystem.
|
# read-only filesystem.
|
||||||
systemd.globalEnvironment.NIX_REMOTE = "daemon";
|
systemd.globalEnvironment.NIX_REMOTE = "daemon";
|
||||||
|
|
||||||
# Under `nixBuilds = "none"` the daemon directory is an empty tmpfs, which
|
|
||||||
# the nix-daemon socket unit's `ConditionPathIsReadWrite` accepts, so the
|
|
||||||
# container's own daemon would listen there and hand out the very socket
|
|
||||||
# the mode withholds. Under `host` the directory is the host's read-only
|
|
||||||
# bind, the condition fails, and the unit stays inactive. `nix.enable`
|
|
||||||
# stays on, so the client and the read-only store remain usable.
|
|
||||||
systemd.sockets.nix-daemon.enable = lib.mkIf (
|
|
||||||
config.services.hyperhive.agent.nixBuilds == "none"
|
|
||||||
) false;
|
|
||||||
systemd.services.nix-daemon.enable = lib.mkIf (
|
|
||||||
config.services.hyperhive.agent.nixBuilds == "none"
|
|
||||||
) false;
|
|
||||||
|
|
||||||
# `claude-code` is unfree. Each per-agent container's nixosConfiguration
|
# `claude-code` is unfree. Each per-agent container's nixosConfiguration
|
||||||
# evaluates its own `nixpkgs` instance, so the operator's host-level
|
# evaluates its own `nixpkgs` instance, so the operator's host-level
|
||||||
# `nixpkgs.config.allowUnfreePredicate` does not propagate into here —
|
# `nixpkgs.config.allowUnfreePredicate` does not propagate into here —
|
||||||
|
|
|
||||||
46
nix/agent-modules/nix-builds.nix
Normal file
46
nix/agent-modules/nix-builds.nix
Normal file
|
|
@ -0,0 +1,46 @@
|
||||||
|
# Where the agent's nix builds run: `services.hyperhive.agent.nixBuilds`.
|
||||||
|
#
|
||||||
|
# The mode reaches the container's start flags through the closure record
|
||||||
|
# (hive-priv adds `--tmpfs=/nix/var/nix/daemon-socket` for `none`); this module
|
||||||
|
# is the part inside the container's own NixOS config.
|
||||||
|
{ lib, config, ... }:
|
||||||
|
{
|
||||||
|
options.services.hyperhive.agent.nixBuilds = lib.mkOption {
|
||||||
|
type = lib.types.enum [
|
||||||
|
"host"
|
||||||
|
"none"
|
||||||
|
];
|
||||||
|
default = "host";
|
||||||
|
example = "none";
|
||||||
|
description = ''
|
||||||
|
Where this agent's nix builds run.
|
||||||
|
|
||||||
|
- `host` (default): builds go through the host's nix-daemon socket,
|
||||||
|
which every agent container gets bound in by default. The daemon runs
|
||||||
|
them outside this container, so the container's CPU and memory caps do
|
||||||
|
not apply to a build.
|
||||||
|
- `none`: the container cannot reach the host's nix-daemon, so
|
||||||
|
`nix build`, `nix run`, `nix shell` and the like fail for anything not
|
||||||
|
already in the store, with a "cannot connect to socket" error. The
|
||||||
|
agent's own system closure stays readable.
|
||||||
|
The container's own nix-daemon socket and service are off, so nothing
|
||||||
|
in the container listens on that path either.
|
||||||
|
|
||||||
|
The swarm builder reads the option from the evaluated config into the
|
||||||
|
agent's closure record, and the hive applies it when it next starts the
|
||||||
|
container, so a running agent keeps its mode until restarted. A host
|
||||||
|
that does not know the option leaves the agent on `host`.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
# Under `none` the daemon directory is an empty tmpfs, which the nix-daemon
|
||||||
|
# socket unit's `ConditionPathIsReadWrite` accepts, so the container's own
|
||||||
|
# daemon would listen there and hand out the very socket the mode withholds.
|
||||||
|
# Under `host` the directory is the host's read-only bind, the condition
|
||||||
|
# fails, and the unit stays inactive. `nix.enable` stays on, so the client and
|
||||||
|
# the read-only store remain usable.
|
||||||
|
config = lib.mkIf (config.services.hyperhive.agent.nixBuilds == "none") {
|
||||||
|
systemd.sockets.nix-daemon.enable = false;
|
||||||
|
systemd.services.nix-daemon.enable = false;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
@ -36,6 +36,28 @@ let
|
||||||
services.hyperhive.agent.user.name = "iris";
|
services.hyperhive.agent.user.name = "iris";
|
||||||
services.hyperhive.agent.nixBuilds = "container";
|
services.hyperhive.agent.nixBuilds = "container";
|
||||||
};
|
};
|
||||||
|
# The VM test's agent stand-in: the user and nix-builds modules on a plain
|
||||||
|
# container, under each mode.
|
||||||
|
standIn =
|
||||||
|
mode:
|
||||||
|
(nixosSystem {
|
||||||
|
system = pkgs.stdenv.hostPlatform.system;
|
||||||
|
modules = [
|
||||||
|
../agent-modules/user.nix
|
||||||
|
../agent-modules/nix-builds.nix
|
||||||
|
{
|
||||||
|
fileSystems."/" = {
|
||||||
|
device = "/dev/null";
|
||||||
|
fsType = "tmpfs";
|
||||||
|
};
|
||||||
|
boot.loader.grub.enable = false;
|
||||||
|
system.stateVersion = "25.11";
|
||||||
|
services.hyperhive.agent.nixBuilds = mode;
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}).config;
|
||||||
|
standInHost = standIn "host";
|
||||||
|
standInNone = standIn "none";
|
||||||
# `privateUsers` with the given pins; the agent name is the one the cases
|
# `privateUsers` with the given pins; the agent name is the one the cases
|
||||||
# above use, so only the pins differ.
|
# above use, so only the pins differ.
|
||||||
withUsers =
|
withUsers =
|
||||||
|
|
@ -143,7 +165,13 @@ let
|
||||||
iris.systemd.sockets.nix-daemon.enable
|
iris.systemd.sockets.nix-daemon.enable
|
||||||
&& iris.systemd.services.nix-daemon.enable
|
&& iris.systemd.services.nix-daemon.enable
|
||||||
&& !noBuilds.systemd.sockets.nix-daemon.enable
|
&& !noBuilds.systemd.sockets.nix-daemon.enable
|
||||||
&& !noBuilds.systemd.services.nix-daemon.enable;
|
&& !noBuilds.systemd.services.nix-daemon.enable
|
||||||
|
# The VM test's agent stand-in imports these two modules and not the
|
||||||
|
# whole agent module set, so the disable has to live in the former.
|
||||||
|
&& standInHost.systemd.sockets.nix-daemon.enable
|
||||||
|
&& standInHost.systemd.services.nix-daemon.enable
|
||||||
|
&& !standInNone.systemd.sockets.nix-daemon.enable
|
||||||
|
&& !standInNone.systemd.services.nix-daemon.enable;
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
in
|
in
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
# `vmTests.<system>.agent-ephemeral-rootfs`: an agent container's switch from a
|
# `vmTests.<system>.agent-ephemeral-rootfs`: an agent container's switch from a
|
||||||
# persistent root to nspawn's `--ephemeral`, through the real hive-priv unit
|
# persistent root to nspawn's `--ephemeral`, through the real hive-priv unit
|
||||||
# (nix/host-modules/hive-priv.nix, sandbox included). The container is an
|
# (nix/host-modules/hive-priv.nix, sandbox included). The container is an
|
||||||
# agent stand-in: the real agent user module (nix/agent-modules/user.nix) on a
|
# agent stand-in: the real agent user and nix-builds modules (nix/agent-modules)
|
||||||
# plain NixOS container, with the binds hive-c0re passes for an agent.
|
# on a plain NixOS container, with the binds hive-c0re passes for an agent.
|
||||||
#
|
#
|
||||||
# What it pins:
|
# What it pins:
|
||||||
# - hive-priv's `WriteNspawnFlags` salvages the persistent root before it
|
# - hive-priv's `WriteNspawnFlags` salvages the persistent root before it
|
||||||
|
|
@ -142,7 +142,10 @@ pkgs.testers.runNixOSTest {
|
||||||
privateNetwork = true;
|
privateNetwork = true;
|
||||||
hostBridge = bridge;
|
hostBridge = bridge;
|
||||||
config = {
|
config = {
|
||||||
imports = [ ../agent-modules/user.nix ];
|
imports = [
|
||||||
|
../agent-modules/user.nix
|
||||||
|
../agent-modules/nix-builds.nix
|
||||||
|
];
|
||||||
system.stateVersion = "25.11";
|
system.stateVersion = "25.11";
|
||||||
services.hyperhive.agent.user.name = agent;
|
services.hyperhive.agent.user.name = agent;
|
||||||
# The agent's forwarder reads /var/log/journal, which journald only
|
# The agent's forwarder reads /var/log/journal, which journald only
|
||||||
|
|
@ -156,7 +159,10 @@ pkgs.testers.runNixOSTest {
|
||||||
privateNetwork = true;
|
privateNetwork = true;
|
||||||
hostBridge = bridge;
|
hostBridge = bridge;
|
||||||
config = {
|
config = {
|
||||||
imports = [ ../agent-modules/user.nix ];
|
imports = [
|
||||||
|
../agent-modules/user.nix
|
||||||
|
../agent-modules/nix-builds.nix
|
||||||
|
];
|
||||||
system.stateVersion = "25.11";
|
system.stateVersion = "25.11";
|
||||||
services.hyperhive.agent.user = {
|
services.hyperhive.agent.user = {
|
||||||
name = nsAgent;
|
name = nsAgent;
|
||||||
|
|
@ -164,6 +170,8 @@ pkgs.testers.runNixOSTest {
|
||||||
gid = 1000;
|
gid = 1000;
|
||||||
privateUsers = true;
|
privateUsers = true;
|
||||||
};
|
};
|
||||||
|
# The mode the test also sends in the nspawn flags request below.
|
||||||
|
services.hyperhive.agent.nixBuilds = "none";
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue