mirror of
https://forge.darkest.space/mara/nixos-configs.git
synced 2026-11-19 11:13:59 +01:00
216 lines
7.3 KiB
Nix
216 lines
7.3 KiB
Nix
{
|
|
lib,
|
|
pkgs,
|
|
...
|
|
}:
|
|
let
|
|
vm = "agent-lab";
|
|
unit = "${vm}-vm";
|
|
stateDir = "/var/lib/${unit}";
|
|
disk = "${stateDir}/disk.qcow2";
|
|
qmp = "/run/${unit}/qmp.sock";
|
|
|
|
sshPort = 2222;
|
|
cpus = 4;
|
|
memoryMiB = 8192;
|
|
diskSize = "60G";
|
|
|
|
# Deliberately a flake *reference*, resolved when the reset script runs, not
|
|
# a store path baked into this host's closure. The host must not pin the
|
|
# guest's config: agents rebuild the guest from inside, and a pinned
|
|
# `system.build.toplevel` here (the microvm.nix model) would fight that. All
|
|
# the host ever does is lay down a first-boot disk.
|
|
seedFlake = "git+https://forge.darkest.space/mara/nixos-configs.git";
|
|
seedAttr = "nixosConfigurations.${vm}.config.system.build.images.qemu";
|
|
|
|
# Credentials the guest needs but the nix store must never see. Both are
|
|
# hand-placed, out-of-store files on this host (same as on every other
|
|
# machine here), and are copied in over ssh after the guest is up.
|
|
provisionKey = "/etc/nix/distributed-build-key";
|
|
netrc = "/etc/nix/netrc";
|
|
|
|
# ssh to the guest as the provisioning identity. The host key changes on
|
|
# every reseed, and this target is 127.0.0.1 behind qemu's user-mode NAT, so
|
|
# host key checking buys nothing here.
|
|
guestSsh = lib.concatStringsSep " " [
|
|
(lib.getExe' pkgs.openssh "ssh")
|
|
"-i ${provisionKey}"
|
|
"-o IdentitiesOnly=yes"
|
|
"-o StrictHostKeyChecking=no"
|
|
"-o UserKnownHostsFile=/dev/null"
|
|
"-o LogLevel=ERROR"
|
|
"-o BatchMode=yes"
|
|
"-o ConnectTimeout=5"
|
|
"-p ${toString sshPort}"
|
|
"root@127.0.0.1"
|
|
];
|
|
|
|
reset = pkgs.writeShellApplication {
|
|
name = "${vm}-reset";
|
|
runtimeInputs = [
|
|
pkgs.qemu_kvm
|
|
pkgs.findutils
|
|
pkgs.coreutils
|
|
pkgs.systemd
|
|
pkgs.nix
|
|
];
|
|
text = ''
|
|
if [ "$(id -u)" != 0 ]; then
|
|
echo "${vm}-reset: needs root (writes ${stateDir})" >&2
|
|
exit 1
|
|
fi
|
|
|
|
flake="''${1:-${seedFlake}}"
|
|
echo "${vm}-reset: building seed image from $flake#${seedAttr}" >&2
|
|
out=$(nix build --no-link --print-out-paths "$flake#${seedAttr}")
|
|
|
|
img=$(find "$out" -maxdepth 1 -name '*.qcow2' -print -quit)
|
|
if [ -z "$img" ]; then
|
|
echo "${vm}-reset: no qcow2 in $out" >&2
|
|
exit 1
|
|
fi
|
|
|
|
systemctl stop ${unit}.service || true
|
|
|
|
# The tmpfiles rule above makes this exist at activation; keep the
|
|
# script able to stand alone if the directory was removed by hand.
|
|
install -d -m0700 -o ${unit} -g ${unit} ${stateDir}
|
|
|
|
# `convert`, not a copy or an overlay: the running disk must not keep a
|
|
# backing-file reference into the nix store, or a later GC of the seed
|
|
# would pull the VM's filesystem out from under it.
|
|
rm -f ${disk}.new
|
|
qemu-img convert -O qcow2 "$img" ${disk}.new
|
|
qemu-img resize ${disk}.new ${diskSize}
|
|
chown ${unit}:${unit} ${disk}.new
|
|
chmod 0600 ${disk}.new
|
|
mv -f ${disk}.new ${disk}
|
|
|
|
systemctl start ${unit}.service
|
|
echo "${vm}-reset: fresh disk seeded, VM starting — ssh -p ${toString sshPort} root@<host>" >&2
|
|
'';
|
|
};
|
|
in
|
|
{
|
|
config = {
|
|
users.users.${unit} = {
|
|
isSystemUser = true;
|
|
group = unit;
|
|
home = stateDir;
|
|
description = "agent-lab sandbox VM";
|
|
};
|
|
users.groups.${unit} = { };
|
|
|
|
# `StateDirectory=` in the unit below only creates this when the unit
|
|
# starts, and the unit cannot start until the disk exists -- so the very
|
|
# first `agent-lab-reset` would have nowhere to write. Declare it here.
|
|
systemd.tmpfiles.rules = [ "d ${stateDir} 0700 ${unit} ${unit} -" ];
|
|
|
|
systemd.services.${unit} = {
|
|
description = "agent-lab sandbox VM (ssh target for hyperhive agents)";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [ "network.target" ];
|
|
# No disk yet means the VM was never seeded; `agent-lab-reset` makes one.
|
|
unitConfig.ConditionPathExists = disk;
|
|
|
|
serviceConfig = {
|
|
User = unit;
|
|
Group = unit;
|
|
SupplementaryGroups = [ "kvm" ];
|
|
StateDirectory = unit;
|
|
StateDirectoryMode = "0700";
|
|
RuntimeDirectory = unit;
|
|
|
|
ExecStart = lib.concatStringsSep " " [
|
|
(lib.getExe' pkgs.qemu_kvm "qemu-system-x86_64")
|
|
"-name ${vm}"
|
|
"-machine q35,accel=kvm"
|
|
"-cpu host"
|
|
"-smp ${toString cpus}"
|
|
"-m ${toString memoryMiB}"
|
|
"-nodefaults"
|
|
"-no-user-config"
|
|
"-display none"
|
|
"-serial stdio"
|
|
"-device virtio-rng-pci"
|
|
"-drive file=${disk},if=virtio,format=qcow2,cache=writeback,discard=unmap"
|
|
# User-mode NAT: outbound works (the guest needs to reach the caches
|
|
# to rebuild itself), and only the forwarded ssh port is reachable
|
|
# from outside. Nothing the agents do in here can squat a LAN IP.
|
|
"-netdev user,id=net0,hostfwd=tcp::${toString sshPort}-:22"
|
|
"-device virtio-net-pci,netdev=net0"
|
|
"-qmp unix:${qmp},server=on,wait=off"
|
|
];
|
|
|
|
# ACPI powerdown over QMP rather than `ssh poweroff`: on a box whose
|
|
# whole job is surviving agent experiments, sshd is exactly the thing
|
|
# that may be broken when it is time to stop.
|
|
ExecStop = pkgs.writeShellScript "${unit}-stop" ''
|
|
printf '%s\n' \
|
|
'{"execute":"qmp_capabilities"}' \
|
|
'{"execute":"system_powerdown"}' \
|
|
| ${lib.getExe pkgs.socat} -t2 - UNIX-CONNECT:${qmp} || true
|
|
'';
|
|
TimeoutStopSec = "90s";
|
|
Restart = "always";
|
|
RestartSec = "5s";
|
|
StandardInput = "null";
|
|
|
|
NoNewPrivileges = true;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
PrivateTmp = true;
|
|
RestrictSUIDSGID = true;
|
|
};
|
|
};
|
|
|
|
# Runs on every VM start, not just after a reseed: if an agent wipes
|
|
# /etc/nix in there, the next restart puts the cache credential back.
|
|
systemd.services."${unit}-netrc" = {
|
|
description = "Copy the private cache netrc into the agent-lab VM";
|
|
wantedBy = [ "${unit}.service" ];
|
|
after = [ "${unit}.service" ];
|
|
unitConfig.ConditionPathExists = [
|
|
netrc
|
|
provisionKey
|
|
];
|
|
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
ExecStart = pkgs.writeShellScript "${unit}-netrc" ''
|
|
set -eu
|
|
|
|
for _ in $(${lib.getExe' pkgs.coreutils "seq"} 1 60); do
|
|
if ${guestSsh} true 2>/dev/null; then
|
|
# Piped from the host's file at runtime; the credential is never
|
|
# an argument, never in a derivation, never in the store.
|
|
${guestSsh} 'umask 022; cat > /etc/nix/netrc' < ${netrc}
|
|
${guestSsh} 'systemctl restart nix-daemon' || true
|
|
echo "netrc installed in ${vm}"
|
|
exit 0
|
|
fi
|
|
${lib.getExe' pkgs.coreutils "sleep"} 5
|
|
done
|
|
|
|
echo "${vm} did not answer ssh within 5min; netrc not installed" >&2
|
|
exit 1
|
|
'';
|
|
};
|
|
};
|
|
|
|
# Reachable from lpt2 (where the hive-controller's agents live) over
|
|
# tailscale/LAN as <this host>:2222.
|
|
networking.firewall.allowedTCPPorts = [ sshPort ];
|
|
|
|
environment.systemPackages = [ reset ];
|
|
|
|
programs.ssh.extraConfig = ''
|
|
Host ${vm}
|
|
HostName 127.0.0.1
|
|
Port ${toString sshPort}
|
|
User root
|
|
StrictHostKeyChecking no
|
|
UserKnownHostsFile /dev/null
|
|
'';
|
|
};
|
|
}
|