{ lib, pkgs, ... }: let vm = "agent-lab"; unit = "${vm}-vm"; stateDir = "/var/lib/${unit}"; disk = "${stateDir}/disk.qcow2"; qmp = "/run/${unit}/qmp.sock"; sshPort = 2222; cpus = 4; memoryMiB = 8192; diskSize = "60G"; # Deliberately a flake *reference*, resolved when the reset script runs, not # a store path baked into this host's closure. The host must not pin the # guest's config: agents rebuild the guest from inside, and a pinned # `system.build.toplevel` here (the microvm.nix model) would fight that. All # the host ever does is lay down a first-boot disk. seedFlake = "git+https://forge.darkest.space/mara/nixos-configs.git"; seedAttr = "nixosConfigurations.${vm}.config.system.build.images.qemu"; # Credentials the guest needs but the nix store must never see. Both are # hand-placed, out-of-store files on this host (same as on every other # machine here), and are copied in over ssh after the guest is up. provisionKey = "/etc/nix/distributed-build-key"; netrc = "/etc/nix/netrc"; # ssh to the guest as the provisioning identity. The host key changes on # every reseed, and this target is 127.0.0.1 behind qemu's user-mode NAT, so # host key checking buys nothing here. guestSsh = lib.concatStringsSep " " [ (lib.getExe' pkgs.openssh "ssh") "-i ${provisionKey}" "-o IdentitiesOnly=yes" "-o StrictHostKeyChecking=no" "-o UserKnownHostsFile=/dev/null" "-o LogLevel=ERROR" "-o BatchMode=yes" "-o ConnectTimeout=5" "-p ${toString sshPort}" "root@127.0.0.1" ]; reset = pkgs.writeShellApplication { name = "${vm}-reset"; runtimeInputs = [ pkgs.qemu_kvm pkgs.findutils pkgs.coreutils pkgs.systemd pkgs.nix ]; text = '' if [ "$(id -u)" != 0 ]; then echo "${vm}-reset: needs root (writes ${stateDir})" >&2 exit 1 fi flake="''${1:-${seedFlake}}" echo "${vm}-reset: building seed image from $flake#${seedAttr}" >&2 out=$(nix build --no-link --print-out-paths "$flake#${seedAttr}") img=$(find "$out" -maxdepth 1 -name '*.qcow2' -print -quit) if [ -z "$img" ]; then echo "${vm}-reset: no qcow2 in $out" >&2 exit 1 fi systemctl stop ${unit}.service || true # The tmpfiles rule above makes this exist at activation; keep the # script able to stand alone if the directory was removed by hand. install -d -m0700 -o ${unit} -g ${unit} ${stateDir} # `convert`, not a copy or an overlay: the running disk must not keep a # backing-file reference into the nix store, or a later GC of the seed # would pull the VM's filesystem out from under it. rm -f ${disk}.new qemu-img convert -O qcow2 "$img" ${disk}.new qemu-img resize ${disk}.new ${diskSize} chown ${unit}:${unit} ${disk}.new chmod 0600 ${disk}.new mv -f ${disk}.new ${disk} systemctl start ${unit}.service echo "${vm}-reset: fresh disk seeded, VM starting — ssh -p ${toString sshPort} root@" >&2 ''; }; in { config = { users.users.${unit} = { isSystemUser = true; group = unit; home = stateDir; description = "agent-lab sandbox VM"; }; users.groups.${unit} = { }; # `StateDirectory=` in the unit below only creates this when the unit # starts, and the unit cannot start until the disk exists -- so the very # first `agent-lab-reset` would have nowhere to write. Declare it here. systemd.tmpfiles.rules = [ "d ${stateDir} 0700 ${unit} ${unit} -" ]; systemd.services.${unit} = { description = "agent-lab sandbox VM (ssh target for hyperhive agents)"; wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; # No disk yet means the VM was never seeded; `agent-lab-reset` makes one. unitConfig.ConditionPathExists = disk; serviceConfig = { User = unit; Group = unit; SupplementaryGroups = [ "kvm" ]; StateDirectory = unit; StateDirectoryMode = "0700"; RuntimeDirectory = unit; ExecStart = lib.concatStringsSep " " [ (lib.getExe' pkgs.qemu_kvm "qemu-system-x86_64") "-name ${vm}" "-machine q35,accel=kvm" "-cpu host" "-smp ${toString cpus}" "-m ${toString memoryMiB}" "-nodefaults" "-no-user-config" "-display none" "-serial stdio" "-device virtio-rng-pci" "-drive file=${disk},if=virtio,format=qcow2,cache=writeback,discard=unmap" # User-mode NAT: outbound works (the guest needs to reach the caches # to rebuild itself), and only the forwarded ssh port is reachable # from outside. Nothing the agents do in here can squat a LAN IP. "-netdev user,id=net0,hostfwd=tcp::${toString sshPort}-:22" "-device virtio-net-pci,netdev=net0" "-qmp unix:${qmp},server=on,wait=off" ]; # ACPI powerdown over QMP rather than `ssh poweroff`: on a box whose # whole job is surviving agent experiments, sshd is exactly the thing # that may be broken when it is time to stop. ExecStop = pkgs.writeShellScript "${unit}-stop" '' printf '%s\n' \ '{"execute":"qmp_capabilities"}' \ '{"execute":"system_powerdown"}' \ | ${lib.getExe pkgs.socat} -t2 - UNIX-CONNECT:${qmp} || true ''; TimeoutStopSec = "90s"; Restart = "always"; RestartSec = "5s"; StandardInput = "null"; NoNewPrivileges = true; ProtectSystem = "strict"; ProtectHome = true; PrivateTmp = true; RestrictSUIDSGID = true; }; }; # Runs on every VM start, not just after a reseed: if an agent wipes # /etc/nix in there, the next restart puts the cache credential back. systemd.services."${unit}-netrc" = { description = "Copy the private cache netrc into the agent-lab VM"; wantedBy = [ "${unit}.service" ]; after = [ "${unit}.service" ]; unitConfig.ConditionPathExists = [ netrc provisionKey ]; serviceConfig = { Type = "oneshot"; ExecStart = pkgs.writeShellScript "${unit}-netrc" '' set -eu for _ in $(${lib.getExe' pkgs.coreutils "seq"} 1 60); do if ${guestSsh} true 2>/dev/null; then # Piped from the host's file at runtime; the credential is never # an argument, never in a derivation, never in the store. ${guestSsh} 'umask 022; cat > /etc/nix/netrc' < ${netrc} ${guestSsh} 'systemctl restart nix-daemon' || true echo "netrc installed in ${vm}" exit 0 fi ${lib.getExe' pkgs.coreutils "sleep"} 5 done echo "${vm} did not answer ssh within 5min; netrc not installed" >&2 exit 1 ''; }; }; # Reachable from lpt2 (where the hive-controller's agents live) over # tailscale/LAN as :2222. networking.firewall.allowedTCPPorts = [ sshPort ]; environment.systemPackages = [ reset ]; programs.ssh.extraConfig = '' Host ${vm} HostName 127.0.0.1 Port ${toString sshPort} User root StrictHostKeyChecking no UserKnownHostsFile /dev/null ''; }; }