From 873622dc8a0599aac6dab3d28c2f45bb866a4a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?m=C3=BCde?= Date: Wed, 18 Nov 2026 18:40:12 +0100 Subject: [PATCH] add agent-lab sandbox VM on muede-pc2 --- README.md | 18 ++ devices.nix | 4 + nixosConfigurations/agent-lab/default.nix | 39 ++++ nixosConfigurations/agent-lab/hardware.nix | 46 ++++ .../muede-pc2/agent-lab-vm.nix | 216 ++++++++++++++++++ nixosConfigurations/muede-pc2/default.nix | 1 + 6 files changed, 324 insertions(+) create mode 100644 nixosConfigurations/agent-lab/default.nix create mode 100644 nixosConfigurations/agent-lab/hardware.nix create mode 100644 nixosConfigurations/muede-pc2/agent-lab-vm.nix diff --git a/README.md b/README.md index 3ad5e88..7696955 100644 --- a/README.md +++ b/README.md @@ -44,3 +44,21 @@ Machines are configured to act as build servers / binary caches for each other i ``` 4. Rebuild all machines so they trust the new signing key. + +## agent-lab sandbox VM + +`agent-lab` is a throwaway NixOS VM on muede-pc2 for hyperhive agents to ssh +into and apply risky configs to. The host (`nixosConfigurations/muede-pc2/agent-lab-vm.nix`) +only seeds its disk and runs qemu — it never declares the guest's system, so +the guest can `nixos-rebuild switch` itself freely. + +- Reachable at `muede-pc2:2222` (qemu user-mode NAT forward), root ssh only, + authorized for the lpt2 and pc2 keys plus muede-pc2's build key. +- `sudo agent-lab-reset [flake-ref]` rebuilds the seed image, replaces the + disk, and restarts the VM. Takes seconds once the image is in the store. + Defaults to the upstream flake; pass a local path to test uncommitted work. +- `/etc/nix/netrc` is copied in over ssh on every VM start by + `agent-lab-vm-netrc.service`, so the private cache credential stays out of + the nix store. The host needs `/etc/nix/distributed-build-key` for this. +- The guest's own config is `nixosConfigurations/agent-lab/`; it is the seed + only. Whatever the agents switch to afterwards is not tracked here. diff --git a/devices.nix b/devices.nix index f9bb8c8..b4fc9dc 100644 --- a/devices.nix +++ b/devices.nix @@ -15,6 +15,10 @@ in # system = "aarch64-linux"; # nixosSystem = nixos-raspberrypi.lib.nixosInstaller; # }; + agent-lab = { + # sandbox VM on muede-pc2; see nixosConfigurations/muede-pc2/agent-lab-vm.nix + system = "x86_64-linux"; + }; compute1-vm = { system = "x86_64-linux"; }; diff --git a/nixosConfigurations/agent-lab/default.nix b/nixosConfigurations/agent-lab/default.nix new file mode 100644 index 0000000..ccaa921 --- /dev/null +++ b/nixosConfigurations/agent-lab/default.nix @@ -0,0 +1,39 @@ +{ + imports = [ ./hardware.nix ]; + + config = { + # A deliberately disposable box: hyperhive agents ssh in as root and apply + # whatever config they are testing, up to and including breaking it. The + # host (see ../muede-pc2/agent-lab-vm.nix) only ever seeds the disk and + # runs qemu — it never declares this system's config, so the guest is free + # to `nixos-rebuild switch` itself out from under the seed. + + # Agents log in by certificate, not by key. sshd accepts one the swarm's + # SSH CA signed, and the single principal on it names the agent, so the + # list below is what admits atlas and maia and nobody else. Dropping a name + # ends that agent's logins at its next attempt; a signed certificate + # expires within three hours either way. + # + # Spelled out rather than imported from hyperhive.nixosModules.agent-ssh-target + # on purpose: this VM should not take the hyperhive flake as an input. + # Re-read the CA key from the swarm's store after the CA is regenerated: + # bao read -field=public_key ssh-agents/config/ca + environment.etc."ssh/hyperhive-agent-ssh-ca.pub".text = + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILHGXr3Voq/SVorEoBY5cDMqTumQyHMweNQvWcSvTXuN\n"; + services.openssh.settings.TrustedUserCAKeys = "/etc/ssh/hyperhive-agent-ssh-ca.pub"; + users.users.root.openssh.authorizedPrincipals = [ + "atlas" + "maia" + ]; + + users.users.root.openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO1CRn4yYTL4XUdCebE8Z4ZeuMujBjorTdWifg911EOv pc2 home roaming" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPDNpLDmctyqGpow/ElQvdhY4BLBPS/sigDJ1QEcC7wC lpt2-roaming" + # muede-pc2's build key, reused as the host's provisioning identity: it is + # how /etc/nix/netrc reaches the guest after boot without the credential + # ever passing through the nix store. Private half stays on the host in + # /etc/nix/distributed-build-key. + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHmnyhP6L+kGHV15cb/d31AQr50wSEaQhkUBwy2+OEKk muede-pc2-nix-builds" + ]; + }; +} diff --git a/nixosConfigurations/agent-lab/hardware.nix b/nixosConfigurations/agent-lab/hardware.nix new file mode 100644 index 0000000..c27b3b6 --- /dev/null +++ b/nixosConfigurations/agent-lab/hardware.nix @@ -0,0 +1,46 @@ +{ + lib, + modulesPath, + ... +}: +{ + imports = [ (modulesPath + "/profiles/qemu-guest.nix") ]; + + config = { + # Legacy BIOS boot via GRUB on the virtio disk, matching the `qemu` image + # variant (`image.efiSupport = false`). The repo-wide systemd-boot default + # would need an ESP plus EFI firmware and a writable varstore in the + # runner, and a guest that reinstalls its own bootloader on every + # `nixos-rebuild switch` is one less thing for an agent to brick. Drop that + # module wholesale -- it also owns boot.loader.timeout and the EFI vars. + my.systemdBoot.enable = lib.mkForce false; + + boot = { + loader = { + grub = { + enable = true; + devices = [ "/dev/vda" ]; + }; + timeout = 1; + }; + + # The runner resizes the seeded image before first boot (the image itself + # is sized to its closure), so / has to grow into the rest of the disk. + growPartition = true; + + kernelParams = [ "console=ttyS0" ]; + tmp.cleanOnBoot = true; + }; + + # Labels, not UUIDs: every reseed makes a brand new filesystem, and the + # point of this machine is being thrown away and remade. + fileSystems."/" = { + device = "/dev/disk/by-label/nixos"; + autoResize = true; + fsType = "ext4"; + }; + + # qemu user-mode networking hands out a lease on the single virtio NIC. + networking.useDHCP = lib.mkDefault true; + }; +} diff --git a/nixosConfigurations/muede-pc2/agent-lab-vm.nix b/nixosConfigurations/muede-pc2/agent-lab-vm.nix new file mode 100644 index 0000000..45846f0 --- /dev/null +++ b/nixosConfigurations/muede-pc2/agent-lab-vm.nix @@ -0,0 +1,216 @@ +{ + lib, + pkgs, + ... +}: +let + vm = "agent-lab"; + unit = "${vm}-vm"; + stateDir = "/var/lib/${unit}"; + disk = "${stateDir}/disk.qcow2"; + qmp = "/run/${unit}/qmp.sock"; + + sshPort = 2222; + cpus = 4; + memoryMiB = 8192; + diskSize = "60G"; + + # Deliberately a flake *reference*, resolved when the reset script runs, not + # a store path baked into this host's closure. The host must not pin the + # guest's config: agents rebuild the guest from inside, and a pinned + # `system.build.toplevel` here (the microvm.nix model) would fight that. All + # the host ever does is lay down a first-boot disk. + seedFlake = "git+https://forge.darkest.space/mara/nixos-configs.git"; + seedAttr = "nixosConfigurations.${vm}.config.system.build.images.qemu"; + + # Credentials the guest needs but the nix store must never see. Both are + # hand-placed, out-of-store files on this host (same as on every other + # machine here), and are copied in over ssh after the guest is up. + provisionKey = "/etc/nix/distributed-build-key"; + netrc = "/etc/nix/netrc"; + + # ssh to the guest as the provisioning identity. The host key changes on + # every reseed, and this target is 127.0.0.1 behind qemu's user-mode NAT, so + # host key checking buys nothing here. + guestSsh = lib.concatStringsSep " " [ + (lib.getExe' pkgs.openssh "ssh") + "-i ${provisionKey}" + "-o IdentitiesOnly=yes" + "-o StrictHostKeyChecking=no" + "-o UserKnownHostsFile=/dev/null" + "-o LogLevel=ERROR" + "-o BatchMode=yes" + "-o ConnectTimeout=5" + "-p ${toString sshPort}" + "root@127.0.0.1" + ]; + + reset = pkgs.writeShellApplication { + name = "${vm}-reset"; + runtimeInputs = [ + pkgs.qemu_kvm + pkgs.findutils + pkgs.coreutils + pkgs.systemd + pkgs.nix + ]; + text = '' + if [ "$(id -u)" != 0 ]; then + echo "${vm}-reset: needs root (writes ${stateDir})" >&2 + exit 1 + fi + + flake="''${1:-${seedFlake}}" + echo "${vm}-reset: building seed image from $flake#${seedAttr}" >&2 + out=$(nix build --no-link --print-out-paths "$flake#${seedAttr}") + + img=$(find "$out" -maxdepth 1 -name '*.qcow2' -print -quit) + if [ -z "$img" ]; then + echo "${vm}-reset: no qcow2 in $out" >&2 + exit 1 + fi + + systemctl stop ${unit}.service || true + + # The tmpfiles rule above makes this exist at activation; keep the + # script able to stand alone if the directory was removed by hand. + install -d -m0700 -o ${unit} -g ${unit} ${stateDir} + + # `convert`, not a copy or an overlay: the running disk must not keep a + # backing-file reference into the nix store, or a later GC of the seed + # would pull the VM's filesystem out from under it. + rm -f ${disk}.new + qemu-img convert -O qcow2 "$img" ${disk}.new + qemu-img resize ${disk}.new ${diskSize} + chown ${unit}:${unit} ${disk}.new + chmod 0600 ${disk}.new + mv -f ${disk}.new ${disk} + + systemctl start ${unit}.service + echo "${vm}-reset: fresh disk seeded, VM starting — ssh -p ${toString sshPort} root@" >&2 + ''; + }; +in +{ + config = { + users.users.${unit} = { + isSystemUser = true; + group = unit; + home = stateDir; + description = "agent-lab sandbox VM"; + }; + users.groups.${unit} = { }; + + # `StateDirectory=` in the unit below only creates this when the unit + # starts, and the unit cannot start until the disk exists -- so the very + # first `agent-lab-reset` would have nowhere to write. Declare it here. + systemd.tmpfiles.rules = [ "d ${stateDir} 0700 ${unit} ${unit} -" ]; + + systemd.services.${unit} = { + description = "agent-lab sandbox VM (ssh target for hyperhive agents)"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + # No disk yet means the VM was never seeded; `agent-lab-reset` makes one. + unitConfig.ConditionPathExists = disk; + + serviceConfig = { + User = unit; + Group = unit; + SupplementaryGroups = [ "kvm" ]; + StateDirectory = unit; + StateDirectoryMode = "0700"; + RuntimeDirectory = unit; + + ExecStart = lib.concatStringsSep " " [ + (lib.getExe' pkgs.qemu_kvm "qemu-system-x86_64") + "-name ${vm}" + "-machine q35,accel=kvm" + "-cpu host" + "-smp ${toString cpus}" + "-m ${toString memoryMiB}" + "-nodefaults" + "-no-user-config" + "-display none" + "-serial stdio" + "-device virtio-rng-pci" + "-drive file=${disk},if=virtio,format=qcow2,cache=writeback,discard=unmap" + # User-mode NAT: outbound works (the guest needs to reach the caches + # to rebuild itself), and only the forwarded ssh port is reachable + # from outside. Nothing the agents do in here can squat a LAN IP. + "-netdev user,id=net0,hostfwd=tcp::${toString sshPort}-:22" + "-device virtio-net-pci,netdev=net0" + "-qmp unix:${qmp},server=on,wait=off" + ]; + + # ACPI powerdown over QMP rather than `ssh poweroff`: on a box whose + # whole job is surviving agent experiments, sshd is exactly the thing + # that may be broken when it is time to stop. + ExecStop = pkgs.writeShellScript "${unit}-stop" '' + printf '%s\n' \ + '{"execute":"qmp_capabilities"}' \ + '{"execute":"system_powerdown"}' \ + | ${lib.getExe pkgs.socat} -t2 - UNIX-CONNECT:${qmp} || true + ''; + TimeoutStopSec = "90s"; + Restart = "always"; + RestartSec = "5s"; + StandardInput = "null"; + + NoNewPrivileges = true; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + RestrictSUIDSGID = true; + }; + }; + + # Runs on every VM start, not just after a reseed: if an agent wipes + # /etc/nix in there, the next restart puts the cache credential back. + systemd.services."${unit}-netrc" = { + description = "Copy the private cache netrc into the agent-lab VM"; + wantedBy = [ "${unit}.service" ]; + after = [ "${unit}.service" ]; + unitConfig.ConditionPathExists = [ + netrc + provisionKey + ]; + + serviceConfig = { + Type = "oneshot"; + ExecStart = pkgs.writeShellScript "${unit}-netrc" '' + set -eu + + for _ in $(${lib.getExe' pkgs.coreutils "seq"} 1 60); do + if ${guestSsh} true 2>/dev/null; then + # Piped from the host's file at runtime; the credential is never + # an argument, never in a derivation, never in the store. + ${guestSsh} 'umask 022; cat > /etc/nix/netrc' < ${netrc} + ${guestSsh} 'systemctl restart nix-daemon' || true + echo "netrc installed in ${vm}" + exit 0 + fi + ${lib.getExe' pkgs.coreutils "sleep"} 5 + done + + echo "${vm} did not answer ssh within 5min; netrc not installed" >&2 + exit 1 + ''; + }; + }; + + # Reachable from lpt2 (where the hive-controller's agents live) over + # tailscale/LAN as :2222. + networking.firewall.allowedTCPPorts = [ sshPort ]; + + environment.systemPackages = [ reset ]; + + programs.ssh.extraConfig = '' + Host ${vm} + HostName 127.0.0.1 + Port ${toString sshPort} + User root + StrictHostKeyChecking no + UserKnownHostsFile /dev/null + ''; + }; +} diff --git a/nixosConfigurations/muede-pc2/default.nix b/nixosConfigurations/muede-pc2/default.nix index 6ea8bc7..34c15ca 100644 --- a/nixosConfigurations/muede-pc2/default.nix +++ b/nixosConfigurations/muede-pc2/default.nix @@ -4,6 +4,7 @@ ./hardware.nix ./gitea-runner.nix ./hive-ci-runner.nix + ./agent-lab-vm.nix ./k8s-test-swarm.nix # ./vscode-server.nix # ./hass.nix