mirror of
https://forge.darkest.space/mara/nixos-configs.git
synced 2026-11-19 11:13:59 +01:00
add agent-lab sandbox VM on muede-pc2
This commit is contained in:
parent
bb4bef7aef
commit
873622dc8a
6 changed files with 324 additions and 0 deletions
18
README.md
18
README.md
|
|
@ -44,3 +44,21 @@ Machines are configured to act as build servers / binary caches for each other i
|
|||
```
|
||||
|
||||
4. Rebuild all machines so they trust the new signing key.
|
||||
|
||||
## agent-lab sandbox VM
|
||||
|
||||
`agent-lab` is a throwaway NixOS VM on muede-pc2 for hyperhive agents to ssh
|
||||
into and apply risky configs to. The host (`nixosConfigurations/muede-pc2/agent-lab-vm.nix`)
|
||||
only seeds its disk and runs qemu — it never declares the guest's system, so
|
||||
the guest can `nixos-rebuild switch` itself freely.
|
||||
|
||||
- Reachable at `muede-pc2:2222` (qemu user-mode NAT forward), root ssh only,
|
||||
authorized for the lpt2 and pc2 keys plus muede-pc2's build key.
|
||||
- `sudo agent-lab-reset [flake-ref]` rebuilds the seed image, replaces the
|
||||
disk, and restarts the VM. Takes seconds once the image is in the store.
|
||||
Defaults to the upstream flake; pass a local path to test uncommitted work.
|
||||
- `/etc/nix/netrc` is copied in over ssh on every VM start by
|
||||
`agent-lab-vm-netrc.service`, so the private cache credential stays out of
|
||||
the nix store. The host needs `/etc/nix/distributed-build-key` for this.
|
||||
- The guest's own config is `nixosConfigurations/agent-lab/`; it is the seed
|
||||
only. Whatever the agents switch to afterwards is not tracked here.
|
||||
|
|
|
|||
|
|
@ -15,6 +15,10 @@ in
|
|||
# system = "aarch64-linux";
|
||||
# nixosSystem = nixos-raspberrypi.lib.nixosInstaller;
|
||||
# };
|
||||
agent-lab = {
|
||||
# sandbox VM on muede-pc2; see nixosConfigurations/muede-pc2/agent-lab-vm.nix
|
||||
system = "x86_64-linux";
|
||||
};
|
||||
compute1-vm = {
|
||||
system = "x86_64-linux";
|
||||
};
|
||||
|
|
|
|||
39
nixosConfigurations/agent-lab/default.nix
Normal file
39
nixosConfigurations/agent-lab/default.nix
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
{
|
||||
imports = [ ./hardware.nix ];
|
||||
|
||||
config = {
|
||||
# A deliberately disposable box: hyperhive agents ssh in as root and apply
|
||||
# whatever config they are testing, up to and including breaking it. The
|
||||
# host (see ../muede-pc2/agent-lab-vm.nix) only ever seeds the disk and
|
||||
# runs qemu — it never declares this system's config, so the guest is free
|
||||
# to `nixos-rebuild switch` itself out from under the seed.
|
||||
|
||||
# Agents log in by certificate, not by key. sshd accepts one the swarm's
|
||||
# SSH CA signed, and the single principal on it names the agent, so the
|
||||
# list below is what admits atlas and maia and nobody else. Dropping a name
|
||||
# ends that agent's logins at its next attempt; a signed certificate
|
||||
# expires within three hours either way.
|
||||
#
|
||||
# Spelled out rather than imported from hyperhive.nixosModules.agent-ssh-target
|
||||
# on purpose: this VM should not take the hyperhive flake as an input.
|
||||
# Re-read the CA key from the swarm's store after the CA is regenerated:
|
||||
# bao read -field=public_key ssh-agents/config/ca
|
||||
environment.etc."ssh/hyperhive-agent-ssh-ca.pub".text =
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILHGXr3Voq/SVorEoBY5cDMqTumQyHMweNQvWcSvTXuN\n";
|
||||
services.openssh.settings.TrustedUserCAKeys = "/etc/ssh/hyperhive-agent-ssh-ca.pub";
|
||||
users.users.root.openssh.authorizedPrincipals = [
|
||||
"atlas"
|
||||
"maia"
|
||||
];
|
||||
|
||||
users.users.root.openssh.authorizedKeys.keys = [
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO1CRn4yYTL4XUdCebE8Z4ZeuMujBjorTdWifg911EOv pc2 home roaming"
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPDNpLDmctyqGpow/ElQvdhY4BLBPS/sigDJ1QEcC7wC lpt2-roaming"
|
||||
# muede-pc2's build key, reused as the host's provisioning identity: it is
|
||||
# how /etc/nix/netrc reaches the guest after boot without the credential
|
||||
# ever passing through the nix store. Private half stays on the host in
|
||||
# /etc/nix/distributed-build-key.
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHmnyhP6L+kGHV15cb/d31AQr50wSEaQhkUBwy2+OEKk muede-pc2-nix-builds"
|
||||
];
|
||||
};
|
||||
}
|
||||
46
nixosConfigurations/agent-lab/hardware.nix
Normal file
46
nixosConfigurations/agent-lab/hardware.nix
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
{
|
||||
lib,
|
||||
modulesPath,
|
||||
...
|
||||
}:
|
||||
{
|
||||
imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
|
||||
|
||||
config = {
|
||||
# Legacy BIOS boot via GRUB on the virtio disk, matching the `qemu` image
|
||||
# variant (`image.efiSupport = false`). The repo-wide systemd-boot default
|
||||
# would need an ESP plus EFI firmware and a writable varstore in the
|
||||
# runner, and a guest that reinstalls its own bootloader on every
|
||||
# `nixos-rebuild switch` is one less thing for an agent to brick. Drop that
|
||||
# module wholesale -- it also owns boot.loader.timeout and the EFI vars.
|
||||
my.systemdBoot.enable = lib.mkForce false;
|
||||
|
||||
boot = {
|
||||
loader = {
|
||||
grub = {
|
||||
enable = true;
|
||||
devices = [ "/dev/vda" ];
|
||||
};
|
||||
timeout = 1;
|
||||
};
|
||||
|
||||
# The runner resizes the seeded image before first boot (the image itself
|
||||
# is sized to its closure), so / has to grow into the rest of the disk.
|
||||
growPartition = true;
|
||||
|
||||
kernelParams = [ "console=ttyS0" ];
|
||||
tmp.cleanOnBoot = true;
|
||||
};
|
||||
|
||||
# Labels, not UUIDs: every reseed makes a brand new filesystem, and the
|
||||
# point of this machine is being thrown away and remade.
|
||||
fileSystems."/" = {
|
||||
device = "/dev/disk/by-label/nixos";
|
||||
autoResize = true;
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
# qemu user-mode networking hands out a lease on the single virtio NIC.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
};
|
||||
}
|
||||
216
nixosConfigurations/muede-pc2/agent-lab-vm.nix
Normal file
216
nixosConfigurations/muede-pc2/agent-lab-vm.nix
Normal file
|
|
@ -0,0 +1,216 @@
|
|||
{
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
vm = "agent-lab";
|
||||
unit = "${vm}-vm";
|
||||
stateDir = "/var/lib/${unit}";
|
||||
disk = "${stateDir}/disk.qcow2";
|
||||
qmp = "/run/${unit}/qmp.sock";
|
||||
|
||||
sshPort = 2222;
|
||||
cpus = 4;
|
||||
memoryMiB = 8192;
|
||||
diskSize = "60G";
|
||||
|
||||
# Deliberately a flake *reference*, resolved when the reset script runs, not
|
||||
# a store path baked into this host's closure. The host must not pin the
|
||||
# guest's config: agents rebuild the guest from inside, and a pinned
|
||||
# `system.build.toplevel` here (the microvm.nix model) would fight that. All
|
||||
# the host ever does is lay down a first-boot disk.
|
||||
seedFlake = "git+https://forge.darkest.space/mara/nixos-configs.git";
|
||||
seedAttr = "nixosConfigurations.${vm}.config.system.build.images.qemu";
|
||||
|
||||
# Credentials the guest needs but the nix store must never see. Both are
|
||||
# hand-placed, out-of-store files on this host (same as on every other
|
||||
# machine here), and are copied in over ssh after the guest is up.
|
||||
provisionKey = "/etc/nix/distributed-build-key";
|
||||
netrc = "/etc/nix/netrc";
|
||||
|
||||
# ssh to the guest as the provisioning identity. The host key changes on
|
||||
# every reseed, and this target is 127.0.0.1 behind qemu's user-mode NAT, so
|
||||
# host key checking buys nothing here.
|
||||
guestSsh = lib.concatStringsSep " " [
|
||||
(lib.getExe' pkgs.openssh "ssh")
|
||||
"-i ${provisionKey}"
|
||||
"-o IdentitiesOnly=yes"
|
||||
"-o StrictHostKeyChecking=no"
|
||||
"-o UserKnownHostsFile=/dev/null"
|
||||
"-o LogLevel=ERROR"
|
||||
"-o BatchMode=yes"
|
||||
"-o ConnectTimeout=5"
|
||||
"-p ${toString sshPort}"
|
||||
"root@127.0.0.1"
|
||||
];
|
||||
|
||||
reset = pkgs.writeShellApplication {
|
||||
name = "${vm}-reset";
|
||||
runtimeInputs = [
|
||||
pkgs.qemu_kvm
|
||||
pkgs.findutils
|
||||
pkgs.coreutils
|
||||
pkgs.systemd
|
||||
pkgs.nix
|
||||
];
|
||||
text = ''
|
||||
if [ "$(id -u)" != 0 ]; then
|
||||
echo "${vm}-reset: needs root (writes ${stateDir})" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
flake="''${1:-${seedFlake}}"
|
||||
echo "${vm}-reset: building seed image from $flake#${seedAttr}" >&2
|
||||
out=$(nix build --no-link --print-out-paths "$flake#${seedAttr}")
|
||||
|
||||
img=$(find "$out" -maxdepth 1 -name '*.qcow2' -print -quit)
|
||||
if [ -z "$img" ]; then
|
||||
echo "${vm}-reset: no qcow2 in $out" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
systemctl stop ${unit}.service || true
|
||||
|
||||
# The tmpfiles rule above makes this exist at activation; keep the
|
||||
# script able to stand alone if the directory was removed by hand.
|
||||
install -d -m0700 -o ${unit} -g ${unit} ${stateDir}
|
||||
|
||||
# `convert`, not a copy or an overlay: the running disk must not keep a
|
||||
# backing-file reference into the nix store, or a later GC of the seed
|
||||
# would pull the VM's filesystem out from under it.
|
||||
rm -f ${disk}.new
|
||||
qemu-img convert -O qcow2 "$img" ${disk}.new
|
||||
qemu-img resize ${disk}.new ${diskSize}
|
||||
chown ${unit}:${unit} ${disk}.new
|
||||
chmod 0600 ${disk}.new
|
||||
mv -f ${disk}.new ${disk}
|
||||
|
||||
systemctl start ${unit}.service
|
||||
echo "${vm}-reset: fresh disk seeded, VM starting — ssh -p ${toString sshPort} root@<host>" >&2
|
||||
'';
|
||||
};
|
||||
in
|
||||
{
|
||||
config = {
|
||||
users.users.${unit} = {
|
||||
isSystemUser = true;
|
||||
group = unit;
|
||||
home = stateDir;
|
||||
description = "agent-lab sandbox VM";
|
||||
};
|
||||
users.groups.${unit} = { };
|
||||
|
||||
# `StateDirectory=` in the unit below only creates this when the unit
|
||||
# starts, and the unit cannot start until the disk exists -- so the very
|
||||
# first `agent-lab-reset` would have nowhere to write. Declare it here.
|
||||
systemd.tmpfiles.rules = [ "d ${stateDir} 0700 ${unit} ${unit} -" ];
|
||||
|
||||
systemd.services.${unit} = {
|
||||
description = "agent-lab sandbox VM (ssh target for hyperhive agents)";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network.target" ];
|
||||
# No disk yet means the VM was never seeded; `agent-lab-reset` makes one.
|
||||
unitConfig.ConditionPathExists = disk;
|
||||
|
||||
serviceConfig = {
|
||||
User = unit;
|
||||
Group = unit;
|
||||
SupplementaryGroups = [ "kvm" ];
|
||||
StateDirectory = unit;
|
||||
StateDirectoryMode = "0700";
|
||||
RuntimeDirectory = unit;
|
||||
|
||||
ExecStart = lib.concatStringsSep " " [
|
||||
(lib.getExe' pkgs.qemu_kvm "qemu-system-x86_64")
|
||||
"-name ${vm}"
|
||||
"-machine q35,accel=kvm"
|
||||
"-cpu host"
|
||||
"-smp ${toString cpus}"
|
||||
"-m ${toString memoryMiB}"
|
||||
"-nodefaults"
|
||||
"-no-user-config"
|
||||
"-display none"
|
||||
"-serial stdio"
|
||||
"-device virtio-rng-pci"
|
||||
"-drive file=${disk},if=virtio,format=qcow2,cache=writeback,discard=unmap"
|
||||
# User-mode NAT: outbound works (the guest needs to reach the caches
|
||||
# to rebuild itself), and only the forwarded ssh port is reachable
|
||||
# from outside. Nothing the agents do in here can squat a LAN IP.
|
||||
"-netdev user,id=net0,hostfwd=tcp::${toString sshPort}-:22"
|
||||
"-device virtio-net-pci,netdev=net0"
|
||||
"-qmp unix:${qmp},server=on,wait=off"
|
||||
];
|
||||
|
||||
# ACPI powerdown over QMP rather than `ssh poweroff`: on a box whose
|
||||
# whole job is surviving agent experiments, sshd is exactly the thing
|
||||
# that may be broken when it is time to stop.
|
||||
ExecStop = pkgs.writeShellScript "${unit}-stop" ''
|
||||
printf '%s\n' \
|
||||
'{"execute":"qmp_capabilities"}' \
|
||||
'{"execute":"system_powerdown"}' \
|
||||
| ${lib.getExe pkgs.socat} -t2 - UNIX-CONNECT:${qmp} || true
|
||||
'';
|
||||
TimeoutStopSec = "90s";
|
||||
Restart = "always";
|
||||
RestartSec = "5s";
|
||||
StandardInput = "null";
|
||||
|
||||
NoNewPrivileges = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateTmp = true;
|
||||
RestrictSUIDSGID = true;
|
||||
};
|
||||
};
|
||||
|
||||
# Runs on every VM start, not just after a reseed: if an agent wipes
|
||||
# /etc/nix in there, the next restart puts the cache credential back.
|
||||
systemd.services."${unit}-netrc" = {
|
||||
description = "Copy the private cache netrc into the agent-lab VM";
|
||||
wantedBy = [ "${unit}.service" ];
|
||||
after = [ "${unit}.service" ];
|
||||
unitConfig.ConditionPathExists = [
|
||||
netrc
|
||||
provisionKey
|
||||
];
|
||||
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = pkgs.writeShellScript "${unit}-netrc" ''
|
||||
set -eu
|
||||
|
||||
for _ in $(${lib.getExe' pkgs.coreutils "seq"} 1 60); do
|
||||
if ${guestSsh} true 2>/dev/null; then
|
||||
# Piped from the host's file at runtime; the credential is never
|
||||
# an argument, never in a derivation, never in the store.
|
||||
${guestSsh} 'umask 022; cat > /etc/nix/netrc' < ${netrc}
|
||||
${guestSsh} 'systemctl restart nix-daemon' || true
|
||||
echo "netrc installed in ${vm}"
|
||||
exit 0
|
||||
fi
|
||||
${lib.getExe' pkgs.coreutils "sleep"} 5
|
||||
done
|
||||
|
||||
echo "${vm} did not answer ssh within 5min; netrc not installed" >&2
|
||||
exit 1
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
# Reachable from lpt2 (where the hive-controller's agents live) over
|
||||
# tailscale/LAN as <this host>:2222.
|
||||
networking.firewall.allowedTCPPorts = [ sshPort ];
|
||||
|
||||
environment.systemPackages = [ reset ];
|
||||
|
||||
programs.ssh.extraConfig = ''
|
||||
Host ${vm}
|
||||
HostName 127.0.0.1
|
||||
Port ${toString sshPort}
|
||||
User root
|
||||
StrictHostKeyChecking no
|
||||
UserKnownHostsFile /dev/null
|
||||
'';
|
||||
};
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@
|
|||
./hardware.nix
|
||||
./gitea-runner.nix
|
||||
./hive-ci-runner.nix
|
||||
./agent-lab-vm.nix
|
||||
./k8s-test-swarm.nix
|
||||
# ./vscode-server.nix
|
||||
# ./hass.nix
|
||||
|
|
|
|||
Loading…
Reference in a new issue