diff --git a/README.md b/README.md index f97d4b5..3ad5e88 100644 --- a/README.md +++ b/README.md @@ -9,28 +9,38 @@ Machines are configured to act as build servers / binary caches for each other i ### Onboarding a device as a build client 1. Generate a key pair on the device: + + ```sh + sudo ssh-keygen -t ed25519 -f /etc/nix/distributed-build-key -N "" -C "$(hostname)-nix-builds" && sudo cat /etc/nix/distributed-build-key.pub ``` - sudo ssh-keygen -t ed25519 -f /etc/nix/distributed-build-key -N "" -C "$(hostname)-nix-builds" - ``` + 2. Add the public key to the device entry in `devices.nix`: + ```nix distributedBuilds.clientPublicKey = "ssh-ed25519 AAAA... -nix-builds"; ``` + 3. Rebuild all build machines so they pick up the new authorized key. ### Adding a build server 1. Add to its entry in `devices.nix`: + ```nix distributedBuilds.isBuilder = true; - distributedBuilds.hostPublicKey = "ssh-ed25519 AAAA..."; # from: ssh-keyscan -t ed25519 + distributedBuilds.hostPublicKey = "ssh-ed25519 AAAA..."; # from: ssh-keyscan -t ed25519 "$(hostname)" ``` + 2. Generate a store signing key on the builder: - ``` + + ```sh sudo nix key generate-secret --key-name "$(hostname)" | sudo tee /etc/nix/signing-key.sec | sudo nix key convert-secret-to-public ``` + 3. Add the printed public key to `devices.nix`: + ```nix distributedBuilds.storeSigningPublicKey = ":"; ``` + 4. Rebuild all machines so they trust the new signing key. diff --git a/devices.nix b/devices.nix index c17d02e..ea3c6f6 100644 --- a/devices.nix +++ b/devices.nix @@ -26,6 +26,12 @@ in }; forgejo-runner-1 = { system = "aarch64-linux"; + distributedBuilds = { + isBuilder = true; + clientPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK0NLgg0sFobBWz/bjYs9WkrMvlcvJC5F6+3jQ/b+AnD forgejo-runner-1-nix-builds"; + hostPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIANGC89GiT5xCsFICwrharrbV3q7acWHqk6ZwOUXbtGT"; + storeSigningPublicKey = "forgejo-runner-1:ln1FVLL8G5+IveQuBi/Kn3SaqFZ1gaiQrE3yPlMhCMA="; + }; }; hetzner-vpn2 = { system = "aarch64-linux"; diff --git a/flake.nix b/flake.nix index 01c4906..e95af0e 100644 --- a/flake.nix +++ b/flake.nix @@ -146,7 +146,11 @@ niri = niri.overlays.niri; }; - nixosModules = importModuleDir ./nixosModules; + nixosModules = (importModuleDir ./nixosModules) // { + default = { + imports = builtins.attrValues (builtins.removeAttrs self.nixosModules [ "default" ]); + }; + }; homeModules = importModuleDir ./homeModules; homeConfigurations = { diff --git a/nixosConfigurations.nix b/nixosConfigurations.nix index 9fb2cf2..29e78c3 100644 --- a/nixosConfigurations.nix +++ b/nixosConfigurations.nix @@ -3,7 +3,7 @@ lib, }: let - devices = import ./devices.nix { inherit (inputs) self; }; + allDevices = import ./devices.nix { inherit (inputs) self; }; inherit (inputs) self home-manager @@ -15,7 +15,7 @@ let stylix zerforschen-plus ; - forDevice = f: lib.mapAttrs (device: value: f (value // { inherit device; })) devices; + forDevice = f: lib.mapAttrs (device: value: f (value // { inherit device; })) allDevices; in forDevice ( { @@ -24,10 +24,15 @@ forDevice ( home-manager-users ? { }, nixosSystem ? inputs.nixpkgs.lib.nixosSystem, ... - }: + }@thisDevice: let specialArgs = inputs // { - inherit device home-manager-users devices; + inherit + device + home-manager-users + allDevices + thisDevice + ; }; in nixosSystem { diff --git a/nixosModules/default.nix b/nixosModules/default.nix deleted file mode 100644 index 2808b2a..0000000 --- a/nixosModules/default.nix +++ /dev/null @@ -1,39 +0,0 @@ -{ ... }: -{ - imports = [ - # keep-sorted start - ./allowed-unfree-list.nix - ./amd-graphics.nix - ./autoupdate.nix - ./distributed-builds.nix - ./en-de.nix - ./extra-caches.nix - ./firmware-updates.nix - ./git.nix - ./globalinstalls.nix - ./gnome.nix - ./intel-graphics.nix - ./kdeconnect.nix - ./latex.nix - ./lix-is-nix.nix - ./modern-desktop.nix - ./muede-desktop-settings.nix - ./nix-ld.nix - ./nixpkgs-overlays.nix - ./openssh.nix - ./podman.nix - ./printing.nix - ./prometheus-node.nix - ./pxvirt-guest.nix - ./quiet-boot.nix - ./secure-boot.nix - ./steam.nix - ./stylix.nix - ./systemd-boot.nix - ./tailscale.nix - ./user-muede.nix - ./user-ronja.nix - ./wine-gaming.nix - # keep-sorted end - ]; -} diff --git a/nixosModules/distributed-builds.nix b/nixosModules/distributed-builds.nix index 32a8f34..1d0a55e 100644 --- a/nixosModules/distributed-builds.nix +++ b/nixosModules/distributed-builds.nix @@ -1,24 +1,27 @@ { config, lib, - devices, + allDevices, + thisDevice, ... }: let - sshKeyPath = "/etc/nix/distributed-build-key"; + clientSshKeyPath = "/etc/nix/distributed-build-key"; buildUser = "remotebuild"; # Collect all per-device public keys that have been registered. - authorizedPublicKeys = lib.pipe devices [ + allClientPublicKeys = lib.pipe allDevices [ (lib.filterAttrs (_: v: (v.distributedBuilds or { }) ? clientPublicKey)) (lib.mapAttrsToList (_: v: v.distributedBuilds.clientPublicKey)) ]; + isClient = (thisDevice.distributedBuilds or { }) ? clientPublicKey; + buildServerDevices = lib.filterAttrs ( _: v: (v.distributedBuilds or { }).isBuilder or false - ) devices; + ) allDevices; - knownHosts = lib.pipe buildServerDevices [ + buildServerKnownHosts = lib.pipe buildServerDevices [ (lib.filterAttrs (_: v: v.distributedBuilds ? hostPublicKey)) (lib.mapAttrs ( _: v: { @@ -27,17 +30,21 @@ let )) ]; - buildMachineList = lib.mapAttrsToList ( - hostName: v: + remoteBuildServerDevices = builtins.filter ( + m: m.hostName != config.networking.hostName + ) (lib.mapAttrsToList (name: v: v // { hostName = name; }) buildServerDevices); + + buildMachines = map ( + m: { - inherit hostName; - systems = [ v.system ]; + hostName = m.hostName; + systems = [ m.system ]; sshUser = buildUser; - sshKey = sshKeyPath; + sshKey = clientSshKeyPath; protocol = "ssh-ng"; } - // lib.optionalAttrs (v.distributedBuilds ? speedFactor) { - speedFactor = v.distributedBuilds.speedFactor; + // lib.optionalAttrs (m.distributedBuilds ? speedFactor) { + speedFactor = m.distributedBuilds.speedFactor; } // { supportedFeatures = [ @@ -47,60 +54,73 @@ let "benchmark" ]; } - ) buildServerDevices; - - remoteMachines = builtins.filter (m: m.hostName != config.networking.hostName) buildMachineList; + ) remoteBuildServerDevices; in { options.my.distributedBuilds.enable = lib.mkEnableOption "distributed Nix builds"; - config = lib.mkIf config.my.distributedBuilds.enable { - programs.ssh.knownHosts = knownHosts; + config = lib.mkIf config.my.distributedBuilds.enable ( + lib.mkMerge [ - # Dedicated user for receiving distributed build connections - users.users.${buildUser} = { - isSystemUser = true; - group = buildUser; - useDefaultShell = true; - openssh.authorizedKeys.keys = map ( - k: ''command="nix daemon --stdio",restrict ${k}'' - ) authorizedPublicKeys; - }; - users.groups.${buildUser} = { }; + # All machines + { + nix.settings = { + trusted-public-keys = lib.pipe buildServerDevices [ + (lib.mapAttrsToList (_: v: v.distributedBuilds.storeSigningPublicKey or null)) + (builtins.filter (k: k != null)) + ]; + max-jobs = (thisDevice.distributedBuilds or { }).maxJobs or "auto"; + cores = 0; + min-free = 10 * 1024 * 1024; + max-free = 200 * 1024 * 1024; + }; + systemd.services.nix-daemon.serviceConfig = { + MemoryAccounting = true; + MemoryMax = "90%"; + OOMScoreAdjust = 500; + }; + } - nix = { - distributedBuilds = remoteMachines != [ ]; - buildMachines = remoteMachines; - settings = { - trusted-users = [ buildUser ]; - builders-use-substitutes = true; - # Use build machines as binary caches so already-built paths are downloaded - # rather than rebuilt. Only machines with a storeSigningPublicKey are used. - substituters = lib.pipe buildServerDevices [ - (lib.filterAttrs (_: v: v.distributedBuilds ? storeSigningPublicKey)) - (lib.mapAttrsToList (hostName: _: "ssh-ng://${buildUser}@${hostName}")) - (lib.filter (s: s != "ssh-ng://${buildUser}@${config.networking.hostName}")) - ]; - trusted-public-keys = lib.pipe buildServerDevices [ - (lib.mapAttrsToList (_: v: v.distributedBuilds.storeSigningPublicKey or null)) - (builtins.filter (k: k != null)) - ]; - secret-key-files = - let - thisDevice = devices.${config.networking.hostName} or { }; - in - lib.optional (thisDevice.distributedBuilds.isBuilder or false) "/etc/nix/signing-key.sec"; - max-jobs = (devices.${config.networking.hostName}.distributedBuilds or { }).maxJobs or "auto"; - cores = 0; - min-free = 10 * 1024 * 1024; - max-free = 200 * 1024 * 1024; - }; - }; + # Server: accept incoming build connections + (lib.mkIf (thisDevice.distributedBuilds.isBuilder or false) { + users.users.${buildUser} = { + isSystemUser = true; + group = buildUser; + useDefaultShell = true; + openssh.authorizedKeys.keys = map ( + k: ''command="nix daemon --stdio",restrict ${k}'' + ) allClientPublicKeys; + }; + users.groups.${buildUser} = { }; + nix.settings = { + trusted-users = [ buildUser ]; + secret-key-files = [ "/etc/nix/signing-key.sec" ]; + }; + }) - systemd.services.nix-daemon.serviceConfig = { - MemoryAccounting = true; - MemoryMax = "90%"; - OOMScoreAdjust = 500; - }; - }; + # Client: connect to build servers for building and substitution + (lib.mkIf isClient { + programs.ssh = { + knownHosts = buildServerKnownHosts; + extraConfig = '' + Host ${lib.concatStringsSep " " (lib.attrNames buildServerDevices)} + User ${buildUser} + IdentityFile ${clientSshKeyPath} + IdentitiesOnly yes + ''; + }; + nix = { + distributedBuilds = buildMachines != [ ]; + buildMachines = buildMachines; + settings = { + builders-use-substitutes = true; + substituters = map (m: "ssh-ng://${buildUser}@${m.hostName}") ( + builtins.filter (m: m.distributedBuilds ? storeSigningPublicKey) remoteBuildServerDevices + ); + }; + }; + }) + + ] + ); }