diff --git a/README.md b/README.md index 3ad5e88..f97d4b5 100644 --- a/README.md +++ b/README.md @@ -9,38 +9,28 @@ Machines are configured to act as build servers / binary caches for each other i ### Onboarding a device as a build client 1. Generate a key pair on the device: - - ```sh - sudo ssh-keygen -t ed25519 -f /etc/nix/distributed-build-key -N "" -C "$(hostname)-nix-builds" && sudo cat /etc/nix/distributed-build-key.pub ``` - + sudo ssh-keygen -t ed25519 -f /etc/nix/distributed-build-key -N "" -C "$(hostname)-nix-builds" + ``` 2. Add the public key to the device entry in `devices.nix`: - ```nix distributedBuilds.clientPublicKey = "ssh-ed25519 AAAA... -nix-builds"; ``` - 3. Rebuild all build machines so they pick up the new authorized key. ### Adding a build server 1. Add to its entry in `devices.nix`: - ```nix distributedBuilds.isBuilder = true; - distributedBuilds.hostPublicKey = "ssh-ed25519 AAAA..."; # from: ssh-keyscan -t ed25519 "$(hostname)" + distributedBuilds.hostPublicKey = "ssh-ed25519 AAAA..."; # from: ssh-keyscan -t ed25519 ``` - 2. Generate a store signing key on the builder: - - ```sh + ``` sudo nix key generate-secret --key-name "$(hostname)" | sudo tee /etc/nix/signing-key.sec | sudo nix key convert-secret-to-public ``` - 3. Add the printed public key to `devices.nix`: - ```nix distributedBuilds.storeSigningPublicKey = ":"; ``` - 4. Rebuild all machines so they trust the new signing key. diff --git a/devices.nix b/devices.nix index ea3c6f6..c17d02e 100644 --- a/devices.nix +++ b/devices.nix @@ -26,12 +26,6 @@ in }; forgejo-runner-1 = { system = "aarch64-linux"; - distributedBuilds = { - isBuilder = true; - clientPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK0NLgg0sFobBWz/bjYs9WkrMvlcvJC5F6+3jQ/b+AnD forgejo-runner-1-nix-builds"; - hostPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIANGC89GiT5xCsFICwrharrbV3q7acWHqk6ZwOUXbtGT"; - storeSigningPublicKey = "forgejo-runner-1:ln1FVLL8G5+IveQuBi/Kn3SaqFZ1gaiQrE3yPlMhCMA="; - }; }; hetzner-vpn2 = { system = "aarch64-linux"; diff --git a/flake.nix b/flake.nix index e95af0e..01c4906 100644 --- a/flake.nix +++ b/flake.nix @@ -146,11 +146,7 @@ niri = niri.overlays.niri; }; - nixosModules = (importModuleDir ./nixosModules) // { - default = { - imports = builtins.attrValues (builtins.removeAttrs self.nixosModules [ "default" ]); - }; - }; + nixosModules = importModuleDir ./nixosModules; homeModules = importModuleDir ./homeModules; homeConfigurations = { diff --git a/nixosConfigurations.nix b/nixosConfigurations.nix index 29e78c3..9fb2cf2 100644 --- a/nixosConfigurations.nix +++ b/nixosConfigurations.nix @@ -3,7 +3,7 @@ lib, }: let - allDevices = import ./devices.nix { inherit (inputs) self; }; + devices = import ./devices.nix { inherit (inputs) self; }; inherit (inputs) self home-manager @@ -15,7 +15,7 @@ let stylix zerforschen-plus ; - forDevice = f: lib.mapAttrs (device: value: f (value // { inherit device; })) allDevices; + forDevice = f: lib.mapAttrs (device: value: f (value // { inherit device; })) devices; in forDevice ( { @@ -24,15 +24,10 @@ forDevice ( home-manager-users ? { }, nixosSystem ? inputs.nixpkgs.lib.nixosSystem, ... - }@thisDevice: + }: let specialArgs = inputs // { - inherit - device - home-manager-users - allDevices - thisDevice - ; + inherit device home-manager-users devices; }; in nixosSystem { diff --git a/nixosModules/default.nix b/nixosModules/default.nix new file mode 100644 index 0000000..2808b2a --- /dev/null +++ b/nixosModules/default.nix @@ -0,0 +1,39 @@ +{ ... }: +{ + imports = [ + # keep-sorted start + ./allowed-unfree-list.nix + ./amd-graphics.nix + ./autoupdate.nix + ./distributed-builds.nix + ./en-de.nix + ./extra-caches.nix + ./firmware-updates.nix + ./git.nix + ./globalinstalls.nix + ./gnome.nix + ./intel-graphics.nix + ./kdeconnect.nix + ./latex.nix + ./lix-is-nix.nix + ./modern-desktop.nix + ./muede-desktop-settings.nix + ./nix-ld.nix + ./nixpkgs-overlays.nix + ./openssh.nix + ./podman.nix + ./printing.nix + ./prometheus-node.nix + ./pxvirt-guest.nix + ./quiet-boot.nix + ./secure-boot.nix + ./steam.nix + ./stylix.nix + ./systemd-boot.nix + ./tailscale.nix + ./user-muede.nix + ./user-ronja.nix + ./wine-gaming.nix + # keep-sorted end + ]; +} diff --git a/nixosModules/distributed-builds.nix b/nixosModules/distributed-builds.nix index 1d0a55e..32a8f34 100644 --- a/nixosModules/distributed-builds.nix +++ b/nixosModules/distributed-builds.nix @@ -1,27 +1,24 @@ { config, lib, - allDevices, - thisDevice, + devices, ... }: let - clientSshKeyPath = "/etc/nix/distributed-build-key"; + sshKeyPath = "/etc/nix/distributed-build-key"; buildUser = "remotebuild"; # Collect all per-device public keys that have been registered. - allClientPublicKeys = lib.pipe allDevices [ + authorizedPublicKeys = lib.pipe devices [ (lib.filterAttrs (_: v: (v.distributedBuilds or { }) ? clientPublicKey)) (lib.mapAttrsToList (_: v: v.distributedBuilds.clientPublicKey)) ]; - isClient = (thisDevice.distributedBuilds or { }) ? clientPublicKey; - buildServerDevices = lib.filterAttrs ( _: v: (v.distributedBuilds or { }).isBuilder or false - ) allDevices; + ) devices; - buildServerKnownHosts = lib.pipe buildServerDevices [ + knownHosts = lib.pipe buildServerDevices [ (lib.filterAttrs (_: v: v.distributedBuilds ? hostPublicKey)) (lib.mapAttrs ( _: v: { @@ -30,21 +27,17 @@ let )) ]; - remoteBuildServerDevices = builtins.filter ( - m: m.hostName != config.networking.hostName - ) (lib.mapAttrsToList (name: v: v // { hostName = name; }) buildServerDevices); - - buildMachines = map ( - m: + buildMachineList = lib.mapAttrsToList ( + hostName: v: { - hostName = m.hostName; - systems = [ m.system ]; + inherit hostName; + systems = [ v.system ]; sshUser = buildUser; - sshKey = clientSshKeyPath; + sshKey = sshKeyPath; protocol = "ssh-ng"; } - // lib.optionalAttrs (m.distributedBuilds ? speedFactor) { - speedFactor = m.distributedBuilds.speedFactor; + // lib.optionalAttrs (v.distributedBuilds ? speedFactor) { + speedFactor = v.distributedBuilds.speedFactor; } // { supportedFeatures = [ @@ -54,73 +47,60 @@ let "benchmark" ]; } - ) remoteBuildServerDevices; + ) buildServerDevices; + + remoteMachines = builtins.filter (m: m.hostName != config.networking.hostName) buildMachineList; in { options.my.distributedBuilds.enable = lib.mkEnableOption "distributed Nix builds"; - config = lib.mkIf config.my.distributedBuilds.enable ( - lib.mkMerge [ + config = lib.mkIf config.my.distributedBuilds.enable { + programs.ssh.knownHosts = knownHosts; - # All machines - { - nix.settings = { - trusted-public-keys = lib.pipe buildServerDevices [ - (lib.mapAttrsToList (_: v: v.distributedBuilds.storeSigningPublicKey or null)) - (builtins.filter (k: k != null)) - ]; - max-jobs = (thisDevice.distributedBuilds or { }).maxJobs or "auto"; - cores = 0; - min-free = 10 * 1024 * 1024; - max-free = 200 * 1024 * 1024; - }; - systemd.services.nix-daemon.serviceConfig = { - MemoryAccounting = true; - MemoryMax = "90%"; - OOMScoreAdjust = 500; - }; - } + # Dedicated user for receiving distributed build connections + users.users.${buildUser} = { + isSystemUser = true; + group = buildUser; + useDefaultShell = true; + openssh.authorizedKeys.keys = map ( + k: ''command="nix daemon --stdio",restrict ${k}'' + ) authorizedPublicKeys; + }; + users.groups.${buildUser} = { }; - # Server: accept incoming build connections - (lib.mkIf (thisDevice.distributedBuilds.isBuilder or false) { - users.users.${buildUser} = { - isSystemUser = true; - group = buildUser; - useDefaultShell = true; - openssh.authorizedKeys.keys = map ( - k: ''command="nix daemon --stdio",restrict ${k}'' - ) allClientPublicKeys; - }; - users.groups.${buildUser} = { }; - nix.settings = { - trusted-users = [ buildUser ]; - secret-key-files = [ "/etc/nix/signing-key.sec" ]; - }; - }) + nix = { + distributedBuilds = remoteMachines != [ ]; + buildMachines = remoteMachines; + settings = { + trusted-users = [ buildUser ]; + builders-use-substitutes = true; + # Use build machines as binary caches so already-built paths are downloaded + # rather than rebuilt. Only machines with a storeSigningPublicKey are used. + substituters = lib.pipe buildServerDevices [ + (lib.filterAttrs (_: v: v.distributedBuilds ? storeSigningPublicKey)) + (lib.mapAttrsToList (hostName: _: "ssh-ng://${buildUser}@${hostName}")) + (lib.filter (s: s != "ssh-ng://${buildUser}@${config.networking.hostName}")) + ]; + trusted-public-keys = lib.pipe buildServerDevices [ + (lib.mapAttrsToList (_: v: v.distributedBuilds.storeSigningPublicKey or null)) + (builtins.filter (k: k != null)) + ]; + secret-key-files = + let + thisDevice = devices.${config.networking.hostName} or { }; + in + lib.optional (thisDevice.distributedBuilds.isBuilder or false) "/etc/nix/signing-key.sec"; + max-jobs = (devices.${config.networking.hostName}.distributedBuilds or { }).maxJobs or "auto"; + cores = 0; + min-free = 10 * 1024 * 1024; + max-free = 200 * 1024 * 1024; + }; + }; - # Client: connect to build servers for building and substitution - (lib.mkIf isClient { - programs.ssh = { - knownHosts = buildServerKnownHosts; - extraConfig = '' - Host ${lib.concatStringsSep " " (lib.attrNames buildServerDevices)} - User ${buildUser} - IdentityFile ${clientSshKeyPath} - IdentitiesOnly yes - ''; - }; - nix = { - distributedBuilds = buildMachines != [ ]; - buildMachines = buildMachines; - settings = { - builders-use-substitutes = true; - substituters = map (m: "ssh-ng://${buildUser}@${m.hostName}") ( - builtins.filter (m: m.distributedBuilds ? storeSigningPublicKey) remoteBuildServerDevices - ); - }; - }; - }) - - ] - ); + systemd.services.nix-daemon.serviceConfig = { + MemoryAccounting = true; + MemoryMax = "90%"; + OOMScoreAdjust = 500; + }; + }; }