diff --git a/nixosConfigurations.nix b/nixosConfigurations.nix index 9fb2cf2..29e78c3 100644 --- a/nixosConfigurations.nix +++ b/nixosConfigurations.nix @@ -3,7 +3,7 @@ lib, }: let - devices = import ./devices.nix { inherit (inputs) self; }; + allDevices = import ./devices.nix { inherit (inputs) self; }; inherit (inputs) self home-manager @@ -15,7 +15,7 @@ let stylix zerforschen-plus ; - forDevice = f: lib.mapAttrs (device: value: f (value // { inherit device; })) devices; + forDevice = f: lib.mapAttrs (device: value: f (value // { inherit device; })) allDevices; in forDevice ( { @@ -24,10 +24,15 @@ forDevice ( home-manager-users ? { }, nixosSystem ? inputs.nixpkgs.lib.nixosSystem, ... - }: + }@thisDevice: let specialArgs = inputs // { - inherit device home-manager-users devices; + inherit + device + home-manager-users + allDevices + thisDevice + ; }; in nixosSystem { diff --git a/nixosModules/distributed-builds.nix b/nixosModules/distributed-builds.nix index 32a8f34..1d0a55e 100644 --- a/nixosModules/distributed-builds.nix +++ b/nixosModules/distributed-builds.nix @@ -1,24 +1,27 @@ { config, lib, - devices, + allDevices, + thisDevice, ... }: let - sshKeyPath = "/etc/nix/distributed-build-key"; + clientSshKeyPath = "/etc/nix/distributed-build-key"; buildUser = "remotebuild"; # Collect all per-device public keys that have been registered. - authorizedPublicKeys = lib.pipe devices [ + allClientPublicKeys = lib.pipe allDevices [ (lib.filterAttrs (_: v: (v.distributedBuilds or { }) ? clientPublicKey)) (lib.mapAttrsToList (_: v: v.distributedBuilds.clientPublicKey)) ]; + isClient = (thisDevice.distributedBuilds or { }) ? clientPublicKey; + buildServerDevices = lib.filterAttrs ( _: v: (v.distributedBuilds or { }).isBuilder or false - ) devices; + ) allDevices; - knownHosts = lib.pipe buildServerDevices [ + buildServerKnownHosts = lib.pipe buildServerDevices [ (lib.filterAttrs (_: v: v.distributedBuilds ? hostPublicKey)) (lib.mapAttrs ( _: v: { @@ -27,17 +30,21 @@ let )) ]; - buildMachineList = lib.mapAttrsToList ( - hostName: v: + remoteBuildServerDevices = builtins.filter ( + m: m.hostName != config.networking.hostName + ) (lib.mapAttrsToList (name: v: v // { hostName = name; }) buildServerDevices); + + buildMachines = map ( + m: { - inherit hostName; - systems = [ v.system ]; + hostName = m.hostName; + systems = [ m.system ]; sshUser = buildUser; - sshKey = sshKeyPath; + sshKey = clientSshKeyPath; protocol = "ssh-ng"; } - // lib.optionalAttrs (v.distributedBuilds ? speedFactor) { - speedFactor = v.distributedBuilds.speedFactor; + // lib.optionalAttrs (m.distributedBuilds ? speedFactor) { + speedFactor = m.distributedBuilds.speedFactor; } // { supportedFeatures = [ @@ -47,60 +54,73 @@ let "benchmark" ]; } - ) buildServerDevices; - - remoteMachines = builtins.filter (m: m.hostName != config.networking.hostName) buildMachineList; + ) remoteBuildServerDevices; in { options.my.distributedBuilds.enable = lib.mkEnableOption "distributed Nix builds"; - config = lib.mkIf config.my.distributedBuilds.enable { - programs.ssh.knownHosts = knownHosts; + config = lib.mkIf config.my.distributedBuilds.enable ( + lib.mkMerge [ - # Dedicated user for receiving distributed build connections - users.users.${buildUser} = { - isSystemUser = true; - group = buildUser; - useDefaultShell = true; - openssh.authorizedKeys.keys = map ( - k: ''command="nix daemon --stdio",restrict ${k}'' - ) authorizedPublicKeys; - }; - users.groups.${buildUser} = { }; + # All machines + { + nix.settings = { + trusted-public-keys = lib.pipe buildServerDevices [ + (lib.mapAttrsToList (_: v: v.distributedBuilds.storeSigningPublicKey or null)) + (builtins.filter (k: k != null)) + ]; + max-jobs = (thisDevice.distributedBuilds or { }).maxJobs or "auto"; + cores = 0; + min-free = 10 * 1024 * 1024; + max-free = 200 * 1024 * 1024; + }; + systemd.services.nix-daemon.serviceConfig = { + MemoryAccounting = true; + MemoryMax = "90%"; + OOMScoreAdjust = 500; + }; + } - nix = { - distributedBuilds = remoteMachines != [ ]; - buildMachines = remoteMachines; - settings = { - trusted-users = [ buildUser ]; - builders-use-substitutes = true; - # Use build machines as binary caches so already-built paths are downloaded - # rather than rebuilt. Only machines with a storeSigningPublicKey are used. - substituters = lib.pipe buildServerDevices [ - (lib.filterAttrs (_: v: v.distributedBuilds ? storeSigningPublicKey)) - (lib.mapAttrsToList (hostName: _: "ssh-ng://${buildUser}@${hostName}")) - (lib.filter (s: s != "ssh-ng://${buildUser}@${config.networking.hostName}")) - ]; - trusted-public-keys = lib.pipe buildServerDevices [ - (lib.mapAttrsToList (_: v: v.distributedBuilds.storeSigningPublicKey or null)) - (builtins.filter (k: k != null)) - ]; - secret-key-files = - let - thisDevice = devices.${config.networking.hostName} or { }; - in - lib.optional (thisDevice.distributedBuilds.isBuilder or false) "/etc/nix/signing-key.sec"; - max-jobs = (devices.${config.networking.hostName}.distributedBuilds or { }).maxJobs or "auto"; - cores = 0; - min-free = 10 * 1024 * 1024; - max-free = 200 * 1024 * 1024; - }; - }; + # Server: accept incoming build connections + (lib.mkIf (thisDevice.distributedBuilds.isBuilder or false) { + users.users.${buildUser} = { + isSystemUser = true; + group = buildUser; + useDefaultShell = true; + openssh.authorizedKeys.keys = map ( + k: ''command="nix daemon --stdio",restrict ${k}'' + ) allClientPublicKeys; + }; + users.groups.${buildUser} = { }; + nix.settings = { + trusted-users = [ buildUser ]; + secret-key-files = [ "/etc/nix/signing-key.sec" ]; + }; + }) - systemd.services.nix-daemon.serviceConfig = { - MemoryAccounting = true; - MemoryMax = "90%"; - OOMScoreAdjust = 500; - }; - }; + # Client: connect to build servers for building and substitution + (lib.mkIf isClient { + programs.ssh = { + knownHosts = buildServerKnownHosts; + extraConfig = '' + Host ${lib.concatStringsSep " " (lib.attrNames buildServerDevices)} + User ${buildUser} + IdentityFile ${clientSshKeyPath} + IdentitiesOnly yes + ''; + }; + nix = { + distributedBuilds = buildMachines != [ ]; + buildMachines = buildMachines; + settings = { + builders-use-substitutes = true; + substituters = map (m: "ssh-ng://${buildUser}@${m.hostName}") ( + builtins.filter (m: m.distributedBuilds ? storeSigningPublicKey) remoteBuildServerDevices + ); + }; + }; + }) + + ] + ); }