diff --git a/flake.nix b/flake.nix index dce3929..57d4037 100644 --- a/flake.nix +++ b/flake.nix @@ -128,9 +128,11 @@ }; damocles = { system = "x86_64-linux"; + distributedBuilds.maxJobs = 0; }; damocles-lab = { system = "x86_64-linux"; + distributedBuilds.maxJobs = 0; }; epimetheus = { system = "aarch64-linux"; @@ -146,12 +148,17 @@ home-manager-users = { inherit (self.homeConfigurations) muede; }; + distributedBuilds.isBuilder = true; }; muede-pc2 = { system = "x86_64-linux"; home-manager-users = { inherit (self.homeConfigurations) muede; }; + distributedBuilds = { + isBuilder = true; + publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKAbojdhb3PfazSRmudvo381Y+zUFVLMa7AbWbfK/Zp2 muede-pc2-nix-builds"; + }; }; ronja-pc = { system = "x86_64-linux"; @@ -236,10 +243,11 @@ system, home-manager-users ? { }, nixosSystem ? nixpkgs.lib.nixosSystem, + ... }: let specialArgs = inputs // { - inherit device home-manager-users; + inherit device home-manager-users devices; }; in nixosSystem { diff --git a/nixosConfigurations/muede-lpt2/default.nix b/nixosConfigurations/muede-lpt2/default.nix index 711a578..434b046 100644 --- a/nixosConfigurations/muede-lpt2/default.nix +++ b/nixosConfigurations/muede-lpt2/default.nix @@ -65,6 +65,10 @@ autoStart = false; privateNetwork = false; path = self.nixosConfigurations.damocles.config.system.build.toplevel; + bindMounts."/etc/nix/distributed-build-key" = { + hostPath = "/etc/nix/distributed-build-key"; + isReadOnly = true; + }; bindMounts."/persist/damocles-ssh" = { hostPath = "/persist/damocles-ssh"; isReadOnly = true; @@ -79,6 +83,10 @@ autoStart = false; privateNetwork = false; path = self.nixosConfigurations.damocles-lab.config.system.build.toplevel; + bindMounts."/etc/nix/distributed-build-key" = { + hostPath = "/etc/nix/distributed-build-key"; + isReadOnly = true; + }; bindMounts."/workspace" = { hostPath = "/persist/damocles-lab"; isReadOnly = false; diff --git a/nixosModules/distributed-builds.nix b/nixosModules/distributed-builds.nix new file mode 100644 index 0000000..a811a71 --- /dev/null +++ b/nixosModules/distributed-builds.nix @@ -0,0 +1,80 @@ +{ + config, + lib, + devices, + ... +}: +let + sshKeyPath = "/etc/nix/distributed-build-key"; + buildUser = "remotebuild"; + + # Collect all per-device public keys that have been registered. + authorizedPublicKeys = lib.pipe devices [ + (lib.filterAttrs (_: v: (v.distributedBuilds or { }) ? publicKey)) + (lib.mapAttrsToList (_: v: v.distributedBuilds.publicKey)) + ]; + + # === Onboarding a device as a build client === + # + # 1. Generate a key pair on the device: + # sudo ssh-keygen -t ed25519 -f /etc/nix/distributed-build-key -N "" -C "-nix-builds" + # (owned by root, mode 0600) + # + # 2. Add the public key to the device entry in flake.nix: + # distributedBuilds.publicKey = "ssh-ed25519 AAAA... -nix-builds"; + # + # 3. Rebuild all machines so they pick up the new authorized key. + # + # === Marking a device as a build server === + # + # Add to its entry in flake.nix: + # distributedBuilds.isBuilder = true; + # All machines automatically discover and use it after the next rebuild. + + buildServerDevices = lib.filterAttrs (_: v: (v.distributedBuilds or { }).isBuilder or false) devices; + + buildMachineList = lib.mapAttrsToList (hostName: v: { + inherit hostName; + systems = [ v.system ]; + sshUser = buildUser; + sshKey = sshKeyPath; + protocol = "ssh-ng"; + supportedFeatures = [ + "nixos-test" + "big-parallel" + "kvm" + "benchmark" + ]; + }) buildServerDevices; + + remoteMachines = builtins.filter (m: m.hostName != config.networking.hostName) buildMachineList; +in +{ + # Dedicated user for receiving distributed build connections + users.users.${buildUser} = { + isSystemUser = true; + group = buildUser; + useDefaultShell = true; + openssh.authorizedKeys.keys = authorizedPublicKeys; + }; + users.groups.${buildUser} = { }; + + nix = { + distributedBuilds = remoteMachines != [ ]; + buildMachines = remoteMachines; + settings = { + trusted-users = [ buildUser ]; + builders-use-substitutes = true; + max-jobs = (devices.${config.networking.hostName}.distributedBuilds or { }).maxJobs or "auto"; + cores = 0; + min-free = 10 * 1024 * 1024; + max-free = 200 * 1024 * 1024; + }; + }; + + systemd.services.nix-daemon.serviceConfig = { + MemoryAccounting = true; + MemoryMax = "90%"; + OOMScoreAdjust = 500; + }; +} diff --git a/nixosModules/global-settings.nix b/nixosModules/global-settings.nix index cd92140..2d1c5b3 100644 --- a/nixosModules/global-settings.nix +++ b/nixosModules/global-settings.nix @@ -12,6 +12,7 @@ self.nixosModules.allowed-unfree-list self.nixosModules.autoupdate self.nixosModules.default + self.nixosModules.distributed-builds self.nixosModules.extra-caches self.nixosModules.globalinstalls self.nixosModules.lix-is-nix