distributed builds - infra and lpt2

flake.nix

@@ -128,9 +128,11 @@
         };
         damocles = {
           system = "x86_64-linux";
+          distributedBuilds.maxJobs = 0;
         };
         damocles-lab = {
           system = "x86_64-linux";
+          distributedBuilds.maxJobs = 0;
         };
         epimetheus = {
           system = "aarch64-linux";
@@ -146,12 +148,17 @@
           home-manager-users = {
             inherit (self.homeConfigurations) muede;
           };
+          distributedBuilds.isBuilder = true;
         };
         muede-pc2 = {
           system = "x86_64-linux";
           home-manager-users = {
             inherit (self.homeConfigurations) muede;
           };
+          distributedBuilds = {
+            isBuilder = true;
+            publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKAbojdhb3PfazSRmudvo381Y+zUFVLMa7AbWbfK/Zp2 muede-pc2-nix-builds";
+          };
         };
         ronja-pc = {
           system = "x86_64-linux";
@@ -236,10 +243,11 @@
           system,
           home-manager-users ? { },
           nixosSystem ? nixpkgs.lib.nixosSystem,
+          ...
         }:
         let
           specialArgs = inputs // {
-            inherit device home-manager-users;
+            inherit device home-manager-users devices;
           };
         in
         nixosSystem {

nixosConfigurations/muede-lpt2/default.nix

@@ -65,6 +65,10 @@
       autoStart = false;
       privateNetwork = false;
       path = self.nixosConfigurations.damocles.config.system.build.toplevel;
+      bindMounts."/etc/nix/distributed-build-key" = {
+        hostPath = "/etc/nix/distributed-build-key";
+        isReadOnly = true;
+      };
       bindMounts."/persist/damocles-ssh" = {
         hostPath = "/persist/damocles-ssh";
         isReadOnly = true;
@@ -79,6 +83,10 @@
       autoStart = false;
       privateNetwork = false;
       path = self.nixosConfigurations.damocles-lab.config.system.build.toplevel;
+      bindMounts."/etc/nix/distributed-build-key" = {
+        hostPath = "/etc/nix/distributed-build-key";
+        isReadOnly = true;
+      };
       bindMounts."/workspace" = {
         hostPath = "/persist/damocles-lab";
         isReadOnly = false;

nixosModules/distributed-builds.nix

@@ -0,0 +1,80 @@
+{
+  config,
+  lib,
+  devices,
+  ...
+}:
+let
+  sshKeyPath = "/etc/nix/distributed-build-key";
+  buildUser = "remotebuild";
+
+  # Collect all per-device public keys that have been registered.
+  authorizedPublicKeys = lib.pipe devices [
+    (lib.filterAttrs (_: v: (v.distributedBuilds or { }) ? publicKey))
+    (lib.mapAttrsToList (_: v: v.distributedBuilds.publicKey))
+  ];
+
+  # === Onboarding a device as a build client ===
+  #
+  # 1. Generate a key pair on the device:
+  #      sudo ssh-keygen -t ed25519 -f /etc/nix/distributed-build-key -N "" -C "<hostname>-nix-builds"
+  #    (owned by root, mode 0600)
+  #
+  # 2. Add the public key to the device entry in flake.nix:
+  #      distributedBuilds.publicKey = "ssh-ed25519 AAAA... <hostname>-nix-builds";
+  #
+  # 3. Rebuild all machines so they pick up the new authorized key.
+  #
+  # === Marking a device as a build server ===
+  #
+  # Add to its entry in flake.nix:
+  #   distributedBuilds.isBuilder = true;
+  # All machines automatically discover and use it after the next rebuild.
+
+  buildServerDevices = lib.filterAttrs (_: v: (v.distributedBuilds or { }).isBuilder or false) devices;
+
+  buildMachineList = lib.mapAttrsToList (hostName: v: {
+    inherit hostName;
+    systems = [ v.system ];
+    sshUser = buildUser;
+    sshKey = sshKeyPath;
+    protocol = "ssh-ng";
+    supportedFeatures = [
+      "nixos-test"
+      "big-parallel"
+      "kvm"
+      "benchmark"
+    ];
+  }) buildServerDevices;
+
+  remoteMachines = builtins.filter (m: m.hostName != config.networking.hostName) buildMachineList;
+in
+{
+  # Dedicated user for receiving distributed build connections
+  users.users.${buildUser} = {
+    isSystemUser = true;
+    group = buildUser;
+    useDefaultShell = true;
+    openssh.authorizedKeys.keys = authorizedPublicKeys;
+  };
+  users.groups.${buildUser} = { };
+
+  nix = {
+    distributedBuilds = remoteMachines != [ ];
+    buildMachines = remoteMachines;
+    settings = {
+      trusted-users = [ buildUser ];
+      builders-use-substitutes = true;
+      max-jobs = (devices.${config.networking.hostName}.distributedBuilds or { }).maxJobs or "auto";
+      cores = 0;
+      min-free = 10 * 1024 * 1024;
+      max-free = 200 * 1024 * 1024;
+    };
+  };
+
+  systemd.services.nix-daemon.serviceConfig = {
+    MemoryAccounting = true;
+    MemoryMax = "90%";
+    OOMScoreAdjust = 500;
+  };
+}

nixosModules/global-settings.nix

@@ -12,6 +12,7 @@
     self.nixosModules.allowed-unfree-list
     self.nixosModules.autoupdate
     self.nixosModules.default
+    self.nixosModules.distributed-builds
     self.nixosModules.extra-caches
     self.nixosModules.globalinstalls
     self.nixosModules.lix-is-nix