distributed builds: cleanups, fixes

nixosConfigurations.nix

@@ -3,7 +3,7 @@
   lib,
 }:
 let
-  devices = import ./devices.nix { inherit (inputs) self; };
+  allDevices = import ./devices.nix { inherit (inputs) self; };
   inherit (inputs)
     self
     home-manager
@@ -15,7 +15,7 @@ let
     stylix
     zerforschen-plus
     ;
-  forDevice = f: lib.mapAttrs (device: value: f (value // { inherit device; })) devices;
+  forDevice = f: lib.mapAttrs (device: value: f (value // { inherit device; })) allDevices;
 in
 forDevice (
   {
@@ -24,10 +24,15 @@ forDevice (
     home-manager-users ? { },
     nixosSystem ? inputs.nixpkgs.lib.nixosSystem,
     ...
-  }:
+  }@thisDevice:
   let
     specialArgs = inputs // {
-      inherit device home-manager-users devices;
+      inherit
+        device
+        home-manager-users
+        allDevices
+        thisDevice
+        ;
     };
   in
   nixosSystem {

nixosModules/distributed-builds.nix

@@ -1,24 +1,27 @@
 {
   config,
   lib,
-  devices,
+  allDevices,
+  thisDevice,
   ...
 }:
 let
-  sshKeyPath = "/etc/nix/distributed-build-key";
+  clientSshKeyPath = "/etc/nix/distributed-build-key";
   buildUser = "remotebuild";
 
   # Collect all per-device public keys that have been registered.
-  authorizedPublicKeys = lib.pipe devices [
+  allClientPublicKeys = lib.pipe allDevices [
     (lib.filterAttrs (_: v: (v.distributedBuilds or { }) ? clientPublicKey))
     (lib.mapAttrsToList (_: v: v.distributedBuilds.clientPublicKey))
   ];
 
+  isClient = (thisDevice.distributedBuilds or { }) ? clientPublicKey;
+
   buildServerDevices = lib.filterAttrs (
     _: v: (v.distributedBuilds or { }).isBuilder or false
-  ) devices;
+  ) allDevices;
 
-  knownHosts = lib.pipe buildServerDevices [
+  buildServerKnownHosts = lib.pipe buildServerDevices [
     (lib.filterAttrs (_: v: v.distributedBuilds ? hostPublicKey))
     (lib.mapAttrs (
       _: v: {
@@ -27,17 +30,21 @@ let
     ))
   ];
 
-  buildMachineList = lib.mapAttrsToList (
+  remoteBuildServerDevices = builtins.filter (
-    hostName: v:
+    m: m.hostName != config.networking.hostName
+  ) (lib.mapAttrsToList (name: v: v // { hostName = name; }) buildServerDevices);
+
+  buildMachines = map (
+    m:
     {
-      inherit hostName;
+      hostName = m.hostName;
-      systems = [ v.system ];
+      systems = [ m.system ];
       sshUser = buildUser;
-      sshKey = sshKeyPath;
+      sshKey = clientSshKeyPath;
       protocol = "ssh-ng";
     }
-    // lib.optionalAttrs (v.distributedBuilds ? speedFactor) {
+    // lib.optionalAttrs (m.distributedBuilds ? speedFactor) {
-      speedFactor = v.distributedBuilds.speedFactor;
+      speedFactor = m.distributedBuilds.speedFactor;
     }
     // {
       supportedFeatures = [
@@ -47,60 +54,73 @@ let
         "benchmark"
       ];
     }
-  ) buildServerDevices;
+  ) remoteBuildServerDevices;
-
-  remoteMachines = builtins.filter (m: m.hostName != config.networking.hostName) buildMachineList;
 in
 {
   options.my.distributedBuilds.enable = lib.mkEnableOption "distributed Nix builds";
 
-  config = lib.mkIf config.my.distributedBuilds.enable {
+  config = lib.mkIf config.my.distributedBuilds.enable (
-    programs.ssh.knownHosts = knownHosts;
+    lib.mkMerge [
 
-    # Dedicated user for receiving distributed build connections
+      # All machines
+      {
+        nix.settings = {
+          trusted-public-keys = lib.pipe buildServerDevices [
+            (lib.mapAttrsToList (_: v: v.distributedBuilds.storeSigningPublicKey or null))
+            (builtins.filter (k: k != null))
+          ];
+          max-jobs = (thisDevice.distributedBuilds or { }).maxJobs or "auto";
+          cores = 0;
+          min-free = 10 * 1024 * 1024;
+          max-free = 200 * 1024 * 1024;
+        };
+        systemd.services.nix-daemon.serviceConfig = {
+          MemoryAccounting = true;
+          MemoryMax = "90%";
+          OOMScoreAdjust = 500;
+        };
+      }
+
+      # Server: accept incoming build connections
+      (lib.mkIf (thisDevice.distributedBuilds.isBuilder or false) {
         users.users.${buildUser} = {
           isSystemUser = true;
           group = buildUser;
           useDefaultShell = true;
           openssh.authorizedKeys.keys = map (
             k: ''command="nix daemon --stdio",restrict ${k}''
-      ) authorizedPublicKeys;
+          ) allClientPublicKeys;
         };
         users.groups.${buildUser} = { };
-
+        nix.settings = {
-    nix = {
-      distributedBuilds = remoteMachines != [ ];
-      buildMachines = remoteMachines;
-      settings = {
           trusted-users = [ buildUser ];
-        builders-use-substitutes = true;
+          secret-key-files = [ "/etc/nix/signing-key.sec" ];
-        # Use build machines as binary caches so already-built paths are downloaded
-        # rather than rebuilt. Only machines with a storeSigningPublicKey are used.
-        substituters = lib.pipe buildServerDevices [
-          (lib.filterAttrs (_: v: v.distributedBuilds ? storeSigningPublicKey))
-          (lib.mapAttrsToList (hostName: _: "ssh-ng://${buildUser}@${hostName}"))
-          (lib.filter (s: s != "ssh-ng://${buildUser}@${config.networking.hostName}"))
-        ];
-        trusted-public-keys = lib.pipe buildServerDevices [
-          (lib.mapAttrsToList (_: v: v.distributedBuilds.storeSigningPublicKey or null))
-          (builtins.filter (k: k != null))
-        ];
-        secret-key-files =
-          let
-            thisDevice = devices.${config.networking.hostName} or { };
-          in
-          lib.optional (thisDevice.distributedBuilds.isBuilder or false) "/etc/nix/signing-key.sec";
-        max-jobs = (devices.${config.networking.hostName}.distributedBuilds or { }).maxJobs or "auto";
-        cores = 0;
-        min-free = 10 * 1024 * 1024;
-        max-free = 200 * 1024 * 1024;
-      };
         };
+      })
 
-    systemd.services.nix-daemon.serviceConfig = {
+      # Client: connect to build servers for building and substitution
-      MemoryAccounting = true;
+      (lib.mkIf isClient {
-      MemoryMax = "90%";
+        programs.ssh = {
-      OOMScoreAdjust = 500;
+          knownHosts = buildServerKnownHosts;
+          extraConfig = ''
+            Host ${lib.concatStringsSep " " (lib.attrNames buildServerDevices)}
+              User ${buildUser}
+              IdentityFile ${clientSshKeyPath}
+              IdentitiesOnly yes
+          '';
+        };
+        nix = {
+          distributedBuilds = buildMachines != [ ];
+          buildMachines = buildMachines;
+          settings = {
+            builders-use-substitutes = true;
+            substituters = map (m: "ssh-ng://${buildUser}@${m.hostName}") (
+              builtins.filter (m: m.distributedBuilds ? storeSigningPublicKey) remoteBuildServerDevices
+            );
           };
         };
+      })
+
+    ]
+  );
 }