distributed builds: use builders as binary caches

README.md

@@ -0,0 +1,36 @@
+# nixos-configuration
+
+Personal NixOS configuration for all machines. Devices are declared in `devices.nix`, per-device configs live in `nixosConfigurations/<name>/`, and shared modules in `nixosModules/`.
+
+## Distributed builds
+
+Machines are configured to act as build servers / binary caches for each other in devices.nix.
+
+### Onboarding a device as a build client
+
+1. Generate a key pair on the device:
+   ```
+   sudo ssh-keygen -t ed25519 -f /etc/nix/distributed-build-key -N "" -C "$(hostname)-nix-builds"
+   ```
+2. Add the public key to the device entry in `devices.nix`:
+   ```nix
+   distributedBuilds.clientPublicKey = "ssh-ed25519 AAAA... <hostname>-nix-builds";
+   ```
+3. Rebuild all build machines so they pick up the new authorized key.
+
+### Adding a build server
+
+1. Add to its entry in `devices.nix`:
+   ```nix
+   distributedBuilds.isBuilder = true;
+   distributedBuilds.hostPublicKey = "ssh-ed25519 AAAA..."; # from: ssh-keyscan -t ed25519 <hostname>
+   ```
+2. Generate a store signing key on the builder:
+   ```
+   sudo nix key generate-secret --key-name "$(hostname)" | sudo tee /etc/nix/signing-key.sec | sudo nix key convert-secret-to-public
+   ```
+3. Add the printed public key to `devices.nix`:
+   ```nix
+   distributedBuilds.storeSigningPublicKey = "<hostname>:<base64...>";
+   ```
+4. Rebuild all machines so they trust the new signing key.

devices.nix

@@ -39,6 +39,7 @@ in
       isBuilder = true;
       hostPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHGKoZ68wwyVRmPB0SkvpJUyUMDWeFbC5Je9zukyEOh7";
       clientPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKAbojdhb3PfazSRmudvo381Y+zUFVLMa7AbWbfK/Zp2 muede-lpt2-nix-builds";
+      storeSigningPublicKey = "muede-lpt2:3csut7FW6oZK/ztRLBRC80LSBfFE3qzl+aIYgOixB6U=";
     };
   };
   muede-pc2 = {
@@ -51,6 +52,7 @@ in
       speedFactor = 2;
       hostPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKEQQS5XNoj62Oj85xQfIuLORwoBRwfqjvfBHHsiI+RH";
       clientPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHmnyhP6L+kGHV15cb/d31AQr50wSEaQhkUBwy2+OEKk muede-pc2-nix-builds";
+      storeSigningPublicKey = "muede-pc2:fqQO0E0y65MjUWlQnrgWt5ZsmQKlKCv4jls3CmUXDEQ=";
     };
   };
   ronja-pc = {

nixosConfigurations/muede-pc2/default.nix

@@ -15,6 +15,7 @@
     my.amdGraphics.enable = true;
     my.secureBoot.enable = true;
 
+    boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
     nix.settings.extra-platforms = [
       "aarch64-linux"
       "i686-linux"

nixosModules/distributed-builds.nix

@@ -14,24 +14,6 @@ let
     (lib.mapAttrsToList (_: v: v.distributedBuilds.clientPublicKey))
   ];
 
-  # === Onboarding a device as a build client ===
-  #
-  # 1. Generate a key pair on the device:
-  #      sudo ssh-keygen -t ed25519 -f /etc/nix/distributed-build-key -N "" -C "$(hostname)-nix-builds"
-  #    (owned by root, mode 0600)
-  #
-  # 2. Add the public key to the device entry in flake.nix:
-  #      distributedBuilds.clientPublicKey = "ssh-ed25519 AAAA... <hostname>-nix-builds";
-  #
-  # 3. Rebuild all machines so they pick up the new authorized key.
-  #
-  # === Marking a device as a build server ===
-  #
-  # Add to its entry in flake.nix:
-  #   distributedBuilds.isBuilder = true;
-  #   distributedBuilds.hostPublicKey = "ssh-ed25519 AAAA...";  # from: ssh-keyscan -t ed25519 <hostname>
-  # All machines automatically discover and use it after the next rebuild.
-
   buildServerDevices = lib.filterAttrs (
     _: v: (v.distributedBuilds or { }).isBuilder or false
   ) devices;
@@ -92,6 +74,22 @@ in
       settings = {
         trusted-users = [ buildUser ];
         builders-use-substitutes = true;
+        # Use build machines as binary caches so already-built paths are downloaded
+        # rather than rebuilt. Only machines with a storeSigningPublicKey are used.
+        substituters = lib.pipe buildServerDevices [
+          (lib.filterAttrs (_: v: v.distributedBuilds ? storeSigningPublicKey))
+          (lib.mapAttrsToList (hostName: _: "ssh-ng://${buildUser}@${hostName}"))
+          (lib.filter (s: s != "ssh-ng://${buildUser}@${config.networking.hostName}"))
+        ];
+        trusted-public-keys = lib.pipe buildServerDevices [
+          (lib.mapAttrsToList (_: v: v.distributedBuilds.storeSigningPublicKey or null))
+          (builtins.filter (k: k != null))
+        ];
+        secret-key-files =
+          let
+            thisDevice = devices.${config.networking.hostName} or { };
+          in
+          lib.optional (thisDevice.distributedBuilds.isBuilder or false) "/etc/nix/signing-key.sec";
         max-jobs = (devices.${config.networking.hostName}.distributedBuilds or { }).maxJobs or "auto";
         cores = 0;
         min-free = 10 * 1024 * 1024;