vinzenz/nixos-configuration
0
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

distributed builds: cleanups, fixes

Browse source
This commit is contained in:
müde 2026-05-02 02:01:30 +02:00
parent f035f1300f
commit 736557bb8a
2 changed files with 90 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

142
nixosModules/distributed-builds.nix
View file

@ -1,24 +1,27 @@
{
config,
lib,
devices,
allDevices,
thisDevice,
...
}:
let
sshKeyPath = "/etc/nix/distributed-build-key";
clientSshKeyPath = "/etc/nix/distributed-build-key";
buildUser = "remotebuild";
# Collect all per-device public keys that have been registered.
authorizedPublicKeys = lib.pipe devices [
allClientPublicKeys = lib.pipe allDevices [
(lib.filterAttrs (_: v: (v.distributedBuilds or { }) ? clientPublicKey))
(lib.mapAttrsToList (_: v: v.distributedBuilds.clientPublicKey))
];
isClient = (thisDevice.distributedBuilds or { }) ? clientPublicKey;
buildServerDevices = lib.filterAttrs (
_: v: (v.distributedBuilds or { }).isBuilder or false
) devices;
) allDevices;
knownHosts = lib.pipe buildServerDevices [
buildServerKnownHosts = lib.pipe buildServerDevices [
(lib.filterAttrs (_: v: v.distributedBuilds ? hostPublicKey))
(lib.mapAttrs (
_: v: {
@ -27,17 +30,21 @@ let
))
];
buildMachineList = lib.mapAttrsToList (
hostName: v:
remoteBuildServerDevices = builtins.filter (
m: m.hostName != config.networking.hostName
) (lib.mapAttrsToList (name: v: v // { hostName = name; }) buildServerDevices);
buildMachines = map (
m:
{
inherit hostName;
systems = [ v.system ];
hostName = m.hostName;
systems = [ m.system ];
sshUser = buildUser;
sshKey = sshKeyPath;
sshKey = clientSshKeyPath;
protocol = "ssh-ng";
}
// lib.optionalAttrs (v.distributedBuilds ? speedFactor) {
speedFactor = v.distributedBuilds.speedFactor;
// lib.optionalAttrs (m.distributedBuilds ? speedFactor) {
speedFactor = m.distributedBuilds.speedFactor;
}
// {
supportedFeatures = [
@ -47,60 +54,73 @@ let
"benchmark"
];
}
) buildServerDevices;
remoteMachines = builtins.filter (m: m.hostName != config.networking.hostName) buildMachineList;
) remoteBuildServerDevices;
in
{
options.my.distributedBuilds.enable = lib.mkEnableOption "distributed Nix builds";
config = lib.mkIf config.my.distributedBuilds.enable {
programs.ssh.knownHosts = knownHosts;
config = lib.mkIf config.my.distributedBuilds.enable (
lib.mkMerge [
# Dedicated user for receiving distributed build connections
users.users.${buildUser} = {
isSystemUser = true;
group = buildUser;
useDefaultShell = true;
openssh.authorizedKeys.keys = map (
k: ''command="nix daemon --stdio",restrict ${k}''
) authorizedPublicKeys;
};
users.groups.${buildUser} = { };
# All machines
{
nix.settings = {
trusted-public-keys = lib.pipe buildServerDevices [
(lib.mapAttrsToList (_: v: v.distributedBuilds.storeSigningPublicKey or null))
(builtins.filter (k: k != null))
];
max-jobs = (thisDevice.distributedBuilds or { }).maxJobs or "auto";
cores = 0;
min-free = 10 * 1024 * 1024;
max-free = 200 * 1024 * 1024;
};
systemd.services.nix-daemon.serviceConfig = {
MemoryAccounting = true;
MemoryMax = "90%";
OOMScoreAdjust = 500;
};
}
nix = {
distributedBuilds = remoteMachines != [ ];
buildMachines = remoteMachines;
settings = {
trusted-users = [ buildUser ];
builders-use-substitutes = true;
# Use build machines as binary caches so already-built paths are downloaded
# rather than rebuilt. Only machines with a storeSigningPublicKey are used.
substituters = lib.pipe buildServerDevices [
(lib.filterAttrs (_: v: v.distributedBuilds ? storeSigningPublicKey))
(lib.mapAttrsToList (hostName: _: "ssh-ng://${buildUser}@${hostName}"))
(lib.filter (s: s != "ssh-ng://${buildUser}@${config.networking.hostName}"))
];
trusted-public-keys = lib.pipe buildServerDevices [
(lib.mapAttrsToList (_: v: v.distributedBuilds.storeSigningPublicKey or null))
(builtins.filter (k: k != null))
];
secret-key-files =
let
thisDevice = devices.${config.networking.hostName} or { };
in
lib.optional (thisDevice.distributedBuilds.isBuilder or false) "/etc/nix/signing-key.sec";
max-jobs = (devices.${config.networking.hostName}.distributedBuilds or { }).maxJobs or "auto";
cores = 0;
min-free = 10 * 1024 * 1024;
max-free = 200 * 1024 * 1024;
};
};
# Server: accept incoming build connections
(lib.mkIf (thisDevice.distributedBuilds.isBuilder or false) {
users.users.${buildUser} = {
isSystemUser = true;
group = buildUser;
useDefaultShell = true;
openssh.authorizedKeys.keys = map (
k: ''command="nix daemon --stdio",restrict ${k}''
) allClientPublicKeys;
};
users.groups.${buildUser} = { };
nix.settings = {
trusted-users = [ buildUser ];
secret-key-files = [ "/etc/nix/signing-key.sec" ];
};
})
systemd.services.nix-daemon.serviceConfig = {
MemoryAccounting = true;
MemoryMax = "90%";
OOMScoreAdjust = 500;
};
};
# Client: connect to build servers for building and substitution
(lib.mkIf isClient {
programs.ssh = {
knownHosts = buildServerKnownHosts;
extraConfig = ''
Host ${lib.concatStringsSep " " (lib.attrNames buildServerDevices)}
User ${buildUser}
IdentityFile ${clientSshKeyPath}
IdentitiesOnly yes
'';
};
nix = {
distributedBuilds = buildMachines != [ ];
buildMachines = buildMachines;
settings = {
builders-use-substitutes = true;
substituters = map (m: "ssh-ng://${buildUser}@${m.hostName}") (
builtins.filter (m: m.distributedBuilds ? storeSigningPublicKey) remoteBuildServerDevices
);
};
};
})
]
);
}