{self}: { _class = "clan.service"; manifest.name = "remote-builders"; # Define what roles exist roles.worker = { interface = {lib, ...}: { # These options can be set via 'roles.client.settings' options.supportedFeatures = lib.mkOption { type = with lib.types; listOf (enum [ "nixos-test" "benchmark" "big-parallel" "kvm" ]); default = []; description = '' kvm | Everything which builds inside a vm, like NixOS tests nixos-test | Machine can run NixOS tests big-parallel | kernel config, libreoffice, evolution, llvm and chromium benchmark | Machine can generate metrics (means the builds usually takes the same amount of time) ''; }; }; # Maps over all instances and produces one result per instance. perInstance = { roles, machine, ... }: { # Analog to 'perSystem' of flake-parts. # For every instance of this service we will add a nixosModule to a client-machine nixosModule = { config, lib, ... }: let inherit (lib) filterAttrs hasAttr mapAttrsToList; clients = filterAttrs (name: _value: hasAttr name roles.client.machines) self.nixosConfigurations; others = filterAttrs (name: _value: name != machine.name) clients; remotebuildKeys = mapAttrsToList ( _name: attrs: attrs.config.clan.core.vars.generators.remotebuild.files."ssh.id_ed25519.pub".value ) others; in { # Interaction examples what you could do here: # - Get some settings of this machine # settings.ipRanges # # - Get all controller names: # allControllerNames = lib.attrNames roles.controller.machines # # - Get all roles of the machine: # machine.roles # # - Get the settings that where applied to a specific controller machine: # roles.controller.machines.jon.settings # # Add one systemd service for every instance users.users.remotebuild = { isNormalUser = true; createHome = false; group = "remotebuild"; openssh.authorizedKeys.keys = remotebuildKeys; }; users.groups.remotebuild = {}; }; }; }; roles.client = { interface = {}; perInstance = { roles, machine, ... }: { nixosModule = { config, pkgs, lib, ... }: let inherit (lib) filterAttrs hasAttr mapAttrsToList concatLines optional; workers = filterAttrs (name: _value: hasAttr name roles.worker.machines) self.nixosConfigurations; others = filterAttrs (name: _value: name != machine.name) workers; mkBuilder = name: attrs: let config' = attrs.config; cfg' = roles.worker.machines.${name}.settings; pkgs' = attrs.pkgs; in { hostName = name; sshUser = "remotebuild"; # CPU architecture of the builder, and the operating system it runs. # If your builder supports multiple architectures # (e.g. search for "binfmt" for emulation), systems = [pkgs'.system] ++ config'.boot.binfmt.emulatedSystems; # Nix custom ssh-variant that avoids lots of "trusted-users" settings pain protocol = "ssh-ng"; # default is 1 but may keep the builder idle in between builds maxJobs = 3; speedFactor = 1; supportedFeatures = cfg'.supportedFeatures; mandatoryFeatures = []; }; otherBuildMachines = mapAttrsToList mkBuilder others; buildMachines = otherBuildMachines ++ optional (hasAttr machine.name roles.worker.machines) { # NOTE: https://github.com/NixOS/nix/issues/3177 hostName = "local?root=/nix/store"; sshUser = null; # CPU architecture of the builder, and the operating system it runs. # If your builder supports multiple architectures # (e.g. search for "binfmt" for emulation), systems = [pkgs.system] ++ config.boot.binfmt.emulatedSystems; protocol = null; # default is 1 but may keep the builder idle in between builds maxJobs = 3; speedFactor = 1; supportedFeatures = roles.worker.machines.${machine.name}.settings.supportedFeatures; mandatoryFeatures = []; }; mkMatch = name: _value: '' Match User remotebuild Host ${name} IdentityFile ${config.clan.core.vars.generators.remotebuild.files."ssh.id_ed25519".path} ConnectTimeout 1 ''; sshConfig = concatLines (mapAttrsToList mkMatch others); in { programs.ssh.extraConfig = sshConfig; clan.core.vars.generators.remotebuild = { files."ssh.id_ed25519" = {}; files."ssh.id_ed25519.pub".secret = false; runtimeInputs = [ pkgs.coreutils pkgs.openssh ]; script = '' ssh-keygen -t ed25519 -N "" -f "$out"/ssh.id_ed25519 ''; }; nix = { buildMachines = buildMachines; # required, otherwise remote buildMachines above aren't used distributedBuilds = true; # optional, useful when the builder has a faster internet connection than yours settings = { builders-use-substitutes = true; trusted-users = ["remotebuild"]; }; }; }; }; }; # Maps over all machines and produces one result per machine. perMachine = {...}: { # Analog to 'perSystem' of flake-parts. # For every machine of this service we will add exactly one nixosModule to a machine nixosModule = {...}: { # Interaction examples what you could do here: # - Get the name of this machine # machine.name # # - Get all roles of this machine across all instances: # machine.roles # # - Get the settings of a specific instance of a specific machine # instances.foo.roles.peer.machines.jon.settings # }; }; }