moritz/clan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: remote-builders to clan service

Browse source
This commit is contained in:
Moritz Böhme 2025-04-20 10:22:37 +02:00
parent f761ede7ff
commit fde70ab2bb
No known key found for this signature in database
GPG key ID: 970C6E89EB0547A9
8 changed files with 208 additions and 150 deletions
Download patch file Download diff file Expand all files Collapse all files

9
clanServices/flake-module.nix Normal file
View file

@ -0,0 +1,9 @@
{
self,
lib,
...
}: {
clan.inventory.modules = {
remote-builders = lib.modules.importApply ./remote-builders/default.nix {inherit self;};
};
}

171
clanServices/remote-builders/default.nix Normal file
View file

@ -0,0 +1,171 @@
{self}: {
_class = "clan.service";
manifest.name = "remote-builders";
# Define what roles exist
roles.worker = {
interface = {lib, ...}: {
# These options can be set via 'roles.client.settings'
options.supportedFeatures = lib.mkOption {
type = with lib.types;
listOf (
oneOf [
"nixos-test"
"benchmark"
"big-parallel"
"kvm"
]
);
default = [];
description = ''
kvm | Everything which builds inside a vm, like NixOS tests
nixos-test | Machine can run NixOS tests
big-parallel | kernel config, libreoffice, evolution, llvm and chromium
benchmark | Machine can generate metrics (means the builds usually takes the same amount of time)
'';
};
};
# Maps over all instances and produces one result per instance.
perInstance = {roles, ...}: {
# Analog to 'perSystem' of flake-parts.
# For every instance of this service we will add a nixosModule to a client-machine
nixosModule = {
config,
lib,
...
}: let
inherit (lib) filterAttrs hasAttr mapAttrsToList;
clients = filterAttrs (name: _value: hasAttr name roles.client.machines) self.nixosConfigurations;
others = filterAttrs (_name: value: value.config.networking.hostName != config.networking.hostName) clients;
remotebuildKeys =
mapAttrsToList (
_name: attrs: attrs.config.clan.core.vars.generators.remotebuild.files."ssh.id_ed25519.pub".value
)
others;
in {
# Interaction examples what you could do here:
# - Get some settings of this machine
# settings.ipRanges
#
# - Get all controller names:
# allControllerNames = lib.attrNames roles.controller.machines
#
# - Get all roles of the machine:
# machine.roles
#
# - Get the settings that where applied to a specific controller machine:
# roles.controller.machines.jon.settings
#
# Add one systemd service for every instance
users.users.remotebuild = {
isNormalUser = true;
createHome = false;
group = "remotebuild";
openssh.authorizedKeys.keys = remotebuildKeys;
};
users.groups.remotebuild = {};
};
};
};
roles.client = {
interface = {};
perInstance = {roles, ...}: {
nixosModule = {
config,
pkgs,
lib,
...
}: let
inherit (lib) filterAttrs hasAttr mapAttrsToList concatLines;
workers = filterAttrs (name: _value: hasAttr name roles.worker.machines) self.nixosConfigurations;
mkBuilder = hostName: attrs: let
config' = attrs.config;
cfg' = roles.worker.machines.${hostName}.settings;
pkgs' = attrs.pkgs;
in {
# NOTE: https://github.com/NixOS/nix/issues/3177
hostName =
if config'.networking.hostName == config.networking.hostName
then "local?root=/nix/store"
else hostName;
sshUser =
if config'.networking.hostName == config.networking.hostName
then null
else "remotebuild";
# CPU architecture of the builder, and the operating system it runs.
# If your builder supports multiple architectures
# (e.g. search for "binfmt" for emulation),
systems = [pkgs'.system] ++ config'.boot.binfmt.emulatedSystems;
# Nix custom ssh-variant that avoids lots of "trusted-users" settings pain
protocol =
if config'.networking.hostName == config.networking.hostName
then null
else "ssh-ng";
# default is 1 but may keep the builder idle in between builds
maxJobs = 3;
speedFactor = 1;
supportedFeatures = cfg'.supportedFeatures;
mandatoryFeatures = [];
};
buildMachines = mapAttrsToList mkBuilder workers;
others = filterAttrs (_name: value: value.config.networking.hostName != config.networking.hostName) workers;
mkMatch = _name: value: ''
Match User remotebuild Host ${value.config.networking.hostName}
IdentityFile ${config.clan.core.vars.generators.remotebuild.files."ssh.id_ed25519".path}
'';
sshConfig = concatLines (mapAttrsToList mkMatch others);
in {
programs.ssh.extraConfig = sshConfig;
clan.core.vars.generators.remotebuild = {
files."ssh.id_ed25519" = {};
files."ssh.id_ed25519.pub".secret = false;
runtimeInputs = [
pkgs.coreutils
pkgs.openssh
];
script = ''
ssh-keygen -t ed25519 -N "" -f "$out"/ssh.id_ed25519
'';
};
nix = {
buildMachines = buildMachines;
# required, otherwise remote buildMachines above aren't used
distributedBuilds = true;
# optional, useful when the builder has a faster internet connection than yours
settings = {
builders-use-substitutes = true;
trusted-users = ["remotebuild"];
};
};
};
};
};
# Maps over all machines and produces one result per machine.
perMachine = {...}: {
# Analog to 'perSystem' of flake-parts.
# For every machine of this service we will add exactly one nixosModule to a machine
nixosModule = {...}: {
# Interaction examples what you could do here:
# - Get the name of this machine
# machine.name
#
# - Get all roles of this machine across all instances:
# machine.roles
#
# - Get the settings of a specific instance of a specific machine
# instances.foo.roles.peer.machines.jon.settings
#
};
};
}

40
flake.lock generated
View file

@ -50,11 +50,11 @@
"treefmt-nix": "treefmt-nix" "treefmt-nix": "treefmt-nix"
}, },
"locked": { "locked": {
"lastModified": 1744472425, "lastModified": 1745001614,
"narHash": "sha256-zBx1nBO+vlaPgk1+prUjaREbJLMVW2omMtY2ppWtBPc=", "narHash": "sha256-NAADglNN3lh/3hgTbkynbYC2JuQCCGvv3Yizm9I64ns=",
"ref": "refs/heads/main", "ref": "refs/heads/main",
"rev": "d69ae6049a4bf325aaabe646c235d5777cbb56aa", "rev": "779d28577de180c60ed4774c002cc5814162a0f9",
"revCount": 6272, "revCount": 6386,
"type": "git", "type": "git",
"url": "https://git.clan.lol/clan/clan-core" "url": "https://git.clan.lol/clan/clan-core"
}, },
@ -98,11 +98,11 @@
] ]
}, },
"locked": { "locked": {
"lastModified": 1744470659, "lastModified": 1744961442,
"narHash": "sha256-syTeDvkDg9eiFpn8gUOz3DLOHJoJFS1Md7DXJQBIQFg=", "narHash": "sha256-ymbmZjOaIoVwEuTpFDjlXPGufqmdlzdDEVpQAjNi8Ls=",
"ref": "refs/heads/main", "ref": "refs/heads/main",
"rev": "35cc96c4f952a93ecb1abc8caefddc52f69fdd98", "rev": "0328b8c5912db9562e1ab115f760ca7ba0044aff",
"revCount": 401, "revCount": 409,
"type": "git", "type": "git",
"url": "https://git.clan.lol/clan/data-mesher" "url": "https://git.clan.lol/clan/data-mesher"
}, },
@ -119,11 +119,11 @@
] ]
}, },
"locked": { "locked": {
"lastModified": 1744145203, "lastModified": 1744940522,
"narHash": "sha256-I2oILRiJ6G+BOSjY+0dGrTPe080L3pbKpc+gCV3Nmyk=", "narHash": "sha256-TNoetfICvd29DhxRPpmyKItQBDlqSvKcV+wGNkn14jk=",
"owner": "nix-community", "owner": "nix-community",
"repo": "disko", "repo": "disko",
"rev": "76c0a6dba345490508f36c1aa3c7ba5b6b460989", "rev": "51d33bbb7f1e74ba5f9d9a77357735149da99081",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -712,11 +712,11 @@
] ]
}, },
"locked": { "locked": {
"lastModified": 1744224272, "lastModified": 1744478979,
"narHash": "sha256-cqePj5nuC7flJWNncaVAFq1YZncU0PSyO0DEqGn+vYc=", "narHash": "sha256-dyN+teG9G82G+m+PX/aSAagkC+vUv0SgUw3XkPhQodQ=",
"owner": "nix-darwin", "owner": "nix-darwin",
"repo": "nix-darwin", "repo": "nix-darwin",
"rev": "113883e37d985d26ecb65282766e5719f2539103", "rev": "43975d782b418ebf4969e9ccba82466728c2851b",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -1081,11 +1081,11 @@
] ]
}, },
"locked": { "locked": {
"lastModified": 1744103455, "lastModified": 1744669848,
"narHash": "sha256-SR6+qjkPjGQG+8eM4dCcVtss8r9bre/LAxFMPJpaZeU=", "narHash": "sha256-pXyanHLUzLNd3MX9vsWG+6Z2hTU8niyphWstYEP3/GU=",
"owner": "Mic92", "owner": "Mic92",
"repo": "sops-nix", "repo": "sops-nix",
"rev": "69d5a5a4635c27dae5a742f36108beccc506c1ba", "rev": "61154300d945f0b147b30d24ddcafa159148026a",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -1198,11 +1198,11 @@
] ]
}, },
"locked": { "locked": {
"lastModified": 1743748085, "lastModified": 1744961264,
"narHash": "sha256-uhjnlaVTWo5iD3LXics1rp9gaKgDRQj6660+gbUU3cE=", "narHash": "sha256-aRmUh0AMwcbdjJHnytg1e5h5ECcaWtIFQa6d9gI85AI=",
"owner": "numtide", "owner": "numtide",
"repo": "treefmt-nix", "repo": "treefmt-nix",
"rev": "815e4121d6a5d504c0f96e5be2dd7f871e4fd99d", "rev": "8d404a69efe76146368885110f29a2ca3700bee6",
"type": "github" "type": "github"
}, },
"original": { "original": {

10
flake.nix
View file

@ -66,6 +66,7 @@
imports = [ imports = [
inputs.clan-core.flakeModules.default inputs.clan-core.flakeModules.default
inputs.git-hooks-nix.flakeModule inputs.git-hooks-nix.flakeModule
./clanServices/flake-module.nix
]; ];
perSystem = { perSystem = {
@ -99,9 +100,14 @@
inventory = { inventory = {
machines = { machines = {
moritz-desktop.tags = ["peer"]; moritz-desktop.tags = ["peer" "client" "worker"];
moritz-laptop.tags = ["peer"]; moritz-laptop.tags = ["peer"];
moritz-server.tags = []; moritz-server.tags = ["client" "worker"];
};
instances."remote-builders" = {
module.name = "remote-builders";
roles.client.tags.client = {};
roles.worker.tags.worker = {};
}; };
services.zerotier.default = { services.zerotier.default = {
roles.peer.tags = [ roles.peer.tags = [

2
machines/moritz-desktop/configuration.nix
View file

@ -23,8 +23,6 @@
# This only works however if you have avahi running on your admin machine else use IP # This only works however if you have avahi running on your admin machine else use IP
clan.core.networking.targetHost = "root@moritz-desktop"; clan.core.networking.targetHost = "root@moritz-desktop";
our.buildMachines.enable = true;
my = { my = {
ai.enable = true; ai.enable = true;
profiles = { profiles = {

2
machines/moritz-server/configuration.nix
View file

@ -11,8 +11,6 @@
# This only works however if you have avahi running on your admin machine else use IP # This only works however if you have avahi running on your admin machine else use IP
clan.core.networking.targetHost = "root@moritz-server"; clan.core.networking.targetHost = "root@moritz-server";
our.buildMachines.enable = true;
networking = { networking = {
interfaces.enp2s0 = { interfaces.enp2s0 = {
ipv4.addresses = [ ipv4.addresses = [

123
modules/remote_builders.nix
View file

@ -1,123 +0,0 @@
{
lib,
config,
pkgs,
self,
...
}: let
inherit
(lib)
filterAttrs
mkEnableOption
mkIf
mapAttrsToList
concatLines
mkOption
types
;
cfg = config.our.buildMachines;
builders = filterAttrs (_n: v: v.config.our.buildMachines.enable) self.nixosConfigurations;
others = filterAttrs (n: _v: n != config.networking.hostName) builders;
mkBuilder = hostName: attrs: let
config' = attrs.config;
cfg' = config'.our.buildMachines;
pkgs' = attrs.pkgs;
in {
# NOTE: https://github.com/NixOS/nix/issues/3177
hostName =
if hostName == config.networking.hostName
then "local?root=/nix/store"
else hostName;
sshUser =
if hostName == config.networking.hostName
then null
else "remotebuild";
# CPU architecture of the builder, and the operating system it runs.
# If your builder supports multiple architectures
# (e.g. search for "binfmt" for emulation),
systems = [pkgs'.system] ++ config'.boot.binfmt.emulatedSystems;
# Nix custom ssh-variant that avoids lots of "trusted-users" settings pain
protocol =
if hostName == config.networking.hostName
then null
else "ssh-ng";
# default is 1 but may keep the builder idle in between builds
maxJobs = 3;
speedFactor = 1;
supportedFeatures = cfg'.supportedFeatures;
mandatoryFeatures = [];
};
buildMachines = mapAttrsToList mkBuilder builders;
remotebuildKeys =
mapAttrsToList (
_name: attrs: attrs.config.clan.core.vars.generators.remotebuild.files."ssh.id_ed25519.pub".value
)
others;
mkMatch = hostName: _v: ''
Match User remotebuild Host ${hostName}
IdentityFile ${config.clan.core.vars.generators.remotebuild.files."ssh.id_ed25519".path}
'';
sshConfig = concatLines (mapAttrsToList mkMatch others);
in {
options.our.buildMachines = {
enable = mkEnableOption "Use this machine as a remoteBuilder for others and vice versa.";
supportedFeatures = mkOption {
type = types.listOf (
types.oneOf [
"nixos-test"
"benchmark"
"big-parallel"
"kvm"
]
);
default = [];
description = ''
kvm | Everything which builds inside a vm, like NixOS tests
nixos-test | Machine can run NixOS tests
big-parallel | kernel config, libreoffice, evolution, llvm and chromium
benchmark | Machine can generate metrics (means the builds usually takes the same amount of time)
'';
};
};
config = mkIf cfg.enable {
users.users.remotebuild = {
isNormalUser = true;
createHome = false;
group = "remotebuild";
openssh.authorizedKeys.keys = remotebuildKeys;
};
users.groups.remotebuild = {};
clan.core.vars.generators.remotebuild = {
files."ssh.id_ed25519" = {};
files."ssh.id_ed25519.pub".secret = false;
runtimeInputs = [
pkgs.coreutils
pkgs.openssh
];
script = ''
ssh-keygen -t ed25519 -N "" -f "$out"/ssh.id_ed25519
'';
};
programs.ssh.extraConfig = sshConfig;
nix = {
buildMachines = buildMachines;
# required, otherwise remote buildMachines above aren't used
distributedBuilds = true;
# optional, useful when the builder has a faster internet connection than yours
settings = {
builders-use-substitutes = true;
trusted-users = ["remotebuild"];
};
};
};
}

1
modules/shared.nix
View file

@ -10,7 +10,6 @@
clan-core.clanModules.state-version clan-core.clanModules.state-version
clan-core.clanModules.static-hosts clan-core.clanModules.static-hosts
clan-core.clanModules.machine-id clan-core.clanModules.machine-id
./remote_builders.nix
]; ];
networking.hosts."fd77:acc0:1d56:2265:499:9367:28e0:97d3" = ["moritz-remarkable"]; networking.hosts."fd77:acc0:1d56:2265:499:9367:28e0:97d3" = ["moritz-remarkable"];