From 05ea59d38a5e202dad4039c01bdc0c474ea1d763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 31 Oct 2024 14:51:55 +0100 Subject: [PATCH 1/3] inventory: make sure we always define all roles, even if we don't have machines this makes write writing modules easier add missing roles assertions --- lib/inventory/build-inventory/default.nix | 56 +++++++++++++---------- lib/inventory/tests/default.nix | 2 +- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/lib/inventory/build-inventory/default.nix b/lib/inventory/build-inventory/default.nix index cf7b8394e..9d6fe2439 100644 --- a/lib/inventory/build-inventory/default.nix +++ b/lib/inventory/build-inventory/default.nix @@ -39,11 +39,9 @@ let checkService = serviceName: - let - frontmatter = clan-core.lib.modules.getFrontmatter serviceName; - in - if builtins.elem "inventory" frontmatter.features or [ ] then true else false; + builtins.elem "inventory" (clan-core.lib.modules.getFrontmatter serviceName).features or [ ]; + trimExtension = name: builtins.substring 0 (builtins.stringLength name - 4) name; /* Returns a NixOS configuration for every machine in the inventory. @@ -65,18 +63,29 @@ let acc2: instanceName: serviceConfig: let - resolvedRoles = builtins.mapAttrs ( - roleName: members: + roles = lib.mapAttrsToList (name: _value: trimExtension name) ( + lib.filterAttrs (name: type: type == "regular" && lib.hasSuffix ".nix" name) ( + builtins.readDir ( + if clan-core.clanModules ? ${serviceName} then + clan-core.clanModules.${serviceName} + "/roles" + else + throw "ClanModule not found: '${serviceName}'. Make sure the module is added in the 'clanModules' attribute of clan-core." + ) + ) + ); + + resolvedRoles = lib.genAttrs roles ( + roleName: resolveTags { + members = serviceConfig.roles.${roleName} or { }; inherit serviceName instanceName roleName inventory - members ; } - ) serviceConfig.roles; + ); isInService = builtins.any (members: builtins.elem machineName members.machines) ( builtins.attrValues resolvedRoles @@ -98,18 +107,12 @@ let # TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy roleModules = builtins.map ( role: - let - # Check the module exists - module = - clan-core.clanModules.${serviceName} - or (throw "ClanModule not found: '${serviceName}'. Make sure the module is added in the 'clanModules' attribute of clan-core."); - - path = module + "/roles/${role}.nix"; - in - if builtins.pathExists path then - path + if builtins.elem role roles && clan-core.clanModules ? ${serviceName} then + clan-core.clanModules.${serviceName} + "/roles/${role}.nix" else - throw "Module doesn't have role: '${role}'. Role: ${role}.nix not found." + throw "Module ${serviceName} doesn't have role: '${role}'. Role: ${ + clan-core.clanModules.${serviceName} + }/roles/${role}.nix not found." ) machineRoles; roleServiceConfigs = builtins.filter (m: m != { }) ( @@ -119,8 +122,14 @@ let extraModules = map (s: if builtins.typeOf s == "string" then "${directory}/${s}" else s) ( globalExtraModules ++ machineExtraModules ++ roleServiceExtraModules ); + + nonExistingRoles = builtins.filter (role: !(builtins.elem role roles)) ( + builtins.attrNames (serviceConfig.roles or { }) + ); in - if !(serviceConfig.enabled or true) then + if (nonExistingRoles != [ ]) then + throw "Roles ${builtins.toString nonExistingRoles} are not defined in the service ${serviceName}." + else if !(serviceConfig.enabled or true) then acc2 else if isInService then acc2 @@ -139,14 +148,13 @@ let ); } ) - - { - config.clan.inventory.services.${serviceName}.${instanceName} = { + ({ + clan.inventory.services.${serviceName}.${instanceName} = { roles = resolvedRoles; # TODO: Add inverseRoles to the service config if needed # inherit inverseRoles; }; - } + }) ] else acc2 diff --git a/lib/inventory/tests/default.nix b/lib/inventory/tests/default.nix index b275246a1..8fdffdcf0 100644 --- a/lib/inventory/tests/default.nix +++ b/lib/inventory/tests/default.nix @@ -171,7 +171,7 @@ in expr = configs; expectedError = { type = "ThrownError"; - msg = "Module doesn't have role.*"; + msg = "Roles roleXYZ are not defined in the service borgbackup."; }; }; test_inventory_tag_doesnt_exist = From c02ecc2f673a3b2338ceb851b80ed4da6940ef03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 31 Oct 2024 16:50:14 +0100 Subject: [PATCH 2/3] tests/command: also wait for exit status --- pkgs/clan-cli/tests/command.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/clan-cli/tests/command.py b/pkgs/clan-cli/tests/command.py index 608ae439f..8284c7e69 100644 --- a/pkgs/clan-cli/tests/command.py +++ b/pkgs/clan-cli/tests/command.py @@ -49,6 +49,7 @@ class Command: for p in reversed(self.processes): with contextlib.suppress(OSError): os.killpg(os.getpgid(p.pid), signal.SIGKILL) + p.wait() @pytest.fixture From 2f3369cca5599471a5d51303ad968f81a26109af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 4 Nov 2024 13:19:00 +0100 Subject: [PATCH 3/3] add build-on-remote flag --- checks/installation/flake-module.nix | 1 + pkgs/clan-cli/clan_cli/machines/install.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/checks/installation/flake-module.nix b/checks/installation/flake-module.nix index dd39b94ed..f2bc20dee 100644 --- a/checks/installation/flake-module.nix +++ b/checks/installation/flake-module.nix @@ -62,6 +62,7 @@ "flakes" ]; }; + system.extraDependencies = dependencies; }; nodes.client = { environment.systemPackages = [ diff --git a/pkgs/clan-cli/clan_cli/machines/install.py b/pkgs/clan-cli/clan_cli/machines/install.py index 98fbf2105..414e836c7 100644 --- a/pkgs/clan-cli/clan_cli/machines/install.py +++ b/pkgs/clan-cli/clan_cli/machines/install.py @@ -31,6 +31,7 @@ def install_nixos( password: str | None = None, no_reboot: bool = False, extra_args: list[str] | None = None, + build_on_remote: bool = False, ) -> None: if extra_args is None: extra_args = [] @@ -70,6 +71,9 @@ def install_nixos( if no_reboot: cmd.append("--no-reboot") + if build_on_remote: + cmd.append("--build-on-remote") + if password: cmd += [ "--env-password", @@ -104,6 +108,7 @@ class InstallOptions: debug: bool = False no_reboot: bool = False json_ssh_deploy: dict[str, str] | None = None + build_on_remote: bool = False nix_options: list[str] = field(default_factory=list) @@ -119,6 +124,7 @@ def install_machine(opts: InstallOptions, password: str | None) -> None: password=password, no_reboot=opts.no_reboot, extra_args=opts.nix_options, + build_on_remote=opts.build_on_remote, ) @@ -159,6 +165,7 @@ def install_command(args: argparse.Namespace) -> None: no_reboot=args.no_reboot, json_ssh_deploy=json_ssh_deploy, nix_options=args.option, + build_on_remote=args.build_on_remote, ), password, ) @@ -192,6 +199,12 @@ def register_install_parser(parser: argparse.ArgumentParser) -> None: help="do not reboot after installation", default=False, ) + parser.add_argument( + "--build-on-remote", + action="store_true", + help="build the NixOS configuration on the remote machine", + default=False, + ) parser.add_argument( "--yes", action="store_true",