From 382e27a20b9a22faa200941a27a794e1d73fcadf Mon Sep 17 00:00:00 2001 From: Qubasa Date: Fri, 8 Nov 2024 14:11:26 +0700 Subject: [PATCH 1/4] clan: Remove pkgsForSystem --- lib/build-clan/interface.nix | 14 ++------------ lib/build-clan/module.nix | 11 +++++++---- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/lib/build-clan/interface.nix b/lib/build-clan/interface.nix index 8dd3abf74..df4743980 100644 --- a/lib/build-clan/interface.nix +++ b/lib/build-clan/interface.nix @@ -3,6 +3,7 @@ let types = lib.types; in { + options = { # Required options directory = lib.mkOption { @@ -69,18 +70,6 @@ in default = { }; }; - pkgsForSystem = lib.mkOption { - type = types.functionTo (types.nullOr types.attrs); - default = _: null; - defaultText = "Lambda :: String -> { ... } | null"; - description = '' - A function that maps from architecture to pkg. `( string -> pkgs )` - - If specified this nixpkgs will be only imported once for each system. - This improves performance, but all nipxkgs.* options will be ignored. - ''; - }; - # Outputs nixosConfigurations = lib.mkOption { # Hide from documentation. @@ -112,4 +101,5 @@ in }; }; }; + } diff --git a/lib/build-clan/module.nix b/lib/build-clan/module.nix index 079a7762c..cf2a34e54 100644 --- a/lib/build-clan/module.nix +++ b/lib/build-clan/module.nix @@ -9,7 +9,6 @@ let inherit (config) directory machines - pkgsForSystem specialArgs ; @@ -58,7 +57,7 @@ let # Settings clan.core.clanDir = directory; # Inherited from clan wide settings - # TODO: remove these + # TODO: remove these` clan.core.name = config.inventory.meta.name; clan.core.icon = config.inventory.meta.icon; @@ -105,7 +104,11 @@ let name: _: nixosConfiguration { inherit name system; - pkgs = pkgsForSystem system; + + # We removed pkgsForSystems because we have the problem that nixpkgs.* options are then ignored + # However our current model is to have the hardware config read and then set nixpkgs.hostPlatform automatically + # which gets ignored if we set pkgsForSystems + pkgs = nixpkgs.legacyPackages.${system}; } ) allMachines ) @@ -122,7 +125,7 @@ let args // { inherit name system; - pkgs = pkgsForSystem system; + pkgs = nixpkgs.legacyPackages.${system}; } ) ) allMachines From 9fc4e4c7d4806eca384d0a51cdc8be9763cf0d62 Mon Sep 17 00:00:00 2001 From: Qubasa Date: Fri, 8 Nov 2024 14:13:37 +0700 Subject: [PATCH 2/4] clan-cli: Fix nixos-anywhere for systems that can not be build locally --- .../clan_cli/machines/host_platform.py | 47 +++++++++++++++++++ pkgs/clan-cli/clan_cli/machines/install.py | 12 ++--- pkgs/clan-cli/clan_cli/machines/machines.py | 43 ++++++++++++----- 3 files changed, 84 insertions(+), 18 deletions(-) create mode 100644 pkgs/clan-cli/clan_cli/machines/host_platform.py diff --git a/pkgs/clan-cli/clan_cli/machines/host_platform.py b/pkgs/clan-cli/clan_cli/machines/host_platform.py new file mode 100644 index 000000000..bdc8258d4 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/machines/host_platform.py @@ -0,0 +1,47 @@ +from dataclasses import dataclass, fields +from typing import Any + + +@dataclass +class HostPlatform: + config: str + darwinArch: str + darwinMinVersion: str + darwinSdkVersion: str + is32bit: bool + is64bit: bool + isx86_64: bool + isAarch: bool + isDarwin: bool + isFreeBSD: bool + isLinux: bool + isMacOS: bool + isWindows: bool + isAndroid: bool + linuxArch: str + qemuArch: str + system: str + ubootArch: str + + # ruff: noqa: N815 + + @staticmethod + def from_dict(data: dict[str, Any]) -> "HostPlatform": + """ + Factory method that creates an instance of HostPlatform from a dictionary. + Extra fields in the dictionary are ignored. + + Args: + data (dict): A dictionary containing values for initializing HostPlatform. + + Returns: + HostPlatform: An instance of the HostPlatform class. + """ + # Dynamically extract field names from the dataclass + valid_keys = {field.name for field in fields(HostPlatform)} + + # Filter the dictionary to only include items with keys that are valid + filtered_data = {k: v for k, v in data.items() if k in valid_keys} + + # Pass the filtered data to the HostPlatform constructor + return HostPlatform(**filtered_data) diff --git a/pkgs/clan-cli/clan_cli/machines/install.py b/pkgs/clan-cli/clan_cli/machines/install.py index 05fbadc9c..52d2ff440 100644 --- a/pkgs/clan-cli/clan_cli/machines/install.py +++ b/pkgs/clan-cli/clan_cli/machines/install.py @@ -15,6 +15,7 @@ from clan_cli.completions import ( complete_machines, complete_target_host, ) +from clan_cli.errors import ClanError from clan_cli.facts.generate import generate_facts from clan_cli.machines.hardware import HardwareConfig from clan_cli.machines.machines import Machine @@ -25,10 +26,6 @@ from clan_cli.vars.generate import generate_vars log = logging.getLogger(__name__) -class ClanError(Exception): - pass - - @dataclass class InstallOptions: # flake to install @@ -85,9 +82,6 @@ def install_machine(opts: InstallOptions) -> None: if opts.no_reboot: cmd.append("--no-reboot") - if opts.build_on_remote: - cmd.append("--build-on-remote") - if opts.update_hardware_config is not HardwareConfig.NONE: cmd.extend( [ @@ -108,6 +102,10 @@ def install_machine(opts: InstallOptions) -> None: "IdentitiesOnly=yes", ] + if not machine.can_build_locally or opts.build_on_remote: + log.info("Architecture mismatch. Building on remote machine") + cmd.append("--build-on-remote") + if machine.target_host.port: cmd += ["--ssh-port", str(machine.target_host.port)] if opts.kexec: diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 2818414a6..6e836b96b 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -12,6 +12,7 @@ from clan_cli.cmd import run_no_stdout from clan_cli.errors import ClanError from clan_cli.facts import public_modules as facts_public_modules from clan_cli.facts import secret_modules as facts_secret_modules +from clan_cli.machines import host_platform from clan_cli.nix import nix_build, nix_config, nix_eval, nix_metadata from clan_cli.ssh import Host, HostKeyCheck, parse_deployment_address from clan_cli.vars.public_modules import FactStoreBase @@ -46,6 +47,35 @@ class Machine: def __repr__(self) -> str: return str(self) + @property + def host_platform(self) -> host_platform.HostPlatform: + # We filter out function attributes because they are not serializable. + attr = f""" + (let + machine = ((builtins.getFlake "{self.flake}").nixosConfigurations.{self.name}); + lib = machine.lib; + removeFunctionAttrs = attrset: + lib.filterAttrs (name: value: lib.isFunction value == false && name != "parsed") attrset; + in + {{ x = removeFunctionAttrs machine.pkgs.stdenv.hostPlatform; }}).x + """ + if attr in self._eval_cache: + output = self._eval_cache[attr] + else: + output = run_no_stdout( + nix_eval(["--impure", "--expr", attr]) + ).stdout.strip() + self._eval_cache[attr] = output + value = json.loads(output) + return host_platform.HostPlatform.from_dict(value) + + @property + def can_build_locally(self) -> bool: + # TODO: We could also use the function pkgs.stdenv.hostPlatform.canExecute + # but this is good enough for now. + output = nix_config() + return self.host_platform.system == output["system"] + @property def deployment(self) -> dict: if self.cached_deployment is not None: @@ -173,7 +203,6 @@ class Machine: method: Literal["eval", "build"], attr: str, extra_config: None | dict = None, - impure: bool = False, nix_options: list[str] | None = None, ) -> str | Path: """ @@ -215,12 +244,6 @@ class Machine: "dirtyRevision" in metadata or "dirtyRev" in metadata["locks"]["nodes"]["clan-core"]["locked"] ): - # if not impure: - # raise ClanError( - # "The machine has a dirty revision, and impure mode is not allowed" - # ) - # else: - # args += ["--impure"] args += ["--impure"] args += [ @@ -254,7 +277,6 @@ class Machine: attr: str, refresh: bool = False, extra_config: None | dict = None, - impure: bool = False, nix_options: list[str] | None = None, ) -> str: """ @@ -266,7 +288,7 @@ class Machine: if attr in self._eval_cache and not refresh and extra_config is None: return self._eval_cache[attr] - output = self.nix("eval", attr, extra_config, impure, nix_options) + output = self.nix("eval", attr, extra_config, nix_options) if isinstance(output, str): self._eval_cache[attr] = output return output @@ -278,7 +300,6 @@ class Machine: attr: str, refresh: bool = False, extra_config: None | dict = None, - impure: bool = False, nix_options: list[str] | None = None, ) -> Path: """ @@ -291,7 +312,7 @@ class Machine: if attr in self._build_cache and not refresh and extra_config is None: return self._build_cache[attr] - output = self.nix("build", attr, extra_config, impure, nix_options) + output = self.nix("build", attr, extra_config, nix_options) if isinstance(output, Path): self._build_cache[attr] = output return output From 6ac6d626df2db2ce6fc65d3a1089b99367f5fd8c Mon Sep 17 00:00:00 2001 From: Qubasa Date: Mon, 11 Nov 2024 19:50:06 +0700 Subject: [PATCH 3/4] module.nix: add better explanation --- lib/build-clan/module.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/build-clan/module.nix b/lib/build-clan/module.nix index cf2a34e54..072b16c98 100644 --- a/lib/build-clan/module.nix +++ b/lib/build-clan/module.nix @@ -107,7 +107,8 @@ let # We removed pkgsForSystems because we have the problem that nixpkgs.* options are then ignored # However our current model is to have the hardware config read and then set nixpkgs.hostPlatform automatically - # which gets ignored if we set pkgsForSystems + # which gets ignored if we set pkgsForSystems. pkgsForSystems also needed to be set as else + # pkgs was equals to null in nixosConfiguration above which broke something else pkgs = nixpkgs.legacyPackages.${system}; } ) allMachines From 9da872a26716ebb6599302ec4a9855a69853c7aa Mon Sep 17 00:00:00 2001 From: Qubasa Date: Mon, 11 Nov 2024 20:54:48 +0700 Subject: [PATCH 4/4] clan-cli: Simplify hostPlatform and just use 'system' --- .../clan_cli/machines/host_platform.py | 47 ------------------- pkgs/clan-cli/clan_cli/machines/machines.py | 12 ++--- 2 files changed, 4 insertions(+), 55 deletions(-) delete mode 100644 pkgs/clan-cli/clan_cli/machines/host_platform.py diff --git a/pkgs/clan-cli/clan_cli/machines/host_platform.py b/pkgs/clan-cli/clan_cli/machines/host_platform.py deleted file mode 100644 index bdc8258d4..000000000 --- a/pkgs/clan-cli/clan_cli/machines/host_platform.py +++ /dev/null @@ -1,47 +0,0 @@ -from dataclasses import dataclass, fields -from typing import Any - - -@dataclass -class HostPlatform: - config: str - darwinArch: str - darwinMinVersion: str - darwinSdkVersion: str - is32bit: bool - is64bit: bool - isx86_64: bool - isAarch: bool - isDarwin: bool - isFreeBSD: bool - isLinux: bool - isMacOS: bool - isWindows: bool - isAndroid: bool - linuxArch: str - qemuArch: str - system: str - ubootArch: str - - # ruff: noqa: N815 - - @staticmethod - def from_dict(data: dict[str, Any]) -> "HostPlatform": - """ - Factory method that creates an instance of HostPlatform from a dictionary. - Extra fields in the dictionary are ignored. - - Args: - data (dict): A dictionary containing values for initializing HostPlatform. - - Returns: - HostPlatform: An instance of the HostPlatform class. - """ - # Dynamically extract field names from the dataclass - valid_keys = {field.name for field in fields(HostPlatform)} - - # Filter the dictionary to only include items with keys that are valid - filtered_data = {k: v for k, v in data.items() if k in valid_keys} - - # Pass the filtered data to the HostPlatform constructor - return HostPlatform(**filtered_data) diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 6e836b96b..b02599eac 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -12,7 +12,6 @@ from clan_cli.cmd import run_no_stdout from clan_cli.errors import ClanError from clan_cli.facts import public_modules as facts_public_modules from clan_cli.facts import secret_modules as facts_secret_modules -from clan_cli.machines import host_platform from clan_cli.nix import nix_build, nix_config, nix_eval, nix_metadata from clan_cli.ssh import Host, HostKeyCheck, parse_deployment_address from clan_cli.vars.public_modules import FactStoreBase @@ -48,16 +47,13 @@ class Machine: return str(self) @property - def host_platform(self) -> host_platform.HostPlatform: + def system(self) -> str: # We filter out function attributes because they are not serializable. attr = f""" (let machine = ((builtins.getFlake "{self.flake}").nixosConfigurations.{self.name}); - lib = machine.lib; - removeFunctionAttrs = attrset: - lib.filterAttrs (name: value: lib.isFunction value == false && name != "parsed") attrset; in - {{ x = removeFunctionAttrs machine.pkgs.stdenv.hostPlatform; }}).x + {{ x = machine.pkgs.stdenv.hostPlatform.system; }}).x """ if attr in self._eval_cache: output = self._eval_cache[attr] @@ -67,14 +63,14 @@ class Machine: ).stdout.strip() self._eval_cache[attr] = output value = json.loads(output) - return host_platform.HostPlatform.from_dict(value) + return value @property def can_build_locally(self) -> bool: # TODO: We could also use the function pkgs.stdenv.hostPlatform.canExecute # but this is good enough for now. output = nix_config() - return self.host_platform.system == output["system"] + return self.system == output["system"] @property def deployment(self) -> dict: