diff --git a/checks/flake-module.nix b/checks/flake-module.nix index 5e47cdccd..0af739b25 100644 --- a/checks/flake-module.nix +++ b/checks/flake-module.nix @@ -21,6 +21,7 @@ in pkgs, lib, self', + system, ... }: { @@ -83,7 +84,10 @@ in schema = (self.clanLib.inventory.evalClanService { modules = [ m ]; - key = "checks"; + prefix = [ + "checks" + system + ]; }).config.result.api.schema; in schema diff --git a/clanServices/hello-world/default.nix b/clanServices/hello-world/default.nix index 7a835bede..86bfbdb1e 100644 --- a/clanServices/hello-world/default.nix +++ b/clanServices/hello-world/default.nix @@ -4,7 +4,15 @@ _class = "clan.service"; manifest.name = "clan-core/hello-word"; - roles.peer = { }; + roles.peer = { + interface = + { lib, ... }: + { + options.foo = lib.mkOption { + type = lib.types.str; + }; + }; + }; perMachine = { machine, ... }: diff --git a/clanServices/hello-world/flake-module.nix b/clanServices/hello-world/flake-module.nix index 9a1c14a27..0c85e0630 100644 --- a/clanServices/hello-world/flake-module.nix +++ b/clanServices/hello-world/flake-module.nix @@ -50,6 +50,7 @@ in hello-service = import ./tests/vm/default.nix { inherit module; inherit self inputs pkgs; + # clanLib is exposed from inputs.clan-core clanLib = self.clanLib; }; }; diff --git a/lib/build-clan/interface.nix b/lib/build-clan/interface.nix index b90ee8830..5b48a6f48 100644 --- a/lib/build-clan/interface.nix +++ b/lib/build-clan/interface.nix @@ -10,6 +10,11 @@ let in { options = { + _prefix = lib.mkOption { + type = types.listOf types.str; + internal = true; + default = [ ]; + }; self = lib.mkOption { type = types.raw; default = self; @@ -168,6 +173,7 @@ in inventoryFile = lib.mkOption { type = lib.types.raw; }; # The machine 'imports' generated by the inventory per machine inventoryClass = lib.mkOption { type = lib.types.raw; }; + evalServiceSchema = lib.mkOption { }; # clan-core's modules clanModules = lib.mkOption { type = lib.types.raw; }; source = lib.mkOption { type = lib.types.raw; }; diff --git a/lib/build-clan/module.nix b/lib/build-clan/module.nix index 73ce2b2eb..301b57d85 100644 --- a/lib/build-clan/module.nix +++ b/lib/build-clan/module.nix @@ -44,6 +44,7 @@ let buildInventory { inherit inventory directory; flakeInputs = config.self.inputs; + prefix = config._prefix ++ [ "inventoryClass" ]; } ); @@ -204,6 +205,9 @@ in inherit inventoryClass; + # Endpoint that can be called to get a service schema + evalServiceSchema = clan-core.clanLib.evalServiceSchema config.self; + # TODO: unify this interface # We should have only clan.modules. (consistent with clan.templates) inherit (clan-core) clanModules clanLib; diff --git a/lib/default.nix b/lib/default.nix index 724fb3d9a..7ea118d51 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -15,10 +15,27 @@ lib.fix (clanLib: { */ callLib = file: args: import file ({ inherit lib clanLib; } // args); + # ------------------------------------ buildClan = clanLib.buildClanModule.buildClanWith { clan-core = self; inherit nixpkgs nix-darwin; }; + evalServiceSchema = + self: + { + moduleSpec, + flakeInputs ? self.inputs, + localModuleSet ? self.clan.modules, + }: + let + resolvedModule = clanLib.inventory.resolveModule { + inherit moduleSpec flakeInputs localModuleSet; + }; + in + (clanLib.inventory.evalClanService { + modules = [ resolvedModule ]; + prefix = [ ]; + }).config.result.api.schema; # ------------------------------------ # ClanLib functions evalClan = clanLib.callLib ./inventory/eval-clan-modules { }; diff --git a/lib/inventory/build-inventory/default.nix b/lib/inventory/build-inventory/default.nix index b5aee39ff..33de3902f 100644 --- a/lib/inventory/build-inventory/default.nix +++ b/lib/inventory/build-inventory/default.nix @@ -12,8 +12,10 @@ let inventory, directory, flakeInputs, + prefix ? [ ], }: (lib.evalModules { + # TODO: remove clanLib from specialArgs specialArgs = { inherit clanLib; }; @@ -24,13 +26,16 @@ let # config.distributedServices.allMachines.${name} or [ ]; { config, ... }: { + distributedServices = clanLib.inventory.mapInstances { inherit (config) inventory; inherit flakeInputs; + prefix = prefix ++ [ "distributedServices" ]; }; machines = lib.mapAttrs (_machineName: v: { machineImports = v; }) config.distributedServices.allMachines; + } ) (lib.modules.importApply ./inventory-introspection.nix { inherit clanLib; }) diff --git a/lib/inventory/default.nix b/lib/inventory/default.nix index d53a20e06..9b3112f37 100644 --- a/lib/inventory/default.nix +++ b/lib/inventory/default.nix @@ -3,7 +3,7 @@ let services = clanLib.callLib ./distributed-service/inventory-adapter.nix { }; in { - inherit (services) evalClanService mapInstances; + inherit (services) evalClanService mapInstances resolveModule; inherit (import ./build-inventory { inherit lib clanLib; }) buildInventory; interface = ./build-inventory/interface.nix; # Returns the list of machine names diff --git a/lib/inventory/distributed-service/api-feature.nix b/lib/inventory/distributed-service/api-feature.nix index 1266bb68a..f745251f0 100644 --- a/lib/inventory/distributed-service/api-feature.nix +++ b/lib/inventory/distributed-service/api-feature.nix @@ -1,7 +1,7 @@ # This module enables itself if # manifest.features.API = true # It converts the roles.interface to a json-schema -{ clanLib, attrName }: +{ clanLib, prefix }: let converter = clanLib.jsonschema { includeDefaults = true; @@ -45,7 +45,7 @@ in To see the evaluation problem run - nix eval .#clanInternals.inventoryClass.distributedServices.importedModulesEvaluated.${attrName}.config.result.api.schema.${roleName} + nix eval .#${lib.concatStringsSep "." prefix}.config.result.api.schema.${roleName} ''; assertion = (builtins.tryEval (lib.deepSeq config.result.api.schema.${roleName} true)).success; }; diff --git a/lib/inventory/distributed-service/inventory-adapter.nix b/lib/inventory/distributed-service/inventory-adapter.nix index e899564c8..65347918e 100644 --- a/lib/inventory/distributed-service/inventory-adapter.nix +++ b/lib/inventory/distributed-service/inventory-adapter.nix @@ -16,27 +16,72 @@ }: let evalClanService = - { modules, key }: + { modules, prefix }: (lib.evalModules { class = "clan.service"; modules = [ ./service-module.nix # feature modules (lib.modules.importApply ./api-feature.nix { - inherit clanLib; - attrName = key; + inherit clanLib prefix; }) ] ++ modules; }); + + resolveModule = + { + moduleSpec, + flakeInputs, + localModuleSet, + }: + let + # TODO: + resolvedModuleSet = + # If the module.name is self then take the modules defined in the flake + # Otherwise its an external input which provides the modules via 'clan.modules' attribute + if moduleSpec.input == null then + localModuleSet + else + let + input = + flakeInputs.${moduleSpec.input} or (throw '' + Flake doesn't provide input with name '${moduleSpec.input}' + + Choose one of the following inputs: + - ${ + builtins.concatStringsSep "\n- " ( + lib.attrNames (lib.filterAttrs (_name: input: input ? clan) flakeInputs) + ) + } + + To import a local module from 'inventory.modules' remove the 'input' attribute from the module definition + Remove the following line from the module definition: + + ... + - module.input = "${moduleSpec.input}" + + ''); + clanAttrs = + input.clan + or (throw "It seems the flake input ${moduleSpec.input} doesn't export any clan resources"); + in + clanAttrs.modules; + + resolvedModule = + resolvedModuleSet.${moduleSpec.name} + or (throw "flake doesn't provide clan-module with name ${moduleSpec.name}"); + in + resolvedModule; in { - inherit evalClanService; + inherit evalClanService resolveModule; mapInstances = { # This is used to resolve the module imports from 'flake.inputs' flakeInputs, # The clan inventory inventory, + prefix ? [ ], }: let # machineHasTag = machineName: tagName: lib.elem tagName inventory.machines.${machineName}.tags; @@ -45,42 +90,11 @@ in importedModuleWithInstances = lib.mapAttrs ( instanceName: instance: let - # TODO: - resolvedModuleSet = - # If the module.name is self then take the modules defined in the flake - # Otherwise its an external input which provides the modules via 'clan.modules' attribute - if instance.module.input == null then - inventory.modules - else - let - input = - flakeInputs.${instance.module.input} or (throw '' - Flake doesn't provide input with name '${instance.module.input}' - - Choose one of the following inputs: - - ${ - builtins.concatStringsSep "\n- " ( - lib.attrNames (lib.filterAttrs (_name: input: input ? clan) flakeInputs) - ) - } - - To import a local module from 'inventory.modules' remove the 'input' attribute from the module definition - Remove the following line from the module definition: - - ... - - module.input = "${instance.module.input}" - - - ''); - clanAttrs = - input.clan - or (throw "It seems the flake input ${instance.module.input} doesn't export any clan resources"); - in - clanAttrs.modules; - - resolvedModule = - resolvedModuleSet.${instance.module.name} - or (throw "flake doesn't provide clan-module with name ${instance.module.name}"); + resolvedModule = resolveModule { + moduleSpec = instance.module; + localModuleSet = inventory.modules; + inherit flakeInputs; + }; # Every instance includes machines via roles # :: { client :: ... } @@ -138,7 +152,7 @@ in importedModulesEvaluated = lib.mapAttrs ( module_ident: instances: evalClanService { - key = module_ident; + prefix = prefix ++ [ module_ident ]; modules = [ # Import the resolved module. diff --git a/lib/test/default.nix b/lib/test/default.nix index e60cf7cf5..90deddc70 100644 --- a/lib/test/default.nix +++ b/lib/test/default.nix @@ -22,6 +22,9 @@ in pkgs, self, useContainers ? true, + # Displayed for better error messages, otherwise the placeholder + system ? "", + attrName ? "", ... }: let @@ -60,6 +63,15 @@ in }; modules = [ clanLib.buildClanModule.flakePartsModule + { + _prefix = [ + "checks" + system + attrName + "config" + "clan" + ]; + } ]; }; }; diff --git a/pkgs/clan-cli/clan_cli/flake.py b/pkgs/clan-cli/clan_cli/flake.py index 1ce708bb5..dbd239b04 100644 --- a/pkgs/clan-cli/clan_cli/flake.py +++ b/pkgs/clan-cli/clan_cli/flake.py @@ -14,6 +14,7 @@ from clan_cli.nix import ( nix_build, nix_command, nix_config, + nix_eval, nix_metadata, nix_test_store, ) @@ -619,11 +620,9 @@ class Flake: except Exception as e: log.warning(f"Failed load eval cache: {e}. Continue without cache") - def invalidate_cache(self) -> None: + def prefetch(self) -> None: """ - Invalidate the cache and reload it. - - This method is used to refresh the cache by reloading it from the flake. + Loads the flake into the store and populates self.store_path and self.hash such that the flake can evaluate locally and offline """ cmd = [ "flake", @@ -642,6 +641,15 @@ class Flake: flake_metadata = json.loads(flake_prefetch.stdout) self.store_path = flake_metadata["storePath"] self.hash = flake_metadata["hash"] + self.flake_metadata = flake_metadata + + def invalidate_cache(self) -> None: + """ + Invalidate the cache and reload it. + + This method is used to refresh the cache by reloading it from the flake. + """ + self.prefetch() self._cache = FlakeCache() assert self.hash is not None @@ -651,17 +659,17 @@ class Flake: ) self.load_cache() - if "original" not in flake_metadata: - flake_metadata = nix_metadata(self.identifier) + if "original" not in self.flake_metadata: + self.flake_metadata = nix_metadata(self.identifier) - if flake_metadata["original"].get("url", "").startswith("file:"): + if self.flake_metadata["original"].get("url", "").startswith("file:"): self._is_local = True - path = flake_metadata["original"]["url"].removeprefix("file://") + path = self.flake_metadata["original"]["url"].removeprefix("file://") path = path.removeprefix("file:") self._path = Path(path) - elif flake_metadata["original"].get("path"): + elif self.flake_metadata["original"].get("path"): self._is_local = True - self._path = Path(flake_metadata["original"]["path"]) + self._path = Path(self.flake_metadata["original"]["path"]) else: self._is_local = False assert self.store_path is not None @@ -755,6 +763,56 @@ class Flake: if self.flake_cache_path: self._cache.save_to_file(self.flake_cache_path) + def uncached_nix_eval_with_args( + self, + attr_path: str, + f_args: dict[str, str], + nix_options: list[str] | None = None, + ) -> str: + """ + Calls a nix function with the provided arguments 'f_args' + The argument must be an attribute set. + + Args: + attr_path (str): The attribute path to the nix function + f_args (dict[str, nix_expr]): A python dictionary mapping from the name of the argument to a raw nix expression. + + Example + + flake.uncached_nix_eval_with_args( + "clanInternals.evalServiceSchema", + { "moduleSpec": "{ name = \"hello-world\"; input = null; }" } + ) + > '{ ...JSONSchema... }' + + """ + # Always prefetch, so we don't get any stale information + self.prefetch() + + if nix_options is None: + nix_options = [] + + arg_expr = "{" + for arg_name, arg_value in f_args.items(): + arg_expr += f" {arg_name} = {arg_value}; " + arg_expr += "}" + + nix_code = f""" + let + flake = builtins.getFlake "path:{self.store_path}?narHash={self.hash}"; + in + flake.{attr_path} {arg_expr} + """ + if tmp_store := nix_test_store(): + nix_options += ["--store", str(tmp_store)] + nix_options.append("--impure") + + output = run( + nix_eval(["--expr", nix_code, *nix_options]), RunOpts(log=Log.NONE) + ).stdout.strip() + + return output + def precache( self, selectors: list[str],