Merge pull request 'API(cli): add method to Flake class to allow calling nix functions' (#3502) from hsjobeki/clan-core:improvements-2 into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3502

checks/flake-module.nix

@@ -21,6 +21,7 @@ in
       pkgs,
       lib,
       self',
+      system,
       ...
     }:
     {
@@ -83,7 +84,10 @@ in
                   schema =
                     (self.clanLib.inventory.evalClanService {
                       modules = [ m ];
-                      key = "checks";
+                      prefix = [
+                        "checks"
+                        system
+                      ];
                     }).config.result.api.schema;
                 in
                 schema

clanServices/hello-world/default.nix

@@ -4,7 +4,15 @@
   _class = "clan.service";
   manifest.name = "clan-core/hello-word";
 
-  roles.peer = { };
+  roles.peer = {
+    interface =
+      { lib, ... }:
+      {
+        options.foo = lib.mkOption {
+          type = lib.types.str;
+        };
+      };
+  };
 
   perMachine =
     { machine, ... }:

clanServices/hello-world/flake-module.nix

@@ -50,6 +50,7 @@ in
           hello-service = import ./tests/vm/default.nix {
             inherit module;
             inherit self inputs pkgs;
+            # clanLib is exposed from inputs.clan-core
             clanLib = self.clanLib;
           };
         };

lib/build-clan/interface.nix

@@ -10,6 +10,11 @@ let
 in
 {
   options = {
+    _prefix = lib.mkOption {
+      type = types.listOf types.str;
+      internal = true;
+      default = [ ];
+    };
     self = lib.mkOption {
       type = types.raw;
       default = self;
@@ -168,6 +173,7 @@ in
           inventoryFile = lib.mkOption { type = lib.types.raw; };
           # The machine 'imports' generated by the inventory per machine
           inventoryClass = lib.mkOption { type = lib.types.raw; };
+          evalServiceSchema = lib.mkOption { };
           # clan-core's modules
           clanModules = lib.mkOption { type = lib.types.raw; };
           source = lib.mkOption { type = lib.types.raw; };

lib/build-clan/module.nix

@@ -44,6 +44,7 @@ let
     buildInventory {
       inherit inventory directory;
       flakeInputs = config.self.inputs;
+      prefix = config._prefix ++ [ "inventoryClass" ];
     }
   );
 
@@ -204,6 +205,9 @@ in
 
     inherit inventoryClass;
 
+    # Endpoint that can be called to get a service schema
+    evalServiceSchema = clan-core.clanLib.evalServiceSchema config.self;
+
     # TODO: unify this interface
     # We should have only clan.modules. (consistent with clan.templates)
     inherit (clan-core) clanModules clanLib;

lib/default.nix

@@ -15,10 +15,27 @@ lib.fix (clanLib: {
   */
   callLib = file: args: import file ({ inherit lib clanLib; } // args);
 
+  # ------------------------------------
   buildClan = clanLib.buildClanModule.buildClanWith {
     clan-core = self;
     inherit nixpkgs nix-darwin;
   };
+  evalServiceSchema =
+    self:
+    {
+      moduleSpec,
+      flakeInputs ? self.inputs,
+      localModuleSet ? self.clan.modules,
+    }:
+    let
+      resolvedModule = clanLib.inventory.resolveModule {
+        inherit moduleSpec flakeInputs localModuleSet;
+      };
+    in
+    (clanLib.inventory.evalClanService {
+      modules = [ resolvedModule ];
+      prefix = [ ];
+    }).config.result.api.schema;
   # ------------------------------------
   # ClanLib functions
   evalClan = clanLib.callLib ./inventory/eval-clan-modules { };

lib/inventory/build-inventory/default.nix

@@ -12,8 +12,10 @@ let
       inventory,
       directory,
       flakeInputs,
+      prefix ? [ ],
     }:
     (lib.evalModules {
+      # TODO: remove clanLib from specialArgs
       specialArgs = {
         inherit clanLib;
       };
@@ -24,13 +26,16 @@ let
           # config.distributedServices.allMachines.${name} or [ ];
           { config, ... }:
           {
+
             distributedServices = clanLib.inventory.mapInstances {
               inherit (config) inventory;
               inherit flakeInputs;
+              prefix = prefix ++ [ "distributedServices" ];
             };
             machines = lib.mapAttrs (_machineName: v: {
               machineImports = v;
             }) config.distributedServices.allMachines;
+
           }
         )
         (lib.modules.importApply ./inventory-introspection.nix { inherit clanLib; })

lib/inventory/default.nix

@@ -3,7 +3,7 @@ let
   services = clanLib.callLib ./distributed-service/inventory-adapter.nix { };
 in
 {
-  inherit (services) evalClanService mapInstances;
+  inherit (services) evalClanService mapInstances resolveModule;
   inherit (import ./build-inventory { inherit lib clanLib; }) buildInventory;
   interface = ./build-inventory/interface.nix;
   # Returns the list of machine names

lib/inventory/distributed-service/api-feature.nix

@@ -1,7 +1,7 @@
 # This module enables itself if
 # manifest.features.API = true
 # It converts the roles.interface to a json-schema
-{ clanLib, attrName }:
+{ clanLib, prefix }:
 let
   converter = clanLib.jsonschema {
     includeDefaults = true;
@@ -45,7 +45,7 @@ in
 
           To see the evaluation problem run
 
-          nix eval .#clanInternals.inventoryClass.distributedServices.importedModulesEvaluated.${attrName}.config.result.api.schema.${roleName}
+          nix eval .#${lib.concatStringsSep "." prefix}.config.result.api.schema.${roleName}
         '';
         assertion = (builtins.tryEval (lib.deepSeq config.result.api.schema.${roleName} true)).success;
       };

lib/inventory/distributed-service/inventory-adapter.nix

@@ -16,27 +16,72 @@
 }:
 let
   evalClanService =
-    { modules, key }:
+    { modules, prefix }:
     (lib.evalModules {
       class = "clan.service";
       modules = [
         ./service-module.nix
         # feature modules
         (lib.modules.importApply ./api-feature.nix {
-          inherit clanLib;
+          inherit clanLib prefix;
-          attrName = key;
         })
       ] ++ modules;
     });
+
+  resolveModule =
+    {
+      moduleSpec,
+      flakeInputs,
+      localModuleSet,
+    }:
+    let
+      # TODO:
+      resolvedModuleSet =
+        # If the module.name is self then take the modules defined in the flake
+        # Otherwise its an external input which provides the modules via 'clan.modules' attribute
+        if moduleSpec.input == null then
+          localModuleSet
+        else
+          let
+            input =
+              flakeInputs.${moduleSpec.input} or (throw ''
+                Flake doesn't provide input with name '${moduleSpec.input}'
+
+                Choose one of the following inputs:
+                - ${
+                  builtins.concatStringsSep "\n- " (
+                    lib.attrNames (lib.filterAttrs (_name: input: input ? clan) flakeInputs)
+                  )
+                }
+
+                To import a local module from 'inventory.modules' remove the 'input' attribute from the module definition
+                Remove the following line from the module definition:
+
+                ...
+                - module.input = "${moduleSpec.input}"
+
+              '');
+            clanAttrs =
+              input.clan
+                or (throw "It seems the flake input ${moduleSpec.input} doesn't export any clan resources");
+          in
+          clanAttrs.modules;
+
+      resolvedModule =
+        resolvedModuleSet.${moduleSpec.name}
+          or (throw "flake doesn't provide clan-module with name ${moduleSpec.name}");
+    in
+    resolvedModule;
 in
 {
-  inherit evalClanService;
+  inherit evalClanService resolveModule;
   mapInstances =
     {
       # This is used to resolve the module imports from 'flake.inputs'
       flakeInputs,
       # The clan inventory
       inventory,
+      prefix ? [ ],
     }:
     let
       # machineHasTag = machineName: tagName: lib.elem tagName inventory.machines.${machineName}.tags;
@@ -45,42 +90,11 @@ in
       importedModuleWithInstances = lib.mapAttrs (
         instanceName: instance:
         let
-          # TODO:
+          resolvedModule = resolveModule {
-          resolvedModuleSet =
+            moduleSpec = instance.module;
-            # If the module.name is self then take the modules defined in the flake
+            localModuleSet = inventory.modules;
-            # Otherwise its an external input which provides the modules via 'clan.modules' attribute
+            inherit flakeInputs;
-            if instance.module.input == null then
+          };
-              inventory.modules
-            else
-              let
-                input =
-                  flakeInputs.${instance.module.input} or (throw ''
-                    Flake doesn't provide input with name '${instance.module.input}'
-
-                    Choose one of the following inputs:
-                    - ${
-                      builtins.concatStringsSep "\n- " (
-                        lib.attrNames (lib.filterAttrs (_name: input: input ? clan) flakeInputs)
-                      )
-                    }
-
-                    To import a local module from 'inventory.modules' remove the 'input' attribute from the module definition
-                    Remove the following line from the module definition:
-
-                    ...
-                    - module.input = "${instance.module.input}"
-
-
-                  '');
-                clanAttrs =
-                  input.clan
-                    or (throw "It seems the flake input ${instance.module.input} doesn't export any clan resources");
-              in
-              clanAttrs.modules;
-
-          resolvedModule =
-            resolvedModuleSet.${instance.module.name}
-              or (throw "flake doesn't provide clan-module with name ${instance.module.name}");
 
           # Every instance includes machines via roles
           # :: { client :: ... }
@@ -138,7 +152,7 @@ in
       importedModulesEvaluated = lib.mapAttrs (
         module_ident: instances:
         evalClanService {
-          key = module_ident;
+          prefix = prefix ++ [ module_ident ];
           modules =
             [
               # Import the resolved module.

lib/test/default.nix

@@ -22,6 +22,9 @@ in
       pkgs,
       self,
       useContainers ? true,
+      # Displayed for better error messages, otherwise the placeholder
+      system ? "<system>",
+      attrName ? "<check_name>",
       ...
     }:
     let
@@ -60,6 +63,15 @@ in
               };
               modules = [
                 clanLib.buildClanModule.flakePartsModule
+                {
+                  _prefix = [
+                    "checks"
+                    system
+                    attrName
+                    "config"
+                    "clan"
+                  ];
+                }
               ];
             };
           };

pkgs/clan-cli/clan_cli/flake.py

@@ -14,6 +14,7 @@ from clan_cli.nix import (
     nix_build,
     nix_command,
     nix_config,
+    nix_eval,
     nix_metadata,
     nix_test_store,
 )
@@ -619,11 +620,9 @@ class Flake:
         except Exception as e:
             log.warning(f"Failed load eval cache: {e}. Continue without cache")
 
-    def invalidate_cache(self) -> None:
+    def prefetch(self) -> None:
         """
-        Invalidate the cache and reload it.
+        Loads the flake into the store and populates self.store_path and self.hash such that the flake can evaluate locally and offline
-
-        This method is used to refresh the cache by reloading it from the flake.
         """
         cmd = [
             "flake",
@@ -642,6 +641,15 @@ class Flake:
         flake_metadata = json.loads(flake_prefetch.stdout)
         self.store_path = flake_metadata["storePath"]
         self.hash = flake_metadata["hash"]
+        self.flake_metadata = flake_metadata
+
+    def invalidate_cache(self) -> None:
+        """
+        Invalidate the cache and reload it.
+
+        This method is used to refresh the cache by reloading it from the flake.
+        """
+        self.prefetch()
 
         self._cache = FlakeCache()
         assert self.hash is not None
@@ -651,17 +659,17 @@ class Flake:
         )
         self.load_cache()
 
-        if "original" not in flake_metadata:
+        if "original" not in self.flake_metadata:
-            flake_metadata = nix_metadata(self.identifier)
+            self.flake_metadata = nix_metadata(self.identifier)
 
-        if flake_metadata["original"].get("url", "").startswith("file:"):
+        if self.flake_metadata["original"].get("url", "").startswith("file:"):
             self._is_local = True
-            path = flake_metadata["original"]["url"].removeprefix("file://")
+            path = self.flake_metadata["original"]["url"].removeprefix("file://")
             path = path.removeprefix("file:")
             self._path = Path(path)
-        elif flake_metadata["original"].get("path"):
+        elif self.flake_metadata["original"].get("path"):
             self._is_local = True
-            self._path = Path(flake_metadata["original"]["path"])
+            self._path = Path(self.flake_metadata["original"]["path"])
         else:
             self._is_local = False
             assert self.store_path is not None
@@ -755,6 +763,56 @@ class Flake:
         if self.flake_cache_path:
             self._cache.save_to_file(self.flake_cache_path)
 
+    def uncached_nix_eval_with_args(
+        self,
+        attr_path: str,
+        f_args: dict[str, str],
+        nix_options: list[str] | None = None,
+    ) -> str:
+        """
+        Calls a nix function with the provided arguments 'f_args'
+        The argument must be an attribute set.
+
+         Args:
+            attr_path (str): The attribute path to the nix function
+            f_args (dict[str, nix_expr]): A python dictionary mapping from the name of the argument to a raw nix expression.
+
+        Example
+
+        flake.uncached_nix_eval_with_args(
+            "clanInternals.evalServiceSchema",
+            { "moduleSpec": "{ name = \"hello-world\"; input = null; }" }
+        )
+        > '{ ...JSONSchema... }'
+
+        """
+        # Always prefetch, so we don't get any stale information
+        self.prefetch()
+
+        if nix_options is None:
+            nix_options = []
+
+        arg_expr = "{"
+        for arg_name, arg_value in f_args.items():
+            arg_expr += f"  {arg_name} = {arg_value}; "
+        arg_expr += "}"
+
+        nix_code = f"""
+            let
+              flake = builtins.getFlake "path:{self.store_path}?narHash={self.hash}";
+            in
+              flake.{attr_path} {arg_expr}
+        """
+        if tmp_store := nix_test_store():
+            nix_options += ["--store", str(tmp_store)]
+            nix_options.append("--impure")
+
+        output = run(
+            nix_eval(["--expr", nix_code, *nix_options]), RunOpts(log=Log.NONE)
+        ).stdout.strip()
+
+        return output
+
     def precache(
         self,
         selectors: list[str],