Merge pull request 'lib-cleanup' (#3173) from hsjobeki/clan-core:lib-cleanup into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3173

flakeModules/clan.nix

@@ -35,6 +35,8 @@ in
       specialArgs = {
         inherit clan-core self;
         inherit (inputs) nixpkgs;
+        # TODO: inject the inventory interface
+        # inventoryInterface = {};
       };
       modules = [
         buildClanModule.flakePartsModule

lib/README.md

@@ -45,15 +45,15 @@ Example filetree
 .
 ├── default.nix
 ├── build-clan
-│   ├── impl.nix
+│   ├── default.nix
 │   └── test.nix
 └── inventory
-    ├── impl.nix
+    ├── default.nix
     ├── services-subfeature
-    │   ├── impl.nix
+    │   ├── default.nix
     │   └── test.nix
     ├── instances-subfeature # <- We immediately see that this feature is not tested on itself.
-    │   └── impl.nix
+    │   └── default.nix
     └── test.nix
 ```
 

lib/build-clan/default.nix

@@ -4,6 +4,7 @@
 {
   lib,
   nixpkgs,
+  ...
 }:
 {
   flakePartsModule = {

lib/build-clan/interface.nix

@@ -1,6 +1,8 @@
 {
   lib,
   self,
+  # TODO: Use dependency injection to allow for testing
+  # inventoryInterface,
   ...
 }:
 let
@@ -128,6 +130,7 @@ in
       visible = false;
       # ClanInternals
       type = types.submodule {
+        freeformType = types.attrsOf types.raw;
         options = {
           # Those options are interfaced by the CLI
           # We don't specify the type here, for better performance.

lib/build-clan/module.nix

@@ -201,9 +201,9 @@ in
   clanInternals = {
     moduleSchemas = clan-core.lib.modules.getModulesSchema config.inventory.modules;
     inherit inventoryClass;
-    distributedServices = import ../distributed-service/inventory-adapter.nix {
-      inherit lib inventory;
-      flake = config.self;
+    distributedServices = clan-core.clanLib.inventory.mapInstances {
+      inherit inventory;
+      flakeInputs = config.self.inputs;
     };
     # TODO: unify this interface
     # We should have only clan.modules. (consistent with clan.templates)

lib/default.nix

@@ -13,24 +13,22 @@ lib.fix (clanLib: {
   # We should reduce the dependency on 'self' aka the 'flake' object
   # This makes it easier to test
   # most of the time passing the whole flake is unnecessary
-  callLib = file: args: import file { inherit lib clanLib; } // args;
+  callLib = file: args: import file ({ inherit lib clanLib; } // args);
 
-  evalClan = import ./eval-clan-modules {
+  evalClan = import ./inventory/eval-clan-modules {
     inherit lib;
     clan-core = self;
     pkgs = nixpkgs.legacyPackages.x86_64-linux;
   };
+  buildClanModule = clanLib.callLib ./build-clan { inherit nixpkgs; };
 
-  buildClanModule = import ./build-clan {
-    inherit lib nixpkgs;
-  };
   buildClan = clanLib.buildClanModule.buildClanWith { clan-core = self; };
   # ------------------------------------
   # Lib functions that don't depend on 'self'
   inventory = clanLib.callLib ./inventory { };
-  modules = clanLib.callLib ./frontmatter { };
-  facts = import ./facts.nix { inherit lib; };
-  values = import ./values { inherit lib; };
+  modules = clanLib.callLib ./inventory/frontmatter { };
+  values = import ./introspection { inherit lib; };
   jsonschema = import ./jsonschema { inherit lib; };
   select = import ./select.nix;
+  facts = import ./facts.nix { inherit lib; };
 })

lib/flake-module.nix

@@ -4,21 +4,13 @@
   self,
   ...
 }:
-let
-  inherit (lib)
-    filter
-    pathExists
-    ;
-in
 rec {
-  # We should remove this.
-  # It would enforce treating at least 'lib' as a module in a whole
-  imports = filter pathExists [
-    ./jsonschema/flake-module.nix
-    ./inventory/flake-module.nix
+  # TODO: automatically generate this from the directory conventions
+  imports = [
     ./build-clan/flake-module.nix
-    ./values/flake-module.nix
-    ./distributed-service/flake-module.nix
+    ./inventory/flake-module.nix
+    ./jsonschema/flake-module.nix
+    ./introspection/flake-module.nix
   ];
   flake.clanLib = import ./default.nix {
     inherit lib inputs self;

lib/inventory/default.nix

@@ -2,4 +2,41 @@
 {
   inherit (import ./build-inventory { inherit lib clanLib; }) buildInventory;
   interface = ./build-inventory/interface.nix;
+  mapInstances = clanLib.callLib ./distributed-service/inventory-adapter.nix { };
+  # Returns the list of machine names
+  # { ... } -> [ string ]
+  resolveTags =
+    {
+      # Available InventoryMachines :: { {name} :: { tags = [ string ]; }; }
+      machines,
+      # Requested members :: { machines, tags }
+      # Those will be resolved against the available machines
+      members,
+      # Not needed for resolution - only for error reporting
+      roleName,
+      instanceName,
+    }:
+    {
+      machines =
+        members.machines or [ ]
+        ++ (builtins.foldl' (
+          acc: tag:
+          let
+            # For error printing
+            availableTags = lib.foldlAttrs (
+              acc: _: v:
+              v.tags or [ ] ++ acc
+            ) [ ] (machines);
+
+            tagMembers = builtins.attrNames (lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) machines);
+          in
+          if tagMembers == [ ] then
+            lib.warn ''
+              Service instance '${instanceName}': - ${roleName} tags: no machine with tag '${tag}' found.
+              Available tags: ${builtins.toJSON (lib.unique availableTags)}
+            '' acc
+          else
+            acc ++ tagMembers
+        ) [ ] members.tags or [ ]);
+    };
 }

lib/inventory/distributed-service/flake-module.nix

@@ -15,7 +15,8 @@ in
     {
       # Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.<attrName>
       legacyPackages.evalTests-distributedServices = import ./tests {
-        inherit lib self;
+        inherit lib;
+        clanLib = self.clanLib;
       };
 
       checks = {

lib/inventory/distributed-service/inventory-adapter.nix

@@ -11,49 +11,16 @@
 #   Also combines the settings for 'machines' and 'tags'.
 {
   lib,
+  clanLib,
+  ...
+}:
+{
   # This is used to resolve the module imports from 'flake.inputs'
-  flake,
+  flakeInputs,
   # The clan inventory
   inventory,
 }:
 let
-  # Returns the list of machine names
-  # { ... } -> [ string ]
-  resolveTags =
-    {
-      # Available InventoryMachines :: { {name} :: { tags = [ string ]; }; }
-      machines,
-      # Requested members :: { machines, tags }
-      # Those will be resolved against the available machines
-      members,
-      # Not needed for resolution - only for error reporting
-      roleName,
-      instanceName,
-    }:
-    {
-      machines =
-        members.machines or [ ]
-        ++ (builtins.foldl' (
-          acc: tag:
-          let
-            # For error printing
-            availableTags = lib.foldlAttrs (
-              acc: _: v:
-              v.tags or [ ] ++ acc
-            ) [ ] (machines);
-
-            tagMembers = builtins.attrNames (lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) machines);
-          in
-          if tagMembers == [ ] then
-            lib.warn ''
-              Service instance '${instanceName}': - ${roleName} tags: no machine with tag '${tag}' found.
-              Available tags: ${builtins.toJSON (lib.unique availableTags)}
-            '' acc
-          else
-            acc ++ tagMembers
-        ) [ ] members.tags or [ ]);
-    };
-
   # machineHasTag = machineName: tagName: lib.elem tagName inventory.machines.${machineName}.tags;
 
   # map the instances into the module
@@ -69,13 +36,13 @@ let
         else
           let
             input =
-              flake.inputs.${instance.module.input} or (throw ''
+              flakeInputs.${instance.module.input} or (throw ''
                 Flake doesn't provide input with name '${instance.module.input}'
 
                 Choose one of the following inputs:
                 - ${
                   builtins.concatStringsSep "\n- " (
-                    lib.attrNames (lib.filterAttrs (_name: input: input ? clan) flake.inputs)
+                    lib.attrNames (lib.filterAttrs (_name: input: input ? clan) flakeInputs)
                   )
                 }
 
@@ -102,7 +69,7 @@ let
       instanceRoles = lib.mapAttrs (
         roleName: role:
         let
-          resolvedMachines = resolveTags {
+          resolvedMachines = clanLib.inventory.resolveTags {
             members = {
               # Explicit members
               machines = lib.attrNames role.machines;
@@ -150,7 +117,7 @@ let
   ) inventory.instances;
 
   # TODO: Eagerly check the _class of the resolved module
-  evals = lib.mapAttrs (
+  importedModulesEvaluated = lib.mapAttrs (
     _module_ident: instances:
     (lib.evalModules {
       class = "clan.service";
@@ -189,18 +156,19 @@ let
 
   # TODO: Return an attribute set of resources instead of a plain list of nixosModules
   allMachines = lib.foldlAttrs (
-    acc: _name: eval:
+    acc: _module_ident: eval:
     acc
     // lib.mapAttrs (
       machineName: result: acc.${machineName} or [ ] ++ [ result.nixosModule ]
     ) eval.config.result.final
-  ) { } evals;
+  ) { } importedModulesEvaluated;
 in
 {
   inherit
     importedModuleWithInstances
     grouped
-    evals
+
     allMachines
+    importedModulesEvaluated
     ;
 }

lib/inventory/distributed-service/tests/default.nix

@@ -1,5 +1,6 @@
 {
   lib,
+  clanLib,
   ...
 }:
 let
@@ -12,7 +13,7 @@ let
     (evalModules {
       # Static modules
       modules = [
-        ../../inventory/build-inventory/interface.nix
+        clanLib.inventory.interface
         {
           modules.test = { };
         }
@@ -20,15 +21,13 @@ let
       ];
     }).config;
 
-  flakeFixture = {
-    inputs = { };
+  flakeInputsFixture = {
   };
 
   callInventoryAdapter =
     inventoryModule:
-    import ../inventory-adapter.nix {
-      inherit lib;
-      flake = flakeFixture;
+    clanLib.inventory.mapInstances {
+      flakeInputs = flakeInputsFixture;
       inventory = evalInventory inventoryModule;
     };
 in
@@ -56,7 +55,7 @@ in
     {
       # Test that the module is mapped into the output
       # We might change the attribute name in the future
-      expr = res.evals ? "self-simple-module";
+      expr = res.importedModulesEvaluated ? "self-simple-module";
       expected = true;
     };
 
@@ -148,7 +147,7 @@ in
     {
       # Test that the module is mapped into the output
       # We might change the attribute name in the future
-      expr = lib.attrNames res.evals.self-A.config.instances;
+      expr = lib.attrNames res.importedModulesEvaluated.self-A.config.instances;
       expected = [
         "instance_bar"
         "instance_foo"
@@ -200,7 +199,7 @@ in
     {
       # Test that the module is mapped into the output
       # We might change the attribute name in the future
-      expr = lib.attrNames res.evals.self-A.config.result.allMachines;
+      expr = lib.attrNames res.importedModulesEvaluated.self-A.config.result.allMachines;
       expected = [
         "jon"
         "sara"
@@ -250,7 +249,7 @@ in
     {
       # Test that the module is mapped into the output
       # We might change the attribute name in the future
-      expr = lib.attrNames res.evals.self-A.config.result.allMachines;
+      expr = lib.attrNames res.importedModulesEvaluated.self-A.config.result.allMachines;
       expected = [
         "jon"
         "sara"

lib/inventory/distributed-service/tests/per_instance_args.nix

@@ -100,25 +100,25 @@ in
   test_per_instance_arguments = {
     expr = {
       instanceName =
-        res.evals.self-A.config.result.allRoles.peer.allInstances."instance_foo".allMachines.jon.nixosModule.instanceName;
+        res.importedModulesEvaluated.self-A.config.result.allRoles.peer.allInstances."instance_foo".allMachines.jon.nixosModule.instanceName;
 
       # settings are specific.
       # Below we access:
       # instance = instance_foo
       # roles = peer
       # machines = jon
-      settings = filterInternals res.evals.self-A.config.result.allRoles.peer.allInstances.instance_foo.allMachines.jon.nixosModule.settings;
-      machine = mapInternalsRecursive res.evals.self-A.config.result.allRoles.peer.allInstances.instance_foo.allMachines.jon.nixosModule.machine;
+      settings = filterInternals res.importedModulesEvaluated.self-A.config.result.allRoles.peer.allInstances.instance_foo.allMachines.jon.nixosModule.settings;
+      machine = mapInternalsRecursive res.importedModulesEvaluated.self-A.config.result.allRoles.peer.allInstances.instance_foo.allMachines.jon.nixosModule.machine;
 
       # hasRoleSettings =
-      #   res.evals.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer ? settings;
+      #   res.importedModulesEvaluated.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer ? settings;
 
       # # settings are specific.
       # # Below we access:
       # # instance = instance_foo
       # # roles = peer
       # # machines = *
-      # specificRoleSettings = filterInternals res.evals.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.settings;
+      # specificRoleSettings = filterInternals res.importedModulesEvaluated.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.settings;
     };
     expected = {
       instanceName = "instance_foo";
@@ -150,7 +150,7 @@ in
   test_per_instance_settings_vendoring = {
     expr =
       mapInternalsRecursive
-        res.evals.self-A.config.result.allRoles.peer.allInstances."instance_foo".allMachines.jon.nixosModule.vendoredSettings;
+        res.importedModulesEvaluated.self-A.config.result.allRoles.peer.allInstances."instance_foo".allMachines.jon.nixosModule.vendoredSettings;
     expected = {
       # Returns another override
       __functor = "__functor";

lib/inventory/distributed-service/tests/per_machine_args.nix

@@ -73,7 +73,7 @@ in
   test_per_machine_receives_instance_settings = {
     expr = {
       hasMachineSettings =
-        res.evals.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.machines.jon
+        res.importedModulesEvaluated.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.machines.jon
         ? settings;
 
       # settings are specific.
@@ -81,17 +81,18 @@ in
       # instance = instance_foo
       # roles = peer
       # machines = jon
-      specificMachineSettings = filterInternals res.evals.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.machines.jon.settings;
+      specificMachineSettings = filterInternals res.importedModulesEvaluated.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.machines.jon.settings;
 
       hasRoleSettings =
-        res.evals.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer ? settings;
+        res.importedModulesEvaluated.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer
+        ? settings;
 
       # settings are specific.
       # Below we access:
       # instance = instance_foo
       # roles = peer
       # machines = *
-      specificRoleSettings = filterInternals res.evals.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.settings;
+      specificRoleSettings = filterInternals res.importedModulesEvaluated.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.settings;
     };
     expected = {
       hasMachineSettings = true;

lib/inventory/flake-module.nix

@@ -5,6 +5,9 @@ let
   );
 in
 {
+  imports = [
+    ./distributed-service/flake-module.nix
+  ];
   perSystem =
     {
       pkgs,