From 2ab53b05fe3413332274dd1c1535de11356d96e1 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 2 Apr 2025 10:32:58 +0200 Subject: [PATCH] chore(lib/treewide): cleanup directory struture of lib. See lib/readme.md for details --- lib/README.md | 8 +-- lib/build-clan/interface.nix | 2 +- lib/build-clan/module.nix | 6 +- lib/default.nix | 11 ++-- lib/flake-module.nix | 18 ++---- lib/{values => introspection}/default.nix | 0 .../flake-module.nix | 0 lib/{values => introspection}/test.nix | 0 lib/{ => inventory}/constraints/default.nix | 0 lib/{ => inventory}/constraints/interface.nix | 0 lib/inventory/default.nix | 37 ++++++++++++ .../distributed-service/flake-module.nix | 3 +- .../distributed-service/inventory-adapter.nix | 59 ++++--------------- .../distributed-service/service-module.nix | 0 .../distributed-service/tests/default.nix | 19 +++--- .../tests/per_instance_args.nix | 12 ++-- .../tests/per_machine_args.nix | 8 +-- .../eval-clan-modules/default.nix | 0 lib/inventory/flake-module.nix | 3 + lib/{ => inventory}/frontmatter/default.nix | 0 lib/{ => inventory}/frontmatter/interface.nix | 0 21 files changed, 92 insertions(+), 94 deletions(-) rename lib/{values => introspection}/default.nix (100%) rename lib/{values => introspection}/flake-module.nix (100%) rename lib/{values => introspection}/test.nix (100%) rename lib/{ => inventory}/constraints/default.nix (100%) rename lib/{ => inventory}/constraints/interface.nix (100%) rename lib/{ => inventory}/distributed-service/flake-module.nix (94%) rename lib/{ => inventory}/distributed-service/inventory-adapter.nix (80%) rename lib/{ => inventory}/distributed-service/service-module.nix (100%) rename lib/{ => inventory}/distributed-service/tests/default.nix (92%) rename lib/{ => inventory}/distributed-service/tests/per_instance_args.nix (81%) rename lib/{ => inventory}/distributed-service/tests/per_machine_args.nix (79%) rename lib/{ => inventory}/eval-clan-modules/default.nix (100%) rename lib/{ => inventory}/frontmatter/default.nix (100%) rename lib/{ => inventory}/frontmatter/interface.nix (100%) diff --git a/lib/README.md b/lib/README.md index bbe24ed65..7ab34b5b1 100644 --- a/lib/README.md +++ b/lib/README.md @@ -45,15 +45,15 @@ Example filetree . ├── default.nix ├── build-clan -│ ├── impl.nix +│ ├── default.nix │ └── test.nix └── inventory - ├── impl.nix + ├── default.nix ├── services-subfeature - │ ├── impl.nix + │ ├── default.nix │ └── test.nix ├── instances-subfeature # <- We immediately see that this feature is not tested on itself. - │ └── impl.nix + │ └── default.nix └── test.nix ``` diff --git a/lib/build-clan/interface.nix b/lib/build-clan/interface.nix index 010d86819..12a0690c3 100644 --- a/lib/build-clan/interface.nix +++ b/lib/build-clan/interface.nix @@ -79,7 +79,7 @@ in }; inventory = lib.mkOption { - type = types.submodule { imports = [ ../inventory/build-inventory/interface.nix ]; }; + type = types.submodule { imports = [ self.clanLib.inventory.interface ]; }; description = '' The `Inventory` submodule. diff --git a/lib/build-clan/module.nix b/lib/build-clan/module.nix index 9311c53c4..e55e5ff5c 100644 --- a/lib/build-clan/module.nix +++ b/lib/build-clan/module.nix @@ -201,9 +201,9 @@ in clanInternals = { moduleSchemas = clan-core.lib.modules.getModulesSchema config.inventory.modules; inherit inventoryClass; - distributedServices = import ../distributed-service/inventory-adapter.nix { - inherit lib inventory; - flake = config.self; + distributedServices = clan-core.clanLib.inventory.mapInstances { + inherit inventory; + flakeInputs = config.self.inputs; }; # TODO: unify this interface # We should have only clan.modules. (consistent with clan.templates) diff --git a/lib/default.nix b/lib/default.nix index 53fd23021..7ec821e4b 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -13,14 +13,13 @@ lib.fix (clanLib: { # We should reduce the dependency on 'self' aka the 'flake' object # This makes it easier to test # most of the time passing the whole flake is unnecessary - callLib = file: args: import file { inherit lib clanLib; } // args; + callLib = file: args: import file ({ inherit lib clanLib; } // args); - evalClan = import ./eval-clan-modules { + evalClan = import ./inventory/eval-clan-modules { inherit lib; clan-core = self; pkgs = nixpkgs.legacyPackages.x86_64-linux; }; - buildClanModule = import ./build-clan { inherit lib nixpkgs; }; @@ -28,9 +27,9 @@ lib.fix (clanLib: { # ------------------------------------ # Lib functions that don't depend on 'self' inventory = clanLib.callLib ./inventory { }; - modules = clanLib.callLib ./frontmatter { }; - facts = import ./facts.nix { inherit lib; }; - values = import ./values { inherit lib; }; + modules = clanLib.callLib ./inventory/frontmatter { }; + values = import ./introspection { inherit lib; }; jsonschema = import ./jsonschema { inherit lib; }; select = import ./select.nix; + facts = import ./facts.nix { inherit lib; }; }) diff --git a/lib/flake-module.nix b/lib/flake-module.nix index 5d729ee68..1367e7e60 100644 --- a/lib/flake-module.nix +++ b/lib/flake-module.nix @@ -4,21 +4,13 @@ self, ... }: -let - inherit (lib) - filter - pathExists - ; -in rec { - # We should remove this. - # It would enforce treating at least 'lib' as a module in a whole - imports = filter pathExists [ - ./jsonschema/flake-module.nix - ./inventory/flake-module.nix + # TODO: automatically generate this from the directory conventions + imports = [ ./build-clan/flake-module.nix - ./values/flake-module.nix - ./distributed-service/flake-module.nix + ./inventory/flake-module.nix + ./jsonschema/flake-module.nix + ./introspection/flake-module.nix ]; flake.clanLib = import ./default.nix { inherit lib inputs self; diff --git a/lib/values/default.nix b/lib/introspection/default.nix similarity index 100% rename from lib/values/default.nix rename to lib/introspection/default.nix diff --git a/lib/values/flake-module.nix b/lib/introspection/flake-module.nix similarity index 100% rename from lib/values/flake-module.nix rename to lib/introspection/flake-module.nix diff --git a/lib/values/test.nix b/lib/introspection/test.nix similarity index 100% rename from lib/values/test.nix rename to lib/introspection/test.nix diff --git a/lib/constraints/default.nix b/lib/inventory/constraints/default.nix similarity index 100% rename from lib/constraints/default.nix rename to lib/inventory/constraints/default.nix diff --git a/lib/constraints/interface.nix b/lib/inventory/constraints/interface.nix similarity index 100% rename from lib/constraints/interface.nix rename to lib/inventory/constraints/interface.nix diff --git a/lib/inventory/default.nix b/lib/inventory/default.nix index cb6f3e11e..7be7a2c51 100644 --- a/lib/inventory/default.nix +++ b/lib/inventory/default.nix @@ -2,4 +2,41 @@ { inherit (import ./build-inventory { inherit lib clanLib; }) buildInventory; interface = ./build-inventory/interface.nix; + mapInstances = clanLib.callLib ./distributed-service/inventory-adapter.nix {}; + # Returns the list of machine names + # { ... } -> [ string ] + resolveTags = + { + # Available InventoryMachines :: { {name} :: { tags = [ string ]; }; } + machines, + # Requested members :: { machines, tags } + # Those will be resolved against the available machines + members, + # Not needed for resolution - only for error reporting + roleName, + instanceName, + }: + { + machines = + members.machines or [ ] + ++ (builtins.foldl' ( + acc: tag: + let + # For error printing + availableTags = lib.foldlAttrs ( + acc: _: v: + v.tags or [ ] ++ acc + ) [ ] (machines); + + tagMembers = builtins.attrNames (lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) machines); + in + if tagMembers == [ ] then + lib.warn '' + Service instance '${instanceName}': - ${roleName} tags: no machine with tag '${tag}' found. + Available tags: ${builtins.toJSON (lib.unique availableTags)} + '' acc + else + acc ++ tagMembers + ) [ ] members.tags or [ ]); + }; } diff --git a/lib/distributed-service/flake-module.nix b/lib/inventory/distributed-service/flake-module.nix similarity index 94% rename from lib/distributed-service/flake-module.nix rename to lib/inventory/distributed-service/flake-module.nix index 52f6c7e68..87bacd8ee 100644 --- a/lib/distributed-service/flake-module.nix +++ b/lib/inventory/distributed-service/flake-module.nix @@ -15,7 +15,8 @@ in { # Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux. legacyPackages.evalTests-distributedServices = import ./tests { - inherit lib self; + inherit lib; + clanLib = self.clanLib; }; checks = { diff --git a/lib/distributed-service/inventory-adapter.nix b/lib/inventory/distributed-service/inventory-adapter.nix similarity index 80% rename from lib/distributed-service/inventory-adapter.nix rename to lib/inventory/distributed-service/inventory-adapter.nix index d99c1bf74..4075833c8 100644 --- a/lib/distributed-service/inventory-adapter.nix +++ b/lib/inventory/distributed-service/inventory-adapter.nix @@ -11,49 +11,16 @@ # Also combines the settings for 'machines' and 'tags'. { lib, + clanLib, + ... +}: +{ # This is used to resolve the module imports from 'flake.inputs' - flake, + flakeInputs, # The clan inventory inventory, }: let - # Returns the list of machine names - # { ... } -> [ string ] - resolveTags = - { - # Available InventoryMachines :: { {name} :: { tags = [ string ]; }; } - machines, - # Requested members :: { machines, tags } - # Those will be resolved against the available machines - members, - # Not needed for resolution - only for error reporting - roleName, - instanceName, - }: - { - machines = - members.machines or [ ] - ++ (builtins.foldl' ( - acc: tag: - let - # For error printing - availableTags = lib.foldlAttrs ( - acc: _: v: - v.tags or [ ] ++ acc - ) [ ] (machines); - - tagMembers = builtins.attrNames (lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) machines); - in - if tagMembers == [ ] then - lib.warn '' - Service instance '${instanceName}': - ${roleName} tags: no machine with tag '${tag}' found. - Available tags: ${builtins.toJSON (lib.unique availableTags)} - '' acc - else - acc ++ tagMembers - ) [ ] members.tags or [ ]); - }; - # machineHasTag = machineName: tagName: lib.elem tagName inventory.machines.${machineName}.tags; # map the instances into the module @@ -69,13 +36,13 @@ let else let input = - flake.inputs.${instance.module.input} or (throw '' + flakeInputs.${instance.module.input} or (throw '' Flake doesn't provide input with name '${instance.module.input}' Choose one of the following inputs: - ${ builtins.concatStringsSep "\n- " ( - lib.attrNames (lib.filterAttrs (_name: input: input ? clan) flake.inputs) + lib.attrNames (lib.filterAttrs (_name: input: input ? clan) flakeInputs) ) } @@ -102,7 +69,7 @@ let instanceRoles = lib.mapAttrs ( roleName: role: let - resolvedMachines = resolveTags { + resolvedMachines = clanLib.inventory.resolveTags { members = { # Explicit members machines = lib.attrNames role.machines; @@ -150,7 +117,7 @@ let ) inventory.instances; # TODO: Eagerly check the _class of the resolved module - evals = lib.mapAttrs ( + importedModulesEvaluated = lib.mapAttrs ( _module_ident: instances: (lib.evalModules { class = "clan.service"; @@ -189,18 +156,18 @@ let # TODO: Return an attribute set of resources instead of a plain list of nixosModules allMachines = lib.foldlAttrs ( - acc: _name: eval: + acc: _module_ident: eval: acc // lib.mapAttrs ( machineName: result: acc.${machineName} or [ ] ++ [ result.nixosModule ] ) eval.config.result.final - ) { } evals; + ) { } importedModulesEvaluated; in { inherit importedModuleWithInstances grouped - evals + allMachines - ; + importedModulesEvaluated; } diff --git a/lib/distributed-service/service-module.nix b/lib/inventory/distributed-service/service-module.nix similarity index 100% rename from lib/distributed-service/service-module.nix rename to lib/inventory/distributed-service/service-module.nix diff --git a/lib/distributed-service/tests/default.nix b/lib/inventory/distributed-service/tests/default.nix similarity index 92% rename from lib/distributed-service/tests/default.nix rename to lib/inventory/distributed-service/tests/default.nix index 7275f8ce3..cc04b8436 100644 --- a/lib/distributed-service/tests/default.nix +++ b/lib/inventory/distributed-service/tests/default.nix @@ -1,5 +1,6 @@ { lib, + clanLib, ... }: let @@ -12,7 +13,7 @@ let (evalModules { # Static modules modules = [ - ../../inventory/build-inventory/interface.nix + clanLib.inventory.interface { modules.test = { }; } @@ -20,15 +21,13 @@ let ]; }).config; - flakeFixture = { - inputs = { }; + flakeInputsFixture = { }; callInventoryAdapter = inventoryModule: - import ../inventory-adapter.nix { - inherit lib; - flake = flakeFixture; + clanLib.inventory.mapInstances { + flakeInputs = flakeInputsFixture; inventory = evalInventory inventoryModule; }; in @@ -56,7 +55,7 @@ in { # Test that the module is mapped into the output # We might change the attribute name in the future - expr = res.evals ? "self-simple-module"; + expr = res.importedModulesEvaluated ? "self-simple-module"; expected = true; }; @@ -148,7 +147,7 @@ in { # Test that the module is mapped into the output # We might change the attribute name in the future - expr = lib.attrNames res.evals.self-A.config.instances; + expr = lib.attrNames res.importedModulesEvaluated.self-A.config.instances; expected = [ "instance_bar" "instance_foo" @@ -200,7 +199,7 @@ in { # Test that the module is mapped into the output # We might change the attribute name in the future - expr = lib.attrNames res.evals.self-A.config.result.allMachines; + expr = lib.attrNames res.importedModulesEvaluated.self-A.config.result.allMachines; expected = [ "jon" "sara" @@ -250,7 +249,7 @@ in { # Test that the module is mapped into the output # We might change the attribute name in the future - expr = lib.attrNames res.evals.self-A.config.result.allMachines; + expr = lib.attrNames res.importedModulesEvaluated.self-A.config.result.allMachines; expected = [ "jon" "sara" diff --git a/lib/distributed-service/tests/per_instance_args.nix b/lib/inventory/distributed-service/tests/per_instance_args.nix similarity index 81% rename from lib/distributed-service/tests/per_instance_args.nix rename to lib/inventory/distributed-service/tests/per_instance_args.nix index 88394e740..38ef41a6d 100644 --- a/lib/distributed-service/tests/per_instance_args.nix +++ b/lib/inventory/distributed-service/tests/per_instance_args.nix @@ -100,25 +100,25 @@ in test_per_instance_arguments = { expr = { instanceName = - res.evals.self-A.config.result.allRoles.peer.allInstances."instance_foo".allMachines.jon.nixosModule.instanceName; + res.importedModulesEvaluated.self-A.config.result.allRoles.peer.allInstances."instance_foo".allMachines.jon.nixosModule.instanceName; # settings are specific. # Below we access: # instance = instance_foo # roles = peer # machines = jon - settings = filterInternals res.evals.self-A.config.result.allRoles.peer.allInstances.instance_foo.allMachines.jon.nixosModule.settings; - machine = mapInternalsRecursive res.evals.self-A.config.result.allRoles.peer.allInstances.instance_foo.allMachines.jon.nixosModule.machine; + settings = filterInternals res.importedModulesEvaluated.self-A.config.result.allRoles.peer.allInstances.instance_foo.allMachines.jon.nixosModule.settings; + machine = mapInternalsRecursive res.importedModulesEvaluated.self-A.config.result.allRoles.peer.allInstances.instance_foo.allMachines.jon.nixosModule.machine; # hasRoleSettings = - # res.evals.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer ? settings; + # res.importedModulesEvaluated.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer ? settings; # # settings are specific. # # Below we access: # # instance = instance_foo # # roles = peer # # machines = * - # specificRoleSettings = filterInternals res.evals.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.settings; + # specificRoleSettings = filterInternals res.importedModulesEvaluated.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.settings; }; expected = { instanceName = "instance_foo"; @@ -150,7 +150,7 @@ in test_per_instance_settings_vendoring = { expr = mapInternalsRecursive - res.evals.self-A.config.result.allRoles.peer.allInstances."instance_foo".allMachines.jon.nixosModule.vendoredSettings; + res.importedModulesEvaluated.self-A.config.result.allRoles.peer.allInstances."instance_foo".allMachines.jon.nixosModule.vendoredSettings; expected = { # Returns another override __functor = "__functor"; diff --git a/lib/distributed-service/tests/per_machine_args.nix b/lib/inventory/distributed-service/tests/per_machine_args.nix similarity index 79% rename from lib/distributed-service/tests/per_machine_args.nix rename to lib/inventory/distributed-service/tests/per_machine_args.nix index 2f5868e3e..95931683e 100644 --- a/lib/distributed-service/tests/per_machine_args.nix +++ b/lib/inventory/distributed-service/tests/per_machine_args.nix @@ -73,7 +73,7 @@ in test_per_machine_receives_instance_settings = { expr = { hasMachineSettings = - res.evals.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.machines.jon + res.importedModulesEvaluated.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.machines.jon ? settings; # settings are specific. @@ -81,17 +81,17 @@ in # instance = instance_foo # roles = peer # machines = jon - specificMachineSettings = filterInternals res.evals.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.machines.jon.settings; + specificMachineSettings = filterInternals res.importedModulesEvaluated.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.machines.jon.settings; hasRoleSettings = - res.evals.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer ? settings; + res.importedModulesEvaluated.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer ? settings; # settings are specific. # Below we access: # instance = instance_foo # roles = peer # machines = * - specificRoleSettings = filterInternals res.evals.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.settings; + specificRoleSettings = filterInternals res.importedModulesEvaluated.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.settings; }; expected = { hasMachineSettings = true; diff --git a/lib/eval-clan-modules/default.nix b/lib/inventory/eval-clan-modules/default.nix similarity index 100% rename from lib/eval-clan-modules/default.nix rename to lib/inventory/eval-clan-modules/default.nix diff --git a/lib/inventory/flake-module.nix b/lib/inventory/flake-module.nix index 8cb751dd1..64a5e4f20 100644 --- a/lib/inventory/flake-module.nix +++ b/lib/inventory/flake-module.nix @@ -5,6 +5,9 @@ let ); in { + imports = [ + ./distributed-service/flake-module.nix + ]; perSystem = { pkgs, diff --git a/lib/frontmatter/default.nix b/lib/inventory/frontmatter/default.nix similarity index 100% rename from lib/frontmatter/default.nix rename to lib/inventory/frontmatter/default.nix diff --git a/lib/frontmatter/interface.nix b/lib/inventory/frontmatter/interface.nix similarity index 100% rename from lib/frontmatter/interface.nix rename to lib/inventory/frontmatter/interface.nix