Merge pull request 'enable clan services for machines' (#3134) from hsjobeki/clan-core:clan-services into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3134

checks/backups/flake-module.nix

@@ -160,14 +160,10 @@
           "flake.lock"
           "flakeModules"
           "inventory.json"
-          "lib/build-clan"
-          "lib/default.nix"
-          "lib/select.nix"
-          "lib/flake-module.nix"
-          "lib/frontmatter"
-          "lib/inventory"
-          "lib/constraints"
           "nixosModules"
+          # Just include everything in 'lib'
+          # If anything changes in /lib that may affect everything
+          "lib"
         ];
       };
     in

lib/README.md

@@ -0,0 +1,72 @@
+# ClanLib
+
+This folder is supposed to contain clan specific nix functions.
+
+Such as:
+
+- build-clan function
+- select
+- build-inventory function
+- json-schema-converter
+
+## Structure
+
+Similar to `nixpkgs/lib` this produces a recursive attribute set in a fixed-point.
+Functions within lib can depend on each other to create new abstractions.
+
+### Conventions
+
+Note: This is not consistently enforced yet.
+If you start a new feature, or refactoring/touching existing ones, please help us to move towards the below illustrated.
+
+A single feature-set/module may be organized like this:
+
+```nix
+#     ↓ The final clanLib
+{lib, clanLib, ...}:
+# ↓ portion to add to clanLib
+{
+    inventory.resolveTags = tags: inventory.machines; # implementation
+    inventory.buildMachines = x: clanLib.inventory.resolveTags x; # implementation
+}
+```
+
+Every bigger feature should live in a subfolder with the feature name.
+It should contain two files:
+
+- `impl.nix`
+- `test.nix`
+- Everything else may be adopted as needed.
+
+```
+Example filetree
+```
+```sh
+.
+├── default.nix
+├── feature_foo
+│   ├── impl.nix
+│   └── test.nix
+└── feature_bar
+    ├── impl.nix
+    ├── complex-subfeature
+    │   ├── impl.nix
+    │   └── test.nix
+    ├── testless-subfeature # <- We immediately see that this feature is not tested on itself.
+    │   └── impl.nix
+    └── test.nix
+```
+
+```nix
+# default.nix
+{lib, clanLib, ...}:
+{
+    inventory.resolveTags = import ./resolveTags { inherit lib clanLib; };
+}
+```
+
+## Testing
+
+For testing we use [nix-unit](https://github.com/nix-community/nix-unit)
+
+TODO: define a helper that automatically hooks up `tests` in `flake.legacyPackages` and a corresponding buildable `checks` attribute

lib/build-clan/auto-imports.nix

@@ -1,23 +0,0 @@
-{
-  lib,
-  self,
-  ...
-}:
-
-let
-  # Returns an attrset with inputs that have the attribute `clanModules`
-  inputsWithClanModules = lib.filterAttrs (
-    _name: value: builtins.hasAttr "clanModules" value
-  ) self.inputs;
-
-  flattenedClanModules = lib.foldl' (
-    acc: input:
-    lib.mkMerge [
-      acc
-      input.clanModules
-    ]
-  ) { } (lib.attrValues inputsWithClanModules);
-in
-{
-  inventory.modules = flattenedClanModules;
-}

lib/build-clan/flake-module.nix

@@ -43,10 +43,7 @@ in
                 include = [
                   "flakeModules"
                   "inventory.json"
-                  "lib/build-clan"
-                  "lib/default.nix"
-                  "lib/flake-module.nix"
-                  "lib/inventory"
+                  "lib"
                   "machines"
                   "nixosModules"
                 ];

lib/build-clan/module.nix

@@ -77,6 +77,9 @@ let
           # Inherit the inventory assertions ?
           # { inherit (mergedInventory) assertions; }
           { imports = inventoryClass.machines.${name}.machineImports or [ ]; }
+
+          # Import the distribute services
+          { imports = config.clanInternals.distributedServices.allMachines.${name} or [ ]; }
           (
             {
               # Settings
@@ -165,18 +168,6 @@ let
 in
 {
   imports = [
-    # Temporarily disable auto-imports since the type of the modules is not a plain path anymore we cant "merge" multiple definitions
-    # That this feature worked previously seems like a coincidence.
-    # TODO(@Qubasa): make sure modules are not imported twice.
-    # Example error:
-    # The option `inventory.modules.admin' is defined multiple times while it's expected to be unique.
-    # - In `/nix/store/a0iqxl7r1spqsf2b886kn3i5sj8p37nc-source/lib/build-clan/auto-imports.nix': /nix/store/a0iqxl7r1spqsf2b886kn3i5sj8p37nc-source/clanModules/admin
-    # - In `/nix/store/a0iqxl7r1spqsf2b886kn3i5sj8p37nc-source/lib/build-clan/module.nix': /nix/store/a0iqxl7r1spqsf2b886kn3i5sj8p37nc-source/clanModules/admin
-    #
-    # After the inventory refactoring we might not need this anymore
-    # People can just import the module they want to use: `module = { input = "inputName"; name = "moduleName"; };`
-    # ./auto-imports.nix
-
     # Merge the inventory file
     {
       inventory = _: {

lib/distributed-service/flake-module.nix

@@ -21,7 +21,6 @@ in
       checks = {
         lib-distributedServices-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } ''
           export HOME="$(realpath .)"
-          export NIX_ABORT_ON_WARN=1
           nix-unit --eval-store "$HOME" \
             --extra-experimental-features flakes \
             ${inputOverrides} \

lib/distributed-service/inventory-adapter.nix

@@ -183,8 +183,17 @@ let
       ];
     }
   ) { } importedModuleWithInstances;
+
+  # TODO: Return an attribute set of resources instead of a plain list of nixosModules
+  allMachines = lib.foldlAttrs (
+    acc: _name: eval:
+    acc
+    // lib.mapAttrs (
+      machineName: result: acc.${machineName} or [ ] ++ [ result.nixosModule ]
+    ) eval.config.result.final
+  ) { } evals;
 in
 {
   inherit importedModuleWithInstances grouped;
-  inherit evals;
+  inherit evals allMachines;
 }

lib/distributed-service/service-module.nix

@@ -503,7 +503,8 @@ in
           nixosModule = {
             imports = [
               # For error backtracing. This module was produced by the 'perMachine' function
-              (lib.setDefaultModuleLocation "via perMachine" machineResult.nixosModule)
+              # TODO: check if we need this or if it leads to better errors if we pass the underlying module locations
+              (lib.setDefaultModuleLocation "clan.service: ${config.manifest.name} - via perMachine" machineResult.nixosModule)
             ] ++ instanceResults;
           };
         }

lib/inventory/build-inventory/interface.nix

@@ -343,6 +343,7 @@ in
         }
       );
       default = { };
+      apply = lib.warn "Inventory.instances and related features are still under development. Please use with care.";
     };
     services = lib.mkOption {
       description = ''

lib/inventory/flake-module.nix

@@ -50,11 +50,7 @@ in
               self.filter {
                 include = [
                   "flakeModules"
-                  "lib/default.nix"
-                  "lib/flake-module.nix"
-                  "lib/inventory"
-                  "lib/constraints"
-                  "lib/frontmatter"
+                  "lib"
                   "clanModules/flake-module.nix"
                   "clanModules/borgbackup"
                 ];

lib/values/flake-module.nix

@@ -28,9 +28,7 @@ in
               self.filter {
                 include = [
                   "flakeModules"
-                  "lib/default.nix"
-                  "lib/flake-module.nix"
-                  "lib/values"
+                  "lib"
                 ];
               }
             }#legacyPackages.${system}.evalTests-values