Merge pull request 'feat(clan-services): enable recursive services' (#3972) from hsjobeki/nested-services into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3972

lib/inventory/distributed-service/inventory-adapter.nix

@@ -19,6 +19,7 @@ let
     { modules, prefix }:
     (lib.evalModules {
       class = "clan.service";
+      specialArgs._ctx = prefix;
       modules = [
         ./service-module.nix
         # feature modules

lib/inventory/distributed-service/service-module.nix

@@ -1,31 +1,18 @@
-{ lib, config, ... }:
+{
+  lib,
+  config,
+  _ctx,
+  ...
+}:
 let
   inherit (lib) mkOption types;
   inherit (types) attrsWith submoduleWith;
 
+  errorContext = "Error context: ${lib.concatStringsSep "." _ctx}";
   # TODO:
   # Remove once this gets merged upstream; performs in O(n*log(n) instead of O(n^2))
   # https://github.com/NixOS/nixpkgs/pull/355616/files
   uniqueStrings = list: builtins.attrNames (builtins.groupBy lib.id list);
-
-  checkInstanceRoles =
-    instanceName: instanceRoles:
-    let
-      unmatchedRoles = lib.filter (roleName: !lib.elem roleName (lib.attrNames config.roles)) (
-        lib.attrNames instanceRoles
-      );
-    in
-    if unmatchedRoles == [ ] then
-      true
-    else
-      throw ''
-        inventory instance: 'instances.${instanceName}' defines the following roles:
-        ${builtins.toJSON unmatchedRoles}
-
-        But the clan-service module '${config.manifest.name}' defines roles:
-        ${builtins.toJSON (lib.attrNames config.roles)}
-      '';
-
   /**
     Merges the role- and machine-settings using the role interface
 
@@ -39,8 +26,6 @@ let
 
     The caller is responsible to use .config or .extendModules
   */
-  # TODO: evaluate against the role.settings statically and use extendModules to get the machineSettings
-  # Doing this might improve performance
   evalMachineSettings =
     {
       roleName,
@@ -53,7 +38,8 @@ let
       # This prints the path where the option should be defined rather than the plain path within settings
       # "The option `instances.foo.roles.server.machines.test.settings.<>' was accessed but has no value defined. Try setting the option."
       prefix =
-        [
+        _ctx
+        ++ [
           "instances"
           instanceName
           "roles"
@@ -78,7 +64,7 @@ let
         (lib.setDefaultModuleLocation "Via clan.service module: roles.${roleName}.interface"
           config.roles.${roleName}.interface
         )
-        (lib.setDefaultModuleLocation "inventory.instances.${instanceName}.roles.${roleName}.settings"
+        (lib.setDefaultModuleLocation "instances.${instanceName}.roles.${roleName}.settings"
           config.instances.${instanceName}.roles.${roleName}.settings
         )
         settings
@@ -88,32 +74,6 @@ let
       ];
     };
 
-  /**
-    Makes a module extensible
-    returning its config
-    and making it extensible via '__functor' polymorphism
-
-    Example:
-
-    ```nix-repl
-    res = makeExtensibleConfig (evalModules { options.foo = mkOption { default = 42; };)
-    res
-    =>
-    {
-      foo = 42;
-      _functor = <function>;
-    }
-
-    # This allows to override using mkDefault, mkForce, etc.
-    res { foo = 100; }
-    =>
-    {
-      foo = 100;
-      _functor = <function>;
-    }
-    ```
-  */
-
   # Extend evalModules result by a module, returns .config.
   extendEval = eval: m: (eval.extendModules { modules = lib.toList m; }).config;
 
@@ -129,15 +89,12 @@ let
     instanceName: instance:
     lib.mapAttrs (roleName: role: {
       machines = lib.mapAttrs (machineName: v: {
-        # TODO: evaluate the settings against the interface
-        # settings = (evalMachineSettings { inherit roleName instanceName; inherit (v) settings; }).config;
         settings =
           (evalMachineSettings {
             inherit roleName instanceName machineName;
             inherit (v) settings;
           }).config;
       }) role.machines;
-      # TODO: evaluate the settings against the interface
       settings =
         (evalMachineSettings {
           inherit roleName instanceName;
@@ -147,16 +104,15 @@ let
 in
 {
   options = {
-    # TODO: deduplicate this with inventory.instances
-    # Although inventory has stricter constraints
     instances = mkOption {
-      # Instances are created in the inventory
       visible = false;
       defaultText = "Throws: 'The service must define its instances' when not defined";
       default = throw ''
         The clan service module ${config.manifest.name} doesn't define any instances.
 
-        Did you forget to create instances via 'inventory.instances'?
+        Did you forget to create instances via 'instances'?
+
+        ${errorContext}
       '';
       description = ''
         Instances of the service.
@@ -179,11 +135,6 @@ in
             (
               { name, ... }:
               {
-                # options.settings = mkOption {
-                #   description = "settings of 'instance': ${name}";
-                #   default = {};
-                #   apply = v: lib.seq (checkInstanceSettings name v) v;
-                # };
                 options.roles = mkOption {
                   description = ''
                     Roles of the instance.
@@ -204,7 +155,9 @@ in
                     Instance '${name}' of service '${config.manifest.name}' mut define members via 'roles'.
 
                     To include a machine:
-                    'instances.${name}.roles.<role-name>.machines.<your-machine-name>' must be set.
+                    'instances.${name}.roles.<role-name>.machines.<machine-name>' must be set.
+
+                    ${errorContext}
                   '';
                   type = attrsWith {
                     placeholder = "roleName";
@@ -258,7 +211,34 @@ in
                       ];
                     };
                   };
-                  apply = v: lib.seq (checkInstanceRoles name v) v;
+                  apply =
+                    v:
+                    lib.seq (
+                      (
+
+                        instanceName: instanceRoles:
+                        let
+                          unmatchedRoles = lib.filter (roleName: !lib.elem roleName (lib.attrNames config.roles)) (
+                            lib.attrNames instanceRoles
+                          );
+                        in
+                        if unmatchedRoles == [ ] then
+                          true
+                        else
+                          throw ''
+                            Instance: 'instances.${instanceName}' uses the following roles:
+                            ${builtins.toJSON unmatchedRoles}
+
+                            But the clan-service module '${config.manifest.name}' only defines roles:
+                            ${builtins.toJSON (lib.attrNames config.roles)}
+
+                            ${errorContext}
+                          ''
+
+                      )
+                      name
+                      v
+                    ) v;
                 };
               }
             )
@@ -301,6 +281,8 @@ in
 
         To define multiple instance behavior:
         `roles.client.perInstance = { ... }: {}`
+
+        ${errorContext}
       '';
       type = attrsWith {
         placeholder = "roleName";
@@ -336,8 +318,6 @@ in
                     - *defaults* that depend on the *machine* or *instance* should be added to *settings* later in 'perInstance' or 'perMachine'
                   '';
                   type = types.deferredModule;
-                  # TODO: Default to an empty module
-                  # need to test that an the empty module can be evaluated to empty settings
                   default = { };
                 };
                 options.perInstance = mkOption {
@@ -379,7 +359,7 @@ in
                       };
                       ```
 
-                    - `settings`: The settings of the role, as defined in `inventory`
+                    - `settings`: The settings of the role, as defined in `instances`
                       ```nix
                       {
                         timeout = 30;
@@ -432,16 +412,25 @@ in
                             ```
                           '';
                         };
-                        # TODO: Recursive services
                         options.services = mkOption {
                           visible = false;
                           type = attrsWith {
                             placeholder = "serviceName";
                             elemType = submoduleWith {
-                              modules = [ ./service-module.nix ];
+                              modules = [
+                                {
+                                  _module.args._ctx = _ctx ++ [
+                                    config.manifest.name
+                                    "roles"
+                                    roleName
+                                    "perInstance"
+                                    "services"
+                                  ];
+                                }
+                                ./service-module.nix
+                              ];
                             };
                           };
-                          apply = _: throw "Not implemented yet";
                           default = { };
                         };
                       })
@@ -548,16 +537,23 @@ in
                 ```
               '';
             };
-            # TODO: Recursive services
             options.services = mkOption {
               visible = false;
               type = attrsWith {
                 placeholder = "serviceName";
                 elemType = submoduleWith {
-                  modules = [ ./service-module.nix ];
+                  modules = [
+                    {
+                      _module.args._ctx = _ctx ++ [
+                        config.manifest.name
+                        "perMachine"
+                        "services"
+                      ];
+                    }
+                    ./service-module.nix
+                  ];
                 };
               };
-              apply = _: throw "Not implemented yet";
               default = { };
             };
           })
@@ -591,7 +587,6 @@ in
                 in
                 uniqueStrings (collectRoles machineScope.instances);
             };
-            # TODO: instances.<instanceName>.roles should contain all roles, even if nobody has the role
             inherit (machineScope) instances;
 
             # There are no machine settings.
@@ -605,6 +600,8 @@ in
               - 'instances.<instanceName>.roles.<roleName>.machines.<machineName>.settings' should be used instead.
 
               If that is insufficient, you might also consider using 'roles.<roleName>.perInstance' instead of 'perMachine'.
+
+              ${errorContext}
             '';
           };
 
@@ -627,7 +624,7 @@ in
               allMachines :: {
                 <machineName> :: {
                   nixosModule :: NixOSModule;
-                  services :: { }; # TODO: nested services
+                  services :: { };
                 };
               };
             };
@@ -666,6 +663,7 @@ in
       type = types.attrsOf types.raw;
     };
 
+    # The result collected from 'perMachine'
     result.allMachines = mkOption {
       visible = false;
       readOnly = true;
@@ -720,43 +718,93 @@ in
       default = lib.mapAttrs (
         machineName: machineResult:
         let
-          instanceResults = lib.foldlAttrs (
+          instanceResults =
-            acc: roleName: role:
+            lib.foldlAttrs
-            acc
+              (
-            ++ lib.foldlAttrs (
+                roleAcc: roleName: role:
-              acc: instanceName: instance:
+                roleAcc
-              if instance.allMachines.${machineName}.nixosModule or { } != { } then
+                // lib.foldlAttrs (
-                acc
+                  instanceAcc: instanceName: instance:
-                ++ [
+                  instanceAcc
-                  (lib.setDefaultModuleLocation
+                  // {
-                    "Via instances.${instanceName}.roles.${roleName}.machines.${machineName}"
+                    nixosModules =
-                    instance.allMachines.${machineName}.nixosModule
+                      (
-                  )
+                        (lib.mapAttrsToList (
-                ]
+                          nestedServiceName: serviceModule:
-              else
+                          let
-                acc
+                            unmatchedMachines = lib.attrNames (
-            ) [ ] role.allInstances
+                              lib.removeAttrs serviceModule.result.final (lib.attrNames config.result.allMachines)
-          ) [ ] config.result.allRoles;
+                            );
+                          in
+                          if unmatchedMachines != [ ] then
+                            throw ''
+                              The following machines are not part of the parent service: ${builtins.toJSON unmatchedMachines}
+                              Either remove the machines, or include them into the parent via a role.
+                              (Added via roles.${roleName}.perInstance.services.${nestedServiceName})
+
+                              ${errorContext}
+                            ''
+                          else
+                            serviceModule.result.final.${machineName}.nixosModule
+                        ) instance.allMachines.${machineName}.services or { })
+
+                      )
+                      ++ (
+                        if instance.allMachines.${machineName}.nixosModule or { } != { } then
+                          instanceAcc.nixosModules
+                          ++ [
+                            (lib.setDefaultModuleLocation
+                              "Via instances.${instanceName}.roles.${roleName}.machines.${machineName}"
+                              instance.allMachines.${machineName}.nixosModule
+                            )
+                          ]
+                        else
+                          instanceAcc.nixosModules
+                      );
+                  }
+                ) roleAcc role.allInstances
+              )
+              {
+                nixosModules = [ ];
+                # ...
+              }
+              config.result.allRoles;
         in
         {
-          inherit instanceResults;
+          inherit instanceResults machineResult;
           nixosModule = {
-            imports = [
+            imports =
-              # include service assertions:
+              [
-              (
+                # include service assertions:
+                (
+                  let
+                    failedAssertions = (lib.filterAttrs (_: v: !v.assertion) config.result.assertions);
+                  in
+                  {
+                    assertions = lib.attrValues failedAssertions;
+                  }
+                )
+                (lib.setDefaultModuleLocation "Via ${config.manifest.name}.perMachine - machine='${machineName}';" machineResult.nixosModule)
+              ]
+              ++ (lib.mapAttrsToList (
+                nestedServiceName: serviceModule:
                 let
-                  failedAssertions = (lib.filterAttrs (_: v: !v.assertion) config.result.assertions);
+                  unmatchedMachines = lib.attrNames (
+                    lib.removeAttrs serviceModule.result.final (lib.attrNames config.result.allMachines)
+                  );
                 in
-                {
+                if unmatchedMachines != [ ] then
-                  assertions = lib.attrValues failedAssertions;
+                  throw ''
-                }
+                    The following machines are not part of the parent service: ${builtins.toJSON unmatchedMachines}
-              )
+                    Either remove the machines, or include them into the parent via a role.
+                    (Added via perMachine.services.${nestedServiceName})
 
-              # For error backtracing. This module was produced by the 'perMachine' function
+                    ${errorContext}
-              # TODO: check if we need this or if it leads to better errors if we pass the underlying module locations
+                  ''
-              # (lib.setDefaultModuleLocation "clan.service: ${config.manifest.name} - via perMachine" machineResult.nixosModule)
+                else
-              (machineResult.nixosModule)
+                  serviceModule.result.final.${machineName}.nixosModule
-            ] ++ instanceResults;
+              ) machineResult.services)
+              ++ instanceResults.nixosModules;
           };
         }
       ) config.result.allMachines;

lib/inventory/distributed-service/tests/default.nix

@@ -278,4 +278,5 @@ in
 
   per_machine_args = import ./per_machine_args.nix { inherit lib callInventoryAdapter; };
   per_instance_args = import ./per_instance_args.nix { inherit lib callInventoryAdapter; };
+  nested = import ./nested_services { inherit lib clanLib; };
 }

lib/inventory/distributed-service/tests/nested_services/default.nix

@@ -0,0 +1,8 @@
+{ clanLib, lib, ... }:
+{
+  test_simple = import ./simple.nix { inherit clanLib lib; };
+
+  test_multi_machine = import ./multi_machine.nix { inherit clanLib lib; };
+
+  test_multi_import_duplication = import ./multi_import_duplication.nix { inherit clanLib lib; };
+}

lib/inventory/distributed-service/tests/nested_services/multi_import_duplication.nix

@@ -0,0 +1,125 @@
+{ clanLib, lib, ... }:
+let
+  # Potentially imported many times
+  # To add the ssh key
+  example-admin = (
+    { lib, ... }:
+    {
+      manifest.name = "example-admin";
+
+      roles.client.interface = {
+        options.keys = lib.mkOption { };
+      };
+
+      roles.client.perInstance =
+        { settings, ... }:
+        {
+          nixosModule = {
+            inherit (settings) keys;
+          };
+        };
+    }
+  );
+
+  consumer-A =
+    { ... }:
+    {
+      manifest.name = "consumer-A";
+
+      instances.foo = {
+        roles.server.machines."jon" = { };
+      };
+      instances.bar = {
+        roles.server.machines."jon" = { };
+      };
+
+      roles.server = {
+        perInstance =
+          { machine, instanceName, ... }:
+          {
+            services."example-admin" = {
+              imports = [
+                example-admin
+              ];
+              instances."${instanceName}" = {
+                roles.client.machines.${machine.name} = {
+                  settings.keys = [ "pubkey-1" ];
+                };
+              };
+            };
+          };
+      };
+    };
+  consumer-B =
+    { ... }:
+    {
+      manifest.name = "consumer-A";
+
+      instances.foo = {
+        roles.server.machines."jon" = { };
+      };
+      instances.bar = {
+        roles.server.machines."jon" = { };
+      };
+
+      roles.server = {
+        perInstance =
+          { machine, instanceName, ... }:
+          {
+            services."example-admin" = {
+              imports = [
+                example-admin
+              ];
+              instances."${instanceName}" = {
+                roles.client.machines.${machine.name} = {
+                  settings.keys = [
+                    "pubkey-1"
+                  ];
+                };
+              };
+            };
+          };
+      };
+    };
+
+  eval = clanLib.inventory.evalClanService {
+    modules = [
+      (consumer-A)
+    ];
+    prefix = [ ];
+  };
+  eval2 = clanLib.inventory.evalClanService {
+    modules = [
+      (consumer-B)
+    ];
+    prefix = [ ];
+  };
+
+  evalNixos = lib.evalModules {
+    modules = [
+      {
+        options.assertions = lib.mkOption { };
+        # This is suboptimal
+        options.keys = lib.mkOption { };
+      }
+      eval.config.result.final.jon.nixosModule
+      eval2.config.result.final.jon.nixosModule
+    ];
+  };
+in
+{
+  # Check that the nixos system has the settings from the nested module, as well as those from the "perMachine" and "perInstance"
+  inherit eval;
+  expr = evalNixos.config;
+  expected = {
+    assertions = [ ];
+    # TODO: Some deduplication mechanism is nice
+    # Could add types.set or do 'apply = unique', or something else ?
+    keys = [
+      "pubkey-1"
+      "pubkey-1"
+      "pubkey-1"
+      "pubkey-1"
+    ];
+  };
+}

lib/inventory/distributed-service/tests/nested_services/multi_machine.nix

@@ -0,0 +1,108 @@
+{ clanLib, lib, ... }:
+let
+  service-B = (
+    { lib, ... }:
+    {
+      manifest.name = "service-B";
+
+      roles.client.interface = {
+        options.user = lib.mkOption { };
+        options.host = lib.mkOption { };
+      };
+      roles.client.perInstance =
+        { settings, instanceName, ... }:
+        {
+          nixosModule = {
+            units.${instanceName} = {
+              script = settings.user + "@" + settings.host;
+            };
+          };
+        };
+      perMachine =
+        { ... }:
+        {
+          nixosModule = {
+            ssh.enable = true;
+          };
+        };
+    }
+  );
+  service-A =
+    { ... }:
+    {
+      manifest.name = "service-A";
+
+      instances.foo = {
+        roles.server.machines."jon" = { };
+        roles.server.machines."sara" = { };
+      };
+
+      roles.server = {
+        perInstance =
+          { machine, instanceName, ... }:
+          {
+            services."B" = {
+              imports = [
+                service-B
+              ];
+              instances."A-${instanceName}-B" = {
+                roles.client.machines.${machine.name} = {
+                  settings.user = "johnny";
+                  settings.host = machine.name;
+                };
+              };
+            };
+          };
+      };
+    };
+
+  eval = clanLib.inventory.evalClanService {
+    modules = [
+      (service-A)
+    ];
+    prefix = [ ];
+  };
+
+  evalNixos = lib.mapAttrs (
+    _n: v:
+    (lib.evalModules {
+      modules = [
+        {
+          options.assertions = lib.mkOption { };
+          options.units = lib.mkOption { };
+          options.ssh = lib.mkOption { };
+        }
+        v.nixosModule
+      ];
+    }).config
+  ) eval.config.result.final;
+in
+{
+  # Check that the nixos system has the settings from the nested module, as well as those from the "perMachine" and "perInstance"
+  inherit eval;
+  expr = evalNixos;
+  expected = {
+    jon = {
+      assertions = [ ];
+      ssh = {
+        enable = true;
+      };
+      units = {
+        A-foo-B = {
+          script = "johnny@jon";
+        };
+      };
+    };
+    sara = {
+      assertions = [ ];
+      ssh = {
+        enable = true;
+      };
+      units = {
+        A-foo-B = {
+          script = "johnny@sara";
+        };
+      };
+    };
+  };
+}

lib/inventory/distributed-service/tests/nested_services/simple.nix

@@ -0,0 +1,117 @@
+/*
+  service-B :: Service
+    exports a nixosModule which set "address" and "hostname"
+    Note: How we use null together with mkIf to create optional values.
+    This is a method, to create mergable modules
+
+  service-A :: Service
+
+  service-A.roles.server.perInstance.services."B"
+    imports service-B
+    configures a client with hostname = "johnny"
+
+  service-A.perMachine.services."B"
+    imports service-B
+    configures a client with address = "root"
+*/
+{ clanLib, lib, ... }:
+let
+  service-B = (
+    { lib, ... }:
+    {
+      manifest.name = "service-B";
+
+      roles.client.interface = {
+        options.hostname = lib.mkOption { default = null; };
+        options.address = lib.mkOption { default = null; };
+      };
+      roles.client.perInstance =
+        { settings, ... }:
+        {
+          nixosModule = {
+            imports = [
+              # Only export the value that is actually set.
+              (lib.mkIf (settings.hostname != null) {
+                hostname = settings.hostname;
+              })
+              (lib.mkIf (settings.address != null) {
+                address = settings.address;
+              })
+            ];
+          };
+        };
+    }
+  );
+  service-A =
+    { ... }:
+    {
+      manifest.name = "service-A";
+
+      instances.foo = {
+        roles.server.machines."jon" = { };
+      };
+      instances.bar = {
+        roles.server.machines."jon" = { };
+      };
+
+      roles.server = {
+        perInstance =
+          { machine, instanceName, ... }:
+          {
+            services."B" = {
+              imports = [
+                service-B
+              ];
+              instances."B-for-A" = {
+                roles.client.machines.${machine.name} = {
+                  settings.hostname = instanceName + "+johnny";
+                };
+              };
+            };
+          };
+      };
+      perMachine =
+        { machine, ... }:
+        {
+          services."B" = {
+            imports = [
+              service-B
+            ];
+            instances."B-for-A" = {
+              roles.client.machines.${machine.name} = {
+                settings.address = "root";
+              };
+            };
+          };
+        };
+    };
+
+  eval = clanLib.inventory.evalClanService {
+    modules = [
+      (service-A)
+    ];
+    prefix = [ ];
+  };
+
+  evalNixos = lib.evalModules {
+    modules = [
+      {
+        options.assertions = lib.mkOption { };
+        options.hostname = lib.mkOption { type = lib.types.separatedString " "; };
+        options.address = lib.mkOption { type = lib.types.str; };
+      }
+      eval.config.result.final."jon".nixosModule
+    ];
+  };
+in
+{
+  # Check that the nixos system has the settings from the nested module, as well as those from the "perMachine" and "perInstance"
+  inherit eval;
+  expr = evalNixos.config;
+  expected = {
+    address = "root";
+    assertions = [ ];
+    # Concatenates hostnames from both instances
+    hostname = "bar+johnny foo+johnny";
+  };
+}