Merge pull request 'Add build-clan module' (#1843) from hsjobeki/clan-core:hsjobeki-flake-parts into main

flakeModules/clan.nix

@@ -4,125 +4,34 @@ clan-core:
   lib,
   flake-parts-lib,
   inputs,
-  self,
   ...
 }:
 let
   inherit (lib) mkOption types;
-  buildClan = import ../lib/build-clan {
-    inherit lib clan-core;
-    inherit (inputs) nixpkgs;
-  };
-  cfg = config.clan;
 in
 {
-  imports = [
+
-    # TODO: figure out how to print the deprecation warning
+  options.clan = lib.mkOption {
-    # "${inputs.nixpkgs}/nixos/modules/misc/assertions.nix"
+    type = types.submoduleWith {
-    (lib.mkRenamedOptionModule
+      # _module.args = {
-      [
+      # };
-        "clan"
+      specialArgs = {
-        "clanName"
+        inherit clan-core;
-      ]
+        inherit (inputs) nixpkgs;
-      [
+      };
-        "clan"
+      modules = [
-        "meta"
+        ../lib/build-clan/interface.nix
-        "name"
+        ../lib/build-clan/module.nix
-      ]
-    )
-    (lib.mkRenamedOptionModule
-      [
-        "clan"
-        "clanIcon"
-      ]
-      [
-        "clan"
-        "meta"
-        "icon"
-      ]
-    )
       ];
-
-  options.clan = {
-    directory = mkOption {
-      type = types.path;
-      description = "The directory containing the clan subdirectory";
-      default = self; # default to the directory of the flake
-    };
-    specialArgs = mkOption {
-      type = types.attrsOf types.raw;
-      default = { };
-      description = "Extra arguments to pass to nixosSystem i.e. useful to make self available";
-    };
-    machines = mkOption {
-      type = types.attrsOf types.raw;
-      default = { };
-      description = "Allows to include machine-specific modules i.e. machines.\${name} = { ... }";
-    };
-    inventory = mkOption {
-      #type = types.submodule { imports = [ ../lib/inventory/build-inventory/interface.nix ]; };
-      type = types.attrsOf types.raw;
-      default = { };
-      description = ''
-        An abstract service layer for consistently configuring distributed services across machine boundaries.
-        See https://docs.clan.lol/concepts/inventory/ for more details.
-      '';
-    };
-
-    # Checks are performed in 'buildClan'
-    # Not everyone uses flake-parts
-    meta = {
-      name = lib.mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = "Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.";
-      };
-      icon = mkOption {
-        type = types.nullOr types.path;
-        default = null;
-        description = "A path to an icon to be used for the clan in the GUI";
-      };
-      description = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = "A short description of the clan";
     };
   };
 
-    pkgsForSystem = mkOption {
-      type = types.functionTo types.raw;
-      default = _system: null;
-      description = "A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system.";
-    };
-  };
   options.flake = flake-parts-lib.mkSubmoduleOptions {
-    clanInternals = lib.mkOption {
+    clanInternals = lib.mkOption { type = types.raw; };
-      type = lib.types.submodule {
-        options = {
-          inventory = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; };
-          inventoryFile = lib.mkOption { type = lib.types.unspecified; };
-
-          clanModules = lib.mkOption { type = lib.types.attrsOf lib.types.path; };
-          source = lib.mkOption { type = lib.types.path; };
-          meta = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; };
-          all-machines-json = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; };
-          machines = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); };
-          machinesFunc = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); };
-        };
-      };
-    };
   };
   config = {
-    flake = buildClan {
+    flake.clanInternals = config.clan.clanInternals;
-      inherit (cfg)
+    flake.nixosConfigurations = config.clan.nixosConfigurations;
-        directory
-        specialArgs
-        machines
-        pkgsForSystem
-        meta
-        inventory
-        ;
-    };
   };
   _file = __curPos.file;
 }

lib/build-clan/default.nix

@@ -1,306 +1,42 @@
+## WARNING: Do not add core logic here.
+## This is only a wrapper such that buildClan can be called as a function.
+## Add any logic to ./module.nix
 {
-  clan-core,
-  nixpkgs,
   lib,
+  nixpkgs,
+  clan-core,
 }:
 {
-  directory, # The directory containing the machines subdirectory
+  ## Inputs
-  specialArgs ? { }, # Extra arguments to pass to nixosSystem i.e. useful to make self available
+  directory, # The directory containing the machines subdirectory # allows to include machine-specific modules i.e. machines.${name} = { ... }
-  machines ? { }, # allows to include machine-specific modules i.e. machines.${name} = { ... }
-  # DEPRECATED: use meta.name instead
-  clanName ? null, # Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.
-  # DEPRECATED: use meta.icon instead
-  clanIcon ? null, # A path to an icon to be used for the clan, should be the same for all machines
-  meta ? { }, # A set containing clan meta: name :: string, icon :: string, description :: string
   # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system.
   # This improves performance, but all nipxkgs.* options will be ignored.
-  pkgsForSystem ? (_system: null),
-  /*
-    Low level inventory configuration.
-    Overrides the services configuration.
-  */
   inventory ? { },
-}:
+  ## Sepcial inputs (not passed to the module system as config)
+  specialArgs ? { }, # Extra arguments to pass to nixosSystem i.e. useful to make self available # A set containing clan meta: name :: string, icon :: string, description :: string
+  ##
+  ...
+}@attrs:
 let
-  # Internal inventory, this is the result of merging all potential inventory sources:
+  eval = import ./eval.nix {
-  # - Default instances configured via 'services'
+    inherit
-  # - The inventory overrides
+      lib
-  #   - Machines that exist in inventory.machines
+      nixpkgs
-  # - Machines explicitly configured via 'machines' argument
+      specialArgs
-  # - Machines that exist in the machines directory
+      clan-core
-  # Checks on the module level:
+      ;
-  # - Each service role must reference a valid machine after all machines are merged
+  } { self = directory; };
-
+  meta = attrs.meta or { };
-  clanToInventory =
+  rest = builtins.removeAttrs attrs [
-    config:
-    { clanPath, inventoryPath }:
-    let
-      v = lib.attrByPath clanPath null config;
-    in
-    lib.optionalAttrs (v != null) (lib.setAttrByPath inventoryPath v);
-
-  mergedInventory =
-    (lib.evalModules {
-      modules = [
-        clan-core.lib.inventory.interface
-        { inherit meta; }
-        (
-          if
-            builtins.pathExists "${directory}/inventory.json"
-          # Is recursively applied. Any explicit nix will override.
-          then
-            (builtins.fromJSON (builtins.readFile "${directory}/inventory.json"))
-          else
-            { }
-        )
-        inventory
-        # Machines explicitly configured via 'machines' argument
-        {
-          # { ${name} :: meta // { name, tags } }
-          machines = lib.mapAttrs (
-            name: machineConfig:
-            (lib.attrByPath [
-              "clan"
     "meta"
-            ] { } machineConfig)
+    "specialArgs"
-            // {
-              # meta.name default is the attribute name of the machine
-              name = lib.mkDefault (
-                lib.attrByPath [
-                  "clan"
-                  "meta"
-                  "name"
-                ] name machineConfig
-              );
-            }
-            # tags
-            // (clanToInventory machineConfig {
-              clanPath = [
-                "clan"
-                "tags"
   ];
-              inventoryPath = [ "tags" ];
-            })
-            # system
-            // (clanToInventory machineConfig {
-              clanPath = [
-                "nixpkgs"
-                "hostPlatform"
-              ];
-              inventoryPath = [ "system" ];
-            })
-            # deploy.targetHost
-            // (clanToInventory machineConfig {
-              clanPath = [
-                "clan"
-                "core"
-                "networking"
-                "targetHost"
-              ];
-              inventoryPath = [
-                "deploy"
-                "targetHost"
-              ];
-            })
-          ) machines;
-        }
-
-        # Will be deprecated
-        {
-          machines =
-            lib.mapAttrs
-              (
-                name: _:
-                # Use mkForce to make sure users migrate to the inventory system.
-                # When the settings.json exists the evaluation will print the deprecation warning.
-                lib.mkForce {
-                  inherit name;
-                  system = (machineSettings name).nixpkgs.hostSystem or null;
-                }
-              )
-              (
-                lib.filterAttrs (
-                  machineName: _: builtins.pathExists "${directory}/machines/${machineName}/settings.json"
-                ) machinesDirs
-              );
-        }
-
-        # Deprecated interface
-        (if clanName != null then { meta.name = clanName; } else { })
-        (if clanIcon != null then { meta.icon = clanIcon; } else { })
-      ];
-    }).config;
-
-  inherit (clan-core.lib.inventory) buildInventory;
-
-  # map from machine name to service configuration
-  # { ${machineName} :: Config }
-  serviceConfigs = buildInventory {
-    inventory = mergedInventory;
-    inherit directory;
-  };
-
-  machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") (
-    builtins.readDir (directory + /machines)
-  );
-
-  machineSettings =
-    machineName:
-    let
-      warn = lib.warn ''
-        The use of ./machines/<machine>/settings.json is deprecated.
-        If your settings.json is empty, you can safely remove it.
-        !!! Consider using the inventory system. !!!
-
-        File: ${directory + /machines/${machineName}/settings.json}
-
-        If there are still features missing in the inventory system, please open an issue on the clan-core repository.
-      '';
-    in
-    # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily
-    # This is useful for doing a dry-run before writing changes into the settings.json
-    # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval
-    if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then
-      warn (builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE")))
-    else
-      lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") (
-        warn (builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json)))
-      );
-
-  machineImports =
-    machineSettings: map (module: clan-core.clanModules.${module}) (machineSettings.clanImports or [ ]);
-
-  deprecationWarnings = [
-    (lib.warnIf (
-      clanName != null
-    ) "clanName in buildClan is deprecated, please use meta.name instead." null)
-    (lib.warnIf (clanIcon != null) "clanIcon is deprecated, please use meta.icon instead" null)
-  ];
-
-  # TODO: remove default system once we have a hardware-config mechanism
-  nixosConfiguration =
-    {
-      system ? "x86_64-linux",
-      name,
-      pkgs ? null,
-      extraConfig ? { },
-    }:
-    nixpkgs.lib.nixosSystem {
-      modules =
-        let
-          settings = machineSettings name;
-        in
-        (machineImports settings)
-        ++ [
-          {
-            # Autoinclude configuration.nix and hardware-configuration.nix
-            imports = builtins.filter (p: builtins.pathExists p) [
-              "${directory}/machines/${name}/configuration.nix"
-              "${directory}/machines/${name}/hardware-configuration.nix"
-            ];
-          }
-          settings
-          clan-core.nixosModules.clanCore
-          extraConfig
-          (machines.${name} or { })
-          # Inherit the inventory assertions ?
-          { inherit (mergedInventory) assertions; }
-          { imports = serviceConfigs.${name} or { }; }
-          (
-            {
-              # Settings
-              clan.core.clanDir = directory;
-              # Inherited from clan wide settings
-              clan.core.clanName = meta.name or clanName;
-              clan.core.clanIcon = meta.icon or clanIcon;
-
-              # Machine specific settings
-              clan.core.machineName = name;
-              networking.hostName = lib.mkDefault name;
-              nixpkgs.hostPlatform = lib.mkDefault system;
-
-              # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs)
-              nix.registry.nixpkgs.to = {
-                type = "path";
-                path = lib.mkDefault nixpkgs;
-              };
-            }
-            // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; }
-          )
-        ];
-      specialArgs = {
-        inherit clan-core;
-      } // specialArgs;
-    };
-
-  allMachines = mergedInventory.machines or { };
-
-  supportedSystems = [
-    "x86_64-linux"
-    "aarch64-linux"
-    "riscv64-linux"
-    "x86_64-darwin"
-    "aarch64-darwin"
-  ];
-
-  nixosConfigurations = lib.mapAttrs (name: _: nixosConfiguration { inherit name; }) allMachines;
-
-  # This instantiates nixos for each system that we support:
-  # configPerSystem = <system>.<machine>.nixosConfiguration
-  # We need this to build nixos secret generators for each system
-  configsPerSystem = builtins.listToAttrs (
-    builtins.map (
-      system:
-      lib.nameValuePair system (
-        lib.mapAttrs (
-          name: _:
-          nixosConfiguration {
-            inherit name system;
-            pkgs = pkgsForSystem system;
-          }
-        ) allMachines
-      )
-    ) supportedSystems
-  );
-
-  configsFuncPerSystem = builtins.listToAttrs (
-    builtins.map (
-      system:
-      lib.nameValuePair system (
-        lib.mapAttrs (
-          name: _: args:
-          nixosConfiguration (
-            args
-            // {
-              inherit name system;
-              pkgs = pkgsForSystem system;
-            }
-          )
-        ) allMachines
-      )
-    ) supportedSystems
-  );
 in
-builtins.deepSeq deprecationWarnings {
+eval {
-  inherit nixosConfigurations;
+  inventory.meta = lib.mapAttrs (_: lib.mkDefault) meta;
-
+  imports = [
-  clanInternals = {
+    rest
-    inherit (clan-core) clanModules;
+    # implementation
-    source = "${clan-core}";
+    ./module.nix
-
+  ];
-    meta = mergedInventory.meta;
-    inventory = mergedInventory;
-
-    inventoryFile = "${directory}/inventory.json";
-
-    # machine specifics
-    machines = configsPerSystem;
-    machinesFunc = configsFuncPerSystem;
-    all-machines-json = lib.mapAttrs (
-      system: configs:
-      nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" (
-        lib.mapAttrs (_: m: m.config.system.clan.deployment.data) configs
-      )
-    ) configsPerSystem;
-  };
 }

lib/build-clan/eval.nix

@@ -0,0 +1,18 @@
+{
+  lib,
+  nixpkgs,
+  clan-core,
+  specialArgs ? { },
+}:
+# Returns a function that takes self, which should point to the directory of the flake
+{ self }:
+module:
+(lib.evalModules {
+  specialArgs = {
+    inherit self clan-core nixpkgs;
+  } // specialArgs;
+  modules = [
+    ./interface.nix
+    module
+  ];
+}).config

lib/build-clan/flake-module.nix

@@ -0,0 +1,41 @@
+{ self, inputs, ... }:
+let
+  inputOverrides = builtins.concatStringsSep " " (
+    builtins.map (input: " --override-input ${input} ${inputs.${input}}") (builtins.attrNames inputs)
+  );
+in
+{
+
+  perSystem =
+    {
+      pkgs,
+      lib,
+      system,
+      ...
+    }:
+    # let
+
+    # in
+    {
+
+      # Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests
+      legacyPackages.evalTests-build-clan = import ./tests.nix {
+        inherit lib;
+        inherit (inputs) nixpkgs;
+        clan-core = self;
+        buildClan = self.lib.buildClan;
+      };
+      checks = {
+        lib-build-clan-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } ''
+          export HOME="$(realpath .)"
+
+          nix-unit --eval-store "$HOME" \
+            --extra-experimental-features flakes \
+            ${inputOverrides} \
+            --flake ${self}#legacyPackages.${system}.evalTests-build-clan
+
+          touch $out
+        '';
+      };
+    };
+}

lib/build-clan/interface.nix

@@ -0,0 +1,77 @@
+{ lib, ... }:
+let
+  types = lib.types;
+in
+{
+  options = {
+    # Required options
+    directory = lib.mkOption {
+      type = types.path;
+      description = "The directory containing the clan subdirectory";
+    };
+
+    specialArgs = lib.mkOption {
+      type = types.attrsOf types.raw;
+      default = { };
+      description = "Extra arguments to pass to nixosSystem i.e. useful to make self available";
+    };
+
+    # Optional
+    machines = lib.mkOption {
+      type = types.attrsOf types.deferredModule;
+      default = { };
+    };
+    inventory = lib.mkOption {
+      type = types.submodule { imports = [ ../inventory/build-inventory/interface.nix ]; };
+    };
+
+    # Meta
+    meta = {
+      name = lib.mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = "Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.";
+      };
+      icon = lib.mkOption {
+        type = types.nullOr types.path;
+        default = null;
+        description = "A path to an icon to be used for the clan in the GUI";
+      };
+      description = lib.mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = "A short description of the clan";
+      };
+    };
+
+    pkgsForSystem = lib.mkOption {
+      type = types.functionTo (types.nullOr types.attrs);
+      default = _: null;
+    };
+
+    # Outputs
+    nixosConfigurations = lib.mkOption {
+      type = types.lazyAttrsOf types.raw;
+      default = { };
+    };
+    # flake.clanInternals
+    clanInternals = lib.mkOption {
+      # type = types.raw;
+      # ClanInternals
+      type = types.submodule {
+        options = {
+          # Those options are interfaced by the CLI
+          # We don't speficy the type here, for better performance.
+          inventory = lib.mkOption { type = lib.types.raw; };
+          inventoryFile = lib.mkOption { type = lib.types.raw; };
+          clanModules = lib.mkOption { type = lib.types.raw; };
+          source = lib.mkOption { type = lib.types.raw; };
+          meta = lib.mkOption { type = lib.types.raw; };
+          all-machines-json = lib.mkOption { type = lib.types.raw; };
+          machines = lib.mkOption { type = lib.types.raw; };
+          machinesFunc = lib.mkOption { type = lib.types.raw; };
+        };
+      };
+    };
+  };
+}

lib/build-clan/module.nix

@@ -0,0 +1,227 @@
+{
+  config,
+  clan-core,
+  nixpkgs,
+  lib,
+  ...
+}:
+let
+  inherit (config)
+    directory
+    machines
+    pkgsForSystem
+    specialArgs
+    ;
+
+  # Final inventory
+  inherit (config.clanInternals) inventory;
+
+  inherit (clan-core.lib.inventory) buildInventory;
+
+  # map from machine name to service configuration
+  # { ${machineName} :: Config }
+  serviceConfigs = (
+    buildInventory {
+      inherit inventory;
+      inherit directory;
+    }
+  );
+
+  machineSettings =
+    machineName:
+    let
+      warn = lib.warn ''
+        The use of ./machines/<machine>/settings.json is deprecated.
+        If your settings.json is empty, you can safely remove it.
+        !!! Consider using the inventory system. !!!
+
+        File: ${directory + /machines/${machineName}/settings.json}
+
+        If there are still features missing in the inventory system, please open an issue on the clan-core repository.
+      '';
+    in
+    # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily
+    # This is useful for doing a dry-run before writing changes into the settings.json
+    # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval
+    if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then
+      warn (builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE")))
+    else
+      lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") (
+        warn (builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json)))
+      );
+
+  machineImports =
+    machineSettings: map (module: clan-core.clanModules.${module}) (machineSettings.clanImports or [ ]);
+
+  # TODO: remove default system once we have a hardware-config mechanism
+  nixosConfiguration =
+    {
+      system ? "x86_64-linux",
+      name,
+      pkgs ? null,
+      extraConfig ? { },
+    }:
+    nixpkgs.lib.nixosSystem {
+      modules =
+        let
+          settings = machineSettings name;
+        in
+        (machineImports settings)
+        ++ [
+          {
+            # Autoinclude configuration.nix and hardware-configuration.nix
+            imports = builtins.filter builtins.pathExists [
+              "${directory}/machines/${name}/configuration.nix"
+              "${directory}/machines/${name}/hardware-configuration.nix"
+            ];
+          }
+          settings
+          clan-core.nixosModules.clanCore
+          extraConfig
+          (machines.${name} or { })
+          # Inherit the inventory assertions ?
+          # { inherit (mergedInventory) assertions; }
+          { imports = serviceConfigs.${name} or [ ]; }
+          (
+            {
+              # Settings
+              clan.core.clanDir = directory;
+              # Inherited from clan wide settings
+              # TODO: remove these
+              clan.core.clanName = config.inventory.meta.name;
+              clan.core.clanIcon = config.inventory.meta.icon;
+
+              # Machine specific settings
+              clan.core.machineName = name;
+              networking.hostName = lib.mkDefault name;
+              nixpkgs.hostPlatform = lib.mkDefault system;
+
+              # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs)
+              nix.registry.nixpkgs.to = {
+                type = "path";
+                path = lib.mkDefault nixpkgs;
+              };
+            }
+            // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; }
+          )
+        ];
+
+      specialArgs = {
+        inherit clan-core;
+      } // specialArgs;
+    };
+
+  # TODO: Will be deprecated
+  # We must migrate the tests, that create a settings.json to add a machine.
+  ##################################################
+  testMachines =
+    lib.mapAttrs
+      (name: _: {
+        inherit name;
+        system = (machineSettings name).nixpkgs.hostSystem or null;
+      })
+      (
+        lib.filterAttrs (
+          machineName: _:
+          if builtins.pathExists "${directory}/machines/${machineName}/settings.json" then
+            lib.warn ''
+              The use of ./machines/<machine>/settings.json is deprecated.
+              If your settings.json is empty, you can safely remove it.
+              !!! Consider using the inventory system. !!!
+
+              File: ${directory + /machines/${machineName}/settings.json}
+
+              If there are still features missing in the inventory system, please open an issue on the clan-core repository.
+            '' true
+          else
+            false
+        ) machinesDirs
+      );
+  machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") (
+    builtins.readDir (directory + /machines)
+  );
+  ##################################################
+
+  allMachines = inventory.machines or { } // config.machines or { } // testMachines;
+
+  supportedSystems = [
+    "x86_64-linux"
+    "aarch64-linux"
+    "riscv64-linux"
+    "x86_64-darwin"
+    "aarch64-darwin"
+  ];
+
+  nixosConfigurations = lib.mapAttrs (name: _: nixosConfiguration { inherit name; }) allMachines;
+
+  # This instantiates nixos for each system that we support:
+  # configPerSystem = <system>.<machine>.nixosConfiguration
+  # We need this to build nixos secret generators for each system
+  configsPerSystem = builtins.listToAttrs (
+    builtins.map (
+      system:
+      lib.nameValuePair system (
+        lib.mapAttrs (
+          name: _:
+          nixosConfiguration {
+            inherit name system;
+            pkgs = pkgsForSystem system;
+          }
+        ) allMachines
+      )
+    ) supportedSystems
+  );
+
+  configsFuncPerSystem = builtins.listToAttrs (
+    builtins.map (
+      system:
+      lib.nameValuePair system (
+        lib.mapAttrs (
+          name: _: args:
+          nixosConfiguration (
+            args
+            // {
+              inherit name system;
+              pkgs = pkgsForSystem system;
+            }
+          )
+        ) allMachines
+      )
+    ) supportedSystems
+  );
+
+  inventoryFile = "${directory}/inventory.json";
+
+  inventoryLoaded =
+    if builtins.pathExists inventoryFile then
+      (builtins.fromJSON (builtins.readFile inventoryFile))
+    else
+      { };
+in
+{
+  imports = [
+    # Merge the inventory file
+    { inventory = inventoryLoaded; }
+  ];
+
+  inherit nixosConfigurations;
+
+  clanInternals = {
+    inherit (clan-core) clanModules;
+    inherit inventoryFile;
+    inventory = config.inventory;
+    meta = config.inventory.meta;
+
+    source = "${clan-core}";
+
+    # machine specifics
+    machines = configsPerSystem;
+    machinesFunc = configsFuncPerSystem;
+    all-machines-json = lib.mapAttrs (
+      system: configs:
+      nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" (
+        lib.mapAttrs (_: m: m.config.system.clan.deployment.data) configs
+      )
+    ) configsPerSystem;
+  };
+}

lib/build-clan/tests.nix

@@ -0,0 +1,134 @@
+{
+  lib,
+  nixpkgs,
+  clan-core,
+  buildClan,
+  ...
+}:
+let
+  eval = import ./eval.nix { inherit lib nixpkgs clan-core; };
+
+  self = ./.;
+  evalClan = eval { inherit self; };
+
+in
+#######
+{
+  test_only_required =
+    let
+      config = evalClan { directory = ./.; };
+    in
+    {
+      expr = config.pkgsForSystem null == null;
+      expected = true;
+    };
+
+  test_all_simple =
+    let
+      config = evalClan {
+        directory = ./.;
+        machines = { };
+        inventory = {
+          meta.name = "test";
+        };
+        pkgsForSystem = _system: { };
+      };
+    in
+    {
+      expr = config ? inventory;
+      expected = true;
+    };
+
+  test_outputs_clanInternals =
+    let
+      config = evalClan {
+        imports = [
+          # What the user needs to specif
+          {
+            directory = ./.;
+            inventory.meta.name = "test";
+          }
+
+          ./module.nix
+          # Explicit output, usually defined by flake-parts
+          { options.nixosConfigurations = lib.mkOption { type = lib.types.raw; }; }
+        ];
+      };
+    in
+    {
+      expr = config.clanInternals.meta;
+      expected = {
+        description = null;
+        icon = null;
+        name = "test";
+      };
+    };
+
+  test_fn_simple =
+    let
+      result = buildClan {
+        directory = ./.;
+        meta.name = "test";
+      };
+    in
+    {
+      expr = result.clanInternals.meta;
+      expected = {
+        description = null;
+        icon = null;
+        name = "test";
+      };
+    };
+
+  test_fn_extensiv_meta =
+    let
+      result = buildClan {
+        directory = ./.;
+        meta.name = "test";
+        meta.description = "test";
+        meta.icon = "test";
+        inventory.meta.name = "superclan";
+        inventory.meta.description = "description";
+        inventory.meta.icon = "icon";
+      };
+    in
+    {
+      expr = result.clanInternals.meta;
+      expected = {
+        description = "description";
+        icon = "icon";
+        name = "superclan";
+      };
+    };
+
+  test_fn_clan_core =
+    let
+      result = buildClan {
+        directory = ../../.;
+        meta.name = "test-clan-core";
+      };
+    in
+    {
+      expr = builtins.attrNames result.nixosConfigurations;
+      expected = [ "test-inventory-machine" ];
+    };
+
+  test_buildClan_all_machines =
+    let
+      result = buildClan {
+        directory = ./.;
+        meta.name = "test";
+        inventory.machines.machine1.meta.name = "machine1";
+
+        machines.machine2 = { };
+
+      };
+    in
+    {
+      expr = builtins.attrNames result.nixosConfigurations;
+      expected = [
+        "machine1"
+        "machine2"
+      ];
+    };
+}

lib/default.nix

@@ -6,7 +6,7 @@
 }:
 {
   evalClanModules = import ./eval-clan-modules { inherit clan-core nixpkgs lib; };
-  buildClan = import ./build-clan { inherit clan-core lib nixpkgs; };
+  buildClan = import ./build-clan { inherit lib nixpkgs clan-core; };
   facts = import ./facts.nix { inherit lib; };
   inventory = import ./inventory { inherit lib clan-core; };
   jsonschema = import ./jsonschema { inherit lib; };

lib/flake-module.nix

@@ -8,6 +8,7 @@
   imports = [
     ./jsonschema/flake-module.nix
     ./inventory/flake-module.nix
+    ./build-clan/flake-module.nix
   ];
   flake.lib = import ./default.nix {
     inherit lib inputs;

nixosModules/clanCore/meta/interface.nix

@@ -6,5 +6,4 @@ in
   options.clan.meta.name = lib.mkOption { type = lib.types.str; };
   options.clan.meta.description = lib.mkOption { type = optStr; };
   options.clan.meta.icon = lib.mkOption { type = optStr; };
-  options.clan.tags = lib.mkOption { type = lib.types.listOf lib.types.str; };
 }

pkgs/clan-cli/clan_cli/clan/inspect.py

@@ -6,7 +6,7 @@ from ..clan_uri import FlakeId
 from ..cmd import run
 from ..dirs import machine_gcroot
 from ..errors import ClanError
-from ..machines.list import list_machines
+from ..machines.list import list_nixos_machines
 from ..machines.machines import Machine
 from ..nix import nix_add_to_gcroots, nix_build, nix_config, nix_eval, nix_metadata
 from ..vms.inspect import VmConfig, inspect_vm
@@ -40,7 +40,7 @@ def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig:
     system = config["system"]
 
     # Check if the machine exists
-    machines = list_machines(flake_url, False)
+    machines: list[str] = list_nixos_machines(flake_url, False)
     if machine_name not in machines:
         raise ClanError(
             f"Machine {machine_name} not found in {flake_url}. Available machines: {', '.join(machines)}"

pkgs/clan-cli/clan_cli/history/add.py

@@ -7,7 +7,7 @@ import logging
 from typing import Any
 
 from clan_cli.clan.inspect import FlakeConfig, inspect_flake
-from clan_cli.machines.list import list_machines
+from clan_cli.machines.list import list_nixos_machines
 
 from ..clan_uri import ClanURI
 from ..dirs import user_history_file
@@ -72,7 +72,7 @@ def new_history_entry(url: str, machine: str) -> HistoryEntry:
 def add_all_to_history(uri: ClanURI) -> list[HistoryEntry]:
     history = list_history()
     new_entries: list[HistoryEntry] = []
-    for machine in list_machines(uri.get_url()):
+    for machine in list_nixos_machines(uri.get_url()):
         new_entry = _add_maschine_to_history_list(uri.get_url(), machine, history)
         new_entries.append(new_entry)
     write_history_file(history)

pkgs/clan-cli/clan_cli/machines/list.py

@@ -1,22 +1,48 @@
 import argparse
+import json
 import logging
 from pathlib import Path
 
 from clan_cli.api import API
+from clan_cli.cmd import run_no_stdout
+from clan_cli.errors import ClanError
 from clan_cli.inventory import Machine, load_inventory_eval
+from clan_cli.nix import nix_eval
 
 log = logging.getLogger(__name__)
 
 
 @API.register
-def list_machines(flake_url: str | Path, debug: bool = False) -> dict[str, Machine]:
+def list_inventory_machines(
+    flake_url: str | Path, debug: bool = False
+) -> dict[str, Machine]:
     inventory = load_inventory_eval(flake_url)
     return inventory.machines
 
 
+@API.register
+def list_nixos_machines(flake_url: str | Path, debug: bool = False) -> list[str]:
+    cmd = nix_eval(
+        [
+            f"{flake_url}#nixosConfigurations",
+            "--apply",
+            "builtins.attrNames",
+            "--json",
+        ]
+    )
+    proc = run_no_stdout(cmd)
+
+    try:
+        res = proc.stdout.strip()
+        data = json.loads(res)
+        return data
+    except json.JSONDecodeError as e:
+        raise ClanError(f"Error decoding machines from flake: {e}")
+
+
 def list_command(args: argparse.Namespace) -> None:
     flake_path = args.flake.path
-    for name in list_machines(flake_path, args.debug).keys():
+    for name in list_nixos_machines(flake_path, args.debug):
         print(name)
 
 

pkgs/clan-cli/clan_cli/secrets/machines.py

@@ -47,10 +47,16 @@ def get_machine(flake_dir: Path, name: str) -> str:
 
 
 def has_machine(flake_dir: Path, name: str) -> bool:
+    """
+    Checks if a machine exists in the sops machines folder
+    """
     return (sops_machines_folder(flake_dir) / name / "key.json").exists()
 
 
-def list_machines(flake_dir: Path) -> list[str]:
+def list_sops_machines(flake_dir: Path) -> list[str]:
+    """
+    Lists all machines in the sops machines folder
+    """
     path = sops_machines_folder(flake_dir)
 
     def validate(name: str) -> bool:
@@ -86,7 +92,7 @@ def remove_secret(flake_dir: Path, machine: str, secret: str) -> None:
 def list_command(args: argparse.Namespace) -> None:
     if args.flake is None:
         raise ClanError("Could not find clan flake toplevel directory")
-    lst = list_machines(args.flake.path)
+    lst = list_sops_machines(args.flake.path)
     if len(lst) > 0:
         print("\n".join(lst))
 

pkgs/clan-vm-manager/clan_vm_manager/views/list.py

@@ -113,7 +113,7 @@ class ClanList(Gtk.Box):
 
         # menu_model = Gio.Menu()
         # TODO: Make this lazy, blocks UI startup for too long
-        # for vm in machines.list.list_machines(flake_url=vm.data.flake.flake_url):
+        # for vm in machines.list.list_nixos_machines(flake_url=vm.data.flake.flake_url):
         #     if vm not in vm_store:
         #         menu_model.append(vm, f"app.add::{vm}")
 

pkgs/webview-ui/app/src/components/MachineListItem.tsx

@@ -1,7 +1,7 @@
 import { createSignal, Match, Show, Switch } from "solid-js";
 import { ErrorData, pyApi, SuccessData } from "../api";
 
-type MachineDetails = SuccessData<"list_machines">["data"][string];
+type MachineDetails = SuccessData<"list_inventory_machines">["data"][string];
 
 interface MachineListItemProps {
   name: string;

pkgs/webview-ui/app/src/routes/machines/view.tsx

@@ -23,7 +23,7 @@ import { MachineListItem } from "@/src/components/MachineListItem";
 // >["data"]["services"];
 
 type MachinesModel = Extract<
-  OperationResponse<"list_machines">,
+  OperationResponse<"list_inventory_machines">,
   { status: "success" }
 >["data"];
 
@@ -63,7 +63,7 @@ export const MachineListView: Component = () => {
       return;
     }
     setLoading(true);
-    const response = await callApi("list_machines", {
+    const response = await callApi("list_inventory_machines", {
       flake_url: uri,
     });
     setLoading(false);

pkgs/webview-ui/app/tests/types.test.ts

@@ -29,7 +29,7 @@ describe.concurrent("API types work properly", () => {
   });
 
   it("Machine list receives a records of names and machine info.", async () => {
-    expectTypeOf(pyApi.list_machines.receive)
+    expectTypeOf(pyApi.list_inventory_machines.receive)
       .parameter(0)
       .parameter(0)
       .toMatchTypeOf<

templates/flake-module.nix

@@ -17,8 +17,10 @@
       initialized = inputs.nixpkgs.legacyPackages.x86_64-linux.runCommand "minimal-clan-flake" { } ''
         mkdir $out
         cp -r ${path}/* $out
-        mkdir -p $out/machines/foo
+        rm $out/inventory.json
-        echo '{ "nixpkgs": { "hostPlatform": "x86_64-linux" } }' > $out/machines/foo/settings.json
+
+        # TODO: Instead create a machine by calling the API, this wont break in future tests and is much closer to what the user performs
+        echo '{ "machines": { "foo": {  "name": "foo" } } }' > $out/inventory.json
       '';
       evaled = (import "${initialized}/flake.nix").outputs {
         self = evaled // {