From 8b0212b8281a90c7a094c670b7bddf9ac9aca010 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 31 Jul 2024 18:37:17 +0200 Subject: [PATCH] Add build-clan module --- lib/build-clan/default.nix | 327 +++-------------------- lib/build-clan/eval.nix | 18 ++ lib/build-clan/flake-module.nix | 41 +++ lib/build-clan/interface.nix | 68 +++++ lib/build-clan/module.nix | 192 +++++++++++++ lib/build-clan/old_default.nix | 306 +++++++++++++++++++++ lib/build-clan/tests.nix | 135 ++++++++++ lib/default.nix | 2 +- lib/flake-module.nix | 1 + nixosModules/clanCore/meta/interface.nix | 1 - pkgs/clan-cli/clan_cli/machines/list.py | 26 +- 11 files changed, 820 insertions(+), 297 deletions(-) create mode 100644 lib/build-clan/eval.nix create mode 100644 lib/build-clan/flake-module.nix create mode 100644 lib/build-clan/interface.nix create mode 100644 lib/build-clan/module.nix create mode 100644 lib/build-clan/old_default.nix create mode 100644 lib/build-clan/tests.nix diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 01808d6db..23875981e 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -1,306 +1,45 @@ +## WARNING: Do not add core logic here. +## This is only a wrapper such that buildClan can be called as a function. +## Add any logic to ./module.nix { - clan-core, - nixpkgs, lib, + nixpkgs, + clan-core, }: { - directory, # The directory containing the machines subdirectory - specialArgs ? { }, # Extra arguments to pass to nixosSystem i.e. useful to make self available - machines ? { }, # allows to include machine-specific modules i.e. machines.${name} = { ... } - # DEPRECATED: use meta.name instead - clanName ? null, # Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to. - # DEPRECATED: use meta.icon instead - clanIcon ? null, # A path to an icon to be used for the clan, should be the same for all machines - meta ? { }, # A set containing clan meta: name :: string, icon :: string, description :: string + ## Inputs + directory, # The directory containing the machines subdirectory # allows to include machine-specific modules i.e. machines.${name} = { ... } # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system. # This improves performance, but all nipxkgs.* options will be ignored. - pkgsForSystem ? (_system: null), - /* - Low level inventory configuration. - Overrides the services configuration. - */ inventory ? { }, -}: + ## Sepcial inputs (not passed to the module system as config) + specialArgs ? { }, # Extra arguments to pass to nixosSystem i.e. useful to make self available # A set containing clan meta: name :: string, icon :: string, description :: string + ## + ... +}@attrs: let - # Internal inventory, this is the result of merging all potential inventory sources: - # - Default instances configured via 'services' - # - The inventory overrides - # - Machines that exist in inventory.machines - # - Machines explicitly configured via 'machines' argument - # - Machines that exist in the machines directory - # Checks on the module level: - # - Each service role must reference a valid machine after all machines are merged - - clanToInventory = - config: - { clanPath, inventoryPath }: - let - v = lib.attrByPath clanPath null config; - in - lib.optionalAttrs (v != null) (lib.setAttrByPath inventoryPath v); - - mergedInventory = - (lib.evalModules { - modules = [ - clan-core.lib.inventory.interface - { inherit meta; } - ( - if - builtins.pathExists "${directory}/inventory.json" - # Is recursively applied. Any explicit nix will override. - then - (builtins.fromJSON (builtins.readFile "${directory}/inventory.json")) - else - { } - ) - inventory - # Machines explicitly configured via 'machines' argument - { - # { ${name} :: meta // { name, tags } } - machines = lib.mapAttrs ( - name: machineConfig: - (lib.attrByPath [ - "clan" - "meta" - ] { } machineConfig) - // { - # meta.name default is the attribute name of the machine - name = lib.mkDefault ( - lib.attrByPath [ - "clan" - "meta" - "name" - ] name machineConfig - ); - } - # tags - // (clanToInventory machineConfig { - clanPath = [ - "clan" - "tags" - ]; - inventoryPath = [ "tags" ]; - }) - # system - // (clanToInventory machineConfig { - clanPath = [ - "nixpkgs" - "hostPlatform" - ]; - inventoryPath = [ "system" ]; - }) - # deploy.targetHost - // (clanToInventory machineConfig { - clanPath = [ - "clan" - "core" - "networking" - "targetHost" - ]; - inventoryPath = [ - "deploy" - "targetHost" - ]; - }) - ) machines; - } - - # Will be deprecated - { - machines = - lib.mapAttrs - ( - name: _: - # Use mkForce to make sure users migrate to the inventory system. - # When the settings.json exists the evaluation will print the deprecation warning. - lib.mkForce { - inherit name; - system = (machineSettings name).nixpkgs.hostSystem or null; - } - ) - ( - lib.filterAttrs ( - machineName: _: builtins.pathExists "${directory}/machines/${machineName}/settings.json" - ) machinesDirs - ); - } - - # Deprecated interface - (if clanName != null then { meta.name = clanName; } else { }) - (if clanIcon != null then { meta.icon = clanIcon; } else { }) - ]; - }).config; - - inherit (clan-core.lib.inventory) buildInventory; - - # map from machine name to service configuration - # { ${machineName} :: Config } - serviceConfigs = buildInventory { - inventory = mergedInventory; - inherit directory; - }; - - machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") ( - builtins.readDir (directory + /machines) - ); - - machineSettings = - machineName: - let - warn = lib.warn '' - The use of ./machines//settings.json is deprecated. - If your settings.json is empty, you can safely remove it. - !!! Consider using the inventory system. !!! - - File: ${directory + /machines/${machineName}/settings.json} - - If there are still features missing in the inventory system, please open an issue on the clan-core repository. - ''; - in - # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily - # This is useful for doing a dry-run before writing changes into the settings.json - # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval - if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then - warn (builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE"))) - else - lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") ( - warn (builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json))) - ); - - machineImports = - machineSettings: map (module: clan-core.clanModules.${module}) (machineSettings.clanImports or [ ]); - - deprecationWarnings = [ - (lib.warnIf ( - clanName != null - ) "clanName in buildClan is deprecated, please use meta.name instead." null) - (lib.warnIf (clanIcon != null) "clanIcon is deprecated, please use meta.icon instead" null) + eval = import ./eval.nix { + inherit + lib + nixpkgs + specialArgs + clan-core + ; + } { self = directory; }; + meta = attrs.meta or { }; + rest = builtins.removeAttrs attrs [ + "meta" + "specialArgs" ]; - - # TODO: remove default system once we have a hardware-config mechanism - nixosConfiguration = - { - system ? "x86_64-linux", - name, - pkgs ? null, - extraConfig ? { }, - }: - nixpkgs.lib.nixosSystem { - modules = - let - settings = machineSettings name; - in - (machineImports settings) - ++ [ - { - # Autoinclude configuration.nix and hardware-configuration.nix - imports = builtins.filter (p: builtins.pathExists p) [ - "${directory}/machines/${name}/configuration.nix" - "${directory}/machines/${name}/hardware-configuration.nix" - ]; - } - settings - clan-core.nixosModules.clanCore - extraConfig - (machines.${name} or { }) - # Inherit the inventory assertions ? - { inherit (mergedInventory) assertions; } - { imports = serviceConfigs.${name} or { }; } - ( - { - # Settings - clan.core.clanDir = directory; - # Inherited from clan wide settings - clan.core.clanName = meta.name or clanName; - clan.core.clanIcon = meta.icon or clanIcon; - - # Machine specific settings - clan.core.machineName = name; - networking.hostName = lib.mkDefault name; - nixpkgs.hostPlatform = lib.mkDefault system; - - # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs) - nix.registry.nixpkgs.to = { - type = "path"; - path = lib.mkDefault nixpkgs; - }; - } - // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; } - ) - ]; - specialArgs = { - inherit clan-core; - } // specialArgs; - }; - - allMachines = mergedInventory.machines or { }; - - supportedSystems = [ - "x86_64-linux" - "aarch64-linux" - "riscv64-linux" - "x86_64-darwin" - "aarch64-darwin" - ]; - - nixosConfigurations = lib.mapAttrs (name: _: nixosConfiguration { inherit name; }) allMachines; - - # This instantiates nixos for each system that we support: - # configPerSystem = ..nixosConfiguration - # We need this to build nixos secret generators for each system - configsPerSystem = builtins.listToAttrs ( - builtins.map ( - system: - lib.nameValuePair system ( - lib.mapAttrs ( - name: _: - nixosConfiguration { - inherit name system; - pkgs = pkgsForSystem system; - } - ) allMachines - ) - ) supportedSystems - ); - - configsFuncPerSystem = builtins.listToAttrs ( - builtins.map ( - system: - lib.nameValuePair system ( - lib.mapAttrs ( - name: _: args: - nixosConfiguration ( - args - // { - inherit name system; - pkgs = pkgsForSystem system; - } - ) - ) allMachines - ) - ) supportedSystems - ); in -builtins.deepSeq deprecationWarnings { - inherit nixosConfigurations; +eval { + inventory.meta = lib.mapAttrs (_: lib.mkDefault) meta; + imports = [ + rest + # implementation + ./module.nix - clanInternals = { - inherit (clan-core) clanModules; - source = "${clan-core}"; - - meta = mergedInventory.meta; - inventory = mergedInventory; - - inventoryFile = "${directory}/inventory.json"; - - # machine specifics - machines = configsPerSystem; - machinesFunc = configsFuncPerSystem; - all-machines-json = lib.mapAttrs ( - system: configs: - nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" ( - lib.mapAttrs (_: m: m.config.system.clan.deployment.data) configs - ) - ) configsPerSystem; - }; + # Explicit output, usually defined by flake-parts + { options.nixosConfigurations = lib.mkOption { type = lib.types.raw; }; } + ]; } diff --git a/lib/build-clan/eval.nix b/lib/build-clan/eval.nix new file mode 100644 index 000000000..a1ed4a623 --- /dev/null +++ b/lib/build-clan/eval.nix @@ -0,0 +1,18 @@ +{ + lib, + nixpkgs, + clan-core, + specialArgs ? { }, +}: +# Returns a function that takes self, which should point to the directory of the flake +{ self }: +module: +(lib.evalModules { + specialArgs = { + inherit self clan-core nixpkgs; + } // specialArgs; + modules = [ + ./interface.nix + module + ]; +}).config diff --git a/lib/build-clan/flake-module.nix b/lib/build-clan/flake-module.nix new file mode 100644 index 000000000..87dfa7c92 --- /dev/null +++ b/lib/build-clan/flake-module.nix @@ -0,0 +1,41 @@ +{ self, inputs, ... }: +let + inputOverrides = builtins.concatStringsSep " " ( + builtins.map (input: " --override-input ${input} ${inputs.${input}}") (builtins.attrNames inputs) + ); +in +{ + + perSystem = + { + pkgs, + lib, + system, + ... + }: + # let + + # in + { + + # Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests + legacyPackages.evalTests-build-clan = import ./tests.nix { + inherit lib; + inherit (inputs) nixpkgs; + clan-core = self; + buildClan = self.lib.buildClan; + }; + checks = { + lib-inventory-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } '' + export HOME="$(realpath .)" + + nix-unit --eval-store "$HOME" \ + --extra-experimental-features flakes \ + ${inputOverrides} \ + --flake ${self}#legacyPackages.${system}.evalTests-build-clan + + touch $out + ''; + }; + }; +} diff --git a/lib/build-clan/interface.nix b/lib/build-clan/interface.nix new file mode 100644 index 000000000..169a0bb6f --- /dev/null +++ b/lib/build-clan/interface.nix @@ -0,0 +1,68 @@ +{ lib, ... }: +let + types = lib.types; +in +{ + options = { + # Meta + meta = { + name = lib.mkOption { + type = types.nullOr types.str; + default = null; + description = "Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to."; + }; + icon = lib.mkOption { + type = types.nullOr types.path; + default = null; + description = "A path to an icon to be used for the clan in the GUI"; + }; + description = lib.mkOption { + type = types.nullOr types.str; + default = null; + description = "A short description of the clan"; + }; + }; + + # Required options + directory = lib.mkOption { + type = types.path; + description = "The directory containing the clan subdirectory"; + }; + specialArgs = lib.mkOption { + type = types.attrsOf types.raw; + default = { }; + description = "Extra arguments to pass to nixosSystem i.e. useful to make self available"; + }; + + # Optional + machines = lib.mkOption { + type = types.attrsOf types.deferredModule; + default = { }; + }; + pkgsForSystem = lib.mkOption { + type = types.functionTo (types.nullOr types.attrs); + default = _: null; + }; + inventory = lib.mkOption { + type = types.submodule { imports = [ ../inventory/build-inventory/interface.nix ]; }; + }; + + # Outputs + # flake.clanInternals + clanInternals = lib.mkOption { + type = types.submodule { + options = { + # Those options are interfaced by the CLI + inventory = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; + inventoryFile = lib.mkOption { type = lib.types.unspecified; }; + clanModules = lib.mkOption { type = lib.types.attrsOf lib.types.path; }; + source = lib.mkOption { type = lib.types.path; }; + meta = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; + all-machines-json = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; + machines = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); }; + machinesFunc = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); }; + }; + }; + }; + }; +} diff --git a/lib/build-clan/module.nix b/lib/build-clan/module.nix new file mode 100644 index 000000000..b868230bb --- /dev/null +++ b/lib/build-clan/module.nix @@ -0,0 +1,192 @@ +{ + config, + clan-core, + nixpkgs, + lib, + specialArgs ? { }, + ... +}: +let + inherit (config) directory machines pkgsForSystem; + + # Final inventory + inherit (config.clanInternals) inventory; + + inherit (clan-core.lib.inventory) buildInventory; + + # map from machine name to service configuration + # { ${machineName} :: Config } + serviceConfigs = ( + buildInventory { + inherit inventory; + inherit directory; + } + ); + + machineSettings = + machineName: + let + warn = lib.warn '' + The use of ./machines//settings.json is deprecated. + If your settings.json is empty, you can safely remove it. + !!! Consider using the inventory system. !!! + + File: ${directory + /machines/${machineName}/settings.json} + + If there are still features missing in the inventory system, please open an issue on the clan-core repository. + ''; + in + # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily + # This is useful for doing a dry-run before writing changes into the settings.json + # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval + if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then + warn (builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE"))) + else + lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") ( + warn (builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json))) + ); + + machineImports = + machineSettings: map (module: clan-core.clanModules.${module}) (machineSettings.clanImports or [ ]); + + # TODO: remove default system once we have a hardware-config mechanism + nixosConfiguration = + { + system ? "x86_64-linux", + name, + pkgs ? null, + extraConfig ? { }, + }: + nixpkgs.lib.nixosSystem { + modules = + let + settings = machineSettings name; + in + (machineImports settings) + ++ [ + { + # Autoinclude configuration.nix and hardware-configuration.nix + imports = builtins.filter (p: builtins.pathExists p) [ + "${directory}/machines/${name}/configuration.nix" + "${directory}/machines/${name}/hardware-configuration.nix" + ]; + } + settings + clan-core.nixosModules.clanCore + extraConfig + (machines.${name} or { }) + # Inherit the inventory assertions ? + # { inherit (mergedInventory) assertions; } + { imports = serviceConfigs.${name} or { }; } + ( + { + # Settings + clan.core.clanDir = directory; + # Inherited from clan wide settings + # clan.core.clanName = meta.name or clanName; + # clan.core.clanIcon = meta.icon or clanIcon; + + # Machine specific settings + clan.core.machineName = name; + networking.hostName = lib.mkDefault name; + nixpkgs.hostPlatform = lib.mkDefault system; + + # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs) + nix.registry.nixpkgs.to = { + type = "path"; + path = lib.mkDefault nixpkgs; + }; + } + // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; } + ) + ]; + specialArgs = { + inherit clan-core; + } // specialArgs; + }; + + allMachines = (inventory.machines or { } // config.machines or { }); + + supportedSystems = [ + "x86_64-linux" + "aarch64-linux" + "riscv64-linux" + "x86_64-darwin" + "aarch64-darwin" + ]; + + nixosConfigurations = lib.mapAttrs (name: _: nixosConfiguration { inherit name; }) allMachines; + + # This instantiates nixos for each system that we support: + # configPerSystem = ..nixosConfiguration + # We need this to build nixos secret generators for each system + configsPerSystem = builtins.listToAttrs ( + builtins.map ( + system: + lib.nameValuePair system ( + lib.mapAttrs ( + name: _: + nixosConfiguration { + inherit name system; + pkgs = pkgsForSystem system; + } + ) allMachines + ) + ) supportedSystems + ); + + configsFuncPerSystem = builtins.listToAttrs ( + builtins.map ( + system: + lib.nameValuePair system ( + lib.mapAttrs ( + name: _: args: + nixosConfiguration ( + args + // { + inherit name system; + pkgs = pkgsForSystem system; + } + ) + ) allMachines + ) + ) supportedSystems + ); + + inventoryFile = "${directory}/inventory.json"; + + inventoryLoaded = + if builtins.pathExists inventoryFile then + (builtins.fromJSON (builtins.readFile inventoryFile)) + else + { }; + +in +{ + imports = [ + # Merge the inventory file + { clanInternals.inventory = inventoryLoaded; } + { clanInternals.inventory = config.inventory; } + # Derived meta from the merged inventory + { clanInternals.meta = config.clanInternals.inventory.meta; } + ]; + + inherit nixosConfigurations; + + clanInternals = { + inherit (clan-core) clanModules; + inherit inventoryFile; + + source = "${clan-core}"; + + # machine specifics + machines = configsPerSystem; + machinesFunc = configsFuncPerSystem; + all-machines-json = lib.mapAttrs ( + system: configs: + nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" ( + lib.mapAttrs (_: m: m.config.system.clan.deployment.data) configs + ) + ) configsPerSystem; + }; +} diff --git a/lib/build-clan/old_default.nix b/lib/build-clan/old_default.nix new file mode 100644 index 000000000..01808d6db --- /dev/null +++ b/lib/build-clan/old_default.nix @@ -0,0 +1,306 @@ +{ + clan-core, + nixpkgs, + lib, +}: +{ + directory, # The directory containing the machines subdirectory + specialArgs ? { }, # Extra arguments to pass to nixosSystem i.e. useful to make self available + machines ? { }, # allows to include machine-specific modules i.e. machines.${name} = { ... } + # DEPRECATED: use meta.name instead + clanName ? null, # Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to. + # DEPRECATED: use meta.icon instead + clanIcon ? null, # A path to an icon to be used for the clan, should be the same for all machines + meta ? { }, # A set containing clan meta: name :: string, icon :: string, description :: string + # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system. + # This improves performance, but all nipxkgs.* options will be ignored. + pkgsForSystem ? (_system: null), + /* + Low level inventory configuration. + Overrides the services configuration. + */ + inventory ? { }, +}: +let + # Internal inventory, this is the result of merging all potential inventory sources: + # - Default instances configured via 'services' + # - The inventory overrides + # - Machines that exist in inventory.machines + # - Machines explicitly configured via 'machines' argument + # - Machines that exist in the machines directory + # Checks on the module level: + # - Each service role must reference a valid machine after all machines are merged + + clanToInventory = + config: + { clanPath, inventoryPath }: + let + v = lib.attrByPath clanPath null config; + in + lib.optionalAttrs (v != null) (lib.setAttrByPath inventoryPath v); + + mergedInventory = + (lib.evalModules { + modules = [ + clan-core.lib.inventory.interface + { inherit meta; } + ( + if + builtins.pathExists "${directory}/inventory.json" + # Is recursively applied. Any explicit nix will override. + then + (builtins.fromJSON (builtins.readFile "${directory}/inventory.json")) + else + { } + ) + inventory + # Machines explicitly configured via 'machines' argument + { + # { ${name} :: meta // { name, tags } } + machines = lib.mapAttrs ( + name: machineConfig: + (lib.attrByPath [ + "clan" + "meta" + ] { } machineConfig) + // { + # meta.name default is the attribute name of the machine + name = lib.mkDefault ( + lib.attrByPath [ + "clan" + "meta" + "name" + ] name machineConfig + ); + } + # tags + // (clanToInventory machineConfig { + clanPath = [ + "clan" + "tags" + ]; + inventoryPath = [ "tags" ]; + }) + # system + // (clanToInventory machineConfig { + clanPath = [ + "nixpkgs" + "hostPlatform" + ]; + inventoryPath = [ "system" ]; + }) + # deploy.targetHost + // (clanToInventory machineConfig { + clanPath = [ + "clan" + "core" + "networking" + "targetHost" + ]; + inventoryPath = [ + "deploy" + "targetHost" + ]; + }) + ) machines; + } + + # Will be deprecated + { + machines = + lib.mapAttrs + ( + name: _: + # Use mkForce to make sure users migrate to the inventory system. + # When the settings.json exists the evaluation will print the deprecation warning. + lib.mkForce { + inherit name; + system = (machineSettings name).nixpkgs.hostSystem or null; + } + ) + ( + lib.filterAttrs ( + machineName: _: builtins.pathExists "${directory}/machines/${machineName}/settings.json" + ) machinesDirs + ); + } + + # Deprecated interface + (if clanName != null then { meta.name = clanName; } else { }) + (if clanIcon != null then { meta.icon = clanIcon; } else { }) + ]; + }).config; + + inherit (clan-core.lib.inventory) buildInventory; + + # map from machine name to service configuration + # { ${machineName} :: Config } + serviceConfigs = buildInventory { + inventory = mergedInventory; + inherit directory; + }; + + machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") ( + builtins.readDir (directory + /machines) + ); + + machineSettings = + machineName: + let + warn = lib.warn '' + The use of ./machines//settings.json is deprecated. + If your settings.json is empty, you can safely remove it. + !!! Consider using the inventory system. !!! + + File: ${directory + /machines/${machineName}/settings.json} + + If there are still features missing in the inventory system, please open an issue on the clan-core repository. + ''; + in + # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily + # This is useful for doing a dry-run before writing changes into the settings.json + # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval + if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then + warn (builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE"))) + else + lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") ( + warn (builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json))) + ); + + machineImports = + machineSettings: map (module: clan-core.clanModules.${module}) (machineSettings.clanImports or [ ]); + + deprecationWarnings = [ + (lib.warnIf ( + clanName != null + ) "clanName in buildClan is deprecated, please use meta.name instead." null) + (lib.warnIf (clanIcon != null) "clanIcon is deprecated, please use meta.icon instead" null) + ]; + + # TODO: remove default system once we have a hardware-config mechanism + nixosConfiguration = + { + system ? "x86_64-linux", + name, + pkgs ? null, + extraConfig ? { }, + }: + nixpkgs.lib.nixosSystem { + modules = + let + settings = machineSettings name; + in + (machineImports settings) + ++ [ + { + # Autoinclude configuration.nix and hardware-configuration.nix + imports = builtins.filter (p: builtins.pathExists p) [ + "${directory}/machines/${name}/configuration.nix" + "${directory}/machines/${name}/hardware-configuration.nix" + ]; + } + settings + clan-core.nixosModules.clanCore + extraConfig + (machines.${name} or { }) + # Inherit the inventory assertions ? + { inherit (mergedInventory) assertions; } + { imports = serviceConfigs.${name} or { }; } + ( + { + # Settings + clan.core.clanDir = directory; + # Inherited from clan wide settings + clan.core.clanName = meta.name or clanName; + clan.core.clanIcon = meta.icon or clanIcon; + + # Machine specific settings + clan.core.machineName = name; + networking.hostName = lib.mkDefault name; + nixpkgs.hostPlatform = lib.mkDefault system; + + # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs) + nix.registry.nixpkgs.to = { + type = "path"; + path = lib.mkDefault nixpkgs; + }; + } + // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; } + ) + ]; + specialArgs = { + inherit clan-core; + } // specialArgs; + }; + + allMachines = mergedInventory.machines or { }; + + supportedSystems = [ + "x86_64-linux" + "aarch64-linux" + "riscv64-linux" + "x86_64-darwin" + "aarch64-darwin" + ]; + + nixosConfigurations = lib.mapAttrs (name: _: nixosConfiguration { inherit name; }) allMachines; + + # This instantiates nixos for each system that we support: + # configPerSystem = ..nixosConfiguration + # We need this to build nixos secret generators for each system + configsPerSystem = builtins.listToAttrs ( + builtins.map ( + system: + lib.nameValuePair system ( + lib.mapAttrs ( + name: _: + nixosConfiguration { + inherit name system; + pkgs = pkgsForSystem system; + } + ) allMachines + ) + ) supportedSystems + ); + + configsFuncPerSystem = builtins.listToAttrs ( + builtins.map ( + system: + lib.nameValuePair system ( + lib.mapAttrs ( + name: _: args: + nixosConfiguration ( + args + // { + inherit name system; + pkgs = pkgsForSystem system; + } + ) + ) allMachines + ) + ) supportedSystems + ); +in +builtins.deepSeq deprecationWarnings { + inherit nixosConfigurations; + + clanInternals = { + inherit (clan-core) clanModules; + source = "${clan-core}"; + + meta = mergedInventory.meta; + inventory = mergedInventory; + + inventoryFile = "${directory}/inventory.json"; + + # machine specifics + machines = configsPerSystem; + machinesFunc = configsFuncPerSystem; + all-machines-json = lib.mapAttrs ( + system: configs: + nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" ( + lib.mapAttrs (_: m: m.config.system.clan.deployment.data) configs + ) + ) configsPerSystem; + }; +} diff --git a/lib/build-clan/tests.nix b/lib/build-clan/tests.nix new file mode 100644 index 000000000..366e795be --- /dev/null +++ b/lib/build-clan/tests.nix @@ -0,0 +1,135 @@ +{ + lib, + nixpkgs, + clan-core, + buildClan, + ... +}: +let + eval = import ./eval.nix { inherit lib nixpkgs clan-core; }; + + self = ./.; + evalClan = eval { inherit self; }; + +in +####### +{ + test_only_required = + let + config = evalClan { directory = ./.; }; + in + { + expr = config.pkgsForSystem null == null; + expected = true; + }; + + test_all_simple = + let + config = evalClan { + directory = ./.; + machines = { }; + inventory = { + meta.name = "test"; + }; + pkgsForSystem = _system: { }; + }; + in + { + expr = config ? inventory; + expected = true; + }; + + test_outputs_clanInternals = + let + config = evalClan { + imports = [ + # What the user needs to specif + { + directory = ./.; + inventory.meta.name = "test"; + } + + # Build-clan implementation + ./module.nix + # Explicit output, usually defined by flake-parts + { options.nixosConfigurations = lib.mkOption { type = lib.types.raw; }; } + ]; + }; + in + { + expr = config.clanInternals.meta; + expected = { + description = null; + icon = null; + name = "test"; + }; + }; + + test_fn_simple = + let + result = buildClan { + directory = ./.; + meta.name = "test"; + }; + in + { + expr = result.clanInternals.meta; + expected = { + description = null; + icon = null; + name = "test"; + }; + }; + + test_fn_extensiv_meta = + let + result = buildClan { + directory = ./.; + meta.name = "test"; + meta.description = "test"; + meta.icon = "test"; + inventory.meta.name = "superclan"; + inventory.meta.description = "description"; + inventory.meta.icon = "icon"; + }; + in + { + expr = result.clanInternals.meta; + expected = { + description = "description"; + icon = "icon"; + name = "superclan"; + }; + }; + + test_fn_clan_core = + let + result = buildClan { + directory = ../../.; + meta.name = "test-clan-core"; + }; + in + { + expr = builtins.attrNames result.nixosConfigurations; + expected = [ "test-inventory-machine" ]; + }; + + test_buildClan_all_machines = + let + result = buildClan { + directory = ./.; + meta.name = "test"; + inventory.machines.machine1.meta.name = "machine1"; + + machines.machine2 = { }; + + }; + in + { + expr = builtins.attrNames result.nixosConfigurations; + expected = [ + "machine1" + "machine2" + ]; + }; +} diff --git a/lib/default.nix b/lib/default.nix index 91a23d6df..23556ee91 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -6,7 +6,7 @@ }: { evalClanModules = import ./eval-clan-modules { inherit clan-core nixpkgs lib; }; - buildClan = import ./build-clan { inherit clan-core lib nixpkgs; }; + buildClan = import ./build-clan { inherit lib nixpkgs clan-core; }; facts = import ./facts.nix { inherit lib; }; inventory = import ./inventory { inherit lib clan-core; }; jsonschema = import ./jsonschema { inherit lib; }; diff --git a/lib/flake-module.nix b/lib/flake-module.nix index 0785872a5..e9ab69a54 100644 --- a/lib/flake-module.nix +++ b/lib/flake-module.nix @@ -8,6 +8,7 @@ imports = [ ./jsonschema/flake-module.nix ./inventory/flake-module.nix + ./build-clan/flake-module.nix ]; flake.lib = import ./default.nix { inherit lib inputs; diff --git a/nixosModules/clanCore/meta/interface.nix b/nixosModules/clanCore/meta/interface.nix index 3b44046e7..aef373d57 100644 --- a/nixosModules/clanCore/meta/interface.nix +++ b/nixosModules/clanCore/meta/interface.nix @@ -6,5 +6,4 @@ in options.clan.meta.name = lib.mkOption { type = lib.types.str; }; options.clan.meta.description = lib.mkOption { type = optStr; }; options.clan.meta.icon = lib.mkOption { type = optStr; }; - options.clan.tags = lib.mkOption { type = lib.types.listOf lib.types.str; }; } diff --git a/pkgs/clan-cli/clan_cli/machines/list.py b/pkgs/clan-cli/clan_cli/machines/list.py index f8df311d5..ec3011a2a 100644 --- a/pkgs/clan-cli/clan_cli/machines/list.py +++ b/pkgs/clan-cli/clan_cli/machines/list.py @@ -1,9 +1,13 @@ import argparse +import json import logging from pathlib import Path from clan_cli.api import API +from clan_cli.cmd import run_no_stdout +from clan_cli.errors import ClanError from clan_cli.inventory import Machine, load_inventory_eval +from clan_cli.nix import nix_eval log = logging.getLogger(__name__) @@ -14,9 +18,29 @@ def list_machines(flake_url: str | Path, debug: bool = False) -> dict[str, Machi return inventory.machines +@API.register +def list_nixos_machines(flake_url: str | Path, debug: bool = False) -> list[str]: + cmd = nix_eval( + [ + f"{flake_url}#nixosConfigurations", + "--apply", + "builtins.attrNames", + "--json", + ] + ) + proc = run_no_stdout(cmd) + + try: + res = proc.stdout.strip() + data = json.loads(res) + return data + except json.JSONDecodeError as e: + raise ClanError(f"Error decoding machines from flake: {e}") + + def list_command(args: argparse.Namespace) -> None: flake_path = args.flake.path - for name in list_machines(flake_path, args.debug).keys(): + for name in list_nixos_machines(flake_path, args.debug): print(name)