From 8b0212b8281a90c7a094c670b7bddf9ac9aca010 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 31 Jul 2024 18:37:17 +0200 Subject: [PATCH 1/8] Add build-clan module --- lib/build-clan/default.nix | 327 +++-------------------- lib/build-clan/eval.nix | 18 ++ lib/build-clan/flake-module.nix | 41 +++ lib/build-clan/interface.nix | 68 +++++ lib/build-clan/module.nix | 192 +++++++++++++ lib/build-clan/old_default.nix | 306 +++++++++++++++++++++ lib/build-clan/tests.nix | 135 ++++++++++ lib/default.nix | 2 +- lib/flake-module.nix | 1 + nixosModules/clanCore/meta/interface.nix | 1 - pkgs/clan-cli/clan_cli/machines/list.py | 26 +- 11 files changed, 820 insertions(+), 297 deletions(-) create mode 100644 lib/build-clan/eval.nix create mode 100644 lib/build-clan/flake-module.nix create mode 100644 lib/build-clan/interface.nix create mode 100644 lib/build-clan/module.nix create mode 100644 lib/build-clan/old_default.nix create mode 100644 lib/build-clan/tests.nix diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 01808d6db..23875981e 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -1,306 +1,45 @@ +## WARNING: Do not add core logic here. +## This is only a wrapper such that buildClan can be called as a function. +## Add any logic to ./module.nix { - clan-core, - nixpkgs, lib, + nixpkgs, + clan-core, }: { - directory, # The directory containing the machines subdirectory - specialArgs ? { }, # Extra arguments to pass to nixosSystem i.e. useful to make self available - machines ? { }, # allows to include machine-specific modules i.e. machines.${name} = { ... } - # DEPRECATED: use meta.name instead - clanName ? null, # Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to. - # DEPRECATED: use meta.icon instead - clanIcon ? null, # A path to an icon to be used for the clan, should be the same for all machines - meta ? { }, # A set containing clan meta: name :: string, icon :: string, description :: string + ## Inputs + directory, # The directory containing the machines subdirectory # allows to include machine-specific modules i.e. machines.${name} = { ... } # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system. # This improves performance, but all nipxkgs.* options will be ignored. - pkgsForSystem ? (_system: null), - /* - Low level inventory configuration. - Overrides the services configuration. - */ inventory ? { }, -}: + ## Sepcial inputs (not passed to the module system as config) + specialArgs ? { }, # Extra arguments to pass to nixosSystem i.e. useful to make self available # A set containing clan meta: name :: string, icon :: string, description :: string + ## + ... +}@attrs: let - # Internal inventory, this is the result of merging all potential inventory sources: - # - Default instances configured via 'services' - # - The inventory overrides - # - Machines that exist in inventory.machines - # - Machines explicitly configured via 'machines' argument - # - Machines that exist in the machines directory - # Checks on the module level: - # - Each service role must reference a valid machine after all machines are merged - - clanToInventory = - config: - { clanPath, inventoryPath }: - let - v = lib.attrByPath clanPath null config; - in - lib.optionalAttrs (v != null) (lib.setAttrByPath inventoryPath v); - - mergedInventory = - (lib.evalModules { - modules = [ - clan-core.lib.inventory.interface - { inherit meta; } - ( - if - builtins.pathExists "${directory}/inventory.json" - # Is recursively applied. Any explicit nix will override. - then - (builtins.fromJSON (builtins.readFile "${directory}/inventory.json")) - else - { } - ) - inventory - # Machines explicitly configured via 'machines' argument - { - # { ${name} :: meta // { name, tags } } - machines = lib.mapAttrs ( - name: machineConfig: - (lib.attrByPath [ - "clan" - "meta" - ] { } machineConfig) - // { - # meta.name default is the attribute name of the machine - name = lib.mkDefault ( - lib.attrByPath [ - "clan" - "meta" - "name" - ] name machineConfig - ); - } - # tags - // (clanToInventory machineConfig { - clanPath = [ - "clan" - "tags" - ]; - inventoryPath = [ "tags" ]; - }) - # system - // (clanToInventory machineConfig { - clanPath = [ - "nixpkgs" - "hostPlatform" - ]; - inventoryPath = [ "system" ]; - }) - # deploy.targetHost - // (clanToInventory machineConfig { - clanPath = [ - "clan" - "core" - "networking" - "targetHost" - ]; - inventoryPath = [ - "deploy" - "targetHost" - ]; - }) - ) machines; - } - - # Will be deprecated - { - machines = - lib.mapAttrs - ( - name: _: - # Use mkForce to make sure users migrate to the inventory system. - # When the settings.json exists the evaluation will print the deprecation warning. - lib.mkForce { - inherit name; - system = (machineSettings name).nixpkgs.hostSystem or null; - } - ) - ( - lib.filterAttrs ( - machineName: _: builtins.pathExists "${directory}/machines/${machineName}/settings.json" - ) machinesDirs - ); - } - - # Deprecated interface - (if clanName != null then { meta.name = clanName; } else { }) - (if clanIcon != null then { meta.icon = clanIcon; } else { }) - ]; - }).config; - - inherit (clan-core.lib.inventory) buildInventory; - - # map from machine name to service configuration - # { ${machineName} :: Config } - serviceConfigs = buildInventory { - inventory = mergedInventory; - inherit directory; - }; - - machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") ( - builtins.readDir (directory + /machines) - ); - - machineSettings = - machineName: - let - warn = lib.warn '' - The use of ./machines//settings.json is deprecated. - If your settings.json is empty, you can safely remove it. - !!! Consider using the inventory system. !!! - - File: ${directory + /machines/${machineName}/settings.json} - - If there are still features missing in the inventory system, please open an issue on the clan-core repository. - ''; - in - # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily - # This is useful for doing a dry-run before writing changes into the settings.json - # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval - if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then - warn (builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE"))) - else - lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") ( - warn (builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json))) - ); - - machineImports = - machineSettings: map (module: clan-core.clanModules.${module}) (machineSettings.clanImports or [ ]); - - deprecationWarnings = [ - (lib.warnIf ( - clanName != null - ) "clanName in buildClan is deprecated, please use meta.name instead." null) - (lib.warnIf (clanIcon != null) "clanIcon is deprecated, please use meta.icon instead" null) + eval = import ./eval.nix { + inherit + lib + nixpkgs + specialArgs + clan-core + ; + } { self = directory; }; + meta = attrs.meta or { }; + rest = builtins.removeAttrs attrs [ + "meta" + "specialArgs" ]; - - # TODO: remove default system once we have a hardware-config mechanism - nixosConfiguration = - { - system ? "x86_64-linux", - name, - pkgs ? null, - extraConfig ? { }, - }: - nixpkgs.lib.nixosSystem { - modules = - let - settings = machineSettings name; - in - (machineImports settings) - ++ [ - { - # Autoinclude configuration.nix and hardware-configuration.nix - imports = builtins.filter (p: builtins.pathExists p) [ - "${directory}/machines/${name}/configuration.nix" - "${directory}/machines/${name}/hardware-configuration.nix" - ]; - } - settings - clan-core.nixosModules.clanCore - extraConfig - (machines.${name} or { }) - # Inherit the inventory assertions ? - { inherit (mergedInventory) assertions; } - { imports = serviceConfigs.${name} or { }; } - ( - { - # Settings - clan.core.clanDir = directory; - # Inherited from clan wide settings - clan.core.clanName = meta.name or clanName; - clan.core.clanIcon = meta.icon or clanIcon; - - # Machine specific settings - clan.core.machineName = name; - networking.hostName = lib.mkDefault name; - nixpkgs.hostPlatform = lib.mkDefault system; - - # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs) - nix.registry.nixpkgs.to = { - type = "path"; - path = lib.mkDefault nixpkgs; - }; - } - // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; } - ) - ]; - specialArgs = { - inherit clan-core; - } // specialArgs; - }; - - allMachines = mergedInventory.machines or { }; - - supportedSystems = [ - "x86_64-linux" - "aarch64-linux" - "riscv64-linux" - "x86_64-darwin" - "aarch64-darwin" - ]; - - nixosConfigurations = lib.mapAttrs (name: _: nixosConfiguration { inherit name; }) allMachines; - - # This instantiates nixos for each system that we support: - # configPerSystem = ..nixosConfiguration - # We need this to build nixos secret generators for each system - configsPerSystem = builtins.listToAttrs ( - builtins.map ( - system: - lib.nameValuePair system ( - lib.mapAttrs ( - name: _: - nixosConfiguration { - inherit name system; - pkgs = pkgsForSystem system; - } - ) allMachines - ) - ) supportedSystems - ); - - configsFuncPerSystem = builtins.listToAttrs ( - builtins.map ( - system: - lib.nameValuePair system ( - lib.mapAttrs ( - name: _: args: - nixosConfiguration ( - args - // { - inherit name system; - pkgs = pkgsForSystem system; - } - ) - ) allMachines - ) - ) supportedSystems - ); in -builtins.deepSeq deprecationWarnings { - inherit nixosConfigurations; +eval { + inventory.meta = lib.mapAttrs (_: lib.mkDefault) meta; + imports = [ + rest + # implementation + ./module.nix - clanInternals = { - inherit (clan-core) clanModules; - source = "${clan-core}"; - - meta = mergedInventory.meta; - inventory = mergedInventory; - - inventoryFile = "${directory}/inventory.json"; - - # machine specifics - machines = configsPerSystem; - machinesFunc = configsFuncPerSystem; - all-machines-json = lib.mapAttrs ( - system: configs: - nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" ( - lib.mapAttrs (_: m: m.config.system.clan.deployment.data) configs - ) - ) configsPerSystem; - }; + # Explicit output, usually defined by flake-parts + { options.nixosConfigurations = lib.mkOption { type = lib.types.raw; }; } + ]; } diff --git a/lib/build-clan/eval.nix b/lib/build-clan/eval.nix new file mode 100644 index 000000000..a1ed4a623 --- /dev/null +++ b/lib/build-clan/eval.nix @@ -0,0 +1,18 @@ +{ + lib, + nixpkgs, + clan-core, + specialArgs ? { }, +}: +# Returns a function that takes self, which should point to the directory of the flake +{ self }: +module: +(lib.evalModules { + specialArgs = { + inherit self clan-core nixpkgs; + } // specialArgs; + modules = [ + ./interface.nix + module + ]; +}).config diff --git a/lib/build-clan/flake-module.nix b/lib/build-clan/flake-module.nix new file mode 100644 index 000000000..87dfa7c92 --- /dev/null +++ b/lib/build-clan/flake-module.nix @@ -0,0 +1,41 @@ +{ self, inputs, ... }: +let + inputOverrides = builtins.concatStringsSep " " ( + builtins.map (input: " --override-input ${input} ${inputs.${input}}") (builtins.attrNames inputs) + ); +in +{ + + perSystem = + { + pkgs, + lib, + system, + ... + }: + # let + + # in + { + + # Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests + legacyPackages.evalTests-build-clan = import ./tests.nix { + inherit lib; + inherit (inputs) nixpkgs; + clan-core = self; + buildClan = self.lib.buildClan; + }; + checks = { + lib-inventory-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } '' + export HOME="$(realpath .)" + + nix-unit --eval-store "$HOME" \ + --extra-experimental-features flakes \ + ${inputOverrides} \ + --flake ${self}#legacyPackages.${system}.evalTests-build-clan + + touch $out + ''; + }; + }; +} diff --git a/lib/build-clan/interface.nix b/lib/build-clan/interface.nix new file mode 100644 index 000000000..169a0bb6f --- /dev/null +++ b/lib/build-clan/interface.nix @@ -0,0 +1,68 @@ +{ lib, ... }: +let + types = lib.types; +in +{ + options = { + # Meta + meta = { + name = lib.mkOption { + type = types.nullOr types.str; + default = null; + description = "Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to."; + }; + icon = lib.mkOption { + type = types.nullOr types.path; + default = null; + description = "A path to an icon to be used for the clan in the GUI"; + }; + description = lib.mkOption { + type = types.nullOr types.str; + default = null; + description = "A short description of the clan"; + }; + }; + + # Required options + directory = lib.mkOption { + type = types.path; + description = "The directory containing the clan subdirectory"; + }; + specialArgs = lib.mkOption { + type = types.attrsOf types.raw; + default = { }; + description = "Extra arguments to pass to nixosSystem i.e. useful to make self available"; + }; + + # Optional + machines = lib.mkOption { + type = types.attrsOf types.deferredModule; + default = { }; + }; + pkgsForSystem = lib.mkOption { + type = types.functionTo (types.nullOr types.attrs); + default = _: null; + }; + inventory = lib.mkOption { + type = types.submodule { imports = [ ../inventory/build-inventory/interface.nix ]; }; + }; + + # Outputs + # flake.clanInternals + clanInternals = lib.mkOption { + type = types.submodule { + options = { + # Those options are interfaced by the CLI + inventory = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; + inventoryFile = lib.mkOption { type = lib.types.unspecified; }; + clanModules = lib.mkOption { type = lib.types.attrsOf lib.types.path; }; + source = lib.mkOption { type = lib.types.path; }; + meta = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; + all-machines-json = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; + machines = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); }; + machinesFunc = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); }; + }; + }; + }; + }; +} diff --git a/lib/build-clan/module.nix b/lib/build-clan/module.nix new file mode 100644 index 000000000..b868230bb --- /dev/null +++ b/lib/build-clan/module.nix @@ -0,0 +1,192 @@ +{ + config, + clan-core, + nixpkgs, + lib, + specialArgs ? { }, + ... +}: +let + inherit (config) directory machines pkgsForSystem; + + # Final inventory + inherit (config.clanInternals) inventory; + + inherit (clan-core.lib.inventory) buildInventory; + + # map from machine name to service configuration + # { ${machineName} :: Config } + serviceConfigs = ( + buildInventory { + inherit inventory; + inherit directory; + } + ); + + machineSettings = + machineName: + let + warn = lib.warn '' + The use of ./machines//settings.json is deprecated. + If your settings.json is empty, you can safely remove it. + !!! Consider using the inventory system. !!! + + File: ${directory + /machines/${machineName}/settings.json} + + If there are still features missing in the inventory system, please open an issue on the clan-core repository. + ''; + in + # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily + # This is useful for doing a dry-run before writing changes into the settings.json + # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval + if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then + warn (builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE"))) + else + lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") ( + warn (builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json))) + ); + + machineImports = + machineSettings: map (module: clan-core.clanModules.${module}) (machineSettings.clanImports or [ ]); + + # TODO: remove default system once we have a hardware-config mechanism + nixosConfiguration = + { + system ? "x86_64-linux", + name, + pkgs ? null, + extraConfig ? { }, + }: + nixpkgs.lib.nixosSystem { + modules = + let + settings = machineSettings name; + in + (machineImports settings) + ++ [ + { + # Autoinclude configuration.nix and hardware-configuration.nix + imports = builtins.filter (p: builtins.pathExists p) [ + "${directory}/machines/${name}/configuration.nix" + "${directory}/machines/${name}/hardware-configuration.nix" + ]; + } + settings + clan-core.nixosModules.clanCore + extraConfig + (machines.${name} or { }) + # Inherit the inventory assertions ? + # { inherit (mergedInventory) assertions; } + { imports = serviceConfigs.${name} or { }; } + ( + { + # Settings + clan.core.clanDir = directory; + # Inherited from clan wide settings + # clan.core.clanName = meta.name or clanName; + # clan.core.clanIcon = meta.icon or clanIcon; + + # Machine specific settings + clan.core.machineName = name; + networking.hostName = lib.mkDefault name; + nixpkgs.hostPlatform = lib.mkDefault system; + + # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs) + nix.registry.nixpkgs.to = { + type = "path"; + path = lib.mkDefault nixpkgs; + }; + } + // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; } + ) + ]; + specialArgs = { + inherit clan-core; + } // specialArgs; + }; + + allMachines = (inventory.machines or { } // config.machines or { }); + + supportedSystems = [ + "x86_64-linux" + "aarch64-linux" + "riscv64-linux" + "x86_64-darwin" + "aarch64-darwin" + ]; + + nixosConfigurations = lib.mapAttrs (name: _: nixosConfiguration { inherit name; }) allMachines; + + # This instantiates nixos for each system that we support: + # configPerSystem = ..nixosConfiguration + # We need this to build nixos secret generators for each system + configsPerSystem = builtins.listToAttrs ( + builtins.map ( + system: + lib.nameValuePair system ( + lib.mapAttrs ( + name: _: + nixosConfiguration { + inherit name system; + pkgs = pkgsForSystem system; + } + ) allMachines + ) + ) supportedSystems + ); + + configsFuncPerSystem = builtins.listToAttrs ( + builtins.map ( + system: + lib.nameValuePair system ( + lib.mapAttrs ( + name: _: args: + nixosConfiguration ( + args + // { + inherit name system; + pkgs = pkgsForSystem system; + } + ) + ) allMachines + ) + ) supportedSystems + ); + + inventoryFile = "${directory}/inventory.json"; + + inventoryLoaded = + if builtins.pathExists inventoryFile then + (builtins.fromJSON (builtins.readFile inventoryFile)) + else + { }; + +in +{ + imports = [ + # Merge the inventory file + { clanInternals.inventory = inventoryLoaded; } + { clanInternals.inventory = config.inventory; } + # Derived meta from the merged inventory + { clanInternals.meta = config.clanInternals.inventory.meta; } + ]; + + inherit nixosConfigurations; + + clanInternals = { + inherit (clan-core) clanModules; + inherit inventoryFile; + + source = "${clan-core}"; + + # machine specifics + machines = configsPerSystem; + machinesFunc = configsFuncPerSystem; + all-machines-json = lib.mapAttrs ( + system: configs: + nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" ( + lib.mapAttrs (_: m: m.config.system.clan.deployment.data) configs + ) + ) configsPerSystem; + }; +} diff --git a/lib/build-clan/old_default.nix b/lib/build-clan/old_default.nix new file mode 100644 index 000000000..01808d6db --- /dev/null +++ b/lib/build-clan/old_default.nix @@ -0,0 +1,306 @@ +{ + clan-core, + nixpkgs, + lib, +}: +{ + directory, # The directory containing the machines subdirectory + specialArgs ? { }, # Extra arguments to pass to nixosSystem i.e. useful to make self available + machines ? { }, # allows to include machine-specific modules i.e. machines.${name} = { ... } + # DEPRECATED: use meta.name instead + clanName ? null, # Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to. + # DEPRECATED: use meta.icon instead + clanIcon ? null, # A path to an icon to be used for the clan, should be the same for all machines + meta ? { }, # A set containing clan meta: name :: string, icon :: string, description :: string + # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system. + # This improves performance, but all nipxkgs.* options will be ignored. + pkgsForSystem ? (_system: null), + /* + Low level inventory configuration. + Overrides the services configuration. + */ + inventory ? { }, +}: +let + # Internal inventory, this is the result of merging all potential inventory sources: + # - Default instances configured via 'services' + # - The inventory overrides + # - Machines that exist in inventory.machines + # - Machines explicitly configured via 'machines' argument + # - Machines that exist in the machines directory + # Checks on the module level: + # - Each service role must reference a valid machine after all machines are merged + + clanToInventory = + config: + { clanPath, inventoryPath }: + let + v = lib.attrByPath clanPath null config; + in + lib.optionalAttrs (v != null) (lib.setAttrByPath inventoryPath v); + + mergedInventory = + (lib.evalModules { + modules = [ + clan-core.lib.inventory.interface + { inherit meta; } + ( + if + builtins.pathExists "${directory}/inventory.json" + # Is recursively applied. Any explicit nix will override. + then + (builtins.fromJSON (builtins.readFile "${directory}/inventory.json")) + else + { } + ) + inventory + # Machines explicitly configured via 'machines' argument + { + # { ${name} :: meta // { name, tags } } + machines = lib.mapAttrs ( + name: machineConfig: + (lib.attrByPath [ + "clan" + "meta" + ] { } machineConfig) + // { + # meta.name default is the attribute name of the machine + name = lib.mkDefault ( + lib.attrByPath [ + "clan" + "meta" + "name" + ] name machineConfig + ); + } + # tags + // (clanToInventory machineConfig { + clanPath = [ + "clan" + "tags" + ]; + inventoryPath = [ "tags" ]; + }) + # system + // (clanToInventory machineConfig { + clanPath = [ + "nixpkgs" + "hostPlatform" + ]; + inventoryPath = [ "system" ]; + }) + # deploy.targetHost + // (clanToInventory machineConfig { + clanPath = [ + "clan" + "core" + "networking" + "targetHost" + ]; + inventoryPath = [ + "deploy" + "targetHost" + ]; + }) + ) machines; + } + + # Will be deprecated + { + machines = + lib.mapAttrs + ( + name: _: + # Use mkForce to make sure users migrate to the inventory system. + # When the settings.json exists the evaluation will print the deprecation warning. + lib.mkForce { + inherit name; + system = (machineSettings name).nixpkgs.hostSystem or null; + } + ) + ( + lib.filterAttrs ( + machineName: _: builtins.pathExists "${directory}/machines/${machineName}/settings.json" + ) machinesDirs + ); + } + + # Deprecated interface + (if clanName != null then { meta.name = clanName; } else { }) + (if clanIcon != null then { meta.icon = clanIcon; } else { }) + ]; + }).config; + + inherit (clan-core.lib.inventory) buildInventory; + + # map from machine name to service configuration + # { ${machineName} :: Config } + serviceConfigs = buildInventory { + inventory = mergedInventory; + inherit directory; + }; + + machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") ( + builtins.readDir (directory + /machines) + ); + + machineSettings = + machineName: + let + warn = lib.warn '' + The use of ./machines//settings.json is deprecated. + If your settings.json is empty, you can safely remove it. + !!! Consider using the inventory system. !!! + + File: ${directory + /machines/${machineName}/settings.json} + + If there are still features missing in the inventory system, please open an issue on the clan-core repository. + ''; + in + # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily + # This is useful for doing a dry-run before writing changes into the settings.json + # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval + if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then + warn (builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE"))) + else + lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") ( + warn (builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json))) + ); + + machineImports = + machineSettings: map (module: clan-core.clanModules.${module}) (machineSettings.clanImports or [ ]); + + deprecationWarnings = [ + (lib.warnIf ( + clanName != null + ) "clanName in buildClan is deprecated, please use meta.name instead." null) + (lib.warnIf (clanIcon != null) "clanIcon is deprecated, please use meta.icon instead" null) + ]; + + # TODO: remove default system once we have a hardware-config mechanism + nixosConfiguration = + { + system ? "x86_64-linux", + name, + pkgs ? null, + extraConfig ? { }, + }: + nixpkgs.lib.nixosSystem { + modules = + let + settings = machineSettings name; + in + (machineImports settings) + ++ [ + { + # Autoinclude configuration.nix and hardware-configuration.nix + imports = builtins.filter (p: builtins.pathExists p) [ + "${directory}/machines/${name}/configuration.nix" + "${directory}/machines/${name}/hardware-configuration.nix" + ]; + } + settings + clan-core.nixosModules.clanCore + extraConfig + (machines.${name} or { }) + # Inherit the inventory assertions ? + { inherit (mergedInventory) assertions; } + { imports = serviceConfigs.${name} or { }; } + ( + { + # Settings + clan.core.clanDir = directory; + # Inherited from clan wide settings + clan.core.clanName = meta.name or clanName; + clan.core.clanIcon = meta.icon or clanIcon; + + # Machine specific settings + clan.core.machineName = name; + networking.hostName = lib.mkDefault name; + nixpkgs.hostPlatform = lib.mkDefault system; + + # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs) + nix.registry.nixpkgs.to = { + type = "path"; + path = lib.mkDefault nixpkgs; + }; + } + // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; } + ) + ]; + specialArgs = { + inherit clan-core; + } // specialArgs; + }; + + allMachines = mergedInventory.machines or { }; + + supportedSystems = [ + "x86_64-linux" + "aarch64-linux" + "riscv64-linux" + "x86_64-darwin" + "aarch64-darwin" + ]; + + nixosConfigurations = lib.mapAttrs (name: _: nixosConfiguration { inherit name; }) allMachines; + + # This instantiates nixos for each system that we support: + # configPerSystem = ..nixosConfiguration + # We need this to build nixos secret generators for each system + configsPerSystem = builtins.listToAttrs ( + builtins.map ( + system: + lib.nameValuePair system ( + lib.mapAttrs ( + name: _: + nixosConfiguration { + inherit name system; + pkgs = pkgsForSystem system; + } + ) allMachines + ) + ) supportedSystems + ); + + configsFuncPerSystem = builtins.listToAttrs ( + builtins.map ( + system: + lib.nameValuePair system ( + lib.mapAttrs ( + name: _: args: + nixosConfiguration ( + args + // { + inherit name system; + pkgs = pkgsForSystem system; + } + ) + ) allMachines + ) + ) supportedSystems + ); +in +builtins.deepSeq deprecationWarnings { + inherit nixosConfigurations; + + clanInternals = { + inherit (clan-core) clanModules; + source = "${clan-core}"; + + meta = mergedInventory.meta; + inventory = mergedInventory; + + inventoryFile = "${directory}/inventory.json"; + + # machine specifics + machines = configsPerSystem; + machinesFunc = configsFuncPerSystem; + all-machines-json = lib.mapAttrs ( + system: configs: + nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" ( + lib.mapAttrs (_: m: m.config.system.clan.deployment.data) configs + ) + ) configsPerSystem; + }; +} diff --git a/lib/build-clan/tests.nix b/lib/build-clan/tests.nix new file mode 100644 index 000000000..366e795be --- /dev/null +++ b/lib/build-clan/tests.nix @@ -0,0 +1,135 @@ +{ + lib, + nixpkgs, + clan-core, + buildClan, + ... +}: +let + eval = import ./eval.nix { inherit lib nixpkgs clan-core; }; + + self = ./.; + evalClan = eval { inherit self; }; + +in +####### +{ + test_only_required = + let + config = evalClan { directory = ./.; }; + in + { + expr = config.pkgsForSystem null == null; + expected = true; + }; + + test_all_simple = + let + config = evalClan { + directory = ./.; + machines = { }; + inventory = { + meta.name = "test"; + }; + pkgsForSystem = _system: { }; + }; + in + { + expr = config ? inventory; + expected = true; + }; + + test_outputs_clanInternals = + let + config = evalClan { + imports = [ + # What the user needs to specif + { + directory = ./.; + inventory.meta.name = "test"; + } + + # Build-clan implementation + ./module.nix + # Explicit output, usually defined by flake-parts + { options.nixosConfigurations = lib.mkOption { type = lib.types.raw; }; } + ]; + }; + in + { + expr = config.clanInternals.meta; + expected = { + description = null; + icon = null; + name = "test"; + }; + }; + + test_fn_simple = + let + result = buildClan { + directory = ./.; + meta.name = "test"; + }; + in + { + expr = result.clanInternals.meta; + expected = { + description = null; + icon = null; + name = "test"; + }; + }; + + test_fn_extensiv_meta = + let + result = buildClan { + directory = ./.; + meta.name = "test"; + meta.description = "test"; + meta.icon = "test"; + inventory.meta.name = "superclan"; + inventory.meta.description = "description"; + inventory.meta.icon = "icon"; + }; + in + { + expr = result.clanInternals.meta; + expected = { + description = "description"; + icon = "icon"; + name = "superclan"; + }; + }; + + test_fn_clan_core = + let + result = buildClan { + directory = ../../.; + meta.name = "test-clan-core"; + }; + in + { + expr = builtins.attrNames result.nixosConfigurations; + expected = [ "test-inventory-machine" ]; + }; + + test_buildClan_all_machines = + let + result = buildClan { + directory = ./.; + meta.name = "test"; + inventory.machines.machine1.meta.name = "machine1"; + + machines.machine2 = { }; + + }; + in + { + expr = builtins.attrNames result.nixosConfigurations; + expected = [ + "machine1" + "machine2" + ]; + }; +} diff --git a/lib/default.nix b/lib/default.nix index 91a23d6df..23556ee91 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -6,7 +6,7 @@ }: { evalClanModules = import ./eval-clan-modules { inherit clan-core nixpkgs lib; }; - buildClan = import ./build-clan { inherit clan-core lib nixpkgs; }; + buildClan = import ./build-clan { inherit lib nixpkgs clan-core; }; facts = import ./facts.nix { inherit lib; }; inventory = import ./inventory { inherit lib clan-core; }; jsonschema = import ./jsonschema { inherit lib; }; diff --git a/lib/flake-module.nix b/lib/flake-module.nix index 0785872a5..e9ab69a54 100644 --- a/lib/flake-module.nix +++ b/lib/flake-module.nix @@ -8,6 +8,7 @@ imports = [ ./jsonschema/flake-module.nix ./inventory/flake-module.nix + ./build-clan/flake-module.nix ]; flake.lib = import ./default.nix { inherit lib inputs; diff --git a/nixosModules/clanCore/meta/interface.nix b/nixosModules/clanCore/meta/interface.nix index 3b44046e7..aef373d57 100644 --- a/nixosModules/clanCore/meta/interface.nix +++ b/nixosModules/clanCore/meta/interface.nix @@ -6,5 +6,4 @@ in options.clan.meta.name = lib.mkOption { type = lib.types.str; }; options.clan.meta.description = lib.mkOption { type = optStr; }; options.clan.meta.icon = lib.mkOption { type = optStr; }; - options.clan.tags = lib.mkOption { type = lib.types.listOf lib.types.str; }; } diff --git a/pkgs/clan-cli/clan_cli/machines/list.py b/pkgs/clan-cli/clan_cli/machines/list.py index f8df311d5..ec3011a2a 100644 --- a/pkgs/clan-cli/clan_cli/machines/list.py +++ b/pkgs/clan-cli/clan_cli/machines/list.py @@ -1,9 +1,13 @@ import argparse +import json import logging from pathlib import Path from clan_cli.api import API +from clan_cli.cmd import run_no_stdout +from clan_cli.errors import ClanError from clan_cli.inventory import Machine, load_inventory_eval +from clan_cli.nix import nix_eval log = logging.getLogger(__name__) @@ -14,9 +18,29 @@ def list_machines(flake_url: str | Path, debug: bool = False) -> dict[str, Machi return inventory.machines +@API.register +def list_nixos_machines(flake_url: str | Path, debug: bool = False) -> list[str]: + cmd = nix_eval( + [ + f"{flake_url}#nixosConfigurations", + "--apply", + "builtins.attrNames", + "--json", + ] + ) + proc = run_no_stdout(cmd) + + try: + res = proc.stdout.strip() + data = json.loads(res) + return data + except json.JSONDecodeError as e: + raise ClanError(f"Error decoding machines from flake: {e}") + + def list_command(args: argparse.Namespace) -> None: flake_path = args.flake.path - for name in list_machines(flake_path, args.debug).keys(): + for name in list_nixos_machines(flake_path, args.debug): print(name) From 35891f7165093206f7ce986af9e2d18652e2dd90 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Thu, 1 Aug 2024 13:59:36 +0200 Subject: [PATCH 2/8] Add flake-parts module --- flakeModules/clan.nix | 121 ++----------- lib/build-clan/flake-module.nix | 2 +- lib/build-clan/interface.nix | 63 ++++--- lib/build-clan/module.nix | 15 +- lib/build-clan/old_default.nix | 306 -------------------------------- lib/build-clan/tests.nix | 1 - 6 files changed, 61 insertions(+), 447 deletions(-) delete mode 100644 lib/build-clan/old_default.nix diff --git a/flakeModules/clan.nix b/flakeModules/clan.nix index f8a60d1fb..45f5e382f 100644 --- a/flakeModules/clan.nix +++ b/flakeModules/clan.nix @@ -4,125 +4,34 @@ clan-core: lib, flake-parts-lib, inputs, - self, ... }: let inherit (lib) mkOption types; - buildClan = import ../lib/build-clan { - inherit lib clan-core; - inherit (inputs) nixpkgs; - }; - cfg = config.clan; in { - imports = [ - # TODO: figure out how to print the deprecation warning - # "${inputs.nixpkgs}/nixos/modules/misc/assertions.nix" - (lib.mkRenamedOptionModule - [ - "clan" - "clanName" - ] - [ - "clan" - "meta" - "name" - ] - ) - (lib.mkRenamedOptionModule - [ - "clan" - "clanIcon" - ] - [ - "clan" - "meta" - "icon" - ] - ) - ]; - options.clan = { - directory = mkOption { - type = types.path; - description = "The directory containing the clan subdirectory"; - default = self; # default to the directory of the flake - }; - specialArgs = mkOption { - type = types.attrsOf types.raw; - default = { }; - description = "Extra arguments to pass to nixosSystem i.e. useful to make self available"; - }; - machines = mkOption { - type = types.attrsOf types.raw; - default = { }; - description = "Allows to include machine-specific modules i.e. machines.\${name} = { ... }"; - }; - inventory = mkOption { - #type = types.submodule { imports = [ ../lib/inventory/build-inventory/interface.nix ]; }; - type = types.attrsOf types.raw; - default = { }; - description = '' - An abstract service layer for consistently configuring distributed services across machine boundaries. - See https://docs.clan.lol/concepts/inventory/ for more details. - ''; - }; - - # Checks are performed in 'buildClan' - # Not everyone uses flake-parts - meta = { - name = lib.mkOption { - type = types.nullOr types.str; - default = null; - description = "Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to."; + options.clan = lib.mkOption { + type = types.submoduleWith { + # _module.args = { + # }; + specialArgs = { + inherit clan-core; + inherit (inputs) nixpkgs; }; - icon = mkOption { - type = types.nullOr types.path; - default = null; - description = "A path to an icon to be used for the clan in the GUI"; - }; - description = mkOption { - type = types.nullOr types.str; - default = null; - description = "A short description of the clan"; - }; - }; - - pkgsForSystem = mkOption { - type = types.functionTo types.raw; - default = _system: null; - description = "A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system."; + modules = [ + ../lib/build-clan/interface.nix + ../lib/build-clan/module.nix + ]; }; }; - options.flake = flake-parts-lib.mkSubmoduleOptions { - clanInternals = lib.mkOption { - type = lib.types.submodule { - options = { - inventory = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; - inventoryFile = lib.mkOption { type = lib.types.unspecified; }; - clanModules = lib.mkOption { type = lib.types.attrsOf lib.types.path; }; - source = lib.mkOption { type = lib.types.path; }; - meta = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; - all-machines-json = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; - machines = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); }; - machinesFunc = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); }; - }; - }; - }; + options.flake = flake-parts-lib.mkSubmoduleOptions { + clanInternals = lib.mkOption { type = types.raw; }; }; config = { - flake = buildClan { - inherit (cfg) - directory - specialArgs - machines - pkgsForSystem - meta - inventory - ; - }; + flake.clanInternals = config.clan.clanInternals; + flake.nixosConfigurations = config.clan.nixosConfigurations; }; _file = __curPos.file; } diff --git a/lib/build-clan/flake-module.nix b/lib/build-clan/flake-module.nix index 87dfa7c92..13310663a 100644 --- a/lib/build-clan/flake-module.nix +++ b/lib/build-clan/flake-module.nix @@ -26,7 +26,7 @@ in buildClan = self.lib.buildClan; }; checks = { - lib-inventory-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } '' + lib-build-clan-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } '' export HOME="$(realpath .)" nix-unit --eval-store "$HOME" \ diff --git a/lib/build-clan/interface.nix b/lib/build-clan/interface.nix index 169a0bb6f..275aa1664 100644 --- a/lib/build-clan/interface.nix +++ b/lib/build-clan/interface.nix @@ -4,6 +4,27 @@ let in { options = { + # Required options + directory = lib.mkOption { + type = types.path; + description = "The directory containing the clan subdirectory"; + }; + + specialArgs = lib.mkOption { + type = types.attrsOf types.raw; + default = { }; + description = "Extra arguments to pass to nixosSystem i.e. useful to make self available"; + }; + + # Optional + machines = lib.mkOption { + type = types.attrsOf types.deferredModule; + default = { }; + }; + inventory = lib.mkOption { + type = types.submodule { imports = [ ../inventory/build-inventory/interface.nix ]; }; + }; + # Meta meta = { name = lib.mkOption { @@ -23,44 +44,32 @@ in }; }; - # Required options - directory = lib.mkOption { - type = types.path; - description = "The directory containing the clan subdirectory"; - }; - specialArgs = lib.mkOption { - type = types.attrsOf types.raw; - default = { }; - description = "Extra arguments to pass to nixosSystem i.e. useful to make self available"; - }; - - # Optional - machines = lib.mkOption { - type = types.attrsOf types.deferredModule; - default = { }; - }; pkgsForSystem = lib.mkOption { type = types.functionTo (types.nullOr types.attrs); default = _: null; }; - inventory = lib.mkOption { - type = types.submodule { imports = [ ../inventory/build-inventory/interface.nix ]; }; - }; # Outputs + nixosConfigurations = lib.mkOption { + type = types.lazyAttrsOf types.raw; + default = { }; + }; # flake.clanInternals clanInternals = lib.mkOption { + # type = types.raw; + # ClanInternals type = types.submodule { options = { # Those options are interfaced by the CLI - inventory = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; - inventoryFile = lib.mkOption { type = lib.types.unspecified; }; - clanModules = lib.mkOption { type = lib.types.attrsOf lib.types.path; }; - source = lib.mkOption { type = lib.types.path; }; - meta = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; - all-machines-json = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; - machines = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); }; - machinesFunc = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); }; + # We don't speficy the type here, for better performance. + inventory = lib.mkOption { type = lib.types.raw; }; + inventoryFile = lib.mkOption { type = lib.types.raw; }; + clanModules = lib.mkOption { type = lib.types.raw; }; + source = lib.mkOption { type = lib.types.raw; }; + meta = lib.mkOption { type = lib.types.raw; }; + all-machines-json = lib.mkOption { type = lib.types.raw; }; + machines = lib.mkOption { type = lib.types.raw; }; + machinesFunc = lib.mkOption { type = lib.types.raw; }; }; }; }; diff --git a/lib/build-clan/module.nix b/lib/build-clan/module.nix index b868230bb..e0e6d3473 100644 --- a/lib/build-clan/module.nix +++ b/lib/build-clan/module.nix @@ -3,11 +3,15 @@ clan-core, nixpkgs, lib, - specialArgs ? { }, ... }: let - inherit (config) directory machines pkgsForSystem; + inherit (config) + directory + machines + pkgsForSystem + specialArgs + ; # Final inventory inherit (config.clanInternals) inventory; @@ -165,10 +169,7 @@ in { imports = [ # Merge the inventory file - { clanInternals.inventory = inventoryLoaded; } - { clanInternals.inventory = config.inventory; } - # Derived meta from the merged inventory - { clanInternals.meta = config.clanInternals.inventory.meta; } + { inventory = inventoryLoaded; } ]; inherit nixosConfigurations; @@ -176,6 +177,8 @@ in clanInternals = { inherit (clan-core) clanModules; inherit inventoryFile; + inventory = config.inventory; + meta = config.inventory.meta; source = "${clan-core}"; diff --git a/lib/build-clan/old_default.nix b/lib/build-clan/old_default.nix deleted file mode 100644 index 01808d6db..000000000 --- a/lib/build-clan/old_default.nix +++ /dev/null @@ -1,306 +0,0 @@ -{ - clan-core, - nixpkgs, - lib, -}: -{ - directory, # The directory containing the machines subdirectory - specialArgs ? { }, # Extra arguments to pass to nixosSystem i.e. useful to make self available - machines ? { }, # allows to include machine-specific modules i.e. machines.${name} = { ... } - # DEPRECATED: use meta.name instead - clanName ? null, # Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to. - # DEPRECATED: use meta.icon instead - clanIcon ? null, # A path to an icon to be used for the clan, should be the same for all machines - meta ? { }, # A set containing clan meta: name :: string, icon :: string, description :: string - # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system. - # This improves performance, but all nipxkgs.* options will be ignored. - pkgsForSystem ? (_system: null), - /* - Low level inventory configuration. - Overrides the services configuration. - */ - inventory ? { }, -}: -let - # Internal inventory, this is the result of merging all potential inventory sources: - # - Default instances configured via 'services' - # - The inventory overrides - # - Machines that exist in inventory.machines - # - Machines explicitly configured via 'machines' argument - # - Machines that exist in the machines directory - # Checks on the module level: - # - Each service role must reference a valid machine after all machines are merged - - clanToInventory = - config: - { clanPath, inventoryPath }: - let - v = lib.attrByPath clanPath null config; - in - lib.optionalAttrs (v != null) (lib.setAttrByPath inventoryPath v); - - mergedInventory = - (lib.evalModules { - modules = [ - clan-core.lib.inventory.interface - { inherit meta; } - ( - if - builtins.pathExists "${directory}/inventory.json" - # Is recursively applied. Any explicit nix will override. - then - (builtins.fromJSON (builtins.readFile "${directory}/inventory.json")) - else - { } - ) - inventory - # Machines explicitly configured via 'machines' argument - { - # { ${name} :: meta // { name, tags } } - machines = lib.mapAttrs ( - name: machineConfig: - (lib.attrByPath [ - "clan" - "meta" - ] { } machineConfig) - // { - # meta.name default is the attribute name of the machine - name = lib.mkDefault ( - lib.attrByPath [ - "clan" - "meta" - "name" - ] name machineConfig - ); - } - # tags - // (clanToInventory machineConfig { - clanPath = [ - "clan" - "tags" - ]; - inventoryPath = [ "tags" ]; - }) - # system - // (clanToInventory machineConfig { - clanPath = [ - "nixpkgs" - "hostPlatform" - ]; - inventoryPath = [ "system" ]; - }) - # deploy.targetHost - // (clanToInventory machineConfig { - clanPath = [ - "clan" - "core" - "networking" - "targetHost" - ]; - inventoryPath = [ - "deploy" - "targetHost" - ]; - }) - ) machines; - } - - # Will be deprecated - { - machines = - lib.mapAttrs - ( - name: _: - # Use mkForce to make sure users migrate to the inventory system. - # When the settings.json exists the evaluation will print the deprecation warning. - lib.mkForce { - inherit name; - system = (machineSettings name).nixpkgs.hostSystem or null; - } - ) - ( - lib.filterAttrs ( - machineName: _: builtins.pathExists "${directory}/machines/${machineName}/settings.json" - ) machinesDirs - ); - } - - # Deprecated interface - (if clanName != null then { meta.name = clanName; } else { }) - (if clanIcon != null then { meta.icon = clanIcon; } else { }) - ]; - }).config; - - inherit (clan-core.lib.inventory) buildInventory; - - # map from machine name to service configuration - # { ${machineName} :: Config } - serviceConfigs = buildInventory { - inventory = mergedInventory; - inherit directory; - }; - - machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") ( - builtins.readDir (directory + /machines) - ); - - machineSettings = - machineName: - let - warn = lib.warn '' - The use of ./machines//settings.json is deprecated. - If your settings.json is empty, you can safely remove it. - !!! Consider using the inventory system. !!! - - File: ${directory + /machines/${machineName}/settings.json} - - If there are still features missing in the inventory system, please open an issue on the clan-core repository. - ''; - in - # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily - # This is useful for doing a dry-run before writing changes into the settings.json - # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval - if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then - warn (builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE"))) - else - lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") ( - warn (builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json))) - ); - - machineImports = - machineSettings: map (module: clan-core.clanModules.${module}) (machineSettings.clanImports or [ ]); - - deprecationWarnings = [ - (lib.warnIf ( - clanName != null - ) "clanName in buildClan is deprecated, please use meta.name instead." null) - (lib.warnIf (clanIcon != null) "clanIcon is deprecated, please use meta.icon instead" null) - ]; - - # TODO: remove default system once we have a hardware-config mechanism - nixosConfiguration = - { - system ? "x86_64-linux", - name, - pkgs ? null, - extraConfig ? { }, - }: - nixpkgs.lib.nixosSystem { - modules = - let - settings = machineSettings name; - in - (machineImports settings) - ++ [ - { - # Autoinclude configuration.nix and hardware-configuration.nix - imports = builtins.filter (p: builtins.pathExists p) [ - "${directory}/machines/${name}/configuration.nix" - "${directory}/machines/${name}/hardware-configuration.nix" - ]; - } - settings - clan-core.nixosModules.clanCore - extraConfig - (machines.${name} or { }) - # Inherit the inventory assertions ? - { inherit (mergedInventory) assertions; } - { imports = serviceConfigs.${name} or { }; } - ( - { - # Settings - clan.core.clanDir = directory; - # Inherited from clan wide settings - clan.core.clanName = meta.name or clanName; - clan.core.clanIcon = meta.icon or clanIcon; - - # Machine specific settings - clan.core.machineName = name; - networking.hostName = lib.mkDefault name; - nixpkgs.hostPlatform = lib.mkDefault system; - - # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs) - nix.registry.nixpkgs.to = { - type = "path"; - path = lib.mkDefault nixpkgs; - }; - } - // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; } - ) - ]; - specialArgs = { - inherit clan-core; - } // specialArgs; - }; - - allMachines = mergedInventory.machines or { }; - - supportedSystems = [ - "x86_64-linux" - "aarch64-linux" - "riscv64-linux" - "x86_64-darwin" - "aarch64-darwin" - ]; - - nixosConfigurations = lib.mapAttrs (name: _: nixosConfiguration { inherit name; }) allMachines; - - # This instantiates nixos for each system that we support: - # configPerSystem = ..nixosConfiguration - # We need this to build nixos secret generators for each system - configsPerSystem = builtins.listToAttrs ( - builtins.map ( - system: - lib.nameValuePair system ( - lib.mapAttrs ( - name: _: - nixosConfiguration { - inherit name system; - pkgs = pkgsForSystem system; - } - ) allMachines - ) - ) supportedSystems - ); - - configsFuncPerSystem = builtins.listToAttrs ( - builtins.map ( - system: - lib.nameValuePair system ( - lib.mapAttrs ( - name: _: args: - nixosConfiguration ( - args - // { - inherit name system; - pkgs = pkgsForSystem system; - } - ) - ) allMachines - ) - ) supportedSystems - ); -in -builtins.deepSeq deprecationWarnings { - inherit nixosConfigurations; - - clanInternals = { - inherit (clan-core) clanModules; - source = "${clan-core}"; - - meta = mergedInventory.meta; - inventory = mergedInventory; - - inventoryFile = "${directory}/inventory.json"; - - # machine specifics - machines = configsPerSystem; - machinesFunc = configsFuncPerSystem; - all-machines-json = lib.mapAttrs ( - system: configs: - nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" ( - lib.mapAttrs (_: m: m.config.system.clan.deployment.data) configs - ) - ) configsPerSystem; - }; -} diff --git a/lib/build-clan/tests.nix b/lib/build-clan/tests.nix index 366e795be..2d78f42b3 100644 --- a/lib/build-clan/tests.nix +++ b/lib/build-clan/tests.nix @@ -49,7 +49,6 @@ in inventory.meta.name = "test"; } - # Build-clan implementation ./module.nix # Explicit output, usually defined by flake-parts { options.nixosConfigurations = lib.mkOption { type = lib.types.raw; }; } From bf7b8fca7bc548dc3e132c2ec9516c8f43c740c6 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Thu, 1 Aug 2024 14:09:31 +0200 Subject: [PATCH 3/8] Fix wrong default for imports --- lib/build-clan/module.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/build-clan/module.nix b/lib/build-clan/module.nix index e0e6d3473..47c012053 100644 --- a/lib/build-clan/module.nix +++ b/lib/build-clan/module.nix @@ -70,7 +70,7 @@ let ++ [ { # Autoinclude configuration.nix and hardware-configuration.nix - imports = builtins.filter (p: builtins.pathExists p) [ + imports = builtins.filter builtins.pathExists [ "${directory}/machines/${name}/configuration.nix" "${directory}/machines/${name}/hardware-configuration.nix" ]; @@ -81,7 +81,7 @@ let (machines.${name} or { }) # Inherit the inventory assertions ? # { inherit (mergedInventory) assertions; } - { imports = serviceConfigs.${name} or { }; } + { imports = (serviceConfigs.${name} or [ ]); } ( { # Settings @@ -104,6 +104,7 @@ let // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; } ) ]; + specialArgs = { inherit clan-core; } // specialArgs; From 440122d4709c89f853f62ce0a5d9975aa52d172f Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Thu, 1 Aug 2024 14:27:07 +0200 Subject: [PATCH 4/8] Fix wrong unit tests --- lib/build-clan/default.nix | 3 --- templates/flake-module.nix | 6 ++++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 23875981e..773036424 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -38,8 +38,5 @@ eval { rest # implementation ./module.nix - - # Explicit output, usually defined by flake-parts - { options.nixosConfigurations = lib.mkOption { type = lib.types.raw; }; } ]; } diff --git a/templates/flake-module.nix b/templates/flake-module.nix index e48375b57..639a5db5d 100644 --- a/templates/flake-module.nix +++ b/templates/flake-module.nix @@ -17,8 +17,10 @@ initialized = inputs.nixpkgs.legacyPackages.x86_64-linux.runCommand "minimal-clan-flake" { } '' mkdir $out cp -r ${path}/* $out - mkdir -p $out/machines/foo - echo '{ "nixpkgs": { "hostPlatform": "x86_64-linux" } }' > $out/machines/foo/settings.json + rm $out/inventory.json + + # TODO: Instead create a machine by calling the API, this wont break in future tests and is much closer to what the user performs + echo '{ "machines": { "foo": { "name": "foo" } } }' > $out/inventory.json ''; evaled = (import "${initialized}/flake.nix").outputs { self = evaled // { From b893b33d3edb6a42e4c1f10bd0fd0d0de44cc8c1 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Thu, 1 Aug 2024 14:47:26 +0200 Subject: [PATCH 5/8] Forward clan name for vm test compatibility --- lib/build-clan/module.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/build-clan/module.nix b/lib/build-clan/module.nix index 47c012053..aff63ffe1 100644 --- a/lib/build-clan/module.nix +++ b/lib/build-clan/module.nix @@ -87,8 +87,9 @@ let # Settings clan.core.clanDir = directory; # Inherited from clan wide settings - # clan.core.clanName = meta.name or clanName; - # clan.core.clanIcon = meta.icon or clanIcon; + # TODO: remove these + clan.core.clanName = config.inventory.meta.name; + clan.core.clanIcon = config.inventory.meta.icon; # Machine specific settings clan.core.machineName = name; From a4ae204857da6da02c79ce6d2a54fc1400e1ae6a Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sat, 3 Aug 2024 12:04:23 +0200 Subject: [PATCH 6/8] Add: legacy support for implicitly creating machines through creating a directory This will be deprecated as soon as the tests dont rely on this behavior anymore --- lib/build-clan/module.nix | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/lib/build-clan/module.nix b/lib/build-clan/module.nix index aff63ffe1..e0c7767af 100644 --- a/lib/build-clan/module.nix +++ b/lib/build-clan/module.nix @@ -81,7 +81,7 @@ let (machines.${name} or { }) # Inherit the inventory assertions ? # { inherit (mergedInventory) assertions; } - { imports = (serviceConfigs.${name} or [ ]); } + { imports = serviceConfigs.${name} or [ ]; } ( { # Settings @@ -111,7 +111,38 @@ let } // specialArgs; }; - allMachines = (inventory.machines or { } // config.machines or { }); + # TODO: Will be deprecated + # We must migrate the tests, that create a settings.json to add a machine. + ################################################## + testMachines = + lib.mapAttrs + (name: _: { + inherit name; + system = (machineSettings name).nixpkgs.hostSystem or null; + }) + ( + lib.filterAttrs ( + machineName: _: + if builtins.pathExists "${directory}/machines/${machineName}/settings.json" then + lib.warn '' + The use of ./machines//settings.json is deprecated. + If your settings.json is empty, you can safely remove it. + !!! Consider using the inventory system. !!! + + File: ${directory + /machines/${machineName}/settings.json} + + If there are still features missing in the inventory system, please open an issue on the clan-core repository. + '' true + else + false + ) machinesDirs + ); + machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") ( + builtins.readDir (directory + /machines) + ); + ################################################## + + allMachines = inventory.machines or { } // config.machines or { } // testMachines; supportedSystems = [ "x86_64-linux" @@ -166,7 +197,6 @@ let (builtins.fromJSON (builtins.readFile inventoryFile)) else { }; - in { imports = [ From ede5cbcf139ed12c4872c0b058e45fdc86cdc0f6 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sat, 3 Aug 2024 12:32:37 +0200 Subject: [PATCH 7/8] Fix: regression list_machines. Split into multiple functions list_inventory_machines, list_nixos_machine, list_sops_machines The caller of the function should specify which machines they wants to see --- pkgs/clan-cli/clan_cli/clan/inspect.py | 4 ++-- pkgs/clan-cli/clan_cli/history/add.py | 4 ++-- pkgs/clan-cli/clan_cli/machines/list.py | 4 +++- pkgs/clan-cli/clan_cli/secrets/machines.py | 10 ++++++++-- pkgs/clan-vm-manager/clan_vm_manager/views/list.py | 2 +- pkgs/webview-ui/app/src/components/MachineListItem.tsx | 2 +- pkgs/webview-ui/app/src/routes/machines/view.tsx | 4 ++-- 7 files changed, 19 insertions(+), 11 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/clan/inspect.py b/pkgs/clan-cli/clan_cli/clan/inspect.py index 0c5b28636..bf83c9b5b 100644 --- a/pkgs/clan-cli/clan_cli/clan/inspect.py +++ b/pkgs/clan-cli/clan_cli/clan/inspect.py @@ -6,7 +6,7 @@ from ..clan_uri import FlakeId from ..cmd import run from ..dirs import machine_gcroot from ..errors import ClanError -from ..machines.list import list_machines +from ..machines.list import list_nixos_machines from ..machines.machines import Machine from ..nix import nix_add_to_gcroots, nix_build, nix_config, nix_eval, nix_metadata from ..vms.inspect import VmConfig, inspect_vm @@ -40,7 +40,7 @@ def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig: system = config["system"] # Check if the machine exists - machines = list_machines(flake_url, False) + machines: list[str] = list_nixos_machines(flake_url, False) if machine_name not in machines: raise ClanError( f"Machine {machine_name} not found in {flake_url}. Available machines: {', '.join(machines)}" diff --git a/pkgs/clan-cli/clan_cli/history/add.py b/pkgs/clan-cli/clan_cli/history/add.py index e13938e86..93646be30 100644 --- a/pkgs/clan-cli/clan_cli/history/add.py +++ b/pkgs/clan-cli/clan_cli/history/add.py @@ -7,7 +7,7 @@ import logging from typing import Any from clan_cli.clan.inspect import FlakeConfig, inspect_flake -from clan_cli.machines.list import list_machines +from clan_cli.machines.list import list_nixos_machines from ..clan_uri import ClanURI from ..dirs import user_history_file @@ -72,7 +72,7 @@ def new_history_entry(url: str, machine: str) -> HistoryEntry: def add_all_to_history(uri: ClanURI) -> list[HistoryEntry]: history = list_history() new_entries: list[HistoryEntry] = [] - for machine in list_machines(uri.get_url()): + for machine in list_nixos_machines(uri.get_url()): new_entry = _add_maschine_to_history_list(uri.get_url(), machine, history) new_entries.append(new_entry) write_history_file(history) diff --git a/pkgs/clan-cli/clan_cli/machines/list.py b/pkgs/clan-cli/clan_cli/machines/list.py index ec3011a2a..b473a0c2f 100644 --- a/pkgs/clan-cli/clan_cli/machines/list.py +++ b/pkgs/clan-cli/clan_cli/machines/list.py @@ -13,7 +13,9 @@ log = logging.getLogger(__name__) @API.register -def list_machines(flake_url: str | Path, debug: bool = False) -> dict[str, Machine]: +def list_inventory_machines( + flake_url: str | Path, debug: bool = False +) -> dict[str, Machine]: inventory = load_inventory_eval(flake_url) return inventory.machines diff --git a/pkgs/clan-cli/clan_cli/secrets/machines.py b/pkgs/clan-cli/clan_cli/secrets/machines.py index af7543ff3..5ca91fd65 100644 --- a/pkgs/clan-cli/clan_cli/secrets/machines.py +++ b/pkgs/clan-cli/clan_cli/secrets/machines.py @@ -47,10 +47,16 @@ def get_machine(flake_dir: Path, name: str) -> str: def has_machine(flake_dir: Path, name: str) -> bool: + """ + Checks if a machine exists in the sops machines folder + """ return (sops_machines_folder(flake_dir) / name / "key.json").exists() -def list_machines(flake_dir: Path) -> list[str]: +def list_sops_machines(flake_dir: Path) -> list[str]: + """ + Lists all machines in the sops machines folder + """ path = sops_machines_folder(flake_dir) def validate(name: str) -> bool: @@ -86,7 +92,7 @@ def remove_secret(flake_dir: Path, machine: str, secret: str) -> None: def list_command(args: argparse.Namespace) -> None: if args.flake is None: raise ClanError("Could not find clan flake toplevel directory") - lst = list_machines(args.flake.path) + lst = list_sops_machines(args.flake.path) if len(lst) > 0: print("\n".join(lst)) diff --git a/pkgs/clan-vm-manager/clan_vm_manager/views/list.py b/pkgs/clan-vm-manager/clan_vm_manager/views/list.py index 65444fb4c..95c37eb05 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/views/list.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/views/list.py @@ -113,7 +113,7 @@ class ClanList(Gtk.Box): # menu_model = Gio.Menu() # TODO: Make this lazy, blocks UI startup for too long - # for vm in machines.list.list_machines(flake_url=vm.data.flake.flake_url): + # for vm in machines.list.list_nixos_machines(flake_url=vm.data.flake.flake_url): # if vm not in vm_store: # menu_model.append(vm, f"app.add::{vm}") diff --git a/pkgs/webview-ui/app/src/components/MachineListItem.tsx b/pkgs/webview-ui/app/src/components/MachineListItem.tsx index d3a396119..26079d7e4 100644 --- a/pkgs/webview-ui/app/src/components/MachineListItem.tsx +++ b/pkgs/webview-ui/app/src/components/MachineListItem.tsx @@ -1,7 +1,7 @@ import { createSignal, Match, Show, Switch } from "solid-js"; import { ErrorData, pyApi, SuccessData } from "../api"; -type MachineDetails = SuccessData<"list_machines">["data"][string]; +type MachineDetails = SuccessData<"list_inventory_machines">["data"][string]; interface MachineListItemProps { name: string; diff --git a/pkgs/webview-ui/app/src/routes/machines/view.tsx b/pkgs/webview-ui/app/src/routes/machines/view.tsx index 404a6af56..c30ea1f86 100644 --- a/pkgs/webview-ui/app/src/routes/machines/view.tsx +++ b/pkgs/webview-ui/app/src/routes/machines/view.tsx @@ -23,7 +23,7 @@ import { MachineListItem } from "@/src/components/MachineListItem"; // >["data"]["services"]; type MachinesModel = Extract< - OperationResponse<"list_machines">, + OperationResponse<"list_inventory_machines">, { status: "success" } >["data"]; @@ -63,7 +63,7 @@ export const MachineListView: Component = () => { return; } setLoading(true); - const response = await callApi("list_machines", { + const response = await callApi("list_inventory_machines", { flake_url: uri, }); setLoading(false); From 7b4b0800008c20e98aaf65a52a6427bbcafac8c3 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sat, 3 Aug 2024 13:14:08 +0200 Subject: [PATCH 8/8] Fix: type test --- pkgs/webview-ui/app/tests/types.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/webview-ui/app/tests/types.test.ts b/pkgs/webview-ui/app/tests/types.test.ts index f27fd4485..c8184a2ec 100644 --- a/pkgs/webview-ui/app/tests/types.test.ts +++ b/pkgs/webview-ui/app/tests/types.test.ts @@ -29,7 +29,7 @@ describe.concurrent("API types work properly", () => { }); it("Machine list receives a records of names and machine info.", async () => { - expectTypeOf(pyApi.list_machines.receive) + expectTypeOf(pyApi.list_inventory_machines.receive) .parameter(0) .parameter(0) .toMatchTypeOf<