From 091c5675705c38a7c6f32a201b683963bfc210ba Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 12:39:17 +0200 Subject: [PATCH 1/6] templates_url: add clan template url test --- .../clan_lib/templates/template_url_test.py | 61 ++++++++++++++----- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/templates/template_url_test.py b/pkgs/clan-cli/clan_lib/templates/template_url_test.py index 3193f0447..de8a7e7b9 100644 --- a/pkgs/clan-cli/clan_lib/templates/template_url_test.py +++ b/pkgs/clan-cli/clan_lib/templates/template_url_test.py @@ -5,7 +5,7 @@ import pytest from clan_lib.errors import ClanError from clan_lib.templates.template_url import transform_url -template_type = "machine" +machine_template_type = "machine" class DummyFlake: @@ -23,7 +23,18 @@ def test_transform_url_self_explizit_dot() -> None: user_input = ".#new-machine" expected_selector = 'clan.templates.machine."new-machine"' - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) + assert flake_ref == str(local_path.path) + assert selector == expected_selector + + +def test_default_clan_template() -> None: + user_input = ".#default" + expected_selector = 'clan.templates.clan."default"' + + flake_ref, selector = transform_url("clan", user_input, flake=local_path) assert flake_ref == str(local_path.path) assert selector == expected_selector @@ -32,7 +43,9 @@ def test_transform_url_self_no_dot() -> None: user_input = "#new-machine" expected_selector = 'clan.templates.machine."new-machine"' - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == str(local_path.path) assert selector == expected_selector @@ -41,7 +54,9 @@ def test_transform_url_builtin_template() -> None: user_input = "new-machine" expected_selector = 'clanInternals.templates.machine."new-machine"' - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == str(local_path.path) assert selector == expected_selector @@ -50,7 +65,9 @@ def test_transform_url_remote_template() -> None: user_input = "github:/org/repo#new-machine" expected_selector = 'clan.templates.machine."new-machine"' - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == "github:/org/repo" assert selector == expected_selector @@ -60,7 +77,9 @@ def test_transform_url_explicit_path() -> None: user_input = ".#clan.templates.machine.new-machine" expected_selector = "clan.templates.machine.new-machine" - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == str(local_path.path) assert selector == expected_selector @@ -69,7 +88,9 @@ def test_transform_url_explicit_path() -> None: def test_transform_url_quoted_selector() -> None: user_input = '.#"new.machine"' expected_selector = '"new.machine"' - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == str(local_path.path) assert selector == expected_selector @@ -77,7 +98,9 @@ def test_transform_url_quoted_selector() -> None: def test_single_quote_selector() -> None: user_input = ".#'new.machine'" expected_selector = "'new.machine'" - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == str(local_path.path) assert selector == expected_selector @@ -86,7 +109,9 @@ def test_custom_template_path() -> None: user_input = "github:/org/repo#my.templates.custom.machine" expected_selector = "my.templates.custom.machine" - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == "github:/org/repo" assert selector == expected_selector @@ -96,7 +121,9 @@ def test_full_url_query_and_fragment() -> None: expected_flake_ref = "github:/org/repo?query=param" expected_selector = "my.templates.custom.machine" - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == expected_flake_ref assert selector == expected_selector @@ -114,7 +141,7 @@ def test_malformed_identifier() -> None: user_input = "github:/org/repo#my.templates.custom.machine#extra" with pytest.raises(ClanError) as exc_info: _flake_ref, _selector = transform_url( - template_type, user_input, flake=local_path + machine_template_type, user_input, flake=local_path ) assert isinstance(exc_info.value, ClanError) @@ -128,7 +155,9 @@ def test_locked_input_template() -> None: user_input = "locked-input#new-machine" expected_selector = 'inputs.locked-input.clan.templates.machine."new-machine"' - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == str(local_path.path) assert selector == expected_selector @@ -137,7 +166,9 @@ def test_locked_input_template_no_quotes() -> None: user_input = 'locked-input#"new.machine"' expected_selector = 'inputs.locked-input."new.machine"' - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert selector == expected_selector assert flake_ref == str(local_path.path) @@ -146,6 +177,8 @@ def test_locked_input_template_no_dot() -> None: user_input = "locked-input#new.machine" expected_selector = "inputs.locked-input.new.machine" - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert selector == expected_selector assert flake_ref == str(local_path.path) From 8de70da4753f53a2717ae72dfd03ffa9ed180be0 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 13:03:17 +0200 Subject: [PATCH 2/6] Templates: replace leftover MachineID, by Machine --- pkgs/clan-cli/clan_lib/machines/actions.py | 12 ++---------- pkgs/clan-cli/clan_lib/machines/machines.py | 4 +++- pkgs/clan-cli/clan_lib/templates/handler.py | 5 +++-- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/machines/actions.py b/pkgs/clan-cli/clan_lib/machines/actions.py index 0fd7748ce..4c19e7a23 100644 --- a/pkgs/clan-cli/clan_lib/machines/actions.py +++ b/pkgs/clan-cli/clan_lib/machines/actions.py @@ -1,9 +1,9 @@ -from dataclasses import dataclass from typing import TypedDict from clan_lib.api import API from clan_lib.errors import ClanError from clan_lib.flake.flake import Flake +from clan_lib.machines.machines import Machine from clan_lib.nix_models.clan import ( InventoryMachine, ) @@ -65,16 +65,8 @@ def get_machine(flake: Flake, name: str) -> InventoryMachine: return InventoryMachine(**machine_inv) -# TODO: remove this machine, once the Machine class is refactored -# We added this now, to allow for dispatching actions. To require only 'name' and 'flake' of a machine. -@dataclass(frozen=True) -class MachineID: - name: str - flake: Flake - - @API.register -def set_machine(machine: MachineID, update: InventoryMachine) -> None: +def set_machine(machine: Machine, update: InventoryMachine) -> None: """ Update the machine information in the inventory. """ diff --git a/pkgs/clan-cli/clan_lib/machines/machines.py b/pkgs/clan-cli/clan_lib/machines/machines.py index 489b0bccf..ce4a51b6d 100644 --- a/pkgs/clan-cli/clan_lib/machines/machines.py +++ b/pkgs/clan-cli/clan_lib/machines/machines.py @@ -13,7 +13,6 @@ from clan_cli.vars._types import StoreBase from clan_lib.api import API from clan_lib.errors import ClanCmdError, ClanError from clan_lib.flake import Flake -from clan_lib.machines.actions import get_machine from clan_lib.nix import nix_config from clan_lib.nix_models.clan import InventoryMachine from clan_lib.ssh.remote import Remote @@ -39,6 +38,9 @@ class Machine: return cls(name=name, flake=flake) def get_inv_machine(self) -> "InventoryMachine": + # Import on demand to avoid circular imports + from clan_lib.machines.actions import get_machine + return get_machine(self.flake, self.name) def get_id(self) -> str: diff --git a/pkgs/clan-cli/clan_lib/templates/handler.py b/pkgs/clan-cli/clan_lib/templates/handler.py index 72d28a627..74ce167d6 100644 --- a/pkgs/clan-cli/clan_lib/templates/handler.py +++ b/pkgs/clan-cli/clan_lib/templates/handler.py @@ -7,7 +7,8 @@ from pathlib import Path from clan_lib.dirs import specific_machine_dir from clan_lib.errors import ClanError from clan_lib.flake import Flake -from clan_lib.machines.actions import MachineID, list_machines +from clan_lib.machines.actions import list_machines +from clan_lib.machines.machines import Machine from clan_lib.templates.filesystem import copy_from_nixstore, realize_nix_path from clan_lib.templates.template_url import transform_url @@ -84,7 +85,7 @@ def machine_template( description="Template machine must contain a configuration.nix", ) - tmp_machine = MachineID(flake=flake, name=dst_machine_name) + tmp_machine = Machine(flake=flake, name=dst_machine_name) dst_machine_dir = specific_machine_dir(tmp_machine) From e18fe134308354507c981ee16295ffe12cbdb1d0 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 13:03:46 +0200 Subject: [PATCH 3/6] Templates: remove outdated check for 'configuration.nix' in machine templates --- pkgs/clan-cli/clan_lib/templates/handler.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/templates/handler.py b/pkgs/clan-cli/clan_lib/templates/handler.py index 74ce167d6..185552243 100644 --- a/pkgs/clan-cli/clan_lib/templates/handler.py +++ b/pkgs/clan-cli/clan_lib/templates/handler.py @@ -77,14 +77,6 @@ def machine_template( msg = f"Template {printable_template_ref} is not a directory at {src_path}" raise ClanError(msg) - # TODO: Do we really need to check for a specific file in the template? - if not (src_path / "configuration.nix").exists(): - msg = f"Template {printable_template_ref} does not contain a configuration.nix" - raise ClanError( - msg, - description="Template machine must contain a configuration.nix", - ) - tmp_machine = Machine(flake=flake, name=dst_machine_name) dst_machine_dir = specific_machine_dir(tmp_machine) From f2db543651583aa160c67e74ce2d88c17e32156c Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 13:31:46 +0200 Subject: [PATCH 4/6] Templates: migrate clan templates to flake identifiers --- pkgs/clan-cli/clan_cli/clan/create.py | 36 ++--------- pkgs/clan-cli/clan_lib/clan/create.py | 67 ++++++++------------ pkgs/clan-cli/clan_lib/templates/handler.py | 70 +++++++++++++++++++++ pkgs/clan-cli/clan_lib/tests/test_create.py | 2 +- 4 files changed, 100 insertions(+), 75 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/clan/create.py b/pkgs/clan-cli/clan_cli/clan/create.py index 2604456e0..8d8cf2def 100644 --- a/pkgs/clan-cli/clan_cli/clan/create.py +++ b/pkgs/clan-cli/clan_cli/clan/create.py @@ -4,36 +4,17 @@ import logging from pathlib import Path from clan_lib.clan.create import CreateOptions, create_clan -from clan_lib.templates import ( - InputPrio, -) log = logging.getLogger(__name__) def register_create_parser(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - "--input", - type=str, - help="""Flake input name to use as template source - can be specified multiple times, inputs are tried in order of definition - Example: --input clan --input clan-core - """, - action="append", - default=[], - ) - - parser.add_argument( - "--no-self", - help="Do not look into own flake for templates", - action="store_true", - default=False, - ) - parser.add_argument( "--template", type=str, - help="Clan template name", + help="""Reference to the template to use for the clan. default="default". In the format '#template_name' Where is a flake reference (e.g. github:org/repo) or a local path (e.g. '.' ). + Omitting '#' will use the builtin templates (e.g. just 'default' from clan-core ). + """, default="default", ) @@ -59,19 +40,10 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None: ) def create_flake_command(args: argparse.Namespace) -> None: - if len(args.input) == 0: - args.input = ["clan", "clan-core"] - - if args.no_self: - input_prio = InputPrio.try_inputs(tuple(args.input)) - else: - input_prio = InputPrio.try_self_then_inputs(tuple(args.input)) - create_clan( CreateOptions( - input_prio=input_prio, dest=args.path, - template_name=args.template, + template=args.template, setup_git=not args.no_git, src_flake=args.flake, update_clan=not args.no_update, diff --git a/pkgs/clan-cli/clan_lib/clan/create.py b/pkgs/clan-cli/clan_lib/clan/create.py index ddea36969..81272977b 100644 --- a/pkgs/clan-cli/clan_lib/clan/create.py +++ b/pkgs/clan-cli/clan_lib/clan/create.py @@ -4,16 +4,12 @@ from pathlib import Path from clan_lib.api import API from clan_lib.cmd import RunOpts, run +from clan_lib.dirs import clan_templates from clan_lib.errors import ClanError from clan_lib.flake import Flake from clan_lib.nix import nix_command, nix_metadata, nix_shell from clan_lib.persist.inventory_store import InventorySnapshot, InventoryStore -from clan_lib.templates import ( - InputPrio, - TemplateName, - get_template, -) -from clan_lib.templates.filesystem import copy_from_nixstore +from clan_lib.templates.handler import clan_template log = logging.getLogger(__name__) @@ -21,9 +17,9 @@ log = logging.getLogger(__name__) @dataclass class CreateOptions: dest: Path - template_name: str + template: str + src_flake: Flake | None = None - input_prio: InputPrio | None = None setup_git: bool = True initial: InventorySnapshot | None = None update_clan: bool = True @@ -47,44 +43,31 @@ def create_clan(opts: CreateOptions) -> None: log.warning("Setting src_flake to None") opts.src_flake = None - template = get_template( - TemplateName(opts.template_name), - "clan", - input_prio=opts.input_prio, - clan_dir=opts.src_flake, - ) - log.info(f"Found template '{template.name}' in '{template.input_variant}'") + if opts.src_flake is None: + opts.src_flake = Flake(str(clan_templates())) - if dest.exists(): - dest /= template.name + with clan_template( + opts.src_flake, template_ident=opts.template, dst_dir=opts.dest + ) as _clan_dir: + if opts.setup_git: + run(git_command(dest, "init")) + run(git_command(dest, "add", ".")) - if dest.exists(): - msg = f"Destination directory {dest} already exists" - raise ClanError(msg) + # check if username is set + has_username = run( + git_command(dest, "config", "user.name"), RunOpts(check=False) + ) + if has_username.returncode != 0: + run(git_command(dest, "config", "user.name", "clan-tool")) - src = Path(template.src["path"]) + has_username = run( + git_command(dest, "config", "user.email"), RunOpts(check=False) + ) + if has_username.returncode != 0: + run(git_command(dest, "config", "user.email", "clan@example.com")) - copy_from_nixstore(src, dest) - - if opts.setup_git: - run(git_command(dest, "init")) - run(git_command(dest, "add", ".")) - - # check if username is set - has_username = run( - git_command(dest, "config", "user.name"), RunOpts(check=False) - ) - if has_username.returncode != 0: - run(git_command(dest, "config", "user.name", "clan-tool")) - - has_username = run( - git_command(dest, "config", "user.email"), RunOpts(check=False) - ) - if has_username.returncode != 0: - run(git_command(dest, "config", "user.email", "clan@example.com")) - - if opts.update_clan: - run(nix_command(["flake", "update"]), RunOpts(cwd=dest)) + if opts.update_clan: + run(nix_command(["flake", "update"]), RunOpts(cwd=dest)) if opts.initial: inventory_store = InventoryStore(flake=Flake(str(opts.dest))) diff --git a/pkgs/clan-cli/clan_lib/templates/handler.py b/pkgs/clan-cli/clan_lib/templates/handler.py index 185552243..dd44f0438 100644 --- a/pkgs/clan-cli/clan_lib/templates/handler.py +++ b/pkgs/clan-cli/clan_lib/templates/handler.py @@ -99,3 +99,73 @@ def machine_template( finally: # If no error occurred, the machine directory is kept pass + + +@contextmanager +def clan_template(flake: Flake, template_ident: str, dst_dir: Path) -> Iterator[Path]: + """ + Create a clan from a template. + This function will copy the template files to a new clan directory + + :param flake: The flake to create the machine in. + :param template_ident: The identifier of the template to use. Example ".#template_name" + :param dst: The name of the directory to create. + + + Example usage: + + >>> with clan_template( + ... Flake("/home/johannes/git/clan-core"), ".#new-machine", "my-machine" + ... ) as clan_dir: + ... # Use `clan_dir` here if you want to access the created directory + + ... The directory is removed if the context raised any errors. + ... Only if the context is exited without errors, it is kept. + """ + + # Get the clan template from the specifier + [flake_ref, template_selector] = transform_url("clan", template_ident, flake=flake) + # For pretty error messages + printable_template_ref = f"{flake_ref}#{template_selector}" + + template_flake = Flake(flake_ref) + + try: + template = template_flake.select(template_selector) + except ClanError as e: + msg = f"Failed to select template '{template_ident}' from flake '{flake_ref}' (via attribute path: {printable_template_ref})" + raise ClanError(msg) from e + + src = template.get("path") + if not src: + msg = f"Malformed template: {printable_template_ref} does not have a 'path' attribute" + raise ClanError(msg) + + src_path = Path(src).resolve() + + realize_nix_path(template_flake, str(src_path)) + + if not src_path.exists(): + msg = f"Template {printable_template_ref} does not exist at {src_path}" + raise ClanError(msg) + + if not src_path.is_dir(): + msg = f"Template {printable_template_ref} is not a directory at {src_path}" + raise ClanError(msg) + + if dst_dir.exists(): + msg = f"Destination directory {dst_dir} already exists" + raise ClanError(msg) + + copy_from_nixstore(src_path, dst_dir) + + try: + yield dst_dir + except Exception as e: + log.error(f"An error occurred inside the 'clan_template' context: {e}") + log.info(f"Removing left-over directory: {dst_dir}") + shutil.rmtree(dst_dir, ignore_errors=True) + raise + finally: + # If no error occurred, the directory is kept + pass diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index 29b460a19..ed43e442f 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -137,7 +137,7 @@ def test_clan_create_api( # TODO: We need to generate a lock file for the templates clan_cli.clan.create.create_clan( clan_cli.clan.create.CreateOptions( - template_name="minimal", dest=dest_clan_dir, update_clan=False + template="minimal", dest=dest_clan_dir, update_clan=False ) ) assert dest_clan_dir.is_dir() From ad169d4d522b554b48ad092f1e33386c1998be68 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 15:48:35 +0200 Subject: [PATCH 5/6] Fix/ui: update create argument --- pkgs/clan-app/ui/src/routes/clans/create.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-app/ui/src/routes/clans/create.tsx b/pkgs/clan-app/ui/src/routes/clans/create.tsx index 002109c27..6a1b8fa39 100644 --- a/pkgs/clan-app/ui/src/routes/clans/create.tsx +++ b/pkgs/clan-app/ui/src/routes/clans/create.tsx @@ -49,7 +49,7 @@ export const CreateClan = () => { const r = await callApi("create_clan", { opts: { dest: target_dir[0], - template_name: template, + template: template, initial: { meta, services: {}, From 072654abd6547f0489776d710cef805576dc6c15 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 16:43:38 +0200 Subject: [PATCH 6/6] Templates: fix invalid mock flake --- pkgs/clan-cli/clan_lib/flake/flake.py | 9 ++++++--- templates/flake.nix | 10 ++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/flake/flake.py b/pkgs/clan-cli/clan_lib/flake/flake.py index e4beaf2c2..4aed6e820 100644 --- a/pkgs/clan-cli/clan_lib/flake/flake.py +++ b/pkgs/clan-cli/clan_lib/flake/flake.py @@ -832,7 +832,8 @@ nix repl --expr 'rec {{ """) # fmt: on elif len(selectors) == 1: - log.debug(f""" + log.debug( + f""" selecting: {selectors[0]} to debug run: nix repl --expr 'rec {{ @@ -840,11 +841,13 @@ nix repl --expr 'rec {{ selectLib = (builtins.getFlake "path:{select_source()}?narHash={select_hash}").lib; query = selectLib.select '"''{selectors[0]}''"' flake; }}' - """) + """ + ) build_output = Path( run( - nix_build(["--expr", nix_code, *nix_options]), RunOpts(log=Log.NONE, trace=False), + nix_build(["--expr", nix_code, *nix_options]), + RunOpts(log=Log.NONE, trace=False), ).stdout.strip() ) diff --git a/templates/flake.nix b/templates/flake.nix index e349af18b..c7e76fb92 100644 --- a/templates/flake.nix +++ b/templates/flake.nix @@ -1,8 +1,8 @@ { outputs = { ... }: - { - clan.templates = { + let + templates = { disko = { single-disk = { description = "A simple ext4 disk with a single partition"; @@ -41,5 +41,11 @@ }; }; }; + in + rec { + inherit (clan) clanInternals; + + clan.clanInternals.templates = templates; + clan.templates = templates; }; }