Merge pull request 'templates_url: add clan template url test' (#4216) from clan-templates into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/4216

pkgs/clan-app/ui/src/routes/clans/create.tsx

@@ -49,7 +49,7 @@ export const CreateClan = () => {
     const r = await callApi("create_clan", {
       opts: {
         dest: target_dir[0],
-        template_name: template,
+        template: template,
         initial: {
           meta,
           services: {},

pkgs/clan-cli/clan_cli/clan/create.py

@@ -4,36 +4,17 @@ import logging
 from pathlib import Path
 
 from clan_lib.clan.create import CreateOptions, create_clan
-from clan_lib.templates import (
-    InputPrio,
-)
 
 log = logging.getLogger(__name__)
 
 
 def register_create_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
-        "--input",
-        type=str,
-        help="""Flake input name to use as template source
-        can be specified multiple times, inputs are tried in order of definition
-        Example: --input clan --input clan-core
-        """,
-        action="append",
-        default=[],
-    )
-
-    parser.add_argument(
-        "--no-self",
-        help="Do not look into own flake for templates",
-        action="store_true",
-        default=False,
-    )
-
     parser.add_argument(
         "--template",
         type=str,
-        help="Clan template name",
+        help="""Reference to the template to use for the clan. default="default". In the format '<flake_ref>#template_name' Where <flake_ref> is a flake reference (e.g. github:org/repo) or a local path (e.g. '.' ).
+        Omitting '<flake_ref>#' will use the builtin templates (e.g. just 'default' from clan-core ).
+        """,
         default="default",
     )
 
@@ -59,19 +40,10 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
     )
 
     def create_flake_command(args: argparse.Namespace) -> None:
-        if len(args.input) == 0:
-            args.input = ["clan", "clan-core"]
-
-        if args.no_self:
-            input_prio = InputPrio.try_inputs(tuple(args.input))
-        else:
-            input_prio = InputPrio.try_self_then_inputs(tuple(args.input))
-
         create_clan(
             CreateOptions(
-                input_prio=input_prio,
                 dest=args.path,
-                template_name=args.template,
+                template=args.template,
                 setup_git=not args.no_git,
                 src_flake=args.flake,
                 update_clan=not args.no_update,

pkgs/clan-cli/clan_lib/clan/create.py

@@ -4,16 +4,12 @@ from pathlib import Path
 
 from clan_lib.api import API
 from clan_lib.cmd import RunOpts, run
+from clan_lib.dirs import clan_templates
 from clan_lib.errors import ClanError
 from clan_lib.flake import Flake
 from clan_lib.nix import nix_command, nix_metadata, nix_shell
 from clan_lib.persist.inventory_store import InventorySnapshot, InventoryStore
-from clan_lib.templates import (
+from clan_lib.templates.handler import clan_template
-    InputPrio,
-    TemplateName,
-    get_template,
-)
-from clan_lib.templates.filesystem import copy_from_nixstore
 
 log = logging.getLogger(__name__)
 
@@ -21,9 +17,9 @@ log = logging.getLogger(__name__)
 @dataclass
 class CreateOptions:
     dest: Path
-    template_name: str
+    template: str
+
     src_flake: Flake | None = None
-    input_prio: InputPrio | None = None
     setup_git: bool = True
     initial: InventorySnapshot | None = None
     update_clan: bool = True
@@ -47,44 +43,31 @@ def create_clan(opts: CreateOptions) -> None:
             log.warning("Setting src_flake to None")
             opts.src_flake = None
 
-    template = get_template(
+    if opts.src_flake is None:
-        TemplateName(opts.template_name),
+        opts.src_flake = Flake(str(clan_templates()))
-        "clan",
-        input_prio=opts.input_prio,
-        clan_dir=opts.src_flake,
-    )
-    log.info(f"Found template '{template.name}' in '{template.input_variant}'")
 
-    if dest.exists():
+    with clan_template(
-        dest /= template.name
+        opts.src_flake, template_ident=opts.template, dst_dir=opts.dest
+    ) as _clan_dir:
+        if opts.setup_git:
+            run(git_command(dest, "init"))
+            run(git_command(dest, "add", "."))
 
-    if dest.exists():
+            # check if username is set
-        msg = f"Destination directory {dest} already exists"
+            has_username = run(
-        raise ClanError(msg)
+                git_command(dest, "config", "user.name"), RunOpts(check=False)
+            )
+            if has_username.returncode != 0:
+                run(git_command(dest, "config", "user.name", "clan-tool"))
 
-    src = Path(template.src["path"])
+            has_username = run(
+                git_command(dest, "config", "user.email"), RunOpts(check=False)
+            )
+            if has_username.returncode != 0:
+                run(git_command(dest, "config", "user.email", "clan@example.com"))
 
-    copy_from_nixstore(src, dest)
+        if opts.update_clan:
-
+            run(nix_command(["flake", "update"]), RunOpts(cwd=dest))
-    if opts.setup_git:
-        run(git_command(dest, "init"))
-        run(git_command(dest, "add", "."))
-
-        # check if username is set
-        has_username = run(
-            git_command(dest, "config", "user.name"), RunOpts(check=False)
-        )
-        if has_username.returncode != 0:
-            run(git_command(dest, "config", "user.name", "clan-tool"))
-
-        has_username = run(
-            git_command(dest, "config", "user.email"), RunOpts(check=False)
-        )
-        if has_username.returncode != 0:
-            run(git_command(dest, "config", "user.email", "clan@example.com"))
-
-    if opts.update_clan:
-        run(nix_command(["flake", "update"]), RunOpts(cwd=dest))
 
     if opts.initial:
         inventory_store = InventoryStore(flake=Flake(str(opts.dest)))

pkgs/clan-cli/clan_lib/flake/flake.py

@@ -832,7 +832,8 @@ nix repl --expr 'rec {{
             """)
         # fmt: on
         elif len(selectors) == 1:
-            log.debug(f"""
+            log.debug(
+                f"""
 selecting: {selectors[0]}
 to debug run:
 nix repl --expr 'rec {{
@@ -840,11 +841,13 @@ nix repl --expr 'rec {{
   selectLib = (builtins.getFlake "path:{select_source()}?narHash={select_hash}").lib;
   query = selectLib.select '"''{selectors[0]}''"' flake;
 }}'
-            """)
+            """
+            )
 
         build_output = Path(
             run(
-                nix_build(["--expr", nix_code, *nix_options]), RunOpts(log=Log.NONE, trace=False),
+                nix_build(["--expr", nix_code, *nix_options]),
+                RunOpts(log=Log.NONE, trace=False),
             ).stdout.strip()
         )
 

pkgs/clan-cli/clan_lib/machines/actions.py

@@ -1,9 +1,9 @@
-from dataclasses import dataclass
 from typing import TypedDict
 
 from clan_lib.api import API
 from clan_lib.errors import ClanError
 from clan_lib.flake.flake import Flake
+from clan_lib.machines.machines import Machine
 from clan_lib.nix_models.clan import (
     InventoryMachine,
 )
@@ -65,16 +65,8 @@ def get_machine(flake: Flake, name: str) -> InventoryMachine:
     return InventoryMachine(**machine_inv)
 
 
-# TODO: remove this machine, once the Machine class is refactored
-# We added this now, to allow for dispatching actions. To require only 'name' and 'flake' of a machine.
-@dataclass(frozen=True)
-class MachineID:
-    name: str
-    flake: Flake
-
-
 @API.register
-def set_machine(machine: MachineID, update: InventoryMachine) -> None:
+def set_machine(machine: Machine, update: InventoryMachine) -> None:
     """
     Update the machine information in the inventory.
     """

pkgs/clan-cli/clan_lib/machines/machines.py

@@ -13,7 +13,6 @@ from clan_cli.vars._types import StoreBase
 from clan_lib.api import API
 from clan_lib.errors import ClanCmdError, ClanError
 from clan_lib.flake import Flake
-from clan_lib.machines.actions import get_machine
 from clan_lib.nix import nix_config
 from clan_lib.nix_models.clan import InventoryMachine
 from clan_lib.ssh.remote import Remote
@@ -39,6 +38,9 @@ class Machine:
         return cls(name=name, flake=flake)
 
     def get_inv_machine(self) -> "InventoryMachine":
+        # Import on demand to avoid circular imports
+        from clan_lib.machines.actions import get_machine
+
         return get_machine(self.flake, self.name)
 
     def get_id(self) -> str:

pkgs/clan-cli/clan_lib/templates/handler.py

@@ -7,7 +7,8 @@ from pathlib import Path
 from clan_lib.dirs import specific_machine_dir
 from clan_lib.errors import ClanError
 from clan_lib.flake import Flake
-from clan_lib.machines.actions import MachineID, list_machines
+from clan_lib.machines.actions import list_machines
+from clan_lib.machines.machines import Machine
 from clan_lib.templates.filesystem import copy_from_nixstore, realize_nix_path
 from clan_lib.templates.template_url import transform_url
 
@@ -76,15 +77,7 @@ def machine_template(
         msg = f"Template {printable_template_ref} is not a directory at {src_path}"
         raise ClanError(msg)
 
-    # TODO: Do we really need to check for a specific file in the template?
+    tmp_machine = Machine(flake=flake, name=dst_machine_name)
-    if not (src_path / "configuration.nix").exists():
-        msg = f"Template {printable_template_ref} does not contain a configuration.nix"
-        raise ClanError(
-            msg,
-            description="Template machine must contain a configuration.nix",
-        )
-
-    tmp_machine = MachineID(flake=flake, name=dst_machine_name)
 
     dst_machine_dir = specific_machine_dir(tmp_machine)
 
@@ -106,3 +99,73 @@ def machine_template(
     finally:
         # If no error occurred, the machine directory is kept
         pass
+
+
+@contextmanager
+def clan_template(flake: Flake, template_ident: str, dst_dir: Path) -> Iterator[Path]:
+    """
+    Create a clan from a template.
+    This function will copy the template files to a new clan directory
+
+    :param flake: The flake to create the machine in.
+    :param template_ident: The identifier of the template to use. Example ".#template_name"
+    :param dst: The name of the directory to create.
+
+
+    Example usage:
+
+    >>> with clan_template(
+    ...     Flake("/home/johannes/git/clan-core"), ".#new-machine", "my-machine"
+    ... ) as clan_dir:
+    ...     # Use `clan_dir` here if you want to access the created directory
+
+    ... The directory is removed if the context raised any errors.
+    ... Only if the context is exited without errors, it is kept.
+    """
+
+    # Get the clan template from the specifier
+    [flake_ref, template_selector] = transform_url("clan", template_ident, flake=flake)
+    # For pretty error messages
+    printable_template_ref = f"{flake_ref}#{template_selector}"
+
+    template_flake = Flake(flake_ref)
+
+    try:
+        template = template_flake.select(template_selector)
+    except ClanError as e:
+        msg = f"Failed to select template '{template_ident}' from flake '{flake_ref}' (via attribute path: {printable_template_ref})"
+        raise ClanError(msg) from e
+
+    src = template.get("path")
+    if not src:
+        msg = f"Malformed template: {printable_template_ref} does not have a 'path' attribute"
+        raise ClanError(msg)
+
+    src_path = Path(src).resolve()
+
+    realize_nix_path(template_flake, str(src_path))
+
+    if not src_path.exists():
+        msg = f"Template {printable_template_ref} does not exist at {src_path}"
+        raise ClanError(msg)
+
+    if not src_path.is_dir():
+        msg = f"Template {printable_template_ref} is not a directory at {src_path}"
+        raise ClanError(msg)
+
+    if dst_dir.exists():
+        msg = f"Destination directory {dst_dir} already exists"
+        raise ClanError(msg)
+
+    copy_from_nixstore(src_path, dst_dir)
+
+    try:
+        yield dst_dir
+    except Exception as e:
+        log.error(f"An error occurred inside the 'clan_template' context: {e}")
+        log.info(f"Removing left-over directory: {dst_dir}")
+        shutil.rmtree(dst_dir, ignore_errors=True)
+        raise
+    finally:
+        # If no error occurred, the directory is kept
+        pass

pkgs/clan-cli/clan_lib/templates/template_url_test.py

@@ -5,7 +5,7 @@ import pytest
 from clan_lib.errors import ClanError
 from clan_lib.templates.template_url import transform_url
 
-template_type = "machine"
+machine_template_type = "machine"
 
 
 class DummyFlake:
@@ -23,7 +23,18 @@ def test_transform_url_self_explizit_dot() -> None:
     user_input = ".#new-machine"
     expected_selector = 'clan.templates.machine."new-machine"'
 
-    flake_ref, selector = transform_url(template_type, user_input, flake=local_path)
+    flake_ref, selector = transform_url(
+        machine_template_type, user_input, flake=local_path
+    )
+    assert flake_ref == str(local_path.path)
+    assert selector == expected_selector
+
+
+def test_default_clan_template() -> None:
+    user_input = ".#default"
+    expected_selector = 'clan.templates.clan."default"'
+
+    flake_ref, selector = transform_url("clan", user_input, flake=local_path)
     assert flake_ref == str(local_path.path)
     assert selector == expected_selector
 
@@ -32,7 +43,9 @@ def test_transform_url_self_no_dot() -> None:
     user_input = "#new-machine"
     expected_selector = 'clan.templates.machine."new-machine"'
 
-    flake_ref, selector = transform_url(template_type, user_input, flake=local_path)
+    flake_ref, selector = transform_url(
+        machine_template_type, user_input, flake=local_path
+    )
     assert flake_ref == str(local_path.path)
     assert selector == expected_selector
 
@@ -41,7 +54,9 @@ def test_transform_url_builtin_template() -> None:
     user_input = "new-machine"
     expected_selector = 'clanInternals.templates.machine."new-machine"'
 
-    flake_ref, selector = transform_url(template_type, user_input, flake=local_path)
+    flake_ref, selector = transform_url(
+        machine_template_type, user_input, flake=local_path
+    )
     assert flake_ref == str(local_path.path)
     assert selector == expected_selector
 
@@ -50,7 +65,9 @@ def test_transform_url_remote_template() -> None:
     user_input = "github:/org/repo#new-machine"
     expected_selector = 'clan.templates.machine."new-machine"'
 
-    flake_ref, selector = transform_url(template_type, user_input, flake=local_path)
+    flake_ref, selector = transform_url(
+        machine_template_type, user_input, flake=local_path
+    )
 
     assert flake_ref == "github:/org/repo"
     assert selector == expected_selector
@@ -60,7 +77,9 @@ def test_transform_url_explicit_path() -> None:
     user_input = ".#clan.templates.machine.new-machine"
     expected_selector = "clan.templates.machine.new-machine"
 
-    flake_ref, selector = transform_url(template_type, user_input, flake=local_path)
+    flake_ref, selector = transform_url(
+        machine_template_type, user_input, flake=local_path
+    )
     assert flake_ref == str(local_path.path)
     assert selector == expected_selector
 
@@ -69,7 +88,9 @@ def test_transform_url_explicit_path() -> None:
 def test_transform_url_quoted_selector() -> None:
     user_input = '.#"new.machine"'
     expected_selector = '"new.machine"'
-    flake_ref, selector = transform_url(template_type, user_input, flake=local_path)
+    flake_ref, selector = transform_url(
+        machine_template_type, user_input, flake=local_path
+    )
     assert flake_ref == str(local_path.path)
     assert selector == expected_selector
 
@@ -77,7 +98,9 @@ def test_transform_url_quoted_selector() -> None:
 def test_single_quote_selector() -> None:
     user_input = ".#'new.machine'"
     expected_selector = "'new.machine'"
-    flake_ref, selector = transform_url(template_type, user_input, flake=local_path)
+    flake_ref, selector = transform_url(
+        machine_template_type, user_input, flake=local_path
+    )
     assert flake_ref == str(local_path.path)
     assert selector == expected_selector
 
@@ -86,7 +109,9 @@ def test_custom_template_path() -> None:
     user_input = "github:/org/repo#my.templates.custom.machine"
     expected_selector = "my.templates.custom.machine"
 
-    flake_ref, selector = transform_url(template_type, user_input, flake=local_path)
+    flake_ref, selector = transform_url(
+        machine_template_type, user_input, flake=local_path
+    )
     assert flake_ref == "github:/org/repo"
     assert selector == expected_selector
 
@@ -96,7 +121,9 @@ def test_full_url_query_and_fragment() -> None:
     expected_flake_ref = "github:/org/repo?query=param"
     expected_selector = "my.templates.custom.machine"
 
-    flake_ref, selector = transform_url(template_type, user_input, flake=local_path)
+    flake_ref, selector = transform_url(
+        machine_template_type, user_input, flake=local_path
+    )
     assert flake_ref == expected_flake_ref
     assert selector == expected_selector
 
@@ -114,7 +141,7 @@ def test_malformed_identifier() -> None:
     user_input = "github:/org/repo#my.templates.custom.machine#extra"
     with pytest.raises(ClanError) as exc_info:
         _flake_ref, _selector = transform_url(
-            template_type, user_input, flake=local_path
+            machine_template_type, user_input, flake=local_path
         )
 
     assert isinstance(exc_info.value, ClanError)
@@ -128,7 +155,9 @@ def test_locked_input_template() -> None:
     user_input = "locked-input#new-machine"
     expected_selector = 'inputs.locked-input.clan.templates.machine."new-machine"'
 
-    flake_ref, selector = transform_url(template_type, user_input, flake=local_path)
+    flake_ref, selector = transform_url(
+        machine_template_type, user_input, flake=local_path
+    )
     assert flake_ref == str(local_path.path)
     assert selector == expected_selector
 
@@ -137,7 +166,9 @@ def test_locked_input_template_no_quotes() -> None:
     user_input = 'locked-input#"new.machine"'
     expected_selector = 'inputs.locked-input."new.machine"'
 
-    flake_ref, selector = transform_url(template_type, user_input, flake=local_path)
+    flake_ref, selector = transform_url(
+        machine_template_type, user_input, flake=local_path
+    )
     assert selector == expected_selector
     assert flake_ref == str(local_path.path)
 
@@ -146,6 +177,8 @@ def test_locked_input_template_no_dot() -> None:
     user_input = "locked-input#new.machine"
     expected_selector = "inputs.locked-input.new.machine"
 
-    flake_ref, selector = transform_url(template_type, user_input, flake=local_path)
+    flake_ref, selector = transform_url(
+        machine_template_type, user_input, flake=local_path
+    )
     assert selector == expected_selector
     assert flake_ref == str(local_path.path)

pkgs/clan-cli/clan_lib/tests/test_create.py

@@ -137,7 +137,7 @@ def test_clan_create_api(
     # TODO: We need to generate a lock file for the templates
     clan_cli.clan.create.create_clan(
         clan_cli.clan.create.CreateOptions(
-            template_name="minimal", dest=dest_clan_dir, update_clan=False
+            template="minimal", dest=dest_clan_dir, update_clan=False
         )
     )
     assert dest_clan_dir.is_dir()

templates/flake.nix

@@ -1,8 +1,8 @@
 {
   outputs =
     { ... }:
-    {
+    let
-      clan.templates = {
+      templates = {
         disko = {
           single-disk = {
             description = "A simple ext4 disk with a single partition";
@@ -41,5 +41,11 @@
           };
         };
       };
+    in
+    rec {
+      inherit (clan) clanInternals;
+
+      clan.clanInternals.templates = templates;
+      clan.templates = templates;
     };
 }