diff --git a/checks/flake-module.nix b/checks/flake-module.nix index 7bedb0c61..73fc31aa1 100644 --- a/checks/flake-module.nix +++ b/checks/flake-module.nix @@ -12,6 +12,7 @@ in ./flash/flake-module.nix ./impure/flake-module.nix ./installation/flake-module.nix + ./morph/flake-module.nix ./nixos-documentation/flake-module.nix ]; perSystem = diff --git a/checks/lib/container-test.nix b/checks/lib/container-test.nix index 4fea982e1..f4da6ed67 100644 --- a/checks/lib/container-test.nix +++ b/checks/lib/container-test.nix @@ -31,6 +31,7 @@ in }; # to accept external dependencies such as disko node.specialArgs.self = self; + _module.args = { inherit self; }; imports = [ test ./container-driver/module.nix diff --git a/checks/lib/test-base.nix b/checks/lib/test-base.nix index 858f6a34c..594615e68 100644 --- a/checks/lib/test-base.nix +++ b/checks/lib/test-base.nix @@ -19,6 +19,7 @@ in } ); + _module.args = { inherit self; }; # to accept external dependencies such as disko node.specialArgs.self = self; imports = [ test ]; diff --git a/checks/morph/flake-module.nix b/checks/morph/flake-module.nix new file mode 100644 index 000000000..75d10ac39 --- /dev/null +++ b/checks/morph/flake-module.nix @@ -0,0 +1,62 @@ +{ + self, + ... +}: +{ + clan.machines.test-morph-machine = { + imports = [ + ./template/configuration.nix + self.nixosModules.clanCore + ]; + nixpkgs.hostPlatform = "x86_64-linux"; + environment.etc."testfile".text = "morphed"; + }; + + clan.templates.machine.test-morph-template = { + description = "Morph a machine"; + path = ./template; + }; + + perSystem = + { + pkgs, + ... + }: + { + checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux && pkgs.stdenv.hostPlatform.system != "aarch64-linux") { + test-morph = (import ../lib/test-base.nix) { + name = "morph"; + + nodes = { + actual = + { pkgs, ... }: + let + dependencies = [ + self + pkgs.nixos-anywhere + pkgs.stdenv.drvPath + pkgs.stdenvNoCC + self.nixosConfigurations.test-morph-machine.config.system.build.toplevel + self.nixosConfigurations.test-morph-machine.config.system.clan.deployment.file + ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs); + closureInfo = pkgs.closureInfo { rootPaths = dependencies; }; + in + + { + environment.etc."install-closure".source = "${closureInfo}/store-paths"; + system.extraDependencies = dependencies; + virtualisation.memorySize = 1024; + environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ]; + }; + }; + testScript = '' + start_all() + actual.fail("cat /etc/testfile") + actual.succeed("env CLAN_DIR=${self} clan machines morph test-morph-template --i-will-be-fired-for-using-this --debug --name test-morph-machine") + assert actual.succeed("cat /etc/testfile") == "morphed" + ''; + } { inherit pkgs self; }; + }; + + }; +} diff --git a/checks/morph/template/configuration.nix b/checks/morph/template/configuration.nix new file mode 100644 index 000000000..561385a3f --- /dev/null +++ b/checks/morph/template/configuration.nix @@ -0,0 +1,12 @@ +{ modulesPath, ... }: +{ + imports = [ + # we need these 2 modules always to be able to run the tests + (modulesPath + "/testing/test-instrumentation.nix") + (modulesPath + "/virtualisation/qemu-vm.nix") + + (modulesPath + "/profiles/minimal.nix") + ]; + + clan.core.setDefaults = false; +} diff --git a/nixosModules/clanCore/vars/secret/fs.nix b/nixosModules/clanCore/vars/secret/fs.nix new file mode 100644 index 000000000..d77278b22 --- /dev/null +++ b/nixosModules/clanCore/vars/secret/fs.nix @@ -0,0 +1,17 @@ +{ + config, + lib, + ... +}: +{ + config.clan.core.vars.settings = lib.mkIf (config.clan.core.vars.settings.secretStore == "fs") { + fileModule = file: { + path = + if file.config.neededFor == "partitioning" then + throw "${file.config.generatorName}/${file.config.name}: FS backend does not support partitioning." + else + "/run/secrets/${file.config.generatorName}/${file.config.name}"; + }; + secretModule = "clan_cli.vars.secret_modules.fs"; + }; +} diff --git a/pkgs/clan-cli/clan_cli/machines/cli.py b/pkgs/clan-cli/clan_cli/machines/cli.py index ac91199d2..674d66f66 100644 --- a/pkgs/clan-cli/clan_cli/machines/cli.py +++ b/pkgs/clan-cli/clan_cli/machines/cli.py @@ -6,6 +6,7 @@ from .delete import register_delete_parser from .hardware import register_update_hardware_config from .install import register_install_parser from .list import register_list_parser +from .morph import register_morph_parser from .update import register_update_parser @@ -46,6 +47,9 @@ For more detailed information, visit: https://docs.clan.lol/getting-started/depl delete_parser = subparser.add_parser("delete", help="Delete a machine") register_delete_parser(delete_parser) + morph_parser = subparser.add_parser("morph", help="morph a machine") + register_morph_parser(morph_parser) + list_parser = subparser.add_parser( "list", help="List machines", diff --git a/pkgs/clan-cli/clan_cli/machines/morph.py b/pkgs/clan-cli/clan_cli/machines/morph.py new file mode 100644 index 000000000..46981b9a3 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/machines/morph.py @@ -0,0 +1,180 @@ +import argparse +import json +import logging +import os +import random +import re +from pathlib import Path +from tempfile import TemporaryDirectory + +from clan_cli.cmd import Log, RunOpts, run +from clan_cli.dirs import get_clan_flake_toplevel_or_env +from clan_cli.errors import ClanError +from clan_cli.flake import Flake +from clan_cli.inventory import Machine as InventoryMachine +from clan_cli.machines.create import CreateOptions, create_machine +from clan_cli.machines.machines import Machine +from clan_cli.nix import nix_build, nix_command +from clan_cli.vars.generate import generate_vars + +log = logging.getLogger(__name__) + + +def is_local_input(node: dict[str, dict[str, str]]) -> bool: + locked = node.get("locked") + if not locked: + return False + # matches path and git+file:// + return ( + locked["type"] == "path" + or re.match(r"^\w+\+file://", locked.get("url", "")) is not None + ) + + +def random_hostname() -> str: + adjectives = ["wacky", "happy", "fluffy", "silly", "quirky", "zany", "bouncy"] + nouns = ["unicorn", "penguin", "goose", "ninja", "octopus", "hamster", "robot"] + adjective = random.choice(adjectives) + noun = random.choice(nouns) + return f"{adjective}-{noun}" + + +def morph_machine( + flake: Flake, template_name: str, ask_confirmation: bool, name: str | None = None +) -> None: + cmd = nix_command( + [ + "flake", + "archive", + "--json", + f"{flake}", + ] + ) + + archive_json = run( + cmd, RunOpts(error_msg="Failed to archive flake for morphing") + ).stdout.rstrip() + archive_path = json.loads(archive_json)["path"] + + with TemporaryDirectory(prefix="morph-") as temp_dir: + flakedir = Path(temp_dir) / "flake" + + flakedir.mkdir(parents=True, exist_ok=True) + run(["cp", "-r", archive_path + "/.", str(flakedir)]) + run(["chmod", "-R", "+w", str(flakedir)]) + + os.chdir(flakedir) + + if name is None: + name = random_hostname() + + create_opts = CreateOptions( + template_name=template_name, + machine=InventoryMachine(name=name), + clan_dir=Flake(str(flakedir)), + ) + create_machine(create_opts, commit=False) + + machine = Machine(name=name, flake=Flake(str(flakedir))) + + generate_vars([machine], generator_name=None, regenerate=False) + + machine.secret_vars_store.populate_dir( + output_dir=Path("/run/secrets"), phases=["activation", "users", "services"] + ) + + # run(["nixos-facter", "-o", f"{flakedir}/machines/{name}/facter.json"]).stdout + + # facter_json = run(["nixos-facter"]).stdout + # run(["cp", "facter.json", f"{flakedir}/machines/{name}/facter.json"]).stdout + + Path(f"{flakedir}/machines/{name}/facter.json").write_text( + '{"system": "x86_64-linux"}' + ) + result_path = run( + nix_build( + [f"{flakedir}#nixosConfigurations.{name}.config.system.build.toplevel"] + ) + ).stdout.rstrip() + + ropts = RunOpts(log=Log.BOTH) + + run( + [ + f"{result_path}/sw/bin/nixos-rebuild", + "dry-activate", + "--flake", + f"{flakedir}#{name}", + ], + ropts, + ).stdout.rstrip() + + if ask_confirmation: + log.warning("ARE YOU SURE YOU WANT TO DO THIS?") + log.warning( + "You should have read and understood all of the above and know what you are doing." + ) + + ask = input( + f"Do you really want convert this machine into {name}? If to continue, type in the new machine name: " + ) + if ask != name: + return + + run( + [ + f"{result_path}/sw/bin/nixos-rebuild", + "test", + "--flake", + f"{flakedir}#{name}", + ], + ropts, + ).stdout.rstrip() + + +def morph_command(args: argparse.Namespace) -> None: + if args.flake: + clan_dir = args.flake + else: + tmp = get_clan_flake_toplevel_or_env() + clan_dir = Flake(str(tmp)) if tmp else None + + if not clan_dir: + msg = "No clan found." + description = ( + "Run this command in a clan directory or specify the --flake option" + ) + raise ClanError(msg, description=description) + + morph_machine( + flake=Flake(str(args.flake)), + template_name=args.template_name, + ask_confirmation=args.confirm_firing, + name=args.name, + ) + + +def register_morph_parser(parser: argparse.ArgumentParser) -> None: + parser.set_defaults(func=morph_command) + + parser.add_argument( + "template_name", + default="new-machine", + type=str, + help="The name of the template to use", + nargs="?", + ) + + parser.add_argument( + "--name", + type=str, + help="The name of the machine", + ) + + parser.add_argument( + "--i-will-be-fired-for-using-this", + dest="confirm_firing", + default=True, + action="store_false", + help="Don't use unless you know what you are doing!", + ) diff --git a/pkgs/clan-cli/clan_cli/templates.py b/pkgs/clan-cli/clan_cli/templates.py index c3cc8aeae..97a7d897b 100644 --- a/pkgs/clan-cli/clan_cli/templates.py +++ b/pkgs/clan-cli/clan_cli/templates.py @@ -139,7 +139,7 @@ def copy_from_nixstore(src: Path, dest: Path) -> None: return # Walk through the source directory - for root, _dirs, files in src.walk(on_error=log.error): + for root, dirs, files in src.walk(on_error=log.error): relative_path = Path(root).relative_to(src) dest_dir = dest / relative_path @@ -150,6 +150,9 @@ def copy_from_nixstore(src: Path, dest: Path) -> None: stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC | stat.S_IRGRP | stat.S_IXGRP ) + for d in dirs: + (dest_dir / d).mkdir() + for file_name in files: src_file = Path(root) / file_name dest_file = dest_dir / file_name diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/fs.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/fs.py new file mode 100644 index 000000000..b4a57d204 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/fs.py @@ -0,0 +1,48 @@ +import shutil +from pathlib import Path + +from clan_cli.machines.machines import Machine +from clan_cli.vars._types import StoreBase +from clan_cli.vars.generate import Generator, Var + + +class SecretStore(StoreBase): + @property + def is_secret_store(self) -> bool: + return True + + def __init__(self, machine: Machine) -> None: + self.machine = machine + self.dir = Path("/run/secrets") + self.dir.mkdir(parents=True, exist_ok=True) + + @property + def store_name(self) -> str: + return "fs" + + def _set( + self, + generator: Generator, + var: Var, + value: bytes, + ) -> Path | None: + secret_file = self.dir / generator.name / var.name + secret_file.parent.mkdir(parents=True, exist_ok=True) + secret_file.write_bytes(value) + return None # we manage the files outside of the git repo + + def exists(self, generator: "Generator", name: str) -> bool: + return (self.dir / generator.name / name).exists() + + def get(self, generator: Generator, name: str) -> bytes: + secret_file = self.dir / generator.name / name + return secret_file.read_bytes() + + def populate_dir(self, output_dir: Path, phases: list[str]) -> None: + if output_dir.exists(): + shutil.rmtree(output_dir) + shutil.copytree(self.dir, output_dir) + + def upload(self, phases: list[str]) -> None: + msg = "Cannot upload secrets with FS backend" + raise NotImplementedError(msg)