Revert "Merge pull request 'Fix deploying with sudo + password' (#3470) from target-host into main"

This reverts commit 8a849eb90f, reversing
changes made to 3b5c22ebcf.

.gitignore

@@ -16,6 +16,9 @@ nixos.qcow2
 /docs/out
 **/.local.env
 
+# MacOS stuff
+**/.DS_store
+
 # dream2nix
 .dream2nix
 
@@ -39,3 +42,6 @@ repo
 node_modules
 dist
 .webui
+
+# TODO: remove after bug in select is fixed
+select

checks/flake-module.nix

@@ -55,11 +55,17 @@ in
             syncthing = import ./syncthing nixosTestArgs;
           };
 
+          packagesToBuild = lib.removeAttrs self'.packages [
+            # exclude the check that checks that nothing depends on the repo root
+            # We might want to include this later once everything is fixed
+            "dont-depend-on-repo-root"
+          ];
+
           flakeOutputs =
             lib.mapAttrs' (
               name: config: lib.nameValuePair "nixos-${name}" config.config.system.build.toplevel
             ) (lib.filterAttrs (n: _: !lib.hasPrefix "test-" n) self.nixosConfigurations)
-            // lib.mapAttrs' (n: lib.nameValuePair "package-${n}") self'.packages
+            // lib.mapAttrs' (n: lib.nameValuePair "package-${n}") packagesToBuild
             // lib.mapAttrs' (n: lib.nameValuePair "devShell-${n}") self'.devShells
             // lib.mapAttrs' (name: config: lib.nameValuePair "home-manager-${name}" config.activation-script) (
               self'.legacyPackages.homeConfigurations or { }

checks/installation/flake-module.nix

@@ -8,7 +8,6 @@ let
     { modulesPath, pkgs, ... }:
     let
       dependencies = [
-        self
         self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.toplevel
         self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.diskoScript
         self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.clan.deployment.file

checks/morph/flake-module.nix

@@ -44,7 +44,11 @@
               {
                 environment.etc."install-closure".source = "${closureInfo}/store-paths";
                 system.extraDependencies = dependencies;
+
                 virtualisation.memorySize = 2048;
+                virtualisation.useNixStoreImage = true;
+                virtualisation.writableStore = true;
+
                 environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli-full ];
               };
           };

checks/morph/template/configuration.nix

@@ -8,5 +8,8 @@
     (modulesPath + "/profiles/minimal.nix")
   ];
 
+  virtualisation.useNixStoreImage = true;
+  virtualisation.writableStore = true;
+
   clan.core.enableRecommendedDefaults = false;
 }

checks/sanity-checks/dont-depend-on-repo-root.nix

@@ -27,6 +27,7 @@
                   clan-core.checks.${system}
                   [
                     "dont-depend-on-repo-root"
+                    "package-dont-depend-on-repo-root"
                     "package-clan-core-flake"
                   ];
               checksOutPaths = map (x: "''${x}") (builtins.attrValues checks);
@@ -47,7 +48,7 @@
       '';
     in
     lib.optionalAttrs (system == "x86_64-linux") {
-      checks.dont-depend-on-repo-root =
+      packages.dont-depend-on-repo-root =
         pkgs.runCommand
           # append repo hash to this tests name to ensure it gets invalidated on each chain
           # This is needed because this test is an FOD (due to networking) and would get cached indefinitely.

clanModules/dyndns/default.nix

@@ -210,14 +210,18 @@ in
                   data_dir = Path('data')
                   data_dir.mkdir(mode=0o770, exist_ok=True)
 
+                  # Create a temporary config file
+                  # with appropriate permissions
+                  tmp_config_path = data_dir / '.config.json'
+                  tmp_config_path.touch(mode=0o660, exist_ok=False)
+
                   # Write the config with secrets back
-                  config_path = data_dir / 'config.json'
-                  with open(config_path, 'w') as f:
+                  with open(tmp_config_path, 'w') as f:
                       f.write(json.dumps(config, indent=4))
 
-                  # Set file permissions to read and write
-                  # only by the user and group
-                  config_path.chmod(0o660)
+                  # Move config into place
+                  config_path = data_dir / 'config.json'
+                  tmp_config_path.rename(config_path)
 
                   # Set file permissions to read
                   # and write only by the user and group

clanModules/root-password/README.md

@@ -7,8 +7,12 @@ features = [ "inventory" ]
 After the system was installed/deployed the following command can be used to display the root-password:
 
 ```bash
-clan secrets get {machine_name}-password
+clan vars get [machine_name] root-password/root-password
 ```
 
+See also: [Vars](../../manual/vars-backend.md)
 
-See also: [Facts / Secrets](../../getting-started/secrets.md)
+To regenerate the password run:
+```
+clan vars generate --regenerate [machine_name] --generator root-password
+```

clanModules/user-password/README.md

@@ -13,9 +13,12 @@ If setting the option prompt to true, the user will be prompted to type in their
 After the system was installed/deployed the following command can be used to display the user-password:
 
 ```bash
-clan secrets get {machine_name}-user-password
+clan vars get [machine_name] root-password/root-password
 ```
 
-See also: [Facts / Secrets](../../getting-started/secrets.md)
+See also: [Vars](../../manual/vars-backend.md)
 
-To regenerate the password, delete the password files in the clan directory and redeploy the machine.
+To regenerate the password run:
+```
+clan vars generate --regenerate [machine_name] --generator user-password
+```

docs/nix/deploy-docs.nix

@@ -26,8 +26,7 @@ writeShellScriptBin "deploy-docs" ''
   trap "rm -rf $tmpdir" EXIT
 
   if [ -n "''${SSH_HOMEPAGE_KEY-}" ]; then
-    echo "$SSH_HOMEPAGE_KEY" > "$tmpdir/ssh_key"
-    chmod 600 "$tmpdir/ssh_key"
+    ( umask 0177 && echo "$SSH_HOMEPAGE_KEY" > "$tmpdir/ssh_key" )
     sshExtraArgs="-i $tmpdir/ssh_key"
   else
     sshExtraArgs=

docs/site/reference/index.md

@@ -1,6 +1,6 @@
 # :material-api: Overview
 
-This section of the site provides an **automatically extracted** overview of the available options and commands within the Clan Framework.
+This section of the site provides an overview of available options and commands within the Clan Framework.
 
 ---
 

flake.lock

@@ -16,17 +16,15 @@
         ]
       },
       "locked": {
-        "lastModified": 1745889637,
-        "narHash": "sha256-+BW9rppchFYIiJldD+fZA3MS2OtPNrb8l27SC3GyoSk=",
-        "ref": "refs/heads/main",
-        "rev": "11b5673d9c7290a6b96c2b6c6c5be600304f310f",
-        "revCount": 415,
-        "type": "git",
-        "url": "https://git.clan.lol/clan/data-mesher"
+        "lastModified": 1746334246,
+        "narHash": "sha256-YU4wtH9Y5yRjqbMwczOdDakOjSiTkOUP/JAYd1f3jBc=",
+        "rev": "607ce65fbfe20bb38170b76826a11006f526c05d",
+        "type": "tarball",
+        "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/607ce65fbfe20bb38170b76826a11006f526c05d.tar.gz"
       },
       "original": {
-        "type": "git",
-        "url": "https://git.clan.lol/clan/data-mesher"
+        "type": "tarball",
+        "url": "https://git.clan.lol/clan/data-mesher/archive/main.tar.gz"
       }
     },
     "disko": {
@@ -76,11 +74,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1745816321,
-        "narHash": "sha256-Gyh/fkCDqVNGM0BWvk+4UAS17w2UI6iwnbQQCmc1TDI=",
+        "lastModified": 1746254942,
+        "narHash": "sha256-Y062AuRx6l+TJNX8wxZcT59SSLsqD9EedAY0mqgTtQE=",
         "owner": "nix-darwin",
         "repo": "nix-darwin",
-        "rev": "4515dacafb0ccd42e5395aacc49fd58a43027e01",
+        "rev": "760a11c87009155afa0140d55c40e7c336d62d7a",
         "type": "github"
       },
       "original": {
@@ -93,15 +91,13 @@
       "locked": {
         "lastModified": 1745005516,
         "narHash": "sha256-IVaoOGDIvAa/8I0sdiiZuKptDldrkDWUNf/+ezIRhyc=",
-        "ref": "refs/heads/main",
         "rev": "69d8bf596194c5c35a4e90dd02c52aa530caddf8",
-        "revCount": 40,
-        "type": "git",
-        "url": "https://git.clan.lol/clan/nix-select"
+        "type": "tarball",
+        "url": "https://git.clan.lol/api/v1/repos/clan/nix-select/archive/69d8bf596194c5c35a4e90dd02c52aa530caddf8.tar.gz"
       },
       "original": {
-        "type": "git",
-        "url": "https://git.clan.lol/clan/nix-select"
+        "type": "tarball",
+        "url": "https://git.clan.lol/clan/nix-select/archive/main.tar.gz"
       }
     },
     "nixos-facter-modules": {
@@ -122,10 +118,10 @@
     "nixpkgs": {
       "locked": {
         "lastModified": 315532800,
-        "narHash": "sha256-+Elxpf3FLkgKfh81xrEjVolpJEn8+fKWqEJ3ZXbAbS4=",
-        "rev": "29335f23bea5e34228349ea739f31ee79e267b88",
+        "narHash": "sha256-pxwYhAgOyComW58BCfboADZWr4b5oS8hP9E9fQ489HM=",
+        "rev": "f21e4546e3ede7ae34d12a84602a22246b31f7e0",
         "type": "tarball",
-        "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.05pre791229.29335f23bea5/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.05pre793694.f21e4546e3ed/nixexprs.tar.xz"
       },
       "original": {
         "type": "tarball",
@@ -188,11 +184,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1745929750,
-        "narHash": "sha256-k5ELLpTwRP/OElcLpNaFWLNf8GRDq4/eHBmFy06gGko=",
+        "lastModified": 1746216483,
+        "narHash": "sha256-4h3s1L/kKqt3gMDcVfN8/4v2jqHrgLIe4qok4ApH5x4=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "82bf32e541b30080d94e46af13d46da0708609ea",
+        "rev": "29ec5026372e0dec56f890e50dbe4f45930320fd",
         "type": "github"
       },
       "original": {

flake.nix

@@ -23,10 +23,10 @@
     treefmt-nix.url = "github:numtide/treefmt-nix";
     treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
 
-    nix-select.url = "git+https://git.clan.lol/clan/nix-select";
+    nix-select.url = "https://git.clan.lol/clan/nix-select/archive/main.tar.gz";
 
     data-mesher = {
-      url = "git+https://git.clan.lol/clan/data-mesher";
+      url = "https://git.clan.lol/clan/data-mesher/archive/main.tar.gz";
       inputs = {
         flake-parts.follows = "flake-parts";
         nixpkgs.follows = "nixpkgs";
@@ -40,7 +40,6 @@
     inputs@{
       flake-parts,
       nixpkgs,
-      self,
       systems,
       ...
     }:

lib/build-clan/interface.nix

@@ -160,7 +160,6 @@ in
           # Those options are interfaced by the CLI
           # We don't specify the type here, for better performance.
           inventory = lib.mkOption { type = lib.types.raw; };
-          inventoryValuesPrios = lib.mkOption { type = lib.types.raw; };
           # all exported clan templates from this clan
           templates = lib.mkOption { type = lib.types.raw; };
           # all exported clan modules from this clan

lib/build-clan/module.nix

@@ -210,14 +210,12 @@ in
     modules = config.modules;
 
     inherit inventoryFile;
-    inventoryValuesPrios =
-      # Temporary workaround
-      builtins.removeAttrs (clan-core.clanLib.introspection.getPrios { options = inventory.options; })
-        # tags are freeformType which is not supported yet.
-        [ "tags" ];
 
     templates = config.templates;
     inventory = config.inventory;
+    # TODO: Remove this in about a month
+    # It is only here for backwards compatibility for people with older CLI versions
+    inventoryValuesPrios = inventoryClass.introspection;
     meta = config.inventory.meta;
     secrets = config.secrets;
 

lib/inventory/build-inventory/default.nix

@@ -33,6 +33,7 @@ let
             }) config.distributedServices.allMachines;
           }
         )
+        (lib.modules.importApply ./inventory-introspection.nix { inherit clanLib; })
       ];
     }).config;
 in

lib/inventory/build-inventory/inventory-introspection.nix

@@ -0,0 +1,17 @@
+{ clanLib }:
+{
+  config,
+  options,
+  lib,
+  ...
+}:
+{
+  options.introspection = lib.mkOption {
+    readOnly = true;
+    # TODO: use options.inventory instead of the evaluate config attribute
+    default =
+      builtins.removeAttrs (clanLib.introspection.getPrios { options = config.inventory.options; })
+        # tags are freeformType which is not supported yet.
+        [ "tags" ];
+  };
+}

nixosModules/clanCore/vars/interface.nix

@@ -241,12 +241,30 @@ in
                       type = bool;
                       default = true;
                     };
+                    flakePath = lib.mkOption {
+                      description = ''
+                        The path to the file containing the content of the generated value.
+                        This will be set automatically
+                      '';
+                      type = nullOr str;
+                      default = null;
+                    };
                     path = lib.mkOption {
                       description = ''
                         The path to the file containing the content of the generated value.
                         This will be set automatically
                       '';
                       type = str;
+                      defaultText = ''
+                        builtins.path {
+                          name = "$${generator.config._module.args.name}_$${file.config._module.args.name}";
+                          path = file.config.inRepoPath;
+                        }
+                      '';
+                      default = builtins.path {
+                        name = "${generator.config._module.args.name}_${file.config._module.args.name}";
+                        path = file.config.flakePath;
+                      };
                     };
                     neededFor = lib.mkOption {
                       description = ''

nixosModules/clanCore/vars/public/in_repo.nix

@@ -11,7 +11,7 @@ in
   config.clan.core.vars.settings = mkIf (config.clan.core.vars.settings.publicStore == "in_repo") {
     publicModule = "clan_cli.vars.public_modules.in_repo";
     fileModule = file: {
-      path = mkIf (file.config.secret == false) (
+      flakePath = mkIf (file.config.secret == false) (
         if file.config.share then
           (
             config.clan.core.settings.directory
@@ -25,9 +25,9 @@ in
       );
       value = mkIf (file.config.secret == false) (
         # dynamically adjust priority to allow overriding with mkDefault in case the file is not found
-        if (pathExists file.config.path) then
+        if (pathExists file.config.flakePath) then
           # if the file is found it should have normal priority
-          readFile file.config.path
+          readFile file.config.flakePath
         else
           # if the file is not found, we want to downgrade the priority, to allow overriding via mkDefault
           mkOptionDefault (

nixosModules/clanCore/vars/secret/sops/default.nix

@@ -49,7 +49,10 @@ in
             mode
             neededForUsers
             ;
-          sopsFile = secretPath secret;
+          sopsFile = builtins.path {
+            name = "${secret.generator}_${secret.name}";
+            path = secretPath secret;
+          };
           format = "binary";
         };
       }) (builtins.filter (x: builtins.pathExists (secretPath x)) vars)

pkgs/clan-cli/clan_cli/clan/show.py

@@ -15,7 +15,10 @@ log = logging.getLogger(__name__)
 
 
 @API.register
-def show_clan_meta(uri: str | Path) -> Meta:
+def show_clan_meta(uri: str) -> Meta:
+    if uri.startswith("/") and not Path(uri).exists():
+        msg = f"Path {uri} does not exist"
+        raise ClanError(msg, description="clan directory does not exist")
     cmd = nix_eval(
         [
             f"{uri}#clanInternals.inventory.meta",

pkgs/clan-cli/clan_cli/cmd.py

@@ -244,12 +244,12 @@ class TimeTable:
                     # Print in default color
                     print(f"Took {v} for command: '{k}'")
 
-    def add(self, cmd: str, time: float) -> None:
+    def add(self, cmd: str, duration: float) -> None:
         with self.lock:
             if cmd in self.table:
-                self.table[cmd] += time
+                self.table[cmd] += duration
             else:
-                self.table[cmd] = time
+                self.table[cmd] = duration
 
 
 TIME_TABLE = None
@@ -259,7 +259,7 @@ if os.environ.get("CLAN_CLI_PERF"):
 
 @dataclass
 class RunOpts:
-    input: bytes | None = None
+    input: IO[bytes] | bytes | None = None
     stdout: IO[bytes] | None = None
     stderr: IO[bytes] | None = None
     env: dict[str, str] | None = None
@@ -329,7 +329,7 @@ def run(
     if options.requires_root_perm:
         cmd = cmd_with_root(cmd, options.graphical_perm)
 
-    if options.input:
+    if options.input and isinstance(options.input, bytes):
         if any(not ch.isprintable() for ch in options.input.decode("ascii", "replace")):
             filtered_input = "<<binary_blob>>"
         else:
@@ -344,7 +344,7 @@ def run(
 
     start = timeit.default_timer()
     with ExitStack() as stack:
-        stdin = subprocess.PIPE if options.input is not None else None
+        stdin = subprocess.PIPE if isinstance(options.input, bytes) else options.input
         process = stack.enter_context(
             subprocess.Popen(
                 cmd,
@@ -364,13 +364,18 @@ def run(
         else:
             stack.enter_context(terminate_process_group(process))
 
+        if isinstance(options.input, bytes):
+            input_bytes = options.input
+        else:
+            input_bytes = None
+
         stdout_buf, stderr_buf = handle_io(
             process,
             options.log,
             prefix=options.prefix,
             msg_color=options.msg_color,
             timeout=options.timeout,
-            input_bytes=options.input,
+            input_bytes=input_bytes,
             stdout=options.stdout,
             stderr=options.stderr,
         )
@@ -418,6 +423,3 @@ def run_no_stdout(
         cmd,
         opts,
     )
-
-
-# type: ignore

pkgs/clan-cli/clan_cli/facts/secret_modules/sops.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from typing import override
 
 from clan_cli.machines.machines import Machine
 from clan_cli.secrets.folders import sops_secrets_folder
@@ -58,13 +59,10 @@ class SecretStore(SecretStoreBase):
             sops_secrets_folder(self.machine.flake_dir) / f"{self.machine.name}-{name}",
         )
 
+    @override
+    def needs_upload(self) -> bool:
+        return False
+
+    # We rely now on the vars backend to upload the age key
     def upload(self, output_dir: Path) -> None:
-        key_name = f"{self.machine.name}-age.key"
-        if not has_secret(sops_secrets_folder(self.machine.flake_dir) / key_name):
-            # skip uploading the secret, not managed by us
-            return
-        key = decrypt_secret(
-            self.machine.flake_dir,
-            sops_secrets_folder(self.machine.flake_dir) / key_name,
-        )
-        (output_dir / "key.txt").write_text(key)
+        pass

pkgs/clan-cli/clan_cli/flake.py

@@ -344,9 +344,6 @@ class FlakeCacheEntry:
 
     def is_cached(self, selectors: list[Selector]) -> bool:
         selector: Selector
-        if selectors == []:
-            return self.fetched_all
-        selector = selectors[0]
 
         # for store paths we have to check if they still exist, otherwise they have to be rebuild and are thus not cached
         if isinstance(self.value, str) and self.value.startswith("/nix/store/"):
@@ -356,6 +353,10 @@ class FlakeCacheEntry:
         if isinstance(self.value, str | float | int | None):
             return True
 
+        if selectors == []:
+            return self.fetched_all
+        selector = selectors[0]
+
         # we just fetch all subkeys, so we need to check of we inserted all keys at this level before
         if selector.type == SelectorType.ALL:
             assert isinstance(self.value, dict)
@@ -458,7 +459,7 @@ class FlakeCacheEntry:
                 result = []
                 for index in keys_to_select:
                     result.append(self.value[index].select(selectors[1:]))
-                    return result
+                return result
 
             # otherwise return a dict
             return {k: self.value[k].select(selectors[1:]) for k in keys_to_select}

pkgs/clan-cli/clan_cli/inventory/__init__.py

@@ -375,7 +375,7 @@ def get_inventory_current_priority(flake_dir: str | Path) -> dict:
     """
     cmd = nix_eval(
         [
-            f"{flake_dir}#clanInternals.inventoryValuesPrios",
+            f"{flake_dir}#clanInternals.inventoryClass.introspection",
             "--json",
         ]
     )

pkgs/clan-cli/clan_cli/machines/hardware.py

@@ -7,14 +7,14 @@ from pathlib import Path
 
 from clan_lib.api import API
 
-from clan_cli.cmd import RunOpts, run, run_no_stdout
+from clan_cli.cmd import RunOpts, run_no_stdout
 from clan_cli.completions import add_dynamic_completer, complete_machines
 from clan_cli.dirs import specific_machine_dir
 from clan_cli.errors import ClanCmdError, ClanError
 from clan_cli.flake import Flake
 from clan_cli.git import commit_file
 from clan_cli.machines.machines import Machine
-from clan_cli.nix import nix_config, nix_eval, nix_shell
+from clan_cli.nix import nix_config, nix_eval
 
 from .types import machine_name_type
 
@@ -119,6 +119,10 @@ def generate_machine_hardware_info(opts: HardwareGenerateOptions) -> HardwareCon
     """
 
     machine = Machine(opts.machine, flake=opts.flake)
+
+    if opts.keyfile is not None:
+        machine.private_key = Path(opts.keyfile)
+
     if opts.target_host is not None:
         machine.override_target_host = opts.target_host
 
@@ -136,41 +140,19 @@ def generate_machine_hardware_info(opts: HardwareGenerateOptions) -> HardwareCon
         ]
 
     host = machine.target_host
-
-    # HACK: to make non-root user work
-    if host.user != "root":
-        config_command.insert(0, "sudo")
-
-    deps = ["openssh"]
+    host.ssh_options["StrictHostKeyChecking"] = "accept-new"
+    host.ssh_options["UserKnownHostsFile"] = "/dev/null"
     if opts.password:
-        deps += ["sshpass"]
+        host.password = opts.password
 
-    cmd = nix_shell(
-        deps,
-        [
-            *(["sshpass", "-p", opts.password] if opts.password else []),
-            "ssh",
-            *(["-i", f"{opts.keyfile}"] if opts.keyfile else []),
-            # Disable known hosts file
-            "-o",
-            "UserKnownHostsFile=/dev/null",
-            # Disable strict host key checking. The GUI user cannot type "yes" into the ssh terminal.
-            "-o",
-            "StrictHostKeyChecking=accept-new",
-            *(
-                ["-p", str(machine.target_host.port)]
-                if machine.target_host.port
-                else []
-            ),
-            host.target,
-            *config_command,
-        ],
-    )
-    out = run(cmd, RunOpts(needs_user_terminal=True, prefix=machine.name, check=False))
+    out = host.run(config_command, become_root=True, opts=RunOpts(check=False))
     if out.returncode != 0:
         if "nixos-facter" in out.stderr and "not found" in out.stderr:
             machine.error(str(out.stderr))
-            msg = "Please use our custom nixos install images. nixos-factor only works on nixos / clan systems currently."
+            msg = (
+                "Please use our custom nixos install images from https://github.com/nix-community/nixos-images/releases/tag/nixos-unstable. "
+                "nixos-factor only works on nixos / clan systems currently."
+            )
             raise ClanError(msg)
 
         machine.error(str(out))

pkgs/clan-cli/clan_cli/machines/update.py

@@ -69,7 +69,12 @@ def upload_sources(machine: Machine) -> str:
         )
         run(
             cmd,
-            RunOpts(env=env, error_msg="failed to upload sources", prefix=machine.name),
+            RunOpts(
+                env=env,
+                needs_user_terminal=True,
+                error_msg="failed to upload sources",
+                prefix=machine.name,
+            ),
         )
         return path
 
@@ -84,7 +89,12 @@ def upload_sources(machine: Machine) -> str:
             flake_url,
         ]
     )
-    proc = run(cmd, RunOpts(env=env, error_msg="failed to upload sources"))
+    proc = run(
+        cmd,
+        RunOpts(
+            env=env, needs_user_terminal=True, error_msg="failed to upload sources"
+        ),
+    )
 
     try:
         return json.loads(proc.stdout)["path"]
@@ -108,13 +118,9 @@ def update_machines(base_path: str, machines: list[InventoryMachine]) -> None:
             name,
             flake=flake,
         )
-        if not machine.get("deploy", {}).get("targetHost"):
-            msg = f"'TargetHost' is not set for machine '{name}'"
-            raise ClanError(msg)
-        # Copy targetHost to machine
-        m.override_target_host = machine.get("deploy", {}).get("targetHost")
-        # Would be nice to have?
-        # m.override_build_host = machine.deploy.buildHost
+        # prefer target host set via inventory, but fallback to the one set in the machine
+        if target_host := machine.get("deploy", {}).get("targetHost"):
+            m.override_target_host = target_host
         group_machines.append(m)
 
     deploy_machines(group_machines)
@@ -197,7 +203,10 @@ def deploy_machines(machines: list[Machine]) -> None:
                 )
             ret = host.run(
                 test_cmd if is_mobile else switch_cmd,
-                RunOpts(msg_color=MsgColor(stderr=AnsiColor.DEFAULT)),
+                RunOpts(
+                    msg_color=MsgColor(stderr=AnsiColor.DEFAULT),
+                    needs_user_terminal=True,
+                ),
                 extra_env=env,
                 become_root=become_root,
             )
@@ -206,7 +215,7 @@ def deploy_machines(machines: list[Machine]) -> None:
         for machine in machines:
             if machine._class_ == "darwin":
                 if not machine.deploy_as_root and machine.target_host.user == "root":
-                    msg = f"'TargetHost' should be set to a non-root user for deploying to nix-darwin on machine '{machine.name}'"
+                    msg = f"'targetHost' should be set to a non-root user for deploying to nix-darwin on machine '{machine.name}'"
                     raise ClanError(msg)
 
             machine.info(f"Updating {machine.name}")

pkgs/clan-cli/clan_cli/secrets/sops.py

@@ -18,7 +18,8 @@ from clan_lib.api import API
 from clan_cli.cmd import Log, RunOpts, run
 from clan_cli.dirs import user_config_dir
 from clan_cli.errors import ClanError
-from clan_cli.nix import nix_eval, nix_shell
+from clan_cli.flake import Flake
+from clan_cli.nix import nix_shell
 
 from .folders import sops_users_folder
 
@@ -196,26 +197,11 @@ def load_age_plugins(flake_dir: str | Path) -> list[str]:
         msg = "Missing flake directory"
         raise ClanError(msg)
 
-    cmd = nix_eval(
-        [
-            f"{flake_dir}#clanInternals.secrets.age.plugins",
-            "--json",
-        ]
-    )
-
-    try:
-        result = run(cmd)
-    except Exception as e:
-        msg = f"Failed to load age plugins {flake_dir}"
-        raise ClanError(msg) from e
-
-    json_str = result.stdout.strip()
-
-    try:
-        plugins = json.loads(json_str)
-    except json.JSONDecodeError as e:
-        msg = f"Failed to decode '{json_str}': {e}"
-        raise ClanError(msg) from e
+    flake = Flake(str(flake_dir))
+    result = flake.select("clanInternals.?secrets.?age.?plugins")
+    plugins = result["secrets"]["age"]["plugins"]
+    if plugins == {}:
+        plugins = []
 
     if isinstance(plugins, list):
         return plugins

pkgs/clan-cli/clan_cli/ssh/deploy_info.py

@@ -87,7 +87,7 @@ def ssh_shell_from_deploy(
     deploy_info: DeployInfo, runtime: AsyncRuntime, host_key_check: HostKeyCheck
 ) -> None:
     if host := find_reachable_host(deploy_info, host_key_check):
-        host.connect_ssh_shell(password=deploy_info.pwd)
+        host.interactive_ssh()
     else:
         log.info("Could not reach host via clearnet 'addrs'")
         log.info(f"Trying to reach host via tor '{deploy_info.tor}'")
@@ -96,8 +96,7 @@ def ssh_shell_from_deploy(
             msg = "No tor address provided, please provide a tor address."
             raise ClanError(msg)
         if ssh_tor_reachable(TorTarget(onion=deploy_info.tor, port=22)):
-            host = Host(host=deploy_info.tor)
-            host.connect_ssh_shell(password=deploy_info.pwd, tor_socks=True)
+            host = Host(host=deploy_info.tor, password=deploy_info.pwd, tor_socks=True)
         else:
             msg = "Could not reach host via tor either."
             raise ClanError(msg)

pkgs/clan-cli/clan_cli/ssh/host.py

@@ -5,6 +5,8 @@ import os
 import shlex
 import socket
 import subprocess
+import errno
+import stat
 from dataclasses import dataclass, field
 from pathlib import Path
 from shlex import quote
@@ -29,18 +31,44 @@ class Host:
     user: str | None = None
     port: int | None = None
     private_key: Path | None = None
+    password: str | None = None
     forward_agent: bool = False
     command_prefix: str | None = None
     host_key_check: HostKeyCheck = HostKeyCheck.ASK
     meta: dict[str, Any] = field(default_factory=dict)
     verbose_ssh: bool = False
     ssh_options: dict[str, str] = field(default_factory=dict)
+    tor_socks: bool = False
+
+    def setup_control_master(self) -> None:
+        home = Path.home()
+        if not home.exists():
+            return
+        control_path = home / ".ssh"
+        try:
+            if not stat.S_ISDIR(control_path.stat().st_mode):
+                return
+        except OSError as e:
+            if e.errno == errno.ENOENT:
+                try:
+                    control_path.mkdir(exist_ok=True)
+                except OSError:
+                    return
+            else:
+                return
+
+        self.ssh_options["ControlMaster"] = "auto"
+        # Can we make this a temporary directory?
+        self.ssh_options["ControlPath"] = str(control_path / "clan-%h-%p-%r")
+        # We use a short ttl because we want to mainly re-use the connection during the cli run
+        self.ssh_options["ControlPersist"] = "1m"
 
     def __post_init__(self) -> None:
         if not self.command_prefix:
             self.command_prefix = self.host
         if not self.user:
             self.user = "root"
+        self.setup_control_master()
 
     def __str__(self) -> str:
         return self.target
@@ -106,6 +134,9 @@ class Host:
         if extra_env is None:
             extra_env = {}
 
+        if opts is None:
+            opts = RunOpts()
+
         # If we are not root and we need to become root, prepend sudo
         sudo = ""
         if become_root and self.user != "root":
@@ -116,11 +147,10 @@ class Host:
         for k, v in extra_env.items():
             env_vars.append(f"{shlex.quote(k)}={shlex.quote(v)}")
 
-        if opts is None:
-            opts = RunOpts()
-        else:
-            opts.needs_user_terminal = True
+        if opts.prefix is None:
             opts.prefix = self.command_prefix
+        # always set needs_user_terminal to True because ssh asks for passwords
+        opts.needs_user_terminal = True
 
         if opts.cwd is not None:
             msg = "cwd is not supported for remote commands"
@@ -185,18 +215,16 @@ class Host:
     def ssh_cmd(
         self,
         verbose_ssh: bool = False,
-        tor_socks: bool = False,
         tty: bool = False,
-        password: str | None = None,
     ) -> list[str]:
         packages = []
         password_args = []
-        if password:
+        if self.password:
             packages.append("sshpass")
             password_args = [
                 "sshpass",
                 "-p",
-                password,
+                self.password,
             ]
 
         ssh_opts = self.ssh_cmd_opts
@@ -205,7 +233,7 @@ class Host:
         if tty:
             ssh_opts.extend(["-t"])
 
-        if tor_socks:
+        if self.tor_socks:
             packages.append("netcat")
             ssh_opts.append("-o")
             ssh_opts.append("ProxyCommand=nc -x 127.0.0.1:9050 -X 5 %h %p")
@@ -219,12 +247,8 @@ class Host:
 
         return nix_shell(packages, cmd)
 
-    def connect_ssh_shell(
-        self, *, password: str | None = None, tor_socks: bool = False
-    ) -> None:
-        cmd = self.ssh_cmd(tor_socks=tor_socks, password=password)
-
-        subprocess.run(cmd)
+    def interactive_ssh(self) -> None:
+        subprocess.run(self.ssh_cmd())
 
 
 def is_ssh_reachable(host: Host) -> bool:

pkgs/clan-cli/clan_cli/tests/test_secrets_upload.py

@@ -1,98 +0,0 @@
-from pathlib import Path
-from typing import TYPE_CHECKING
-
-import pytest
-from clan_cli.ssh.host import Host, HostKeyCheck
-from clan_cli.ssh.upload import upload
-from clan_cli.tests.fixtures_flakes import ClanFlake
-from clan_cli.tests.helpers import cli
-
-if TYPE_CHECKING:
-    from .age_keys import KeyPair
-
-
-@pytest.mark.with_core
-def test_upload_single_file(
-    monkeypatch: pytest.MonkeyPatch,
-    temporary_home: Path,
-    hosts: list[Host],
-) -> None:
-    host = hosts[0]
-    host.host_key_check = HostKeyCheck.NONE
-
-    src_file = temporary_home / "test.txt"
-    src_file.write_text("test")
-    dest_file = temporary_home / "test_dest.txt"
-
-    upload(host, src_file, dest_file)
-
-    assert dest_file.exists()
-    assert dest_file.read_text() == "test"
-
-
-@pytest.mark.with_core
-def test_secrets_upload(
-    monkeypatch: pytest.MonkeyPatch,
-    flake: ClanFlake,
-    hosts: list[Host],
-    age_keys: list["KeyPair"],
-) -> None:
-    config = flake.machines["vm1"]
-    config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
-    host = hosts[0]
-    addr = f"{host.user}@{host.host}:{host.port}?StrictHostKeyChecking=no&UserKnownHostsFile=/dev/null&IdentityFile={host.private_key}"
-    config["clan"]["networking"]["targetHost"] = addr
-    config["clan"]["core"]["facts"]["secretUploadDirectory"] = str(flake.path / "facts")
-    flake.refresh()
-
-    with monkeypatch.context():
-        monkeypatch.chdir(str(flake.path))
-        monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
-
-        sops_dir = flake.path / "facts"
-
-        # the flake defines this path as the location where the sops key should be installed
-        sops_key = sops_dir / "key.txt"
-        sops_key2 = sops_dir / "key2.txt"
-
-        # Create old state, which should be cleaned up
-        sops_dir.mkdir()
-        sops_key.write_text("OLD STATE")
-        sops_key2.write_text("OLD STATE2")
-
-        cli.run(
-            [
-                "secrets",
-                "users",
-                "add",
-                "--flake",
-                str(flake.path),
-                "user1",
-                age_keys[0].pubkey,
-            ]
-        )
-
-        cli.run(
-            [
-                "secrets",
-                "machines",
-                "add",
-                "--flake",
-                str(flake.path),
-                "vm1",
-                age_keys[1].pubkey,
-            ]
-        )
-
-        with monkeypatch.context() as m:
-            m.setenv("SOPS_NIX_SECRET", age_keys[0].privkey)
-
-            cli.run(["secrets", "set", "--flake", str(flake.path), "vm1-age.key"])
-
-            flake_path = flake.path.joinpath("flake.nix")
-
-            cli.run(["facts", "upload", "--flake", str(flake_path), "vm1"])
-
-            assert sops_key.exists()
-            assert sops_key.read_text() == age_keys[0].privkey
-            assert not sops_key2.exists()

pkgs/clan-cli/clan_cli/tests/test_ssh_remote.py

@@ -117,7 +117,9 @@ def test_parse_deployment_address(
     assert result.user == expected_user or (
         expected_user == "" and result.user == "root"
     )
-    assert result.ssh_options == expected_options
+
+    for key, value in expected_options.items():
+        assert result.ssh_options[key] == value
 
 
 def test_parse_ssh_options() -> None:

pkgs/clan-cli/clan_cli/tests/test_upload_single_file.py

@@ -0,0 +1,24 @@
+from pathlib import Path
+
+import pytest
+from clan_cli.ssh.host import Host, HostKeyCheck
+from clan_cli.ssh.upload import upload
+
+
+@pytest.mark.with_core
+def test_upload_single_file(
+    monkeypatch: pytest.MonkeyPatch,
+    temporary_home: Path,
+    hosts: list[Host],
+) -> None:
+    host = hosts[0]
+    host.host_key_check = HostKeyCheck.NONE
+
+    src_file = temporary_home / "test.txt"
+    src_file.write_text("test")
+    dest_file = temporary_home / "test_dest.txt"
+
+    upload(host, src_file, dest_file)
+
+    assert dest_file.exists()
+    assert dest_file.read_text() == "test"

pkgs/clan-cli/clan_cli/tests/test_vars.py

@@ -12,7 +12,7 @@ from clan_cli.tests.age_keys import SopsSetup
 from clan_cli.tests.fixtures_flakes import ClanFlake
 from clan_cli.tests.helpers import cli
 from clan_cli.vars.check import check_vars
-from clan_cli.vars.generate import Generator, generate_vars_for_machine
+from clan_cli.vars.generate import Generator, generate_vars_for_machine_interactive
 from clan_cli.vars.get import get_var
 from clan_cli.vars.graph import all_missing_closure, requested_closure
 from clan_cli.vars.list import stringify_all_vars
@@ -706,11 +706,11 @@ def test_stdout_of_generate(
     flake_.refresh()
     monkeypatch.chdir(flake_.path)
     flake = Flake(str(flake_.path))
-    from clan_cli.vars.generate import generate_vars_for_machine
+    from clan_cli.vars.generate import generate_vars_for_machine_interactive
 
     # with capture_output as output:
     with caplog.at_level(logging.INFO):
-        generate_vars_for_machine(
+        generate_vars_for_machine_interactive(
             Machine(name="my_machine", flake=flake),
             "my_generator",
             regenerate=False,
@@ -723,7 +723,7 @@ def test_stdout_of_generate(
 
     set_var("my_machine", "my_generator/my_value", b"world", flake)
     with caplog.at_level(logging.INFO):
-        generate_vars_for_machine(
+        generate_vars_for_machine_interactive(
             Machine(name="my_machine", flake=flake),
             "my_generator",
             regenerate=True,
@@ -734,7 +734,7 @@ def test_stdout_of_generate(
     caplog.clear()
     # check the output when nothing gets regenerated
     with caplog.at_level(logging.INFO):
-        generate_vars_for_machine(
+        generate_vars_for_machine_interactive(
             Machine(name="my_machine", flake=flake),
             "my_generator",
             regenerate=True,
@@ -743,7 +743,7 @@ def test_stdout_of_generate(
     assert "hello" in caplog.text
     caplog.clear()
     with caplog.at_level(logging.INFO):
-        generate_vars_for_machine(
+        generate_vars_for_machine_interactive(
             Machine(name="my_machine", flake=flake),
             "my_secret_generator",
             regenerate=False,
@@ -758,7 +758,7 @@ def test_stdout_of_generate(
         Flake(str(flake.path)),
     )
     with caplog.at_level(logging.INFO):
-        generate_vars_for_machine(
+        generate_vars_for_machine_interactive(
             Machine(name="my_machine", flake=flake),
             "my_secret_generator",
             regenerate=True,
@@ -848,7 +848,7 @@ def test_fails_when_files_are_left_from_other_backend(
     flake.refresh()
     monkeypatch.chdir(flake.path)
     for generator in ["my_secret_generator", "my_value_generator"]:
-        generate_vars_for_machine(
+        generate_vars_for_machine_interactive(
             Machine(name="my_machine", flake=Flake(str(flake.path))),
             generator,
             regenerate=False,
@@ -865,13 +865,13 @@ def test_fails_when_files_are_left_from_other_backend(
         # This should raise an error
         if generator == "my_secret_generator":
             with pytest.raises(ClanError):
-                generate_vars_for_machine(
+                generate_vars_for_machine_interactive(
                     Machine(name="my_machine", flake=Flake(str(flake.path))),
                     generator,
                     regenerate=False,
                 )
         else:
-            generate_vars_for_machine(
+            generate_vars_for_machine_interactive(
                 Machine(name="my_machine", flake=Flake(str(flake.path))),
                 generator,
                 regenerate=False,
@@ -970,7 +970,7 @@ def test_dynamic_invalidation(
     custom_nix.write_text(
         """
         { config, ... }: let
-            p = config.clan.core.vars.generators.my_generator.files.my_value.path;
+            p = config.clan.core.vars.generators.my_generator.files.my_value.flakePath;
         in {
             clan.core.vars.generators.dependent_generator.validation = if builtins.pathExists p then builtins.readFile p else null;
         }

pkgs/clan-cli/clan_cli/vars/generate.py

@@ -15,10 +15,13 @@ from clan_cli.completions import (
     complete_services_for_machine,
 )
 from clan_cli.errors import ClanError
+from clan_cli.flake import Flake
 from clan_cli.git import commit_files
 from clan_cli.machines.inventory import get_all_machines, get_selected_machines
 from clan_cli.nix import nix_config, nix_shell, nix_test_store
 from clan_cli.vars._types import StoreBase
+from clan_cli.vars.migration import check_can_migrate, migrate_files
+from clan_lib.api import API
 
 from .check import check_vars
 from .graph import (
@@ -148,12 +151,15 @@ def dependencies_as_dir(
 ) -> None:
     for dep_generator, files in decrypted_dependencies.items():
         dep_generator_dir = tmpdir / dep_generator
-        dep_generator_dir.mkdir()
-        dep_generator_dir.chmod(0o700)
+        # Explicitly specify parents and exist_ok default values for clarity
+        dep_generator_dir.mkdir(mode=0o700, parents=False, exist_ok=False)
         for file_name, file in files.items():
             file_path = dep_generator_dir / file_name
-            file_path.touch()
-            file_path.chmod(0o600)
+            # Avoid the file creation and chmod race
+            # If the file already existed,
+            # we'd have to create a temp one and rename instead;
+            # however, this is a clean dir so there shouldn't be any collisions
+            file_path.touch(mode=0o600, exist_ok=False)
             file_path.write_bytes(file)
 
 
@@ -308,131 +314,64 @@ def get_closure(
     return minimal_closure([generator_name], generators)
 
 
-def _migration_file_exists(
+@API.register
+def get_generators_closure(
+    machine_name: str,
+    base_dir: Path,
+    regenerate: bool = False,
+) -> list[Generator]:
+    from clan_cli.machines.machines import Machine
+
+    return get_closure(
+        machine=Machine(name=machine_name, flake=Flake(str(base_dir))),
+        generator_name=None,
+        regenerate=regenerate,
+    )
+
+
+def _generate_vars_for_machine(
     machine: "Machine",
-    generator: Generator,
-    fact_name: str,
+    generators: list[Generator],
+    all_prompt_values: dict[str, dict],
+    no_sandbox: bool = False,
 ) -> bool:
-    for file in generator.files:
-        if file.name == fact_name:
-            break
-    else:
-        msg = f"Could not find file {fact_name} in generator {generator.name}"
-        raise ClanError(msg)
-
-    is_secret = file.secret
-    if is_secret:
-        if machine.secret_facts_store.exists(generator.name, fact_name):
-            return True
-        machine.debug(
-            f"Cannot migrate fact {fact_name} for service {generator.name}, as it does not exist in the secret fact store"
-        )
-    if not is_secret:
-        if machine.public_facts_store.exists(generator.name, fact_name):
-            return True
-        machine.debug(
-            f"Cannot migrate fact {fact_name} for service {generator.name}, as it does not exist in the public fact store"
-        )
-    return False
-
-
-def _migrate_file(
-    machine: "Machine",
-    generator: Generator,
-    var_name: str,
-    service_name: str,
-    fact_name: str,
-) -> list[Path]:
-    for file in generator.files:
-        if file.name == var_name:
-            break
-    else:
-        msg = f"Could not find file {fact_name} in generator {generator.name}"
-        raise ClanError(msg)
-
-    paths = []
-
-    if file.secret:
-        old_value = machine.secret_facts_store.get(service_name, fact_name)
-        maybe_path = machine.secret_vars_store.set(
-            generator, file, old_value, is_migration=True
-        )
-        if maybe_path:
-            paths.append(maybe_path)
-    else:
-        old_value = machine.public_facts_store.get(service_name, fact_name)
-        maybe_path = machine.public_vars_store.set(
-            generator, file, old_value, is_migration=True
-        )
-        if maybe_path:
-            paths.append(maybe_path)
-
-    return paths
-
-
-def _migrate_files(
-    machine: "Machine",
-    generator: Generator,
-) -> None:
-    not_found = []
-    files_to_commit = []
-    for file in generator.files:
-        if _migration_file_exists(machine, generator, file.name):
-            assert generator.migrate_fact is not None
-            files_to_commit += _migrate_file(
-                machine, generator, file.name, generator.migrate_fact, file.name
+    for generator in generators:
+        if check_can_migrate(machine, generator):
+            migrate_files(machine, generator)
+        else:
+            execute_generator(
+                machine=machine,
+                generator=generator,
+                secret_vars_store=machine.secret_vars_store,
+                public_vars_store=machine.public_vars_store,
+                prompt_values=all_prompt_values[generator.name],
+                no_sandbox=no_sandbox,
             )
-        else:
-            not_found.append(file.name)
-    if len(not_found) > 0:
-        msg = f"Could not migrate the following files for generator {generator.name}, as no fact or secret exists with the same name: {not_found}"
-        raise ClanError(msg)
-    commit_files(
-        files_to_commit,
-        machine.flake_dir,
-        f"migrated facts to vars for generator {generator.name} for machine {machine.name}",
-    )
-
-
-def _check_can_migrate(
-    machine: "Machine",
-    generator: Generator,
-) -> bool:
-    service_name = generator.migrate_fact
-    if not service_name:
-        return False
-    # ensure that none of the generated vars already exist in the store
-    all_files_missing = True
-    all_files_present = True
-    for file in generator.files:
-        if file.secret:
-            if machine.secret_vars_store.exists(generator, file.name):
-                all_files_missing = False
-            else:
-                all_files_present = False
-        else:
-            if machine.public_vars_store.exists(generator, file.name):
-                all_files_missing = False
-            else:
-                all_files_present = False
-
-    if not all_files_present and not all_files_missing:
-        msg = f"Cannot migrate facts for generator {generator.name} as some files already exist in the store"
-        raise ClanError(msg)
-    if all_files_present:
-        # all files already migrated, no need to run migration again
-        return False
-
-    # ensure that all files can be migrated (exists in the corresponding fact store)
-    return bool(
-        all(
-            _migration_file_exists(machine, generator, file.name)
-            for file in generator.files
-        )
-    )
+    return True
 
 
+@API.register
 def generate_vars_for_machine(
+    machine_name: str,
+    generators: list[Generator],
+    all_prompt_values: dict[str, dict[str, str]],
+    base_dir: Path,
+    no_sandbox: bool = False,
+) -> bool:
+    from clan_cli.machines.machines import Machine
+
+    return _generate_vars_for_machine(
+        machine=Machine(
+            name=machine_name,
+            flake=Flake(str(base_dir)),
+        ),
+        generators=generators,
+        all_prompt_values=all_prompt_values,
+        no_sandbox=no_sandbox,
+    )
+
+
+def generate_vars_for_machine_interactive(
     machine: "Machine",
     generator_name: str | None,
     regenerate: bool,
@@ -456,22 +395,18 @@ def generate_vars_for_machine(
             msg += f"Secret vars store: {sec_healtcheck_msg}"
         raise ClanError(msg)
 
-    closure = get_closure(machine, generator_name, regenerate)
-    if len(closure) == 0:
+    generators = get_closure(machine, generator_name, regenerate)
+    if len(generators) == 0:
         return False
-    for generator in closure:
-        if _check_can_migrate(machine, generator):
-            _migrate_files(machine, generator)
-        else:
-            execute_generator(
-                machine=machine,
-                generator=generator,
-                secret_vars_store=machine.secret_vars_store,
-                public_vars_store=machine.public_vars_store,
-                prompt_values=_ask_prompts(generator),
-                no_sandbox=no_sandbox,
-            )
-    return True
+    all_prompt_values = {}
+    for generator in generators:
+        all_prompt_values[generator.name] = _ask_prompts(generator)
+    return _generate_vars_for_machine(
+        machine,
+        generators,
+        all_prompt_values,
+        no_sandbox=no_sandbox,
+    )
 
 
 def generate_vars(
@@ -484,7 +419,7 @@ def generate_vars(
     for machine in machines:
         errors = []
         try:
-            was_regenerated |= generate_vars_for_machine(
+            was_regenerated |= generate_vars_for_machine_interactive(
                 machine, generator_name, regenerate, no_sandbox=no_sandbox
             )
         except Exception as exc:

pkgs/clan-cli/clan_cli/vars/keygen.py

@@ -1,17 +1,19 @@
 import argparse
 import logging
 import os
+from pathlib import Path
 
 from clan_cli.errors import ClanError
-from clan_cli.flake import Flake
 from clan_cli.secrets.key import generate_key
 from clan_cli.secrets.sops import maybe_get_admin_public_key
 from clan_cli.secrets.users import add_user
+from clan_lib.api import API
 
 log = logging.getLogger(__name__)
 
 
-def keygen(user: str | None, flake: Flake, force: bool) -> None:
+@API.register
+def keygen(flake_dir: Path, user: str | None = None, force: bool = False) -> None:
     if user is None:
         user = os.getenv("USER", None)
         if not user:
@@ -22,7 +24,7 @@ def keygen(user: str | None, flake: Flake, force: bool) -> None:
         pub_key = generate_key()
     # TODO set flake_dir=flake.path / "vars"
     add_user(
-        flake_dir=flake.path,
+        flake_dir=flake_dir,
         name=user,
         keys=[pub_key],
         force=force,
@@ -33,8 +35,8 @@ def _command(
     args: argparse.Namespace,
 ) -> None:
     keygen(
+        flake_dir=args.flake.path,
         user=args.user,
-        flake=args.flake,
         force=args.force,
     )
 

pkgs/clan-cli/clan_cli/vars/migration.py

@@ -0,0 +1,136 @@
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from clan_cli.errors import ClanError
+from clan_cli.git import commit_files
+
+log = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    from clan_cli.machines.machines import Machine
+    from clan_cli.vars.generate import Generator
+
+
+def _migration_file_exists(
+    machine: "Machine",
+    generator: "Generator",
+    fact_name: str,
+) -> bool:
+    for file in generator.files:
+        if file.name == fact_name:
+            break
+    else:
+        msg = f"Could not find file {fact_name} in generator {generator.name}"
+        raise ClanError(msg)
+
+    is_secret = file.secret
+    if is_secret:
+        if machine.secret_facts_store.exists(generator.name, fact_name):
+            return True
+        machine.debug(
+            f"Cannot migrate fact {fact_name} for service {generator.name}, as it does not exist in the secret fact store"
+        )
+    if not is_secret:
+        if machine.public_facts_store.exists(generator.name, fact_name):
+            return True
+        machine.debug(
+            f"Cannot migrate fact {fact_name} for service {generator.name}, as it does not exist in the public fact store"
+        )
+    return False
+
+
+def _migrate_file(
+    machine: "Machine",
+    generator: "Generator",
+    var_name: str,
+    service_name: str,
+    fact_name: str,
+) -> list[Path]:
+    for file in generator.files:
+        if file.name == var_name:
+            break
+    else:
+        msg = f"Could not find file {fact_name} in generator {generator.name}"
+        raise ClanError(msg)
+
+    paths = []
+
+    if file.secret:
+        old_value = machine.secret_facts_store.get(service_name, fact_name)
+        maybe_path = machine.secret_vars_store.set(
+            generator, file, old_value, is_migration=True
+        )
+        if maybe_path:
+            paths.append(maybe_path)
+    else:
+        old_value = machine.public_facts_store.get(service_name, fact_name)
+        maybe_path = machine.public_vars_store.set(
+            generator, file, old_value, is_migration=True
+        )
+        if maybe_path:
+            paths.append(maybe_path)
+
+    return paths
+
+
+def migrate_files(
+    machine: "Machine",
+    generator: "Generator",
+) -> None:
+    not_found = []
+    files_to_commit = []
+    for file in generator.files:
+        if _migration_file_exists(machine, generator, file.name):
+            assert generator.migrate_fact is not None
+            files_to_commit += _migrate_file(
+                machine, generator, file.name, generator.migrate_fact, file.name
+            )
+        else:
+            not_found.append(file.name)
+    if len(not_found) > 0:
+        msg = f"Could not migrate the following files for generator {generator.name}, as no fact or secret exists with the same name: {not_found}"
+        raise ClanError(msg)
+    commit_files(
+        files_to_commit,
+        machine.flake_dir,
+        f"migrated facts to vars for generator {generator.name} for machine {machine.name}",
+    )
+
+
+def check_can_migrate(
+    machine: "Machine",
+    generator: "Generator",
+) -> bool:
+    service_name = generator.migrate_fact
+    if not service_name:
+        return False
+    # ensure that none of the generated vars already exist in the store
+    all_files_missing = True
+    all_files_present = True
+    for file in generator.files:
+        if file.secret:
+            if machine.secret_vars_store.exists(generator, file.name):
+                all_files_missing = False
+            else:
+                all_files_present = False
+        else:
+            if machine.public_vars_store.exists(generator, file.name):
+                all_files_missing = False
+            else:
+                all_files_present = False
+
+    if not all_files_present and not all_files_missing:
+        msg = f"Cannot migrate facts for generator {generator.name} as some files already exist in the store"
+        raise ClanError(msg)
+    if all_files_present:
+        # all files already migrated, no need to run migration again
+        return False
+
+    # ensure that all files can be migrated (exists in the corresponding fact store)
+    return bool(
+        all(
+            _migration_file_exists(machine, generator, file.name)
+            for file in generator.files
+        )
+    )

pkgs/clan-cli/clan_cli/vars/prompt.py

@@ -33,7 +33,6 @@ class Prompt:
             description=data["description"],
             prompt_type=PromptType(data["type"]),
             persist=data.get("persist", data["persist"]),
-            previous_value=data.get("previousValue"),
         )
 
 

pkgs/clan-cli/default.nix

@@ -41,7 +41,12 @@ let
   allDependencies = lib.importJSON ./clan_cli/nix/allowed-packages.json;
   generateRuntimeDependenciesMap =
     deps:
-    lib.filterAttrs (_: pkg: !pkg.meta.unsupported or false) (lib.genAttrs deps (name: pkgs.${name}));
+    lib.filterAttrs (
+      attr: pkg:
+      !pkg.meta.unsupported or false
+      # Currently fails to build because of swift
+      && !(stdenv.hostPlatform.system == "aarch64-linux" && attr == "age-plugin-se")
+    ) (lib.genAttrs deps (name: pkgs.${name}));
   testRuntimeDependenciesMap = generateRuntimeDependenciesMap allDependencies;
   testRuntimeDependencies = lib.attrValues testRuntimeDependenciesMap;
   bundledRuntimeDependenciesMap = generateRuntimeDependenciesMap includedRuntimeDeps;

pkgs/webview-ui/app/eslint.config.mjs

@@ -3,7 +3,7 @@ import tseslint from "typescript-eslint";
 import tailwind from "eslint-plugin-tailwindcss";
 import pluginQuery from "@tanstack/eslint-plugin-query";
 
-export default tseslint.config(
+const config = tseslint.config(
   eslint.configs.recommended,
   ...pluginQuery.configs["flat/recommended"],
   ...tseslint.configs.strict,
@@ -30,3 +30,5 @@ export default tseslint.config(
     },
   },
 );
+
+export default config;

pkgs/webview-ui/app/src/App.tsx

@@ -1,5 +1,6 @@
 import { createSignal } from "solid-js";
 import { makePersisted } from "@solid-primitives/storage";
+import { callApi } from "./api";
 
 const [activeURI, setActiveURI] = makePersisted(
   createSignal<string | null>(null),
@@ -17,3 +18,22 @@ const [clanList, setClanList] = makePersisted(createSignal<string[]>([]), {
 });
 
 export { clanList, setClanList };
+
+(async function () {
+  const curr = activeURI();
+  if (curr) {
+    const result = await callApi("show_clan_meta", { uri: curr });
+    console.log("refetched meta for ", curr);
+    if (result.status === "error") {
+      result.errors.forEach((error) => {
+        if (error.description === "clan directory does not exist") {
+          setActiveURI(null);
+          setClanList((clans) => clans.filter((clan) => clan !== curr));
+        }
+      });
+    }
+  }
+})();
+
+// ensure to null out activeURI on startup if the clan was deleted
+// => throws user back to the view for selecting a clan

pkgs/webview-ui/app/src/Form/fieldset/index.tsx

@@ -0,0 +1,32 @@
+import { JSX } from "solid-js";
+
+import { Typography } from "@/src/components/Typography";
+
+interface FieldsetProps {
+  legend?: string;
+  children: JSX.Element;
+  class?: string;
+}
+
+export default function Fieldset(props: FieldsetProps) {
+  return (
+    <fieldset class="flex flex-col gap-y-2.5">
+      {props.legend && (
+        <div class="px-2">
+          <Typography
+            hierarchy="body"
+            tag="p"
+            size="s"
+            color="primary"
+            weight="medium"
+          >
+            {props.legend}
+          </Typography>
+        </div>
+      )}
+      <div class="flex flex-col gap-y-3 rounded-md border border-secondary-200 bg-secondary-50 p-5">
+        {props.children}
+      </div>
+    </fieldset>
+  );
+}

pkgs/webview-ui/app/src/Form/form/index.tsx

@@ -368,10 +368,10 @@ export function ListValueDisplay<T extends FieldValues, R extends ResponseData>(
   const bottomMost = () => props.idx === 0;
 
   return (
-    <div class="w-full border-l-4 border-gray-300">
-      <div class="flex w-full items-end gap-2 px-4">
+    <div class="w-full border-b border-secondary-200 px-2 pb-4">
+      <div class="flex w-full items-center gap-2">
         {props.children}
-        <div class="ml-4 min-w-fit pb-4">
+        <div class="ml-4 min-w-fit">
           <Button
             variant="ghost"
             size="s"
@@ -541,7 +541,6 @@ export function ArrayFields<T extends FieldValues, R extends ResponseData>(
                     )}
                   </Field>
                 </Match>
-
                 <Match
                   when={
                     itemsSchema().type === "string" ||
@@ -629,9 +628,11 @@ export function ArrayFields<T extends FieldValues, R extends ResponseData>(
                             </ListValueDisplay>
                           )}
                         </For>
-                        <span class=" font-bold text-error-700">
-                          {fieldArray.error}
-                        </span>
+                        <Show when={fieldArray.error}>
+                          <span class="font-bold text-error-700">
+                            {fieldArray.error}
+                          </span>
+                        </Show>
 
                         {/* Add new item */}
                         <DynForm
@@ -651,14 +652,16 @@ export function ArrayFields<T extends FieldValues, R extends ResponseData>(
                           // Button for adding new items
                           components={{
                             before: (
-                              <Button
-                                variant="ghost"
-                                type="submit"
-                                endIcon={<Icon icon={"Plus"} />}
-                                class="capitalize"
-                              >
-                                Add {itemsSchema().title}
-                              </Button>
+                              <div class="flex w-full justify-end pb-2">
+                                <Button
+                                  variant="ghost"
+                                  type="submit"
+                                  endIcon={<Icon size={14} icon={"Plus"} />}
+                                  class="capitalize"
+                                >
+                                  Add {itemsSchema().title}
+                                </Button>
+                              </div>
                             ),
                           }}
                           // Add the new item to the FieldArray

pkgs/webview-ui/app/src/components/MachineListItem.tsx

@@ -110,18 +110,18 @@ export const MachineListItem = (props: MachineListItemProps) => {
     setUpdating(false);
   };
   return (
-    <div class="border rounded-lg border-def-2 p-3 m-2 w-64">
+    <div class="m-2 w-64 rounded-lg border p-3 border-def-2">
       <figure class="h-fit rounded-xl border bg-def-2 border-def-5">
         <RndThumbnail name={name} width={220} height={120} />
       </figure>
-      <div class="flex-row justify-between gap-4 pt-2 px-2">
+      <div class="flex-row justify-between gap-4 px-2 pt-2">
         <div class="flex flex-col">
           <A href={`/machines/${name}`}>
             <Typography hierarchy="title" size="m" weight="bold">
               {name}
             </Typography>
           </A>
-          <div class="text-slate-600 flex justify-between">
+          <div class="flex justify-between text-slate-600">
             <div class="flex flex-nowrap">
               <span class="h-4">
                 <Icon icon="Flash" class="h-4" font-size="inherit" />
@@ -138,7 +138,7 @@ export const MachineListItem = (props: MachineListItemProps) => {
                 popoverid={`menu-${props.name}`}
                 label={<Icon icon={"More"} />}
               >
-                <ul class="z-[1] w-64 p-2 shadow bg-white ">
+                <ul class="z-[1] w-64 bg-white p-2 shadow ">
                   <li>
                     <Button
                       variant="ghost"

pkgs/webview-ui/app/src/components/Sidebar/SidebarListItem.tsx

@@ -1,6 +1,6 @@
 import { A } from "@solidjs/router";
-
 import { Typography } from "@/src/components/Typography";
+import "./css/sidebar.css";
 
 interface SidebarListItem {
   title: string;
@@ -11,13 +11,13 @@ export const SidebarListItem = (props: SidebarListItem) => {
   const { title, href } = props;
 
   return (
-    <li class="sidebar__list__item">
+    <li class="">
       <A class="sidebar__list__link" href={href}>
         <Typography
           class="sidebar__list__content"
           tag="span"
           hierarchy="body"
-          size="s"
+          size="xs"
           weight="normal"
           color="primary"
           inverted={true}

pkgs/webview-ui/app/src/components/Sidebar/css/sidebar-list-item.css

@@ -1,4 +1,4 @@
-.sidebar__list__item {
+.sidebar__list__link {
   position: relative;
   cursor: theme(cursor.pointer);
 
@@ -19,12 +19,12 @@
   &:hover:after {
     background: var(--clr-bg-inv-acc-2);
     transform: scale(theme(scale.100));
-    transition: transform 0.24s ease-in-out;
+    transition: transform 0.32s ease-in-out;
   }
 
   &:active {
     transform: scale(0.99);
-    transition: transform 0.08s ease-in-out;
+    transition: transform 0.12s ease-in-out;
   }
 
   &:active:after {
@@ -37,7 +37,13 @@
   position: relative;
   z-index: 20;
   display: block;
-  padding: theme(padding.3);
+  padding: theme(padding.2) theme(padding.3);
+}
+
+.sidebar__list__link.active {
+  &:after {
+    background: var(--clr-bg-inv-acc-3);
+  }
 }
 
 .sidebar__list__content {

pkgs/webview-ui/app/src/components/Sidebar/css/sidebar.css

@@ -9,6 +9,8 @@
 
 .sidebar {
   @apply bg-inv-2 h-full border border-solid border-inv-2 min-w-72 rounded-xl;
+  display: flex;
+  flex-direction: column;
 }
 
 .sidebar__body {
@@ -19,9 +21,9 @@
 }
 
 .sidebar__section {
-  padding: theme(padding.2);
-  /* background-color: rgba(var(--clr-bg-inv-3) / 0.9); */
   @apply bg-primary-800/90;
+
+  padding: theme(padding.2);
   border-radius: theme(borderRadius.md);
 
   ::marker {

pkgs/webview-ui/app/src/components/Sidebar/index.tsx

@@ -19,20 +19,22 @@ export const SidebarSection = (props: {
 
   return (
     <details class="sidebar__section accordeon" open>
-      <summary class="accordeon__header">
-        <Typography
-          class="inline-flex w-full gap-2 uppercase"
-          tag="p"
-          hierarchy="body"
-          size="xs"
-          weight="normal"
-          color="tertiary"
-          inverted={true}
-        >
-          <Icon icon={props.icon} />
-          {title}
-          <Icon icon="CaretDown" class="ml-auto" />
-        </Typography>
+      <summary style="display: contents;">
+        <div class="accordeon__header">
+          <Typography
+            class="inline-flex w-full gap-2 uppercase !tracking-wider"
+            tag="p"
+            hierarchy="body"
+            size="xxs"
+            weight="normal"
+            color="tertiary"
+            inverted={true}
+          >
+            <Icon class="opacity-90" icon={props.icon} size={13} />
+            {title}
+            <Icon icon="CaretDown" class="ml-auto" size={10} />
+          </Typography>
+        </div>
       </summary>
       <div class="accordeon__body">{children}</div>
     </details>
@@ -60,7 +62,7 @@ export const Sidebar = (props: RouteSectionProps) => {
   }));
 
   return (
-    <div class="sidebar opacity-95">
+    <div class="sidebar">
       <Show
         when={query.data}
         fallback={<SidebarHeader clanName={"Untitled"} />}
@@ -81,7 +83,7 @@ export const Sidebar = (props: RouteSectionProps) => {
                   title={route.label}
                   icon={route.icon || "Paperclip"}
                 >
-                  <ul>
+                  <ul class="flex flex-col gap-y-0.5">
                     <For each={children().filter((r) => !r.hidden)}>
                       {(child) => (
                         <SidebarListItem

pkgs/webview-ui/app/src/components/Typography/css/typography-hierarchy/typography-body.css

@@ -17,7 +17,7 @@
 }
 
 .fnt-body-xxs {
-  font-size: 0.6875rem;
+  font-size: 0.75rem;
   line-height: 132%;
   letter-spacing: 0.00688rem;
 }

pkgs/webview-ui/app/src/components/accordion/accordion.css

@@ -0,0 +1,10 @@
+.accordion {
+  @apply flex flex-col gap-y-5;
+}
+
+.accordion__title {
+  @apply flex h-5 cursor-pointer items-center justify-end gap-x-0.5 px-1 font-medium;
+}
+
+.accordion__body {
+}

pkgs/webview-ui/app/src/components/accordion/index.tsx

@@ -0,0 +1,45 @@
+import { createSignal, JSX, Show } from "solid-js";
+import Icon from "../icon";
+import { Button } from "../button";
+import cx from "classnames";
+import "./accordion.css";
+
+interface AccordionProps {
+  title: string;
+  children: JSX.Element;
+  class?: string;
+  initiallyOpen?: boolean;
+}
+
+export default function Accordion(props: AccordionProps) {
+  const [isOpen, setIsOpen] = createSignal(props.initiallyOpen ?? false);
+  return (
+    <div class={cx(`accordion`, props.class)} tabindex="0">
+      <div onClick={() => setIsOpen(!isOpen())} class="accordion__title">
+        <Show
+          when={isOpen()}
+          fallback={
+            <Button
+              endIcon={<Icon size={12} icon={"CaretDown"} />}
+              variant="light"
+              size="s"
+            >
+              {props.title}
+            </Button>
+          }
+        >
+          <Button
+            endIcon={<Icon size={12} icon={"CaretUp"} />}
+            variant="dark"
+            size="s"
+          >
+            {props.title}
+          </Button>
+        </Show>
+      </div>
+      <Show when={isOpen()}>
+        <div class="accordion__body">{props.children}</div>
+      </Show>
+    </div>
+  );
+}

pkgs/webview-ui/app/src/components/button/css/button-dark.css

@@ -0,0 +1,31 @@
+/* button DARK and states */
+
+.button--dark {
+  @apply border border-solid border-secondary-950 bg-primary-800 text-white;
+
+  box-shadow: inset 1px 1px theme(backgroundColor.secondary.700);
+
+  &:disabled {
+    @apply disabled:bg-secondary-200 disabled:text-secondary-700 disabled:border-secondary-300;
+  }
+
+  & .button__icon {
+    color: theme(textColor.secondary.200);
+  }
+}
+
+.button--dark-hover:hover {
+  @apply hover:bg-secondary-900;
+}
+
+.button--dark-focus:focus {
+  @apply focus:border-secondary-900;
+}
+
+.button--dark-active:active {
+  @apply focus:border-secondary-900;
+}
+
+.button--dark-active:active {
+  @apply active:border-secondary-900 active:shadow-inner-primary-active;
+}

pkgs/webview-ui/app/src/components/button/css/button-light.css

@@ -0,0 +1,37 @@
+/* button LIGHT and states */
+
+.button--light {
+  @apply border border-solid border-secondary-400 bg-secondary-100 text-secondary-950;
+
+  box-shadow: inset 1px 1px theme(backgroundColor.white);
+
+  &:disabled {
+    @apply disabled:bg-secondary-50 disabled:text-secondary-200 disabled:border-secondary-700;
+  }
+
+  & .button__icon {
+    color: theme(textColor.secondary.900);
+  }
+}
+
+.button--light-hover:hover {
+  @apply hover:bg-secondary-200;
+}
+
+.button--light-focus:focus {
+  @apply focus:bg-secondary-200;
+
+  & .button__label {
+    color: theme(textColor.secondary.900) !important;
+  }
+}
+
+.button--light-active:active {
+  @apply active:bg-secondary-200 border-secondary-600 active:text-secondary-900 active:shadow-inner-primary-active;
+
+  box-shadow: inset 2px 2px theme(backgroundColor.secondary.300);
+
+  & .button__label {
+    color: theme(textColor.secondary.900) !important;
+  }
+}

pkgs/webview-ui/app/src/components/button/css/index.css

@@ -0,0 +1,54 @@
+@import "./button-light.css";
+@import "./button-dark.css";
+
+.button {
+  @apply inline-flex items-center flex-shrink gap-1 justify-center p-4 font-semibold;
+  letter-spacing: 0.0275rem;
+}
+
+/* button SIZES */
+
+.button--default {
+  padding: theme(padding.2) theme(padding.4);
+  height: theme(height.9);
+  border-radius: theme(borderRadius.DEFAULT);
+
+  &:has(> .button__icon--start):has(> .button__label) {
+    padding-left: theme(padding[2.5]);
+  }
+
+  &:has(> .button__icon--end):has(> .button__label) {
+    padding-right: theme(padding[2.5]);
+  }
+}
+
+.button--small {
+  padding: theme(padding[1.5]) theme(padding[3]);
+  height: theme(height.8);
+  border-radius: 3px;
+
+  &:has(> .button__icon--start):has(> .button__label) {
+    padding-left: theme(padding.2);
+  }
+
+  &:has(> .button__label):has(> .button__icon--end) {
+    padding-right: theme(padding.2);
+  }
+}
+
+/* button group */
+
+.button-group .button:first-child {
+  border-top-right-radius: 0;
+  border-bottom-right-radius: 0;
+}
+
+.button-group .button:first-child {
+  border-top-right-radius: 0;
+  border-bottom-right-radius: 0;
+}
+
+.button-group .button:last-child {
+  border-top-left-radius: 0;
+  border-bottom-left-radius: 0;
+}

pkgs/webview-ui/app/src/components/button/index.tsx

@@ -1,6 +1,8 @@
 import { splitProps, type JSX } from "solid-js";
 import cx from "classnames";
 import { Typography } from "../Typography";
+//import './css/index.css'
+import "./css/index.css";
 
 type Variants = "dark" | "light" | "ghost";
 type Size = "default" | "s";
@@ -9,50 +11,31 @@ const variantColors: (
   disabled: boolean | undefined,
 ) => Record<Variants, string> = (disabled) => ({
   dark: cx(
-    "border border-solid",
-    "border-secondary-950 bg-primary-900 text-white",
-    "shadow-inner-primary",
-    // Hover state
-    // Focus state
-    // Active state
-    !disabled && "hover:border-secondary-900 hover:bg-secondary-700",
-    !disabled && "focus:border-secondary-900",
-    !disabled &&
-      "active:border-secondary-900 active:shadow-inner-primary-active",
+    "button--dark",
+    !disabled && "button--dark-hover", // Hover state
+    !disabled && "button--dark-focus", // Focus state
+    !disabled && "button--dark-active", // Active state
     // Disabled
     "disabled:bg-secondary-200 disabled:text-secondary-700 disabled:border-secondary-300",
   ),
   light: cx(
-    "border border-solid",
-    "border-secondary-800 bg-secondary-100 text-secondary-800",
-    "shadow-inner-secondary",
-    // Hover state
-    // Focus state
-    // Active state
-    !disabled && "hover:bg-secondary-200 hover:text-secondary-900",
-    !disabled && "focus:bg-secondary-200 focus:text-secondary-900",
-    !disabled &&
-      "active:bg-secondary-200 active:text-secondary-950 active:shadow-inner-secondary-active",
-    // Disabled
-    "disabled:bg-secondary-50 disabled:text-secondary-200 disabled:border-secondary-700",
+    "button--light",
+
+    !disabled && "button--light-hover", // Hover state
+    !disabled && "button--light-focus", // Focus state
+    !disabled && "button--light-active", // Active state
   ),
   ghost: cx(
     // "shadow-inner-secondary",
-    // Hover state
-    // Focus state
-    // Active state
-    !disabled && "hover:bg-secondary-200 hover:text-secondary-900",
-    !disabled && "focus:bg-secondary-200 focus:text-secondary-900",
-    !disabled &&
-      "active:bg-secondary-200 active:text-secondary-950 active:shadow-inner-secondary-active",
-    // Disabled
-    "disabled:bg-secondary-50 disabled:text-secondary-200 disabled:border-secondary-700",
+    !disabled && "hover:bg-secondary-200 hover:text-secondary-900", // Hover state
+    !disabled && "focus:bg-secondary-200 focus:text-secondary-900", // Focus state
+    !disabled && "button--light-active", // Active state
   ),
 });
 
 const sizePaddings: Record<Size, string> = {
-  default: cx("rounded-[0.1875rem] px-4 py-2"),
-  s: cx("rounded-sm py-[0.375rem] px-3"),
+  default: cx("button--default"),
+  s: cx("button button--small"), //cx("rounded-sm py-[0.375rem] px-3"),
 };
 
 const sizeFont: Record<Size, string> = {
@@ -77,26 +60,38 @@ export const Button = (props: ButtonProps) => {
     "endIcon",
     "class",
   ]);
+
+  const buttonInvertion = (variant: Variants) => {
+    return !(!variant || variant === "ghost" || variant === "light");
+  };
+
   return (
     <button
       class={cx(
         local.class,
-        // Layout
-        "inline-flex items-center flex-shrink gap-2 justify-center",
-        // Styles
-        "p-4",
-        sizePaddings[local.size || "default"],
-        // Colors
-        variantColors(props.disabled)[local.variant || "dark"],
-        //Font
-        "leading-none font-semibold",
-        sizeFont[local.size || "default"],
+        "button", // default button class
+        variantColors(props.disabled)[local.variant || "dark"], // button appereance
+        sizePaddings[local.size || "default"], // button size
       )}
       {...other}
     >
-      {local.startIcon && <span class="h-4">{local.startIcon}</span>}
-      {local.children && <span>{local.children}</span>}
-      {local.endIcon && <span class="h-4">{local.endIcon}</span>}
+      {local.startIcon && (
+        <span class="button__icon--start">{local.startIcon}</span>
+      )}
+      {local.children && (
+        <Typography
+          class="button__label"
+          hierarchy="label"
+          size={local.size || "default"}
+          color="inherit"
+          inverted={buttonInvertion(local.variant || "dark")}
+          weight="medium"
+          tag="span"
+        >
+          {local.children}
+        </Typography>
+      )}
+      {local.endIcon && <span class="button__icon--end">{local.endIcon}</span>}
     </button>
   );
 };

pkgs/webview-ui/app/src/components/icon/index.tsx

@@ -77,6 +77,7 @@ export type IconVariant = keyof typeof icons;
 
 interface IconProps extends JSX.SvgSVGAttributes<SVGElement> {
   icon: IconVariant;
+  size?: number;
 }
 
 const Icon: Component<IconProps> = (props) => {
@@ -85,8 +86,8 @@ const Icon: Component<IconProps> = (props) => {
   const IconComponent = icons[local.icon];
   return IconComponent ? (
     <IconComponent
-      width={16}
-      height={16}
+      width={iconProps.size || 16}
+      height={iconProps.size || 16}
       viewBox="0 0 48 48"
       // @ts-expect-error: dont know, fix this type nit later
       ref={iconProps.ref}

pkgs/webview-ui/app/src/index.css

@@ -11,11 +11,13 @@
   font-weight: 400;
   src: url(../.fonts/ArchivoSemiCondensed-Regular.woff2) format("woff2");
 }
+
 @font-face {
   font-family: "Archivo";
   font-weight: 500;
   src: url(../.fonts/ArchivoSemiCondensed-Medium.woff2) format("woff2");
 }
+
 @font-face {
   font-family: "Archivo";
   font-weight: 600;
@@ -30,7 +32,7 @@
 
 :root {
   --clr-bg-def-1: theme(colors.white);
-  --clr-bg-def-2: theme(colors.secondary.50);
+  --clr-bg-def-2: theme(colors.primary.50);
   --clr-bg-def-3: theme(colors.secondary.100);
   --clr-bg-def-4: theme(colors.secondary.200);
   --clr-bg-def-5: theme(colors.secondary.300);
@@ -72,6 +74,15 @@ html {
   @apply font-sans;
   overflow-x: hidden;
   overflow-y: hidden;
+
+  -webkit-user-select: none;
+  /* Safari */
+  -moz-user-select: none;
+  /* Firefox */
+  -ms-user-select: none;
+  /* Internet Explorer/Edge */
+  user-select: none;
+  /* Standard */
 }
 
 .accordeon {
@@ -81,7 +92,7 @@ html {
 }
 
 .accordeon__header {
-  padding: theme(padding.2) theme(padding[1.5]);
+  padding: theme(padding.2) theme(padding[1.5]) theme(padding.1);
   cursor: pointer;
 }
 
@@ -90,5 +101,4 @@ html {
 }
 
 .accordeon__body {
-  padding: theme(padding.2) 0 theme(padding.1);
 }

pkgs/webview-ui/app/src/layout/header.tsx

@@ -9,7 +9,7 @@ interface HeaderProps {
 }
 export const Header = (props: HeaderProps) => {
   return (
-    <div class="flex border-b px-6 py-4 border-def-3">
+    <div class="sticky top-0 z-20 flex items-center border-b bg-white/80 px-6 py-4 backdrop-blur-md border-def-3">
       <div class="flex-none">
         {props.showBack && <BackButton />}
         <span class=" lg:hidden" data-tip="Menu">

pkgs/webview-ui/app/src/layout/layout.tsx

@@ -17,19 +17,11 @@ export const Layout: Component<RouteSectionProps> = (props) => {
 
   return (
     <div class="h-screen w-full p-4 bg-def-2">
-      <div class="h-full flex">
-        <div
-          class="z-40 h-full overflow-hidden"
-          classList={{
-            hidden:
-              props.location.pathname === "welcome" || clanList().length === 0,
-          }}
-        >
-          <Sidebar {...props} />
-        </div>
-        <div class="w-full my-2 ml-8 overflow-x-hidden overflow-y-scroll rounded-lg border bg-def-1 border-def-3">
+      <div class="flex size-full flex-row-reverse">
+        <div class="my-2 ml-8 flex-1 overflow-x-hidden overflow-y-scroll rounded-lg border bg-def-1 border-def-3">
           {props.children}
         </div>
+        <Sidebar {...props} />
       </div>
     </div>
   );

pkgs/webview-ui/app/src/routes/clans/create.tsx

@@ -7,7 +7,7 @@ import {
   SubmitHandler,
 } from "@modular-forms/solid";
 import toast from "solid-toast";
-import { setActiveURI, setClanList } from "@/src/App";
+import { activeURI, setActiveURI, setClanList } from "@/src/App";
 import { TextInput } from "@/src/Form/fields/TextInput";
 import { useNavigate } from "@solidjs/router";
 import { Button } from "@/src/components/button";
@@ -61,6 +61,17 @@ export const CreateClan = () => {
       toast.error("Failed to create clan");
       return;
     }
+
+    // Will generate a key if it doesn't exist, and add a user to the clan
+    const k = await callApi("keygen", {
+      flake_dir: target_dir[0],
+    });
+
+    if (k.status === "error") {
+      toast.error("Failed to generate key");
+      return;
+    }
+
     if (r.status === "success") {
       toast.success("Clan Successfully Created");
       setActiveURI(target_dir[0]);

pkgs/webview-ui/app/src/routes/machines/create.tsx

@@ -11,6 +11,9 @@ import { Match, Switch } from "solid-js";
 import toast from "solid-toast";
 import { MachineAvatar } from "./avatar";
 import { DynForm } from "@/src/Form/form";
+import { Typography } from "@/src/components/Typography";
+import Fieldset from "@/src/Form/fieldset";
+import Accordion from "@/src/components/accordion";
 
 type CreateMachineForm = OperationArgs<"create_machine">;
 
@@ -72,44 +75,80 @@ export function CreateMachine() {
     <>
       <Header title="Create Machine" />
       <div class="flex w-full p-4">
-        <div class="mt-4 w-full self-stretch px-2">
-          <Form onSubmit={handleSubmit} class="gap-2 flex flex-col">
+        <div class="mt-4 w-full self-stretch px-8">
+          <Form
+            onSubmit={handleSubmit}
+            class="mx-auto flex w-full max-w-2xl flex-col gap-y-6"
+          >
             <Field
               name="opts.machine.name"
               validate={[required("This field is required")]}
             >
               {(field, props) => (
                 <>
-                  <div class="flex justify-center mb-4 pb-4 border-b">
+                  <div class="mb-4 flex justify-center border-b pb-4">
                     <MachineAvatar name={field.value} />
                   </div>
-
-                  <TextInput
-                    inputProps={props}
-                    value={`${field.value}`}
-                    label={"name"}
-                    error={field.error}
-                    required
-                    placeholder="New_machine"
-                  />
                 </>
               )}
             </Field>
-            <Field name="opts.machine.description">
-              {(field, props) => (
-                <TextInput
-                  inputProps={props}
-                  value={`${field.value}`}
-                  label={"description"}
-                  error={field.error}
-                  placeholder="My awesome machine"
-                />
-              )}
-            </Field>
-            <div class=" " tabindex="0">
-              <input type="checkbox" />
-              <div class="  font-medium ">Deployment Settings</div>
-              <div class="">
+            <Fieldset legend="General">
+              <Field
+                name="opts.machine.name"
+                validate={[required("This field is required")]}
+              >
+                {(field, props) => (
+                  <>
+                    <TextInput
+                      inputProps={props}
+                      value={`${field.value}`}
+                      label={"name"}
+                      error={field.error}
+                      required
+                      placeholder="New_machine"
+                    />
+                  </>
+                )}
+              </Field>
+              <Field name="opts.machine.description">
+                {(field, props) => (
+                  <TextInput
+                    inputProps={props}
+                    value={`${field.value}`}
+                    label={"description"}
+                    error={field.error}
+                    placeholder="My awesome machine"
+                  />
+                )}
+              </Field>
+            </Fieldset>
+
+            <Fieldset legend="Tags">
+              <Field name="opts.machine.tags" type="string[]">
+                {(field, props) => (
+                  <div class="p-2">
+                    <DynForm
+                      initialValues={{ tags: ["all"] }}
+                      schema={{
+                        type: "object",
+                        properties: {
+                          tags: {
+                            type: "array",
+                            items: {
+                              title: "Tag",
+                              type: "string",
+                            },
+                            uniqueItems: true,
+                          },
+                        },
+                      }}
+                    />
+                  </div>
+                )}
+              </Field>
+            </Fieldset>
+            <Accordion title="Advanced">
+              <Fieldset>
                 <Field name="opts.machine.deploy.targetHost">
                   {(field, props) => (
                     <>
@@ -123,9 +162,10 @@ export function CreateMachine() {
                     </>
                   )}
                 </Field>
-              </div>
-            </div>
-            <div class="mt-12 flex justify-end">
+              </Fieldset>
+            </Accordion>
+
+            <footer class="flex justify-end gap-y-3 border-t border-secondary-200 pt-5">
               <Button
                 type="submit"
                 disabled={formStore.submitting}
@@ -141,7 +181,7 @@ export function CreateMachine() {
                   <Match when={!formStore.submitting}>Create</Match>
                 </Switch>
               </Button>
-            </div>
+            </footer>
           </Form>
         </div>
       </div>

pkgs/webview-ui/app/src/routes/machines/details.tsx

@@ -459,10 +459,6 @@ const MachineForm = (props: MachineDetailsProps) => {
     }
 
     const target = targetHost();
-    if (!target) {
-      toast.error("Target host is required");
-      return;
-    }
 
     const loading_toast = toast.loading("Updating machine...");
     const r = await callApi("update_machines", {

pkgs/webview-ui/app/src/routes/machines/list.tsx

@@ -82,37 +82,31 @@ export const MachineListView: Component = () => {
                 size="s"
                 onClick={() => refresh()}
                 startIcon={<Icon icon="Update" />}
-              ></Button>
+              />
             </span>
 
-            <div class="border border-def-3">
-              <span class="" data-tip="List View">
-                <Button
-                  onclick={() => setView("list")}
-                  variant={view() == "list" ? "dark" : "light"}
-                  size="s"
-                  startIcon={<Icon icon="List" />}
-                ></Button>
-              </span>
-              <span class="" data-tip="Grid View">
-                <Button
-                  onclick={() => setView("grid")}
-                  variant={view() == "grid" ? "dark" : "light"}
-                  size="s"
-                  startIcon={<Icon icon="Grid" />}
-                ></Button>
-              </span>
-            </div>
-            <span class="" data-tip="New Machine">
+            <div class="button-group">
               <Button
-                onClick={() => navigate("create")}
+                onclick={() => setView("list")}
+                variant={view() == "list" ? "dark" : "light"}
                 size="s"
-                variant="light"
-                startIcon={<Icon icon="Plus" />}
-              >
-                New Machine
-              </Button>
-            </span>
+                startIcon={<Icon icon="List" />}
+              />
+              <Button
+                onclick={() => setView("grid")}
+                variant={view() == "grid" ? "dark" : "light"}
+                size="s"
+                startIcon={<Icon icon="Grid" />}
+              />
+            </div>
+            <Button
+              onClick={() => navigate("create")}
+              size="s"
+              variant="light"
+              startIcon={<Icon size={14} icon="Plus" />}
+            >
+              New Machine
+            </Button>
           </>
         }
       />

pkgs/webview-ui/app/src/routes/modules/list.tsx

@@ -19,9 +19,9 @@ interface CategoryProps {
 }
 const Categories = (props: CategoryProps) => {
   return (
-    <span class="ml-6 inline-flex h-full align-middle">
+    <span class="inline-flex h-full align-middle">
       {props.categories.map((category) => (
-        <span class="">{category}</span>
+        <span class="text-sm font-normal">{category}</span>
       ))}
     </span>
   );
@@ -32,10 +32,10 @@ interface RolesProps {
 }
 const Roles = (props: RolesProps) => {
   return (
-    <div>
+    <div class="flex flex-wrap items-center gap-2">
       <span>
         <Typography hierarchy="body" size="xs">
-          Service Typography{" "}
+          Service
         </Typography>
       </span>
       {props.roles.map((role) => (
@@ -54,9 +54,14 @@ const ModuleItem = (props: {
   const navigate = useNavigate();
 
   return (
-    <div class={cx("rounded-lg shadow-md", props.class)}>
-      <div class="text-primary-800">
-        <div class="">
+    <div
+      class={cx(
+        "col-span-1 flex flex-col gap-3 border-b border-secondary-200 pb-4",
+        props.class,
+      )}
+    >
+      {/* <div class="stat-figure text-primary-800">
+        <div class="join">
           <Menu popoverid={`menu-${props.name}`} label={<Icon icon={"More"} />}>
             <ul class="z-[1] w-52 p-2 shadow">
               <li>
@@ -71,20 +76,26 @@ const ModuleItem = (props: {
             </ul>
           </Menu>
         </div>
-      </div>
+      </div> */}
 
-      <A href={`/modules/details/${name}`}>
-        <div class="underline">
-          {name}
-          <Categories categories={info.categories} />
+      <header class="flex flex-col gap-4">
+        <A href={`/modules/details/${name}`}>
+          <div class="">
+            <div class="flex flex-col">
+              <Categories categories={info.categories} />
+              <Typography hierarchy="title" size="m" weight="medium">
+                {name}
+              </Typography>
+            </div>
+          </div>
+        </A>
+
+        <div class="w-full">
+          <Typography hierarchy="body" size="xs">
+            {info.description}
+          </Typography>
         </div>
-      </A>
-
-      <div class="w-full">
-        <Typography hierarchy="body" size="default">
-          {info.description}
-        </Typography>
-      </div>
+      </header>
       <Roles roles={info.roles || []} />
     </div>
   );
@@ -113,38 +124,33 @@ export const ModuleList = () => {
         title="Modules"
         toolbar={
           <>
-            <span class="" data-tip="Reload">
-              <Button
-                variant="light"
-                size="s"
-                onClick={() => refresh()}
-                startIcon={<Icon icon="Update" />}
-              ></Button>
-            </span>
+            <Button
+              variant="light"
+              size="s"
+              onClick={() => refresh()}
+              startIcon={<Icon icon="Update" />}
+            />
 
-            <div class="border border-def-3">
-              <span class="" data-tip="List View">
-                <Button
-                  onclick={() => setView("list")}
-                  variant={view() == "list" ? "dark" : "light"}
-                  size="s"
-                  startIcon={<Icon icon="List" />}
-                ></Button>
-              </span>
-              <span class="" data-tip="Grid View">
-                <Button
-                  onclick={() => setView("grid")}
-                  variant={view() == "grid" ? "dark" : "light"}
-                  size="s"
-                  startIcon={<Icon icon="Grid" />}
-                ></Button>
-              </span>
+            <div class="button-group">
+              <Button
+                onclick={() => setView("list")}
+                variant={view() == "list" ? "dark" : "light"}
+                size="s"
+                startIcon={<Icon icon="List" />}
+              />
+
+              <Button
+                onclick={() => setView("grid")}
+                variant={view() == "grid" ? "dark" : "light"}
+                size="s"
+                startIcon={<Icon icon="Grid" />}
+              />
             </div>
             <span class="" data-tip="New Machine">
               <Button
                 size="s"
                 variant="light"
-                startIcon={<Icon icon="CaretUp" />}
+                startIcon={<Icon size={14} icon="CaretUp" />}
               >
                 Import Module
               </Button>
@@ -156,10 +162,10 @@ export const ModuleList = () => {
         <Match when={modulesQuery.isFetching}>Loading....</Match>
         <Match when={modulesQuery.data}>
           <div
-            class="my-4 flex flex-wrap gap-6 px-3 py-2"
+            class="grid gap-6 p-6"
             classList={{
-              "flex-col": view() === "list",
-              "": view() === "grid",
+              "grid-cols-1": view() === "list",
+              "grid-cols-2": view() === "grid",
             }}
           >
             <For each={modulesQuery.data}>

pkgs/webview-ui/app/stylelint.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  extends: ["stylelint-config-standard", "stylelint-config-tailwindcss"],
+  rules: {
+    // You can adjust rules here
+  },
+};

pkgs/webview-ui/app/tailwind/core-plugin.ts

@@ -284,7 +284,7 @@ export default plugin.withOptions(
           "inner-primary":
             "2px 2px 0px 0px var(--clr-bg-inv-acc-3, #415E63) inset",
           "inner-primary-active":
-            "0px 0px 0px 1px #FFF, 0px 0px 0px 2px var(--clr-bg-inv-acc-4, #203637), -2px -2px 0px 0px var(--clr-bg-inv-acc-1, #7B9B9F) inset",
+            "0px 0px 0px 1px #FFF, 0px 0px 0px 2px var(--clr-bg-inv-acc-4, #203637)",
           "inner-secondary":
             "-2px -2px 0px 0px #CEDFE2 inset, 2px 2px 0px 0px white inset",
           "inner-secondary-active":

pkgs/webview-ui/app/vite.config.ts

@@ -3,6 +3,26 @@ import solidPlugin from "vite-plugin-solid";
 import solidSvg from "vite-plugin-solid-svg";
 import devtools from "solid-devtools/vite";
 import path from "node:path";
+import { exec } from "child_process";
+
+// watch also clan-cli to catch api changes
+const clanCliDir = path.resolve(__dirname, "../../clan-cli");
+
+function regenPythonApiOnFileChange() {
+  return {
+    name: "run-python-script-on-change",
+    handleHotUpdate({}) {
+      exec("reload-python-api.sh", (err, stdout, stderr) => {
+        if (err) {
+          console.error(`reload-python-api.sh error:\n${stderr}`);
+        }
+      });
+    },
+    configureServer(server: import("vite").ViteDevServer) {
+      server.watcher.add([clanCliDir]);
+    },
+  };
+}
 
 export default defineConfig({
   resolve: {
@@ -18,6 +38,7 @@ export default defineConfig({
     devtools(),
     solidPlugin(),
     solidSvg(),
+    regenPythonApiOnFileChange(),
   ],
   server: {
     port: 3000,

pkgs/webview-ui/bin/reload-python-api.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+script_dir=$(dirname "$(readlink -f "$0")")
+
+clan_cli="$script_dir/../../clan-cli"
+
+trap 'rm -rf "$tmpdir"' EXIT
+tmpdir=$(mktemp -d)
+
+set -x
+python "$clan_cli/api.py" > "$tmpdir/API.json"
+json2ts --input "$tmpdir/API.json" > "$tmpdir/API.ts"
+
+# compare sha256 sums of old and new API.ts
+old_api_hash=$(sha256sum "$script_dir/../app/api/API.ts" | cut -d ' ' -f 1)
+new_api_hash=$(sha256sum "$tmpdir/API.ts" | cut -d ' ' -f 1)
+if [ "$old_api_hash" != "$new_api_hash" ]; then
+  cp "$tmpdir/API.json" "$script_dir/../app/api/API.json"
+  cp "$tmpdir/API.ts" "$script_dir/../app/api/API.ts"
+fi

pkgs/webview-ui/flake-module.nix

@@ -53,11 +53,18 @@
           config.packages.webview-ui
           self'.devShells.default
         ];
+        packages = [
+          # required for reload-python-api.sh script
+          pkgs.python3
+          pkgs.json2ts
+        ];
         shellHook = ''
           export GIT_ROOT="$(git rev-parse --show-toplevel)"
           export PKG_ROOT="$GIT_ROOT/pkgs/webview-ui"
           export NODE_PATH="$PKG_ROOT/app/node_modules"
-          export PATH="$NODE_PATH/.bin:$PATH"
+
+          scriptsPath="$PKG_ROOT/bin"
+          export PATH="$NODE_PATH/.bin:$scriptsPath:$PATH"
 
           cp -r ${self'.packages.fonts} "$PKG_ROOT/app/.fonts"
           chmod -R +w "$PKG_ROOT/app/.fonts"