clan-cli: improve runtime dependency management

Many dependencies of clan-cli  are currently dynamically loaded via nix-shell on each execution.
This is nice, as it reduces the initial closure size of clan, but the overhead introduced by nix-shell piles up quickly, as some commands shell out many times during their lifetime. For example, when adding a secret git is called 10+ times.

This reduces the time of a test which adds a secret from around 50 seconds to 15 seconds.

- add run_cmd() as an alternative to nix_shell()
- introduce the concept of static dependencies which do not need to go through nix-shell
- static dependencies are defined at build time and included into the wrapper for clan-cli
- add package: clan-cli-full which statically ships all required dependencies

TODO: deprecate nix_shell() in favor of run_cmd()

pkgs/clan-cli/clan_cli/git.py

@@ -2,7 +2,7 @@ from pathlib import Path
 
 # from clan_cli.dirs import find_git_repo_root
 from clan_cli.errors import ClanError
-from clan_cli.nix import nix_shell
+from clan_cli.nix import run_cmd
 
 from .cmd import Log, run
 from .locked_open import locked_open
@@ -60,8 +60,8 @@ def _commit_file_to_git(
     """
     with locked_open(repo_dir / ".git" / "clan.lock", "w+"):
         for file_path in file_paths:
-            cmd = nix_shell(
-                ["nixpkgs#git"],
+            cmd = run_cmd(
+                ["git"],
                 ["git", "-C", str(repo_dir), "add", str(file_path)],
             )
             # add the file to the git index
@@ -73,8 +73,8 @@ def _commit_file_to_git(
             )
 
         # check if there is a diff
-        cmd = nix_shell(
-            ["nixpkgs#git"],
+        cmd = run_cmd(
+            ["git"],
             ["git", "-C", str(repo_dir), "diff", "--cached", "--exit-code"]
             + [str(file_path) for file_path in file_paths],
         )
@@ -84,8 +84,8 @@ def _commit_file_to_git(
             return
 
         # commit only that file
-        cmd = nix_shell(
-            ["nixpkgs#git"],
+        cmd = run_cmd(
+            ["git"],
             [
                 "git",
                 "-C",

pkgs/clan-cli/clan_cli/nix/__init__.py

@@ -4,8 +4,8 @@ import tempfile
 from pathlib import Path
 from typing import Any
 
-from .cmd import run, run_no_stdout
-from .dirs import nixpkgs_flake, nixpkgs_source
+from ..cmd import run, run_no_stdout
+from ..dirs import nixpkgs_flake, nixpkgs_source
 
 
 def nix_command(flags: list[str]) -> list[str]:
@@ -111,3 +111,49 @@ def nix_shell(packages: list[str], cmd: list[str]) -> list[str]:
         "-c",
         *cmd,
     ]
+
+
+# lazy loads list of allowed and static programs
+class Programs:
+    allowed_programs = None
+    static_programs = None
+
+    @classmethod
+    def is_allowed(cls: type["Programs"], program: str) -> bool:
+        if cls.allowed_programs is None:
+            with open(Path(__file__).parent / "allowed-programs.json") as f:
+                cls.allowed_programs = json.load(f)
+        return program in cls.allowed_programs
+
+    @classmethod
+    def is_static(cls: type["Programs"], program: str) -> bool:
+        """
+        Determines if a program is statically shipped with this clan distribution
+        """
+        if cls.static_programs is None:
+            cls.static_programs = os.environ.get("CLAN_STATIC_PROGRAMS", "").split(":")
+        return program in cls.static_programs
+
+
+# Alternative implementation of nix_shell() to replace nix_shell() at some point
+#   Features:
+#     - allow list for programs (need to be specified in allowed-programs.json)
+#     - be abe to compute a closure of all deps for testing
+#     - build clan distributions that ship some or all packages (eg. clan-cli-full)
+def run_cmd(programs: list[str], cmd: list[str]) -> list[str]:
+    for program in programs:
+        if not Programs.is_allowed(program):
+            raise ValueError(f"Program not allowed: {program}")
+    if os.environ.get("IN_NIX_SANDBOX"):
+        return cmd
+    missing_packages = [
+        f"nixpkgs#{program}" for program in programs if not Programs.is_static(program)
+    ]
+    if not missing_packages:
+        return cmd
+    return [
+        *nix_command(["shell", "--inputs-from", f"{nixpkgs_flake()!s}"]),
+        *missing_packages,
+        "-c",
+        *cmd,
+    ]

pkgs/clan-cli/clan_cli/nix/allowed-programs.json

@@ -0,0 +1,15 @@
+[
+  "age",
+  "bash",
+  "e2fsprogs",
+  "git",
+  "mypy",
+  "nix",
+  "openssh",
+  "qemu",
+  "rsync",
+  "sops",
+  "sshpass",
+  "tor",
+  "zbar"
+]