Merge branch 'main' into Qubasa-main

docs/quickstart.md

@@ -0,0 +1,112 @@
+# Initializing a New Clan Project
+
+## Clone the Clan Template
+
+To start a new project, execute the following command to clone the Clan Core template:
+
+```bash
+$ nix flake init -t git+https://git.clan.lol/clan/clan-core
+```
+
+This action will generate two primary files: `flake.nix` and `.clan-flake`.
+
+```bash
+$ ls -la
+drwx------ joerg users   5 B  a minute ago   ./
+drwxrwxrwt root  root  139 B  12 seconds ago ../
+.rw-r--r-- joerg users  77 B  a minute ago   .clan-flake
+.rw-r--r-- joerg users 4.8 KB a minute ago   flake.lock
+.rw-r--r-- joerg users 242 B  a minute ago   flake.nix
+```
+
+### Understanding the .clan-flake Marker File
+
+The `.clan-flake` marker file serves an optional purpose: it helps the `clan-cli` utility locate the project's root directory.
+If `.clan-flake` is missing, `clan-cli` will instead search for other indicators like `.git`, `.hg`, `.svn`, or `flake.nix` to identify the project root.
+
+---
+
+# Migrating Existing NixOS Configuration Flake
+
+Absolutely, let's break down the migration step by step, explaining each action in detail:
+
+#### Before You Begin
+
+1. **Backup Your Current Configuration**: Always start by making a backup of your current NixOS configuration to ensure you can revert if needed.
+
+   ```shell
+   cp -r /etc/nixos ~/nixos-backup
+   ```
+
+2. **Update Flake Inputs**: The patch adds a new input named `clan-core` to your `flake.nix`. This input points to a Git repository for Clan Core. Here's the addition:
+
+   ```nix
+   inputs.clan-core = {
+     url = "git+https://git.clan.lol/clan/clan-core";
+     inputs.nixpkgs.follows = "nixpkgs";
+   };
+   ```
+
+   - `url`: Specifies the Git repository URL for Clan Core.
+   - `inputs.nixpkgs.follows`: Tells Nix to use the same `nixpkgs` input as your main input (in this case, it follows `nixpkgs`).
+
+3. **Update Outputs**: Then modify the `outputs` section of your `flake.nix` to adapt to Clan Core's new provisioning method. The key changes are as follows:
+
+   Add `clan-core` to the output
+
+   ```diff
+   -  outputs = { self, nixpkgs,  }:
+   +  outputs = { self, nixpkgs, clan-core }:
+   ```
+
+   Previous configuration:
+
+   ```nix
+   nixosConfigurations.example-desktop = nixpkgs.lib.nixosSystem {
+       system = "x86_64-linux";
+       modules = [
+           ./configuration.nix
+       ];
+       [...]
+   };
+   ```
+
+   After change:
+
+   ```nix
+   nixosConfigurations = clan-core.lib.buildClan {
+       directory = ./.;
+       machines = {
+           example-desktop = {
+               nixpkgs.hostPlatform = "x86_64-linux";
+               imports = [
+                   ./configuration.nix
+               ];
+           };
+       };
+   };
+   ```
+
+   - `nixosConfigurations`: Defines NixOS configurations, using Clan Core’s `buildClan` function to manage the machines.
+   - Inside `machines`, a new machine configuration is defined (in this case, `example-desktop`).
+   - Inside `example-desktop` which is the target machine hostname, `nixpkgs.hostPlatform` specifies the host platform as `x86_64-linux`.
+
+4. **Rebuild and Switch**: Rebuild your NixOS configuration using the updated flake:
+
+   ```shell
+   sudo nixos-rebuild switch --flake .
+   ```
+
+   - This command rebuilds and switches to the new configuration. Make sure to include the `--flake .` argument to use the current directory as the flake source.
+
+5. **Test Configuration**: Before rebooting, verify that your new configuration builds without errors or warnings.
+
+6. **Reboot**: If everything is fine, you can reboot your system to apply the changes:
+
+   ```shell
+   sudo reboot
+   ```
+
+7. **Verify**: After the reboot, confirm that your system is running with the new configuration, and all services and applications are functioning as expected.
+
+By following these steps, you've successfully migrated your NixOS Flake configuration to include the `clan-core` input and adapted the `outputs` section to work with Clan Core's new machine provisioning method.

flake.nix

@@ -24,6 +24,7 @@
         "x86_64-linux"
         "aarch64-linux"
       ];
+      flake.clanModules = { };
       imports = [
         ./checks/flake-module.nix
         ./devShell.nix
@@ -36,7 +37,7 @@
 
         ./lib/flake-module.nix
         ./nixosModules/flake-module.nix
-        ./nixosModules/clanCore/flake-module.nix
+        ./nixosModules/core/flake-module.nix
       ];
     });
 }

flakeModules/clan-config.nix

@@ -1,4 +1,4 @@
-{ ... } @ clanCore: {
+{ ... } @ core: {
   flake.flakeModules.clan-config = { self, inputs, ... }:
     let
 
@@ -29,12 +29,12 @@
       perSystem = { pkgs, ... }: {
         devShells.clan-config = pkgs.mkShell {
           packages = [
-            clanCore.config.flake.packages.${pkgs.system}.clan-cli
+            core.config.flake.packages.${pkgs.system}.clan-cli
           ];
           shellHook = ''
             export CLAN_OPTIONS_FILE=$(nix eval --raw .#clanOptions)
-            export XDG_DATA_DIRS="${clanCore.config.flake.packages.${pkgs.system}.clan-cli}/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}"
+            export XDG_DATA_DIRS="${core.config.flake.packages.${pkgs.system}.clan-cli}/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}"
-            export fish_complete_path="${clanCore.config.flake.packages.${pkgs.system}.clan-cli}/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}"
+            export fish_complete_path="${core.config.flake.packages.${pkgs.system}.clan-cli}/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}"
           '';
         };
       };

lib/build-clan/default.nix

@@ -1,4 +1,4 @@
-{ nixpkgs, clan, lib }:
+{ nixpkgs, lib }:
 { directory  # The directory containing the machines subdirectory
 , specialArgs ? { } # Extra arguments to pass to nixosSystem i.e. useful to make self available
 , machines ? { } # allows to include machine-specific modules i.e. machines.${name} = { ... }
@@ -15,14 +15,14 @@ let
     else { };
 
   nixosConfigurations = lib.mapAttrs
-    (name: _mod:
+    (name: _:
       nixpkgs.lib.nixosSystem {
         modules = [
           (machineSettings name)
           (machines.${name} or { })
-        ] ++ lib.attrValues clan.clanModules;
+        ];
         specialArgs = specialArgs;
       })
-    machinesDirs;
+    (machinesDirs // machines);
 in
 nixosConfigurations

lib/default.nix

@@ -1,4 +1,4 @@
-{ lib, clan, nixpkgs, ... }:
+{ lib, nixpkgs, ... }:
 {
   findNixFiles = folder:
     lib.mapAttrs'
@@ -14,5 +14,5 @@
 
   jsonschema = import ./jsonschema { inherit lib; };
 
-  buildClan = import ./build-clan { inherit lib clan nixpkgs; };
+  buildClan = import ./build-clan { inherit lib nixpkgs; };
 }

lib/flake-module.nix

@@ -7,6 +7,6 @@
   ];
   flake.lib = import ./default.nix {
     inherit lib;
-    inherit (inputs) clan nixpkgs;
+    inherit (inputs) nixpkgs;
   };
 }

nixosModules/core/flake-module.nix

@@ -1,6 +1,6 @@
 { self, inputs, lib, ... }: {
-  flake.nixosModules.clanCore = { pkgs, ... }: {
+  flake.nixosModules.clan.core = { pkgs, ... }: {
-    options.clanCore = {
+    options.clan.core = {
       clanDir = lib.mkOption {
         type = lib.types.str;
         description = ''

nixosModules/core/secrets/default.nix

@@ -1,6 +1,6 @@
 { config, lib, ... }:
 {
-  options.clanCore.secrets = lib.mkOption {
+  options.clan.core.secrets = lib.mkOption {
     type = lib.types.attrsOf
       (lib.types.submodule (secret: {
         options = {
@@ -49,7 +49,7 @@
                   description = ''
                     path to a fact which is generated by the generator
                   '';
-                  default = "${config.clanCore.clanDir}/facts/${config.clanCore.machineName}/${fact.config._module.args.name}";
+                  default = "${config.clan.core.clanDir}/facts/${config.clan.core.machineName}/${fact.config._module.args.name}";
                 };
                 value = lib.mkOption {
                   default = builtins.readFile fact.config.path;

nixosModules/core/secrets/sops.nix

@@ -7,24 +7,24 @@
       set -x # remove for prod
 
       PATH=$PATH:${lib.makeBinPath [
-        config.clanCore.clanPkgs.clan-cli
+        config.clan.core.clanPkgs.clan-cli
       ]}
 
       # initialize secret store
-      if ! clan secrets machines list | grep -q ${config.clanCore.machineName}; then (
+      if ! clan secrets machines list | grep -q ${config.clan.core.machineName}; then (
         INITTMP=$(mktemp -d)
         trap 'rm -rf "$INITTMP"' EXIT
         ${pkgs.age}/bin/age-keygen -o "$INITTMP/secret" 2> "$INITTMP/public"
         PUBKEY=$(cat "$INITTMP/public" | sed 's/.*: //')
-        clan secrets machines add ${config.clanCore.machineName} "$PUBKEY"
+        clan secrets machines add ${config.clan.core.machineName} "$PUBKEY"
-        tail -1 "$INITTMP/secret" | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-age.key
+        tail -1 "$INITTMP/secret" | clan secrets set --machine ${config.clan.core.machineName} ${config.clan.core.machineName}-age.key
       ) fi
 
       ${lib.foldlAttrs (acc: n: v: ''
         ${acc}
         # ${n}
         # if any of the secrets are missing, we regenerate all connected facts/secrets
-        (if ! ${lib.concatMapStringsSep " && " (x: "clan secrets get ${config.clanCore.machineName}-${x.name} >/dev/null") (lib.attrValues v.secrets)}; then
+        (if ! ${lib.concatMapStringsSep " && " (x: "clan secrets get ${config.clan.core.machineName}-${x.name} >/dev/null") (lib.attrValues v.secrets)}; then
 
           facts=$(mktemp -d)
           trap "rm -rf $facts" EXIT
@@ -38,24 +38,24 @@
           '') (lib.attrValues v.facts)}
 
           ${lib.concatMapStrings (secret: ''
-            cat "$secrets"/${secret.name} | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-${secret.name}
+            cat "$secrets"/${secret.name} | clan secrets set --machine ${config.clan.core.machineName} ${config.clan.core.machineName}-${secret.name}
           '') (lib.attrValues v.secrets)}
         fi)
-      '') "" config.clanCore.secrets}
+      '') "" config.clan.core.secrets}
     '';
     sops.secrets =
       let
         encryptedForThisMachine = name: type:
           let
-            symlink = config.clanCore.clanDir + "/sops/secrets/${name}/machines/${config.clanCore.machineName}";
+            symlink = config.clan.core.clanDir + "/sops/secrets/${name}/machines/${config.clan.core.machineName}";
           in
           # WTF, nix bug, my symlink is in the nixos module detected as a directory also it works in the repl
           type == "directory" && (builtins.readFileType symlink == "directory" || builtins.readFileType symlink == "symlink");
-        secrets = lib.filterAttrs encryptedForThisMachine (builtins.readDir (config.clanCore.clanDir + "/sops/secrets"));
+        secrets = lib.filterAttrs encryptedForThisMachine (builtins.readDir (config.clan.core.clanDir + "/sops/secrets"));
       in
       builtins.mapAttrs
         (name: _: {
-          sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret";
+          sopsFile = config.clan.core.clanDir + "/sops/secrets/${name}/secret";
           format = "binary";
         })
         secrets;

nixosModules/core/zerotier.nix

@@ -41,13 +41,13 @@ in
   } // lib.mkIf cfg.controller.enable {
     # only the controller needs to have the key in the repo, the other clients can be dynamic
     # we generate the zerotier code manually for the controller, since it's part of the bootstrap command
-    clanCore.secrets.zerotier = {
+    clan.core.secrets.zerotier = {
       facts."network.id" = { };
       secrets."identity.secret" = { };
       generator = ''
         TMPDIR=$(mktemp -d)
         trap 'rm -rf "$TMPDIR"' EXIT
-        ${config.clanCore.clanPkgs.clan-cli}/bin/clan zerotier --outpath "$TMPDIR"
+        ${config.clan.core.clanPkgs.clan-cli}/bin/clan zerotier --outpath "$TMPDIR"
         cp "$TMPDIR"/network.id "$facts"/network.id
         cp "$TMPDIR"/identity.secret "$secrets"/identity.secret
       '';

pkgs/clan-cli/.envrc

@@ -4,7 +4,9 @@ source_up
 
 if type nix_direnv_watch_file &>/dev/null; then
   nix_direnv_watch_file flake-module.nix
+  nix_direnv_watch_file default.nix
 else
   direnv watch flake-module.nix
+  direnv watch default.nix
 fi
 use flake .#clan-cli --builders ''

pkgs/clan-cli/README.md

@@ -27,3 +27,18 @@ To start a local developement environment instead, use the `--dev` flag:
 ```
 
 This will spawn two webserver, a python one to for the api and a nodejs one that rebuilds the ui on the fly.
+
+## Run locally single-threaded for debugging
+
+By default tests run in parallel using pytest-parallel.
+pytest-parallel however breaks `breakpoint()`. To disable it, use this:
+
+```console
+pytest --workers "" -s
+```
+
+You can also run a single test like this:
+
+```console
+pytest --workers "" -s tests/test_secrets_cli.py::test_users
+```

pkgs/clan-cli/clan_cli/secrets/generate.py

@@ -15,7 +15,7 @@ def get_secret_script(machine: str) -> None:
             "--expr",
             "let f = builtins.getFlake (toString ./.); in "
             f"(f.nixosConfigurations.{machine}.extendModules "
-            "{ modules = [{ clanCore.clanDir = toString ./.; }]; })"
+            "{ modules = [{ clan.core.clanDir = toString ./.; }]; })"
             ".config.system.clan.generateSecrets",
         ],
         check=True,

pkgs/clan-cli/clan_cli/secrets/machines.py

@@ -3,11 +3,8 @@ import argparse
 from ..machines.types import machine_name_type, validate_hostname
 from . import secrets
 from .folders import list_objects, remove_object, sops_machines_folder
-from .sops import write_key
+from .sops import read_key, write_key
-from .types import (
+from .types import public_or_private_age_key_type, secret_name_type
-    public_or_private_age_key_type,
-    secret_name_type,
-)
 
 
 def add_machine(name: str, key: str, force: bool) -> None:
@@ -18,6 +15,10 @@ def remove_machine(name: str) -> None:
     remove_object(sops_machines_folder(), name)
 
 
+def get_machine(name: str) -> str:
+    return read_key(sops_machines_folder() / name)
+
+
 def list_machines() -> list[str]:
     return list_objects(sops_machines_folder(), lambda x: validate_hostname(x))
 
@@ -42,6 +43,10 @@ def add_command(args: argparse.Namespace) -> None:
     add_machine(args.machine, args.key, args.force)
 
 
+def get_command(args: argparse.Namespace) -> None:
+    print(get_machine(args.machine))
+
+
 def remove_command(args: argparse.Namespace) -> None:
     remove_machine(args.machine)
 
@@ -82,6 +87,12 @@ def register_machines_parser(parser: argparse.ArgumentParser) -> None:
     )
     add_parser.set_defaults(func=add_command)
 
+    get_parser = subparser.add_parser("get", help="get a machine public key")
+    get_parser.add_argument(
+        "machine", help="the name of the machine", type=machine_name_type
+    )
+    get_parser.set_defaults(func=get_command)
+
     remove_parser = subparser.add_parser("remove", help="remove a machine")
     remove_parser.add_argument(
         "machine", help="the name of the machine", type=machine_name_type

pkgs/clan-cli/clan_cli/secrets/users.py

@@ -2,7 +2,7 @@ import argparse
 
 from . import secrets
 from .folders import list_objects, remove_object, sops_users_folder
-from .sops import write_key
+from .sops import read_key, write_key
 from .types import (
     VALID_SECRET_NAME,
     public_or_private_age_key_type,
@@ -19,6 +19,10 @@ def remove_user(name: str) -> None:
     remove_object(sops_users_folder(), name)
 
 
+def get_user(name: str) -> str:
+    return read_key(sops_users_folder() / name)
+
+
 def list_users() -> list[str]:
     return list_objects(
         sops_users_folder(), lambda n: VALID_SECRET_NAME.match(n) is not None
@@ -43,6 +47,10 @@ def add_command(args: argparse.Namespace) -> None:
     add_user(args.user, args.key, args.force)
 
 
+def get_command(args: argparse.Namespace) -> None:
+    print(get_user(args.user))
+
+
 def remove_command(args: argparse.Namespace) -> None:
     remove_user(args.user)
 
@@ -77,6 +85,10 @@ def register_users_parser(parser: argparse.ArgumentParser) -> None:
     )
     add_parser.set_defaults(func=add_command)
 
+    get_parser = subparser.add_parser("get", help="get a user public key")
+    get_parser.add_argument("user", help="the name of the user", type=user_name_type)
+    get_parser.set_defaults(func=get_command)
+
     remove_parser = subparser.add_parser("remove", help="remove a user")
     remove_parser.add_argument("user", help="the name of the user", type=user_name_type)
     remove_parser.set_defaults(func=remove_command)

pkgs/clan-cli/tests/test_secrets_cli.py

@@ -36,8 +36,13 @@ def _test_identities(
             age_keys[0].privkey,
         ]
     )
-    capsys.readouterr()  # empty the buffer
 
+    capsys.readouterr()  # empty the buffer
+    cli.run(["secrets", what, "get", "foo"])
+    out = capsys.readouterr()  # empty the buffer
+    assert age_keys[0].pubkey in out.out
+
+    capsys.readouterr()  # empty the buffer
     cli.run(["secrets", what, "list"])
     out = capsys.readouterr()  # empty the buffer
     assert "foo" in out.out

scripts/ci

@@ -26,7 +26,9 @@ fi
 
 rc=0
 
-for job in $(nix shell --inputs-from '.#' "nixpkgs#nix-eval-jobs" -c nix-eval-jobs "${args[@]}" | jq -r '. | @base64'); do
+nix shell --inputs-from '.#' "nixpkgs#nix-eval-jobs" -c nix-eval-jobs "${args[@]}" > "jobs.json"
+
+for job in $(jq -r '. | @base64' < "jobs.json"); do
   job=$(echo "$job" | base64 -d)
   attr=$(echo "$job" | jq -r .attr)
   echo "### $attr"

templates/flake-module.nix

@@ -1,8 +1,9 @@
-{
+{ self, ... }: {
   flake.templates = {
     new-clan = {
       description = "Initialize a new clan flake";
       path = ./new-clan;
     };
+    default = self.templates.new-clan;
   };
 }