diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 000000000..63cb208ca --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,112 @@ +# Initializing a New Clan Project + +## Clone the Clan Template + +To start a new project, execute the following command to clone the Clan Core template: + +```bash +$ nix flake init -t git+https://git.clan.lol/clan/clan-core +``` + +This action will generate two primary files: `flake.nix` and `.clan-flake`. + +```bash +$ ls -la +drwx------ joerg users 5 B a minute ago ./ +drwxrwxrwt root root 139 B 12 seconds ago ../ +.rw-r--r-- joerg users 77 B a minute ago .clan-flake +.rw-r--r-- joerg users 4.8 KB a minute ago flake.lock +.rw-r--r-- joerg users 242 B a minute ago flake.nix +``` + +### Understanding the .clan-flake Marker File + +The `.clan-flake` marker file serves an optional purpose: it helps the `clan-cli` utility locate the project's root directory. +If `.clan-flake` is missing, `clan-cli` will instead search for other indicators like `.git`, `.hg`, `.svn`, or `flake.nix` to identify the project root. + +--- + +# Migrating Existing NixOS Configuration Flake + +Absolutely, let's break down the migration step by step, explaining each action in detail: + +#### Before You Begin + +1. **Backup Your Current Configuration**: Always start by making a backup of your current NixOS configuration to ensure you can revert if needed. + + ```shell + cp -r /etc/nixos ~/nixos-backup + ``` + +2. **Update Flake Inputs**: The patch adds a new input named `clan-core` to your `flake.nix`. This input points to a Git repository for Clan Core. Here's the addition: + + ```nix + inputs.clan-core = { + url = "git+https://git.clan.lol/clan/clan-core"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + ``` + + - `url`: Specifies the Git repository URL for Clan Core. + - `inputs.nixpkgs.follows`: Tells Nix to use the same `nixpkgs` input as your main input (in this case, it follows `nixpkgs`). + +3. **Update Outputs**: Then modify the `outputs` section of your `flake.nix` to adapt to Clan Core's new provisioning method. The key changes are as follows: + + Add `clan-core` to the output + + ```diff + - outputs = { self, nixpkgs, }: + + outputs = { self, nixpkgs, clan-core }: + ``` + + Previous configuration: + + ```nix + nixosConfigurations.example-desktop = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ + ./configuration.nix + ]; + [...] + }; + ``` + + After change: + + ```nix + nixosConfigurations = clan-core.lib.buildClan { + directory = ./.; + machines = { + example-desktop = { + nixpkgs.hostPlatform = "x86_64-linux"; + imports = [ + ./configuration.nix + ]; + }; + }; + }; + ``` + + - `nixosConfigurations`: Defines NixOS configurations, using Clan Core’s `buildClan` function to manage the machines. + - Inside `machines`, a new machine configuration is defined (in this case, `example-desktop`). + - Inside `example-desktop` which is the target machine hostname, `nixpkgs.hostPlatform` specifies the host platform as `x86_64-linux`. + +4. **Rebuild and Switch**: Rebuild your NixOS configuration using the updated flake: + + ```shell + sudo nixos-rebuild switch --flake . + ``` + + - This command rebuilds and switches to the new configuration. Make sure to include the `--flake .` argument to use the current directory as the flake source. + +5. **Test Configuration**: Before rebooting, verify that your new configuration builds without errors or warnings. + +6. **Reboot**: If everything is fine, you can reboot your system to apply the changes: + + ```shell + sudo reboot + ``` + +7. **Verify**: After the reboot, confirm that your system is running with the new configuration, and all services and applications are functioning as expected. + +By following these steps, you've successfully migrated your NixOS Flake configuration to include the `clan-core` input and adapted the `outputs` section to work with Clan Core's new machine provisioning method. diff --git a/flake.nix b/flake.nix index cf9d45ab9..4b908bddf 100644 --- a/flake.nix +++ b/flake.nix @@ -24,6 +24,7 @@ "x86_64-linux" "aarch64-linux" ]; + flake.clanModules = { }; imports = [ ./checks/flake-module.nix ./devShell.nix @@ -36,7 +37,7 @@ ./lib/flake-module.nix ./nixosModules/flake-module.nix - ./nixosModules/clanCore/flake-module.nix + ./nixosModules/core/flake-module.nix ]; }); } diff --git a/flakeModules/clan-config.nix b/flakeModules/clan-config.nix index 236d22592..2d1cd5c87 100644 --- a/flakeModules/clan-config.nix +++ b/flakeModules/clan-config.nix @@ -1,4 +1,4 @@ -{ ... } @ clanCore: { +{ ... } @ core: { flake.flakeModules.clan-config = { self, inputs, ... }: let @@ -29,12 +29,12 @@ perSystem = { pkgs, ... }: { devShells.clan-config = pkgs.mkShell { packages = [ - clanCore.config.flake.packages.${pkgs.system}.clan-cli + core.config.flake.packages.${pkgs.system}.clan-cli ]; shellHook = '' export CLAN_OPTIONS_FILE=$(nix eval --raw .#clanOptions) - export XDG_DATA_DIRS="${clanCore.config.flake.packages.${pkgs.system}.clan-cli}/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}" - export fish_complete_path="${clanCore.config.flake.packages.${pkgs.system}.clan-cli}/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}" + export XDG_DATA_DIRS="${core.config.flake.packages.${pkgs.system}.clan-cli}/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}" + export fish_complete_path="${core.config.flake.packages.${pkgs.system}.clan-cli}/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}" ''; }; }; diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 888eb6b25..fc6c465f2 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -1,4 +1,4 @@ -{ nixpkgs, clan, lib }: +{ nixpkgs, lib }: { directory # The directory containing the machines subdirectory , specialArgs ? { } # Extra arguments to pass to nixosSystem i.e. useful to make self available , machines ? { } # allows to include machine-specific modules i.e. machines.${name} = { ... } @@ -15,14 +15,14 @@ let else { }; nixosConfigurations = lib.mapAttrs - (name: _mod: + (name: _: nixpkgs.lib.nixosSystem { modules = [ (machineSettings name) (machines.${name} or { }) - ] ++ lib.attrValues clan.clanModules; + ]; specialArgs = specialArgs; }) - machinesDirs; + (machinesDirs // machines); in nixosConfigurations diff --git a/lib/default.nix b/lib/default.nix index 89ac0cc3f..066e88eff 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -1,4 +1,4 @@ -{ lib, clan, nixpkgs, ... }: +{ lib, nixpkgs, ... }: { findNixFiles = folder: lib.mapAttrs' @@ -14,5 +14,5 @@ jsonschema = import ./jsonschema { inherit lib; }; - buildClan = import ./build-clan { inherit lib clan nixpkgs; }; + buildClan = import ./build-clan { inherit lib nixpkgs; }; } diff --git a/lib/flake-module.nix b/lib/flake-module.nix index 48c682e2f..64369ec24 100644 --- a/lib/flake-module.nix +++ b/lib/flake-module.nix @@ -7,6 +7,6 @@ ]; flake.lib = import ./default.nix { inherit lib; - inherit (inputs) clan nixpkgs; + inherit (inputs) nixpkgs; }; } diff --git a/nixosModules/clanCore/flake-module.nix b/nixosModules/core/flake-module.nix similarity index 91% rename from nixosModules/clanCore/flake-module.nix rename to nixosModules/core/flake-module.nix index da8fa0e63..55648bf64 100644 --- a/nixosModules/clanCore/flake-module.nix +++ b/nixosModules/core/flake-module.nix @@ -1,6 +1,6 @@ { self, inputs, lib, ... }: { - flake.nixosModules.clanCore = { pkgs, ... }: { - options.clanCore = { + flake.nixosModules.clan.core = { pkgs, ... }: { + options.clan.core = { clanDir = lib.mkOption { type = lib.types.str; description = '' diff --git a/nixosModules/clanCore/secrets/default.nix b/nixosModules/core/secrets/default.nix similarity index 92% rename from nixosModules/clanCore/secrets/default.nix rename to nixosModules/core/secrets/default.nix index f1128a327..1660de63c 100644 --- a/nixosModules/clanCore/secrets/default.nix +++ b/nixosModules/core/secrets/default.nix @@ -1,6 +1,6 @@ { config, lib, ... }: { - options.clanCore.secrets = lib.mkOption { + options.clan.core.secrets = lib.mkOption { type = lib.types.attrsOf (lib.types.submodule (secret: { options = { @@ -49,7 +49,7 @@ description = '' path to a fact which is generated by the generator ''; - default = "${config.clanCore.clanDir}/facts/${config.clanCore.machineName}/${fact.config._module.args.name}"; + default = "${config.clan.core.clanDir}/facts/${config.clan.core.machineName}/${fact.config._module.args.name}"; }; value = lib.mkOption { default = builtins.readFile fact.config.path; diff --git a/nixosModules/clanCore/secrets/sops.nix b/nixosModules/core/secrets/sops.nix similarity index 70% rename from nixosModules/clanCore/secrets/sops.nix rename to nixosModules/core/secrets/sops.nix index 7df0b31d2..cc0507d5d 100644 --- a/nixosModules/clanCore/secrets/sops.nix +++ b/nixosModules/core/secrets/sops.nix @@ -7,24 +7,24 @@ set -x # remove for prod PATH=$PATH:${lib.makeBinPath [ - config.clanCore.clanPkgs.clan-cli + config.clan.core.clanPkgs.clan-cli ]} # initialize secret store - if ! clan secrets machines list | grep -q ${config.clanCore.machineName}; then ( + if ! clan secrets machines list | grep -q ${config.clan.core.machineName}; then ( INITTMP=$(mktemp -d) trap 'rm -rf "$INITTMP"' EXIT ${pkgs.age}/bin/age-keygen -o "$INITTMP/secret" 2> "$INITTMP/public" PUBKEY=$(cat "$INITTMP/public" | sed 's/.*: //') - clan secrets machines add ${config.clanCore.machineName} "$PUBKEY" - tail -1 "$INITTMP/secret" | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-age.key + clan secrets machines add ${config.clan.core.machineName} "$PUBKEY" + tail -1 "$INITTMP/secret" | clan secrets set --machine ${config.clan.core.machineName} ${config.clan.core.machineName}-age.key ) fi ${lib.foldlAttrs (acc: n: v: '' ${acc} # ${n} # if any of the secrets are missing, we regenerate all connected facts/secrets - (if ! ${lib.concatMapStringsSep " && " (x: "clan secrets get ${config.clanCore.machineName}-${x.name} >/dev/null") (lib.attrValues v.secrets)}; then + (if ! ${lib.concatMapStringsSep " && " (x: "clan secrets get ${config.clan.core.machineName}-${x.name} >/dev/null") (lib.attrValues v.secrets)}; then facts=$(mktemp -d) trap "rm -rf $facts" EXIT @@ -38,24 +38,24 @@ '') (lib.attrValues v.facts)} ${lib.concatMapStrings (secret: '' - cat "$secrets"/${secret.name} | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-${secret.name} + cat "$secrets"/${secret.name} | clan secrets set --machine ${config.clan.core.machineName} ${config.clan.core.machineName}-${secret.name} '') (lib.attrValues v.secrets)} fi) - '') "" config.clanCore.secrets} + '') "" config.clan.core.secrets} ''; sops.secrets = let encryptedForThisMachine = name: type: let - symlink = config.clanCore.clanDir + "/sops/secrets/${name}/machines/${config.clanCore.machineName}"; + symlink = config.clan.core.clanDir + "/sops/secrets/${name}/machines/${config.clan.core.machineName}"; in # WTF, nix bug, my symlink is in the nixos module detected as a directory also it works in the repl type == "directory" && (builtins.readFileType symlink == "directory" || builtins.readFileType symlink == "symlink"); - secrets = lib.filterAttrs encryptedForThisMachine (builtins.readDir (config.clanCore.clanDir + "/sops/secrets")); + secrets = lib.filterAttrs encryptedForThisMachine (builtins.readDir (config.clan.core.clanDir + "/sops/secrets")); in builtins.mapAttrs (name: _: { - sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret"; + sopsFile = config.clan.core.clanDir + "/sops/secrets/${name}/secret"; format = "binary"; }) secrets; diff --git a/nixosModules/clanCore/zerotier.nix b/nixosModules/core/zerotier.nix similarity index 95% rename from nixosModules/clanCore/zerotier.nix rename to nixosModules/core/zerotier.nix index 452294d95..67b9b6fec 100644 --- a/nixosModules/clanCore/zerotier.nix +++ b/nixosModules/core/zerotier.nix @@ -41,13 +41,13 @@ in } // lib.mkIf cfg.controller.enable { # only the controller needs to have the key in the repo, the other clients can be dynamic # we generate the zerotier code manually for the controller, since it's part of the bootstrap command - clanCore.secrets.zerotier = { + clan.core.secrets.zerotier = { facts."network.id" = { }; secrets."identity.secret" = { }; generator = '' TMPDIR=$(mktemp -d) trap 'rm -rf "$TMPDIR"' EXIT - ${config.clanCore.clanPkgs.clan-cli}/bin/clan zerotier --outpath "$TMPDIR" + ${config.clan.core.clanPkgs.clan-cli}/bin/clan zerotier --outpath "$TMPDIR" cp "$TMPDIR"/network.id "$facts"/network.id cp "$TMPDIR"/identity.secret "$secrets"/identity.secret ''; diff --git a/pkgs/clan-cli/.envrc b/pkgs/clan-cli/.envrc index 00f84d526..53d6aa325 100644 --- a/pkgs/clan-cli/.envrc +++ b/pkgs/clan-cli/.envrc @@ -4,7 +4,9 @@ source_up if type nix_direnv_watch_file &>/dev/null; then nix_direnv_watch_file flake-module.nix + nix_direnv_watch_file default.nix else direnv watch flake-module.nix + direnv watch default.nix fi use flake .#clan-cli --builders '' diff --git a/pkgs/clan-cli/README.md b/pkgs/clan-cli/README.md index 538916486..df9c50907 100644 --- a/pkgs/clan-cli/README.md +++ b/pkgs/clan-cli/README.md @@ -27,3 +27,18 @@ To start a local developement environment instead, use the `--dev` flag: ``` This will spawn two webserver, a python one to for the api and a nodejs one that rebuilds the ui on the fly. + +## Run locally single-threaded for debugging + +By default tests run in parallel using pytest-parallel. +pytest-parallel however breaks `breakpoint()`. To disable it, use this: + +```console +pytest --workers "" -s +``` + +You can also run a single test like this: + +```console +pytest --workers "" -s tests/test_secrets_cli.py::test_users +``` diff --git a/pkgs/clan-cli/clan_cli/secrets/generate.py b/pkgs/clan-cli/clan_cli/secrets/generate.py index 0b01a8c85..01499db0a 100644 --- a/pkgs/clan-cli/clan_cli/secrets/generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/generate.py @@ -15,7 +15,7 @@ def get_secret_script(machine: str) -> None: "--expr", "let f = builtins.getFlake (toString ./.); in " f"(f.nixosConfigurations.{machine}.extendModules " - "{ modules = [{ clanCore.clanDir = toString ./.; }]; })" + "{ modules = [{ clan.core.clanDir = toString ./.; }]; })" ".config.system.clan.generateSecrets", ], check=True, diff --git a/pkgs/clan-cli/clan_cli/secrets/machines.py b/pkgs/clan-cli/clan_cli/secrets/machines.py index 80a07e6d9..811e88367 100644 --- a/pkgs/clan-cli/clan_cli/secrets/machines.py +++ b/pkgs/clan-cli/clan_cli/secrets/machines.py @@ -3,11 +3,8 @@ import argparse from ..machines.types import machine_name_type, validate_hostname from . import secrets from .folders import list_objects, remove_object, sops_machines_folder -from .sops import write_key -from .types import ( - public_or_private_age_key_type, - secret_name_type, -) +from .sops import read_key, write_key +from .types import public_or_private_age_key_type, secret_name_type def add_machine(name: str, key: str, force: bool) -> None: @@ -18,6 +15,10 @@ def remove_machine(name: str) -> None: remove_object(sops_machines_folder(), name) +def get_machine(name: str) -> str: + return read_key(sops_machines_folder() / name) + + def list_machines() -> list[str]: return list_objects(sops_machines_folder(), lambda x: validate_hostname(x)) @@ -42,6 +43,10 @@ def add_command(args: argparse.Namespace) -> None: add_machine(args.machine, args.key, args.force) +def get_command(args: argparse.Namespace) -> None: + print(get_machine(args.machine)) + + def remove_command(args: argparse.Namespace) -> None: remove_machine(args.machine) @@ -82,6 +87,12 @@ def register_machines_parser(parser: argparse.ArgumentParser) -> None: ) add_parser.set_defaults(func=add_command) + get_parser = subparser.add_parser("get", help="get a machine public key") + get_parser.add_argument( + "machine", help="the name of the machine", type=machine_name_type + ) + get_parser.set_defaults(func=get_command) + remove_parser = subparser.add_parser("remove", help="remove a machine") remove_parser.add_argument( "machine", help="the name of the machine", type=machine_name_type diff --git a/pkgs/clan-cli/clan_cli/secrets/users.py b/pkgs/clan-cli/clan_cli/secrets/users.py index 25cf28ae2..760218af8 100644 --- a/pkgs/clan-cli/clan_cli/secrets/users.py +++ b/pkgs/clan-cli/clan_cli/secrets/users.py @@ -2,7 +2,7 @@ import argparse from . import secrets from .folders import list_objects, remove_object, sops_users_folder -from .sops import write_key +from .sops import read_key, write_key from .types import ( VALID_SECRET_NAME, public_or_private_age_key_type, @@ -19,6 +19,10 @@ def remove_user(name: str) -> None: remove_object(sops_users_folder(), name) +def get_user(name: str) -> str: + return read_key(sops_users_folder() / name) + + def list_users() -> list[str]: return list_objects( sops_users_folder(), lambda n: VALID_SECRET_NAME.match(n) is not None @@ -43,6 +47,10 @@ def add_command(args: argparse.Namespace) -> None: add_user(args.user, args.key, args.force) +def get_command(args: argparse.Namespace) -> None: + print(get_user(args.user)) + + def remove_command(args: argparse.Namespace) -> None: remove_user(args.user) @@ -77,6 +85,10 @@ def register_users_parser(parser: argparse.ArgumentParser) -> None: ) add_parser.set_defaults(func=add_command) + get_parser = subparser.add_parser("get", help="get a user public key") + get_parser.add_argument("user", help="the name of the user", type=user_name_type) + get_parser.set_defaults(func=get_command) + remove_parser = subparser.add_parser("remove", help="remove a user") remove_parser.add_argument("user", help="the name of the user", type=user_name_type) remove_parser.set_defaults(func=remove_command) diff --git a/pkgs/clan-cli/tests/test_secrets_cli.py b/pkgs/clan-cli/tests/test_secrets_cli.py index d614c7737..ccdffd799 100644 --- a/pkgs/clan-cli/tests/test_secrets_cli.py +++ b/pkgs/clan-cli/tests/test_secrets_cli.py @@ -36,8 +36,13 @@ def _test_identities( age_keys[0].privkey, ] ) - capsys.readouterr() # empty the buffer + capsys.readouterr() # empty the buffer + cli.run(["secrets", what, "get", "foo"]) + out = capsys.readouterr() # empty the buffer + assert age_keys[0].pubkey in out.out + + capsys.readouterr() # empty the buffer cli.run(["secrets", what, "list"]) out = capsys.readouterr() # empty the buffer assert "foo" in out.out diff --git a/scripts/ci b/scripts/ci index c39be836b..85e766546 100755 --- a/scripts/ci +++ b/scripts/ci @@ -26,7 +26,9 @@ fi rc=0 -for job in $(nix shell --inputs-from '.#' "nixpkgs#nix-eval-jobs" -c nix-eval-jobs "${args[@]}" | jq -r '. | @base64'); do +nix shell --inputs-from '.#' "nixpkgs#nix-eval-jobs" -c nix-eval-jobs "${args[@]}" > "jobs.json" + +for job in $(jq -r '. | @base64' < "jobs.json"); do job=$(echo "$job" | base64 -d) attr=$(echo "$job" | jq -r .attr) echo "### $attr" diff --git a/templates/flake-module.nix b/templates/flake-module.nix index 2074c12aa..d44882f36 100644 --- a/templates/flake-module.nix +++ b/templates/flake-module.nix @@ -1,8 +1,9 @@ -{ +{ self, ... }: { flake.templates = { new-clan = { description = "Initialize a new clan flake"; path = ./new-clan; }; + default = self.templates.new-clan; }; }