From d88c628f57d66ff937114a64ffd1b98186f96d64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 29 Aug 2023 22:03:57 +0200 Subject: [PATCH 01/14] buildClan: also allow machines without settings --- lib/build-clan/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 888eb6b25..297e94a79 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -15,7 +15,7 @@ let else { }; nixosConfigurations = lib.mapAttrs - (name: _mod: + (name: _: nixpkgs.lib.nixosSystem { modules = [ (machineSettings name) @@ -23,6 +23,6 @@ let ] ++ lib.attrValues clan.clanModules; specialArgs = specialArgs; }) - machinesDirs; + (machinesDirs // machines); in nixosConfigurations From 7c21453b17d8f62c45cd1c89d4eec5affd55d8f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 29 Aug 2023 22:27:13 +0200 Subject: [PATCH 02/14] lib.buildClan: fix flake reference --- lib/flake-module.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/flake-module.nix b/lib/flake-module.nix index 48c682e2f..517e96d49 100644 --- a/lib/flake-module.nix +++ b/lib/flake-module.nix @@ -1,4 +1,5 @@ { lib +, self , inputs , ... }: { @@ -6,7 +7,8 @@ ./jsonschema/flake-module.nix ]; flake.lib = import ./default.nix { + clan = self; inherit lib; - inherit (inputs) clan nixpkgs; + inherit (inputs) nixpkgs; }; } From 72768573db94591d00798b08383d27eed40e09e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 29 Aug 2023 22:29:32 +0200 Subject: [PATCH 03/14] drop non-existent clan modules --- lib/build-clan/default.nix | 4 ++-- lib/default.nix | 4 ++-- lib/flake-module.nix | 2 -- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 297e94a79..fc6c465f2 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -1,4 +1,4 @@ -{ nixpkgs, clan, lib }: +{ nixpkgs, lib }: { directory # The directory containing the machines subdirectory , specialArgs ? { } # Extra arguments to pass to nixosSystem i.e. useful to make self available , machines ? { } # allows to include machine-specific modules i.e. machines.${name} = { ... } @@ -20,7 +20,7 @@ let modules = [ (machineSettings name) (machines.${name} or { }) - ] ++ lib.attrValues clan.clanModules; + ]; specialArgs = specialArgs; }) (machinesDirs // machines); diff --git a/lib/default.nix b/lib/default.nix index 89ac0cc3f..066e88eff 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -1,4 +1,4 @@ -{ lib, clan, nixpkgs, ... }: +{ lib, nixpkgs, ... }: { findNixFiles = folder: lib.mapAttrs' @@ -14,5 +14,5 @@ jsonschema = import ./jsonschema { inherit lib; }; - buildClan = import ./build-clan { inherit lib clan nixpkgs; }; + buildClan = import ./build-clan { inherit lib nixpkgs; }; } diff --git a/lib/flake-module.nix b/lib/flake-module.nix index 517e96d49..64369ec24 100644 --- a/lib/flake-module.nix +++ b/lib/flake-module.nix @@ -1,5 +1,4 @@ { lib -, self , inputs , ... }: { @@ -7,7 +6,6 @@ ./jsonschema/flake-module.nix ]; flake.lib = import ./default.nix { - clan = self; inherit lib; inherit (inputs) nixpkgs; }; From d655c0ceb53e0cbda1cec32cbdfa3d1959b869df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 30 Aug 2023 11:27:43 +0200 Subject: [PATCH 04/14] clan-cli/direnv: also watch default.nix --- pkgs/clan-cli/.envrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/clan-cli/.envrc b/pkgs/clan-cli/.envrc index 00f84d526..53d6aa325 100644 --- a/pkgs/clan-cli/.envrc +++ b/pkgs/clan-cli/.envrc @@ -4,7 +4,9 @@ source_up if type nix_direnv_watch_file &>/dev/null; then nix_direnv_watch_file flake-module.nix + nix_direnv_watch_file default.nix else direnv watch flake-module.nix + direnv watch default.nix fi use flake .#clan-cli --builders '' From 54d855a6cdbcd9f5e6b8c467d6e32cbfba572e8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 30 Aug 2023 11:28:07 +0200 Subject: [PATCH 05/14] clan-cli: Document how to run single-threaded --- pkgs/clan-cli/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkgs/clan-cli/README.md b/pkgs/clan-cli/README.md index 538916486..df9c50907 100644 --- a/pkgs/clan-cli/README.md +++ b/pkgs/clan-cli/README.md @@ -27,3 +27,18 @@ To start a local developement environment instead, use the `--dev` flag: ``` This will spawn two webserver, a python one to for the api and a nodejs one that rebuilds the ui on the fly. + +## Run locally single-threaded for debugging + +By default tests run in parallel using pytest-parallel. +pytest-parallel however breaks `breakpoint()`. To disable it, use this: + +```console +pytest --workers "" -s +``` + +You can also run a single test like this: + +```console +pytest --workers "" -s tests/test_secrets_cli.py::test_users +``` From af38408a3e55684b089bbd9c2d344d889f080220 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 30 Aug 2023 11:30:57 +0200 Subject: [PATCH 06/14] secret cli: add get command that returns the key of users/machines --- pkgs/clan-cli/clan_cli/secrets/machines.py | 21 ++++++++++++++++----- pkgs/clan-cli/clan_cli/secrets/users.py | 14 +++++++++++++- pkgs/clan-cli/tests/test_secrets_cli.py | 7 ++++++- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/machines.py b/pkgs/clan-cli/clan_cli/secrets/machines.py index 80a07e6d9..811e88367 100644 --- a/pkgs/clan-cli/clan_cli/secrets/machines.py +++ b/pkgs/clan-cli/clan_cli/secrets/machines.py @@ -3,11 +3,8 @@ import argparse from ..machines.types import machine_name_type, validate_hostname from . import secrets from .folders import list_objects, remove_object, sops_machines_folder -from .sops import write_key -from .types import ( - public_or_private_age_key_type, - secret_name_type, -) +from .sops import read_key, write_key +from .types import public_or_private_age_key_type, secret_name_type def add_machine(name: str, key: str, force: bool) -> None: @@ -18,6 +15,10 @@ def remove_machine(name: str) -> None: remove_object(sops_machines_folder(), name) +def get_machine(name: str) -> str: + return read_key(sops_machines_folder() / name) + + def list_machines() -> list[str]: return list_objects(sops_machines_folder(), lambda x: validate_hostname(x)) @@ -42,6 +43,10 @@ def add_command(args: argparse.Namespace) -> None: add_machine(args.machine, args.key, args.force) +def get_command(args: argparse.Namespace) -> None: + print(get_machine(args.machine)) + + def remove_command(args: argparse.Namespace) -> None: remove_machine(args.machine) @@ -82,6 +87,12 @@ def register_machines_parser(parser: argparse.ArgumentParser) -> None: ) add_parser.set_defaults(func=add_command) + get_parser = subparser.add_parser("get", help="get a machine public key") + get_parser.add_argument( + "machine", help="the name of the machine", type=machine_name_type + ) + get_parser.set_defaults(func=get_command) + remove_parser = subparser.add_parser("remove", help="remove a machine") remove_parser.add_argument( "machine", help="the name of the machine", type=machine_name_type diff --git a/pkgs/clan-cli/clan_cli/secrets/users.py b/pkgs/clan-cli/clan_cli/secrets/users.py index 25cf28ae2..760218af8 100644 --- a/pkgs/clan-cli/clan_cli/secrets/users.py +++ b/pkgs/clan-cli/clan_cli/secrets/users.py @@ -2,7 +2,7 @@ import argparse from . import secrets from .folders import list_objects, remove_object, sops_users_folder -from .sops import write_key +from .sops import read_key, write_key from .types import ( VALID_SECRET_NAME, public_or_private_age_key_type, @@ -19,6 +19,10 @@ def remove_user(name: str) -> None: remove_object(sops_users_folder(), name) +def get_user(name: str) -> str: + return read_key(sops_users_folder() / name) + + def list_users() -> list[str]: return list_objects( sops_users_folder(), lambda n: VALID_SECRET_NAME.match(n) is not None @@ -43,6 +47,10 @@ def add_command(args: argparse.Namespace) -> None: add_user(args.user, args.key, args.force) +def get_command(args: argparse.Namespace) -> None: + print(get_user(args.user)) + + def remove_command(args: argparse.Namespace) -> None: remove_user(args.user) @@ -77,6 +85,10 @@ def register_users_parser(parser: argparse.ArgumentParser) -> None: ) add_parser.set_defaults(func=add_command) + get_parser = subparser.add_parser("get", help="get a user public key") + get_parser.add_argument("user", help="the name of the user", type=user_name_type) + get_parser.set_defaults(func=get_command) + remove_parser = subparser.add_parser("remove", help="remove a user") remove_parser.add_argument("user", help="the name of the user", type=user_name_type) remove_parser.set_defaults(func=remove_command) diff --git a/pkgs/clan-cli/tests/test_secrets_cli.py b/pkgs/clan-cli/tests/test_secrets_cli.py index d614c7737..ccdffd799 100644 --- a/pkgs/clan-cli/tests/test_secrets_cli.py +++ b/pkgs/clan-cli/tests/test_secrets_cli.py @@ -36,8 +36,13 @@ def _test_identities( age_keys[0].privkey, ] ) - capsys.readouterr() # empty the buffer + capsys.readouterr() # empty the buffer + cli.run(["secrets", what, "get", "foo"]) + out = capsys.readouterr() # empty the buffer + assert age_keys[0].pubkey in out.out + + capsys.readouterr() # empty the buffer cli.run(["secrets", what, "list"]) out = capsys.readouterr() # empty the buffer assert "foo" in out.out From e5db26d4b624eb11352f855f7aab7e42e7526a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 30 Aug 2023 13:28:21 +0200 Subject: [PATCH 07/14] set default template --- templates/flake-module.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/flake-module.nix b/templates/flake-module.nix index 2074c12aa..d44882f36 100644 --- a/templates/flake-module.nix +++ b/templates/flake-module.nix @@ -1,8 +1,9 @@ -{ +{ self, ... }: { flake.templates = { new-clan = { description = "Initialize a new clan flake"; path = ./new-clan; }; + default = self.templates.new-clan; }; } From 8c9c3173fa2dd7cf887be2492aae40763450f536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 30 Aug 2023 13:51:08 +0200 Subject: [PATCH 08/14] implement quickstart --- docs/quickstart.md | 61 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 docs/quickstart.md diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 000000000..950e6783c --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,61 @@ +# Initializing a New Clan Project + +## Clone the Clan Template + +To start a new project, execute the following command to clone the Clan Core template: + +```bash +$ nix flake init -t git+https://git.clan.lol/clan/clan-core +``` + +This action will generate two primary files: `flake.nix` and `.clan-flake`. + +```bash +$ ls -la +drwx------ joerg users 5 B a minute ago ./ +drwxrwxrwt root root 139 B 12 seconds ago ../ +.rw-r--r-- joerg users 77 B a minute ago .clan-flake +.rw-r--r-- joerg users 4.8 KB a minute ago flake.lock +.rw-r--r-- joerg users 242 B a minute ago flake.nix +``` + +### Understanding the .clan-flake Marker File + +The `.clan-flake` marker file serves an optional purpose: it helps the `clan-cli` utility locate the project's root directory. +If `.clan-flake` is missing, `clan-cli` will instead search for other indicators like `.git`, `.hg`, `.svn`, or `flake.nix` to identify the project root. + +--- + +# Migrating Existing NixOS Configuration Flake + +## Integrating with Existing NixOS Machines + +If you already manage NixOS machines using a flake, you can integrate them with the clan-core as shown in the example below: + +```nix +{ + description = "My custom NixOS flake"; + + inputs.clan-core.url = "git+https://git.clan.lol/clan/clan-core"; + + outputs = { clan-core, ... }: { + nixosConfigurations = clan-core.lib.buildClan { + directory = ./.; + machines = { + turingmachine = { + nixpkgs.pkgs = nixpkgs.legacyPackages.aarch64-linux; + imports = [ + ./configuration.nix + ]; + }; + }; + }; + }; +} +``` + +In this configuration: + +- `description`: Provides a brief description of the flake. +- `inputs.clan-core.url`: Specifies the Clan Core template's repository URL. +- `nixosConfigurations`: Defines NixOS configurations, using Clan Core’s `buildClan` function to manage the machines. From a052c414699bd7be83720086cfa69e2707657477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 30 Aug 2023 13:57:55 +0200 Subject: [PATCH 09/14] improve ci check --- flake.nix | 1 + scripts/ci | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index cf9d45ab9..79119ed36 100644 --- a/flake.nix +++ b/flake.nix @@ -24,6 +24,7 @@ "x86_64-linux" "aarch64-linux" ]; + flake.clanModules = { }; imports = [ ./checks/flake-module.nix ./devShell.nix diff --git a/scripts/ci b/scripts/ci index c39be836b..85e766546 100755 --- a/scripts/ci +++ b/scripts/ci @@ -26,7 +26,9 @@ fi rc=0 -for job in $(nix shell --inputs-from '.#' "nixpkgs#nix-eval-jobs" -c nix-eval-jobs "${args[@]}" | jq -r '. | @base64'); do +nix shell --inputs-from '.#' "nixpkgs#nix-eval-jobs" -c nix-eval-jobs "${args[@]}" > "jobs.json" + +for job in $(jq -r '. | @base64' < "jobs.json"); do job=$(echo "$job" | base64 -d) attr=$(echo "$job" | jq -r .attr) echo "### $attr" From 1cd037edd533b5ff0d5c7c072beebfc53d209b01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 30 Aug 2023 14:03:58 +0200 Subject: [PATCH 10/14] quickstart: unify naming --- docs/quickstart.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/quickstart.md b/docs/quickstart.md index 950e6783c..376f5c0ea 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -30,7 +30,7 @@ If `.clan-flake` is missing, `clan-cli` will instead search for other indicators ## Integrating with Existing NixOS Machines -If you already manage NixOS machines using a flake, you can integrate them with the clan-core as shown in the example below: +If you already manage NixOS machines using a flake, you can integrate them with the Clan Core as shown in the example below: ```nix { @@ -58,4 +58,4 @@ In this configuration: - `description`: Provides a brief description of the flake. - `inputs.clan-core.url`: Specifies the Clan Core template's repository URL. -- `nixosConfigurations`: Defines NixOS configurations, using Clan Core’s `buildClan` function to manage the machines. +- `nixosConfigurations`: Defines NixOS configurations, using Clan Core's `buildClan` function to manage the machines. From fef796fa6e27036c4a7dfdb2a2aaec1cff18544f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 30 Aug 2023 14:46:01 +0200 Subject: [PATCH 11/14] rename clanCore to clan.core --- flake.nix | 2 +- flakeModules/clan-config.nix | 8 ++++---- .../{clanCore => core}/flake-module.nix | 4 ++-- .../{clanCore => core}/secrets/default.nix | 4 ++-- .../{clanCore => core}/secrets/sops.nix | 20 +++++++++---------- nixosModules/{clanCore => core}/zerotier.nix | 4 ++-- pkgs/clan-cli/clan_cli/secrets/generate.py | 2 +- 7 files changed, 22 insertions(+), 22 deletions(-) rename nixosModules/{clanCore => core}/flake-module.nix (91%) rename nixosModules/{clanCore => core}/secrets/default.nix (92%) rename nixosModules/{clanCore => core}/secrets/sops.nix (70%) rename nixosModules/{clanCore => core}/zerotier.nix (95%) diff --git a/flake.nix b/flake.nix index 79119ed36..4b908bddf 100644 --- a/flake.nix +++ b/flake.nix @@ -37,7 +37,7 @@ ./lib/flake-module.nix ./nixosModules/flake-module.nix - ./nixosModules/clanCore/flake-module.nix + ./nixosModules/core/flake-module.nix ]; }); } diff --git a/flakeModules/clan-config.nix b/flakeModules/clan-config.nix index 236d22592..2d1cd5c87 100644 --- a/flakeModules/clan-config.nix +++ b/flakeModules/clan-config.nix @@ -1,4 +1,4 @@ -{ ... } @ clanCore: { +{ ... } @ core: { flake.flakeModules.clan-config = { self, inputs, ... }: let @@ -29,12 +29,12 @@ perSystem = { pkgs, ... }: { devShells.clan-config = pkgs.mkShell { packages = [ - clanCore.config.flake.packages.${pkgs.system}.clan-cli + core.config.flake.packages.${pkgs.system}.clan-cli ]; shellHook = '' export CLAN_OPTIONS_FILE=$(nix eval --raw .#clanOptions) - export XDG_DATA_DIRS="${clanCore.config.flake.packages.${pkgs.system}.clan-cli}/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}" - export fish_complete_path="${clanCore.config.flake.packages.${pkgs.system}.clan-cli}/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}" + export XDG_DATA_DIRS="${core.config.flake.packages.${pkgs.system}.clan-cli}/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}" + export fish_complete_path="${core.config.flake.packages.${pkgs.system}.clan-cli}/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}" ''; }; }; diff --git a/nixosModules/clanCore/flake-module.nix b/nixosModules/core/flake-module.nix similarity index 91% rename from nixosModules/clanCore/flake-module.nix rename to nixosModules/core/flake-module.nix index da8fa0e63..55648bf64 100644 --- a/nixosModules/clanCore/flake-module.nix +++ b/nixosModules/core/flake-module.nix @@ -1,6 +1,6 @@ { self, inputs, lib, ... }: { - flake.nixosModules.clanCore = { pkgs, ... }: { - options.clanCore = { + flake.nixosModules.clan.core = { pkgs, ... }: { + options.clan.core = { clanDir = lib.mkOption { type = lib.types.str; description = '' diff --git a/nixosModules/clanCore/secrets/default.nix b/nixosModules/core/secrets/default.nix similarity index 92% rename from nixosModules/clanCore/secrets/default.nix rename to nixosModules/core/secrets/default.nix index f1128a327..1660de63c 100644 --- a/nixosModules/clanCore/secrets/default.nix +++ b/nixosModules/core/secrets/default.nix @@ -1,6 +1,6 @@ { config, lib, ... }: { - options.clanCore.secrets = lib.mkOption { + options.clan.core.secrets = lib.mkOption { type = lib.types.attrsOf (lib.types.submodule (secret: { options = { @@ -49,7 +49,7 @@ description = '' path to a fact which is generated by the generator ''; - default = "${config.clanCore.clanDir}/facts/${config.clanCore.machineName}/${fact.config._module.args.name}"; + default = "${config.clan.core.clanDir}/facts/${config.clan.core.machineName}/${fact.config._module.args.name}"; }; value = lib.mkOption { default = builtins.readFile fact.config.path; diff --git a/nixosModules/clanCore/secrets/sops.nix b/nixosModules/core/secrets/sops.nix similarity index 70% rename from nixosModules/clanCore/secrets/sops.nix rename to nixosModules/core/secrets/sops.nix index 7df0b31d2..cc0507d5d 100644 --- a/nixosModules/clanCore/secrets/sops.nix +++ b/nixosModules/core/secrets/sops.nix @@ -7,24 +7,24 @@ set -x # remove for prod PATH=$PATH:${lib.makeBinPath [ - config.clanCore.clanPkgs.clan-cli + config.clan.core.clanPkgs.clan-cli ]} # initialize secret store - if ! clan secrets machines list | grep -q ${config.clanCore.machineName}; then ( + if ! clan secrets machines list | grep -q ${config.clan.core.machineName}; then ( INITTMP=$(mktemp -d) trap 'rm -rf "$INITTMP"' EXIT ${pkgs.age}/bin/age-keygen -o "$INITTMP/secret" 2> "$INITTMP/public" PUBKEY=$(cat "$INITTMP/public" | sed 's/.*: //') - clan secrets machines add ${config.clanCore.machineName} "$PUBKEY" - tail -1 "$INITTMP/secret" | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-age.key + clan secrets machines add ${config.clan.core.machineName} "$PUBKEY" + tail -1 "$INITTMP/secret" | clan secrets set --machine ${config.clan.core.machineName} ${config.clan.core.machineName}-age.key ) fi ${lib.foldlAttrs (acc: n: v: '' ${acc} # ${n} # if any of the secrets are missing, we regenerate all connected facts/secrets - (if ! ${lib.concatMapStringsSep " && " (x: "clan secrets get ${config.clanCore.machineName}-${x.name} >/dev/null") (lib.attrValues v.secrets)}; then + (if ! ${lib.concatMapStringsSep " && " (x: "clan secrets get ${config.clan.core.machineName}-${x.name} >/dev/null") (lib.attrValues v.secrets)}; then facts=$(mktemp -d) trap "rm -rf $facts" EXIT @@ -38,24 +38,24 @@ '') (lib.attrValues v.facts)} ${lib.concatMapStrings (secret: '' - cat "$secrets"/${secret.name} | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-${secret.name} + cat "$secrets"/${secret.name} | clan secrets set --machine ${config.clan.core.machineName} ${config.clan.core.machineName}-${secret.name} '') (lib.attrValues v.secrets)} fi) - '') "" config.clanCore.secrets} + '') "" config.clan.core.secrets} ''; sops.secrets = let encryptedForThisMachine = name: type: let - symlink = config.clanCore.clanDir + "/sops/secrets/${name}/machines/${config.clanCore.machineName}"; + symlink = config.clan.core.clanDir + "/sops/secrets/${name}/machines/${config.clan.core.machineName}"; in # WTF, nix bug, my symlink is in the nixos module detected as a directory also it works in the repl type == "directory" && (builtins.readFileType symlink == "directory" || builtins.readFileType symlink == "symlink"); - secrets = lib.filterAttrs encryptedForThisMachine (builtins.readDir (config.clanCore.clanDir + "/sops/secrets")); + secrets = lib.filterAttrs encryptedForThisMachine (builtins.readDir (config.clan.core.clanDir + "/sops/secrets")); in builtins.mapAttrs (name: _: { - sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret"; + sopsFile = config.clan.core.clanDir + "/sops/secrets/${name}/secret"; format = "binary"; }) secrets; diff --git a/nixosModules/clanCore/zerotier.nix b/nixosModules/core/zerotier.nix similarity index 95% rename from nixosModules/clanCore/zerotier.nix rename to nixosModules/core/zerotier.nix index 452294d95..67b9b6fec 100644 --- a/nixosModules/clanCore/zerotier.nix +++ b/nixosModules/core/zerotier.nix @@ -41,13 +41,13 @@ in } // lib.mkIf cfg.controller.enable { # only the controller needs to have the key in the repo, the other clients can be dynamic # we generate the zerotier code manually for the controller, since it's part of the bootstrap command - clanCore.secrets.zerotier = { + clan.core.secrets.zerotier = { facts."network.id" = { }; secrets."identity.secret" = { }; generator = '' TMPDIR=$(mktemp -d) trap 'rm -rf "$TMPDIR"' EXIT - ${config.clanCore.clanPkgs.clan-cli}/bin/clan zerotier --outpath "$TMPDIR" + ${config.clan.core.clanPkgs.clan-cli}/bin/clan zerotier --outpath "$TMPDIR" cp "$TMPDIR"/network.id "$facts"/network.id cp "$TMPDIR"/identity.secret "$secrets"/identity.secret ''; diff --git a/pkgs/clan-cli/clan_cli/secrets/generate.py b/pkgs/clan-cli/clan_cli/secrets/generate.py index 0b01a8c85..01499db0a 100644 --- a/pkgs/clan-cli/clan_cli/secrets/generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/generate.py @@ -15,7 +15,7 @@ def get_secret_script(machine: str) -> None: "--expr", "let f = builtins.getFlake (toString ./.); in " f"(f.nixosConfigurations.{machine}.extendModules " - "{ modules = [{ clanCore.clanDir = toString ./.; }]; })" + "{ modules = [{ clan.core.clanDir = toString ./.; }]; })" ".config.system.clan.generateSecrets", ], check=True, From f01581f04cc9d8d2545be7927ca106e1cd98fed4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 30 Aug 2023 14:48:22 +0200 Subject: [PATCH 12/14] quickstart: change hostPlatform --- docs/quickstart.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quickstart.md b/docs/quickstart.md index 376f5c0ea..65dd68e85 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -43,7 +43,7 @@ If you already manage NixOS machines using a flake, you can integrate them with directory = ./.; machines = { turingmachine = { - nixpkgs.pkgs = nixpkgs.legacyPackages.aarch64-linux; + nixpkgs.hostPlatform = "x86_64-linux"; imports = [ ./configuration.nix ]; From bad7d624bdf3f3d2ff759a3db1bf988366c38fc2 Mon Sep 17 00:00:00 2001 From: Luis-Hebendanz Date: Wed, 30 Aug 2023 15:25:20 +0200 Subject: [PATCH 13/14] Improved Quickstart --- docs/quickstart.md | 101 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 76 insertions(+), 25 deletions(-) diff --git a/docs/quickstart.md b/docs/quickstart.md index 950e6783c..63cb208ca 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -28,34 +28,85 @@ If `.clan-flake` is missing, `clan-cli` will instead search for other indicators # Migrating Existing NixOS Configuration Flake -## Integrating with Existing NixOS Machines +Absolutely, let's break down the migration step by step, explaining each action in detail: -If you already manage NixOS machines using a flake, you can integrate them with the clan-core as shown in the example below: +#### Before You Begin -```nix -{ - description = "My custom NixOS flake"; +1. **Backup Your Current Configuration**: Always start by making a backup of your current NixOS configuration to ensure you can revert if needed. - inputs.clan-core.url = "git+https://git.clan.lol/clan/clan-core"; + ```shell + cp -r /etc/nixos ~/nixos-backup + ``` - outputs = { clan-core, ... }: { - nixosConfigurations = clan-core.lib.buildClan { - directory = ./.; - machines = { - turingmachine = { - nixpkgs.pkgs = nixpkgs.legacyPackages.aarch64-linux; - imports = [ - ./configuration.nix - ]; - }; - }; - }; - }; -} -``` +2. **Update Flake Inputs**: The patch adds a new input named `clan-core` to your `flake.nix`. This input points to a Git repository for Clan Core. Here's the addition: -In this configuration: + ```nix + inputs.clan-core = { + url = "git+https://git.clan.lol/clan/clan-core"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + ``` -- `description`: Provides a brief description of the flake. -- `inputs.clan-core.url`: Specifies the Clan Core template's repository URL. -- `nixosConfigurations`: Defines NixOS configurations, using Clan Core’s `buildClan` function to manage the machines. + - `url`: Specifies the Git repository URL for Clan Core. + - `inputs.nixpkgs.follows`: Tells Nix to use the same `nixpkgs` input as your main input (in this case, it follows `nixpkgs`). + +3. **Update Outputs**: Then modify the `outputs` section of your `flake.nix` to adapt to Clan Core's new provisioning method. The key changes are as follows: + + Add `clan-core` to the output + + ```diff + - outputs = { self, nixpkgs, }: + + outputs = { self, nixpkgs, clan-core }: + ``` + + Previous configuration: + + ```nix + nixosConfigurations.example-desktop = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ + ./configuration.nix + ]; + [...] + }; + ``` + + After change: + + ```nix + nixosConfigurations = clan-core.lib.buildClan { + directory = ./.; + machines = { + example-desktop = { + nixpkgs.hostPlatform = "x86_64-linux"; + imports = [ + ./configuration.nix + ]; + }; + }; + }; + ``` + + - `nixosConfigurations`: Defines NixOS configurations, using Clan Core’s `buildClan` function to manage the machines. + - Inside `machines`, a new machine configuration is defined (in this case, `example-desktop`). + - Inside `example-desktop` which is the target machine hostname, `nixpkgs.hostPlatform` specifies the host platform as `x86_64-linux`. + +4. **Rebuild and Switch**: Rebuild your NixOS configuration using the updated flake: + + ```shell + sudo nixos-rebuild switch --flake . + ``` + + - This command rebuilds and switches to the new configuration. Make sure to include the `--flake .` argument to use the current directory as the flake source. + +5. **Test Configuration**: Before rebooting, verify that your new configuration builds without errors or warnings. + +6. **Reboot**: If everything is fine, you can reboot your system to apply the changes: + + ```shell + sudo reboot + ``` + +7. **Verify**: After the reboot, confirm that your system is running with the new configuration, and all services and applications are functioning as expected. + +By following these steps, you've successfully migrated your NixOS Flake configuration to include the `clan-core` input and adapted the `outputs` section to work with Clan Core's new machine provisioning method. From d16bd000da8fcb4c2015f9020630a7b882d3d8f0 Mon Sep 17 00:00:00 2001 From: Luis-Hebendanz Date: Wed, 30 Aug 2023 15:28:24 +0200 Subject: [PATCH 14/14] Improved quickstart.me --- flake.nix | 2 +- flakeModules/clan-config.nix | 8 ++++---- .../{clanCore => core}/flake-module.nix | 4 ++-- .../{clanCore => core}/secrets/default.nix | 4 ++-- .../{clanCore => core}/secrets/sops.nix | 20 +++++++++---------- nixosModules/{clanCore => core}/zerotier.nix | 4 ++-- pkgs/clan-cli/clan_cli/secrets/generate.py | 2 +- 7 files changed, 22 insertions(+), 22 deletions(-) rename nixosModules/{clanCore => core}/flake-module.nix (91%) rename nixosModules/{clanCore => core}/secrets/default.nix (92%) rename nixosModules/{clanCore => core}/secrets/sops.nix (70%) rename nixosModules/{clanCore => core}/zerotier.nix (95%) diff --git a/flake.nix b/flake.nix index 79119ed36..4b908bddf 100644 --- a/flake.nix +++ b/flake.nix @@ -37,7 +37,7 @@ ./lib/flake-module.nix ./nixosModules/flake-module.nix - ./nixosModules/clanCore/flake-module.nix + ./nixosModules/core/flake-module.nix ]; }); } diff --git a/flakeModules/clan-config.nix b/flakeModules/clan-config.nix index 236d22592..2d1cd5c87 100644 --- a/flakeModules/clan-config.nix +++ b/flakeModules/clan-config.nix @@ -1,4 +1,4 @@ -{ ... } @ clanCore: { +{ ... } @ core: { flake.flakeModules.clan-config = { self, inputs, ... }: let @@ -29,12 +29,12 @@ perSystem = { pkgs, ... }: { devShells.clan-config = pkgs.mkShell { packages = [ - clanCore.config.flake.packages.${pkgs.system}.clan-cli + core.config.flake.packages.${pkgs.system}.clan-cli ]; shellHook = '' export CLAN_OPTIONS_FILE=$(nix eval --raw .#clanOptions) - export XDG_DATA_DIRS="${clanCore.config.flake.packages.${pkgs.system}.clan-cli}/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}" - export fish_complete_path="${clanCore.config.flake.packages.${pkgs.system}.clan-cli}/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}" + export XDG_DATA_DIRS="${core.config.flake.packages.${pkgs.system}.clan-cli}/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}" + export fish_complete_path="${core.config.flake.packages.${pkgs.system}.clan-cli}/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}" ''; }; }; diff --git a/nixosModules/clanCore/flake-module.nix b/nixosModules/core/flake-module.nix similarity index 91% rename from nixosModules/clanCore/flake-module.nix rename to nixosModules/core/flake-module.nix index da8fa0e63..55648bf64 100644 --- a/nixosModules/clanCore/flake-module.nix +++ b/nixosModules/core/flake-module.nix @@ -1,6 +1,6 @@ { self, inputs, lib, ... }: { - flake.nixosModules.clanCore = { pkgs, ... }: { - options.clanCore = { + flake.nixosModules.clan.core = { pkgs, ... }: { + options.clan.core = { clanDir = lib.mkOption { type = lib.types.str; description = '' diff --git a/nixosModules/clanCore/secrets/default.nix b/nixosModules/core/secrets/default.nix similarity index 92% rename from nixosModules/clanCore/secrets/default.nix rename to nixosModules/core/secrets/default.nix index f1128a327..1660de63c 100644 --- a/nixosModules/clanCore/secrets/default.nix +++ b/nixosModules/core/secrets/default.nix @@ -1,6 +1,6 @@ { config, lib, ... }: { - options.clanCore.secrets = lib.mkOption { + options.clan.core.secrets = lib.mkOption { type = lib.types.attrsOf (lib.types.submodule (secret: { options = { @@ -49,7 +49,7 @@ description = '' path to a fact which is generated by the generator ''; - default = "${config.clanCore.clanDir}/facts/${config.clanCore.machineName}/${fact.config._module.args.name}"; + default = "${config.clan.core.clanDir}/facts/${config.clan.core.machineName}/${fact.config._module.args.name}"; }; value = lib.mkOption { default = builtins.readFile fact.config.path; diff --git a/nixosModules/clanCore/secrets/sops.nix b/nixosModules/core/secrets/sops.nix similarity index 70% rename from nixosModules/clanCore/secrets/sops.nix rename to nixosModules/core/secrets/sops.nix index 7df0b31d2..cc0507d5d 100644 --- a/nixosModules/clanCore/secrets/sops.nix +++ b/nixosModules/core/secrets/sops.nix @@ -7,24 +7,24 @@ set -x # remove for prod PATH=$PATH:${lib.makeBinPath [ - config.clanCore.clanPkgs.clan-cli + config.clan.core.clanPkgs.clan-cli ]} # initialize secret store - if ! clan secrets machines list | grep -q ${config.clanCore.machineName}; then ( + if ! clan secrets machines list | grep -q ${config.clan.core.machineName}; then ( INITTMP=$(mktemp -d) trap 'rm -rf "$INITTMP"' EXIT ${pkgs.age}/bin/age-keygen -o "$INITTMP/secret" 2> "$INITTMP/public" PUBKEY=$(cat "$INITTMP/public" | sed 's/.*: //') - clan secrets machines add ${config.clanCore.machineName} "$PUBKEY" - tail -1 "$INITTMP/secret" | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-age.key + clan secrets machines add ${config.clan.core.machineName} "$PUBKEY" + tail -1 "$INITTMP/secret" | clan secrets set --machine ${config.clan.core.machineName} ${config.clan.core.machineName}-age.key ) fi ${lib.foldlAttrs (acc: n: v: '' ${acc} # ${n} # if any of the secrets are missing, we regenerate all connected facts/secrets - (if ! ${lib.concatMapStringsSep " && " (x: "clan secrets get ${config.clanCore.machineName}-${x.name} >/dev/null") (lib.attrValues v.secrets)}; then + (if ! ${lib.concatMapStringsSep " && " (x: "clan secrets get ${config.clan.core.machineName}-${x.name} >/dev/null") (lib.attrValues v.secrets)}; then facts=$(mktemp -d) trap "rm -rf $facts" EXIT @@ -38,24 +38,24 @@ '') (lib.attrValues v.facts)} ${lib.concatMapStrings (secret: '' - cat "$secrets"/${secret.name} | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-${secret.name} + cat "$secrets"/${secret.name} | clan secrets set --machine ${config.clan.core.machineName} ${config.clan.core.machineName}-${secret.name} '') (lib.attrValues v.secrets)} fi) - '') "" config.clanCore.secrets} + '') "" config.clan.core.secrets} ''; sops.secrets = let encryptedForThisMachine = name: type: let - symlink = config.clanCore.clanDir + "/sops/secrets/${name}/machines/${config.clanCore.machineName}"; + symlink = config.clan.core.clanDir + "/sops/secrets/${name}/machines/${config.clan.core.machineName}"; in # WTF, nix bug, my symlink is in the nixos module detected as a directory also it works in the repl type == "directory" && (builtins.readFileType symlink == "directory" || builtins.readFileType symlink == "symlink"); - secrets = lib.filterAttrs encryptedForThisMachine (builtins.readDir (config.clanCore.clanDir + "/sops/secrets")); + secrets = lib.filterAttrs encryptedForThisMachine (builtins.readDir (config.clan.core.clanDir + "/sops/secrets")); in builtins.mapAttrs (name: _: { - sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret"; + sopsFile = config.clan.core.clanDir + "/sops/secrets/${name}/secret"; format = "binary"; }) secrets; diff --git a/nixosModules/clanCore/zerotier.nix b/nixosModules/core/zerotier.nix similarity index 95% rename from nixosModules/clanCore/zerotier.nix rename to nixosModules/core/zerotier.nix index 452294d95..67b9b6fec 100644 --- a/nixosModules/clanCore/zerotier.nix +++ b/nixosModules/core/zerotier.nix @@ -41,13 +41,13 @@ in } // lib.mkIf cfg.controller.enable { # only the controller needs to have the key in the repo, the other clients can be dynamic # we generate the zerotier code manually for the controller, since it's part of the bootstrap command - clanCore.secrets.zerotier = { + clan.core.secrets.zerotier = { facts."network.id" = { }; secrets."identity.secret" = { }; generator = '' TMPDIR=$(mktemp -d) trap 'rm -rf "$TMPDIR"' EXIT - ${config.clanCore.clanPkgs.clan-cli}/bin/clan zerotier --outpath "$TMPDIR" + ${config.clan.core.clanPkgs.clan-cli}/bin/clan zerotier --outpath "$TMPDIR" cp "$TMPDIR"/network.id "$facts"/network.id cp "$TMPDIR"/identity.secret "$secrets"/identity.secret ''; diff --git a/pkgs/clan-cli/clan_cli/secrets/generate.py b/pkgs/clan-cli/clan_cli/secrets/generate.py index 0b01a8c85..01499db0a 100644 --- a/pkgs/clan-cli/clan_cli/secrets/generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/generate.py @@ -15,7 +15,7 @@ def get_secret_script(machine: str) -> None: "--expr", "let f = builtins.getFlake (toString ./.); in " f"(f.nixosConfigurations.{machine}.extendModules " - "{ modules = [{ clanCore.clanDir = toString ./.; }]; })" + "{ modules = [{ clan.core.clanDir = toString ./.; }]; })" ".config.system.clan.generateSecrets", ], check=True,