From 5268ecb595ab5c5132ee54baa940553e597df801 Mon Sep 17 00:00:00 2001 From: DavHau Date: Wed, 2 Aug 2023 18:44:36 +0200 Subject: [PATCH 1/2] flake: add input nix-unit --- flake.lock | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 4 ++++ 2 files changed, 53 insertions(+) diff --git a/flake.lock b/flake.lock index 62baf34a5..1196be246 100644 --- a/flake.lock +++ b/flake.lock @@ -95,6 +95,54 @@ "type": "github" } }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "nix-unit", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1688870561, + "narHash": "sha256-4UYkifnPEw1nAzqqPOTL2MvWtm3sNGw1UTYTalkTcGY=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "165b1650b753316aa7f1787f3005a8d2da0f5301", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, + "nix-unit": { + "inputs": { + "flake-parts": [ + "flake-parts" + ], + "nix-github-actions": "nix-github-actions", + "nixpkgs": [ + "nixpkgs" + ], + "treefmt-nix": [ + "treefmt-nix" + ] + }, + "locked": { + "lastModified": 1690289081, + "narHash": "sha256-PCXQAQt8+i2pkUym9P1JY4JGoeZJLzzxWBhprHDdItM=", + "owner": "adisbladis", + "repo": "nix-unit", + "rev": "a9d6f33e50d4dcd9cfc0c92253340437bbae282b", + "type": "github" + }, + "original": { + "owner": "adisbladis", + "repo": "nix-unit", + "type": "github" + } + }, "nixlib": { "locked": { "lastModified": 1689469483, @@ -205,6 +253,7 @@ "inputs": { "disko": "disko", "flake-parts": "flake-parts", + "nix-unit": "nix-unit", "nixos-generators": "nixos-generators", "nixpkgs": "nixpkgs", "pre-commit-hooks-nix": "pre-commit-hooks-nix", diff --git a/flake.nix b/flake.nix index 8b275868e..e509386eb 100644 --- a/flake.nix +++ b/flake.nix @@ -12,6 +12,10 @@ treefmt-nix.url = "github:numtide/treefmt-nix"; treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; pre-commit-hooks-nix.url = "github:cachix/pre-commit-hooks.nix"; + nix-unit.url = "github:adisbladis/nix-unit"; + nix-unit.inputs.flake-parts.follows = "flake-parts"; + nix-unit.inputs.nixpkgs.follows = "nixpkgs"; + nix-unit.inputs.treefmt-nix.follows = "treefmt-nix"; }; outputs = inputs @ { flake-parts, ... }: From b88ac7a2bfc7c50db348ab134f065f584a62e1dd Mon Sep 17 00:00:00 2001 From: DavHau Date: Wed, 2 Aug 2023 20:04:16 +0200 Subject: [PATCH 2/2] clan-config: init - nixos-modules to jsonschema converter - nix unit testing via adisbladis/nix-unit - clan config: configuration CLI for nixos-modules --- pkgs/clan-cli/clan_cli/admin.py | 19 -- pkgs/clan-cli/clan_cli/cli.py | 5 +- pkgs/clan-cli/clan_cli/config/__init__.py | 99 +++++++ pkgs/clan-cli/clan_cli/config/schema-lib.nix | 144 ++++++++++ pkgs/clan-cli/flake-module.nix | 32 ++- pkgs/clan-cli/shell.nix | 1 + pkgs/clan-cli/tests/config/example-data.json | 17 ++ .../tests/config/example-interface.nix | 46 ++++ .../clan-cli/tests/config/example-schema.json | 54 ++++ pkgs/clan-cli/tests/config/test.nix | 8 + .../tests/config/test_parseOption.nix | 249 ++++++++++++++++++ .../tests/config/test_parseOptions.nix | 20 ++ 12 files changed, 668 insertions(+), 26 deletions(-) create mode 100644 pkgs/clan-cli/clan_cli/config/__init__.py create mode 100644 pkgs/clan-cli/clan_cli/config/schema-lib.nix create mode 100644 pkgs/clan-cli/tests/config/example-data.json create mode 100644 pkgs/clan-cli/tests/config/example-interface.nix create mode 100644 pkgs/clan-cli/tests/config/example-schema.json create mode 100644 pkgs/clan-cli/tests/config/test.nix create mode 100644 pkgs/clan-cli/tests/config/test_parseOption.nix create mode 100644 pkgs/clan-cli/tests/config/test_parseOptions.nix diff --git a/pkgs/clan-cli/clan_cli/admin.py b/pkgs/clan-cli/clan_cli/admin.py index cf40532be..2197b5235 100644 --- a/pkgs/clan-cli/clan_cli/admin.py +++ b/pkgs/clan-cli/clan_cli/admin.py @@ -18,22 +18,6 @@ def create(args: argparse.Namespace) -> None: ) -def edit(args: argparse.Namespace) -> None: - # TODO add some cli options to change certain options without relying on a text editor - clan_flake = f"{args.folder}/flake.nix" - if os.path.isfile(clan_flake): - subprocess.Popen( - [ - os.environ["EDITOR"], - clan_flake, - ] - ) - else: - print( - f"{args.folder} has no flake.nix, so it does not seem to be the clan root folder", - ) - - def rebuild(args: argparse.Namespace) -> None: # TODO get clients from zerotier cli? if args.host: @@ -89,9 +73,6 @@ def register_parser(parser: argparse.ArgumentParser) -> None: parser_create = subparser.add_parser("create", help="create a new clan") parser_create.set_defaults(func=create) - parser_edit = subparser.add_parser("edit", help="edit a clan") - parser_edit.set_defaults(func=edit) - parser_rebuild = subparser.add_parser( "rebuild", help="build configuration of a clan and push it to the target" ) diff --git a/pkgs/clan-cli/clan_cli/cli.py b/pkgs/clan-cli/clan_cli/cli.py index 1319a3d32..759b54869 100644 --- a/pkgs/clan-cli/clan_cli/cli.py +++ b/pkgs/clan-cli/clan_cli/cli.py @@ -1,7 +1,7 @@ import argparse import sys -from . import admin, secrets, ssh +from . import admin, config, secrets, ssh from .errors import ClanError has_argcomplete = True @@ -19,6 +19,9 @@ def main() -> None: parser_admin = subparsers.add_parser("admin") admin.register_parser(parser_admin) + parser_config = subparsers.add_parser("config") + config.register_parser(parser_config) + parser_ssh = subparsers.add_parser("ssh", help="ssh to a remote machine") ssh.register_parser(parser_ssh) diff --git a/pkgs/clan-cli/clan_cli/config/__init__.py b/pkgs/clan-cli/clan_cli/config/__init__.py new file mode 100644 index 000000000..e1e652c90 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/config/__init__.py @@ -0,0 +1,99 @@ +# !/usr/bin/env python3 +import argparse +import json +from pathlib import Path +from typing import Any, Optional, Union + + +class Kwargs: + def __init__(self): + self.type = None + self.default: Any = None + self.required: bool = False + self.help: Optional[str] = None + self.action: Optional[str] = None + self.choices: Optional[list] = None + + +# takes a (sub)parser and configures it +def register_parser( + parser: Optional[argparse.ArgumentParser] = None, + schema: Union[dict, str, Path] = "./tests/config/example-schema.json", +) -> dict: + if not isinstance(schema, dict): + with open(str(schema)) as f: + schema: dict = json.load(f) + assert "type" in schema and schema["type"] == "object" + + required_set = set(schema.get("required", [])) + + if parser is None: + parser = argparse.ArgumentParser(description=schema.get("description")) + + type_map = { + "array": list, + "boolean": bool, + "integer": int, + "number": float, + "string": str, + } + + subparsers = parser.add_subparsers( + title="more options", + description="Other options to configure", + help="the option to configure", + required=True, + ) + + for name, value in schema.get("properties", {}).items(): + assert isinstance(value, dict) + + # TODO: add support for nested objects + if value.get("type") == "object": + subparser = subparsers.add_parser(name, help=value.get("description")) + register_parser(parser=subparser, schema=value) + continue + # elif value.get("type") == "array": + # subparser = parser.add_subparsers(dest=name) + # register_parser(subparser, value) + # continue + kwargs = Kwargs() + kwargs.default = value.get("default") + kwargs.help = value.get("description") + kwargs.required = name in required_set + + if kwargs.default is not None: + kwargs.help = f"{kwargs.help}, [{kwargs.default}] in default" + + if "enum" in value: + enum_list = value["enum"] + assert len(enum_list) > 0, "Enum List is Empty" + arg_type = type(enum_list[0]) + assert all( + arg_type is type(item) for item in enum_list + ), f"Items in [{enum_list}] with Different Types" + + kwargs.type = arg_type + kwargs.choices = enum_list + else: + kwargs.type = type_map[value.get("type")] + del kwargs.choices + + name = f"--{name}" + + if kwargs.type is bool: + assert not kwargs.default, "boolean have to be False in default" + kwargs.default = False + kwargs.action = "store_true" + del kwargs.type + else: + del kwargs.action + + parser.add_argument(name, **vars(kwargs)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + register_parser(parser) + args = parser.parse_args() + print(args) diff --git a/pkgs/clan-cli/clan_cli/config/schema-lib.nix b/pkgs/clan-cli/clan_cli/config/schema-lib.nix new file mode 100644 index 000000000..d0f2daa4e --- /dev/null +++ b/pkgs/clan-cli/clan_cli/config/schema-lib.nix @@ -0,0 +1,144 @@ +{ lib ? (import { }).lib }: +let + + # from nixos type to jsonschema type + typeMap = { + bool = "boolean"; + float = "number"; + int = "integer"; + str = "string"; + }; + + # remove _module attribute from options + clean = opts: builtins.removeAttrs opts [ "_module" ]; + + # throw error if option type is not supported + notSupported = option: throw + "option type '${option.type.description}' not supported by jsonschema converter"; + +in +rec { + + # parses a set of evaluated nixos options to a jsonschema + parseOptions = options': + let + options = clean options'; + # parse options to jsonschema properties + properties = lib.mapAttrs (_name: option: parseOption option) options; + isRequired = prop: ! (prop ? default || prop.type == "object"); + requiredProps = lib.filterAttrs (_: prop: isRequired prop) properties; + required = lib.optionalAttrs (requiredProps != { }) { + required = lib.attrNames requiredProps; + }; + in + # return jsonschema + required // { + type = "object"; + inherit properties; + }; + + # parses and evaluated nixos option to a jsonschema property definition + parseOption = option: + let + default = lib.optionalAttrs (option ? default) { + inherit (option) default; + }; + description = lib.optionalAttrs (option ? description) { + inherit (option) description; + }; + in + if option._type != "option" + then throw "parseOption: not an option" + + # parse nullOr + else if option.type.name == "nullOr" + # return jsonschema property definition for nullOr + then default // description // { + type = [ + "null" + (typeMap.${option.type.functor.wrapped.name} or (notSupported option)) + ]; + } + + # parse bool + else if option.type.name == "bool" + # return jsonschema property definition for bool + then default // description // { + type = "boolean"; + } + + # parse float + else if option.type.name == "float" + # return jsonschema property definition for float + then default // description // { + type = "number"; + } + + # parse int + else if option.type.name == "int" + # return jsonschema property definition for int + then default // description // { + type = "integer"; + } + + # parse string + else if option.type.name == "str" + # return jsonschema property definition for string + then default // description // { + type = "string"; + } + + # parse enum + else if option.type.name == "enum" + # return jsonschema property definition for enum + then default // description // { + enum = option.type.functor.payload; + } + + # parse listOf submodule + else if option.type.name == "listOf" && option.type.functor.wrapped.name == "submodule" + # return jsonschema property definition for listOf submodule + then default // description // { + type = "array"; + items = parseOptions (option.type.functor.wrapped.getSubOptions option.loc); + } + + # parse list + else if + (option.type.name == "listOf") + && (typeMap ? "${option.type.functor.wrapped.name}") + # return jsonschema property definition for list + then default // description // { + type = "array"; + items = { + type = typeMap.${option.type.functor.wrapped.name}; + }; + } + + # parse attrsOf submodule + else if option.type.name == "attrsOf" && option.type.nestedTypes.elemType.name == "submodule" + # return jsonschema property definition for attrsOf submodule + then default // description // { + type = "object"; + additionalProperties = parseOptions (option.type.nestedTypes.elemType.getSubOptions option.loc); + } + + # parse attrs + else if option.type.name == "attrsOf" + # return jsonschema property definition for attrs + then default // description // { + type = "object"; + additionalProperties = { + type = typeMap.${option.type.nestedTypes.elemType.name} or (notSupported option); + }; + } + + # parse submodule + else if option.type.name == "submodule" + # return jsonschema property definition for submodule + # then (lib.attrNames (option.type.getSubOptions option.loc).opt) + then parseOptions (option.type.getSubOptions option.loc) + + # throw error if option type is not supported + else notSupported option; +} diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index a617bf145..272a5e2c5 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -1,5 +1,5 @@ { self, ... }: { - perSystem = { self', pkgs, ... }: { + perSystem = { inputs', self', pkgs, ... }: { devShells.clan = pkgs.callPackage ./shell.nix { inherit self; inherit (self'.packages) clan; @@ -18,14 +18,34 @@ openssh sshpass zbar - tor - sops - age; - # Override license so that we can build zerotierone without + tor; + # Override license so that we can build zerotierone without # having to re-import nixpkgs. zerotierone = pkgs.zerotierone.overrideAttrs (_old: { meta = { }; }); ## End optional dependencies }; - checks = self'.packages.clan.tests; + + # check if the `clan config` example jsonschema and data is valid + checks.clan-config-example-schema-valid = pkgs.runCommand "clan-config-example-schema-valid" { } '' + echo "Checking that example-schema.json is valid" + ${pkgs.check-jsonschema}/bin/check-jsonschema \ + --check-metaschema ${./.}/tests/config/example-schema.json + + echo "Checking that example-data.json is valid according to example-schema.json" + ${pkgs.check-jsonschema}/bin/check-jsonschema \ + --schemafile ${./.}/tests/config/example-schema.json \ + ${./.}/tests/config/example-data.json + + touch $out + ''; + + # check if the `clan config` nix jsonschema converter unit tests succeed + checks.clan-config-nix-unit-tests = pkgs.runCommand "clan-edit-unit-tests" { } '' + export NIX_PATH=nixpkgs=${pkgs.path} + ${inputs'.nix-unit.packages.nix-unit}/bin/nix-unit \ + ${./.}/tests/config/test.nix \ + --eval-store $(realpath .) + touch $out + ''; }; } diff --git a/pkgs/clan-cli/shell.nix b/pkgs/clan-cli/shell.nix index c5aeba33d..fb7735c96 100644 --- a/pkgs/clan-cli/shell.nix +++ b/pkgs/clan-cli/shell.nix @@ -15,6 +15,7 @@ in pkgs.mkShell { packages = [ pkgs.ruff + self.inputs.nix-unit.packages.${pkgs.system}.nix-unit pythonWithDeps ]; # sets up an editable install and add enty points to $PATH diff --git a/pkgs/clan-cli/tests/config/example-data.json b/pkgs/clan-cli/tests/config/example-data.json new file mode 100644 index 000000000..a703d539d --- /dev/null +++ b/pkgs/clan-cli/tests/config/example-data.json @@ -0,0 +1,17 @@ +{ + "name": "John Doe", + "age": 42, + "isAdmin": false, + "kernelModules": [ + "usbhid", + "usb_storage" + ], + "userIds": { + "mic92": 1, + "lassulus": 2, + "davhau": 3 + }, + "services": { + "opt": "this option doesn't make sense" + } +} diff --git a/pkgs/clan-cli/tests/config/example-interface.nix b/pkgs/clan-cli/tests/config/example-interface.nix new file mode 100644 index 000000000..1370d3cd9 --- /dev/null +++ b/pkgs/clan-cli/tests/config/example-interface.nix @@ -0,0 +1,46 @@ +/* + An example nixos module declaring an interface. +*/ +{ lib, ... }: { + options = { + name = lib.mkOption { + type = lib.types.str; + default = "John Doe"; + description = "The name of the user"; + }; + age = lib.mkOption { + type = lib.types.int; + default = 42; + description = "The age of the user"; + }; + isAdmin = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Is the user an admin?"; + }; + # a submodule option + services = lib.mkOption { + type = lib.types.submodule { + options.opt = lib.mkOption { + type = lib.types.str; + default = "foo"; + description = "A submodule option"; + }; + }; + }; + userIds = lib.mkOption { + type = lib.types.attrsOf lib.types.int; + description = "Some attributes"; + default = { + horst = 1; + peter = 2; + albrecht = 3; + }; + }; + kernelModules = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ "nvme" "xhci_pci" "ahci" ]; + description = "A list of enabled kernel modules"; + }; + }; +} diff --git a/pkgs/clan-cli/tests/config/example-schema.json b/pkgs/clan-cli/tests/config/example-schema.json new file mode 100644 index 000000000..823a5d6cd --- /dev/null +++ b/pkgs/clan-cli/tests/config/example-schema.json @@ -0,0 +1,54 @@ +{ + "type": "object", + "properties": { + "name": { + "type": "string", + "default": "John Doe", + "description": "The name of the user" + }, + "age": { + "type": "integer", + "default": 42, + "description": "The age of the user" + }, + "isAdmin": { + "type": "boolean", + "default": false, + "description": "Is the user an admin?" + }, + "kernelModules": { + "type": "array", + "items": { + "type": "string" + }, + "default": [ + "nvme", + "xhci_pci", + "ahci" + ], + "description": "A list of enabled kernel modules" + }, + "userIds": { + "type": "object", + "default": { + "horst": 1, + "peter": 2, + "albrecht": 3 + }, + "additionalProperties": { + "type": "integer" + }, + "description": "Some attributes" + }, + "services": { + "type": "object", + "properties": { + "opt": { + "type": "string", + "default": "foo", + "description": "A submodule option" + } + } + } + } +} diff --git a/pkgs/clan-cli/tests/config/test.nix b/pkgs/clan-cli/tests/config/test.nix new file mode 100644 index 000000000..8a39ed248 --- /dev/null +++ b/pkgs/clan-cli/tests/config/test.nix @@ -0,0 +1,8 @@ +# run these tests via `nix-unit ./test.nix` +{ lib ? (import { }).lib +, slib ? import ../../clan_cli/config/schema-lib.nix { inherit lib; } +}: +{ + parseOption = import ./test_parseOption.nix { inherit lib slib; }; + parseOptions = import ./test_parseOptions.nix { inherit lib slib; }; +} diff --git a/pkgs/clan-cli/tests/config/test_parseOption.nix b/pkgs/clan-cli/tests/config/test_parseOption.nix new file mode 100644 index 000000000..b3e6173b5 --- /dev/null +++ b/pkgs/clan-cli/tests/config/test_parseOption.nix @@ -0,0 +1,249 @@ +# tests for the nixos options to jsonschema converter +# run these tests via `nix-unit ./test.nix` +{ lib ? (import { }).lib +, slib ? import ../../clan_cli/config/schema-lib.nix { inherit lib; } +}: +let + description = "Test Description"; + + evalType = type: default: + let + evaledConfig = lib.evalModules { + modules = [{ + options.opt = lib.mkOption { + inherit type; + inherit default; + inherit description; + }; + }]; + }; + in + evaledConfig.options.opt; +in + +{ + testNoDefaultNoDescription = + let + evaledConfig = lib.evalModules { + modules = [{ + options.opt = lib.mkOption { + type = lib.types.bool; + }; + }]; + }; + in + { + expr = slib.parseOption evaledConfig.options.opt; + expected = { + type = "boolean"; + }; + }; + + testBool = + let + default = false; + in + { + expr = slib.parseOption (evalType lib.types.bool default); + expected = { + type = "boolean"; + inherit default description; + }; + }; + + testString = + let + default = "hello"; + in + { + expr = slib.parseOption (evalType lib.types.str default); + expected = { + type = "string"; + inherit default description; + }; + }; + + testInteger = + let + default = 42; + in + { + expr = slib.parseOption (evalType lib.types.int default); + expected = { + type = "integer"; + inherit default description; + }; + }; + + testFloat = + let + default = 42.42; + in + { + expr = slib.parseOption (evalType lib.types.float default); + expected = { + type = "number"; + inherit default description; + }; + }; + + testEnum = + let + default = "foo"; + values = [ "foo" "bar" "baz" ]; + in + { + expr = slib.parseOption (evalType (lib.types.enum values) default); + expected = { + enum = values; + inherit default description; + }; + }; + + testListOfInt = + let + default = [ 1 2 3 ]; + in + { + expr = slib.parseOption (evalType (lib.types.listOf lib.types.int) default); + expected = { + type = "array"; + items = { + type = "integer"; + }; + inherit default description; + }; + }; + + testAttrsOfInt = + let + default = { foo = 1; bar = 2; baz = 3; }; + in + { + expr = slib.parseOption (evalType (lib.types.attrsOf lib.types.int) default); + expected = { + type = "object"; + additionalProperties = { + type = "integer"; + }; + inherit default description; + }; + }; + + testNullOrBool = + let + default = null; # null is a valid value for this type + in + { + expr = slib.parseOption (evalType (lib.types.nullOr lib.types.bool) default); + expected = { + type = [ "null" "boolean" ]; + inherit default description; + }; + }; + + testSubmoduleOption = + let + subModule = { + options.opt = lib.mkOption { + type = lib.types.bool; + default = true; + inherit description; + }; + }; + in + { + expr = slib.parseOption (evalType (lib.types.submodule subModule) { }); + expected = { + type = "object"; + properties = { + opt = { + type = "boolean"; + default = true; + inherit description; + }; + }; + }; + }; + + testSubmoduleOptionWithoutDefault = + let + subModule = { + options.opt = lib.mkOption { + type = lib.types.bool; + inherit description; + }; + }; + in + { + expr = slib.parseOption (evalType (lib.types.submodule subModule) { }); + expected = { + type = "object"; + properties = { + opt = { + type = "boolean"; + inherit description; + }; + }; + required = [ "opt" ]; + }; + }; + + testAttrsOfSubmodule = + let + subModule = { + options.opt = lib.mkOption { + type = lib.types.bool; + default = true; + inherit description; + }; + }; + default = { foo.opt = false; bar.opt = true; }; + in + { + expr = slib.parseOption (evalType (lib.types.attrsOf (lib.types.submodule subModule)) default); + expected = { + type = "object"; + additionalProperties = { + type = "object"; + properties = { + opt = { + type = "boolean"; + default = true; + inherit description; + }; + }; + }; + inherit default description; + }; + }; + + testListOfSubmodule = + let + subModule = { + options.opt = lib.mkOption { + type = lib.types.bool; + default = true; + inherit description; + }; + }; + default = [{ opt = false; } { opt = true; }]; + in + { + expr = slib.parseOption (evalType (lib.types.listOf (lib.types.submodule subModule)) default); + expected = { + type = "array"; + items = { + type = "object"; + properties = { + opt = { + type = "boolean"; + default = true; + inherit description; + }; + }; + }; + inherit default description; + }; + }; +} diff --git a/pkgs/clan-cli/tests/config/test_parseOptions.nix b/pkgs/clan-cli/tests/config/test_parseOptions.nix new file mode 100644 index 000000000..4787d9d95 --- /dev/null +++ b/pkgs/clan-cli/tests/config/test_parseOptions.nix @@ -0,0 +1,20 @@ +# tests for the nixos options to jsonschema converter +# run these tests via `nix-unit ./test.nix` +{ lib ? (import { }).lib +, slib ? import ../../clan_cli/config/schema-lib.nix { inherit lib; } +}: +let + evaledOptions = + let + evaledConfig = lib.evalModules { + modules = [ ./example-interface.nix ]; + }; + in + evaledConfig.options; +in +{ + testParseOptions = { + expr = slib.parseOptions evaledOptions; + expected = builtins.fromJSON (builtins.readFile ./example-schema.json); + }; +}