yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'mic92' (#74) from mic92 into main

Browse Source 
Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/74
This commit is contained in:
Mic92
2023-08-03 11:35:23 +00:00
parent 7f54a64c0a 7199f405ab
commit a405b4e53f
13 changed files with 230 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
flake.lock generated
View File

@@ -95,54 +95,6 @@
"type": "github"
}
},
"nix-github-actions": {
"inputs": {
"nixpkgs": [
"nix-unit",
"nixpkgs"
]
},
"locked": {
"lastModified": 1688870561,
"narHash": "sha256-4UYkifnPEw1nAzqqPOTL2MvWtm3sNGw1UTYTalkTcGY=",
"owner": "nix-community",
"repo": "nix-github-actions",
"rev": "165b1650b753316aa7f1787f3005a8d2da0f5301",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "nix-github-actions",
"type": "github"
}
},
"nix-unit": {
"inputs": {
"flake-parts": [
"flake-parts"
],
"nix-github-actions": "nix-github-actions",
"nixpkgs": [
"nixpkgs"
],
"treefmt-nix": [
"treefmt-nix"
]
},
"locked": {
"lastModified": 1690289081,
"narHash": "sha256-PCXQAQt8+i2pkUym9P1JY4JGoeZJLzzxWBhprHDdItM=",
"owner": "adisbladis",
"repo": "nix-unit",
"rev": "a9d6f33e50d4dcd9cfc0c92253340437bbae282b",
"type": "github"
},
"original": {
"owner": "adisbladis",
"repo": "nix-unit",
"type": "github"
}
},
"nixlib": {
"locked": {
"lastModified": 1689469483,
@@ -253,7 +205,6 @@
"inputs": {
"disko": "disko",
"flake-parts": "flake-parts",
"nix-unit": "nix-unit",
"nixos-generators": "nixos-generators",
"nixpkgs": "nixpkgs",
"pre-commit-hooks-nix": "pre-commit-hooks-nix",

5
flake.nix
View File

@@ -12,10 +12,6 @@
treefmt-nix.url = "github:numtide/treefmt-nix";
treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
pre-commit-hooks-nix.url = "github:cachix/pre-commit-hooks.nix";
nix-unit.url = "github:adisbladis/nix-unit";
nix-unit.inputs.flake-parts.follows = "flake-parts";
nix-unit.inputs.nixpkgs.follows = "nixpkgs";
nix-unit.inputs.treefmt-nix.follows = "treefmt-nix";
};
outputs = inputs @ { flake-parts, ... }:
@@ -35,6 +31,7 @@
./templates/flake-module.nix
./templates/python-project/flake-module.nix
./pkgs/clan-cli/flake-module.nix
./pkgs/nix-unit/flake-module.nix
./lib/flake-module.nix
];
});

7
pkgs/clan-cli/clan_cli/cli.py
View File

@@ -1,8 +1,10 @@
import argparse
import subprocess
import sys
from . import admin, config, secrets, ssh
from .errors import ClanError
from .tty import warn
has_argcomplete = True
try:
@@ -20,7 +22,10 @@ def main() -> None:
admin.register_parser(parser_admin)
parser_config = subparsers.add_parser("config")
config.register_parser(parser_config)
try:
config.register_parser(parser_config)
except subprocess.CalledProcessError:
warn("The config command does not in the nix sandbox")
parser_ssh = subparsers.add_parser("ssh", help="ssh to a remote machine")
ssh.register_parser(parser_ssh)

71
pkgs/clan-cli/clan_cli/config/__init__.py
View File

@@ -4,12 +4,14 @@ import json
import subprocess
import sys
from pathlib import Path
from typing import Any, Optional, Union
from typing import Any, Optional, Type, Union
from clan_cli.errors import ClanError
class Kwargs:
def __init__(self):
self.type = None
def __init__(self) -> None:
self.type: Optional[Type] = None
self.default: Any = None
self.required: bool = False
self.help: Optional[str] = None
@@ -19,7 +21,7 @@ class Kwargs:
def schema_from_module_file(
file: Union[str, Path] = "./tests/config/example-interface.nix",
) -> dict:
) -> dict[str, Any]:
absolute_path = Path(file).absolute()
# define a nix expression that loads the given module file using lib.evalModules
nix_expr = f"""
@@ -37,22 +39,31 @@ def schema_from_module_file(
)
# takes a (sub)parser and configures it
def register_parser(
parser: Optional[argparse.ArgumentParser] = None,
schema: Union[dict, str, Path] = "./tests/config/example-interface.nix",
) -> dict:
parser: argparse.ArgumentParser,
file: Path = Path("./tests/config/example-interface.nix"),
) -> None:
if file.name.endswith(".nix"):
schema = schema_from_module_file(file)
else:
schema = json.loads(file.read_text())
return _register_parser(parser, schema)
# takes a (sub)parser and configures it
def _register_parser(
parser: Optional[argparse.ArgumentParser],
schema: dict[str, Any],
) -> None:
# check if schema is a .nix file and load it in that case
if isinstance(schema, str) and schema.endswith(".nix"):
schema = schema_from_module_file(schema)
elif not isinstance(schema, dict):
with open(str(schema)) as f:
schema: dict = json.load(f)
assert "type" in schema and schema["type"] == "object"
if "type" not in schema:
raise ClanError("Schema has no type")
if schema["type"] != "object":
raise ClanError("Schema is not an object")
required_set = set(schema.get("required", []))
type_map = {
type_map: dict[str, Type] = {
"array": list,
"boolean": bool,
"integer": int,
@@ -60,8 +71,7 @@ def register_parser(
"string": str,
}
if parser is None:
parser = argparse.ArgumentParser(description=schema.get("description"))
parser = argparse.ArgumentParser(description=schema.get("description"))
subparsers = parser.add_subparsers(
title="more options",
@@ -72,11 +82,12 @@ def register_parser(
for name, value in schema.get("properties", {}).items():
assert isinstance(value, dict)
type_ = value.get("type")
# TODO: add support for nested objects
if value.get("type") == "object":
if type_ == "object":
subparser = subparsers.add_parser(name, help=value.get("description"))
register_parser(parser=subparser, schema=value)
_register_parser(parser=subparser, schema=value)
continue
# elif value.get("type") == "array":
# subparser = parser.add_subparsers(dest=name)
@@ -92,22 +103,25 @@ def register_parser(
if "enum" in value:
enum_list = value["enum"]
assert len(enum_list) > 0, "Enum List is Empty"
if len(enum_list) == 0:
raise ClanError("Enum List is Empty")
arg_type = type(enum_list[0])
assert all(
arg_type is type(item) for item in enum_list
), f"Items in [{enum_list}] with Different Types"
if not all(arg_type is type(item) for item in enum_list):
raise ClanError(f"Items in [{enum_list}] with Different Types")
kwargs.type = arg_type
kwargs.choices = enum_list
else:
kwargs.type = type_map[value.get("type")]
elif type_ in type_map:
kwargs.type = type_map[type_]
del kwargs.choices
else:
raise ClanError(f"Unsupported Type '{type_}' in schema")
name = f"--{name}"
if kwargs.type is bool:
assert not kwargs.default, "boolean have to be False in default"
if kwargs.default:
raise ClanError("Boolean have to be False in default")
kwargs.default = False
kwargs.action = "store_true"
del kwargs.type
@@ -117,7 +131,7 @@ def register_parser(
parser.add_argument(name, **vars(kwargs))
def main():
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument(
"schema",
@@ -125,8 +139,7 @@ def main():
type=str,
)
args = parser.parse_args(sys.argv[1:2])
schema = args.schema
register_parser(schema=schema, parser=parser)
register_parser(parser, args.schema)
parser.parse_args(sys.argv[2:])

30
pkgs/clan-cli/clan_cli/secrets/folders.py
View File

@@ -1,4 +1,3 @@
import json
import os
import shutil
from pathlib import Path
@@ -40,32 +39,3 @@ def remove_object(path: Path, name: str) -> None:
raise ClanError(f"{name} not found in {path}")
if not os.listdir(path):
os.rmdir(path)
def add_key(path: Path, publickey: str, overwrite: bool) -> None:
path.mkdir(parents=True, exist_ok=True)
try:
flags = os.O_CREAT | os.O_WRONLY | os.O_TRUNC
if not overwrite:
flags |= os.O_EXCL
fd = os.open(path / "key.json", flags)
except FileExistsError:
raise ClanError(f"{path.name} already exists in {path}")
with os.fdopen(fd, "w") as f:
json.dump({"publickey": publickey, "type": "age"}, f, indent=2)
def read_key(path: Path) -> str:
with open(path / "key.json") as f:
try:
key = json.load(f)
except json.JSONDecodeError as e:
raise ClanError(f"Failed to decode {path.name}: {e}")
if key["type"] != "age":
raise ClanError(
f"{path.name} is not an age key but {key['type']}. This is not supported"
)
publickey = key.get("publickey")
if not publickey:
raise ClanError(f"{path.name} does not contain a public key")
return publickey

3
pkgs/clan-cli/clan_cli/secrets/machines.py
View File

@@ -1,7 +1,8 @@
import argparse
from . import secrets
from .folders import add_key, list_objects, remove_object, sops_machines_folder
from .folders import list_objects, remove_object, sops_machines_folder
from .sops import add_key
from .types import (
machine_name_type,
public_or_private_age_key_type,

33
pkgs/clan-cli/clan_cli/secrets/sops.py
View File

@@ -1,3 +1,4 @@
import json
import os
import shutil
import subprocess
@@ -7,8 +8,9 @@ from typing import IO
from .. import tty
from ..dirs import user_config_dir
from ..errors import ClanError
from ..nix import nix_shell
from .folders import add_key, read_key, sops_users_folder
from .folders import sops_users_folder
class SopsKey:
@@ -122,3 +124,32 @@ def encrypt_file(secret_path: Path, content: IO[str], keys: list[str]) -> None:
os.remove(f.name)
except OSError:
pass
def add_key(path: Path, publickey: str, overwrite: bool) -> None:
path.mkdir(parents=True, exist_ok=True)
try:
flags = os.O_CREAT | os.O_WRONLY | os.O_TRUNC
if not overwrite:
flags |= os.O_EXCL
fd = os.open(path / "key.json", flags)
except FileExistsError:
raise ClanError(f"{path.name} already exists in {path}")
with os.fdopen(fd, "w") as f:
json.dump({"publickey": publickey, "type": "age"}, f, indent=2)
def read_key(path: Path) -> str:
with open(path / "key.json") as f:
try:
key = json.load(f)
except json.JSONDecodeError as e:
raise ClanError(f"Failed to decode {path.name}: {e}")
if key["type"] != "age":
raise ClanError(
f"{path.name} is not an age key but {key['type']}. This is not supported"
)
publickey = key.get("publickey")
if not publickey:
raise ClanError(f"{path.name} does not contain a public key")
return publickey

3
pkgs/clan-cli/clan_cli/secrets/users.py
View File

@@ -1,7 +1,8 @@
import argparse
from . import secrets
from .folders import add_key, list_objects, remove_object, sops_users_folder
from .folders import list_objects, remove_object, sops_users_folder
from .sops import add_key
from .types import (
VALID_SECRET_NAME,
public_or_private_age_key_type,

138
pkgs/clan-cli/default.nix
View File

@@ -1,5 +1,4 @@
{ pkgs
, lib
{ lib
, python3
, ruff
, runCommand
@@ -8,78 +7,75 @@
, bubblewrap
, sops
, age
, black
, nix
, mypy
, setuptools
, self
, argcomplete
, pytest
, pytest-cov
, pytest-subprocess
, wheel
}:
let
pyproject = builtins.fromTOML (builtins.readFile ./pyproject.toml);
name = pyproject.project.name;
dependencies = [ argcomplete ];
src = lib.cleanSource ./.;
testDependencies = [
pytest
pytest-cov
pytest-subprocess
mypy
];
dependencies = lib.attrValues {
inherit (python3.pkgs)
argcomplete
;
};
devDependencies = lib.attrValues {
inherit (pkgs) ruff;
inherit (python3.pkgs)
black
mypy
pytest
pytest-cov
pytest-subprocess
setuptools
wheel
;
};
package = python3.pkgs.buildPythonPackage {
inherit name src;
format = "pyproject";
nativeBuildInputs = [
python3.pkgs.setuptools
installShellFiles
];
propagatedBuildInputs =
dependencies
++ [ ];
passthru.tests = { inherit clan-mypy clan-pytest; };
passthru.devDependencies = devDependencies;
makeWrapperArgs = [
"--set CLAN_FLAKE ${self}"
];
postInstall = ''
installShellCompletion --bash --name clan \
<(${python3.pkgs.argcomplete}/bin/register-python-argcomplete --shell bash clan)
installShellCompletion --fish --name clan.fish \
<(${python3.pkgs.argcomplete}/bin/register-python-argcomplete --shell fish clan)
'';
meta.mainProgram = "clan";
};
checkPython = python3.withPackages (_ps: devDependencies ++ dependencies);
clan-mypy = runCommand "${name}-mypy" { } ''
cp -r ${src} ./src
chmod +w -R ./src
cd src
${checkPython}/bin/mypy .
touch $out
'';
clan-pytest = runCommand "${name}-tests"
{
nativeBuildInputs = [ zerotierone bubblewrap sops age ];
} ''
cp -r ${src} ./src
chmod +w -R ./src
cd src
${checkPython}/bin/python -m pytest ./tests
touch $out
'';
checkPython = python3.withPackages (_ps: dependencies ++ testDependencies);
in
package
python3.pkgs.buildPythonPackage {
name = "clan";
src = lib.cleanSource ./.;
format = "pyproject";
nativeBuildInputs = [
setuptools
installShellFiles
];
propagatedBuildInputs = dependencies;
passthru.tests = {
clan-mypy = runCommand "clan-mypy" { } ''
cp -r ${./.} ./src
chmod +w -R ./src
cd src
${checkPython}/bin/mypy .
touch $out
'';
clan-pytest = runCommand "clan-tests"
{
nativeBuildInputs = [ age zerotierone bubblewrap sops nix ];
} ''
cp -r ${./.} ./src
chmod +w -R ./src
cd src
${checkPython}/bin/python -m pytest ./tests
touch $out
'';
};
passthru.devDependencies = [
ruff
black
setuptools
wheel
] ++ testDependencies;
makeWrapperArgs = [
"--set CLAN_FLAKE ${self}"
];
postInstall = ''
installShellCompletion --bash --name clan \
<(${argcomplete}/bin/register-python-argcomplete --shell bash clan)
installShellCompletion --fish --name clan.fish \
<(${argcomplete}/bin/register-python-argcomplete --shell fish clan)
'';
meta.mainProgram = "clan";
}

49
pkgs/clan-cli/flake-module.nix
View File

@@ -1,11 +1,11 @@
{ self, ... }: {
perSystem = { inputs', self', pkgs, ... }: {
perSystem = { self', pkgs, ... }: {
devShells.clan = pkgs.callPackage ./shell.nix {
inherit self;
inherit (self'.packages) clan;
};
packages = {
clan = pkgs.callPackage ./default.nix {
clan = pkgs.python3.pkgs.callPackage ./default.nix {
inherit self;
zerotierone = self'.packages.zerotierone;
};
@@ -18,34 +18,39 @@
openssh
sshpass
zbar
tor;
tor
age
sops;
# Override license so that we can build zerotierone without
# having to re-import nixpkgs.
zerotierone = pkgs.zerotierone.overrideAttrs (_old: { meta = { }; });
## End optional dependencies
};
# check if the `clan config` example jsonschema and data is valid
checks.clan-config-example-schema-valid = pkgs.runCommand "clan-config-example-schema-valid" { } ''
echo "Checking that example-schema.json is valid"
${pkgs.check-jsonschema}/bin/check-jsonschema \
--check-metaschema ${./.}/tests/config/example-schema.json
checks = self'.packages.clan.tests // {
# check if the `clan config` example jsonschema and data is valid
clan-config-example-schema-valid = pkgs.runCommand "clan-config-example-schema-valid" { } ''
echo "Checking that example-schema.json is valid"
${pkgs.check-jsonschema}/bin/check-jsonschema \
--check-metaschema ${./.}/tests/config/example-schema.json
echo "Checking that example-data.json is valid according to example-schema.json"
${pkgs.check-jsonschema}/bin/check-jsonschema \
--schemafile ${./.}/tests/config/example-schema.json \
${./.}/tests/config/example-data.json
echo "Checking that example-data.json is valid according to example-schema.json"
${pkgs.check-jsonschema}/bin/check-jsonschema \
--schemafile ${./.}/tests/config/example-schema.json \
${./.}/tests/config/example-data.json
touch $out
'';
touch $out
'';
# check if the `clan config` nix jsonschema converter unit tests succeed
checks.clan-config-nix-unit-tests = pkgs.runCommand "clan-edit-unit-tests" { } ''
export NIX_PATH=nixpkgs=${pkgs.path}
${inputs'.nix-unit.packages.nix-unit}/bin/nix-unit \
${./.}/tests/config/test.nix \
--eval-store $(realpath .)
touch $out
'';
# check if the `clan config` nix jsonschema converter unit tests succeed
clan-config-nix-unit-tests = pkgs.runCommand "clan-edit-unit-tests" { } ''
export NIX_PATH=nixpkgs=${pkgs.path}
${self'.packages.nix-unit}/bin/nix-unit \
${./.}/tests/config/test.nix \
--eval-store $(realpath .)
touch $out
'';
};
};
}

2
pkgs/clan-cli/shell.nix
View File

@@ -15,7 +15,7 @@ in
pkgs.mkShell {
packages = [
pkgs.ruff
self.inputs.nix-unit.packages.${pkgs.system}.nix-unit
self.packages.${pkgs.system}.nix-unit
pythonWithDeps
];
# sets up an editable install and add enty points to $PATH

45
pkgs/nix-unit/default.nix Normal file
View File

@@ -0,0 +1,45 @@
{ stdenv
, lib
, nixVersions
, fetchFromGitHub
, nlohmann_json
, boost
, bear
, meson
, pkg-config
, ninja
, cmake
, clang-tools
}:
stdenv.mkDerivation {
pname = "nix-unit";
version = "0.1";
src = fetchFromGitHub {
owner = "adisbladis";
repo = "nix-unit";
rev = "a9d6f33e50d4dcd9cfc0c92253340437bbae282b";
sha256 = "sha256-PCXQAQt8+i2pkUym9P1JY4JGoeZJLzzxWBhprHDdItM=";
};
buildInputs = [
nlohmann_json
nixVersions.unstable
boost
];
nativeBuildInputs = [
bear
meson
pkg-config
ninja
# nlohmann_json can be only discovered via cmake files
cmake
] ++ (lib.optional stdenv.cc.isClang [ bear clang-tools ]);
meta = {
description = "Nix unit test runner";
homepage = "https://github.com/adisbladis/nix-unit";
license = lib.licenses.gpl3;
maintainers = with lib.maintainers; [ adisbladis ];
platforms = lib.platforms.unix;
};
}

5
pkgs/nix-unit/flake-module.nix Normal file
View File

@@ -0,0 +1,5 @@
{
perSystem = { pkgs, ... }: {
packages.nix-unit = pkgs.callPackage ./default.nix { };
};
}