From fc1591a0a8be53be599fa35afa8a4dc20abe456a Mon Sep 17 00:00:00 2001 From: DavHau Date: Thu, 9 Nov 2023 15:30:25 +0700 Subject: [PATCH] api/schema: get schema without having a machine Also done: - add @pytest.mark.with_core indicating that a test depends on clan-core and cannot be cached - separate unit tests into two derivations, ones that can be cached and ones that cannot. - fix frontend typescript code --- checks/impure/flake-module.nix | 2 +- .../singleDiskExt4.nix => diskLayouts.nix} | 0 clanModules/flake-module.nix | 16 ++-- pkgs/clan-cli/clan_cli/config/machine.py | 50 ------------ pkgs/clan-cli/clan_cli/config/schema.py | 79 +++++++++++++++++++ .../clan_cli/webui/routers/machines.py | 16 ++-- pkgs/clan-cli/default.nix | 25 +++++- pkgs/clan-cli/flake-module.nix | 28 ++++++- pkgs/clan-cli/tests/test_clan_modules.py | 2 +- pkgs/clan-cli/tests/test_machines_api.py | 73 +++++++++-------- pkgs/clan-cli/tests/test_machines_config.py | 8 +- .../createMachineForm/clanModules.tsx | 6 +- 12 files changed, 192 insertions(+), 113 deletions(-) rename clanModules/{diskLayouts/singleDiskExt4.nix => diskLayouts.nix} (100%) create mode 100644 pkgs/clan-cli/clan_cli/config/schema.py diff --git a/checks/impure/flake-module.nix b/checks/impure/flake-module.nix index 6e587f916..2e325416f 100644 --- a/checks/impure/flake-module.nix +++ b/checks/impure/flake-module.nix @@ -13,7 +13,7 @@ ]}" ROOT=$(git rev-parse --show-toplevel) cd "$ROOT/pkgs/clan-cli" - nix develop "$ROOT#clan-cli" -c bash -c 'TMPDIR=/tmp python -m pytest -m impure -s ./tests' + nix develop "$ROOT#clan-cli" -c bash -c "TMPDIR=/tmp python -m pytest -m impure -s ./tests $@" ''; runMockApi = pkgs.writeShellScriptBin "run-mock-api" '' diff --git a/clanModules/diskLayouts/singleDiskExt4.nix b/clanModules/diskLayouts.nix similarity index 100% rename from clanModules/diskLayouts/singleDiskExt4.nix rename to clanModules/diskLayouts.nix diff --git a/clanModules/flake-module.nix b/clanModules/flake-module.nix index 332286a3b..5df749d5b 100644 --- a/clanModules/flake-module.nix +++ b/clanModules/flake-module.nix @@ -1,13 +1,11 @@ -{ self, lib, ... }: { +{ inputs, ... }: { flake.clanModules = { - diskLayouts = lib.mapAttrs' - (name: _: lib.nameValuePair (lib.removeSuffix ".nix" name) { - imports = [ - self.inputs.disko.nixosModules.disko - ./diskLayouts/${name} - ]; - }) - (builtins.readDir ./diskLayouts); + diskLayouts = { + imports = [ + ./diskLayouts.nix + inputs.disko.nixosModules.default + ]; + }; deltachat = ./deltachat.nix; xfce = ./xfce.nix; }; diff --git a/pkgs/clan-cli/clan_cli/config/machine.py b/pkgs/clan-cli/clan_cli/config/machine.py index c7433f7a5..eaf8d36cf 100644 --- a/pkgs/clan-cli/clan_cli/config/machine.py +++ b/pkgs/clan-cli/clan_cli/config/machine.py @@ -1,7 +1,6 @@ import json import os import subprocess -import sys from pathlib import Path from tempfile import NamedTemporaryFile from typing import Optional @@ -10,7 +9,6 @@ from fastapi import HTTPException from clan_cli.dirs import ( machine_settings_file, - nixpkgs_source, specific_flake_dir, specific_machine_dir, ) @@ -91,51 +89,3 @@ def set_config_for_machine( if repo_dir is not None: commit_file(settings_path, repo_dir) - - -def schema_for_machine( - flake_name: FlakeName, machine_name: str, config: Optional[dict] = None -) -> dict: - flake = specific_flake_dir(flake_name) - # use nix eval to lib.evalModules .#nixosConfigurations..options.clan - with NamedTemporaryFile(mode="w", dir=flake) as clan_machine_settings_file: - env = os.environ.copy() - inject_config_flags = [] - if config is not None: - json.dump(config, clan_machine_settings_file, indent=2) - clan_machine_settings_file.seek(0) - env["CLAN_MACHINE_SETTINGS_FILE"] = clan_machine_settings_file.name - inject_config_flags = [ - "--impure", # needed to access CLAN_MACHINE_SETTINGS_FILE - ] - proc = subprocess.run( - nix_eval( - flags=inject_config_flags - + [ - "--impure", - "--show-trace", - "--expr", - f""" - let - flake = builtins.getFlake (toString {flake}); - lib = import {nixpkgs_source()}/lib; - options = flake.nixosConfigurations.{machine_name}.options; - clanOptions = options.clan; - jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }}; - jsonschema = jsonschemaLib.parseOptions clanOptions; - in - jsonschema - """, - ], - ), - capture_output=True, - text=True, - cwd=flake, - env=env, - ) - if proc.returncode != 0: - print(proc.stderr, file=sys.stderr) - raise Exception( - f"Failed to read schema for machine {machine_name}:\n{proc.stderr}" - ) - return json.loads(proc.stdout) diff --git a/pkgs/clan-cli/clan_cli/config/schema.py b/pkgs/clan-cli/clan_cli/config/schema.py new file mode 100644 index 000000000..723401d3a --- /dev/null +++ b/pkgs/clan-cli/clan_cli/config/schema.py @@ -0,0 +1,79 @@ +import json +import os +import subprocess +import sys +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import Optional + +from clan_cli.dirs import ( + nixpkgs_source, + specific_flake_dir, +) +from clan_cli.nix import nix_eval + +from ..types import FlakeName + + +def machine_schema( + flake_name: FlakeName, + config: dict, + clan_imports: Optional[list[str]] = None, +) -> dict: + flake = specific_flake_dir(flake_name) + # use nix eval to lib.evalModules .#nixosConfigurations..options.clan + with NamedTemporaryFile(mode="w", dir=flake) as clan_machine_settings_file: + env = os.environ.copy() + inject_config_flags = [] + if clan_imports is not None: + config["clanImports"] = clan_imports + json.dump(config, clan_machine_settings_file, indent=2) + clan_machine_settings_file.seek(0) + env["CLAN_MACHINE_SETTINGS_FILE"] = clan_machine_settings_file.name + inject_config_flags = [ + "--impure", # needed to access CLAN_MACHINE_SETTINGS_FILE + ] + proc = subprocess.run( + nix_eval( + flags=inject_config_flags + + [ + "--impure", + "--show-trace", + "--expr", + f""" + let + system = builtins.currentSystem; + flake = builtins.getFlake (toString {flake}); + clan-core = flake.inputs.clan-core; + nixpkgsSrc = flake.inputs.nixpkgs or {nixpkgs_source()}; + lib = import (nixpkgsSrc + /lib); + pkgs = import nixpkgsSrc {{ inherit system; }}; + config = lib.importJSON (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE"); + fakeMachine = pkgs.nixos {{ + imports = + [ + clan-core.nixosModules.clanCore + # potentially the config might affect submodule options, + # therefore we need to import it + config + ] + # add all clan modules specified via clanImports + ++ (map (name: clan-core.clanModules.${{name}}) config.clanImports or []); + }}; + clanOptions = fakeMachine.options.clan; + jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }}; + jsonschema = jsonschemaLib.parseOptions clanOptions; + in + jsonschema + """, + ], + ), + capture_output=True, + text=True, + cwd=flake, + env=env, + ) + if proc.returncode != 0: + print(proc.stderr, file=sys.stderr) + raise Exception(f"Failed to read schema:\n{proc.stderr}") + return json.loads(proc.stdout) diff --git a/pkgs/clan-cli/clan_cli/webui/routers/machines.py b/pkgs/clan-cli/clan_cli/webui/routers/machines.py index c2d6b364f..d8a5b8e41 100644 --- a/pkgs/clan-cli/clan_cli/webui/routers/machines.py +++ b/pkgs/clan-cli/clan_cli/webui/routers/machines.py @@ -8,10 +8,10 @@ from clan_cli.webui.api_inputs import MachineConfig from ...config.machine import ( config_for_machine, - schema_for_machine, set_config_for_machine, verify_machine_config, ) +from ...config.schema import machine_schema from ...machines.create import create_machine as _create_machine from ...machines.list import list_machines as _list_machines from ...types import FlakeName @@ -68,17 +68,11 @@ async def set_machine_config( set_config_for_machine(flake_name, name, conf) -@router.get("/api/{flake_name}/machines/{name}/schema", tags=[Tags.machine]) -async def get_machine_schema(flake_name: FlakeName, name: str) -> SchemaResponse: - schema = schema_for_machine(flake_name, name) - return SchemaResponse(schema=schema) - - -@router.put("/api/{flake_name}/machines/{name}/schema", tags=[Tags.machine]) -async def set_machine_schema( - flake_name: FlakeName, name: str, config: Annotated[dict, Body()] +@router.put("/api/{flake_name}/schema", tags=[Tags.machine]) +async def get_machine_schema( + flake_name: FlakeName, config: Annotated[dict, Body()] ) -> SchemaResponse: - schema = schema_for_machine(flake_name, name, config) + schema = machine_schema(flake_name, config=config) return SchemaResponse(schema=schema) diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index 173368f9c..3cbed5994 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -36,6 +36,8 @@ , mypy , deal , schemathesis +, rope +, clan-core-path }: let @@ -132,14 +134,30 @@ python3.pkgs.buildPythonApplication { propagatedBuildInputs = dependencies; # also re-expose dependencies so we test them in CI - passthru.tests = (lib.mapAttrs' (n: lib.nameValuePair "clan-dep-${n}") runtimeDependenciesAsSet) // { - clan-pytest = runCommand "clan-pytest" { nativeBuildInputs = [ checkPython ] ++ pytestDependencies; } '' + passthru.tests = (lib.mapAttrs' (n: lib.nameValuePair "clan-dep-${n}") runtimeDependenciesAsSet) // rec { + clan-pytest-without-core = runCommand "clan-pytest-without-core" { nativeBuildInputs = [ checkPython ] ++ pytestDependencies; } '' cp -r ${source} ./src chmod +w -R ./src cd ./src export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 - ${checkPython}/bin/python -m pytest -m "not impure" -s ./tests + ${checkPython}/bin/python -m pytest -m "not impure and not with_core" -s ./tests + touch $out + ''; + # separate the tests that can never be cached + clan-pytest-with-core = runCommand "clan-pytest-with-core" { nativeBuildInputs = [ checkPython ] ++ pytestDependencies; } '' + cp -r ${source} ./src + chmod +w -R ./src + cd ./src + + export CLAN_CORE=${clan-core-path} + export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 + ${checkPython}/bin/python -m pytest -m "not impure and with_core" -s ./tests + touch $out + ''; + clan-pytest = runCommand "clan-pytest" { } '' + echo ${clan-pytest-without-core} + echo ${clan-pytest-with-core} touch $out ''; check-for-breakpoints = runCommand "breakpoints" { } '' @@ -164,6 +182,7 @@ python3.pkgs.buildPythonApplication { passthru.checkPython = checkPython; passthru.devDependencies = [ + rope setuptools wheel ] ++ pytestDependencies; diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index 7a230dabd..7b1e383a6 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -1,8 +1,33 @@ -{ inputs, ... }: +{ inputs, self, lib, ... }: { perSystem = { self', pkgs, system, ... }: let luisPythonPkgs = inputs.luispkgs.legacyPackages.${system}.python3Packages; + flakeLock = lib.importJSON (self + /flake.lock); + flakeInputs = (builtins.removeAttrs inputs [ "self" ]); + flakeLockVendoredDeps = flakeLock // { + nodes = flakeLock.nodes // ( + lib.flip lib.mapAttrs flakeInputs (name: _: flakeLock.nodes.${name} // { + locked = { + inherit (flakeLock.nodes.${name}.locked) narHash; + lastModified = + # lol, nixpkgs has a different timestamp on the fs??? + if name == "nixpkgs" + then 0 + else 1; + path = "${inputs.${name}}"; + type = "path"; + }; + }) + ); + }; + flakeLockFile = builtins.toFile "clan-core-flake.lock" + (builtins.toJSON flakeLockVendoredDeps); + clanCoreWithVendoredDeps = lib.trace flakeLockFile pkgs.runCommand "clan-core-with-vendored-deps" { } '' + cp -r ${self} $out + chmod +w -R $out + cp ${flakeLockFile} $out/flake.lock + ''; in { devShells.clan-cli = pkgs.callPackage ./shell.nix { @@ -14,6 +39,7 @@ inherit (inputs) nixpkgs; deal = luisPythonPkgs.deal; schemathesis = luisPythonPkgs.schemathesis; + clan-core-path = clanCoreWithVendoredDeps; }; inherit (self'.packages.clan-cli) clan-openapi; default = self'.packages.clan-cli; diff --git a/pkgs/clan-cli/tests/test_clan_modules.py b/pkgs/clan-cli/tests/test_clan_modules.py index 3b30750e4..300a7adcd 100644 --- a/pkgs/clan-cli/tests/test_clan_modules.py +++ b/pkgs/clan-cli/tests/test_clan_modules.py @@ -3,7 +3,7 @@ from api import TestClient from fixtures_flakes import FlakeForTest -@pytest.mark.impure() +@pytest.mark.with_core def test_configure_machine(api: TestClient, test_flake_with_core: FlakeForTest) -> None: # retrieve the list of available clanModules response = api.get(f"/api/{test_flake_with_core.name}/clan_modules") diff --git a/pkgs/clan-cli/tests/test_machines_api.py b/pkgs/clan-cli/tests/test_machines_api.py index dde4ae861..84a9eeb7e 100644 --- a/pkgs/clan-cli/tests/test_machines_api.py +++ b/pkgs/clan-cli/tests/test_machines_api.py @@ -1,3 +1,4 @@ +import pytest from api import TestClient from fixtures_flakes import FlakeForTest @@ -21,37 +22,46 @@ def test_machines(api: TestClient, test_flake: FlakeForTest) -> None: assert response.json() == {"machines": [{"name": "test", "status": "unknown"}]} -def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None: +@pytest.mark.with_core +def test_configure_machine(api: TestClient, test_flake_with_core: FlakeForTest) -> None: # ensure error 404 if machine does not exist when accessing the config - response = api.get(f"/api/{test_flake.name}/machines/machine1/config") + response = api.get(f"/api/{test_flake_with_core.name}/machines/machine1/config") assert response.status_code == 404 # ensure error 404 if machine does not exist when writing to the config - response = api.put(f"/api/{test_flake.name}/machines/machine1/config", json={}) + response = api.put( + f"/api/{test_flake_with_core.name}/machines/machine1/config", json={} + ) assert response.status_code == 404 # create the machine - response = api.post(f"/api/{test_flake.name}/machines", json={"name": "machine1"}) + response = api.post( + f"/api/{test_flake_with_core.name}/machines", json={"name": "machine1"} + ) assert response.status_code == 201 # ensure an empty config is returned by default for a new machine - response = api.get(f"/api/{test_flake.name}/machines/machine1/config") + response = api.get(f"/api/{test_flake_with_core.name}/machines/machine1/config") assert response.status_code == 200 assert response.json() == { "clanImports": [], "clan": {}, } - # get jsonschema for machine - response = api.get(f"/api/{test_flake.name}/machines/machine1/schema") + # get jsonschema for without imports + response = api.put( + f"/api/{test_flake_with_core.name}/schema", + json={"clanImports": []}, + ) assert response.status_code == 200 json_response = response.json() assert "schema" in json_response and "properties" in json_response["schema"] # an invalid config missing the fileSystems invalid_config = dict( - clan=dict( - jitsi=dict( + clan=dict(), + services=dict( + nginx=dict( enable=True, ), ), @@ -59,7 +69,7 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None: # verify an invalid config (fileSystems missing) fails response = api.put( - f"/api/{test_flake.name}/machines/machine1/verify", + f"/api/{test_flake_with_core.name}/machines/machine1/verify", json=invalid_config, ) assert response.status_code == 200 @@ -70,13 +80,13 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None: # set come invalid config (fileSystems missing) response = api.put( - f"/api/{test_flake.name}/machines/machine1/config", + f"/api/{test_flake_with_core.name}/machines/machine1/config", json=invalid_config, ) assert response.status_code == 200 # ensure the config has actually been updated - response = api.get(f"/api/{test_flake.name}/machines/machine1/config") + response = api.get(f"/api/{test_flake_with_core.name}/machines/machine1/config") assert response.status_code == 200 assert response.json() == dict(clanImports=[], **invalid_config) @@ -99,8 +109,9 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None: # set some valid config config2 = dict( - clan=dict( - jitsi=dict( + clan=dict(), + services=dict( + nginx=dict( enable=True, ), ), @@ -108,20 +119,20 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None: ) response = api.put( - f"/api/{test_flake.name}/machines/machine1/config", + f"/api/{test_flake_with_core.name}/machines/machine1/config", json=config2, ) assert response.status_code == 200 # ensure the config has been applied response = api.get( - f"/api/{test_flake.name}/machines/machine1/config", + f"/api/{test_flake_with_core.name}/machines/machine1/config", ) assert response.status_code == 200 assert response.json() == dict(clanImports=[], **config2) # get the config again - response = api.get(f"/api/{test_flake.name}/machines/machine1/config") + response = api.get(f"/api/{test_flake_with_core.name}/machines/machine1/config") assert response.status_code == 200 assert response.json() == {"clanImports": [], **config2} @@ -129,36 +140,36 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None: # For example, this should not result in the boot.loader.grub.devices being # set twice (eg. merged) response = api.put( - f"/api/{test_flake.name}/machines/machine1/config", + f"/api/{test_flake_with_core.name}/machines/machine1/config", json=config2, ) assert response.status_code == 200 # ensure the config has been applied response = api.get( - f"/api/{test_flake.name}/machines/machine1/config", + f"/api/{test_flake_with_core.name}/machines/machine1/config", ) assert response.status_code == 200 assert response.json() == dict(clanImports=[], **config2) # verify the machine config evaluates - response = api.get(f"/api/{test_flake.name}/machines/machine1/verify") + response = api.get(f"/api/{test_flake_with_core.name}/machines/machine1/verify") assert response.status_code == 200 assert response.json() == {"success": True, "error": None} # get the schema with an extra module imported response = api.put( - f"/api/{test_flake.name}/machines/machine1/schema", - json={"clanImports": ["fake-module"]}, + f"/api/{test_flake_with_core.name}/schema", + json={"clanImports": ["diskLayouts"]}, ) - # expect the result schema to contain the fake-module.fake-flag option + # expect the result schema to contain the deltachat option assert response.status_code == 200 assert ( - response.json()["schema"]["properties"]["fake-module"]["properties"][ - "fake-flag" - ]["type"] - == "boolean" + response.json()["schema"]["properties"]["diskLayouts"]["properties"][ + "singleDiskExt4" + ]["properties"]["device"]["type"] + == "string" ) # new config importing an extra clanModule (clanModules.fake-module) @@ -174,14 +185,14 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None: # set the fake-module.fake-flag option to true response = api.put( - f"/api/{test_flake.name}/machines/machine1/config", + f"/api/{test_flake_with_core.name}/machines/machine1/config", json=config_with_imports, ) assert response.status_code == 200 # ensure the config has been applied response = api.get( - f"/api/{test_flake.name}/machines/machine1/config", + f"/api/{test_flake_with_core.name}/machines/machine1/config", ) assert response.status_code == 200 assert response.json() == { @@ -200,14 +211,14 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None: **fs_config, ) response = api.put( - f"/api/{test_flake.name}/machines/machine1/config", + f"/api/{test_flake_with_core.name}/machines/machine1/config", json=config_with_empty_imports, ) assert response.status_code == 200 # ensure the config has been applied response = api.get( - f"/api/{test_flake.name}/machines/machine1/config", + f"/api/{test_flake_with_core.name}/machines/machine1/config", ) assert response.status_code == 200 assert response.json() == { diff --git a/pkgs/clan-cli/tests/test_machines_config.py b/pkgs/clan-cli/tests/test_machines_config.py index ee07ad017..2ce8980d8 100644 --- a/pkgs/clan-cli/tests/test_machines_config.py +++ b/pkgs/clan-cli/tests/test_machines_config.py @@ -1,8 +1,10 @@ +import pytest from fixtures_flakes import FlakeForTest -from clan_cli.config import machine +from clan_cli.config.schema import machine_schema -def test_schema_for_machine(test_flake: FlakeForTest) -> None: - schema = machine.schema_for_machine(test_flake.name, "machine1") +@pytest.mark.with_core +def test_schema_for_machine(test_flake_with_core: FlakeForTest) -> None: + schema = machine_schema(test_flake_with_core.name, config={}) assert "properties" in schema diff --git a/pkgs/ui/src/components/createMachineForm/clanModules.tsx b/pkgs/ui/src/components/createMachineForm/clanModules.tsx index e892f5177..e069beafb 100644 --- a/pkgs/ui/src/components/createMachineForm/clanModules.tsx +++ b/pkgs/ui/src/components/createMachineForm/clanModules.tsx @@ -1,4 +1,4 @@ -import { setMachineSchema } from "@/api/machine/machine"; +import { getMachineSchema } from "@/api/machine/machine"; import { useListClanModules } from "@/api/modules/modules"; import { Alert, AlertTitle, FormHelperText, Typography } from "@mui/material"; import Box from "@mui/material/Box"; @@ -32,7 +32,7 @@ export default function ClanModules(props: ClanModulesProps) { const selectedModules = formHooks.watch("modules"); useEffect(() => { - setMachineSchema(clanName, "example_machine", { + getMachineSchema(clanName, { imports: [], }).then((response) => { if (response.statusText == "OK") { @@ -52,7 +52,7 @@ export default function ClanModules(props: ClanModulesProps) { } = event; const newValue = typeof value === "string" ? value.split(",") : value; formHooks.setValue("modules", newValue); - setMachineSchema(clanName, "example_machine", { + getMachineSchema(clanName, { imports: newValue, }) .then((response) => {