api/machines: allow importing extra modules

- add top-level option `clanImports` to clanCore
- clanImports can be set and checked as any other option
- buildClan resolves the clanImports from the settings.json before calling evalModules to prevent infinite recursions
- new endpoint PUT machines/{name}/schema to allow getting the schema for a specific list of imports
- to retrieve the currently imported modules, cimply do a GET or PU on machines/{name}/config which will return `clanImports` as part of the config

Still missing: get list of available modules

lib/build-clan/default.nix

@@ -1,4 +1,4 @@
-{ nixpkgs, self, lib }:
+{ clan-core, nixpkgs, lib }:
 { directory  # The directory containing the machines subdirectory
 , specialArgs ? { } # Extra arguments to pass to nixosSystem i.e. useful to make self available
 , machines ? { } # allows to include machine-specific modules i.e. machines.${name} = { ... }
@@ -7,6 +7,9 @@ let
   machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") (builtins.readDir (directory + /machines));
 
   machineSettings = machineName:
+    # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily
+    # This is useful for doing a dry-run before writing changes into the settings.json
+    # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval
     if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != ""
     then builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE"))
     else
@@ -14,18 +17,33 @@ let
         (builtins.fromJSON
           (builtins.readFile (directory + /machines/${machineName}/settings.json)));
 
+  # Read additional imports specified via a config option in settings.json
+  # This is not an infinite recursion, because the imports are discovered here
+  #   before calling evalModules.
+  # It is still useful to have the imports as an option, as this allows for type
+  #   checking and easy integration with the config frontend(s)
+  machineImports = machineSettings:
+    map
+      (module: clan-core.clanModules.${module})
+      (machineSettings.clanImports or [ ]);
+
   # TODO: remove default system once we have a hardware-config mechanism
   nixosConfiguration = { system ? "x86_64-linux", name }: nixpkgs.lib.nixosSystem {
-    modules = [
+    modules =
-      self.nixosModules.clanCore
+      let
-      (machineSettings name)
+        settings = machineSettings name;
-      (machines.${name} or { })
+      in
-      {
+      (machineImports settings)
-        clanCore.machineName = name;
+      ++ [
-        clanCore.clanDir = directory;
+        settings
-        nixpkgs.hostPlatform = lib.mkForce system;
+        clan-core.nixosModules.clanCore
-      }
+        (machines.${name} or { })
-    ];
+        {
+          clanCore.machineName = name;
+          clanCore.clanDir = directory;
+          nixpkgs.hostPlatform = lib.mkForce system;
+        }
+      ];
     inherit specialArgs;
   };
 

lib/default.nix

@@ -1,6 +1,6 @@
-{ lib, self, nixpkgs, ... }:
+{ lib, clan-core, nixpkgs, ... }:
 {
   jsonschema = import ./jsonschema { inherit lib; };
 
-  buildClan = import ./build-clan { inherit lib self nixpkgs; };
+  buildClan = import ./build-clan { inherit clan-core lib nixpkgs; };
 }

lib/flake-module.nix

@@ -8,7 +8,7 @@
   ];
   flake.lib = import ./default.nix {
     inherit lib;
-    inherit self;
     inherit (inputs) nixpkgs;
+    clan-core = self;
   };
 }

nixosModules/clanCore/flake-module.nix

@@ -1,6 +1,7 @@
 { self, inputs, lib, ... }: {
   flake.nixosModules.clanCore = { config, pkgs, options, ... }: {
     imports = [
+      ../clanImports
       ./secrets
       ./zerotier
       ./networking.nix
@@ -34,6 +35,11 @@
         internal = true;
       };
     };
+    # TODO: factor these out into a separate interface.nix.
+    # Also think about moving these options out of `system.clan`.
+    # Maybe we should not re-use the already polluted confg.system namespace
+    #   and instead have a separate top-level namespace like `clanOutputs`, with
+    #   well defined options marked as `internal = true;`.
     options.system.clan = lib.mkOption {
       type = lib.types.submodule {
         options = {

nixosModules/clanImports/default.nix

@@ -0,0 +1,16 @@
+{ lib
+, ...
+}: {
+  /*
+    Declaring imports inside the module system does not trigger an infinite
+    recursion in this case because buildClan generates the imports from the
+    settings.json file before calling out to evalModules.
+  */
+  options.clanImports = lib.mkOption {
+    type = lib.types.listOf lib.types.str;
+    description = ''
+      A list of imported module names imported from clan-core.clanModules.<name>
+      The buildClan function will automatically import these modules for the current machine.
+    '';
+  };
+}

pkgs/clan-cli/clan_cli/config/machine.py

@@ -45,9 +45,9 @@ def verify_machine_config(
             cwd=flake,
             env=env,
         )
-        if proc.returncode != 0:
+    if proc.returncode != 0:
-            return proc.stderr
+        return proc.stderr
-        return None
+    return None
 
 
 def config_for_machine(machine_name: str) -> dict:
@@ -85,32 +85,47 @@ def set_config_for_machine(machine_name: str, config: dict) -> Optional[str]:
     return None
 
 
-def schema_for_machine(machine_name: str, flake: Optional[Path] = None) -> dict:
+def schema_for_machine(
+    machine_name: str, config: Optional[dict] = None, flake: Optional[Path] = None
+) -> dict:
     if flake is None:
         flake = get_clan_flake_toplevel()
-    # use nix eval to lib.evalModules .#nixosModules.machine-{machine_name}
+    # use nix eval to lib.evalModules .#nixosConfigurations.<machine_name>.options.clan
-    proc = subprocess.run(
+    with NamedTemporaryFile(mode="w") as clan_machine_settings_file:
-        nix_eval(
+        env = os.environ.copy()
-            flags=[
+        inject_config_flags = []
-                "--impure",
+        if config is not None:
-                "--show-trace",
+            json.dump(config, clan_machine_settings_file, indent=2)
-                "--expr",
+            clan_machine_settings_file.seek(0)
-                f"""
+            env["CLAN_MACHINE_SETTINGS_FILE"] = clan_machine_settings_file.name
-                let
+            inject_config_flags = [
-                    flake = builtins.getFlake (toString {flake});
+                "--impure",  # needed to access CLAN_MACHINE_SETTINGS_FILE
-                    lib = import {nixpkgs_source()}/lib;
+            ]
-                    options = flake.nixosConfigurations.{machine_name}.options;
+        proc = subprocess.run(
-                    clanOptions = options.clan;
+            nix_eval(
-                    jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }};
+                flags=inject_config_flags
-                    jsonschema = jsonschemaLib.parseOptions clanOptions;
+                + [
-                in
+                    "--impure",
-                    jsonschema
+                    "--show-trace",
-                """,
+                    "--expr",
-            ],
+                    f"""
-        ),
+                    let
-        capture_output=True,
+                        flake = builtins.getFlake (toString {flake});
-        text=True,
+                        lib = import {nixpkgs_source()}/lib;
-    )
+                        options = flake.nixosConfigurations.{machine_name}.options;
+                        clanOptions = options.clan;
+                        jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }};
+                        jsonschema = jsonschemaLib.parseOptions clanOptions;
+                    in
+                        jsonschema
+                    """,
+                ],
+            ),
+            capture_output=True,
+            text=True,
+            cwd=flake,
+            env=env,
+        )
     if proc.returncode != 0:
         print(proc.stderr, file=sys.stderr)
         raise Exception(

pkgs/clan-cli/clan_cli/webui/routers/machines.py

@@ -69,6 +69,14 @@ async def get_machine_schema(name: str) -> SchemaResponse:
     return SchemaResponse(schema=schema)
 
 
+@router.put("/api/machines/{name}/schema")
+async def set_machine_schema(
+    name: str, config: Annotated[dict, Body()]
+) -> SchemaResponse:
+    schema = schema_for_machine(name, config)
+    return SchemaResponse(schema=schema)
+
+
 @router.get("/api/machines/{name}/verify")
 async def put_verify_machine_config(name: str) -> VerifyMachineResponse:
     error = verify_machine_config(name)

pkgs/clan-cli/tests/test_flake/fake-module.nix

@@ -0,0 +1,11 @@
+{ lib
+, ...
+}: {
+  options.clan.fake-module.fake-flag = lib.mkOption {
+    type = lib.types.bool;
+    default = false;
+    description = ''
+      A useless fake flag fro testing purposes.
+    '';
+  };
+}

pkgs/clan-cli/tests/test_flake/flake.nix

@@ -2,36 +2,57 @@
   # this placeholder is replaced by the path to nixpkgs
   inputs.nixpkgs.url = "__NIXPKGS__";
 
-  outputs = inputs: {
+  outputs = inputs':
-    nixosConfigurations.machine1 = inputs.nixpkgs.lib.nixosSystem {
+    let
-      modules = [
+      # fake clan-core input
-        ./nixosModules/machine1.nix
+      fake-clan-core = {
-        (
+        clanModules.fake-module = ./fake-module.nix;
-          if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != ""
+      };
-          then builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE"))
+      inputs = inputs' // { clan-core = fake-clan-core; };
-          else if builtins.pathExists ./machines/machine1/settings.json
+      machineSettings = (
-          then builtins.fromJSON (builtins.readFile ./machines/machine1/settings.json)
+        if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != ""
-          else { }
+        then builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE"))
-        )
+        else if builtins.pathExists ./machines/machine1/settings.json
-        ({ lib, options, pkgs, ... }: {
+        then builtins.fromJSON (builtins.readFile ./machines/machine1/settings.json)
-          config = {
+        else { }
-            nixpkgs.hostPlatform = "x86_64-linux";
+      );
-            # speed up by not instantiating nixpkgs twice and disable documentation
+      machineImports =
-            nixpkgs.pkgs = inputs.nixpkgs.legacyPackages.x86_64-linux;
+        map
-            documentation.enable = false;
+          (module: fake-clan-core.clanModules.${module})
-          };
+          (machineSettings.clanImports or [ ]);
-          options.clanCore.optionsNix = lib.mkOption {
+    in
-            type = lib.types.raw;
+    {
-            internal = true;
+      nixosConfigurations.machine1 = inputs.nixpkgs.lib.nixosSystem {
-            readOnly = true;
+        modules =
-            default = (pkgs.nixosOptionsDoc { inherit options; }).optionsNix;
+          machineImports ++ [
-            defaultText = "optionsNix";
+            ./nixosModules/machine1.nix
-            description = ''
+            machineSettings
-              This is to export nixos options used for `clan config`
+            ({ lib, options, pkgs, ... }: {
-            '';
+              config = {
-          };
+                nixpkgs.hostPlatform = "x86_64-linux";
-        })
+                # speed up by not instantiating nixpkgs twice and disable documentation
-      ];
+                nixpkgs.pkgs = inputs.nixpkgs.legacyPackages.x86_64-linux;
+                documentation.enable = false;
+              };
+              options.clanCore.optionsNix = lib.mkOption {
+                type = lib.types.raw;
+                internal = true;
+                readOnly = true;
+                default = (pkgs.nixosOptionsDoc { inherit options; }).optionsNix;
+                defaultText = "optionsNix";
+                description = ''
+                  This is to export nixos options used for `clan config`
+                '';
+              };
+              options.clanImports = lib.mkOption {
+                type = lib.types.listOf lib.types.str;
+                description = ''
+                  A list of imported module names imported from clan-core.clanModules.<name>
+                  The buildClan function will automatically import these modules for the current machine.
+                '';
+              };
+            })
+          ];
+      };
     };
-  };
 }

pkgs/clan-cli/tests/test_machines_api.py

@@ -68,20 +68,13 @@ def test_configure_machine(api: TestClient, test_flake: Path) -> None:
     assert response.status_code == 200
     assert response.json() == {"config": {}}
 
-    # set some valid config
+    fs_config = dict(
-    config2 = dict(
-        clan=dict(
-            jitsi=dict(
-                enable=True,
-            ),
-        ),
         fileSystems={
             "/": dict(
                 device="/dev/fake_disk",
                 fsType="ext4",
             ),
         },
-        # set boot.loader.grub.devices
         boot=dict(
             loader=dict(
                 grub=dict(
@@ -90,6 +83,16 @@ def test_configure_machine(api: TestClient, test_flake: Path) -> None:
             ),
         ),
     )
+
+    # set some valid config
+    config2 = dict(
+        clan=dict(
+            jitsi=dict(
+                enable=True,
+            ),
+        ),
+        **fs_config,
+    )
     response = api.put(
         "/api/machines/machine1/config",
         json=config2,
@@ -116,3 +119,58 @@ def test_configure_machine(api: TestClient, test_flake: Path) -> None:
     response = api.get("/api/machines/machine1/verify")
     assert response.status_code == 200
     assert response.json() == {"success": True, "error": None}
+
+    # get the schema with an extra module imported
+    response = api.put(
+        "/api/machines/machine1/schema",
+        json={"clanImports": ["fake-module"]},
+    )
+    # expect the result schema to contain the fake-module.fake-flag option
+    assert response.status_code == 200
+    assert (
+        response.json()["schema"]["properties"]["fake-module"]["properties"][
+            "fake-flag"
+        ]["type"]
+        == "boolean"
+    )
+
+    # new config importing an extra clanModule (clanModules.fake-module)
+    config_with_imports: dict = {
+        "clanImports": ["fake-module"],
+        "clan": {
+            "fake-module": {
+                "fake-flag": True,
+            },
+        },
+        **fs_config,
+    }
+
+    # set the fake-module.fake-flag option to true
+    response = api.put(
+        "/api/machines/machine1/config",
+        json=config_with_imports,
+    )
+    assert response.status_code == 200
+    assert response.json() == {
+        "config": {
+            "clanImports": ["fake-module"],
+            "clan": {
+                "fake-module": {
+                    "fake-flag": True,
+                },
+            },
+            **fs_config,
+        }
+    }
+
+    # remove the import from the config
+    config_with_empty_imports = dict(
+        clanImports=[],
+        **fs_config,
+    )
+    response = api.put(
+        "/api/machines/machine1/config",
+        json=config_with_empty_imports,
+    )
+    assert response.status_code == 200
+    assert response.json() == {"config": config_with_empty_imports}

pkgs/clan-cli/tests/test_machines_config.py

@@ -4,5 +4,5 @@ from clan_cli.config import machine
 
 
 def test_schema_for_machine(test_flake: Path) -> None:
-    schema = machine.schema_for_machine("machine1", test_flake)
+    schema = machine.schema_for_machine("machine1", flake=test_flake)
     assert "properties" in schema