Merge pull request 'Inventory: init: deployment info for machines' (#1767) from hsjobeki/clan-core:hsjobeki-main into main

flakeModules/clan.nix

@@ -120,8 +120,8 @@ in
         machines
         pkgsForSystem
         meta
+        inventory
         ;
-      inventory = (lib.traceValSeq cfg.inventory);
     };
   };
   _file = __curPos.file;

inventory.json

@@ -1,77 +1,99 @@
 {
   "meta": {
-    "name": "clan-core"
+    "name": "clan-core",
+    "description": null,
+    "icon": null
   },
   "machines": {
-    "minimal-inventory-machine": {
+    "test-inventory-machine": {
       "name": "foo",
-      "system": "x86_64-linux",
+      "deploy": {
+        "targetHost": null
+      },
       "description": "A nice thing",
       "icon": "./path/to/icon.png",
-      "tags": ["1", "2", "3"]
+      "tags": ["1", "2", "3"],
+      "system": "x86_64-linux"
     }
   },
   "services": {
     "packages": {
       "editors": {
         "meta": {
-          "name": "Some editor packages"
+          "name": "Some editor packages",
+          "description": null,
+          "icon": null
         },
         "roles": {
           "default": {
-            "machines": ["minimal-inventory-machine"],
             "config": {
               "packages": ["vim"]
-            }
+            },
+            "imports": [],
+            "machines": ["test-inventory-machine"],
+            "tags": []
           }
         },
+        "config": {},
+        "imports": [],
         "machines": {
-          "minimal-inventory-machine": {
+          "test-inventory-machine": {
             "config": {
               "packages": ["zed-editor"]
-            }
-          }
             },
-        "config": {
+            "imports": []
-          "packages": ["vim"]
+          }
         }
       },
       "browsing": {
         "meta": {
-          "name": "Web browsing packages"
+          "name": "Web browsing packages",
+          "description": null,
+          "icon": null
         },
         "roles": {
           "default": {
-            "machines": ["minimal-inventory-machine"]
+            "config": {},
+            "imports": [],
+            "machines": ["test-inventory-machine"],
+            "tags": []
           }
         },
+        "config": {},
+        "imports": [],
         "machines": {
-          "minimal-inventory-machine": {
+          "test-inventory-machine": {
             "config": {
               "packages": ["chromium"]
-            }
-          }
             },
-        "config": {
+            "imports": []
-          "packages": ["firefox"]
+          }
         }
       }
     },
     "single-disk": {
       "default": {
         "meta": {
-          "name": "single-disk"
+          "name": "single-disk",
+          "description": null,
+          "icon": null
         },
         "roles": {
           "default": {
-            "machines": ["minimal-inventory-machine"]
+            "config": {},
+            "imports": [],
+            "machines": ["test-inventory-machine"],
+            "tags": []
           }
         },
+        "config": {},
+        "imports": [],
         "machines": {
-          "minimal-inventory-machine": {
+          "test-inventory-machine": {
             "config": {
               "device": "/dev/null"
-            }
+            },
+            "imports": []
           }
         }
       }

lib/build-clan/default.nix

@@ -30,6 +30,15 @@ let
   # - Machines that exist in the machines directory
   # Checks on the module level:
   # - Each service role must reference a valid machine after all machines are merged
+
+  clanToInventory =
+    config:
+    { clanPath, inventoryPath }:
+    let
+      v = lib.attrByPath clanPath null config;
+    in
+    lib.optionalAttrs (v != null) (lib.setAttrByPath inventoryPath v);
+
   mergedInventory =
     (lib.evalModules {
       modules = [
@@ -63,16 +72,36 @@ let
                   "name"
                 ] name config
               );
-              tags = lib.attrByPath [
+            }
+            # tags
+            // (clanToInventory config {
+              clanPath = [
                 "clan"
                 "tags"
-              ] [ ] config;
+              ];
-
+              inventoryPath = [ "tags" ];
-              system = lib.attrByPath [
+            })
+            # system
+            // (clanToInventory config {
+              clanPath = [
                 "nixpkgs"
                 "hostSystem"
-              ] null config;
+              ];
-            }
+              inventoryPath = [ "system" ];
+            })
+            # deploy.targetHost
+            // (clanToInventory config {
+              clanPath = [
+                "clan"
+                "core"
+                "networking"
+                "targetHost"
+              ];
+              inventoryPath = [
+                "deploy"
+                "targetHost"
+              ];
+            })
           ) machines;
         }
 

lib/inventory/build-inventory/default.nix

@@ -24,7 +24,7 @@ let
             availableTags = lib.foldlAttrs (
               acc: _: v:
               v.tags or [ ] ++ acc
-            ) [ ] (lib.traceValSeq inventory.machines);
+            ) [ ] (inventory.machines);
 
             tagMembers = builtins.attrNames (
               lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) inventory.machines
@@ -148,6 +148,9 @@ let
         (lib.optionalAttrs (machineConfig.system or null != null) {
           config.nixpkgs.hostPlatform = machineConfig.system;
         })
+        (lib.optionalAttrs (machineConfig.deploy.targetHost or null != null) {
+          config.clan.core.networking.targetHost = machineConfig.deploy.targetHost;
+        })
       ]
     ) inventory.machines or { };
 in

lib/inventory/build-inventory/interface.nix

@@ -49,6 +49,17 @@ in
               default = null;
               type = types.nullOr types.str;
             };
+            deploy = lib.mkOption {
+              default = { };
+              type = types.submodule {
+                options = {
+                  targetHost = lib.mkOption {
+                    default = null;
+                    type = types.nullOr types.str;
+                  };
+                };
+              };
+            };
           };
         }
       );

lib/inventory/flake-module.nix

@@ -23,10 +23,19 @@ in
       };
 
       getSchema = import ./interface-to-schema.nix { inherit lib self; };
+
+      # The schema for the inventory, without default values, from the module system.
+      # This is better suited for human reading and for generating code.
+      bareSchema = getSchema { includeDefaults = false; };
+      # The schema for the inventory with default values, from the module system.
+      # This is better suited for validation, since default values are included.
+      fullSchema = getSchema { };
     in
     {
-      legacyPackages.inventorySchema = getSchema { };
+      legacyPackages.inventory = {
-      legacyPackages.inventorySchemaPretty = getSchema { includeDefaults = false; };
+        inherit fullSchema;
+        inherit bareSchema;
+      };
 
       devShells.inventory-schema = pkgs.mkShell {
         inputsFrom = with config.checks; [
@@ -42,7 +51,7 @@ in
         buildInputs = [ pkgs.cue ];
         src = ./.;
         buildPhase = ''
-          export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON self'.legacyPackages.inventorySchema)}
+          export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON fullSchema.schemaWithModules)}
           cp $SCHEMA schema.json
           cue import -f -p compose -l '#Root:' schema.json
           mkdir $out
@@ -55,7 +64,7 @@ in
         buildInputs = [ pkgs.cue ];
         src = ./.;
         buildPhase = ''
-          export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON self'.legacyPackages.inventorySchemaPretty)}
+          export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON bareSchema.schemaWithModules)}
           cp $SCHEMA schema.json
           cue import -f -p compose -l '#Root:' schema.json
           mkdir $out

lib/inventory/interface-to-schema.nix

@@ -53,7 +53,9 @@ let
           properties = {
             meta =
               inventorySchema.properties.services.additionalProperties.additionalProperties.properties.meta;
-            config = moduleSchema;
+            config = {
+              title = "${moduleName}-config";
+            } // moduleSchema;
             roles = {
               type = "object";
               additionalProperties = false;
@@ -62,14 +64,24 @@ let
                 map (role: {
                   name = role;
                   value =
-                    inventorySchema.properties.services.additionalProperties.additionalProperties.properties.roles.additionalProperties;
+                    lib.recursiveUpdate
+                      inventorySchema.properties.services.additionalProperties.additionalProperties.properties.roles.additionalProperties
+                      {
+                        properties.config = {
+                          title = "${moduleName}-config";
+                        } // moduleSchema;
+                      };
                 }) (rolesOf moduleName)
               );
             };
             machines =
               lib.recursiveUpdate
                 inventorySchema.properties.services.additionalProperties.additionalProperties.properties.machines
-                { additionalProperties.properties.config = moduleSchema; };
+                {
+                  additionalProperties.properties.config = {
+                    title = "${moduleName}-config";
+                  } // moduleSchema;
+                };
           };
         };
       };
@@ -95,4 +107,21 @@ let
       };
     };
 in
-schema
+{
+  /*
+    The abstract inventory without the exact schema for each module filled
+
+    InventorySchema<T extends Any> :: {
+      serviceConfig :: dict[str, T];
+    }
+  */
+  abstractSchema = inventorySchema;
+  /*
+    The inventory with each module schema filled.
+
+    InventorySchema<T extends ModuleSchema> :: {
+      ${serviceConfig} :: T; # <- each concrete module name is filled
+    }
+  */
+  schemaWithModules = schema;
+}

lib/inventory/tests/default.nix

@@ -85,9 +85,9 @@
         not_used_machine = builtins.length configs.not_used_machine;
       };
       expected = {
-        client_1_machine = 3;
+        client_1_machine = 4;
-        client_2_machine = 3;
+        client_2_machine = 4;
-        not_used_machine = 1;
+        not_used_machine = 2;
       };
     };
 

pkgs/clan-cli/clan_cli/inventory/__init__.py

@@ -1,3 +1,5 @@
+# ruff: noqa: N815
+# ruff: noqa: N806
 import json
 from dataclasses import asdict, dataclass, field, is_dataclass
 from pathlib import Path
@@ -35,6 +37,15 @@ def dataclass_to_dict(obj: Any) -> Any:
         return obj
 
 
+@dataclass
+class DeploymentInfo:
+    """
+    Deployment information for a machine.
+    """
+
+    targetHost: str | None = None
+
+
 @dataclass
 class Machine:
     """
@@ -49,19 +60,29 @@ class Machine:
     """
 
     name: str
-    system: Literal["x86_64-linux"] | str | None = None
+    deploy: DeploymentInfo = field(default_factory=DeploymentInfo)
     description: str | None = None
     icon: str | None = None
     tags: list[str] = field(default_factory=list)
+    system: Literal["x86_64-linux"] | str | None = None
 
     @staticmethod
-    def from_dict(d: dict[str, Any]) -> "Machine":
+    def from_dict(data: dict[str, Any]) -> "Machine":
-        return Machine(**d)
+        targetHost = data.get("deploy", {}).get("targetHost", None)
+        return Machine(
+            name=data["name"],
+            description=data.get("description", None),
+            icon=data.get("icon", None),
+            tags=data.get("tags", []),
+            system=data.get("system", None),
+            deploy=DeploymentInfo(targetHost),
+        )
 
 
 @dataclass
 class MachineServiceConfig:
-    config: dict[str, Any] | None = None
+    config: dict[str, Any] = field(default_factory=dict)
+    imports: list[str] = field(default_factory=list)
 
 
 @dataclass
@@ -73,6 +94,8 @@ class ServiceMeta:
 
 @dataclass
 class Role:
+    config: dict[str, Any] = field(default_factory=dict)
+    imports: list[str] = field(default_factory=list)
     machines: list[str] = field(default_factory=list)
     tags: list[str] = field(default_factory=list)
 
@@ -81,6 +104,8 @@ class Role:
 class Service:
     meta: ServiceMeta
     roles: dict[str, Role]
+    config: dict[str, Any] = field(default_factory=dict)
+    imports: list[str] = field(default_factory=list)
     machines: dict[str, MachineServiceConfig] = field(default_factory=dict)
 
     @staticmethod
@@ -96,6 +121,8 @@ class Service:
                 if d.get("machines")
                 else {}
             ),
+            config=d.get("config", {}),
+            imports=d.get("imports", []),
         )
 
 

pkgs/clan-cli/tests/fixtures_flakes.py

@@ -32,6 +32,10 @@ def substitute(
             line = line.replace(
                 "git+https://git.clan.lol/clan/clan-core", str(clan_core_flake)
             )
+            line = line.replace(
+                "https://git.clan.lol/clan/clan-core/archive/main.tar.gz",
+                str(clan_core_flake),
+            )
         line = line.replace("__CLAN_SOPS_KEY_PATH__", sops_key)
         line = line.replace("__CLAN_SOPS_KEY_DIR__", str(flake))
         print(line, end="")

pkgs/clan-cli/tests/test_create_flake.py

@@ -3,6 +3,7 @@ import subprocess
 from pathlib import Path
 
 import pytest
+from fixtures_flakes import substitute
 from helpers import cli
 
 
@@ -17,7 +18,15 @@ def test_create_flake(
 
     url = f"{clan_core}#default"
     cli.run(["flakes", "create", str(flake_dir), f"--url={url}"])
+
     assert (flake_dir / ".clan-flake").exists()
+
+    # Replace the inputs.clan.url in the template flake.nix
+    substitute(
+        flake_dir / "flake.nix",
+        clan_core,
+    )
+
     monkeypatch.chdir(flake_dir)
     cli.run(["machines", "create", "machine1"])
     capsys.readouterr()  # flush cache
@@ -55,6 +64,13 @@ def test_ui_template(
     flake_dir = temporary_home / "test-flake"
     url = f"{clan_core}#minimal"
     cli.run(["flakes", "create", str(flake_dir), f"--url={url}"])
+
+    # Replace the inputs.clan.url in the template flake.nix
+    substitute(
+        flake_dir / "flake.nix",
+        clan_core,
+    )
+
     monkeypatch.chdir(flake_dir)
     cli.run(["machines", "create", "machine1"])
     capsys.readouterr()  # flush cache