Merge pull request 'Inventory: init: deployment info for machines' (#1767) from hsjobeki/clan-core:hsjobeki-main into main

2024-07-17 10:18:50 +00:00
parent 6c7a2ae969 2a0acb28d3
commit 483425d6a3
11 changed files with 196 additions and 46 deletions
--- a/flakeModules/clan.nix
+++ b/flakeModules/clan.nix
@@ -120,8 +120,8 @@ in
        machines
        pkgsForSystem
        meta
+        inventory
        ;
-      inventory = (lib.traceValSeq cfg.inventory);
    };
  };
  _file = __curPos.file;
--- a/inventory.json
+++ b/inventory.json
@@ -1,77 +1,99 @@
 {
  "meta": {
-    "name": "clan-core"
+    "name": "clan-core",
+    "description": null,
+    "icon": null
  },
  "machines": {
-    "minimal-inventory-machine": {
+    "test-inventory-machine": {
      "name": "foo",
-      "system": "x86_64-linux",
+      "deploy": {
+        "targetHost": null
+      },
      "description": "A nice thing",
      "icon": "./path/to/icon.png",
-      "tags": ["1", "2", "3"]
+      "tags": ["1", "2", "3"],
+      "system": "x86_64-linux"
    }
  },
  "services": {
    "packages": {
      "editors": {
        "meta": {
-          "name": "Some editor packages"
+          "name": "Some editor packages",
+          "description": null,
+          "icon": null
        },
        "roles": {
          "default": {
-            "machines": ["minimal-inventory-machine"],
            "config": {
              "packages": ["vim"]
-            }
+            },
+            "imports": [],
+            "machines": ["test-inventory-machine"],
+            "tags": []
          }
        },
+        "config": {},
+        "imports": [],
        "machines": {
-          "minimal-inventory-machine": {
+          "test-inventory-machine": {
            "config": {
              "packages": ["zed-editor"]
-            }
-          }
            },
-        "config": {
-          "packages": ["vim"]
+            "imports": []
+          }
        }
      },
      "browsing": {
        "meta": {
-          "name": "Web browsing packages"
+          "name": "Web browsing packages",
+          "description": null,
+          "icon": null
        },
        "roles": {
          "default": {
-            "machines": ["minimal-inventory-machine"]
+            "config": {},
+            "imports": [],
+            "machines": ["test-inventory-machine"],
+            "tags": []
          }
        },
+        "config": {},
+        "imports": [],
        "machines": {
-          "minimal-inventory-machine": {
+          "test-inventory-machine": {
            "config": {
              "packages": ["chromium"]
-            }
-          }
            },
-        "config": {
-          "packages": ["firefox"]
+            "imports": []
+          }
        }
      }
    },
    "single-disk": {
      "default": {
        "meta": {
-          "name": "single-disk"
+          "name": "single-disk",
+          "description": null,
+          "icon": null
        },
        "roles": {
          "default": {
-            "machines": ["minimal-inventory-machine"]
+            "config": {},
+            "imports": [],
+            "machines": ["test-inventory-machine"],
+            "tags": []
          }
        },
+        "config": {},
+        "imports": [],
        "machines": {
-          "minimal-inventory-machine": {
+          "test-inventory-machine": {
            "config": {
              "device": "/dev/null"
-            }
+            },
+            "imports": []
          }
        }
      }
--- a/lib/build-clan/default.nix
+++ b/lib/build-clan/default.nix
@@ -30,6 +30,15 @@ let
  # - Machines that exist in the machines directory
  # Checks on the module level:
  # - Each service role must reference a valid machine after all machines are merged
+
+  clanToInventory =
+    config:
+    { clanPath, inventoryPath }:
+    let
+      v = lib.attrByPath clanPath null config;
+    in
+    lib.optionalAttrs (v != null) (lib.setAttrByPath inventoryPath v);
+
  mergedInventory =
    (lib.evalModules {
      modules = [
@@ -63,16 +72,36 @@ let
                  "name"
                ] name config
              );
-              tags = lib.attrByPath [
+            }
+            # tags
+            // (clanToInventory config {
+              clanPath = [
                "clan"
                "tags"
-              ] [ ] config;
-
-              system = lib.attrByPath [
+              ];
+              inventoryPath = [ "tags" ];
+            })
+            # system
+            // (clanToInventory config {
+              clanPath = [
                "nixpkgs"
                "hostSystem"
-              ] null config;
-            }
+              ];
+              inventoryPath = [ "system" ];
+            })
+            # deploy.targetHost
+            // (clanToInventory config {
+              clanPath = [
+                "clan"
+                "core"
+                "networking"
+                "targetHost"
+              ];
+              inventoryPath = [
+                "deploy"
+                "targetHost"
+              ];
+            })
          ) machines;
        }

--- a/lib/inventory/build-inventory/default.nix
+++ b/lib/inventory/build-inventory/default.nix
@@ -24,7 +24,7 @@ let
            availableTags = lib.foldlAttrs (
              acc: _: v:
              v.tags or [ ] ++ acc
-            ) [ ] (lib.traceValSeq inventory.machines);
+            ) [ ] (inventory.machines);

            tagMembers = builtins.attrNames (
              lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) inventory.machines
@@ -148,6 +148,9 @@ let
        (lib.optionalAttrs (machineConfig.system or null != null) {
          config.nixpkgs.hostPlatform = machineConfig.system;
        })
+        (lib.optionalAttrs (machineConfig.deploy.targetHost or null != null) {
+          config.clan.core.networking.targetHost = machineConfig.deploy.targetHost;
+        })
      ]
    ) inventory.machines or { };
 in
--- a/lib/inventory/build-inventory/interface.nix
+++ b/lib/inventory/build-inventory/interface.nix
@@ -49,6 +49,17 @@ in
              default = null;
              type = types.nullOr types.str;
            };
+            deploy = lib.mkOption {
+              default = { };
+              type = types.submodule {
+                options = {
+                  targetHost = lib.mkOption {
+                    default = null;
+                    type = types.nullOr types.str;
+                  };
+                };
+              };
+            };
          };
        }
      );
--- a/lib/inventory/flake-module.nix
+++ b/lib/inventory/flake-module.nix
@@ -23,10 +23,19 @@ in
      };

      getSchema = import ./interface-to-schema.nix { inherit lib self; };
+
+      # The schema for the inventory, without default values, from the module system.
+      # This is better suited for human reading and for generating code.
+      bareSchema = getSchema { includeDefaults = false; };
+      # The schema for the inventory with default values, from the module system.
+      # This is better suited for validation, since default values are included.
+      fullSchema = getSchema { };
    in
    {
-      legacyPackages.inventorySchema = getSchema { };
-      legacyPackages.inventorySchemaPretty = getSchema { includeDefaults = false; };
+      legacyPackages.inventory = {
+        inherit fullSchema;
+        inherit bareSchema;
+      };

      devShells.inventory-schema = pkgs.mkShell {
        inputsFrom = with config.checks; [
@@ -42,7 +51,7 @@ in
        buildInputs = [ pkgs.cue ];
        src = ./.;
        buildPhase = ''
-          export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON self'.legacyPackages.inventorySchema)}
+          export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON fullSchema.schemaWithModules)}
          cp $SCHEMA schema.json
          cue import -f -p compose -l '#Root:' schema.json
          mkdir $out
@@ -55,7 +64,7 @@ in
        buildInputs = [ pkgs.cue ];
        src = ./.;
        buildPhase = ''
-          export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON self'.legacyPackages.inventorySchemaPretty)}
+          export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON bareSchema.schemaWithModules)}
          cp $SCHEMA schema.json
          cue import -f -p compose -l '#Root:' schema.json
          mkdir $out
--- a/lib/inventory/interface-to-schema.nix
+++ b/lib/inventory/interface-to-schema.nix
@@ -53,7 +53,9 @@ let
          properties = {
            meta =
              inventorySchema.properties.services.additionalProperties.additionalProperties.properties.meta;
-            config = moduleSchema;
+            config = {
+              title = "${moduleName}-config";
+            } // moduleSchema;
            roles = {
              type = "object";
              additionalProperties = false;
@@ -62,14 +64,24 @@ let
                map (role: {
                  name = role;
                  value =
-                    inventorySchema.properties.services.additionalProperties.additionalProperties.properties.roles.additionalProperties;
+                    lib.recursiveUpdate
+                      inventorySchema.properties.services.additionalProperties.additionalProperties.properties.roles.additionalProperties
+                      {
+                        properties.config = {
+                          title = "${moduleName}-config";
+                        } // moduleSchema;
+                      };
                }) (rolesOf moduleName)
              );
            };
            machines =
              lib.recursiveUpdate
                inventorySchema.properties.services.additionalProperties.additionalProperties.properties.machines
-                { additionalProperties.properties.config = moduleSchema; };
+                {
+                  additionalProperties.properties.config = {
+                    title = "${moduleName}-config";
+                  } // moduleSchema;
+                };
          };
        };
      };
@@ -95,4 +107,21 @@ let
      };
    };
 in
-schema
+{
+  /*
+    The abstract inventory without the exact schema for each module filled
+
+    InventorySchema<T extends Any> :: {
+      serviceConfig :: dict[str, T];
+    }
+  */
+  abstractSchema = inventorySchema;
+  /*
+    The inventory with each module schema filled.
+
+    InventorySchema<T extends ModuleSchema> :: {
+      ${serviceConfig} :: T; # <- each concrete module name is filled
+    }
+  */
+  schemaWithModules = schema;
+}
--- a/lib/inventory/tests/default.nix
+++ b/lib/inventory/tests/default.nix
@@ -85,9 +85,9 @@
        not_used_machine = builtins.length configs.not_used_machine;
      };
      expected = {
-        client_1_machine = 3;
-        client_2_machine = 3;
-        not_used_machine = 1;
+        client_1_machine = 4;
+        client_2_machine = 4;
+        not_used_machine = 2;
      };
    };

--- a/pkgs/clan-cli/clan_cli/inventory/init.py
+++ b/pkgs/clan-cli/clan_cli/inventory/init.py
@@ -1,3 +1,5 @@
+# ruff: noqa: N815
+# ruff: noqa: N806
 import json
 from dataclasses import asdict, dataclass, field, is_dataclass
 from pathlib import Path
@@ -35,6 +37,15 @@ def dataclass_to_dict(obj: Any) -> Any:
        return obj


+@dataclass
+class DeploymentInfo:
+    """
+    Deployment information for a machine.
+    """
+
+    targetHost: str | None = None
+
+
@dataclass
 class Machine:
    """
@@ -49,19 +60,29 @@ class Machine:
    """

    name: str
-    system: Literal["x86_64-linux"] | str | None = None
+    deploy: DeploymentInfo = field(default_factory=DeploymentInfo)
    description: str | None = None
    icon: str | None = None
    tags: list[str] = field(default_factory=list)
+    system: Literal["x86_64-linux"] | str | None = None

    @staticmethod
-    def from_dict(d: dict[str, Any]) -> "Machine":
-        return Machine(**d)
+    def from_dict(data: dict[str, Any]) -> "Machine":
+        targetHost = data.get("deploy", {}).get("targetHost", None)
+        return Machine(
+            name=data["name"],
+            description=data.get("description", None),
+            icon=data.get("icon", None),
+            tags=data.get("tags", []),
+            system=data.get("system", None),
+            deploy=DeploymentInfo(targetHost),
+        )


@dataclass
 class MachineServiceConfig:
-    config: dict[str, Any] | None = None
+    config: dict[str, Any] = field(default_factory=dict)
+    imports: list[str] = field(default_factory=list)


@dataclass
@@ -73,6 +94,8 @@ class ServiceMeta:

@dataclass
 class Role:
+    config: dict[str, Any] = field(default_factory=dict)
+    imports: list[str] = field(default_factory=list)
    machines: list[str] = field(default_factory=list)
    tags: list[str] = field(default_factory=list)

@@ -81,6 +104,8 @@ class Role:
 class Service:
    meta: ServiceMeta
    roles: dict[str, Role]
+    config: dict[str, Any] = field(default_factory=dict)
+    imports: list[str] = field(default_factory=list)
    machines: dict[str, MachineServiceConfig] = field(default_factory=dict)

    @staticmethod
@@ -96,6 +121,8 @@ class Service:
                if d.get("machines")
                else {}
            ),
+            config=d.get("config", {}),
+            imports=d.get("imports", []),
        )


--- a/pkgs/clan-cli/tests/fixtures_flakes.py
+++ b/pkgs/clan-cli/tests/fixtures_flakes.py
@@ -32,6 +32,10 @@ def substitute(
            line = line.replace(
                "git+https://git.clan.lol/clan/clan-core", str(clan_core_flake)
            )
+            line = line.replace(
+                "https://git.clan.lol/clan/clan-core/archive/main.tar.gz",
+                str(clan_core_flake),
+            )
        line = line.replace("__CLAN_SOPS_KEY_PATH__", sops_key)
        line = line.replace("__CLAN_SOPS_KEY_DIR__", str(flake))
        print(line, end="")
--- a/pkgs/clan-cli/tests/test_create_flake.py
+++ b/pkgs/clan-cli/tests/test_create_flake.py
@@ -3,6 +3,7 @@ import subprocess
 from pathlib import Path

 import pytest
+from fixtures_flakes import substitute
 from helpers import cli


@@ -17,7 +18,15 @@ def test_create_flake(

    url = f"{clan_core}#default"
    cli.run(["flakes", "create", str(flake_dir), f"--url={url}"])
+
    assert (flake_dir / ".clan-flake").exists()
+
+    # Replace the inputs.clan.url in the template flake.nix
+    substitute(
+        flake_dir / "flake.nix",
+        clan_core,
+    )
+
    monkeypatch.chdir(flake_dir)
    cli.run(["machines", "create", "machine1"])
    capsys.readouterr()  # flush cache
@@ -55,6 +64,13 @@ def test_ui_template(
    flake_dir = temporary_home / "test-flake"
    url = f"{clan_core}#minimal"
    cli.run(["flakes", "create", str(flake_dir), f"--url={url}"])
+
+    # Replace the inputs.clan.url in the template flake.nix
+    substitute(
+        flake_dir / "flake.nix",
+        clan_core,
+    )
+
    monkeypatch.chdir(flake_dir)
    cli.run(["machines", "create", "machine1"])
    capsys.readouterr()  # flush cache