vars/sops: fix loading of vars from directory structure

nixosModules/clanCore/vars/default.nix

@@ -18,7 +18,7 @@ in
     ./public/in_repo.nix
     # ./public/vm.nix
     ./secret/password-store.nix
-    ./secret/sops.nix
+    ./secret/sops
     # ./secret/vm.nix
   ];
   options.clan.core.vars = lib.mkOption {

nixosModules/clanCore/vars/interface.nix

@@ -72,7 +72,7 @@ in
                         name of the generator
                       '';
                       readOnly = true;
-                      default = generator.name;
+                      default = generator.config._module.args.name;
                     };
                     secret = {
                       description = ''
@@ -87,7 +87,6 @@ in
                         This will be set automatically
                       '';
                       type = str;
-                      readOnly = true;
                     };
                     value = {
                       description = ''
@@ -109,32 +108,35 @@ in
                 For example, a prompt named 'prompt1' will be available via $prompts/prompt1
               '';
               default = { };
-              type = attrsOf (submodule {
+              type = attrsOf (
-                options = options {
+                submodule (prompt: {
-                  description = {
+                  options = options {
-                    description = ''
+                    description = {
-                      The description of the prompted value
+                      description = ''
-                    '';
+                        The description of the prompted value
-                    type = str;
+                      '';
-                    example = "SSH private key";
+                      type = str;
+                      example = "SSH private key";
+                      default = prompt.config._module.args.name;
+                    };
+                    type = {
+                      description = ''
+                        The input type of the prompt.
+                        The following types are available:
+                          - hidden: A hidden text (e.g. password)
+                          - line: A single line of text
+                          - multiline: A multiline text
+                      '';
+                      type = enum [
+                        "hidden"
+                        "line"
+                        "multiline"
+                      ];
+                      default = "line";
+                    };
                   };
-                  type = {
+                })
-                    description = ''
+              );
-                      The input type of the prompt.
-                      The following types are available:
-                        - hidden: A hidden text (e.g. password)
-                        - line: A single line of text
-                        - multiline: A multiline text
-                    '';
-                    type = enum [
-                      "hidden"
-                      "line"
-                      "multiline"
-                    ];
-                    default = "line";
-                  };
-                };
-              });
             };
             runtimeInputs = {
               description = ''

nixosModules/clanCore/vars/public/in_repo.nix

@@ -5,8 +5,9 @@
       {
         publicModule = "clan_cli.vars.public_modules.in_repo";
         fileModule = file: {
-          path =
+          path = lib.mkIf (file.config.secret == false) (
-            config.clan.core.clanDir + "/machines/${config.clan.core.machineName}/vars/${file.config.name}";
+            config.clan.core.clanDir + "/machines/${config.clan.core.machineName}/vars/${file.config.name}"
+          );
         };
       };
 }

nixosModules/clanCore/vars/secret/password-store.nix

@@ -4,7 +4,7 @@
     lib.mkIf (config.clan.core.vars.settings.secretStore == "password-store")
       {
         fileModule = file: {
-          path = lib.mkIf file.secret "${config.clan.core.password-store.targetDirectory}/${config.clan.core.machineName}-${file.config.generatorName}-${file.config.name}";
+          path = lib.mkIf file.config.secret "${config.clan.core.password-store.targetDirectory}/${config.clan.core.machineName}-${file.config.generatorName}-${file.config.name}";
         };
         secretUploadDirectory = lib.mkDefault "/etc/secrets";
         secretModule = "clan_cli.vars.secret_modules.password_store";

nixosModules/clanCore/vars/secret/sops.nix

@@ -1,61 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-let
-  secretsDir = config.clan.core.clanDir + "/sops/secrets";
-  groupsDir = config.clan.core.clanDir + "/sops/groups";
-
-  # My symlink is in the nixos module detected as a directory also it works in the repl. Is this because of pure evaluation?
-  containsSymlink =
-    path:
-    builtins.pathExists path
-    && (builtins.readFileType path == "directory" || builtins.readFileType path == "symlink");
-
-  containsMachine =
-    parent: name: type:
-    type == "directory" && containsSymlink "${parent}/${name}/machines/${config.clan.core.machineName}";
-
-  containsMachineOrGroups =
-    name: type:
-    (containsMachine secretsDir name type)
-    || lib.any (
-      group: type == "directory" && containsSymlink "${secretsDir}/${name}/groups/${group}"
-    ) groups;
-
-  filterDir =
-    filter: dir:
-    lib.optionalAttrs (builtins.pathExists dir) (lib.filterAttrs filter (builtins.readDir dir));
-
-  groups = builtins.attrNames (filterDir (containsMachine groupsDir) groupsDir);
-  secrets = filterDir containsMachineOrGroups secretsDir;
-in
-{
-  config.clan.core.vars.settings = lib.mkIf (config.clan.core.vars.settings.secretStore == "sops") {
-    # Before we generate a secret we cannot know the path yet, so we need to set it to an empty string
-    fileModule = file: {
-      path =
-        lib.mkIf file.secret
-          config.sops.secrets.${"vars-${config.clan.core.machineName}-${file.config.generatorName}-${file.config.name}"}.path
-            or "/no-such-path";
-    };
-    secretModule = "clan_cli.vars.secret_modules.sops";
-    secretUploadDirectory = lib.mkDefault "/var/lib/sops-nix";
-  };
-
-  config.sops = lib.mkIf (config.clan.core.vars.settings.secretStore == "sops") {
-    secrets = builtins.mapAttrs (name: _: {
-      sopsFile = config.clan.core.clanDir + "/sops/secrets/${name}/secret";
-      format = "binary";
-    }) secrets;
-    # To get proper error messages about missing secrets we need a dummy secret file that is always present
-    defaultSopsFile = lib.mkIf config.sops.validateSopsFiles (
-      lib.mkDefault (builtins.toString (pkgs.writeText "dummy.yaml" ""))
-    );
-    age.keyFile = lib.mkIf (builtins.pathExists (
-      config.clan.core.clanDir + "/sops/secrets/${config.clan.core.machineName}-age.key/secret"
-    )) (lib.mkDefault "/var/lib/sops-nix/key.txt");
-  };
-}

nixosModules/clanCore/vars/secret/sops/default.nix

@@ -0,0 +1,50 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+
+  inherit (lib) flip;
+
+  inherit (import ./funcs.nix { inherit lib; }) listVars;
+
+  varsDir = config.clan.core.clanDir + "/sops/vars";
+
+  vars = listVars varsDir;
+
+in
+{
+  config.clan.core.vars.settings = lib.mkIf (config.clan.core.vars.settings.secretStore == "sops") {
+    # Before we generate a secret we cannot know the path yet, so we need to set it to an empty string
+    fileModule = file: {
+      path = lib.mkIf file.config.secret (
+        config.sops.secrets.${"vars-${config.clan.core.machineName}-${file.config.generatorName}-${file.config.name}"}.path
+          or "/no-such-path"
+      );
+    };
+    secretModule = "clan_cli.vars.secret_modules.sops";
+    secretUploadDirectory = lib.mkDefault "/var/lib/sops-nix";
+  };
+
+  config.sops = lib.mkIf (config.clan.core.vars.settings.secretStore == "sops") {
+    secrets = lib.listToAttrs (
+      flip map vars (secret: {
+        name = secret.name;
+        value = {
+          sopsFile =
+            config.clan.core.clanDir + "/sops/vars/${secret.machine}/${secret.generator}/${secret.name}/secret";
+          format = "binary";
+        };
+      })
+    );
+    # To get proper error messages about missing secrets we need a dummy secret file that is always present
+    defaultSopsFile = lib.mkIf config.sops.validateSopsFiles (
+      lib.mkDefault (builtins.toString (pkgs.writeText "dummy.yaml" ""))
+    );
+    age.keyFile = lib.mkIf (builtins.pathExists (
+      config.clan.core.clanDir + "/sops/secrets/${config.clan.core.machineName}-age.key/secret"
+    )) (lib.mkDefault "/var/lib/sops-nix/key.txt");
+  };
+}

nixosModules/clanCore/vars/secret/sops/eval-tests/default.nix

@@ -0,0 +1,43 @@
+{
+  lib ? import <nixpkgs/lib>,
+  pkgs ? import <nixpkgs> { },
+}:
+let
+  inherit (import ../funcs.nix { inherit lib; }) readDirNames listVars;
+
+  noVars = pkgs.runCommand "empty-dir" { } ''
+    mkdir $out
+  '';
+
+  emtpyVars = pkgs.runCommand "empty-dir" { } ''
+    mkdir -p $out/vars
+  '';
+
+in
+{
+  test_readDirNames = {
+    expr = readDirNames ./populated/vars;
+    expected = [ "my_machine" ];
+  };
+
+  test_listSecrets = {
+    expr = listVars ./populated/vars;
+    expected = [
+      {
+        machine = "my_machine";
+        generator = "my_generator";
+        name = "my_secret";
+      }
+    ];
+  };
+
+  test_listSecrets_no_vars = {
+    expr = listVars noVars;
+    expected = [ ];
+  };
+
+  test_listSecrets_empty_vars = {
+    expr = listVars emtpyVars;
+    expected = [ ];
+  };
+}

nixosModules/clanCore/vars/secret/sops/funcs.nix

@@ -0,0 +1,28 @@
+{
+  lib ? import <nixpkgs/lib>,
+  ...
+}:
+let
+  inherit (builtins) readDir;
+
+  inherit (lib) concatMap flip;
+in
+rec {
+  readDirNames =
+    dir:
+    if !(builtins.pathExists dir) then [ ] else lib.mapAttrsToList (name: _type: name) (readDir dir);
+
+  listVars =
+    varsDir:
+    flip concatMap (readDirNames varsDir) (
+      machine_name:
+      flip concatMap (readDirNames (varsDir + "/${machine_name}")) (
+        generator_name:
+        flip map (readDirNames (varsDir + "/${machine_name}/${generator_name}")) (secret_name: {
+          machine = machine_name;
+          generator = generator_name;
+          name = secret_name;
+        })
+      )
+    );
+}

nixosModules/clanCore/vars/settings-opts.nix

@@ -30,6 +30,7 @@
     '';
   };
 
+  # TODO: see if this is the right approach. Maybe revert to secretPathFunction
   fileModule = lib.mkOption {
     type = lib.types.deferredModule;
     internal = true;

pkgs/clan-cli/clan_cli/machines/update.py

@@ -13,6 +13,7 @@ from ..facts.upload import upload_secrets
 from ..machines.machines import Machine
 from ..nix import nix_command, nix_metadata
 from ..ssh import HostKeyCheck
+from ..vars.generate import generate_vars
 from .inventory import get_all_machines, get_selected_machines
 from .machine_group import MachineGroup
 
@@ -93,6 +94,7 @@ def deploy_machine(machines: MachineGroup) -> None:
         env["NIX_SSHOPTS"] = ssh_arg
 
         generate_facts([machine], None, False)
+        generate_vars([machine], None, False)
         upload_secrets(machine)
 
         path = upload_sources(".", target)