Merge pull request 'vars/sops: load sops info from nix instead of filesystem' (#2081) from DavHau/clan-core:DavHau-dave into main

nixosModules/clanCore/vars/secret/sops/default.nix

@@ -6,31 +6,21 @@
 }:
 let
 
-  inherit (lib) importJSON flip;
+  inherit (lib) flip;
 
-  inherit (builtins) dirOf pathExists;
+  inherit (import ./funcs.nix { inherit lib; }) collectFiles;
-
-  inherit (import ./funcs.nix { inherit lib; }) listVars;
 
   inherit (config.clan.core) machineName;
 
-  metaFile = sopsFile: dirOf sopsFile + "/meta.json";
+  secretPath =
-
-  metaData = sopsFile: if pathExists (metaFile sopsFile) then importJSON (metaFile sopsFile) else { };
-
-  isSopsSecret =
     secret:
-    let
+    if secret.share then
-      meta = metaData secret.sopsFile;
+      config.clan.core.clanDir + "/vars/shared/${secret.generator}/${secret.name}/secret"
-    in
+    else
-    meta.store or null == "sops" && meta.deployed or true && meta.secret or true;
+      config.clan.core.clanDir
+      + "/vars/per-machine/${machineName}/${secret.generator}/${secret.name}/secret";
 
-  varsDirMachines = config.clan.core.clanDir + "/vars/per-machine/${machineName}";
+  vars = collectFiles config.clan.core.vars;
-  varsDirShared = config.clan.core.clanDir + "/vars/shared";
-
-  vars' = (listVars varsDirMachines) ++ (listVars varsDirShared);
-
-  vars = lib.filter isSopsSecret vars';
 in
 {
   config.clan.core.vars.settings = lib.mkIf (config.clan.core.vars.settings.secretStore == "sops") {
@@ -50,7 +40,7 @@ in
       flip map vars (secret: {
         name = "vars/${secret.generator}/${secret.name}";
         value = {
-          sopsFile = secret.sopsFile;
+          sopsFile = secretPath secret;
           format = "binary";
         };
       })

nixosModules/clanCore/vars/secret/sops/eval-tests/default.nix

@@ -1,43 +1,53 @@
 {
   lib ? import <nixpkgs/lib>,
-  pkgs ? import <nixpkgs> { },
 }:
 let
-  inherit (import ../funcs.nix { inherit lib; }) readDirNames listVars;
+  inherit (import ../funcs.nix { inherit lib; }) collectFiles;
-
-  noVars = pkgs.runCommand "empty-dir" { } ''
-    mkdir $out
-  '';
-
-  emtpyVars = pkgs.runCommand "empty-dir" { } ''
-    mkdir -p $out/vars
-  '';
-
 in
 {
-  test_readDirNames = {
+  test_collectFiles = {
-    expr = readDirNames ./populated/vars;
+    expr = collectFiles {
-    expected = [ "my_machine" ];
+      # secret, deployed
-  };
+      generators.gen_1.files.secret_deployed_file.secret = true;
+      generators.gen_1.files.secret_deployed_file.deploy = true;
+      generators.gen_1.files.secret_deployed_file.share = false;
 
-  test_listSecrets = {
+      # secret, deployed, shared
-    expr = listVars ./populated/vars/my_machine;
+      generators.gen_1.files.secret_deployed_shared_file.secret = true;
+      generators.gen_1.files.secret_deployed_shared_file.deploy = true;
+      generators.gen_1.files.secret_deployed_shared_file.share = true;
+
+      # secret, undeployed
+      generators.gen_1.files.secret_undeployed_file.secret = true;
+      generators.gen_1.files.secret_undeployed_file.deploy = false;
+      generators.gen_1.files.secret_undeployed_file.share = false;
+
+      # public, deployed
+      generators.gen_1.files.public_deployed_file.secret = false;
+      generators.gen_1.files.public_deployed_file.deploy = true;
+      generators.gen_1.files.public_deployed_file.share = false;
+
+      # secret deployed (different generator)
+      generators.gen_2.files.secret_deployed_file.secret = true;
+      generators.gen_2.files.secret_deployed_file.deploy = true;
+      generators.gen_2.files.secret_deployed_file.share = false;
+    };
     expected = [
       {
-        generator = "my_generator";
+        generator = "gen_1";
-        name = "my_secret";
+        name = "secret_deployed_file";
-        sopsFile = "${./populated/vars/my_machine}/my_generator/my_secret/secret";
+        share = false;
+      }
+      {
+        generator = "gen_1";
+        name = "secret_deployed_shared_file";
+        share = true;
+      }
+      {
+        generator = "gen_2";
+        name = "secret_deployed_file";
+        share = false;
       }
     ];
   };
-
-  test_listSecrets_no_vars = {
-    expr = listVars noVars;
-    expected = [ ];
-  };
-
-  test_listSecrets_empty_vars = {
-    expr = listVars emtpyVars;
-    expected = [ ];
-  };
 }

nixosModules/clanCore/vars/secret/sops/funcs.nix

@@ -3,23 +3,33 @@
   ...
 }:
 let
-  inherit (builtins) readDir;
 
-  inherit (lib) concatMap flip;
+  inherit (lib)
+    filterAttrs
+    flatten
+    flip
+    mapAttrsToList
+    ;
 in
-rec {
+{
-  readDirNames =
-    dir:
-    if !(builtins.pathExists dir) then [ ] else lib.mapAttrsToList (name: _type: name) (readDir dir);
 
-  listVars =
+  collectFiles =
-    varsDir:
+    vars:
-    flip concatMap (readDirNames (varsDir)) (
+    let
-      generator_name:
+      relevantFiles = generator: flip filterAttrs generator.files (_name: f: f.secret && f.deploy);
-      flip map (readDirNames (varsDir + "/${generator_name}")) (secret_name: {
+      allFiles = flatten (
-        generator = generator_name;
+        flip mapAttrsToList vars.generators (
-        name = secret_name;
+          gen_name: generator:
-        sopsFile = "${varsDir}/${generator_name}/${secret_name}/secret";
+          flip mapAttrsToList (relevantFiles generator) (
-      })
+            fname: file:
-    );
+            lib.trace file {
+              name = fname;
+              generator = gen_name;
+              inherit (generator) share;
+            }
+          )
+        )
+      );
+    in
+    allFiles;
 }

pkgs/clan-cli/clan_cli/vars/_types.py

@@ -144,7 +144,6 @@ class StoreBase(ABC):
         directory.mkdir(parents=True, exist_ok=True)
         new_file = self._set(generator_name, var_name, value, shared, deployed)
         meta = {
-            "deployed": deployed,
             "secret": self.is_secret_store,
             "store": self.store_name,
         }