Merge pull request 'Fix import group secrets and fix import sops files into the nix store' (#245) from machines-import-fix into main

checks/impure/flake-module.nix

@@ -17,6 +17,7 @@
             pkgs.jq
             pkgs.openssh
             pkgs.nix
+            self'.packages.clan-cli
           ]}"
 
           cd $TMPDIR
@@ -32,10 +33,10 @@
           nix flake show
 
           echo create a machine
-          ${self'.packages.clan-cli}/bin/clan machines create machine1
+          clan machines create machine1
 
           echo check machine1 exists
-          ${self'.packages.clan-cli}/bin/clan machines list | grep -q machine1
+          clan machines list | grep -q machine1
 
           echo check machine1 appears in nixosConfigurations
           nix flake show --json | jq '.nixosConfigurations' | grep -q machine1

checks/secrets/default.nix

@@ -5,13 +5,17 @@
     imports = [
       (self.nixosModules.clanCore)
     ];
-    environment.etc."secret".source = config.sops.secrets.foo.path;
+    environment.etc."secret".source = config.sops.secrets.secret.path;
+    environment.etc."group-secret".source = config.sops.secrets.group-secret.path;
     sops.age.keyFile = ./key.age;
+
     clanCore.clanDir = "${./.}";
     clanCore.machineName = "machine";
+
     networking.hostName = "machine";
   };
   testScript = ''
     machine.succeed("cat /etc/secret >&2")
+    machine.succeed("cat /etc/group-secret >&2")
   '';
 }

checks/secrets/sops/secrets/group-secret/groups/group

@@ -0,0 +1 @@
+../../../groups/group

checks/secrets/sops/secrets/group-secret/secret

@@ -0,0 +1,20 @@
+{
+	"data": "ENC[AES256_GCM,data:FgF3,iv:QBbnqZ6405qmwGKhbolPr9iobngXt8rtfUwCBOnmwRA=,tag:7gqI1zLVnTkZ0xrNn/LEkA==,type:str]",
+	"sops": {
+		"kms": null,
+		"gcp_kms": null,
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": [
+			{
+				"recipient": "age15x8u838dwqflr3t6csf4tlghxm4tx77y379ncqxav7y2n8qp7yzqgrwt00",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArMHcxKzhUZzNHQmQrb28x\nRC9UMlZMeDN3S1l1eHdUWmV4VUVReHhhQ0RnCjAyUXVlY1FmclVmL2lEdFZuTmll\nVENpa3AwbjlDck5zdGdHUTRnNEdEOUkKLS0tIER3ZlNMSVFnRElkRDcxajZnVmFl\nZThyYzcvYUUvaWJYUmlwQ3dsSDdjSjgK+tj34yBzrsIjm6V+T9wTgz5FdNGOR7I/\nVB4fh8meW0vi/PCK/rajC8NbqmK8qq/lwsF/JwfZKDSdG0FOJUB1AA==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2023-09-03T12:44:56Z",
+		"mac": "ENC[AES256_GCM,data:d5a0WfE5ZRLKF1NZkBfOl+cVI8ZZHd2rC+qX/giALjyrzk09rLxBeY4lO827GFfMmVy/oC7ceH9pjv2O7ibUiQtcbGIQVBg/WP+dVn8fRMWtF0jpv9BhYTutkVk3kiddqPGhp3mpwvls2ot5jtCRczTPk3JSxN3B1JSJCmj9GfQ=,iv:YmlkTYFNUaFRWozO8+OpEVKaSQmh+N9zpatwUNMPNyw=,tag:mEGQ4tdo82qlhKWalQuufg==,type:str]",
+		"pgp": null,
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.7.3"
+	}
+}

checks/secrets/sops/secrets/secret/machines/machine

@@ -0,0 +1 @@
+../../../machines/machine

lib/build-clan/default.nix

@@ -4,15 +4,12 @@
 , machines ? { } # allows to include machine-specific modules i.e. machines.${name} = { ... }
 }:
 let
-  machinesDirs =
+  machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") (builtins.readDir (directory + /machines));
-    if builtins.pathExists "${directory}/machines"
-    then builtins.readDir "${directory}/machines"
-    else { };
 
   machineSettings = machineName:
-    if builtins.pathExists "${directory}/machines/${machineName}/settings.json"
+    lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json")
-    then builtins.fromJSON (builtins.readFile "${directory}/machines/${machineName}/settings.json")
+      builtins.fromJSON
-    else { };
+      (builtins.readFile (directory + /machines/${machineName}/settings.json));
 
   nixosConfigurations = lib.mapAttrs
     (name: _:
@@ -23,7 +20,7 @@ let
           (machines.${name} or { })
           {
             clanCore.machineName = name;
-            clanCore.clanDir = builtins.toString directory;
+            clanCore.clanDir = directory;
             # TODO: remove this once we have a hardware-config mechanism
             nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
           }

nixosModules/clanCore/secrets/sops.nix

@@ -1,7 +1,28 @@
 { config, lib, pkgs, ... }:
+let
+  secretsDir = config.clanCore.clanDir + "/sops/secrets";
+  groupsDir = config.clanCore.clanDir + "/sops/groups";
+
+  # My symlink is in the nixos module detected as a directory also it works in the repl. Is this because of pure evaluation?
+  containsSymlink = path:
+    builtins.pathExists path && (builtins.readFileType path == "directory" || builtins.readFileType path == "symlink");
+
+  containsMachine = parent: name: type:
+    type == "directory" && containsSymlink "${parent}/${name}/machines/${config.clanCore.machineName}";
+
+  containsMachineOrGroups = name: type:
+    (containsMachine secretsDir name type) || lib.any (group: type == "directory" && containsSymlink "${secretsDir}/${name}/groups/${group}") groups;
+
+  filterDir = filter: dir:
+    lib.optionalAttrs (builtins.pathExists dir)
+      (lib.filterAttrs filter (builtins.readDir dir));
+
+  groups = builtins.attrNames (filterDir (containsMachine groupsDir) groupsDir);
+  secrets = filterDir containsMachineOrGroups secretsDir;
+in
 {
   config = {
-    system.clan.generateSecrets = pkgs.writeScript "generate_secrets" ''
+    system.clan.generateSecrets = pkgs.writeScript "generate-secrets" ''
       #!/bin/sh
       set -efu
       set -x # remove for prod
@@ -43,26 +64,12 @@
         fi)
       '') "" config.clanCore.secrets}
     '';
-    sops.secrets =
+    sops.secrets = builtins.mapAttrs
-      let
+      (name: _: {
-        secretsDir = config.clanCore.clanDir + "/sops/secrets";
+        sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret";
-        encryptedForThisMachine = name: type:
+        format = "binary";
-          let
+      })
-            symlink = "${secretsDir}/${name}/machines/${config.clanCore.machineName}";
+      secrets;
-          in
-          # WTF, nix bug, my symlink is in the nixos module detected as a directory also it works in the repl
-          type == "directory" && builtins.pathExists symlink && (builtins.readFileType symlink == "directory" || builtins.readFileType symlink == "symlink");
-        secrets =
-          if !(builtins.pathExists secretsDir)
-          then { }
-          else lib.filterAttrs encryptedForThisMachine (builtins.readDir secretsDir);
-      in
-      builtins.mapAttrs
-        (name: _: {
-          sopsFile = "${config.clanCore.clanDir}/sops/secrets/${name}/secret";
-          format = "binary";
-        })
-        secrets;
     # To get proper error messages about missing secrets we need a dummy secret file that is always present
     sops.defaultSopsFile = lib.mkIf config.sops.validateSopsFiles (lib.mkDefault (builtins.toString (pkgs.writeText "dummy.yaml" "")));
   };