ci performance: add check to ensure nothing depends on the whole repo

Since this project is an ever growing monorepo, having derivations depending on the whole repo leads to bad CI performance, as the cache is busted on every commit.

-> We never want any derivations depend on the whole repo

...except: the test that tests that nothing depends on the whole repo, which is added by this commit.

checks/flake-module.nix

@@ -14,6 +14,7 @@ in
     ./installation/flake-module.nix
     ./morph/flake-module.nix
     ./nixos-documentation/flake-module.nix
+    ./sanity-checks/dont-depend-on-repo-root.nix
   ];
   perSystem =
     {

checks/sanity-checks/dont-depend-on-repo-root.nix

@@ -0,0 +1,121 @@
+{
+  ...
+}:
+{
+  perSystem =
+    {
+      system,
+      pkgs,
+      self',
+      lib,
+      ...
+    }:
+    let
+      clanCore = self'.packages.clan-core-flake;
+      clanCoreHash = lib.substring 0 12 (builtins.hashString "sha256" "${clanCore}");
+      /*
+        construct a flake for the test which contains a single check which depends
+        on all checks of clan-core.
+      */
+      testFlakeFile = pkgs.writeText "flake.nix" ''
+        {
+          inputs.clan-core.url = path:///to/nowhere;
+          outputs = {clan-core, ...}:
+            let
+              checks =
+                builtins.removeAttrs
+                  clan-core.checks.${system}
+                  [
+                    "dont-depend-on-repo-root"
+                    "package-clan-core-flake"
+                  ];
+              checksOutPaths = map (x: "''${x}") (builtins.attrValues checks);
+            in
+            {
+              checks.${system}.check = builtins.derivation {
+                name = "all-clan-core-checks";
+                system = "${system}";
+                builder = "/bin/sh";
+                args = ["-c" '''
+                  of outPath in ''${toString checksOutPaths}; do
+                    echo "$outPath" >> $out
+                  done
+                '''];
+              };
+            };
+        }
+      '';
+    in
+    lib.optionalAttrs (system == "x86_64-linux") {
+      checks.dont-depend-on-repo-root =
+        pkgs.runCommand
+          # append repo hash to this tests name to ensure it gets invalidated on each chain
+          # This is needed because this test is an FOD (due to networking) and would get cached indefinitely.
+          "check-dont-depend-on-repo-root-${clanCoreHash}"
+          {
+            buildInputs = [
+              pkgs.nix
+              pkgs.cacert
+              pkgs.nix-diff
+            ];
+            outputHashAlgo = "sha256";
+            outputHash = "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=";
+          }
+          ''
+            mkdir clanCore testFlake store
+            clanCore=$(realpath clanCore)
+            testFlake=$(realpath testFlake)
+
+            # copy clan core flake and make writable
+            cp -r ${clanCore}/* clanCore/
+            chmod +w -R clanCore\
+
+            # copy test flake and make writable
+            cp ${testFlakeFile} testFlake/flake.nix
+            chmod +w -R testFlake
+
+            # enable flakes
+            export NIX_CONFIG="experimental-features = nix-command flakes"
+
+            # give nix a $HOME
+            export HOME=$(realpath ./store)
+
+            # override clan-core flake input to point to $clanCore\
+            echo "locking clan-core to $clanCore"
+            nix flake lock --override-input clan-core "path://$clanCore" "$testFlake" --store "$HOME"
+
+            # evaluate all tests
+            echo "evaluating all tests for clan core"
+            nix eval "$testFlake"#checks.${system}.check.drvPath --store "$HOME" --raw > drvPath1 &
+
+            # slightly modify clan core
+            cp -r $clanCore clanCore2
+            cp -r $testFlake testFlake2
+            export clanCore2=$(realpath clanCore2)
+            export testFlake2=$(realpath testFlake2)
+            touch clanCore2/fly-fpv
+
+            # re-evaluate all tests
+            echo "locking clan-core to $clanCore2"
+            nix flake lock --override-input clan-core "path://$clanCore2" "$testFlake2" --store "$HOME"
+            echo "evaluating all tests for clan core with added file"
+            nix eval "$testFlake2"#checks.${system}.check.drvPath --store "$HOME" --raw > drvPath2
+
+            # wait for first nix eval to return as well
+            while ! grep -q drv drvPath1; do sleep 1; done
+
+            # raise error if outputs are different
+            if [ "$(cat drvPath1)" != "$(cat drvPath2)" ]; then
+              echo -e "\n\nERROR: Something in clan-core depends on the whole repo" > /dev/stderr
+              echo -e "See details in the nix-diff below which shows the difference between two evaluations:"
+              echo -e "  1. Evaluation of clan-core checks without any changes"
+              echo -e "  1. Evaluation of clan-core checks after adding a file to the top-level of the repo"
+              echo "nix-diff:"
+              export NIX_REMOTE="$HOME"
+              nix-diff $(cat drvPath1) $(cat drvPath2)
+              exit 1
+            fi
+            touch $out
+          '';
+    };
+}

pkgs/clan-cli/flake-module.nix

@@ -12,6 +12,7 @@
       ...
     }:
     let
+      clanCoreWithVendoredDeps = self'.packages.clan-core-flake.override {
         clanCore = self.filter {
           include = [
             "clanModules"
@@ -22,65 +23,7 @@
             "templates"
           ];
         };
-      flakeLock = lib.importJSON (clanCore + "/flake.lock");
-      flakeInputs = builtins.removeAttrs inputs [ "self" ];
-      flakeLockVendoredDeps =
-        flakeLock:
-        flakeLock
-        // {
-          nodes =
-            flakeLock.nodes
-            // (lib.flip lib.mapAttrs flakeInputs (
-              name: _:
-              # remove follows and let 'nix flake lock' re-compute it later
-              # (lib.removeAttrs flakeLock.nodes.${name} ["inputs"])
-              flakeLock.nodes.${name}
-              // {
-                locked = {
-                  inherit (flakeLock.nodes.${name}.locked) narHash;
-                  lastModified =
-                    # lol, nixpkgs has a different timestamp on the fs???
-                    if name == "nixpkgs" then 0 else 1;
-                  path = "${inputs.${name}}";
-                  type = "path";
       };
-              }
-            ));
-        };
-      clanCoreLock = flakeLockVendoredDeps flakeLock;
-      clanCoreLockFile = builtins.toFile "clan-core-flake.lock" (builtins.toJSON clanCoreLock);
-
-      clanCoreNode = {
-
-        inputs = lib.mapAttrs (name: _input: name) flakeInputs;
-        locked = {
-          lastModified = 1;
-          path = "${clanCore}";
-          type = "path";
-        };
-        original = {
-          type = "tarball";
-          url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
-        };
-      };
-      # generate a lock file that nix will accept for our flake templates,
-      # in order to not require internet access during tests.
-      templateLock = clanCoreLock // {
-        nodes = clanCoreLock.nodes // {
-          clan-core = clanCoreNode;
-          nixpkgs-lib = clanCoreLock.nodes.nixpkgs; # required by flake-parts
-          flake-parts = clanCoreLock.nodes.flake-parts;
-          root = clanCoreLock.nodes.root // {
-            inputs = clanCoreLock.nodes.root.inputs // {
-              clan-core = "clan-core";
-              nixpkgs = "nixpkgs";
-              clan = "clan-core";
-              flake-parts = "flake-parts";
-            };
-          };
-        };
-      };
-      templateLockFile = builtins.toFile "template-flake.lock" (builtins.toJSON templateLock);
 
       # We need to add the paths of the templates to the nix store such that they are available
       # only adding clanCoreWithVendoredDeps to the nix store is not enough
@@ -89,41 +32,6 @@
           builtins.attrValues (self.clanLib.select "clan.templates.clan.*.path" self)
           ++ builtins.attrValues (self.clanLib.select "clan.templates.machine.*.path" self);
       };
-
-      clanCoreWithVendoredDeps =
-        pkgs.runCommand "clan-core-with-vendored-deps"
-          {
-            buildInputs = [
-              pkgs.findutils
-              pkgs.git
-              pkgs.jq
-              pkgs.nix
-            ];
-          }
-          ''
-            set -e
-            export HOME=$(realpath .)
-            export NIX_STATE_DIR=$HOME
-            export NIX_STORE_DIR=$HOME
-            cp -r ${clanCore} $out
-            chmod +w -R $out
-            cp ${clanCoreLockFile} $out/flake.lock
-            nix flake lock $out --extra-experimental-features 'nix-command flakes'
-            clanCoreHash=$(nix hash path ${clanCore} --extra-experimental-features 'nix-command')
-
-            ## ==> We need this to make nix flake update work on the templates
-            ## however then we have to re-add the clan templates to the nix store
-            ## which is not possible (or I don't know how)
-            # for templateDir in $(find $out/templates/clan -mindepth 1 -maxdepth 1 -type d); do
-            #   if ! [ -e "$templateDir/flake.nix" ]; then
-            #     continue
-            #   fi
-            #   cp ${templateLockFile} $templateDir/flake.lock
-            #   cat $templateDir/flake.lock | jq ".nodes.\"clan-core\".locked.narHash = \"$clanCoreHash\"" > $templateDir/flake.lock.final
-            #   mv $templateDir/flake.lock.final $templateDir/flake.lock
-            #   nix flake lock $templateDir --extra-experimental-features 'nix-command flakes'
-            # done
-          '';
     in
     {
       devShells.clan-cli = pkgs.callPackage ./shell.nix {

pkgs/clan-core-flake/flake-module.nix

@@ -0,0 +1,75 @@
+{ self, inputs, ... }:
+{
+  perSystem =
+    {
+      lib,
+      pkgs,
+      ...
+    }:
+    let
+      # A flake lock for offline use.
+      # All flake inputs are locked to an existing store path
+      clanCoreLockFile =
+        clanCore:
+        let
+          flakeLock = lib.importJSON (clanCore + "/flake.lock");
+          flakeInputs = builtins.removeAttrs inputs [ "self" ];
+          flakeLockVendoredDeps =
+            flakeLock:
+            flakeLock
+            // {
+              nodes =
+                flakeLock.nodes
+                // (lib.flip lib.mapAttrs flakeInputs (
+                  name: _:
+                  # remove follows and let 'nix flake lock' re-compute it later
+                  # (lib.removeAttrs flakeLock.nodes.${name} ["inputs"])
+                  flakeLock.nodes.${name}
+                  // {
+                    locked = {
+                      inherit (flakeLock.nodes.${name}.locked) narHash;
+                      lastModified =
+                        # lol, nixpkgs has a different timestamp on the fs???
+                        if name == "nixpkgs" then 0 else 1;
+                      path = "${inputs.${name}}";
+                      type = "path";
+                    };
+                  }
+                ));
+            };
+          clanCoreLock = flakeLockVendoredDeps flakeLock;
+          clanCoreLockFile = builtins.toFile "clan-core-flake.lock" (builtins.toJSON clanCoreLock);
+        in
+        clanCoreLockFile;
+    in
+    {
+      packages.clan-core-flake =
+        let
+          package =
+            {
+              clanCore,
+            }:
+            pkgs.runCommand "clan-core-flake"
+              {
+                buildInputs = [
+                  pkgs.findutils
+                  pkgs.git
+                  pkgs.jq
+                  pkgs.nix
+                ];
+              }
+              ''
+                set -e
+                export HOME=$(realpath .)
+                export NIX_STATE_DIR=$HOME
+                export NIX_STORE_DIR=$HOME
+                cp -r ${clanCore} $out
+                chmod +w -R $out
+                cp ${clanCoreLockFile clanCore} $out/flake.lock
+                nix flake lock $out --extra-experimental-features 'nix-command flakes'
+                clanCoreHash=$(nix hash path ${clanCore} --extra-experimental-features 'nix-command')
+              '';
+        in
+        pkgs.callPackage package { clanCore = self; };
+    };
+}

pkgs/flake-module.nix

@@ -10,6 +10,7 @@
     ./distro-packages/flake-module.nix
     ./icon-update/flake-module.nix
     ./generate-test-vars/flake-module.nix
+    ./clan-core-flake/flake-module.nix
   ];
 
   flake.packages.x86_64-linux =