From 5ce72dd261533bf4bec63a876ca9f729296515df Mon Sep 17 00:00:00 2001 From: DavHau Date: Fri, 17 Jan 2025 16:21:52 +0700 Subject: [PATCH] tests: reduce unnecessary rebuilds of several tests Some test were referring to the whole source code via ${self} which amde them rebuild on every single commit. This is not mitigated by introduceing `self.filter { include = [...]; }` allowin to a content addressed subset of the source code in tests. --- checks/backups/flake-module.nix | 31 +++- checks/flake-module.nix | 10 +- clanModules/flake-module.nix | 11 +- docs/nix/flake-module.nix | 8 +- flake.nix | 43 +++-- lib/build-clan/flake-module.nix | 16 +- lib/filter-clan-core/flake-module.nix | 18 ++ lib/filter-clan-core/nix-filter.nix | 193 ++++++++++++++++++++ lib/flake-module.nix | 8 +- lib/inventory/flake-module.nix | 14 +- lib/values/flake-module.nix | 11 +- nixosModules/clanCore/vars/flake-module.nix | 9 +- pkgs/clan-cli/flake-module.nix | 17 +- 13 files changed, 355 insertions(+), 34 deletions(-) create mode 100644 lib/filter-clan-core/flake-module.nix create mode 100644 lib/filter-clan-core/nix-filter.nix diff --git a/checks/backups/flake-module.nix b/checks/backups/flake-module.nix index c94fac5a8..80d6eea3c 100644 --- a/checks/backups/flake-module.nix +++ b/checks/backups/flake-module.nix @@ -133,6 +133,29 @@ }; perSystem = { pkgs, ... }: + let + clanCore = self.filter { + include = [ + "checks/backups" + "checks/flake-module.nix" + "clanModules/borgbackup" + "clanModules/flake-module.nix" + "clanModules/localbackup" + "clanModules/packages/roles" + "clanModules/single-disk" + "clanModules/zerotier" + "flake.lock" + "flakeModules" + "inventory.json" + "lib/build-clan" + "lib/default.nix" + "lib/flake-module.nix" + "lib/frontmatter" + "lib/inventory" + "nixosModules" + ]; + }; + in { # Needs investigation on aarch64-linux # vm-test-run-test-backups> qemu-kvm: No machine specified, and there is no default @@ -158,14 +181,14 @@ machine.succeed("echo testing > /var/test-backups/somefile") # create - machine.succeed("clan backups create --debug --flake ${self} test-backup") + machine.succeed("clan backups create --debug --flake ${clanCore} test-backup") machine.wait_until_succeeds("! systemctl is-active borgbackup-job-test-backup >&2") machine.succeed("test -f /run/mount-external-disk") machine.succeed("test -f /run/unmount-external-disk") # list backup_id = json.loads(machine.succeed("borg-job-test-backup list --json"))["archives"][0]["archive"] - out = machine.succeed("clan backups list --debug --flake ${self} test-backup").strip() + out = machine.succeed("clan backups list --debug --flake ${clanCore} test-backup").strip() print(out) assert backup_id in out, f"backup {backup_id} not found in {out}" localbackup_id = "hdd::/mnt/external-disk/snapshot.0" @@ -173,7 +196,7 @@ ## borgbackup restore machine.succeed("rm -f /var/test-backups/somefile") - machine.succeed(f"clan backups restore --debug --flake ${self} test-backup borgbackup 'test-backup::borg@machine:.::{backup_id}' >&2") + machine.succeed(f"clan backups restore --debug --flake ${clanCore} test-backup borgbackup 'test-backup::borg@machine:.::{backup_id}' >&2") assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed" machine.succeed("test -f /var/test-service/pre-restore-command") machine.succeed("test -f /var/test-service/post-restore-command") @@ -181,7 +204,7 @@ ## localbackup restore machine.succeed("rm -rf /var/test-backups/somefile /var/test-service/ && mkdir -p /var/test-service") - machine.succeed(f"clan backups restore --debug --flake ${self} test-backup localbackup '{localbackup_id}' >&2") + machine.succeed(f"clan backups restore --debug --flake ${clanCore} test-backup localbackup '{localbackup_id}' >&2") assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed" machine.succeed("test -f /var/test-service/pre-restore-command") machine.succeed("test -f /var/test-service/post-restore-command") diff --git a/checks/flake-module.nix b/checks/flake-module.nix index a74a4765c..7bedb0c61 100644 --- a/checks/flake-module.nix +++ b/checks/flake-module.nix @@ -1,6 +1,12 @@ -{ self, ... }: +{ self, lib, ... }: +let + inherit (lib) + filter + pathExists + ; +in { - imports = [ + imports = filter pathExists [ ./backups/flake-module.nix ./devshell/flake-module.nix ./flash/flake-module.nix diff --git a/clanModules/flake-module.nix b/clanModules/flake-module.nix index f7978046a..b9e1b1d2d 100644 --- a/clanModules/flake-module.nix +++ b/clanModules/flake-module.nix @@ -1,6 +1,13 @@ -{ ... }: +{ lib, ... }: +let + inherit (lib) + filterAttrs + pathExists + ; +in { - flake.clanModules = { + # only import available files, as this allows to filter the files for tests. + flake.clanModules = filterAttrs (_name: pathExists) { admin = ./admin; borgbackup = ./borgbackup; borgbackup-static = ./borgbackup-static; diff --git a/docs/nix/flake-module.nix b/docs/nix/flake-module.nix index a4523c5d6..e38226f01 100644 --- a/docs/nix/flake-module.nix +++ b/docs/nix/flake-module.nix @@ -69,7 +69,13 @@ ]; } '' - export CLAN_CORE_PATH=${self} + export CLAN_CORE_PATH=${ + self.filter { + include = [ + "clanModules" + ]; + } + } export CLAN_CORE_DOCS=${jsonDocs.clanCore}/share/doc/nixos/options.json # A file that contains the links to all clanModule docs export CLAN_MODULES_VIA_ROLES=${clanModulesViaRoles} diff --git a/flake.nix b/flake.nix index 902ba1e03..a7b8f1437 100644 --- a/flake.nix +++ b/flake.nix @@ -24,10 +24,18 @@ outputs = inputs@{ flake-parts, + nixpkgs, self, systems, ... }: + let + inherit (nixpkgs.lib) + filter + optional + pathExists + ; + in flake-parts.lib.mkFlake { inherit inputs; } ( { ... }: { @@ -36,24 +44,29 @@ directory = self; }; systems = import systems; - imports = [ - ./checks/flake-module.nix - ./clanModules/flake-module.nix - ./flakeModules/flake-module.nix - (import ./flakeModules/clan.nix inputs.self) - ./devShell.nix - # TODO: migrate this @davHau - # ./docs/flake-module - ./docs/nix/flake-module.nix - ./lib/flake-module.nix - ./nixosModules/flake-module.nix - ./nixosModules/clanCore/vars/flake-module.nix - ./pkgs/flake-module.nix - ./templates/flake-module.nix + imports = + # only imporing existing paths allows to minimize the flake for test + # by removing files + filter pathExists [ + ./checks/flake-module.nix + ./clanModules/flake-module.nix + ./devShell.nix + ./docs/nix/flake-module.nix + ./flakeModules/flake-module.nix + ./lib/filter-clan-core/flake-module.nix + ./lib/flake-module.nix + ./nixosModules/clanCore/vars/flake-module.nix + ./nixosModules/flake-module.nix + ./pkgs/flake-module.nix + ./templates/flake-module.nix + ] + ++ [ + (if pathExists ./flakeModules/clan.nix then import ./flakeModules/clan.nix inputs.self else { }) + ] # Make treefmt-nix optional # This only works if you set inputs.clan-core.inputs.treefmt-nix.follows # to a non-empty input that doesn't export a flakeModule - ] ++ inputs.nixpkgs.lib.optional (inputs.treefmt-nix ? flakeModule) ./formatter.nix; + ++ optional (pathExists ./formatter.nix && inputs.treefmt-nix ? flakeModule) ./formatter.nix; } ); } diff --git a/lib/build-clan/flake-module.nix b/lib/build-clan/flake-module.nix index 66a5a665f..6b27a3ac3 100644 --- a/lib/build-clan/flake-module.nix +++ b/lib/build-clan/flake-module.nix @@ -20,7 +20,6 @@ in jsonDocs = import ./eval-docs.nix { inherit pkgs lib; }; - in { legacyPackages.clan-internals-docs = jsonDocs.optionsJSON; @@ -39,7 +38,20 @@ in nix-unit --eval-store "$HOME" \ --extra-experimental-features flakes \ ${inputOverrides} \ - --flake ${self}#legacyPackages.${system}.evalTests-build-clan + --flake ${ + self.filter { + include = [ + "flakeModules" + "inventory.json" + "lib/build-clan" + "lib/default.nix" + "lib/flake-module.nix" + "lib/inventory" + "machines" + "nixosModules" + ]; + } + }#legacyPackages.${system}.evalTests-build-clan touch $out ''; diff --git a/lib/filter-clan-core/flake-module.nix b/lib/filter-clan-core/flake-module.nix new file mode 100644 index 000000000..b6bcfa345 --- /dev/null +++ b/lib/filter-clan-core/flake-module.nix @@ -0,0 +1,18 @@ +{ self, ... }: +let + nixFilter = import ./nix-filter.nix; +in +{ + flake.filter = + { + include ? [ ], + exclude ? [ ], + }: + nixFilter.filter { + inherit exclude; + include = include ++ [ + "flake.nix" + ]; + root = self; + }; +} diff --git a/lib/filter-clan-core/nix-filter.nix b/lib/filter-clan-core/nix-filter.nix new file mode 100644 index 000000000..d89ed25b1 --- /dev/null +++ b/lib/filter-clan-core/nix-filter.nix @@ -0,0 +1,193 @@ +/* + MIT License + + Copyright (c) 2021 Numtide + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +# This is a pure and self-contained library +rec { + # Default to filter when calling this lib. + __functor = _self: filter; + + # A proper source filter + filter = + { + # Base path to include + root, + # Derivation name + name ? "source", + # Only include the following path matches. + # + # Allows all files by default. + include ? [ + ( + _: _: _: + true + ) + ], + # Ignore the following matches + exclude ? [ ], + }: + assert _pathIsDirectory root; + let + callMatcher = args: _toMatcher ({ inherit root; } // args); + include_ = map (callMatcher { matchParents = true; }) include; + exclude_ = map (callMatcher { matchParents = false; }) exclude; + in + builtins.path { + inherit name; + path = root; + filter = + path: type: (builtins.any (f: f path type) include_) && (!builtins.any (f: f path type) exclude_); + }; + + # Match a directory and any path inside of it + inDirectory = + directory: args: + let + # Convert `directory` to a path to clean user input. + directory_ = _toCleanPath args.root directory; + in + path: _type: + directory_ == path + # Add / to the end to make sure we match a full directory prefix + || _hasPrefix (directory_ + "/") path; + + # Match any directory + isDirectory = + _: _: type: + type == "directory"; + + # Combines matchers + and = + a: b: args: + let + toMatcher = _toMatcher args; + in + path: type: (toMatcher a path type) && (toMatcher b path type); + + # Combines matchers + or_ = + a: b: args: + let + toMatcher = _toMatcher args; + in + path: type: (toMatcher a path type) || (toMatcher b path type); + + # Or is actually a keyword, but can also be used as a key in an attrset. + or = or_; + + # Match paths with the given extension + matchExt = + ext: _args: path: _type: + _hasSuffix ".${ext}" path; + + # Filter out files or folders with this exact name + matchName = + name: _root: path: _type: + builtins.baseNameOf path == name; + + # Wrap a matcher with this to debug its results + debugMatch = + label: fn: args: path: type: + let + ret = fn args path type; + retStr = if ret then "true" else "false"; + in + builtins.trace "label=${label} path=${path} type=${type} ret=${retStr}" ret; + + # Add this at the end of the include or exclude, to trace all the unmatched paths + traceUnmatched = + _args: path: type: + builtins.trace "unmatched path=${path} type=${type}" false; + + # Lib stuff + + # If an argument to include or exclude is a path, transform it to a matcher. + # + # This probably needs more work, I don't think that it works on + # sub-folders. + _toMatcher = + args: f: + let + path_ = _toCleanPath args.root f; + pathIsDirectory = _pathIsDirectory path_; + in + if builtins.isFunction f then + f args + else + path: type: + (if pathIsDirectory then inDirectory path_ args path type else path_ == path) + || args.matchParents && type == "directory" && _hasPrefix "${path}/" path_; + + # Makes sure a path is: + # * absolute + # * doesn't contain superfluous slashes or .. + # + # Returns a string so there is no risk of adding it to the store by mistake. + _toCleanPath = + absPath: path: + assert _pathIsDirectory absPath; + if builtins.isPath path then + toString path + else if builtins.isString path then + if builtins.substring 0 1 path == "/" then path else toString (absPath + ("/" + path)) + else + throw "unsupported type ${builtins.typeOf path}, expected string or path"; + + _hasSuffix = + # Suffix to check for + suffix: + # Input string + content: + let + lenContent = builtins.stringLength content; + lenSuffix = builtins.stringLength suffix; + in + lenContent >= lenSuffix && builtins.substring (lenContent - lenSuffix) lenContent content == suffix; + + _hasPrefix = + # Prefix to check for + prefix: + # Input string + content: + let + lenPrefix = builtins.stringLength prefix; + in + prefix == builtins.substring 0 lenPrefix content; + + # Returns true if the path exists and is a directory and false otherwise + _pathIsDirectory = + p: + let + parent = builtins.dirOf p; + base = builtins.unsafeDiscardStringContext (builtins.baseNameOf p); + inNixStore = builtins.storeDir == toString parent; + in + # If the parent folder is /nix/store, we assume p is a directory. Because + # reading /nix/store is very slow, and not allowed in every environments. + inNixStore + || ( + builtins.pathExists p + && (builtins.readDir parent).${builtins.unsafeDiscardStringContext base} == "directory" + ); + +} diff --git a/lib/flake-module.nix b/lib/flake-module.nix index ccfdf3151..f7b8a71b2 100644 --- a/lib/flake-module.nix +++ b/lib/flake-module.nix @@ -4,8 +4,14 @@ self, ... }: +let + inherit (lib) + filter + pathExists + ; +in { - imports = [ + imports = filter pathExists [ ./jsonschema/flake-module.nix ./inventory/flake-module.nix ./build-clan/flake-module.nix diff --git a/lib/inventory/flake-module.nix b/lib/inventory/flake-module.nix index e6b834676..a3c063802 100644 --- a/lib/inventory/flake-module.nix +++ b/lib/inventory/flake-module.nix @@ -46,7 +46,19 @@ in nix-unit --eval-store "$HOME" \ --extra-experimental-features flakes \ ${inputOverrides} \ - --flake ${self}#legacyPackages.${system}.evalTests-inventory + --flake ${ + self.filter { + include = [ + "flakeModules" + "lib/default.nix" + "lib/flake-module.nix" + "lib/inventory" + "lib/frontmatter" + "clanModules/flake-module.nix" + "clanModules/borgbackup" + ]; + } + }#legacyPackages.${system}.evalTests-inventory touch $out ''; diff --git a/lib/values/flake-module.nix b/lib/values/flake-module.nix index 9d997ccc6..a0b14dd1f 100644 --- a/lib/values/flake-module.nix +++ b/lib/values/flake-module.nix @@ -24,7 +24,16 @@ in nix-unit --eval-store "$HOME" \ --extra-experimental-features flakes \ ${inputOverrides} \ - --flake ${self}#legacyPackages.${system}.evalTests-values + --flake ${ + self.filter { + include = [ + "flakeModules" + "lib/default.nix" + "lib/flake-module.nix" + "lib/values" + ]; + } + }#legacyPackages.${system}.evalTests-values touch $out ''; diff --git a/nixosModules/clanCore/vars/flake-module.nix b/nixosModules/clanCore/vars/flake-module.nix index 3be63eebe..32093e7f6 100644 --- a/nixosModules/clanCore/vars/flake-module.nix +++ b/nixosModules/clanCore/vars/flake-module.nix @@ -24,7 +24,14 @@ in nix-unit --eval-store "$HOME" \ --extra-experimental-features flakes \ ${inputOverrides} \ - --flake ${self}#legacyPackages.${system}.evalTests-module-clan-vars + --flake ${ + self.filter { + include = [ + "flakeModules" + "nixosModules" + ]; + } + }#legacyPackages.${system}.evalTests-module-clan-vars touch $out ''; diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index f16928d53..6cc1136f0 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -12,7 +12,16 @@ ... }: let - flakeLock = lib.importJSON (self + /flake.lock); + clanCore = self.filter { + include = [ + "clanModules" + "flakeModules" + "lib" + "nixosModules" + "flake.lock" + ]; + }; + flakeLock = lib.importJSON (clanCore + "/flake.lock"); flakeInputs = builtins.removeAttrs inputs [ "self" ]; flakeLockVendoredDeps = flakeLock: @@ -43,7 +52,7 @@ inputs = lib.mapAttrs (name: _input: name) flakeInputs; locked = { lastModified = 1; - path = "${self}"; + path = "${clanCore}"; type = "path"; }; original = { @@ -81,11 +90,11 @@ set -e export HOME=$(realpath .) export NIX_STATE_DIR=$HOME - cp -r ${self} $out + cp -r ${clanCore} $out chmod +w -R $out cp ${clanCoreLockFile} $out/flake.lock nix flake lock $out --extra-experimental-features 'nix-command flakes' - clanCoreHash=$(nix hash path ${self} --extra-experimental-features 'nix-command') + clanCoreHash=$(nix hash path ${clanCore} --extra-experimental-features 'nix-command') for templateDir in $(find $out/templates -mindepth 1 -maxdepth 1 -type d); do if ! [ -e "$templateDir/flake.nix" ]; then continue