yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

tests: reduce unnecessary rebuilds of several tests

Browse Source 
Some test were referring to the whole source code via ${self} which amde them rebuild on every single commit.

This is not mitigated by introduceing `self.filter { include = [...]; }` allowin to a content addressed subset of the source code in tests.
This commit is contained in:
DavHau
2025-01-17 16:21:52 +07:00
parent 9cd0572734
commit 3ec028d672
13 changed files with 355 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
checks/backups/flake-module.nix
View File

@@ -133,6 +133,29 @@
};
perSystem =
{ pkgs, ... }:
let
clanCore = self.filter {
include = [
"checks/backups"
"checks/flake-module.nix"
"clanModules/borgbackup"
"clanModules/flake-module.nix"
"clanModules/localbackup"
"clanModules/packages/roles"
"clanModules/single-disk"
"clanModules/zerotier"
"flake.lock"
"flakeModules"
"inventory.json"
"lib/build-clan"
"lib/default.nix"
"lib/flake-module.nix"
"lib/frontmatter"
"lib/inventory"
"nixosModules"
];
};
in
{
# Needs investigation on aarch64-linux
# vm-test-run-test-backups> qemu-kvm: No machine specified, and there is no default
@@ -158,14 +181,14 @@
machine.succeed("echo testing > /var/test-backups/somefile")
# create
machine.succeed("clan backups create --debug --flake ${self} test-backup")
machine.succeed("clan backups create --debug --flake ${clanCore} test-backup")
machine.wait_until_succeeds("! systemctl is-active borgbackup-job-test-backup >&2")
machine.succeed("test -f /run/mount-external-disk")
machine.succeed("test -f /run/unmount-external-disk")
# list
backup_id = json.loads(machine.succeed("borg-job-test-backup list --json"))["archives"][0]["archive"]
out = machine.succeed("clan backups list --debug --flake ${self} test-backup").strip()
out = machine.succeed("clan backups list --debug --flake ${clanCore} test-backup").strip()
print(out)
assert backup_id in out, f"backup {backup_id} not found in {out}"
localbackup_id = "hdd::/mnt/external-disk/snapshot.0"
@@ -173,7 +196,7 @@
## borgbackup restore
machine.succeed("rm -f /var/test-backups/somefile")
machine.succeed(f"clan backups restore --debug --flake ${self} test-backup borgbackup 'test-backup::borg@machine:.::{backup_id}' >&2")
machine.succeed(f"clan backups restore --debug --flake ${clanCore} test-backup borgbackup 'test-backup::borg@machine:.::{backup_id}' >&2")
assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
machine.succeed("test -f /var/test-service/pre-restore-command")
machine.succeed("test -f /var/test-service/post-restore-command")
@@ -181,7 +204,7 @@
## localbackup restore
machine.succeed("rm -rf /var/test-backups/somefile /var/test-service/ && mkdir -p /var/test-service")
machine.succeed(f"clan backups restore --debug --flake ${self} test-backup localbackup '{localbackup_id}' >&2")
machine.succeed(f"clan backups restore --debug --flake ${clanCore} test-backup localbackup '{localbackup_id}' >&2")
assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
machine.succeed("test -f /var/test-service/pre-restore-command")
machine.succeed("test -f /var/test-service/post-restore-command")

10
checks/flake-module.nix
View File

@@ -1,6 +1,12 @@
{ self, ... }:
{ self, lib, ... }:
let
inherit (lib)
filter
pathExists
;
in
{
imports = [
imports = filter pathExists [
./backups/flake-module.nix
./devshell/flake-module.nix
./flash/flake-module.nix

11
clanModules/flake-module.nix
View File

@@ -1,6 +1,13 @@
{ ... }:
{ lib, ... }:
let
inherit (lib)
filterAttrs
pathExists
;
in
{
flake.clanModules = {
# only import available files, as this allows to filter the files for tests.
flake.clanModules = filterAttrs (_name: pathExists) {
admin = ./admin;
borgbackup = ./borgbackup;
borgbackup-static = ./borgbackup-static;

8
docs/nix/flake-module.nix
View File

@@ -69,7 +69,13 @@
];
}
''
export CLAN_CORE_PATH=${self}
export CLAN_CORE_PATH=${
self.filter {
include = [
"clanModules"
];
}
}
export CLAN_CORE_DOCS=${jsonDocs.clanCore}/share/doc/nixos/options.json
# A file that contains the links to all clanModule docs
export CLAN_MODULES_VIA_ROLES=${clanModulesViaRoles}

27
flake.nix
View File

@@ -24,10 +24,18 @@
outputs =
inputs@{
flake-parts,
nixpkgs,
self,
systems,
...
}:
let
inherit (nixpkgs.lib)
filter
optional
pathExists
;
in
flake-parts.lib.mkFlake { inherit inputs; } (
{ ... }:
{
@@ -36,24 +44,29 @@
directory = self;
};
systems = import systems;
imports = [
imports =
# only imporing existing paths allows to minimize the flake for test
# by removing files
filter pathExists [
./checks/flake-module.nix
./clanModules/flake-module.nix
./flakeModules/flake-module.nix
(import ./flakeModules/clan.nix inputs.self)
./devShell.nix
# TODO: migrate this @davHau
# ./docs/flake-module
./docs/nix/flake-module.nix
./flakeModules/flake-module.nix
./lib/filter-clan-core/flake-module.nix
./lib/flake-module.nix
./nixosModules/flake-module.nix
./nixosModules/clanCore/vars/flake-module.nix
./nixosModules/flake-module.nix
./pkgs/flake-module.nix
./templates/flake-module.nix
]
++ [
(if pathExists ./flakeModules/clan.nix then import ./flakeModules/clan.nix inputs.self else { })
]
# Make treefmt-nix optional
# This only works if you set inputs.clan-core.inputs.treefmt-nix.follows
# to a non-empty input that doesn't export a flakeModule
] ++ inputs.nixpkgs.lib.optional (inputs.treefmt-nix ? flakeModule) ./formatter.nix;
++ optional (pathExists ./formatter.nix && inputs.treefmt-nix ? flakeModule) ./formatter.nix;
}
);
}

16
lib/build-clan/flake-module.nix
View File

@@ -20,7 +20,6 @@ in
jsonDocs = import ./eval-docs.nix {
inherit pkgs lib;
};
in
{
legacyPackages.clan-internals-docs = jsonDocs.optionsJSON;
@@ -39,7 +38,20 @@ in
nix-unit --eval-store "$HOME" \
--extra-experimental-features flakes \
${inputOverrides} \
--flake ${self}#legacyPackages.${system}.evalTests-build-clan
--flake ${
self.filter {
include = [
"flakeModules"
"inventory.json"
"lib/build-clan"
"lib/default.nix"
"lib/flake-module.nix"
"lib/inventory"
"machines"
"nixosModules"
];
}
}#legacyPackages.${system}.evalTests-build-clan
touch $out
'';

18
lib/filter-clan-core/flake-module.nix Normal file
View File

@@ -0,0 +1,18 @@
{ self, ... }:
let
nixFilter = import ./nix-filter.nix;
in
{
flake.filter =
{
include ? [ ],
exclude ? [ ],
}:
nixFilter.filter {
inherit exclude;
include = include ++ [
"flake.nix"
];
root = self;
};
}

193
lib/filter-clan-core/nix-filter.nix Normal file
View File

@@ -0,0 +1,193 @@
/*
MIT License
Copyright (c) 2021 Numtide
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
# This is a pure and self-contained library
rec {
# Default to filter when calling this lib.
__functor = _self: filter;
# A proper source filter
filter =
{
# Base path to include
root,
# Derivation name
name ? "source",
# Only include the following path matches.
#
# Allows all files by default.
include ? [
(
_: _: _:
true
)
],
# Ignore the following matches
exclude ? [ ],
}:
assert _pathIsDirectory root;
let
callMatcher = args: _toMatcher ({ inherit root; } // args);
include_ = map (callMatcher { matchParents = true; }) include;
exclude_ = map (callMatcher { matchParents = false; }) exclude;
in
builtins.path {
inherit name;
path = root;
filter =
path: type: (builtins.any (f: f path type) include_) && (!builtins.any (f: f path type) exclude_);
};
# Match a directory and any path inside of it
inDirectory =
directory: args:
let
# Convert `directory` to a path to clean user input.
directory_ = _toCleanPath args.root directory;
in
path: _type:
directory_ == path
# Add / to the end to make sure we match a full directory prefix
|| _hasPrefix (directory_ + "/") path;
# Match any directory
isDirectory =
_: _: type:
type == "directory";
# Combines matchers
and =
a: b: args:
let
toMatcher = _toMatcher args;
in
path: type: (toMatcher a path type) && (toMatcher b path type);
# Combines matchers
or_ =
a: b: args:
let
toMatcher = _toMatcher args;
in
path: type: (toMatcher a path type) || (toMatcher b path type);
# Or is actually a keyword, but can also be used as a key in an attrset.
or = or_;
# Match paths with the given extension
matchExt =
ext: _args: path: _type:
_hasSuffix ".${ext}" path;
# Filter out files or folders with this exact name
matchName =
name: _root: path: _type:
builtins.baseNameOf path == name;
# Wrap a matcher with this to debug its results
debugMatch =
label: fn: args: path: type:
let
ret = fn args path type;
retStr = if ret then "true" else "false";
in
builtins.trace "label=${label} path=${path} type=${type} ret=${retStr}" ret;
# Add this at the end of the include or exclude, to trace all the unmatched paths
traceUnmatched =
_args: path: type:
builtins.trace "unmatched path=${path} type=${type}" false;
# Lib stuff
# If an argument to include or exclude is a path, transform it to a matcher.
#
# This probably needs more work, I don't think that it works on
# sub-folders.
_toMatcher =
args: f:
let
path_ = _toCleanPath args.root f;
pathIsDirectory = _pathIsDirectory path_;
in
if builtins.isFunction f then
f args
else
path: type:
(if pathIsDirectory then inDirectory path_ args path type else path_ == path)
|| args.matchParents && type == "directory" && _hasPrefix "${path}/" path_;
# Makes sure a path is:
# * absolute
# * doesn't contain superfluous slashes or ..
#
# Returns a string so there is no risk of adding it to the store by mistake.
_toCleanPath =
absPath: path:
assert _pathIsDirectory absPath;
if builtins.isPath path then
toString path
else if builtins.isString path then
if builtins.substring 0 1 path == "/" then path else toString (absPath + ("/" + path))
else
throw "unsupported type ${builtins.typeOf path}, expected string or path";
_hasSuffix =
# Suffix to check for
suffix:
# Input string
content:
let
lenContent = builtins.stringLength content;
lenSuffix = builtins.stringLength suffix;
in
lenContent >= lenSuffix && builtins.substring (lenContent - lenSuffix) lenContent content == suffix;
_hasPrefix =
# Prefix to check for
prefix:
# Input string
content:
let
lenPrefix = builtins.stringLength prefix;
in
prefix == builtins.substring 0 lenPrefix content;
# Returns true if the path exists and is a directory and false otherwise
_pathIsDirectory =
p:
let
parent = builtins.dirOf p;
base = builtins.unsafeDiscardStringContext (builtins.baseNameOf p);
inNixStore = builtins.storeDir == toString parent;
in
# If the parent folder is /nix/store, we assume p is a directory. Because
# reading /nix/store is very slow, and not allowed in every environments.
inNixStore
|| (
builtins.pathExists p
&& (builtins.readDir parent).${builtins.unsafeDiscardStringContext base} == "directory"
);
}

8
lib/flake-module.nix
View File

@@ -4,8 +4,14 @@
self,
...
}:
let
inherit (lib)
filter
pathExists
;
in
{
imports = [
imports = filter pathExists [
./jsonschema/flake-module.nix
./inventory/flake-module.nix
./build-clan/flake-module.nix

14
lib/inventory/flake-module.nix
View File

@@ -46,7 +46,19 @@ in
nix-unit --eval-store "$HOME" \
--extra-experimental-features flakes \
${inputOverrides} \
--flake ${self}#legacyPackages.${system}.evalTests-inventory
--flake ${
self.filter {
include = [
"flakeModules"
"lib/default.nix"
"lib/flake-module.nix"
"lib/inventory"
"lib/frontmatter"
"clanModules/flake-module.nix"
"clanModules/borgbackup"
];
}
}#legacyPackages.${system}.evalTests-inventory
touch $out
'';

11
lib/values/flake-module.nix
View File

@@ -24,7 +24,16 @@ in
nix-unit --eval-store "$HOME" \
--extra-experimental-features flakes \
${inputOverrides} \
--flake ${self}#legacyPackages.${system}.evalTests-values
--flake ${
self.filter {
include = [
"flakeModules"
"lib/default.nix"
"lib/flake-module.nix"
"lib/values"
];
}
}#legacyPackages.${system}.evalTests-values
touch $out
'';

9
nixosModules/clanCore/vars/flake-module.nix
View File

@@ -24,7 +24,14 @@ in
nix-unit --eval-store "$HOME" \
--extra-experimental-features flakes \
${inputOverrides} \
--flake ${self}#legacyPackages.${system}.evalTests-module-clan-vars
--flake ${
self.filter {
include = [
"flakeModules"
"nixosModules"
];
}
}#legacyPackages.${system}.evalTests-module-clan-vars
touch $out
'';

17
pkgs/clan-cli/flake-module.nix
View File

@@ -12,7 +12,16 @@
...
}:
let
flakeLock = lib.importJSON (self + /flake.lock);
clanCore = self.filter {
include = [
"clanModules"
"flakeModules"
"lib"
"nixosModules"
"flake.lock"
];
};
flakeLock = lib.importJSON (clanCore + "/flake.lock");
flakeInputs = builtins.removeAttrs inputs [ "self" ];
flakeLockVendoredDeps =
flakeLock:
@@ -43,7 +52,7 @@
inputs = lib.mapAttrs (name: _input: name) flakeInputs;
locked = {
lastModified = 1;
path = "${self}";
path = "${clanCore}";
type = "path";
};
original = {
@@ -81,11 +90,11 @@
set -e
export HOME=$(realpath .)
export NIX_STATE_DIR=$HOME
cp -r ${self} $out
cp -r ${clanCore} $out
chmod +w -R $out
cp ${clanCoreLockFile} $out/flake.lock
nix flake lock $out --extra-experimental-features 'nix-command flakes'
clanCoreHash=$(nix hash path ${self} --extra-experimental-features 'nix-command')
clanCoreHash=$(nix hash path ${clanCore} --extra-experimental-features 'nix-command')
for templateDir in $(find $out/templates -mindepth 1 -maxdepth 1 -type d); do
if ! [ -e "$templateDir/flake.nix" ]; then
continue