Merge pull request 'vars-check: replace slow python implementation with pure nix' (#4144) from machine-class into main
Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/4144
This commit is contained in:
Mic92
@@ -6,13 +6,10 @@
|
||||
}:
|
||||
let
|
||||
inherit (lib)
|
||||
flatten
|
||||
mapAttrsToList
|
||||
mkForce
|
||||
mkIf
|
||||
mkOption
|
||||
types
|
||||
unique
|
||||
;
|
||||
|
||||
clanLib = config.flake.clanLib;
|
||||
@@ -93,70 +90,13 @@ in
|
||||
${update-vars-script} $PRJ_ROOT/${relativeDir} ${testName}
|
||||
'';
|
||||
|
||||
testSrc = lib.cleanSource config.clan.directory;
|
||||
# Import the new Nix-based vars execution system
|
||||
varsExecutor = import ./vars-executor.nix { inherit lib; };
|
||||
|
||||
inputsForMachine =
|
||||
machine:
|
||||
flip mapAttrsToList machine.clan.core.vars.generators (_name: generator: generator.runtimeInputs);
|
||||
|
||||
generatorScripts =
|
||||
machine:
|
||||
flip mapAttrsToList machine.clan.core.vars.generators (_name: generator: generator.finalScript);
|
||||
|
||||
generatorRuntimeInputs = unique (
|
||||
flatten (flip mapAttrsToList config.nodes (_machineName: machine: inputsForMachine machine))
|
||||
vars-check = hostPkgs.runCommand "vars-check-${testName}" { } (
|
||||
varsExecutor.generateExecutionScript hostPkgs config.nodes
|
||||
);
|
||||
|
||||
allGeneratorScripts = unique (
|
||||
flatten (flip mapAttrsToList config.nodes (_machineName: machine: generatorScripts machine))
|
||||
);
|
||||
|
||||
vars-check =
|
||||
hostPkgs.runCommand "update-vars-check-${testName}"
|
||||
{
|
||||
nativeBuildInputs = generatorRuntimeInputs ++ [
|
||||
hostPkgs.nix
|
||||
hostPkgs.git
|
||||
hostPkgs.age
|
||||
hostPkgs.sops
|
||||
hostPkgs.bubblewrap
|
||||
];
|
||||
closureInfo = hostPkgs.closureInfo {
|
||||
rootPaths =
|
||||
generatorRuntimeInputs
|
||||
++ allGeneratorScripts
|
||||
++ [
|
||||
hostPkgs.bash
|
||||
hostPkgs.coreutils
|
||||
hostPkgs.jq.dev
|
||||
hostPkgs.stdenv
|
||||
hostPkgs.stdenvNoCC
|
||||
hostPkgs.shellcheck-minimal
|
||||
hostPkgs.age
|
||||
hostPkgs.sops
|
||||
];
|
||||
};
|
||||
}
|
||||
''
|
||||
${self.legacyPackages.${hostPkgs.system}.setupNixInNix}
|
||||
cp -r ${testSrc} ./src
|
||||
chmod +w -R ./src
|
||||
mkdir -p ./src/sops ./src/vars # create dirs case the test has no vars
|
||||
find ./src/sops ./src/vars | sort > filesBefore
|
||||
${update-vars-script} ./src ${testName} \
|
||||
--repo-root ${self.packages.${hostPkgs.system}.clan-core-flake} \
|
||||
--clean
|
||||
mkdir -p ./src/sops ./src/vars
|
||||
find ./src/sops ./src/vars | sort > filesAfter
|
||||
if ! diff -q filesBefore filesAfter; then
|
||||
echo "The update-vars script changed the files in ${testSrc}."
|
||||
echo "Diff:"
|
||||
diff filesBefore filesAfter || true
|
||||
exit 1
|
||||
fi
|
||||
touch $out
|
||||
'';
|
||||
|
||||
# the test's flake.nix with locked clan-core input
|
||||
flakeForSandbox =
|
||||
hostPkgs.runCommand "offline-flake-for-test-${config.name}"
|
||||
@@ -298,11 +238,13 @@ in
|
||||
# Harder to handle advanced setups (like TPM, LUKS, or LVM-on-LUKS) but not needed since we are in a test
|
||||
# No systemd journal logs from initrd.
|
||||
boot.initrd.systemd.enable = false;
|
||||
# Make the test depend on its vars-check derivation to reduce CI jobs
|
||||
environment.etc."clan-vars-check".source = vars-check;
|
||||
}
|
||||
);
|
||||
|
||||
result = {
|
||||
inherit update-vars vars-check machinesCross;
|
||||
inherit update-vars machinesCross;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
217
lib/clanTest/vars-executor.nix
Normal file
217
lib/clanTest/vars-executor.nix
Normal file
@@ -0,0 +1,217 @@
|
||||
{ lib }:
|
||||
|
||||
rec {
|
||||
# Extract dependency graph from generators configuration
|
||||
# Returns: { generatorName = [ dep1 dep2 ... ]; ... }
|
||||
extractDependencyGraph =
|
||||
generators: lib.mapAttrs (_name: generator: generator.dependencies or [ ]) generators;
|
||||
|
||||
# Topologically sort generators based on their dependencies
|
||||
# Returns: [ "gen1" "gen2" ... ] in dependency order
|
||||
toposortGenerators =
|
||||
generators:
|
||||
let
|
||||
depGraph = extractDependencyGraph generators;
|
||||
# lib.toposort expects a comparison function where a < b means a should come before b
|
||||
# If A depends on B, then B should come before A, so we want B < A
|
||||
# This means: B < A if A depends on B
|
||||
compareNodes = a: b: builtins.elem a (depGraph.${b} or [ ]);
|
||||
sortResult = lib.toposort compareNodes (lib.attrNames generators);
|
||||
in
|
||||
sortResult.result;
|
||||
|
||||
# Create execution info for a single generator
|
||||
# Returns: { name = "gen"; finalScript = ...; inputs = {...}; ... }
|
||||
createGenExecInfo =
|
||||
generators: allGenerators: name:
|
||||
let
|
||||
generator = generators.${name};
|
||||
# Collect dependency outputs as inputs - look in allGenerators for deps
|
||||
depInputs = lib.listToAttrs (
|
||||
map (depName: {
|
||||
name = depName;
|
||||
value = allGenerators.${depName}.files or { };
|
||||
}) (generator.dependencies or [ ])
|
||||
);
|
||||
in
|
||||
{
|
||||
inherit name;
|
||||
finalScript = generator.finalScript;
|
||||
dependencies = generator.dependencies or [ ];
|
||||
inputs = depInputs;
|
||||
files = generator.files or { };
|
||||
prompts = generator.prompts or { };
|
||||
runtimeInputs = generator.runtimeInputs or [ ];
|
||||
};
|
||||
|
||||
# Create execution plan for generators in dependency order
|
||||
# Returns: [ { name = "gen1"; finalScript = ...; inputs = {...}; } ... ]
|
||||
createExecutionPlan =
|
||||
config: allGenerators:
|
||||
let
|
||||
generators = config.clan.core.vars.generators;
|
||||
sortedNames = toposortGenerators generators;
|
||||
in
|
||||
map (createGenExecInfo generators allGenerators) sortedNames;
|
||||
|
||||
# Generate execution script for a single generator
|
||||
generateGeneratorScript = pkgs: genInfo: isShared: ''
|
||||
echo "Executing ${if isShared then "shared " else ""}generator: ${genInfo.name}"
|
||||
|
||||
# Create input directory with dependency outputs
|
||||
mkdir -p ./inputs/${genInfo.name}
|
||||
${lib.concatStringsSep "\n" (
|
||||
lib.mapAttrsToList (depName: depFiles: ''
|
||||
mkdir -p ./inputs/${genInfo.name}/${depName}
|
||||
${lib.concatStringsSep "\n" (
|
||||
lib.mapAttrsToList (fileName: _: ''
|
||||
# Check for dependency in machine-specific outputs first
|
||||
if [ -f "./outputs/${depName}/${fileName}" ]; then
|
||||
cp "./outputs/${depName}/${fileName}" "./inputs/${genInfo.name}/${depName}/${fileName}"
|
||||
# Check for dependency in shared outputs
|
||||
elif [ -f "${
|
||||
if isShared then "./outputs" else "../../shared/outputs"
|
||||
}/${depName}/${fileName}" ]; then
|
||||
cp "${
|
||||
if isShared then "./outputs" else "../../shared/outputs"
|
||||
}/${depName}/${fileName}" "./inputs/${genInfo.name}/${depName}/${fileName}"
|
||||
else
|
||||
echo "Error: Dependency file ${fileName} not found for generator ${genInfo.name}"
|
||||
echo "Current directory: $(pwd)"
|
||||
echo "Checked paths:"
|
||||
echo " ./outputs/${depName}/${fileName}"
|
||||
echo " ${if isShared then "./outputs" else "../../shared/outputs"}/${depName}/${fileName}"
|
||||
if [ -d "./inputs/${genInfo.name}/${depName}" ]; then
|
||||
echo "Input directory contents:"
|
||||
ls -la "./inputs/${genInfo.name}/${depName}/"
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
'') depFiles
|
||||
)}
|
||||
'') genInfo.inputs
|
||||
)}
|
||||
|
||||
# Create prompts directory
|
||||
mkdir -p ./prompts/${genInfo.name}
|
||||
${lib.concatStringsSep "\n" (
|
||||
lib.mapAttrsToList (promptName: _prompt: ''
|
||||
echo "mock-prompt-value-${promptName}" > "./prompts/${genInfo.name}/${promptName}"
|
||||
'') genInfo.prompts
|
||||
)}
|
||||
|
||||
# Create output directory
|
||||
mkdir -p ./outputs/${genInfo.name}
|
||||
|
||||
# Execute finalScript with bubblewrap
|
||||
${pkgs.bubblewrap}/bin/bwrap \
|
||||
--dev-bind /dev /dev \
|
||||
--proc /proc \
|
||||
--tmpfs /tmp \
|
||||
--ro-bind /nix/store /nix/store \
|
||||
--bind "./inputs/${genInfo.name}" /input \
|
||||
--ro-bind "./prompts/${genInfo.name}" /prompts \
|
||||
--bind "./outputs/${genInfo.name}" /output \
|
||||
--setenv in /input \
|
||||
--setenv prompts /prompts \
|
||||
--setenv out /output \
|
||||
--setenv PATH "${
|
||||
lib.makeBinPath (
|
||||
(genInfo.runtimeInputs or [ ])
|
||||
++ [
|
||||
pkgs.bash
|
||||
pkgs.coreutils
|
||||
]
|
||||
)
|
||||
}" \
|
||||
${pkgs.runtimeShell} ${genInfo.finalScript}
|
||||
|
||||
# Verify expected outputs were created
|
||||
${lib.concatStringsSep "\n" (
|
||||
lib.mapAttrsToList (fileName: _fileInfo: ''
|
||||
if [ ! -f "./outputs/${genInfo.name}/${fileName}" ]; then
|
||||
echo "✗ Expected output file ${fileName} not found for generator ${genInfo.name}"
|
||||
exit 1
|
||||
else
|
||||
echo "✓ Generated ${if isShared then "shared " else ""}file: ${genInfo.name}/${fileName}"
|
||||
fi
|
||||
'') genInfo.files
|
||||
)}
|
||||
|
||||
echo "✓ ${if isShared then "Shared " else ""}Generator ${genInfo.name} completed"
|
||||
'';
|
||||
|
||||
# Create machine execution info
|
||||
# Returns: { sorted = [ "gen1" "gen2" ... ]; executionPlan = [...]; generators = {...}; }
|
||||
createMachineExecInfo = allGenerators: machine: rec {
|
||||
sorted = toposortGenerators generators;
|
||||
executionPlan = createExecutionPlan { clan.core.vars.generators = generators; } allGenerators;
|
||||
generators = lib.filterAttrs (_name: gen: !(gen.share or false)) machine.clan.core.vars.generators;
|
||||
};
|
||||
|
||||
# Collect all generators from all machines
|
||||
collectAllGenerators =
|
||||
nodes:
|
||||
lib.foldl' (acc: machine: acc // machine.clan.core.vars.generators) { } (lib.attrValues nodes);
|
||||
|
||||
# Generate shell script for executing generators in dependency order with bubblewrap
|
||||
generateExecutionScript =
|
||||
pkgs: nodes:
|
||||
let
|
||||
# Collect all generators from all machines
|
||||
allGenerators = collectAllGenerators nodes;
|
||||
|
||||
# Separate shared and per-machine generators
|
||||
sharedGenerators = lib.filterAttrs (_name: gen: gen.share or false) allGenerators;
|
||||
|
||||
# Create execution plans
|
||||
sharedExecutionPlan =
|
||||
if sharedGenerators != { } then
|
||||
createExecutionPlan { clan.core.vars.generators = sharedGenerators; } allGenerators
|
||||
else
|
||||
[ ];
|
||||
|
||||
machineExecutions = lib.mapAttrs (_machineName: createMachineExecInfo allGenerators) nodes;
|
||||
in
|
||||
''
|
||||
echo "Running vars check using Nix-based executor..."
|
||||
|
||||
# Execute shared generators first
|
||||
${lib.optionalString (sharedGenerators != { }) ''
|
||||
echo "Executing shared generators..."
|
||||
mkdir -p ./shared
|
||||
cd ./shared
|
||||
|
||||
${lib.concatStringsSep "\n" (
|
||||
map (genInfo: generateGeneratorScript pkgs genInfo true) sharedExecutionPlan
|
||||
)}
|
||||
|
||||
cd ..
|
||||
echo "✓ Shared generators completed"
|
||||
''}
|
||||
|
||||
# Execute generators for each machine in topological order
|
||||
${lib.concatStringsSep "\n" (
|
||||
lib.mapAttrsToList (machineName: execInfo: ''
|
||||
echo "Processing machine: ${machineName}"
|
||||
echo "Generator execution order: ${lib.concatStringsSep " -> " execInfo.sorted}"
|
||||
|
||||
# Create machine-specific work directory
|
||||
mkdir -p ./work/${machineName}
|
||||
cd ./work/${machineName}
|
||||
|
||||
# Execute each generator in dependency order
|
||||
${lib.concatStringsSep "\n" (
|
||||
map (genInfo: generateGeneratorScript pkgs genInfo false) execInfo.executionPlan
|
||||
)}
|
||||
|
||||
cd ../..
|
||||
echo "✓ Machine ${machineName} completed"
|
||||
|
||||
'') machineExecutions
|
||||
)}
|
||||
|
||||
echo "✓ All vars checks completed successfully"
|
||||
touch $out
|
||||
'';
|
||||
}
|
||||
@@ -30,65 +30,29 @@ in
|
||||
};
|
||||
|
||||
config.checks = lib.optionalAttrs (pkgs.stdenv.isLinux) (
|
||||
let
|
||||
# Build all individual vars-check derivations
|
||||
varsChecks = lib.mapAttrs' (
|
||||
name: testModule:
|
||||
lib.nameValuePair "vars-check-${name}" (
|
||||
let
|
||||
test = nixosLib.runTest (
|
||||
{ ... }:
|
||||
# Add the VM tests as checks (vars-check is part of the test closure)
|
||||
lib.mapAttrs (
|
||||
_name: testModule:
|
||||
nixosLib.runTest (
|
||||
{ ... }:
|
||||
{
|
||||
imports = [
|
||||
self.modules.nixosTest.clanTest
|
||||
testModule
|
||||
];
|
||||
|
||||
hostPkgs = pkgs;
|
||||
|
||||
defaults = {
|
||||
imports = [
|
||||
{
|
||||
imports = [
|
||||
self.modules.nixosTest.clanTest
|
||||
testModule
|
||||
];
|
||||
|
||||
hostPkgs = pkgs;
|
||||
|
||||
defaults = {
|
||||
imports = [
|
||||
{
|
||||
_module.args.clan-core = self;
|
||||
}
|
||||
];
|
||||
};
|
||||
_module.args.clan-core = self;
|
||||
}
|
||||
);
|
||||
in
|
||||
test.config.result.vars-check
|
||||
)
|
||||
) cfg;
|
||||
in
|
||||
lib.mkMerge [
|
||||
# Add the VM tests as checks
|
||||
(lib.mapAttrs' (
|
||||
name: testModule:
|
||||
lib.nameValuePair "service-${name}" (
|
||||
nixosLib.runTest (
|
||||
{ ... }:
|
||||
{
|
||||
imports = [
|
||||
self.modules.nixosTest.clanTest
|
||||
testModule
|
||||
];
|
||||
|
||||
hostPkgs = pkgs;
|
||||
|
||||
defaults = {
|
||||
imports = [
|
||||
{
|
||||
_module.args.clan-core = self;
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
)
|
||||
)
|
||||
) cfg)
|
||||
|
||||
varsChecks
|
||||
]
|
||||
];
|
||||
};
|
||||
}
|
||||
)
|
||||
) cfg
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user