-
Notifications
You must be signed in to change notification settings - Fork 50
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
adding PR handling, build and comparison of modules and envs
- Loading branch information
1 parent
58277ab
commit beec303
Showing
1 changed file
with
183 additions
and
59 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,79 +1,203 @@ | ||
| #!/bin/bash | ||
| # Start interactive shell to access EESSI through build container | ||
| # mkdir -p /tmp/$USER/EESSI | ||
| # cd /tmp/$USER/EESSI | ||
| # git clone https://github.com/EESSI/software-layer | ||
| # cd software-layer | ||
| # ./eessi_container.sh | ||
|
|
||
| # Initialize EESSI + load/configure EasyBuild | ||
| # source /cvmfs/software.eessi.io/versions/2023.06/init/bash | ||
| # module load EasyBuild/4.9.2 | ||
| # export WORKDIR=/tmp/$USER/EESSI | ||
| # source configure_easybuild | ||
|
|
||
| # .eb directory as an argument | ||
| if [ -z "$1" ]; then | ||
| echo "Usage: $0 <directory>" | ||
|
|
||
| # This script checks the consistency of EB-generated modules and identifies broken or missing modules. | ||
| # Usage: ./module_check.sh <path to easystack file> [<optional path to PR diff>] | ||
|
|
||
| # It uses an adapted approach from check_missing_installations.sh to handling PRs/unmerged PRs | ||
| TOPDIR=$(dirname $(realpath $0)) | ||
|
|
||
| if [ "$#" -eq 1 ]; then | ||
| echo "No PR diff provided. Processing all modules in the easystack file." | ||
| pr_exceptions="" | ||
| elif [ "$#" -eq 2 ]; then | ||
| echo "Using $2 to create exceptions for PR filtering of easystack" | ||
| pr_diff="$2" | ||
| pr_exceptions=$(grep '^+' "$pr_diff" | grep 'from-pr' | uniq | awk '{print $3}' | xargs -I {} echo " || /'{}'/") | ||
| else | ||
| echo "ERROR: Usage: $0 <path to easystack file> [<optional path to PR diff>]" >&2 | ||
| exit 1 | ||
| fi | ||
|
|
||
| base_dir="$1" | ||
| easystack="$1" | ||
|
|
||
| LOCAL_TMPDIR=$(mktemp -d) | ||
| mkdir -p "$LOCAL_TMPDIR" | ||
|
|
||
| # Clone the develop branch of EasyBuild and use that to search for easyconfigs | ||
| git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs | ||
| export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs | ||
|
|
||
| # Dir where the modules will be | ||
| module_install_dir="/tmp/$USER/EESSI/module-only" | ||
| # All PRs used in EESSI are supposed to be merged, so we can strip ou all cases of from-pr | ||
| tmp_easystack="${LOCAL_TMPDIR}/$(basename "${easystack}")" | ||
| grep -v 'from-pr' "${easystack}" > "${tmp_easystack}" | ||
|
|
||
| # If PR exceptions exist, modify the easystack file to include exceptions | ||
| if [ -n "$pr_exceptions" ]; then | ||
| # Use awk to exclude lines containing PR numbers specified in pr_exceptions | ||
| awk_command="awk '!/from-pr/ EXCEPTIONS' ${easystack}" | ||
| awk_command=${awk_command/\\/} | ||
| eval "${awk_command/EXCEPTIONS/$pr_exceptions}" > "${tmp_easystack}" | ||
| fi | ||
|
|
||
| locks_dir="/tmp/$USER/EESSI/locks" | ||
| # Set up temporary directories for module installation and lock files | ||
| TMPDIR=${TMPDIR:-/tmp}/$USER | ||
| module_install_dir="$TMPDIR/EESSI/module-only" | ||
| locks_dir="$TMPDIR/EESSI/locks" | ||
| mkdir -p "$module_install_dir" "$locks_dir" | ||
|
|
||
| # Log file to record broken modules | ||
| # Log file to record broken modules | ||
| broken_modules_log="broken_modules.log" | ||
| > $broken_modules_log | ||
| > "$broken_modules_log" | ||
|
|
||
| # To keep track of already-checked modules and avoid re-checking | ||
| declare -A checked_modules | ||
|
|
||
| # Locate all .eb files within the base dir | ||
| easyconfig_files=$(find $base_dir -name "*.eb") | ||
| # Identify missing easyconfigs based on the temporary easystack file | ||
| echo "Identifying missing easyconfigs using the temporary easystack file..." | ||
| missing_easyconfigs=$(eb --easystack "${tmp_easystack}" --missing --robot 2>&1) | ||
|
|
||
| # Iterate over all eb files found. Package name based on eb file name | ||
| for easyconfig_file in $easyconfig_files; do | ||
| package_name=$(basename $easyconfig_file .eb) | ||
| if [ -z "$missing_easyconfigs" ]; then | ||
| echo "No missing easyconfigs to install." | ||
| rm -rf "$LOCAL_TMPDIR" | ||
| exit 0 | ||
| fi | ||
|
|
||
| # Process each missing easyconfig file | ||
| for easyconfig_file in $missing_easyconfigs; do | ||
| package_name=$(basename "$easyconfig_file" .eb) | ||
|
|
||
| # Building of the easyconfig | ||
| echo "Building $package_name using EasyBuild..." | ||
| eb "$easyconfig_file" --robot | ||
| if [ $? -ne 0 ]; then | ||
| echo "EasyBuild build failed for $package_name. Skipping..." | ||
| echo "$package_name: EasyBuild build failed" >> "$broken_modules_log" | ||
| continue | ||
| fi | ||
|
|
||
| # Run EB to generate the modules. Check if the eb command failed. | ||
| echo "Generating modules for $package_name using EasyBuild..." | ||
| eb $easyconfig_file --module-only --installpath-modules $module_install_dir --locks-dir $locks_dir --force --robot | ||
| # Generate the module using --module-only | ||
| echo "Generating module for $package_name using --module-only..." | ||
| eb "$easyconfig_file" --module-only --installpath-modules "$module_install_dir" --locks-dir "$locks_dir" --force --robot | ||
| if [ $? -ne 0 ]; then | ||
| echo "EasyBuild command failed for $package_name. Skipping..." | ||
| echo "$package_name: EasyBuild command failed" >> $broken_modules_log | ||
| echo "EasyBuild --module-only command failed for $package_name. Skipping..." | ||
| echo "$package_name: EasyBuild --module-only command failed" >> "$broken_modules_log" | ||
| continue | ||
| fi | ||
|
|
||
| # Find the module file generated from the build | ||
| module_relpath=$(eb "$easyconfig_file" --show-module --robot 2>/dev/null) | ||
| if [ -z "$module_relpath" ]; then | ||
| echo "Failed to get module relative path for $package_name" | ||
| echo "$package_name: Failed to get module relative path" >> "$broken_modules_log" | ||
| continue | ||
| fi | ||
|
|
||
| # Modules names and version | ||
| module_software=$(echo "$module_relpath" | sed 's/\.lua$//') | ||
|
|
||
| # Check if the module has already been validated to avoid redundant checks | ||
| if [ -n "${checked_modules[$module_software]}" ]; then | ||
| echo "Module $module_software already checked. Skipping." | ||
| continue | ||
| fi | ||
|
|
||
| # Check the generated modules and iterate over the modules in the 'all' dir | ||
| echo "Checking generated modules for $package_name..." | ||
| for module_category in $(ls $module_install_dir/all); do | ||
| for module_version in $(ls $module_install_dir/all/$module_category); do | ||
| module_name="$module_category/$module_version" | ||
|
|
||
| # Checks if the module has already been tested | ||
| if [ -n "${checked_modules[$module_name]}" ]; then | ||
| echo "Module $module_name already checked. Skipping." | ||
| continue | ||
| fi | ||
|
|
||
| echo "Testing module: $module_name" | ||
|
|
||
| # Try loading the module | ||
| if module --ignore_cache load $module_name 2>/dev/null; then | ||
| echo "$module_name loaded successfully." | ||
| module unload $module_name | ||
| else | ||
| echo "$module_name is broken." | ||
| echo "$package_name: $module_name" >> $broken_modules_log | ||
| fi | ||
|
|
||
| checked_modules[$module_name]=1 | ||
| done | ||
| done | ||
| # Paths to the module files generated from build and the --module-only | ||
| module_file_build="${EASYBUILD_INSTALLPATH}/modules/all/${module_relpath}" | ||
| module_file_module_only="${module_install_dir}/all/${module_relpath}" | ||
|
|
||
| # Check if both module files exist | ||
| if [ ! -f "$module_file_build" ]; then | ||
| echo "Module file from full build not found: $module_file_build" | ||
| echo "$package_name: Module file from full build not found" >> "$broken_modules_log" | ||
| continue | ||
| fi | ||
|
|
||
| if [ ! -f "$module_file_module_only" ]; then | ||
| echo "Module file from --module-only build not found: $module_file_module_only" | ||
| echo "$package_name: Module file from --module-only build not found" >> "$broken_modules_log" | ||
| continue | ||
| fi | ||
|
|
||
| # Compare the module files | ||
| if diff -q "$module_file_build" "$module_file_module_only" >/dev/null; then | ||
| echo "Module files for $package_name match" | ||
| else | ||
| echo "Module files for $package_name differ" | ||
| echo "$package_name: Module files differ" >> "$broken_modules_log" | ||
| # Save differences | ||
| diff_file="${module_software//\//_}_module_diff.txt" | ||
| diff "$module_file_build" "$module_file_module_only" > "$diff_file" | ||
| echo "Module file differences saved to $diff_file" | ||
| fi | ||
|
|
||
| # Proceed to compare the environments | ||
| echo "Testing module: $module_software" | ||
|
|
||
| # Function to get filtered environment variables, excluding lmod-related vars | ||
| get_filtered_env() { | ||
| env | grep -v -E '^(LMOD_|MODULEPATH|MODULESHOME|LOADEDMODULES|BASH_FUNC_module|_ModuleTable_|PWD=|SHLVL=|OLDPWD=|PS1=|PS2=|_LMFILES_)=.*$' | sort | ||
| } | ||
|
|
||
| # Compare the environments of the modules | ||
| module purge | ||
| module unuse "$module_install_dir" | ||
| module load EasyBuild | ||
|
|
||
| # Load the module from the full build | ||
| if module --ignore_cache load "$module_software" 2>/dev/null; then | ||
| original_env=$(get_filtered_env) | ||
| module unload "$module_software" | ||
| else | ||
| echo "Failed to load module from full build: $module_software." | ||
| original_env="" | ||
| fi | ||
|
|
||
| # Load the module from the --module-only | ||
| module purge | ||
| module use "$module_install_dir" | ||
|
|
||
| if module --ignore_cache load "$module_software" 2>/dev/null; then | ||
| new_env=$(get_filtered_env) | ||
| module unload "$module_software" | ||
| else | ||
| echo "Failed to load module from --module-only build: $module_software." | ||
| echo "$package_name: Failed to load module from --module-only build" >> "$broken_modules_log" | ||
| module unuse "$module_install_dir" | ||
| continue | ||
| fi | ||
|
|
||
| # Compare the environments | ||
| if [ -n "$original_env" ]; then | ||
| if diff <(echo "$original_env") <(echo "$new_env") >/dev/null; then | ||
| echo "$module_software loaded with identical environment." | ||
| else | ||
| echo "$module_software environment mismatch." | ||
| echo "$package_name: $module_software (environment mismatch)" >> "$broken_modules_log" | ||
| diff_file="${module_software//\//_}_env_diff.txt" | ||
| diff <(echo "$original_env") <(echo "$new_env") > "$diff_file" | ||
| echo "Environment differences saved to $diff_file" | ||
| fi | ||
| else | ||
| echo "Original environment not available for comparison for $module_software." | ||
| echo "$package_name: $module_software (failed to load module from full build)" >> "$broken_modules_log" | ||
| fi | ||
|
|
||
|
|
||
| module unuse "$module_install_dir" | ||
|
|
||
| # Mark module as checked | ||
| checked_modules[$module_software]=1 | ||
|
|
||
| done | ||
|
|
||
| echo "All module checks completed. Broken modules are listed in $broken_modules_log" | ||
| # Report | ||
| if [ -f "$broken_modules_log" ] && [ -s "$broken_modules_log" ]; then | ||
| echo "Some modules did not match. See $broken_modules_log for details." | ||
| exit 1 | ||
| else | ||
| echo "All modules match between build and --module-only build." | ||
| fi | ||
|
|
||
| # Clean up temporary directories | ||
| rm -rf "$LOCAL_TMPDIR" | ||
|
|