Skip to content

Commit

Permalink
Merge branch '2023.06-software.eessi.io' into 2023.06-software.eessi.…
Browse files Browse the repository at this point in the history
…io_archdetect-a64fx
  • Loading branch information
boegel committed Jan 10, 2025
2 parents cb1672b + 7246674 commit f376b9a
Show file tree
Hide file tree
Showing 111 changed files with 3,284 additions and 507 deletions.
3 changes: 3 additions & 0 deletions .github/workflows/modules/fake_module.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
setenv("INSIDE_GITHUB_ACTIONS", "true")
-- Interfere with PATH so Lmod keeps a record
prepend_path("PATH", "/snap/bin")
2 changes: 1 addition & 1 deletion .github/workflows/scorecards.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ on:
- cron: '25 15 * * 3'
push:
branches:
- '2023.06-software.eessi.io'
- '*-software.eessi.io'

# Declare default permissions as read only.
permissions: read-all
Expand Down
58 changes: 58 additions & 0 deletions .github/workflows/scripts/only_latest_easystacks.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#!/bin/bash
#
# This script figures out the latest version of EasyBuild being used for the installation of easystack
# files.
#
# This file is part of the EESSI software layer, see
# https://github.com/EESSI/software-layer.git
#
# author: Alan O'Cais (CECAM)
#
# license: GPLv2
#

EESSI_VERSION=${EESSI_VERSION:-"2023.06"}

directory="easystacks/software.eessi.io/${EESSI_VERSION}"
# List of example filenames
files=($(find "$directory" -name "*.yml" | grep -e '-eb-'))
[ -n "$DEBUG" ] && echo "${files[@]}"

versions=()
# Loop over each filename
for filename in "${files[@]}"; do
# Extract the semantic version using grep
version=$(echo "$filename" | grep -oP '(?<=eb-)\d+\.\d+\.\d+?(?=-)')

# Output the result
[ -n "$DEBUG" ] && echo "Filename: $filename"
[ -n "$DEBUG" ] && echo "Extracted version: $version"
[ -n "$DEBUG" ] && echo
versions+=("$version")
done
highest_version=$(printf "%s\n" "${versions[@]}" | sort -V | tail -n 1)

[ -n "$DEBUG" ] && echo "Highest version: $highest_version"
[ -n "$DEBUG" ] && echo
[ -n "$DEBUG" ] && echo "Matching files:"
all_latest_easystacks=($(find $directory -type f -name "*eb-$highest_version*.yml"))

accel_latest_easystacks=()
cpu_latest_easystacks=()

# Loop through the array and split based on partial matching of string
accel="/accel/"
for item in "${all_latest_easystacks[@]}"; do
if [[ "$item" == *"$accel"* ]]; then
accel_latest_easystacks+=("$item")
else
cpu_latest_easystacks+=("$item")
fi
done

# Output the results
if [ -n "$ACCEL_EASYSTACKS" ]; then
echo "${accel_latest_easystacks[@]}"
else
echo "${cpu_latest_easystacks[@]}"
fi
50 changes: 50 additions & 0 deletions .github/workflows/scripts/test_init_scripts.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
#!/bin/bash
EESSI_VERSION="2023.06"
export LMOD_PAGER=cat

# initialize assert framework
if [ ! -d assert.sh ]; then
echo "assert.sh not cloned."
echo ""
echo "run \`git clone https://github.com/lehmannro/assert.sh.git\`"
exit 1
fi
. assert.sh/assert.sh

TEST_SHELLS=("bash" "zsh" "fish" "ksh")
SHELLS=$@

for shell in ${SHELLS[@]}; do
echo = | awk 'NF += (OFS = $_) + 100'
echo RUNNING TESTS FOR SHELL: $shell
echo = | awk 'NF += (OFS = $_) + 100'
if [[ ! " ${TEST_SHELLS[*]} " =~ [[:space:]]${shell}[[:space:]] ]]; then
### EXCEPTION FOR CSH ###
echo -e "\033[33mWe don't now how to test the shell '$shell', PRs are Welcome.\033[0m"
else
# TEST 1: Source Script and check Module Output
assert "$shell -c 'source init/lmod/$shell' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully"
# TEST 2: Check if module overviews first section is the loaded EESSI module
MODULE_SECTIONS=($($shell -c "source init/lmod/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'"))
PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/modules/all"
assert_raises 'echo "${MODULE_SECTIONS[1]}" | grep -E "$PATTERN"'
# TEST 3: Check if module overviews second section is the EESSI init module
assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/init/modules"
# Test 4: Load Python module and check version
command="$shell -c 'source init/lmod/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; python --version'"
expected="Python 3.10.8"
assert "$command" "$expected"
# Test 5: Load Python module and check path
PYTHON_PATH=$($shell -c "source init/lmod/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; which python")
PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/software/Python/3\.10\.8-GCCcore-12\.2\.0/bin/python"
echo "$PYTHON_PATH" | grep -E "$PATTERN"
assert_raises 'echo "$PYTHON_PATH" | grep -E "$PATTERN"'

#End Test Suite
assert_end "source_eessi_$shell"
fi
done


# RESET PAGER
export LMOD_PAGER=
53 changes: 49 additions & 4 deletions .github/workflows/test-software.eessi.io.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,21 @@
# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
name: Check for missing software installations in software.eessi.io
on: [push, pull_request, workflow_dispatch]
on:
push:
branches: [ "*-software.eessi.io" ]
pull_request:
workflow_dispatch:
permissions:
contents: read # to fetch code (actions/checkout)
env:
EESSI_ACCELERATOR_TARGETS: |
x86_64/amd/zen2:
- nvidia/cc80
x86_64/amd/zen3:
- nvidia/cc80
jobs:
check_missing:
runs-on: ubuntu-22.04
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
Expand All @@ -17,6 +27,7 @@ jobs:
- aarch64/neoverse_v1
- x86_64/amd/zen2
- x86_64/amd/zen3
- x86_64/amd/zen4
- x86_64/intel/haswell
- x86_64/intel/skylake_avx512
- x86_64/generic
Expand Down Expand Up @@ -44,14 +55,48 @@ jobs:
export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}
export EESSI_OS_TYPE=linux
env | grep ^EESSI | sort
echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)"
for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do
# first check the CPU-only builds for this CPU target
echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml with latest EasyBuild release)"
for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do
if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then
if grep -q 2022b <<<"${easystack_file}"; then
# skip the check of installed software on zen4 for foss/2022b builds
continue
fi
if [[ $easystack_file == *"rebuilds"* ]]; then
# Also handle rebuilds, make a temporary EasyStack file where we clean out all 2022b stuff and use that
new_easystack=$(mktemp pruned_easystackXXX --suffix=.yml)
# first clean out the options then clean out the .eb name
sed '/2022b\|12\.2\.0/,/\.eb/{/\.eb/!d}' "${easystack_file}" | sed '/2022b\|12\.2\.0/d' > $new_easystack
diff --unified=0 "$easystack_file" "$new_easystack" || :
easystack_file="$new_easystack"
fi
fi
echo "check missing installations for ${easystack_file}..."
./check_missing_installations.sh ${easystack_file}
ec=$?
if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
done
# now check the accelerator builds for this CPU target
accelerators=$(echo "${EESSI_ACCELERATOR_TARGETS}" | yq ".${EESSI_SOFTWARE_SUBDIR_OVERRIDE}[]")
if [ -z ${accelerators} ]; then
echo "no accelerator targets defined for ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
else
for accel in ${accelerators}; do
module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all
echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}"
for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do
echo "check missing installations for ${easystack_file}..."
./check_missing_installations.sh ${easystack_file}
ec=$?
if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
done
module unuse ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all
done
fi
- name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
run: |
export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
Expand Down
16 changes: 11 additions & 5 deletions .github/workflows/test_eessi_container_script.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
name: Tests for eessi_container.sh script
on: [push, pull_request, workflow_dispatch]
on:
push:
branches: [ "*-software.eessi.io" ]
pull_request:
workflow_dispatch:
permissions:
contents: read # to fetch code (actions/checkout)
jobs:
eessi_container_script:
runs-on: ubuntu-20.04
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -45,7 +49,8 @@ jobs:
elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then
outfile=out_listrepos.txt
./eessi_container.sh --verbose --list-repos | tee ${outfile}
grep "EESSI" ${outfile}
# make sure that the default EESSI software repository is available
grep "software.eessi.io" ${outfile}
# test use of --list-repos with custom repos.cfg
elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then
Expand All @@ -57,11 +62,12 @@ jobs:
echo "[EESSI/20HT.TP]" >> cfg/repos.cfg
echo "repo_version = 20HT.TP" >> cfg/repos.cfg
./eessi_container.sh --verbose --list-repos | tee ${outfile}
grep "EESSI" ${outfile}
# make sure that the default EESSI software repository is available
grep "software.eessi.io" ${outfile}
export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
./eessi_container.sh --verbose --list-repos | tee ${outfile2}
grep "[EESSI/2023.02]" ${outfile2}
grep "EESSI/20AB.CD" ${outfile2}
# test use of --mode run
elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then
Expand Down
7 changes: 5 additions & 2 deletions .github/workflows/test_licenses.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
name: Test software licenses
on: [push, pull_request]
on:
push:
branches: [ "*-software.eessi.io" ]
pull_request:
permissions:
contents: read # to fetch code (actions/checkout)
jobs:
build:
runs-on: ubuntu-20.04
runs-on: ubuntu-latest
steps:
- name: Check out software-layer repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
Expand Down
9 changes: 6 additions & 3 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
@@ -1,14 +1,17 @@
# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
name: Tests
on: [push, pull_request]
on:
push:
branches: [ "*-software.eessi.io" ]
pull_request:
permissions:
contents: read # to fetch code (actions/checkout)
jobs:
build:
runs-on: ubuntu-20.04
runs-on: ubuntu-latest
strategy:
matrix:
python: [3.6, 3.7, 3.8, 3.9, '3.10']
python: [3.7, 3.8, 3.9, '3.10']
fail-fast: false
steps:
- name: checkout
Expand Down
7 changes: 5 additions & 2 deletions .github/workflows/tests_archdetect.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
name: Tests for eessi_archdetect.sh
on: [push, pull_request]
on:
push:
branches: [ "*-software.eessi.io" ]
pull_request:
permissions:
contents: read # to fetch code (actions/checkout)
jobs:
build:
runs-on: ubuntu-20.04
runs-on: ubuntu-latest
strategy:
matrix:
proc_cpuinfo:
Expand Down
Loading

0 comments on commit f376b9a

Please sign in to comment.