From 1caccaa9310efa488a472974b0bb3e05a8316d41 Mon Sep 17 00:00:00 2001 From: Jeremy L Thompson Date: Thu, 2 Jan 2025 09:16:46 -0700 Subject: [PATCH 1/3] ci - disable ROCm CI --- .gitlab-ci.yml | 140 +++++++++++++++++++++++++------------------------ 1 file changed, 72 insertions(+), 68 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e490cb533a..000b0e650a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -252,90 +252,94 @@ noether-cuda: # ---------------------------------------------------------------------------------------- # ROCm backends # ---------------------------------------------------------------------------------------- -noether-rocm: - stage: test:gpu-and-float - tags: - - rocm - interruptible: true - before_script: -# Environment - - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran HIPCC=hipcc - - export NPROC_POOL=4 - - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU - - echo "-------------- CC ------------------" && $CC --version - - echo "-------------- CXX -----------------" && $CXX --version - - echo "-------------- FC ------------------" && $FC --version - - echo "-------------- HIPCC ---------------" && $HIPCC --version - - echo "-------------- GCOV ----------------" && gcov --version -# Libraries for backends -# -- MAGMA from dev branch - - echo "-------------- MAGMA ---------------" - - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe - script: - - rm -f .SUCCESS -# libCEED - - make configure ROCM_DIR=/opt/rocm-6.1.0 OPT='-O -march=native -ffp-contract=fast' - - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ') - - echo "-------------- libCEED -------------" && make info - - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU - - make clean - - make -j$NPROC_CPU -# -- libCEED only tests - - echo "-------------- core tests ----------" - - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json -# Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests - - export PETSC_DIR= PETSC_ARCH= - - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit realsearch=% -# Libraries for examples -# -- PETSc with HIP (minimal) - - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe - - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info - - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search="petsc fluids solids" -# Clang-tidy - - echo "-------------- clang-tidy ----------" && clang-tidy --version - - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --color=always --exit-code -# Report status - - touch .SUCCESS - after_script: - - | - if [ -f .SUCCESS ]; then - lcov --directory . --capture --output-file coverage.info; - bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface; - bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery; - bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends; - bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests; - bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples; - fi - artifacts: - paths: - - build/*.junit - reports: - junit: build/*.junit - performance: performance.json + +# ROCm tests currently disabled + +#noether-rocm: +# stage: test:gpu-and-float +# tags: +# - rocm +# interruptible: true +# before_script: +## Environment +# - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran HIPCC=hipcc +# - export NPROC_POOL=4 +# - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU +# - echo "-------------- CC ------------------" && $CC --version +# - echo "-------------- CXX -----------------" && $CXX --version +# - echo "-------------- FC ------------------" && $FC --version +# - echo "-------------- HIPCC ---------------" && $HIPCC --version +# - echo "-------------- GCOV ----------------" && gcov --version +## Libraries for backends +## -- MAGMA from dev branch +# - echo "-------------- MAGMA ---------------" +# - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe +# script: +# - rm -f .SUCCESS +## libCEED +# - make configure ROCM_DIR=/opt/rocm-6.1.0 OPT='-O -march=native -ffp-contract=fast' +# - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ') +# - echo "-------------- libCEED -------------" && make info +# - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU +# - make clean +# - make -j$NPROC_CPU +## -- libCEED only tests +# - echo "-------------- core tests ----------" +# - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json +## Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests +# - export PETSC_DIR= PETSC_ARCH= +# - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit realsearch=% +## Libraries for examples +## -- PETSc with HIP (minimal) +# - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe +# - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info +# - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search="petsc fluids solids" +## Clang-tidy +# - echo "-------------- clang-tidy ----------" && clang-tidy --version +# - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --color=always --exit-code +## Report status +# - touch .SUCCESS +# after_script: +# - | +# if [ -f .SUCCESS ]; then +# lcov --directory . --capture --output-file coverage.info; +# bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface; +# bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery; +# bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends; +# bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests; +# bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples; +# fi +# artifacts: +# paths: +# - build/*.junit +# reports: +# junit: build/*.junit +# performance: performance.json # ---------------------------------------------------------------------------------------- -# CPU + ROCm backends with CeedScalar == float (32 bit) +# CPU + CUDA backends with CeedScalar == float (32 bit) # ---------------------------------------------------------------------------------------- noether-float: stage: test:gpu-and-float tags: - cpu - - rocm + - cuda interruptible: true before_script: # Environment - - export COVERAGE=1 CC=gcc CXX=g++ FC= HIPCC=hipcc + - export COVERAGE=1 CC=gcc CXX=g++ FC= NVCC=nvcc - export NPROC_POOL=8 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU - echo "-------------- CC ------------------" && $CC --version - echo "-------------- CXX -----------------" && $CXX --version - - echo "-------------- HIPCC ---------------" && $HIPCC --version + - echo "-------------- NVCC ----------------" && $NVCC --version - echo "-------------- GCOV ----------------" && gcov --version # Libraries for backends +# ROCm tests currently disabled # -- MAGMA from dev branch - - echo "-------------- MAGMA ---------------" - - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe +# - echo "-------------- MAGMA ---------------" +# - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe # -- LIBXSMM 7 April 2024 - cd .. && export XSMM_HASH=94ee71576870152feb62f3f0cf6b061d036dcdb5 && { [[ -d libxsmm-$XSMM_HASH ]] || { curl -L https://github.com/libxsmm/libxsmm/archive/$XSMM_HASH.tar.gz -o xsmm.tar.gz && tar zvxf xsmm.tar.gz && rm xsmm.tar.gz && make -C libxsmm-$XSMM_HASH -j$(nproc); }; } && export XSMM_DIR=$PWD/libxsmm-$XSMM_HASH && cd libCEED - echo "-------------- LIBXSMM -------------" && basename $XSMM_DIR @@ -345,7 +349,7 @@ noether-float: # Change to single precision - sed -i 's/ceed-f64/ceed-f32/1' include/ceed/types.h # Build libCEED - - make configure ROCM_DIR=/opt/rocm-6.1.0 OPT='-O -march=native -ffp-contract=fast' + - make configure OPT='-O -march=native -ffp-contract=fast' CUDA_DIR=/usr - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ') - echo "-------------- libCEED -------------" && make info - echo "-------------- BACKENDS_CPU --------" && echo $BACKENDS_CPU @@ -359,7 +363,7 @@ noether-float: - export PETSC_DIR= PETSC_ARCH= - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="float-cpu" junit realsearch=% - export NPROC_POOL=4 - - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="float-hip" junit realsearch=% + - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="float-cuda" junit realsearch=% # Report status - echo "SUCCESS" > .job_status after_script: From 87d77c19c5dfc2c216f0234817a91dd9ccdabce0 Mon Sep 17 00:00:00 2001 From: Jeremy L Thompson Date: Fri, 3 Jan 2025 10:34:50 -0700 Subject: [PATCH 2/3] minor - style --- .gitlab-ci.yml | 112 ++++++++++++++++++++++++------------------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 000b0e650a..a0ea8afcf8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,37 +19,37 @@ noether-asan: - cpu interruptible: true before_script: -# Environment -# Note: COVERAGE=0 is needed when using ASAN + # Environment + # Note: COVERAGE=0 is needed when using ASAN - export COVERAGE=0 CC=gcc CXX=g++ FC=gfortran - export NPROC_POOL=8 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU - echo "-------------- CC ------------------" && $CC --version - echo "-------------- CXX -----------------" && $CXX --version - echo "-------------- FC ------------------" && $FC --version -# ASAN + # ASAN - echo "-------------- ASAN ----------------" - export ASAN=1 AFLAGS="-fsanitize=address -fsanitize=leak" - echo $AFLAGS script: - rm -f .SUCCESS -# libCEED + # libCEED - make configure OPT='-O -march=native -ffp-contract=fast' - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') - echo "-------------- libCEED -------------" && make info - echo "-------------- BACKENDS_CPU --------" && echo $BACKENDS_CPU - make clean - make -j$NPROC_CPU -# -- libCEED only tests + # -- libCEED only tests - echo "-------------- core tests ----------" - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json -# Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests + # Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests - export PETSC_DIR= PETSC_ARCH= - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="memcheck" junit realsearch=% -# Clang-tidy + # Clang-tidy - echo "-------------- clang-tidy ----------" && clang-tidy --version - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --color=always --exit-code -# Report status + # Report status - touch .SUCCESS artifacts: paths: @@ -68,7 +68,7 @@ noether-cpu: - cpu interruptible: true before_script: -# Environment + # Environment - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran - export NPROC_POOL=8 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU @@ -76,16 +76,16 @@ noether-cpu: - echo "-------------- CXX -----------------" && $CXX --version - echo "-------------- FC ------------------" && $FC --version - echo "-------------- GCOV ----------------" && gcov --version -# Libraries for backends -# -- LIBXSMM 7 April 2024 + # Libraries for backends + # -- LIBXSMM 7 April 2024 - cd .. && export XSMM_HASH=94ee71576870152feb62f3f0cf6b061d036dcdb5 && { [[ -d libxsmm-$XSMM_HASH ]] || { curl -L https://github.com/libxsmm/libxsmm/archive/$XSMM_HASH.tar.gz -o xsmm.tar.gz && tar zvxf xsmm.tar.gz && rm xsmm.tar.gz && make -C libxsmm-$XSMM_HASH -j$(nproc); }; } && export XSMM_DIR=$PWD/libxsmm-$XSMM_HASH && cd libCEED - echo "-------------- LIBXSMM -------------" && basename $XSMM_DIR -# -- OCCA v1.6.0 + # -- OCCA v1.6.0 - cd .. && export OCCA_VERSION=occa-1.6.0 && { [[ -d $OCCA_VERSION ]] || { git clone --depth 1 --branch v1.6.0 https://github.com/libocca/occa.git $OCCA_VERSION && cd $OCCA_VERSION && export ENABLE_OPENCL="OFF" ENABLE_DPCPP="OFF" ENABLE_HIP="OFF" ENABLE_CUDA="OFF" && ./configure-cmake.sh && cmake --build build --parallel $NPROC_CPU && cmake --install build && cd ..; }; } && export OCCA_DIR=$PWD/$OCCA_VERSION/install && cd libCEED - echo "-------------- OCCA ----------------" && git -C $OCCA_DIR describe --tags && LD_LIBRARY_PATH=$OCCA_DIR/lib $OCCA_DIR/bin/occa info script: - rm -f .SUCCESS -# libCEED + # libCEED - make configure OPT='-O -march=native -ffp-contract=fast' - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') - echo "-------------- libCEED -------------" && make info @@ -93,33 +93,33 @@ noether-cpu: - make clean - OCCA_DIR= PEDANTIC=1 make -j$NPROC_CPU - make -j$NPROC_CPU -# -- libCEED only tests + # -- libCEED only tests - echo "-------------- core tests ----------" - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json -# Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests + # Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests - export PETSC_DIR= PETSC_ARCH= - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit realsearch=% -# Libraries for examples -# -- PETSc (minimal) + # Libraries for examples + # -- PETSc (minimal) - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-cpu-int64 && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search="petsc fluids-navierstokes solids" -# -- MFEM v4.7 + # -- MFEM v4.7 - cd .. && export MFEM_VERSION=mfem-4.7 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.7 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED - echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=mfem -# -- Nek5000 v19.0 + # -- Nek5000 v19.0 - export COVERAGE=0 - cd .. && export NEK5K_VERSION=Nek5000-19.0 && { [[ -d $NEK5K_VERSION ]] || { git clone --depth 1 --branch v19.0 https://github.com/Nek5000/Nek5000.git $NEK5K_VERSION && cd $NEK5K_VERSION/tools && ./maketools genbox genmap reatore2 && cd ../..; }; } && export NEK5K_DIR=$PWD/$NEK5K_VERSION && export PATH=$NEK5K_DIR/bin:$PATH MPI=0 && cd libCEED - echo "-------------- Nek5000 -------------" && git -C $NEK5K_DIR describe --tags - export NPROC_POOL=1 - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=nek NEK5K_DIR=$NEK5K_DIR -# -- deal.II 8bd5c262f13e15793aa206b6eed8774a9b25ce11 + # -- deal.II 8bd5c262f13e15793aa206b6eed8774a9b25ce11 - OCCA_DIR= BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') - export DEAL_II_ROOT_DIR=/projects/dealii DEAL_II_DIR=/projects/dealii/install - echo "-------------- deal.II -------------" && git -C $DEAL_II_ROOT_DIR -c safe.directory=$DEAL_II_ROOT_DIR describe --always - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=dealii DEAL_II_DIR=$DEAL_II_DIR -# Report status + # Report status - touch .SUCCESS after_script: - | @@ -148,7 +148,7 @@ noether-sycl: - sycl interruptible: true before_script: -# Environment + # Environment - . /opt/intel/oneapi/setvars.sh - export COVERAGE=1 CC=icx CXX=icpx - export NPROC_POOL=8 @@ -157,14 +157,14 @@ noether-sycl: - echo "-------------- CXX -----------------" && $CXX --version script: - rm -f .SUCCESS -# libCEED + # libCEED - make configure SYCL_DIR=/opt/intel/oneapi/compiler/latest OPT='-O -march=native -ffp-contract=fast' - BACKENDS_SYCL=$(make info-backends-all | grep -o '/sycl[^ ]*' | tr '\n' ' ') - echo "-------------- libCEED -------------" && make info - echo "-------------- BACKENDS_SYCL -------" && echo $BACKENDS_SYCL - make clean - make -j$NPROC_CPU -# Report status + # Report status - touch .SUCCESS @@ -177,7 +177,7 @@ noether-cuda: - cuda interruptible: true before_script: -# Environment + # Environment - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran NVCC=nvcc - export NPROC_POOL=4 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU @@ -186,51 +186,51 @@ noether-cuda: - echo "-------------- FC ------------------" && $FC --version - echo "-------------- NVCC ----------------" && $NVCC --version - echo "-------------- GCOV ----------------" && gcov --version -# ASAN + # ASAN - echo "-------------- ASAN ----------------" - export ASAN=1 AFLAGS="-fsanitize=address -fsanitize=leak" ASAN_OPTIONS=protect_shadow_gap=0 - echo $AFLAGS script: - rm -f .SUCCESS -# libCEED + # libCEED - make configure OPT='-O -march=native -ffp-contract=fast' CUDA_DIR=/usr - echo "-------------- libCEED -------------" && make info - BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ') - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU - make clean - PEDANTIC=1 make -k -j$NPROC_CPU -l$NPROC_CPU -# -- libCEED only tests + # -- libCEED only tests - echo "-------------- core tests ----------" -# Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests + # Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests - export PETSC_DIR= PETSC_ARCH= - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit realsearch=% -# Rebuild without ASAN + # Rebuild without ASAN - unset ASAN AFLAGS ASAN_OPTIONS - make clean - PEDANTIC=1 make -k -j$NPROC_CPU -l$NPROC_CPU -# Libraries for examples -# -- PETSc with CUDA (minimal) + # Libraries for examples + # -- PETSc with CUDA (minimal) - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-cuda-O PETSC_OPTIONS='-use_gpu_aware_mpi 0' && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit search="petsc fluids solids" -# -- MFEM v4.7 + # -- MFEM v4.7 - cd .. && export MFEM_VERSION=mfem-4.7 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.7 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED - echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit search=mfem -# -- Nek5000 v19.0 + # -- Nek5000 v19.0 - export COVERAGE=0 - cd .. && export NEK5K_VERSION=Nek5000-19.0 && { [[ -d $NEK5K_VERSION ]] || { git clone --depth 1 --branch v19.0 https://github.com/Nek5000/Nek5000.git $NEK5K_VERSION && cd $NEK5K_VERSION/tools && ./maketools genbox genmap reatore2 && cd ../..; }; } && export NEK5K_DIR=$PWD/$NEK5K_VERSION && export PATH=$NEK5K_DIR/bin:$PATH MPI=0 && cd libCEED - echo "-------------- Nek5000 -------------" && git -C $NEK5K_DIR describe --tags - export NPROC_POOL=1 - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit search=nek NEK5K_DIR=$NEK5K_DIR -# -- deal.II 8bd5c262f13e15793aa206b6eed8774a9b25ce11 + # -- deal.II 8bd5c262f13e15793aa206b6eed8774a9b25ce11 - export DEAL_II_ROOT_DIR=/projects/dealii DEAL_II_DIR=/projects/dealii/install - echo "-------------- deal.II -------------" && git -C $DEAL_II_ROOT_DIR -c safe.directory=$DEAL_II_ROOT_DIR describe --always - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit search=dealii DEAL_II_DIR=$DEAL_II_DIR -# Clang-tidy + # Clang-tidy - echo "-------------- clang-tidy ----------" && clang-tidy --version - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --color=always --exit-code -# Report status + # Report status - touch .SUCCESS after_script: - | @@ -261,7 +261,7 @@ noether-cuda: # - rocm # interruptible: true # before_script: -## Environment +# # Environment # - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran HIPCC=hipcc # - export NPROC_POOL=4 # - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU @@ -270,34 +270,34 @@ noether-cuda: # - echo "-------------- FC ------------------" && $FC --version # - echo "-------------- HIPCC ---------------" && $HIPCC --version # - echo "-------------- GCOV ----------------" && gcov --version -## Libraries for backends -## -- MAGMA from dev branch +# # Libraries for backends +# # -- MAGMA from dev branch # - echo "-------------- MAGMA ---------------" # - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe # script: # - rm -f .SUCCESS -## libCEED +# # libCEED # - make configure ROCM_DIR=/opt/rocm-6.1.0 OPT='-O -march=native -ffp-contract=fast' # - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ') # - echo "-------------- libCEED -------------" && make info # - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU # - make clean # - make -j$NPROC_CPU -## -- libCEED only tests +# # -- libCEED only tests # - echo "-------------- core tests ----------" # - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json -## Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests +# # Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests # - export PETSC_DIR= PETSC_ARCH= # - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit realsearch=% -## Libraries for examples -## -- PETSc with HIP (minimal) +# # Libraries for examples +# # -- PETSc with HIP (minimal) # - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe # - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info # - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search="petsc fluids solids" -## Clang-tidy +# # Clang-tidy # - echo "-------------- clang-tidy ----------" && clang-tidy --version # - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --color=always --exit-code -## Report status +# # Report status # - touch .SUCCESS # after_script: # - | @@ -327,7 +327,7 @@ noether-float: - cuda interruptible: true before_script: -# Environment + # Environment - export COVERAGE=1 CC=gcc CXX=g++ FC= NVCC=nvcc - export NPROC_POOL=8 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU @@ -335,20 +335,20 @@ noether-float: - echo "-------------- CXX -----------------" && $CXX --version - echo "-------------- NVCC ----------------" && $NVCC --version - echo "-------------- GCOV ----------------" && gcov --version -# Libraries for backends + # Libraries for backends # ROCm tests currently disabled # -- MAGMA from dev branch # - echo "-------------- MAGMA ---------------" # - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe -# -- LIBXSMM 7 April 2024 + # -- LIBXSMM 7 April 2024 - cd .. && export XSMM_HASH=94ee71576870152feb62f3f0cf6b061d036dcdb5 && { [[ -d libxsmm-$XSMM_HASH ]] || { curl -L https://github.com/libxsmm/libxsmm/archive/$XSMM_HASH.tar.gz -o xsmm.tar.gz && tar zvxf xsmm.tar.gz && rm xsmm.tar.gz && make -C libxsmm-$XSMM_HASH -j$(nproc); }; } && export XSMM_DIR=$PWD/libxsmm-$XSMM_HASH && cd libCEED - echo "-------------- LIBXSMM -------------" && basename $XSMM_DIR script: - rm -f .SUCCESS -# libCEED -# Change to single precision + # libCEED + # Change to single precision - sed -i 's/ceed-f64/ceed-f32/1' include/ceed/types.h -# Build libCEED + # Build libCEED - make configure OPT='-O -march=native -ffp-contract=fast' CUDA_DIR=/usr - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ') - echo "-------------- libCEED -------------" && make info @@ -356,15 +356,15 @@ noether-float: - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU - make clean - make -j$NPROC_CPU -# -- libCEED only tests + # -- libCEED only tests - echo "-------------- core tests ----------" - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json -# Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests + # Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests - export PETSC_DIR= PETSC_ARCH= - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="float-cpu" junit realsearch=% - export NPROC_POOL=4 - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="float-cuda" junit realsearch=% -# Report status + # Report status - echo "SUCCESS" > .job_status after_script: - | From ff186337b341b423400898116d8fc0cf1bb9b9bf Mon Sep 17 00:00:00 2001 From: Jeremy L Thompson Date: Fri, 3 Jan 2025 10:35:35 -0700 Subject: [PATCH 3/3] ci - temporarily disable SYCL tests --- .gitlab-ci.yml | 51 ++++++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a0ea8afcf8..27b8f62cc1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -142,30 +142,33 @@ noether-cpu: # ---------------------------------------------------------------------------------------- # Check SYCL backends build # ---------------------------------------------------------------------------------------- -noether-sycl: - stage: test:gpu-and-float - tags: - - sycl - interruptible: true - before_script: - # Environment - - . /opt/intel/oneapi/setvars.sh - - export COVERAGE=1 CC=icx CXX=icpx - - export NPROC_POOL=8 - - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU - - echo "-------------- CC ------------------" && $CC --version - - echo "-------------- CXX -----------------" && $CXX --version - script: - - rm -f .SUCCESS - # libCEED - - make configure SYCL_DIR=/opt/intel/oneapi/compiler/latest OPT='-O -march=native -ffp-contract=fast' - - BACKENDS_SYCL=$(make info-backends-all | grep -o '/sycl[^ ]*' | tr '\n' ' ') - - echo "-------------- libCEED -------------" && make info - - echo "-------------- BACKENDS_SYCL -------" && echo $BACKENDS_SYCL - - make clean - - make -j$NPROC_CPU - # Report status - - touch .SUCCESS + +# SYCL tests currently disabled + +#noether-sycl: +# stage: test:gpu-and-float +# tags: +# - sycl +# interruptible: true +# before_script: +# # Environment +# - . /opt/intel/oneapi/setvars.sh +# - export COVERAGE=1 CC=icx CXX=icpx +# - export NPROC_POOL=8 +# - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU +# - echo "-------------- CC ------------------" && $CC --version +# - echo "-------------- CXX -----------------" && $CXX --version +# script: +# - rm -f .SUCCESS +# # libCEED +# - make configure SYCL_DIR=/opt/intel/oneapi/compiler/latest OPT='-O -march=native -ffp-contract=fast' +# - BACKENDS_SYCL=$(make info-backends-all | grep -o '/sycl[^ ]*' | tr '\n' ' ') +# - echo "-------------- libCEED -------------" && make info +# - echo "-------------- BACKENDS_SYCL -------" && echo $BACKENDS_SYCL +# - make clean +# - make -j$NPROC_CPU +# # Report status +# - touch .SUCCESS # ----------------------------------------------------------------------------------------