Fix SYCL and PBS flags

E3SM-Project · Oct 29, 2024 · 07acaa9 · 07acaa9
1 parent 4aa27c3
commit 07acaa9
Show file tree

Hide file tree

Showing 5 changed files with 38 additions and 21 deletions.
diff --git a/components/omega/CMakeLists.txt b/components/omega/CMakeLists.txt
@@ -44,7 +44,7 @@ if (NOT DEFINED PROJECT_NAME)
 
   set(CMAKE_CXX_STANDARD 17) # used in E3SM
   set(CMAKE_CXX_STANDARD_REQUIRED ON)
-  set(LINKER_LANGUAGE C)     # needed to support Cray C compiler wrapper
+  set(LINKER_LANGUAGE CXX)   # needed to support Cray C compiler wrapper
 
   # update variables for standalone build
   setup_standalone_build()

diff --git a/components/omega/OmegaBuild.cmake b/components/omega/OmegaBuild.cmake
@@ -29,6 +29,10 @@ macro(common)
   option(OMEGA_DEBUG "Turn on error message throwing (default OFF)." OFF)
   option(OMEGA_LOG_FLUSH "Turn on unbuffered logging (default OFF)." OFF)
 
+  if("${OMEGA_BUILD_TYPE}" STREQUAL "Debug" OR "${OMEGA_BUILD_TYPE}" STREQUAL "DEBUG")
+    set(OMEGA_DEBUG ON)
+  endif()
+
   if(NOT DEFINED OMEGA_CXX_FLAGS)
     set(OMEGA_CXX_FLAGS "")
   endif()
@@ -120,7 +124,7 @@ macro(read_cime_config)
                 break()
 
             elseif("${arg}" STREQUAL "-n" OR "${arg}" STREQUAL "-N" OR
-                   "${arg}" STREQUAL "-c")
+                   "${arg}" STREQUAL "-c" OR "${arg}" STREQUAL "-np")
                 set(SKIP_ARG TRUE)
 
             else()
@@ -219,6 +223,9 @@ macro(init_standalone_build)
     elseif(USE_HIP)
       set(OMEGA_ARCH "HIP")
 
+    elseif(USE_SYCL)
+      set(OMEGA_ARCH "SYCL")
+
     else()
 
       execute_process(
@@ -294,7 +301,11 @@ macro(init_standalone_build)
   set(_CtestScript ${OMEGA_BUILD_DIR}/omega_ctest.sh)
   file(WRITE ${_CtestScript}  "#!/usr/bin/env bash\n\n")
   file(APPEND ${_CtestScript} "source ./omega_env.sh\n\n")
-  file(APPEND ${_CtestScript} "ctest --output-on-failure $* # --rerun-failed\n\n")
+  if(OMEGA_DEBUG)
+    file(APPEND ${_CtestScript} "ctest --output-on-failure --verbose $* # --rerun-failed\n\n")
+  else()
+    file(APPEND ${_CtestScript} "ctest --output-on-failure $* # --rerun-failed\n\n")
+  endif()
 
   # create a profile script
   set(_ProfileScript ${OMEGA_BUILD_DIR}/omega_profile.sh)
@@ -422,10 +433,10 @@ macro(init_standalone_build)
     file(APPEND ${_ProfileScript} "    -o \$OUTFILE ./src/omega.exe 1000")
 
   elseif("${OMEGA_ARCH}" STREQUAL "SYCL")
-    set(CMAKE_CXX_COMPILER ${OMEGA_SYCL_COMPILER})
+    set(CMAKE_CXX_COMPILER ${OMEGA_CXX_COMPILER})
 
-    if(OMEGA_SYCL_FLAGS)
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OMEGA_SYCL_FLAGS}")
+    if(SYCL_FLAGS)
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SYCL_FLAGS}")
     endif()
 
   else()
@@ -458,7 +469,7 @@ macro(init_standalone_build)
 
   message(STATUS "CMAKE_CXX_COMPILER     = ${CMAKE_CXX_COMPILER}")
   message(STATUS "CMAKE_CXX_FLAGS        = ${CMAKE_CXX_FLAGS}")
-#  message(STATUS "CMAKE_EXE_LINKER_FLAGS = ${CMAKE_EXE_LINKER_FLAGS}")
+  message(STATUS "CMAKE_EXE_LINKER_FLAGS = ${CMAKE_EXE_LINKER_FLAGS}")
 
 endmacro()
 
@@ -512,10 +523,6 @@ macro(update_variables)
 
   add_definitions(-DOMEGA_BUILD_MODE=${OMEGA_BUILD_MODE})
 
-  if("${OMEGA_BUILD_TYPE}" STREQUAL "Debug" OR "${OMEGA_BUILD_TYPE}" STREQUAL "DEBUG")
-    set(OMEGA_DEBUG ON)
-  endif()
-
   if(NOT DEFINED OMEGA_LOG_LEVEL)
     set(OMEGA_LOG_LEVEL "INFO")
   endif()

diff --git a/components/omega/src/CMakeLists.txt b/components/omega/src/CMakeLists.txt
@@ -41,7 +41,7 @@ target_link_libraries(
 if(GKlib_FOUND)
     target_link_libraries(
         OmegaLibFlags
-        PUBLIC
+        INTERFACE
         gklib
     )
 endif()
@@ -74,6 +74,6 @@ if(OMEGA_BUILD_EXECUTABLE)
     OmegaLibFlags
   )
 
-  set_target_properties(${OMEGA_EXE_NAME} PROPERTIES LINKER_LANGUAGE C)
+  set_target_properties(${OMEGA_EXE_NAME} PROPERTIES LINKER_LANGUAGE CXX)
 
 endif()
diff --git a/components/omega/src/base/DataTypes.h b/components/omega/src/base/DataTypes.h
@@ -43,6 +43,8 @@ KOKKOS_INLINE_FUNCTION constexpr Real operator""_Real(long double x) {
 using MemSpace = Kokkos::CudaSpace;
 #elif OMEGA_ENABLE_HIP
 using MemSpace = Kokkos::Experimental::HIPSpace;
+#elif OMEGA_ENABLE_SYCL
+using MemSpace = Kokkos::Experimental::SYCLDeviceUSMSpace;
 #elif OMEGA_ENABLE_OPENMP
 using MemSpace = Kokkos::HostSpace;
 #elif OMEGA_ENABLE_SERIAL

diff --git a/components/omega/test/CMakeLists.txt b/components/omega/test/CMakeLists.txt
@@ -18,14 +18,22 @@ function(add_omega_test test_name exe_name source_files mpi_args)
     OmegaLibFlags
   )
 
-  set_target_properties(${exe_name} PROPERTIES LINKER_LANGUAGE C)
+  set_target_properties(${exe_name} PROPERTIES LINKER_LANGUAGE CXX)
 
   # Add the test command
   if (mpi_args)
-    add_test(
-      NAME ${test_name}
-      COMMAND ${OMEGA_MPI_EXEC} ${OMEGA_MPI_ARGS} ${mpi_args} -- ./${exe_name}
-    )
+
+    if("${OMEGA_ARCH}" STREQUAL "SYCL")
+      add_test(
+        NAME ${test_name}
+        COMMAND ${OMEGA_MPI_EXEC} ${mpi_args} ${OMEGA_MPI_ARGS} ./${exe_name}
+      )
+    else()
+      add_test(
+        NAME ${test_name}
+        COMMAND ${OMEGA_MPI_EXEC} ${OMEGA_MPI_ARGS} ${mpi_args} -- ./${exe_name}
+      )
+    endif()
 
   else()
     add_test(
@@ -253,7 +261,7 @@ add_omega_test(
     TEND_PLANE_TEST
     testTendencyTermsPlane.exe
     ocn/TendencyTermsTest.cpp
-    "-n 8;--cpu-bind=cores"
+    "-n 8;"
 )
 target_compile_definitions(
   testTendencyTermsPlane.exe
@@ -265,7 +273,7 @@ add_omega_test(
   TEND_SPHERE_TEST
     testTendencyTermsSphere.exe
     ocn/TendencyTermsTest.cpp
-    "-n 8;--cpu-bind=cores"
+    "-n 8;"
 )
 target_compile_definitions(
   testTendencyTermsSphere.exe
@@ -314,7 +322,7 @@ add_omega_test(
     REDUCTIONS_TEST
     testReductions.exe
     base/ReductionsTest.cpp
-    "-n;2;--cpu-bind=cores"
+    "-n;2;"
 )
 
 ###################