Merge pull request #221 from xylar/add-frontier-amd

Add amdclang support on frontier
E3SM-Project · Jan 20, 2025 · a307aa2 · a307aa2
2 parents 93b0d45 + 162050b
commit a307aa2
Show file tree

Hide file tree

Showing 6 changed files with 424 additions and 0 deletions.
diff --git a/mache/spack/frontier_amdclang_mpich.csh b/mache/spack/frontier_amdclang_mpich.csh
@@ -0,0 +1,20 @@
+module reset >& /dev/null
+module switch Core/24.07 >& /dev/null
+module switch PrgEnv-cray PrgEnv-amd/8.3.3 >& /dev/null
+module switch amd amd/5.4.0 >& /dev/null
+
+{% if e3sm_lapack %}
+module load cray-libsci/22.12.1.1
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+module load cray-hdf5-parallel/1.12.2.1
+module load cray-netcdf-hdf5parallel/4.9.0.1
+module load cray-parallel-netcdf/1.12.3.1
+{% endif %}
+
+{% if e3sm_hdf5_netcdf %}
+setenv NETCDF_C_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX
+setenv NETCDF_FORTRAN_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX
+setenv PNETCDF_PATH $CRAY_PARALLEL_NETCDF_PREFIX
+{% endif %}
+setenv HDF5_USE_FILE_LOCKING FALSE
diff --git a/mache/spack/frontier_amdclang_mpich.sh b/mache/spack/frontier_amdclang_mpich.sh
@@ -0,0 +1,20 @@
+module reset >& /dev/null
+module switch Core/24.07 >& /dev/null
+module switch PrgEnv-cray PrgEnv-amd/8.3.3 >& /dev/null
+module switch amd amd/5.4.0 >& /dev/null
+
+{% if e3sm_lapack %}
+module load cray-libsci/22.12.1.1
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+module load cray-hdf5-parallel/1.12.2.1
+module load cray-netcdf-hdf5parallel/4.9.0.1
+module load cray-parallel-netcdf/1.12.3.1
+{% endif %}
+
+{% if e3sm_hdf5_netcdf %}
+export NETCDF_C_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX
+export NETCDF_FORTRAN_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX
+export PNETCDF_PATH=$CRAY_PARALLEL_NETCDF_PREFIX
+{% endif %}
+export HDF5_USE_FILE_LOCKING=FALSE
diff --git a/mache/spack/frontier_amdclang_mpich.yaml b/mache/spack/frontier_amdclang_mpich.yaml
@@ -0,0 +1,168 @@
+spack:
+  specs:
+  - cray-mpich
+{% if e3sm_lapack %}
+  - cray-libsci
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+  - hdf5
+  - netcdf-c
+  - netcdf-fortran
+  - parallel-netcdf
+{% endif %}
+{{ specs }}
+  concretizer:
+    unify: when_possible
+  packages:
+    all:
+      compiler: [rocmcc@5.4.0]
+      providers:
+        mpi: [cray-mpich@8.1.23]
+{% if e3sm_lapack %}
+        lapack: [cray-libsci@22.12.1.1]
+{% endif %}
+    autoconf:
+      externals:
+      - spec: autoconf@2.69
+        prefix: /usr
+      buildable: false
+    automake:
+      externals:
+      - spec: automake@1.15.1
+        prefix: /usr
+      buildable: false
+    bzip2:
+      externals:
+      - spec: bzip2@1.0.8
+        prefix: /usr
+      buildable: false
+    cmake:
+      externals:
+      - spec: cmake@3.27.9
+        modules:
+        - cmake/3.27.9
+      buildable: false
+    curl:
+      externals:
+      - spec: curl@7.79.1
+        prefix: /usr
+      buildable: false
+    gettext:
+      externals:
+      - spec: gettext@0.20.2
+        prefix: /usr
+      buildable: false
+    gmake:
+      externals:
+      - spec: gmake@4.3
+        prefix: /sw/frontier/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/gmake-4.3-32qt54ujpdxcat7ceg4vcjlmf2ze25j5
+      buildable: false
+    libxml2:
+      externals:
+      - spec: libxml2@2.9.14
+        prefix: /usr
+      buildable: false
+    m4:
+      externals:
+      - spec: m4@1.4.18
+        prefix: /usr
+      buildable: false
+    ncurses:
+      externals:
+      - spec: ncurses@6.1
+        prefix: /usr
+      buildable: false
+    openssl:
+      externals:
+      - spec: openssl@1.1.1l
+        prefix: /usr
+      buildable: false
+    perl:
+      externals:
+      - spec: perl@5.26.1
+        prefix: /usr
+      buildable: false
+    python:
+      externals:
+      - spec: python@3.11.5
+        modules:
+        - cray-python/3.11.5
+      buildable: false
+    tar:
+      externals:
+      - spec: tar@1.34
+        prefix: /usr
+      buildable: false
+    xz:
+      externals:
+      - spec: xz@5.2.3
+        prefix: /usr
+      buildable: false
+    cray-mpich:
+      externals:
+      - spec: cray-mpich@8.1.23
+        prefix: /opt/cray/pe/mpich/8.1.23/ofi/amd/5.0
+        modules:
+        - libfabric/1.15.2.0
+        - cray-mpich/8.1.23
+      buildable: false
+    libfabric:
+      externals:
+      - spec: libfabric@1.15.2.0
+        prefix: /opt/cray/libfabric/1.15.2.0
+        modules:
+        - libfabric/1.15.2.0
+      buildable: false
+{% if e3sm_lapack %}
+    cray-libsci:
+      externals:
+      - spec: cray-libsci@22.12.1.1
+        prefix: /opt/cray/pe/libsci/22.12.1.1/amd/4.0/x86_64
+        modules:
+        - cray-libsci/22.12.1.1
+      buildable: false
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+    hdf5:
+      externals:
+      - spec: hdf5@1.12.2.1~cxx+fortran+hl~java+mpi+shared
+        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.1/amd/4.3
+      buildable: false
+    parallel-netcdf:
+      externals:
+      - spec: parallel-netcdf@1.12.3.1+cxx+fortran+pic+shared
+        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.1/amd/4.3
+      buildable: false
+    netcdf-c:
+      externals:
+      - spec: netcdf-c@4.9.0.1+mpi~parallel-netcdf
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.1/amd/4.3
+      buildable: false
+    netcdf-fortran:
+      externals:
+      - spec: netcdf-fortran@4.5.3
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.1/amd/4.3
+      buildable: false
+{% endif %}
+  compilers:
+  - compiler:
+      spec: rocmcc@5.4.0
+      paths:
+        cc: cc
+        cxx: CC
+        f77: ftn
+        fc: ftn
+      flags: {}
+      operating_system: sles15
+      target: any
+      modules:
+      - Core/24.07
+      - PrgEnv-amd/8.3.3
+      - amd/5.4.0
+      - libfabric/1.15.2.0
+      environment:
+        prepend_path:
+          PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.22__gd067c3f.shasta/lib64/pkgconfig"
+      extra_rpaths:
+      - /opt/cray/pe/gcc-libs
+      - /opt/cray/gcc-libs
diff --git a/mache/spack/frontier_amdclanggpu_mpich.csh b/mache/spack/frontier_amdclanggpu_mpich.csh
@@ -0,0 +1,21 @@
+module reset >& /dev/null
+module switch Core/24.07 >& /dev/null
+module switch PrgEnv-cray PrgEnv-amd/8.3.3 >& /dev/null
+module switch amd amd/5.4.0 >& /dev/null
+module load craype-accel-amd-gfx90a >& /dev/null
+
+{% if e3sm_lapack %}
+module load cray-libsci/22.12.1.1
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+module load cray-hdf5-parallel/1.12.2.1
+module load cray-netcdf-hdf5parallel/4.9.0.1
+module load cray-parallel-netcdf/1.12.3.1
+{% endif %}
+
+{% if e3sm_hdf5_netcdf %}
+setenv NETCDF_C_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX
+setenv NETCDF_FORTRAN_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX
+setenv PNETCDF_PATH $CRAY_PARALLEL_NETCDF_PREFIX
+{% endif %}
+setenv HDF5_USE_FILE_LOCKING FALSE
diff --git a/mache/spack/frontier_amdclanggpu_mpich.sh b/mache/spack/frontier_amdclanggpu_mpich.sh
@@ -0,0 +1,21 @@
+module reset >& /dev/null
+module switch Core/24.07 >& /dev/null
+module switch PrgEnv-cray PrgEnv-amd/8.3.3 >& /dev/null
+module switch amd amd/5.4.0 >& /dev/null
+module load craype-accel-amd-gfx90a >& /dev/null
+
+{% if e3sm_lapack %}
+module load cray-libsci/22.12.1.1
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+module load cray-hdf5-parallel/1.12.2.1
+module load cray-netcdf-hdf5parallel/4.9.0.1
+module load cray-parallel-netcdf/1.12.3.1
+{% endif %}
+
+{% if e3sm_hdf5_netcdf %}
+export NETCDF_C_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX
+export NETCDF_FORTRAN_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX
+export PNETCDF_PATH=$CRAY_PARALLEL_NETCDF_PREFIX
+{% endif %}
+export HDF5_USE_FILE_LOCKING=FALSE