Skip to content

Commit

Permalink
adding easyconfigs: CTranslate2-4.5.0-foss-2023a-CUDA-12.1.1.eb, cpu_…
Browse files Browse the repository at this point in the history
…features-0.9.0-GCCcore-12.3.0.eb and patches: CTranslate2-4.5.0_replace-cxxopts.patch, CTranslate2-4.5.0_fix-third-party.patch, CTranslate2-4.5.0_fix-tests.patch
  • Loading branch information
pavelToman committed Jan 6, 2025
1 parent 741e5ea commit affba08
Show file tree
Hide file tree
Showing 5 changed files with 205 additions and 0 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
easyblock = 'CMakeMake'

name = 'CTranslate2'
version = '4.5.0'
versionsuffix = '-CUDA-%(cudaver)s'

homepage = 'https://opennmt.net/CTranslate2/'
description = "Fast inference engine for Transformer models."

toolchain = {'name': 'foss', 'version': '2023a'}
toolchainopts = {'optarch': False}

source_urls = ['https://github.com/OpenNMT/CTranslate2/archive/']
sources = [{
"download_filename": "v%(version)s.tar.gz",
"filename": SOURCE_TAR_GZ,
}]
patches = [
'CTranslate2-4.5.0_fix-third-party.patch',
'CTranslate2-4.5.0_fix-tests.patch',
'CTranslate2-4.5.0_replace-cxxopts.patch',
]
checksums = [
{'CTranslate2-4.5.0.tar.gz': 'f3040c7c3da5dde022fdc16906c279f3f936c6e79f3df8f998c908bb01a77cfe'},
{'CTranslate2-4.5.0_fix-third-party.patch': '07a6e2842e7261901460e0e1e705db645be366008be184db23778668a3bf8de2'},
{'CTranslate2-4.5.0_fix-tests.patch': '73123eafe612538354be5aa96c750199e1a219a5316800848c3894c1cc6ca2ad'},
{'CTranslate2-4.5.0_replace-cxxopts.patch': 'e378969c2968e2fd57863956a4d2f267731a49d1b890dcc45593d6a310531271'},
]

builddependencies = [
('CMake', '3.26.3'),
('pybind11', '2.11.1'),
('cxxopts', '3.0.0', '', SYSTEM),
('spdlog', '1.11.0'),
('cpu_features', '0.9.0'),
]

dependencies = [
('CUDA', '12.1.1', '', SYSTEM),
('Python', '3.11.3'),
('SciPy-bundle', '2023.07'),
('CUTLASS', '3.4.0', versionsuffix),
('googletest', '1.13.0'),
('PyYAML', '6.0'),
]

# make sure that CTranslate2 libraries are linked to FlexiBLAS, not OpenBLAS
preconfigopts = "export CMAKE_INCLUDE_PATH=$EBROOTFLEXIBLAS/include/flexiblas:${CMAKE_INCLUDE_PATH} && "
preconfigopts += "sed -i 's/openblas/flexiblas/g' %(start_dir)s/CMakeLists.txt && "

configopts = '-DOPENMP_RUNTIME=COMP -DWITH_CUDA=ON -DWITH_MKL=OFF '
configopts += '-DOPENBLAS_INCLUDE_DIR="$EBROOTFLEXIBLAS/include" -DWITH_OPENBLAS=ON '
configopts += '-DCUTLASS_INCLUDE_DIRS="$EBROOTCUTLASS/include" '
configopts += '-DBUILD_TESTS=ON '
configopts += '-DWITH_CUDNN=ON '
configopts += '-DENABLE_CPU_DISPATCH=OFF '

exts_defaultclass = 'PythonPackage'

exts_list = [
('ctranslate2', version, {
'download_dep_fail': True,
'runtest': False,
'sources': ['CTranslate2-%(version)s.tar.gz'],
'start_dir': 'python',
'use_pip': True,
'checksums': ['f3040c7c3da5dde022fdc16906c279f3f936c6e79f3df8f998c908bb01a77cfe'],
}),
]

sanity_check_paths = {
'files': ['bin/ct2-translator', 'lib/libctranslate2.%s' % SHLIB_EXT],
'dirs': ['include/ctranslate2', 'lib/python%(pyshortver)s/site-packages'],
}

sanity_check_commands = [
"ct2-translator --help",
"python -c 'import ctranslate2'",
"python -m pip check",
]

modextrapaths = {'PYTHONPATH': 'lib/python%(pyshortver)s/site-packages'}

moduleclass = 'ai'
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
Author: Pavel Tománek (pavelToman)
Fix compilation of tests + replace third_party GTest by one from EB.
--- tests/CMakeLists.txt.orig 2025-01-03 16:35:23.243495904 +0100
+++ tests/CMakeLists.txt 2025-01-03 17:34:04.044755825 +0100
@@ -2,7 +2,7 @@

option(BUILD_GMOCK "" OFF)
option(INSTALL_GTEST "" OFF)
-add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../third_party/googletest ${CMAKE_CURRENT_BINARY_DIR}/googletest)
+find_package(GTest)

add_executable(ctranslate2_test
batching_test.cc
@@ -21,6 +21,7 @@
target_link_libraries(ctranslate2_test
${PROJECT_NAME}
gtest_main
+ gtest
)

add_executable(benchmark_ops
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
Author: Pavel Tománek (pavelToman)
Replace recursively downloaded third party sw with programs from EB.
--- CMakeLists.txt.orig 2025-01-03 16:15:51.023557000 +0100
+++ CMakeLists.txt 2025-01-03 16:29:21.777198000 +0100
@@ -95,11 +95,10 @@
endif()

find_package(Threads)
-add_subdirectory(third_party/spdlog EXCLUDE_FROM_ALL)
+find_package(spdlog)

set(PRIVATE_INCLUDE_DIRECTORIES
${CMAKE_CURRENT_SOURCE_DIR}/src
- ${CMAKE_CURRENT_SOURCE_DIR}/third_party
)
set(SOURCES
src/allocator.cc
@@ -246,7 +245,7 @@
set(BUILD_SHARED_LIBS_SAVED "${BUILD_SHARED_LIBS}")
set(BUILD_SHARED_LIBS OFF)
set(BUILD_TESTING OFF)
- add_subdirectory(third_party/cpu_features EXCLUDE_FROM_ALL)
+ find_package(CpuFeatures)
set(BUILD_SHARED_LIBS "${BUILD_SHARED_LIBS_SAVED}")
list(APPEND LIBRARIES cpu_features)
endif()
@@ -498,19 +497,6 @@
message(STATUS "NVCC host compiler: ${CUDA_HOST_COMPILER}")
message(STATUS "NVCC compilation flags: ${CUDA_NVCC_FLAGS}")

- # We should ensure that the Thrust include directories appear before
- # -I/usr/local/cuda/include for both GCC and NVCC, so that the headers
- # are coming from the submodule and not the system.
- set(THRUST_INCLUDE_DIRS
- ${CMAKE_CURRENT_SOURCE_DIR}/third_party/thrust/dependencies/cub
- ${CMAKE_CURRENT_SOURCE_DIR}/third_party/thrust
- )
- cuda_include_directories(${THRUST_INCLUDE_DIRS})
- list(APPEND PRIVATE_INCLUDE_DIRECTORIES ${THRUST_INCLUDE_DIRS})
-
- set(CUTLASS_INCLUDE_DIRS
- ${CMAKE_CURRENT_SOURCE_DIR}/third_party/cutlass/include
- )
cuda_include_directories(${CUTLASS_INCLUDE_DIRS})
list(APPEND PRIVATE_INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIRS})

Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
Author: Pavel Tománek (pavelToman)
Replace recursively downloaded third party cxxopts with cxxopts from EB.
--- cli/CMakeLists.txt.orig 2025-01-03 16:47:35.863270652 +0100
+++ cli/CMakeLists.txt 2025-01-03 16:52:02.988342754 +0100
@@ -1,15 +1,9 @@
-if (NOT IS_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../third_party/cxxopts/include")
- message(FATAL_ERROR "The client dependency repository (cxxopts) is missing! "
- "You probably didn't clone the project with --recursive. You can include it "
- "by calling \"git submodule update --init --recursive\"")
-endif()
-
add_executable(translator
translator.cc
)
target_include_directories(translator
- PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/cxxopts/include
- )
+ PRIVATE "$ENV{EBROOTCXXOPTS}/include"
+)
target_link_libraries(translator
PRIVATE ${PROJECT_NAME}
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild
# Author: Denis Kristak, update: Pavel Tománek
easyblock = 'CMakeMake'

name = 'cpu_features'
version = '0.9.0'

homepage = 'https://github.com/google/cpu_features'
description = """A cross-platform C library to retrieve CPU features (such as available instructions) at runtime."""

toolchain = {'name': 'GCCcore', 'version': '12.3.0'}
toolchainopts = {'pic': True}

source_urls = ['https://github.com/google/cpu_features/archive/']
sources = ['v%(version)s.tar.gz']
checksums = ['bdb3484de8297c49b59955c3b22dba834401bc2df984ef5cfc17acbe69c5018e']

builddependencies = [
('CMake', '3.26.3'),
('binutils', '2.40'),
]

modextrapaths = {'CPATH': 'include/cpu_features'}

sanity_check_paths = {
'files': ['bin/list_cpu_features', 'lib/libcpu_features.a'],
'dirs': ['include/cpu_features/']
}

sanity_check_commands = ['list_cpu_features']

moduleclass = 'tools'

0 comments on commit affba08

Please sign in to comment.