diff --git a/.github/workflows/ci-build-release-wheels.yaml b/.github/workflows/ci-build-release-wheels.yaml index d57e0ed..11edb6e 100644 --- a/.github/workflows/ci-build-release-wheels.yaml +++ b/.github/workflows/ci-build-release-wheels.yaml @@ -58,7 +58,6 @@ jobs: uses: docker/setup-qemu-action@v2 - uses: docker/setup-buildx-action@v2 - - run: build-support/copy-deps-versionfile.sh - name: Build Manylinux Docker image uses: docker/build-push-action@v3 @@ -95,7 +94,7 @@ jobs: mac-wheels: name: Wheel MacOS Universal2 - Py ${{matrix.py.version}} - runs-on: macos-12 + runs-on: macos-14 timeout-minutes: 300 strategy: @@ -112,30 +111,8 @@ jobs: - name: checkout uses: actions/checkout@v3 - - name: Dependencies cache - uses: actions/cache@v3 - id: cache-deps - with: - path: .pulsar-mac-build/deps/install - key: ${{matrix.py.version_long}}-${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pkg/mac/*') }} - - - name: Build dependencies - if: steps.cache-deps.outputs.cache-hit != 'true' - run: pkg/mac/build-dependencies.sh ${{matrix.py.version}} ${{matrix.py.version_long}} - - - name: Pulsar C++ cache - uses: actions/cache@v3 - id: cache-cpp - with: - path: .pulsar-mac-build/cpp/install - key: ${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pulsar-client-cpp-version.txt') }}-${{ hashFiles('pkg/mac/*') }} - - - name: Build Pulsar C++ client - if: steps.cache-cpp.outputs.cache-hit != 'true' - run: pkg/mac/build-pulsar-cpp.sh - - name: Build and test Mac wheels - run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}} + run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}} ${{matrix.py.version_long}} - name: Upload artifacts uses: actions/upload-artifact@v3 diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 213ddb3..262808d 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -127,7 +127,6 @@ jobs: uses: docker/setup-qemu-action@v2 - uses: docker/setup-buildx-action@v2 - - run: build-support/copy-deps-versionfile.sh - name: Build Manylinux Docker image uses: docker/build-push-action@v3 @@ -157,7 +156,7 @@ jobs: mac-wheels: name: Wheel MacOS Universal2 - Py ${{matrix.py.version}} needs: unit-tests - runs-on: macos-12 + runs-on: macos-14 timeout-minutes: 300 strategy: @@ -169,33 +168,9 @@ jobs: - name: checkout uses: actions/checkout@v3 - - name: Dependencies cache - uses: actions/cache@v3 - id: cache-deps - with: - path: .pulsar-mac-build/deps/install - key: ${{matrix.py.version_long}}-${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pkg/mac/*') }} - - - name: Install Python deps - run: pip3 install -U pyyaml - - - name: Build dependencies - if: steps.cache-deps.outputs.cache-hit != 'true' - run: pkg/mac/build-dependencies.sh ${{matrix.py.version}} ${{matrix.py.version_long}} - - - name: Pulsar C++ cache - uses: actions/cache@v3 - id: cache-cpp - with: - path: .pulsar-mac-build/cpp/install - key: ${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pulsar-client-cpp-version.txt') }}-${{ hashFiles('pkg/mac/*') }} - - - name: Build Pulsar C++ client - if: steps.cache-cpp.outputs.cache-hit != 'true' - run: pkg/mac/build-pulsar-cpp.sh - - name: Build and test Mac wheels - run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}} + run: | + pkg/mac/build-mac-wheels.sh ${{matrix.py.version}} ${{matrix.py.version_long}} windows-wheels: name: "Python ${{ matrix.python.version }} Wheel on Windows x64" diff --git a/CMakeLists.txt b/CMakeLists.txt index 084e078..660a1b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,13 +38,9 @@ if (MSVC) endif () if (LINK_STATIC) - if (MSVC) - find_library(PULSAR_LIBRARY NAMES pulsarWithDeps.lib) - else () - find_library(PULSAR_LIBRARY NAMES libpulsar.a) - endif () + find_library(PULSAR_LIBRARY NAMES pulsarwithdeps pulsarWithDeps.lib) add_definitions("-DPULSAR_STATIC") -else() +else () find_library(PULSAR_LIBRARY NAMES pulsar libpulsar) endif() message(STATUS "PULSAR_LIBRARY: ${PULSAR_LIBRARY}") @@ -88,43 +84,11 @@ set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} Python3::Module) message(STATUS "All libraries: ${PYTHON_WRAPPER_LIBS}") if (LINK_STATIC AND NOT MSVC) - set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") - - # We need to include all the static libs individually because we cannot easily create a universal2 libpulsar.a - # with all the deps included. - find_package(OpenSSL REQUIRED) - message("OPENSSL_LIBRARIES: " ${OPENSSL_LIBRARIES}) - - find_package(Protobuf REQUIRED) - message("Protobuf_LIBRARIES: " ${Protobuf_LIBRARIES}) - - find_package(curl REQUIRED) - message("CURL_LIBRARIES: " ${CURL_LIBRARIES}) - - find_package(zlib) - message("ZLIB_LIBRARIES: " ${ZLIB_LIBRARIES}) - - find_library(LIB_ZSTD NAMES libzstd.a) - message(STATUS "ZStd: ${LIB_ZSTD}") - find_library(LIB_SNAPPY NAMES libsnappy.a) - message(STATUS "LIB_SNAPPY: ${LIB_SNAPPY}") - - set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} - ${OPENSSL_LIBRARIES} - ${Protobuf_LIBRARIES} - ${ZLIB_LIBRARIES} - ${LIB_ZSTD} - ${LIB_SNAPPY} - ${CURL_LIBRARIES} - ) - if (APPLE) set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup") - target_link_libraries(_pulsar -Wl,-all_load ${PYTHON_WRAPPER_LIBS}) + target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS}) else () - if (NOT MSVC) - set (CMAKE_SHARED_LINKER_FLAGS " -static-libgcc -static-libstdc++") - endif() + set (CMAKE_SHARED_LINKER_FLAGS " -static-libgcc -static-libstdc++") target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS}) endif () elseif (LINK_STATIC) # MSVC diff --git a/build-support/copy-deps-versionfile.sh b/build-support/copy-deps-versionfile.sh deleted file mode 100755 index 0e1e135..0000000 --- a/build-support/copy-deps-versionfile.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -e - -ROOT_DIR=$(git rev-parse --show-toplevel) - -for dir in manylinux2014 manylinux_musl; do - mkdir -p pkg/$dir/.build - cp $ROOT_DIR/dependencies.yaml pkg/$dir/.build - cp $ROOT_DIR/build-support/dep-version.py pkg/$dir/.build - cp $ROOT_DIR/build-support/dep-url.sh pkg/$dir/.build -done diff --git a/build-support/dep-url.sh b/build-support/dep-url.sh index 92fb317..7670bb9 100644 --- a/build-support/dep-url.sh +++ b/build-support/dep-url.sh @@ -38,38 +38,15 @@ download_dependency() { VERSION=$(grep $DEP $DEP_FILE | sed 's/://' | awk '{print $2}') case $DEP in - "cmake") - URL=https://github.com/Kitware/CMake/releases/download/v${VERSION}/cmake-${VERSION}-linux-${ARCH}.tar.gz - ;; "pulsar-cpp") URL=$(pulsar_cpp_base_url $VERSION)/apache-pulsar-client-cpp-${VERSION}.tar.gz ;; "pybind11") URL=https://github.com/pybind/pybind11/archive/refs/tags/v${VERSION}.tar.gz ;; - "boost") - VERSION_UNDERSCORE=$(echo $VERSION | sed 's/\./_/g') - URL=https://boostorg.jfrog.io/artifactory/main/release/${VERSION}/source/boost_${VERSION_UNDERSCORE}.tar.gz - ;; - "protobuf") - URL=https://github.com/google/protobuf/releases/download/v${VERSION}/protobuf-cpp-${VERSION}.tar.gz - ;; - "zlib") - URL=https://github.com/madler/zlib/archive/v${VERSION}.tar.gz - ;; - "zstd") - URL=https://github.com/facebook/zstd/releases/download/v${VERSION}/zstd-${VERSION}.tar.gz - ;; - "snappy") - URL=https://github.com/google/snappy/archive/refs/tags/${VERSION}.tar.gz - ;; "openssl") URL=https://github.com/openssl/openssl/archive/OpenSSL_$(echo $VERSION | sed 's/\./_/g').tar.gz ;; - "curl") - VERSION_UNDERSCORE=$(echo $VERSION | sed 's/\./_/g') - URL=https://github.com/curl/curl/releases/download/curl-${VERSION_UNDERSCORE}/curl-${VERSION}.tar.gz - ;; *) echo "Unknown dependency $DEP for version $VERSION" exit 1 diff --git a/build-support/pulsar-test-service-start.sh b/build-support/pulsar-test-service-start.sh index ed1e7ba..f8e26f9 100755 --- a/build-support/pulsar-test-service-start.sh +++ b/build-support/pulsar-test-service-start.sh @@ -25,7 +25,7 @@ cd $SRC_DIR ./build-support/pulsar-test-service-stop.sh -CONTAINER_ID=$(docker run -i --user $(id -u) -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:latest sleep 3600) +CONTAINER_ID=$(docker run -i --user $(id -u) -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:4.0.0 sleep 3600) echo $CONTAINER_ID > .tests-container-id.txt docker cp tests/test-conf $CONTAINER_ID:/pulsar/test-conf diff --git a/dependencies.yaml b/dependencies.yaml index dd435ee..d58263f 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -17,13 +17,7 @@ # under the License. # -cmake: 3.24.2 pulsar-cpp: 3.5.1 pybind11: 2.10.1 -boost: 1.80.0 -protobuf: 3.20.0 -zlib: 1.2.13 -zstd: 1.5.2 -snappy: 1.1.9 +# The OpenSSL dependency is only used when building Python from source openssl: 1.1.1q -curl: 8.4.0 diff --git a/pkg/build-wheel-inside-docker.sh b/pkg/build-wheel-inside-docker.sh index c062607..0e73eae 100755 --- a/pkg/build-wheel-inside-docker.sh +++ b/pkg/build-wheel-inside-docker.sh @@ -21,19 +21,39 @@ set -e -x cd /pulsar-client-python +source build-support/dep-url.sh -PYBIND11_VERSION=$(./build-support/dep-version.py pybind11) -curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz -tar zxf v${PYBIND11_VERSION}.tar.gz -rm -rf pybind11 -mv pybind11-${PYBIND11_VERSION} pybind11 +# Build cpp wheels +PULSAR_CPP_VERSION=$(cat ./dependencies.yaml | grep pulsar-cpp | awk '{print $2}') -rm -f CMakeCache.txt CMakeFiles +if [ $CPP_BINARY_TYPE == "rpm" ]; then + if [ $ARCH == "aarch64" ]; then + RPM_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/rpm-arm64/aarch64 + else + RPM_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/rpm-x86_64/x86_64 + fi + curl -O -L $RPM_ROOT_DIR/apache-pulsar-client-$PULSAR_CPP_VERSION-1.$ARCH.rpm + curl -O -L $RPM_ROOT_DIR/apache-pulsar-client-devel-$PULSAR_CPP_VERSION-1.$ARCH.rpm + curl -O -L $RPM_ROOT_DIR/apache-pulsar-client-debuginfo-$PULSAR_CPP_VERSION-1.$ARCH.rpm + rpm -ivh *.rpm +else # apk + if [ $ARCH == "aarch64" ]; then + APK_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/apk-arm64/aarch64 + else + APK_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/apk-x86_64/x86_64 + fi + curl -O -L $APK_ROOT_DIR/apache-pulsar-client-$PULSAR_CPP_VERSION-r0.apk + curl -O -L $APK_ROOT_DIR/apache-pulsar-client-dev-$PULSAR_CPP_VERSION-r0.apk + apk add --allow-untrusted *.apk +fi -cmake . \ - -DCMAKE_BUILD_TYPE=Release +download_dependency $PWD/dependencies.yaml pybind11 +rm -rf pybind11 +mv pybind11-* pybind11 -make -j4 +cmake -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build -j8 +mv build/lib_pulsar.so . ./setup.py bdist_wheel diff --git a/pkg/mac/build-dependencies.sh b/pkg/mac/build-dependencies.sh deleted file mode 100755 index 43faaba..0000000 --- a/pkg/mac/build-dependencies.sh +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -e -x - -ROOT_DIR=$(git rev-parse --show-toplevel) -cd "${ROOT_DIR}" - -PYTHON_VERSION=$1 -PYTHON_VERSION_LONG=$2 - -source pkg/mac/common.sh -source build-support/dep-url.sh - -pip3 install pyyaml setuptools - -dep=$ROOT_DIR/build-support/dep-version.py -PYBIND11_VERSION=$($dep pybind11) -BOOST_VERSION=$($dep boost) -ZLIB_VERSION=$($dep zlib) -OPENSSL_VERSION=$($dep openssl) -PROTOBUF_VERSION=$($dep protobuf) -ZSTD_VERSION=$($dep zstd) -SNAPPY_VERSION=$($dep snappy) -CURL_VERSION=$($dep curl) - -# Compile and cache dependencies -CACHE_DIR=${CACHE_DIR_DEPS} -mkdir -p $CACHE_DIR -cd $CACHE_DIR - -PREFIX=$CACHE_DIR/install - -############################################################################### -if [ ! -f pybind11/.done ]; then - download_dependency $ROOT_DIR/dependencies.yaml pybind11 - mkdir -p $PREFIX/include/ - cp -rf pybind11-${PYBIND11_VERSION}/include/pybind11 $PREFIX/include/ - mkdir -p pybind11 - touch pybind11/.done -fi - -############################################################################### -if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then - echo "Building ZLib" - download_dependency $ROOT_DIR/dependencies.yaml zlib - pushd zlib-$ZLIB_VERSION - CFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX - make -j16 - make install - touch .done - popd -else - echo "Using cached ZLib" -fi - -############################################################################### -OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') -if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then - echo "Building OpenSSL" - download_dependency $ROOT_DIR/dependencies.yaml openssl - # -arch arm64 -arch x86_64 - tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz - - mv openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64 - pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64 - echo -e "#include \n$(cat test/v3ext.c)" > test/v3ext.c - CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - ./Configure --prefix=$PREFIX no-shared no-unit-test darwin64-arm64-cc - make -j8 - make install_sw - popd - - tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz - mv openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64 - pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64 - echo -e "#include \n$(cat test/v3ext.c)" > test/v3ext.c - CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - ./Configure --prefix=$PREFIX no-shared no-unit-test darwin64-x86_64-cc - make -j8 - make install_sw - popd - - # Create universal binaries - lipo -create openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64/libssl.a openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64/libssl.a \ - -output $PREFIX/lib/libssl.a - lipo -create openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64/libcrypto.a openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64/libcrypto.a \ - -output $PREFIX/lib/libcrypto.a - - touch openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done -else - echo "Using cached OpenSSL" -fi - -############################################################################### -if [ ! -f Python-${PYTHON_VERSION_LONG}/.done ]; then - echo "Building Python $PYTHON_VERSION_LONG" - curl -O -L https://www.python.org/ftp/python/${PYTHON_VERSION_LONG}/Python-${PYTHON_VERSION_LONG}.tgz - tar xfz Python-${PYTHON_VERSION_LONG}.tgz - - pushd Python-${PYTHON_VERSION_LONG} - if [ $PYTHON_VERSION = '3.7' ]; then - patch -p1 < ${ROOT_DIR}/pkg/mac/python-3.7.patch - fi - - CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET} -I${PREFIX}/include ${PY_CFLAGS}" \ - LDFLAGS=" ${PY_CFLAGS} -L${PREFIX}/lib" \ - ./configure --prefix=$PREFIX --enable-shared --enable-universalsdk \ - --with-universal-archs=universal2 \ - --with-openssl=$PREFIX - make -j16 - make install - - curl -O -L https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl - $PREFIX/bin/pip3 install wheel setuptools - $PREFIX/bin/pip3 install wheel-*.whl - - touch .done - popd -else - echo "Using cached Python $PYTHON_VERSION_LONG" -fi - -############################################################################### -BOOST_VERSION_=${BOOST_VERSION//./_} -if [ ! -f boost/.done ]; then - echo "Building Boost for Py $PYTHON_VERSION" - download_dependency $ROOT_DIR/dependencies.yaml boost - cp -rf boost_${BOOST_VERSION_}/boost $PREFIX/include/ - mkdir -p boost - touch .done -fi - -############################################################################### -if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then - echo "Building Protobuf" - download_dependency $ROOT_DIR/dependencies.yaml protobuf - pushd protobuf-${PROTOBUF_VERSION} - # Install by CMake so that the dependency can be found with CMake config mode - pushd cmake/ - cmake -B build -DCMAKE_CXX_FLAGS="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - -DCMAKE_INSTALL_PREFIX=$PREFIX - cmake --build build -j16 --target install - popd - touch .done - popd -else - echo "Using cached Protobuf" -fi - -############################################################################### -if [ ! -f zstd-${ZSTD_VERSION}/.done ]; then - echo "Building ZStd" - download_dependency $ROOT_DIR/dependencies.yaml zstd - pushd zstd-${ZSTD_VERSION} - CFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX \ - make -j16 -C lib install-static install-includes - touch .done - popd -else - echo "Using cached ZStd" -fi - -############################################################################### -if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then - echo "Building Snappy" - download_dependency $ROOT_DIR/dependencies.yaml snappy - pushd snappy-${SNAPPY_VERSION} - CXXFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - cmake . -DCMAKE_INSTALL_PREFIX=$PREFIX -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF - make -j16 - make install - touch .done - popd -else - echo "Using cached Snappy" -fi - -############################################################################### -if [ ! -f curl-${CURL_VERSION}/.done ]; then - echo "Building LibCurl" - CURL_VERSION_=${CURL_VERSION//./_} - download_dependency $ROOT_DIR/dependencies.yaml curl - pushd curl-${CURL_VERSION} - # Force the compiler to find the OpenSSL headers instead of the headers in the system path like /usr/local/include/openssl. - cp -rf $PREFIX/include/openssl include/ - CFLAGS="-I$PREFIX/include -fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - ./configure --with-ssl=$PREFIX \ - --without-nghttp2 \ - --without-libidn2 \ - --disable-ldap \ - --without-brotli \ - --without-secure-transport \ - --disable-ipv6 \ - --prefix=$PREFIX - make -j16 install - touch .done - popd -else - echo "Using cached LibCurl" -fi diff --git a/pkg/mac/build-mac-wheels.sh b/pkg/mac/build-mac-wheels.sh index 917bfec..2aea1de 100755 --- a/pkg/mac/build-mac-wheels.sh +++ b/pkg/mac/build-mac-wheels.sh @@ -20,29 +20,114 @@ set -e -x +if [[ $# -ne 2 ]]; then + echo "Usage: $0 " + exit 1 +fi + ROOT_DIR=$(git rev-parse --show-toplevel) cd "${ROOT_DIR}" -source pkg/mac/common.sh +source build-support/dep-url.sh -PYTHON_CLIENT_VERSION=$(grep -v '^#' pulsar/__about__.py | cut -d "=" -f2 | sed "s/'//g") +CACHE_DIR=$ROOT_DIR/.pulsar-mac-build +PREFIX=${CACHE_DIR}/install +mkdir -p $PREFIX -PYTHON_VERSION=$1 +mkdir -p $PREFIX/lib/ +if [ ! -f $PREFIX/lib/libpulsarwithdeps.a ]; then + VERSION=$(cat ./dependencies.yaml | grep pulsar-cpp | awk '{print $2}') + curl -O -L $(pulsar_cpp_base_url $VERSION)/macos-arm64.zip + curl -O -L $(pulsar_cpp_base_url $VERSION)/macos-x86_64.zip -############################################################################### -############################################################################### -############################################################################### -############################################################################### -PREFIX=${CACHE_DIR_DEPS}/install + unzip -q macos-arm64.zip -d arm64 + unzip -q macos-x86_64.zip -d x86_64 + libtool -static -o libpulsarwithdeps.a arm64/lib/libpulsarwithdeps.a x86_64/lib/libpulsarwithdeps.a -echo '----------------------------------------------------------------------------' -echo '----------------------------------------------------------------------------' -echo '----------------------------------------------------------------------------' -echo "Build wheel for Python $PYTHON_VERSION" + mv arm64/include/ $PREFIX/ + mv libpulsarwithdeps.a $PREFIX/lib/ + rm -rf arm64/ x86_64/ macos-arm64.zip macos-x86_64.zip +fi -cd "${ROOT_DIR}" +PYTHON_VERSION=$1 +PYTHON_VERSION_LONG=$2 + +MACOSX_DEPLOYMENT_TARGET=13 +pushd $CACHE_DIR + +# We need to build OpenSSL from source to have universal2 binaries +OPENSSL_VERSION=$(cat $ROOT_DIR/dependencies.yaml | grep openssl | awk '{print $2}') +OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') +if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then + echo "Building OpenSSL" + download_dependency $ROOT_DIR/dependencies.yaml openssl + tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz + + mv openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64 + pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64 + echo -e "#include \n$(cat test/v3ext.c)" > test/v3ext.c + CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./Configure --prefix=$PREFIX no-shared no-unit-test darwin64-arm64-cc + make -j8 + make install_sw + popd + + tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz + mv openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64 + pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64 + echo -e "#include \n$(cat test/v3ext.c)" > test/v3ext.c + CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./Configure --prefix=$PREFIX no-shared no-unit-test darwin64-x86_64-cc + make -j8 + make install_sw + popd + + # Create universal binaries + lipo -create openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64/libssl.a openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64/libssl.a \ + -output $PREFIX/lib/libssl.a + lipo -create openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64/libcrypto.a openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64/libcrypto.a \ + -output $PREFIX/lib/libcrypto.a + + touch openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done +else + echo "Using cached OpenSSL" +fi + +if [ ! -f Python-${PYTHON_VERSION_LONG}/.done ]; then + echo "Building Python $PYTHON_VERSION_LONG" + curl -O -L https://www.python.org/ftp/python/${PYTHON_VERSION_LONG}/Python-${PYTHON_VERSION_LONG}.tgz + tar xfz Python-${PYTHON_VERSION_LONG}.tgz + + pushd Python-${PYTHON_VERSION_LONG} + ./configure --prefix=$PREFIX --enable-shared --enable-universalsdk --with-universal-archs=universal2 --with-openssl=$PREFIX + make -j16 + make install + + curl -O -L https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl + export SSL_CERT_FILE=/etc/ssl/cert.pem + $PREFIX/bin/pip3 install wheel setuptools + $PREFIX/bin/pip3 install wheel-*.whl + + touch .done + popd +else + echo "Using cached Python $PYTHON_VERSION_LONG" +fi + +PYBIND11_VERSION=$(cat $ROOT_DIR/dependencies.yaml | grep pybind11 | awk '{print $2}') +if [ ! -f pybind11/.done ]; then + download_dependency $ROOT_DIR/dependencies.yaml pybind11 + mkdir -p $PREFIX/include/ + cp -rf pybind11-${PYBIND11_VERSION}/include/pybind11 $PREFIX/include/ + mkdir -p pybind11 + touch pybind11/.done +fi + +popd # $CACHE_DIR -rm -f CMakeCache.txt +PYTHON_CLIENT_VERSION=$(grep -v '^#' pulsar/__about__.py | cut -d "=" -f2 | sed "s/'//g") + +echo "Build wheel for Python $PYTHON_VERSION" PY_EXE=$PREFIX/bin/python3 PIP_EXE=$PREFIX/bin/pip3 @@ -50,19 +135,15 @@ PIP_EXE=$PREFIX/bin/pip3 ARCHS='arm64;x86_64' PIP_TAG='universal2' -cmake . \ +cmake -B build \ -DCMAKE_OSX_ARCHITECTURES=${ARCHS} \ -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_PREFIX_PATH=$PREFIX \ - -DCMAKE_CXX_FLAGS=-I$PREFIX/include \ -DLINK_STATIC=ON \ - -DPULSAR_LIBRARY=${CACHE_DIR_CPP_CLIENT}/install/lib/libpulsar.a \ - -DPULSAR_INCLUDE=${CACHE_DIR_CPP_CLIENT}/install/include \ -DPython3_ROOT_DIR=$PREFIX - -make clean -make -j16 +cmake --build build --config Release -j8 +cp -f build/lib_pulsar.so . $PY_EXE setup.py bdist_wheel @@ -70,6 +151,5 @@ PY_SPEC=$(echo $PYTHON_VERSION | sed 's/\.//g') cd /tmp $PIP_EXE install --no-dependencies --force-reinstall \ - $ROOT_DIR/dist/pulsar_client-${PYTHON_CLIENT_VERSION}-cp$PY_SPEC-*-macosx_10_15_${PIP_TAG}.whl + $ROOT_DIR/dist/pulsar_client-${PYTHON_CLIENT_VERSION}-cp$PY_SPEC-*-macosx*_${PIP_TAG}.whl $PY_EXE -c 'import pulsar' - diff --git a/pkg/mac/build-pulsar-cpp.sh b/pkg/mac/build-pulsar-cpp.sh deleted file mode 100755 index 51f1ef3..0000000 --- a/pkg/mac/build-pulsar-cpp.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -e -x - -ROOT_DIR=$(git rev-parse --show-toplevel) -cd "${ROOT_DIR}" - -source pkg/mac/common.sh -source build-support/dep-url.sh - -PULSAR_CPP_VERSION=$(./build-support/dep-version.py pulsar-cpp) - -# Compile and cache dependencies -mkdir -p $CACHE_DIR_CPP_CLIENT -cd $CACHE_DIR_CPP_CLIENT - -PREFIX=$CACHE_DIR_CPP_CLIENT/install - -DEPS_PREFIX=${CACHE_DIR_DEPS}/install - -############################################################################### -download_dependency $ROOT_DIR/dependencies.yaml pulsar-cpp -tar xfz apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz - -if [ ! -f apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}/.done ]; then - pushd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} - ARCHS='arm64;x86_64' - - cmake . \ - -DCMAKE_CXX_STANDARD=11 \ - -DCMAKE_OSX_ARCHITECTURES=${ARCHS} \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH=${DEPS_PREFIX} \ - -DCMAKE_CXX_FLAGS=-I${DEPS_PREFIX}/include \ - -DOPENSSL_ROOT_DIR=${DEPS_PREFIX} \ - -DLINK_STATIC=OFF \ - -DBUILD_TESTS=OFF \ - -DBUILD_WIRESHARK=OFF \ - -DBUILD_DYNAMIC_LIB=OFF \ - -DBUILD_STATIC_LIB=ON \ - -DPROTOC_PATH=${DEPS_PREFIX}/bin/protoc - - make -j16 install - popd -fi diff --git a/pkg/mac/common.sh b/pkg/mac/common.sh deleted file mode 100644 index fa2ed35..0000000 --- a/pkg/mac/common.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -export MACOSX_DEPLOYMENT_TARGET=10.15 -export MACOSX_DEPLOYMENT_TARGET_MAJOR=${MACOSX_DEPLOYMENT_TARGET%%.*} - -CACHE_DIR_DEPS=$PWD/.pulsar-mac-build/deps -CACHE_DIR_CPP_CLIENT=$PWD/.pulsar-mac-build/cpp diff --git a/pkg/mac/python-3.7.patch b/pkg/mac/python-3.7.patch deleted file mode 100644 index f788920..0000000 --- a/pkg/mac/python-3.7.patch +++ /dev/null @@ -1,185 +0,0 @@ -diff -r -u Python-3.7.15.orig/Lib/_osx_support.py Python-3.7.15/Lib/_osx_support.py ---- Python-3.7.15.orig/Lib/_osx_support.py 2022-10-10 05:34:53.000000000 -0700 -+++ Python-3.7.15/Lib/_osx_support.py 2022-10-14 08:10:34.000000000 -0700 -@@ -491,6 +491,8 @@ - machine = 'fat64' - elif archs == ('i386', 'ppc', 'ppc64', 'x86_64'): - machine = 'universal' -+ elif archs == ('arm64', 'x86_64'): -+ machine = 'universal2' - else: - raise ValueError( - "Don't know machine value for archs=%r" % (archs,)) -diff -r -u Python-3.7.15.orig/Modules/_ctypes/libffi_osx/include/ffi.h Python-3.7.15/Modules/_ctypes/libffi_osx/include/ffi.h ---- Python-3.7.15.orig/Modules/_ctypes/libffi_osx/include/ffi.h 2022-10-10 05:34:53.000000000 -0700 -+++ Python-3.7.15/Modules/_ctypes/libffi_osx/include/ffi.h 2022-10-14 08:54:33.000000000 -0700 -@@ -62,7 +62,8 @@ - # elif defined(__ppc__) || defined(__ppc64__) - # define POWERPC_DARWIN - # else --# error "Unsupported MacOS X CPU type" -+/* Temp fix to allow for universal2 to build through */ -+# define X86_DARWIN - # endif - #else - #error "Unsupported OS type" -@@ -199,9 +200,9 @@ - - void - ffi_raw_call( --/*@dependent@*/ ffi_cif* cif, -- void (*fn)(void), --/*@out@*/ void* rvalue, -+/*@dependent@*/ ffi_cif* cif, -+ void (*fn)(void), -+/*@out@*/ void* rvalue, - /*@dependent@*/ ffi_raw* avalue); - - void -@@ -225,9 +226,9 @@ - longs and doubles are followed by an empty 64-bit word. */ - void - ffi_java_raw_call( --/*@dependent@*/ ffi_cif* cif, -- void (*fn)(void), --/*@out@*/ void* rvalue, -+/*@dependent@*/ ffi_cif* cif, -+ void (*fn)(void), -+/*@out@*/ void* rvalue, - /*@dependent@*/ ffi_raw* avalue); - - void -@@ -272,8 +273,8 @@ - ffi_cif* cif; - - #if !FFI_NATIVE_RAW_API -- /* if this is enabled, then a raw closure has the same layout -- as a regular closure. We use this to install an intermediate -+ /* if this is enabled, then a raw closure has the same layout -+ as a regular closure. We use this to install an intermediate - handler to do the transaltion, void** -> ffi_raw*. */ - void (*translate_args)(ffi_cif*,void*,void**,void*); - void* this_closure; -@@ -303,17 +304,17 @@ - - ffi_status - ffi_prep_cif( --/*@out@*/ /*@partial@*/ ffi_cif* cif, -+/*@out@*/ /*@partial@*/ ffi_cif* cif, - ffi_abi abi, -- unsigned int nargs, --/*@dependent@*/ /*@out@*/ /*@partial@*/ ffi_type* rtype, -+ unsigned int nargs, -+/*@dependent@*/ /*@out@*/ /*@partial@*/ ffi_type* rtype, - /*@dependent@*/ ffi_type** atypes); - - void - ffi_call( --/*@dependent@*/ ffi_cif* cif, -- void (*fn)(void), --/*@out@*/ void* rvalue, -+/*@dependent@*/ ffi_cif* cif, -+ void (*fn)(void), -+/*@out@*/ void* rvalue, - /*@dependent@*/ void** avalue); - - /* Useful for eliminating compiler warnings */ -diff -r -u Python-3.7.15.orig/Modules/_ctypes/libffi_osx/include/fficonfig.h Python-3.7.15/Modules/_ctypes/libffi_osx/include/fficonfig.h ---- Python-3.7.15.orig/Modules/_ctypes/libffi_osx/include/fficonfig.h 2022-10-10 05:34:53.000000000 -0700 -+++ Python-3.7.15/Modules/_ctypes/libffi_osx/include/fficonfig.h 2022-10-14 13:19:13.000000000 -0700 -@@ -1,4 +1,4 @@ --/* Manually created fficonfig.h for Darwin on PowerPC or Intel -+/* Manually created fficonfig.h for Darwin on PowerPC or Intel - - This file is manually generated to do away with the need for autoconf and - therefore make it easier to cross-compile and build fat binaries. -@@ -18,7 +18,7 @@ - # define HAVE_LONG_DOUBLE 1 - # define SIZEOF_LONG_DOUBLE 16 - --#elif defined(__x86_64__) -+#elif defined(__x86_64__) || defined(__arm64__) - # define BYTEORDER 1234 - # undef HOST_WORDS_BIG_ENDIAN - # undef WORDS_BIGENDIAN -@@ -33,10 +33,10 @@ - # define SIZEOF_DOUBLE 8 - # if __GNUC__ >= 4 - # define HAVE_LONG_DOUBLE 1 --# define SIZEOF_LONG_DOUBLE 16 -+# define SIZEOF_LONG_DOUBLE 16 - # else - # undef HAVE_LONG_DOUBLE --# define SIZEOF_LONG_DOUBLE 8 -+# define SIZEOF_LONG_DOUBLE 8 - # endif - - #elif defined(__ppc64__) -@@ -147,4 +147,4 @@ - # else - # define FFI_HIDDEN - # endif --#endif -\ No newline at end of file -+#endif -diff -r -u Python-3.7.15.orig/Modules/_ctypes/libffi_osx/include/ffitarget.h Python-3.7.15/Modules/_ctypes/libffi_osx/include/ffitarget.h ---- Python-3.7.15.orig/Modules/_ctypes/libffi_osx/include/ffitarget.h 2022-10-10 05:34:53.000000000 -0700 -+++ Python-3.7.15/Modules/_ctypes/libffi_osx/include/ffitarget.h 2022-10-14 09:35:48.000000000 -0700 -@@ -4,10 +4,10 @@ - make building fat binaries harder. - */ - --#if defined(__i386__) || defined(__x86_64__) -+#if defined(__i386__) || defined(__x86_64__) || defined(__arm64__) - #include "x86-ffitarget.h" - #elif defined(__ppc__) || defined(__ppc64__) - #include "ppc-ffitarget.h" - #else - #error "Unsupported CPU type" --#endif -\ No newline at end of file -+#endif -diff -r -u Python-3.7.15.orig/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h Python-3.7.15/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h ---- Python-3.7.15.orig/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h 2022-10-10 05:34:53.000000000 -0700 -+++ Python-3.7.15/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h 2022-10-14 11:58:41.000000000 -0700 -@@ -33,7 +33,7 @@ - # define X86 - #endif - --#if defined(__x86_64__) -+#if defined(__x86_64__) || defined(__arm64__) - # ifndef X86_64 - # define X86_64 - # endif -@@ -58,7 +58,7 @@ - #endif - - /* ---- Intel x86 and AMD x86-64 - */ --#if !defined(X86_WIN32) && (defined(__i386__) || defined(__x86_64__)) -+#if !defined(X86_WIN32) && (defined(__i386__) || defined(__x86_64__) || defined(__arm64__)) - FFI_SYSV, - FFI_UNIX64, /* Unix variants all use the same ABI for x86-64 */ - # ifdef __i386__ -@@ -85,4 +85,4 @@ - # define FFI_NATIVE_RAW_API 1 /* x86 has native raw api support */ - #endif - --#endif // #ifndef LIBFFI_TARGET_H -\ No newline at end of file -+#endif // #ifndef LIBFFI_TARGET_H -Only in Python-3.7.15: config.log -diff -r -u Python-3.7.15.orig/configure Python-3.7.15/configure ---- Python-3.7.15.orig/configure 2022-10-10 05:34:53.000000000 -0700 -+++ Python-3.7.15/configure 2022-10-14 08:10:34.000000000 -0700 -@@ -7449,6 +7449,11 @@ - LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" - ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" - ;; -+ universal2) -+ UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" -+ LIPO_32BIT_FLAGS="" -+ ARCH_RUN_32BIT="" -+ ;; - *) - as_fn_error $? "proper usage is --with-universal-arch=32-bit|64-bit|all|intel|3-way" "$LINENO" 5 - ;; diff --git a/pkg/manylinux2014/Dockerfile b/pkg/manylinux2014/Dockerfile index c5c4cf0..d59ed3f 100644 --- a/pkg/manylinux2014/Dockerfile +++ b/pkg/manylinux2014/Dockerfile @@ -24,89 +24,11 @@ ARG PYTHON_VERSION ARG PYTHON_SPEC ARG ARCH +ENV CPP_BINARY_TYPE=rpm ENV PYTHON_VERSION=${PYTHON_VERSION} ENV PYTHON_SPEC=${PYTHON_SPEC} ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}" +ENV ARCH=${ARCH} -ENV PYTHON_INCLUDE_DIR /opt/python/${PYTHON_SPEC}/include -ENV PYTHON_LIBRARIES /opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION} - -RUN pip3 install pyyaml setuptools - -ADD .build/dependencies.yaml / -ADD .build/dep-version.py /usr/local/bin -ADD .build/dep-url.sh / - -# Download and install boost -RUN BOOST_VERSION=$(dep-version.py boost) && \ - BOOST_VERSION_UNDESRSCORE=$(echo $BOOST_VERSION | sed 's/\./_/g') && \ - . /dep-url.sh && download_dependency /dependencies.yaml boost && \ - cp -r boost_${BOOST_VERSION_UNDESRSCORE}/boost /usr/include/ && \ - rm -rf /boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz /boost_${BOOST_VERSION_UNDESRSCORE} - -RUN CMAKE_VERSION=$(dep-version.py cmake) && \ - . /dep-url.sh && ARCH=${ARCH} download_dependency /dependencies.yaml cmake && \ - cp cmake-${CMAKE_VERSION}-linux-${ARCH}/bin/* /usr/bin/ && \ - cp -r cmake-${CMAKE_VERSION}-linux-${ARCH}/share/cmake-* /usr/share/ && \ - rm -rf cmake-${CMAKE_VERSION}-linux-${ARCH} cmake-${CMAKE_VERSION}-linux-${ARCH}.tar.gz - -# Download and compile protobuf -RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \ - . /dep-url.sh && download_dependency /dependencies.yaml protobuf && \ - cd protobuf-${PROTOBUF_VERSION}/ && \ - CXXFLAGS=-fPIC ./configure && \ - make -j8 && make install && ldconfig && \ - rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION} - -# ZLib -RUN ZLIB_VERSION=$(dep-version.py zlib) && \ - . /dep-url.sh && download_dependency /dependencies.yaml zlib && \ - cd zlib-${ZLIB_VERSION} && \ - CFLAGS="-fPIC -O3" ./configure && \ - make -j8 && make install && \ - rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION} - -# Zstandard -RUN ZSTD_VERSION=$(dep-version.py zstd) && \ - . /dep-url.sh && download_dependency /dependencies.yaml zstd && \ - cd zstd-${ZSTD_VERSION} && \ - CFLAGS="-fPIC -O3" make -j8 && \ - make install && \ - rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz - -# Snappy -RUN SNAPPY_VERSION=$(dep-version.py snappy) && \ - . /dep-url.sh && download_dependency /dependencies.yaml snappy && \ - cd snappy-${SNAPPY_VERSION} && \ - CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \ - make -j8 && make install && \ - rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz - -RUN OPENSSL_VERSION=$(dep-version.py openssl) && \ - OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \ - . /dep-url.sh && download_dependency /dependencies.yaml openssl && \ - cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \ - ./config -fPIC --prefix=/usr/local/ssl/ && \ - make -j8 && make install && \ - rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} - -ENV LD_LIBRARY_PATH /usr/local/ssl/lib/:/usr/local/lib -ENV OPENSSL_ROOT_DIR /usr/local/ssl/ - -# LibCurl -RUN CURL_VERSION=$(dep-version.py curl) && \ - . /dep-url.sh && download_dependency /dependencies.yaml curl && \ - cd curl-${CURL_VERSION} && \ - CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \ - make -j8 && make install && \ - rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION} - -# Pulsar client C++ -RUN PULSAR_CPP_VERSION=$(dep-version.py pulsar-cpp) && \ - . /dep-url.sh && download_dependency /dependencies.yaml pulsar-cpp && \ - cd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} && \ - cmake . -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=OFF -DBUILD_STATIC_LIB=OFF -DLINK_STATIC=ON && \ - make -j8 && \ - make install && \ - rm -rf apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz +RUN pip3 install setuptools diff --git a/pkg/manylinux_musl/Dockerfile b/pkg/manylinux_musl/Dockerfile index 3002194..2681b88 100644 --- a/pkg/manylinux_musl/Dockerfile +++ b/pkg/manylinux_musl/Dockerfile @@ -30,80 +30,6 @@ ARG ARCH ENV ARCH=${ARCH} ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}" -ENV PYTHON_INCLUDE_DIR /opt/python/${PYTHON_SPEC}/include -ENV PYTHON_LIBRARIES /opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION} -RUN pip install pyyaml setuptools - -RUN apk add cmake - -ADD .build/dependencies.yaml / -ADD .build/dep-version.py /usr/local/bin -ADD .build/dep-url.sh / - -# Download and install boost -RUN BOOST_VERSION=$(dep-version.py boost) && \ - BOOST_VERSION_UNDESRSCORE=$(echo $BOOST_VERSION | sed 's/\./_/g') && \ - . /dep-url.sh && download_dependency /dependencies.yaml boost && \ - cp -r boost_${BOOST_VERSION_UNDESRSCORE}/boost /usr/include/ && \ - rm -rf /boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz /boost_${BOOST_VERSION_UNDESRSCORE} - -# Download and compile protobuf -RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \ - . /dep-url.sh && download_dependency /dependencies.yaml protobuf && \ - cd protobuf-${PROTOBUF_VERSION}/ && \ - CXXFLAGS=-fPIC ./configure && \ - make -j8 && make install && \ - rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION} - -# ZLib -RUN ZLIB_VERSION=$(dep-version.py zlib) && \ - . /dep-url.sh && download_dependency /dependencies.yaml zlib && \ - cd zlib-${ZLIB_VERSION} && \ - CFLAGS="-fPIC -O3" ./configure && \ - make -j8 && make install && \ - rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION} - -# Zstandard -RUN ZSTD_VERSION=$(dep-version.py zstd) && \ - . /dep-url.sh && download_dependency /dependencies.yaml zstd && \ - cd zstd-${ZSTD_VERSION} && \ - CFLAGS="-fPIC -O3" make -j8 && \ - make install && \ - rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz - -# Snappy -RUN SNAPPY_VERSION=$(dep-version.py snappy) && \ - . /dep-url.sh && download_dependency /dependencies.yaml snappy && \ - cd snappy-${SNAPPY_VERSION} && \ - CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \ - make -j8 && make install && \ - rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz - -RUN OPENSSL_VERSION=$(dep-version.py openssl) && \ - OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \ - . /dep-url.sh && download_dependency /dependencies.yaml openssl && \ - cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \ - ./config -fPIC --prefix=/usr/local/ssl/ && \ - make -j8 && make install && \ - rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} - -ENV LD_LIBRARY_PATH /usr/local/ssl/lib/:/usr/local/lib -ENV OPENSSL_ROOT_DIR /usr/local/ssl/ - -# LibCurl -RUN CURL_VERSION=$(dep-version.py curl) && \ - . /dep-url.sh && download_dependency /dependencies.yaml curl && \ - cd curl-${CURL_VERSION} && \ - CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \ - make -j8 && make install && \ - rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION} - -# Pulsar client C++ -RUN PULSAR_CPP_VERSION=$(dep-version.py pulsar-cpp) && \ - . /dep-url.sh && download_dependency /dependencies.yaml pulsar-cpp && \ - cd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} && \ - cmake . -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=OFF -DBUILD_STATIC_LIB=OFF -DLINK_STATIC=ON && \ - make -j8 && \ - make install && \ - rm -rf apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz +RUN pip install setuptools +ENV CPP_BINARY_TYPE="apk"