From fb4523b48a73819678dc38a64ee4750971abbe88 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 10 Jan 2025 09:33:36 +0800 Subject: [PATCH] Fix the broken CI due to the regression of 4.0.1 and some deprecated image and links (#235) Fixes https://github.com/apache/pulsar-client-python/issues/234 1. Downgrade the Pulsar image to 4.0.0 2. Upgrade the macOS runner image from 12 to 14 Since the Boost download link is broken now, this PR also fixes the wheel build process by reusing the pre-built libraries to install pulsar-client-cpp on Linux and macOS. Specially, it links the `libpulsarwithdeps.a` instead of `libpulsar.a` and all 3rd-party libraries on macOS. --- .../workflows/ci-build-release-wheels.yaml | 27 +-- .github/workflows/ci-pr-validation.yaml | 31 +-- CMakeLists.txt | 44 +--- build-support/copy-deps-versionfile.sh | 30 --- build-support/dep-url.sh | 23 -- build-support/pulsar-test-service-start.sh | 2 +- dependencies.yaml | 8 +- pkg/build-wheel-inside-docker.sh | 38 ++- pkg/mac/build-dependencies.sh | 218 ------------------ pkg/mac/build-mac-wheels.sh | 126 ++++++++-- pkg/mac/build-pulsar-cpp.sh | 65 ------ pkg/mac/common.sh | 25 -- pkg/mac/python-3.7.patch | 185 --------------- pkg/manylinux2014/Dockerfile | 84 +------ pkg/manylinux_musl/Dockerfile | 78 +------ 15 files changed, 148 insertions(+), 836 deletions(-) delete mode 100755 build-support/copy-deps-versionfile.sh delete mode 100755 pkg/mac/build-dependencies.sh delete mode 100755 pkg/mac/build-pulsar-cpp.sh delete mode 100644 pkg/mac/common.sh delete mode 100644 pkg/mac/python-3.7.patch diff --git a/.github/workflows/ci-build-release-wheels.yaml b/.github/workflows/ci-build-release-wheels.yaml index d57e0ed..11edb6e 100644 --- a/.github/workflows/ci-build-release-wheels.yaml +++ b/.github/workflows/ci-build-release-wheels.yaml @@ -58,7 +58,6 @@ jobs: uses: docker/setup-qemu-action@v2 - uses: docker/setup-buildx-action@v2 - - run: build-support/copy-deps-versionfile.sh - name: Build Manylinux Docker image uses: docker/build-push-action@v3 @@ -95,7 +94,7 @@ jobs: mac-wheels: name: Wheel MacOS Universal2 - Py ${{matrix.py.version}} - runs-on: macos-12 + runs-on: macos-14 timeout-minutes: 300 strategy: @@ -112,30 +111,8 @@ jobs: - name: checkout uses: actions/checkout@v3 - - name: Dependencies cache - uses: actions/cache@v3 - id: cache-deps - with: - path: .pulsar-mac-build/deps/install - key: ${{matrix.py.version_long}}-${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pkg/mac/*') }} - - - name: Build dependencies - if: steps.cache-deps.outputs.cache-hit != 'true' - run: pkg/mac/build-dependencies.sh ${{matrix.py.version}} ${{matrix.py.version_long}} - - - name: Pulsar C++ cache - uses: actions/cache@v3 - id: cache-cpp - with: - path: .pulsar-mac-build/cpp/install - key: ${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pulsar-client-cpp-version.txt') }}-${{ hashFiles('pkg/mac/*') }} - - - name: Build Pulsar C++ client - if: steps.cache-cpp.outputs.cache-hit != 'true' - run: pkg/mac/build-pulsar-cpp.sh - - name: Build and test Mac wheels - run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}} + run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}} ${{matrix.py.version_long}} - name: Upload artifacts uses: actions/upload-artifact@v3 diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 213ddb3..262808d 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -127,7 +127,6 @@ jobs: uses: docker/setup-qemu-action@v2 - uses: docker/setup-buildx-action@v2 - - run: build-support/copy-deps-versionfile.sh - name: Build Manylinux Docker image uses: docker/build-push-action@v3 @@ -157,7 +156,7 @@ jobs: mac-wheels: name: Wheel MacOS Universal2 - Py ${{matrix.py.version}} needs: unit-tests - runs-on: macos-12 + runs-on: macos-14 timeout-minutes: 300 strategy: @@ -169,33 +168,9 @@ jobs: - name: checkout uses: actions/checkout@v3 - - name: Dependencies cache - uses: actions/cache@v3 - id: cache-deps - with: - path: .pulsar-mac-build/deps/install - key: ${{matrix.py.version_long}}-${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pkg/mac/*') }} - - - name: Install Python deps - run: pip3 install -U pyyaml - - - name: Build dependencies - if: steps.cache-deps.outputs.cache-hit != 'true' - run: pkg/mac/build-dependencies.sh ${{matrix.py.version}} ${{matrix.py.version_long}} - - - name: Pulsar C++ cache - uses: actions/cache@v3 - id: cache-cpp - with: - path: .pulsar-mac-build/cpp/install - key: ${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pulsar-client-cpp-version.txt') }}-${{ hashFiles('pkg/mac/*') }} - - - name: Build Pulsar C++ client - if: steps.cache-cpp.outputs.cache-hit != 'true' - run: pkg/mac/build-pulsar-cpp.sh - - name: Build and test Mac wheels - run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}} + run: | + pkg/mac/build-mac-wheels.sh ${{matrix.py.version}} ${{matrix.py.version_long}} windows-wheels: name: "Python ${{ matrix.python.version }} Wheel on Windows x64" diff --git a/CMakeLists.txt b/CMakeLists.txt index 084e078..660a1b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,13 +38,9 @@ if (MSVC) endif () if (LINK_STATIC) - if (MSVC) - find_library(PULSAR_LIBRARY NAMES pulsarWithDeps.lib) - else () - find_library(PULSAR_LIBRARY NAMES libpulsar.a) - endif () + find_library(PULSAR_LIBRARY NAMES pulsarwithdeps pulsarWithDeps.lib) add_definitions("-DPULSAR_STATIC") -else() +else () find_library(PULSAR_LIBRARY NAMES pulsar libpulsar) endif() message(STATUS "PULSAR_LIBRARY: ${PULSAR_LIBRARY}") @@ -88,43 +84,11 @@ set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} Python3::Module) message(STATUS "All libraries: ${PYTHON_WRAPPER_LIBS}") if (LINK_STATIC AND NOT MSVC) - set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") - - # We need to include all the static libs individually because we cannot easily create a universal2 libpulsar.a - # with all the deps included. - find_package(OpenSSL REQUIRED) - message("OPENSSL_LIBRARIES: " ${OPENSSL_LIBRARIES}) - - find_package(Protobuf REQUIRED) - message("Protobuf_LIBRARIES: " ${Protobuf_LIBRARIES}) - - find_package(curl REQUIRED) - message("CURL_LIBRARIES: " ${CURL_LIBRARIES}) - - find_package(zlib) - message("ZLIB_LIBRARIES: " ${ZLIB_LIBRARIES}) - - find_library(LIB_ZSTD NAMES libzstd.a) - message(STATUS "ZStd: ${LIB_ZSTD}") - find_library(LIB_SNAPPY NAMES libsnappy.a) - message(STATUS "LIB_SNAPPY: ${LIB_SNAPPY}") - - set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} - ${OPENSSL_LIBRARIES} - ${Protobuf_LIBRARIES} - ${ZLIB_LIBRARIES} - ${LIB_ZSTD} - ${LIB_SNAPPY} - ${CURL_LIBRARIES} - ) - if (APPLE) set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup") - target_link_libraries(_pulsar -Wl,-all_load ${PYTHON_WRAPPER_LIBS}) + target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS}) else () - if (NOT MSVC) - set (CMAKE_SHARED_LINKER_FLAGS " -static-libgcc -static-libstdc++") - endif() + set (CMAKE_SHARED_LINKER_FLAGS " -static-libgcc -static-libstdc++") target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS}) endif () elseif (LINK_STATIC) # MSVC diff --git a/build-support/copy-deps-versionfile.sh b/build-support/copy-deps-versionfile.sh deleted file mode 100755 index 0e1e135..0000000 --- a/build-support/copy-deps-versionfile.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -e - -ROOT_DIR=$(git rev-parse --show-toplevel) - -for dir in manylinux2014 manylinux_musl; do - mkdir -p pkg/$dir/.build - cp $ROOT_DIR/dependencies.yaml pkg/$dir/.build - cp $ROOT_DIR/build-support/dep-version.py pkg/$dir/.build - cp $ROOT_DIR/build-support/dep-url.sh pkg/$dir/.build -done diff --git a/build-support/dep-url.sh b/build-support/dep-url.sh index 92fb317..7670bb9 100644 --- a/build-support/dep-url.sh +++ b/build-support/dep-url.sh @@ -38,38 +38,15 @@ download_dependency() { VERSION=$(grep $DEP $DEP_FILE | sed 's/://' | awk '{print $2}') case $DEP in - "cmake") - URL=https://github.com/Kitware/CMake/releases/download/v${VERSION}/cmake-${VERSION}-linux-${ARCH}.tar.gz - ;; "pulsar-cpp") URL=$(pulsar_cpp_base_url $VERSION)/apache-pulsar-client-cpp-${VERSION}.tar.gz ;; "pybind11") URL=https://github.com/pybind/pybind11/archive/refs/tags/v${VERSION}.tar.gz ;; - "boost") - VERSION_UNDERSCORE=$(echo $VERSION | sed 's/\./_/g') - URL=https://boostorg.jfrog.io/artifactory/main/release/${VERSION}/source/boost_${VERSION_UNDERSCORE}.tar.gz - ;; - "protobuf") - URL=https://github.com/google/protobuf/releases/download/v${VERSION}/protobuf-cpp-${VERSION}.tar.gz - ;; - "zlib") - URL=https://github.com/madler/zlib/archive/v${VERSION}.tar.gz - ;; - "zstd") - URL=https://github.com/facebook/zstd/releases/download/v${VERSION}/zstd-${VERSION}.tar.gz - ;; - "snappy") - URL=https://github.com/google/snappy/archive/refs/tags/${VERSION}.tar.gz - ;; "openssl") URL=https://github.com/openssl/openssl/archive/OpenSSL_$(echo $VERSION | sed 's/\./_/g').tar.gz ;; - "curl") - VERSION_UNDERSCORE=$(echo $VERSION | sed 's/\./_/g') - URL=https://github.com/curl/curl/releases/download/curl-${VERSION_UNDERSCORE}/curl-${VERSION}.tar.gz - ;; *) echo "Unknown dependency $DEP for version $VERSION" exit 1 diff --git a/build-support/pulsar-test-service-start.sh b/build-support/pulsar-test-service-start.sh index ed1e7ba..f8e26f9 100755 --- a/build-support/pulsar-test-service-start.sh +++ b/build-support/pulsar-test-service-start.sh @@ -25,7 +25,7 @@ cd $SRC_DIR ./build-support/pulsar-test-service-stop.sh -CONTAINER_ID=$(docker run -i --user $(id -u) -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:latest sleep 3600) +CONTAINER_ID=$(docker run -i --user $(id -u) -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:4.0.0 sleep 3600) echo $CONTAINER_ID > .tests-container-id.txt docker cp tests/test-conf $CONTAINER_ID:/pulsar/test-conf diff --git a/dependencies.yaml b/dependencies.yaml index dd435ee..d58263f 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -17,13 +17,7 @@ # under the License. # -cmake: 3.24.2 pulsar-cpp: 3.5.1 pybind11: 2.10.1 -boost: 1.80.0 -protobuf: 3.20.0 -zlib: 1.2.13 -zstd: 1.5.2 -snappy: 1.1.9 +# The OpenSSL dependency is only used when building Python from source openssl: 1.1.1q -curl: 8.4.0 diff --git a/pkg/build-wheel-inside-docker.sh b/pkg/build-wheel-inside-docker.sh index c062607..0e73eae 100755 --- a/pkg/build-wheel-inside-docker.sh +++ b/pkg/build-wheel-inside-docker.sh @@ -21,19 +21,39 @@ set -e -x cd /pulsar-client-python +source build-support/dep-url.sh -PYBIND11_VERSION=$(./build-support/dep-version.py pybind11) -curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz -tar zxf v${PYBIND11_VERSION}.tar.gz -rm -rf pybind11 -mv pybind11-${PYBIND11_VERSION} pybind11 +# Build cpp wheels +PULSAR_CPP_VERSION=$(cat ./dependencies.yaml | grep pulsar-cpp | awk '{print $2}') -rm -f CMakeCache.txt CMakeFiles +if [ $CPP_BINARY_TYPE == "rpm" ]; then + if [ $ARCH == "aarch64" ]; then + RPM_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/rpm-arm64/aarch64 + else + RPM_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/rpm-x86_64/x86_64 + fi + curl -O -L $RPM_ROOT_DIR/apache-pulsar-client-$PULSAR_CPP_VERSION-1.$ARCH.rpm + curl -O -L $RPM_ROOT_DIR/apache-pulsar-client-devel-$PULSAR_CPP_VERSION-1.$ARCH.rpm + curl -O -L $RPM_ROOT_DIR/apache-pulsar-client-debuginfo-$PULSAR_CPP_VERSION-1.$ARCH.rpm + rpm -ivh *.rpm +else # apk + if [ $ARCH == "aarch64" ]; then + APK_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/apk-arm64/aarch64 + else + APK_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/apk-x86_64/x86_64 + fi + curl -O -L $APK_ROOT_DIR/apache-pulsar-client-$PULSAR_CPP_VERSION-r0.apk + curl -O -L $APK_ROOT_DIR/apache-pulsar-client-dev-$PULSAR_CPP_VERSION-r0.apk + apk add --allow-untrusted *.apk +fi -cmake . \ - -DCMAKE_BUILD_TYPE=Release +download_dependency $PWD/dependencies.yaml pybind11 +rm -rf pybind11 +mv pybind11-* pybind11 -make -j4 +cmake -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build -j8 +mv build/lib_pulsar.so . ./setup.py bdist_wheel diff --git a/pkg/mac/build-dependencies.sh b/pkg/mac/build-dependencies.sh deleted file mode 100755 index 43faaba..0000000 --- a/pkg/mac/build-dependencies.sh +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -e -x - -ROOT_DIR=$(git rev-parse --show-toplevel) -cd "${ROOT_DIR}" - -PYTHON_VERSION=$1 -PYTHON_VERSION_LONG=$2 - -source pkg/mac/common.sh -source build-support/dep-url.sh - -pip3 install pyyaml setuptools - -dep=$ROOT_DIR/build-support/dep-version.py -PYBIND11_VERSION=$($dep pybind11) -BOOST_VERSION=$($dep boost) -ZLIB_VERSION=$($dep zlib) -OPENSSL_VERSION=$($dep openssl) -PROTOBUF_VERSION=$($dep protobuf) -ZSTD_VERSION=$($dep zstd) -SNAPPY_VERSION=$($dep snappy) -CURL_VERSION=$($dep curl) - -# Compile and cache dependencies -CACHE_DIR=${CACHE_DIR_DEPS} -mkdir -p $CACHE_DIR -cd $CACHE_DIR - -PREFIX=$CACHE_DIR/install - -############################################################################### -if [ ! -f pybind11/.done ]; then - download_dependency $ROOT_DIR/dependencies.yaml pybind11 - mkdir -p $PREFIX/include/ - cp -rf pybind11-${PYBIND11_VERSION}/include/pybind11 $PREFIX/include/ - mkdir -p pybind11 - touch pybind11/.done -fi - -############################################################################### -if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then - echo "Building ZLib" - download_dependency $ROOT_DIR/dependencies.yaml zlib - pushd zlib-$ZLIB_VERSION - CFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX - make -j16 - make install - touch .done - popd -else - echo "Using cached ZLib" -fi - -############################################################################### -OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') -if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then - echo "Building OpenSSL" - download_dependency $ROOT_DIR/dependencies.yaml openssl - # -arch arm64 -arch x86_64 - tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz - - mv openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64 - pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64 - echo -e "#include \n$(cat test/v3ext.c)" > test/v3ext.c - CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - ./Configure --prefix=$PREFIX no-shared no-unit-test darwin64-arm64-cc - make -j8 - make install_sw - popd - - tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz - mv openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64 - pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64 - echo -e "#include \n$(cat test/v3ext.c)" > test/v3ext.c - CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - ./Configure --prefix=$PREFIX no-shared no-unit-test darwin64-x86_64-cc - make -j8 - make install_sw - popd - - # Create universal binaries - lipo -create openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64/libssl.a openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64/libssl.a \ - -output $PREFIX/lib/libssl.a - lipo -create openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64/libcrypto.a openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64/libcrypto.a \ - -output $PREFIX/lib/libcrypto.a - - touch openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done -else - echo "Using cached OpenSSL" -fi - -############################################################################### -if [ ! -f Python-${PYTHON_VERSION_LONG}/.done ]; then - echo "Building Python $PYTHON_VERSION_LONG" - curl -O -L https://www.python.org/ftp/python/${PYTHON_VERSION_LONG}/Python-${PYTHON_VERSION_LONG}.tgz - tar xfz Python-${PYTHON_VERSION_LONG}.tgz - - pushd Python-${PYTHON_VERSION_LONG} - if [ $PYTHON_VERSION = '3.7' ]; then - patch -p1 < ${ROOT_DIR}/pkg/mac/python-3.7.patch - fi - - CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET} -I${PREFIX}/include ${PY_CFLAGS}" \ - LDFLAGS=" ${PY_CFLAGS} -L${PREFIX}/lib" \ - ./configure --prefix=$PREFIX --enable-shared --enable-universalsdk \ - --with-universal-archs=universal2 \ - --with-openssl=$PREFIX - make -j16 - make install - - curl -O -L https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl - $PREFIX/bin/pip3 install wheel setuptools - $PREFIX/bin/pip3 install wheel-*.whl - - touch .done - popd -else - echo "Using cached Python $PYTHON_VERSION_LONG" -fi - -############################################################################### -BOOST_VERSION_=${BOOST_VERSION//./_} -if [ ! -f boost/.done ]; then - echo "Building Boost for Py $PYTHON_VERSION" - download_dependency $ROOT_DIR/dependencies.yaml boost - cp -rf boost_${BOOST_VERSION_}/boost $PREFIX/include/ - mkdir -p boost - touch .done -fi - -############################################################################### -if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then - echo "Building Protobuf" - download_dependency $ROOT_DIR/dependencies.yaml protobuf - pushd protobuf-${PROTOBUF_VERSION} - # Install by CMake so that the dependency can be found with CMake config mode - pushd cmake/ - cmake -B build -DCMAKE_CXX_FLAGS="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - -DCMAKE_INSTALL_PREFIX=$PREFIX - cmake --build build -j16 --target install - popd - touch .done - popd -else - echo "Using cached Protobuf" -fi - -############################################################################### -if [ ! -f zstd-${ZSTD_VERSION}/.done ]; then - echo "Building ZStd" - download_dependency $ROOT_DIR/dependencies.yaml zstd - pushd zstd-${ZSTD_VERSION} - CFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX \ - make -j16 -C lib install-static install-includes - touch .done - popd -else - echo "Using cached ZStd" -fi - -############################################################################### -if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then - echo "Building Snappy" - download_dependency $ROOT_DIR/dependencies.yaml snappy - pushd snappy-${SNAPPY_VERSION} - CXXFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - cmake . -DCMAKE_INSTALL_PREFIX=$PREFIX -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF - make -j16 - make install - touch .done - popd -else - echo "Using cached Snappy" -fi - -############################################################################### -if [ ! -f curl-${CURL_VERSION}/.done ]; then - echo "Building LibCurl" - CURL_VERSION_=${CURL_VERSION//./_} - download_dependency $ROOT_DIR/dependencies.yaml curl - pushd curl-${CURL_VERSION} - # Force the compiler to find the OpenSSL headers instead of the headers in the system path like /usr/local/include/openssl. - cp -rf $PREFIX/include/openssl include/ - CFLAGS="-I$PREFIX/include -fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - ./configure --with-ssl=$PREFIX \ - --without-nghttp2 \ - --without-libidn2 \ - --disable-ldap \ - --without-brotli \ - --without-secure-transport \ - --disable-ipv6 \ - --prefix=$PREFIX - make -j16 install - touch .done - popd -else - echo "Using cached LibCurl" -fi diff --git a/pkg/mac/build-mac-wheels.sh b/pkg/mac/build-mac-wheels.sh index 917bfec..2aea1de 100755 --- a/pkg/mac/build-mac-wheels.sh +++ b/pkg/mac/build-mac-wheels.sh @@ -20,29 +20,114 @@ set -e -x +if [[ $# -ne 2 ]]; then + echo "Usage: $0 " + exit 1 +fi + ROOT_DIR=$(git rev-parse --show-toplevel) cd "${ROOT_DIR}" -source pkg/mac/common.sh +source build-support/dep-url.sh -PYTHON_CLIENT_VERSION=$(grep -v '^#' pulsar/__about__.py | cut -d "=" -f2 | sed "s/'//g") +CACHE_DIR=$ROOT_DIR/.pulsar-mac-build +PREFIX=${CACHE_DIR}/install +mkdir -p $PREFIX -PYTHON_VERSION=$1 +mkdir -p $PREFIX/lib/ +if [ ! -f $PREFIX/lib/libpulsarwithdeps.a ]; then + VERSION=$(cat ./dependencies.yaml | grep pulsar-cpp | awk '{print $2}') + curl -O -L $(pulsar_cpp_base_url $VERSION)/macos-arm64.zip + curl -O -L $(pulsar_cpp_base_url $VERSION)/macos-x86_64.zip -############################################################################### -############################################################################### -############################################################################### -############################################################################### -PREFIX=${CACHE_DIR_DEPS}/install + unzip -q macos-arm64.zip -d arm64 + unzip -q macos-x86_64.zip -d x86_64 + libtool -static -o libpulsarwithdeps.a arm64/lib/libpulsarwithdeps.a x86_64/lib/libpulsarwithdeps.a -echo '----------------------------------------------------------------------------' -echo '----------------------------------------------------------------------------' -echo '----------------------------------------------------------------------------' -echo "Build wheel for Python $PYTHON_VERSION" + mv arm64/include/ $PREFIX/ + mv libpulsarwithdeps.a $PREFIX/lib/ + rm -rf arm64/ x86_64/ macos-arm64.zip macos-x86_64.zip +fi -cd "${ROOT_DIR}" +PYTHON_VERSION=$1 +PYTHON_VERSION_LONG=$2 + +MACOSX_DEPLOYMENT_TARGET=13 +pushd $CACHE_DIR + +# We need to build OpenSSL from source to have universal2 binaries +OPENSSL_VERSION=$(cat $ROOT_DIR/dependencies.yaml | grep openssl | awk '{print $2}') +OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') +if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then + echo "Building OpenSSL" + download_dependency $ROOT_DIR/dependencies.yaml openssl + tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz + + mv openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64 + pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64 + echo -e "#include \n$(cat test/v3ext.c)" > test/v3ext.c + CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./Configure --prefix=$PREFIX no-shared no-unit-test darwin64-arm64-cc + make -j8 + make install_sw + popd + + tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz + mv openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64 + pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64 + echo -e "#include \n$(cat test/v3ext.c)" > test/v3ext.c + CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./Configure --prefix=$PREFIX no-shared no-unit-test darwin64-x86_64-cc + make -j8 + make install_sw + popd + + # Create universal binaries + lipo -create openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64/libssl.a openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64/libssl.a \ + -output $PREFIX/lib/libssl.a + lipo -create openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64/libcrypto.a openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64/libcrypto.a \ + -output $PREFIX/lib/libcrypto.a + + touch openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done +else + echo "Using cached OpenSSL" +fi + +if [ ! -f Python-${PYTHON_VERSION_LONG}/.done ]; then + echo "Building Python $PYTHON_VERSION_LONG" + curl -O -L https://www.python.org/ftp/python/${PYTHON_VERSION_LONG}/Python-${PYTHON_VERSION_LONG}.tgz + tar xfz Python-${PYTHON_VERSION_LONG}.tgz + + pushd Python-${PYTHON_VERSION_LONG} + ./configure --prefix=$PREFIX --enable-shared --enable-universalsdk --with-universal-archs=universal2 --with-openssl=$PREFIX + make -j16 + make install + + curl -O -L https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl + export SSL_CERT_FILE=/etc/ssl/cert.pem + $PREFIX/bin/pip3 install wheel setuptools + $PREFIX/bin/pip3 install wheel-*.whl + + touch .done + popd +else + echo "Using cached Python $PYTHON_VERSION_LONG" +fi + +PYBIND11_VERSION=$(cat $ROOT_DIR/dependencies.yaml | grep pybind11 | awk '{print $2}') +if [ ! -f pybind11/.done ]; then + download_dependency $ROOT_DIR/dependencies.yaml pybind11 + mkdir -p $PREFIX/include/ + cp -rf pybind11-${PYBIND11_VERSION}/include/pybind11 $PREFIX/include/ + mkdir -p pybind11 + touch pybind11/.done +fi + +popd # $CACHE_DIR -rm -f CMakeCache.txt +PYTHON_CLIENT_VERSION=$(grep -v '^#' pulsar/__about__.py | cut -d "=" -f2 | sed "s/'//g") + +echo "Build wheel for Python $PYTHON_VERSION" PY_EXE=$PREFIX/bin/python3 PIP_EXE=$PREFIX/bin/pip3 @@ -50,19 +135,15 @@ PIP_EXE=$PREFIX/bin/pip3 ARCHS='arm64;x86_64' PIP_TAG='universal2' -cmake . \ +cmake -B build \ -DCMAKE_OSX_ARCHITECTURES=${ARCHS} \ -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_PREFIX_PATH=$PREFIX \ - -DCMAKE_CXX_FLAGS=-I$PREFIX/include \ -DLINK_STATIC=ON \ - -DPULSAR_LIBRARY=${CACHE_DIR_CPP_CLIENT}/install/lib/libpulsar.a \ - -DPULSAR_INCLUDE=${CACHE_DIR_CPP_CLIENT}/install/include \ -DPython3_ROOT_DIR=$PREFIX - -make clean -make -j16 +cmake --build build --config Release -j8 +cp -f build/lib_pulsar.so . $PY_EXE setup.py bdist_wheel @@ -70,6 +151,5 @@ PY_SPEC=$(echo $PYTHON_VERSION | sed 's/\.//g') cd /tmp $PIP_EXE install --no-dependencies --force-reinstall \ - $ROOT_DIR/dist/pulsar_client-${PYTHON_CLIENT_VERSION}-cp$PY_SPEC-*-macosx_10_15_${PIP_TAG}.whl + $ROOT_DIR/dist/pulsar_client-${PYTHON_CLIENT_VERSION}-cp$PY_SPEC-*-macosx*_${PIP_TAG}.whl $PY_EXE -c 'import pulsar' - diff --git a/pkg/mac/build-pulsar-cpp.sh b/pkg/mac/build-pulsar-cpp.sh deleted file mode 100755 index 51f1ef3..0000000 --- a/pkg/mac/build-pulsar-cpp.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -e -x - -ROOT_DIR=$(git rev-parse --show-toplevel) -cd "${ROOT_DIR}" - -source pkg/mac/common.sh -source build-support/dep-url.sh - -PULSAR_CPP_VERSION=$(./build-support/dep-version.py pulsar-cpp) - -# Compile and cache dependencies -mkdir -p $CACHE_DIR_CPP_CLIENT -cd $CACHE_DIR_CPP_CLIENT - -PREFIX=$CACHE_DIR_CPP_CLIENT/install - -DEPS_PREFIX=${CACHE_DIR_DEPS}/install - -############################################################################### -download_dependency $ROOT_DIR/dependencies.yaml pulsar-cpp -tar xfz apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz - -if [ ! -f apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}/.done ]; then - pushd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} - ARCHS='arm64;x86_64' - - cmake . \ - -DCMAKE_CXX_STANDARD=11 \ - -DCMAKE_OSX_ARCHITECTURES=${ARCHS} \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH=${DEPS_PREFIX} \ - -DCMAKE_CXX_FLAGS=-I${DEPS_PREFIX}/include \ - -DOPENSSL_ROOT_DIR=${DEPS_PREFIX} \ - -DLINK_STATIC=OFF \ - -DBUILD_TESTS=OFF \ - -DBUILD_WIRESHARK=OFF \ - -DBUILD_DYNAMIC_LIB=OFF \ - -DBUILD_STATIC_LIB=ON \ - -DPROTOC_PATH=${DEPS_PREFIX}/bin/protoc - - make -j16 install - popd -fi diff --git a/pkg/mac/common.sh b/pkg/mac/common.sh deleted file mode 100644 index fa2ed35..0000000 --- a/pkg/mac/common.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -export MACOSX_DEPLOYMENT_TARGET=10.15 -export MACOSX_DEPLOYMENT_TARGET_MAJOR=${MACOSX_DEPLOYMENT_TARGET%%.*} - -CACHE_DIR_DEPS=$PWD/.pulsar-mac-build/deps -CACHE_DIR_CPP_CLIENT=$PWD/.pulsar-mac-build/cpp diff --git a/pkg/mac/python-3.7.patch b/pkg/mac/python-3.7.patch deleted file mode 100644 index f788920..0000000 --- a/pkg/mac/python-3.7.patch +++ /dev/null @@ -1,185 +0,0 @@ -diff -r -u Python-3.7.15.orig/Lib/_osx_support.py Python-3.7.15/Lib/_osx_support.py ---- Python-3.7.15.orig/Lib/_osx_support.py 2022-10-10 05:34:53.000000000 -0700 -+++ Python-3.7.15/Lib/_osx_support.py 2022-10-14 08:10:34.000000000 -0700 -@@ -491,6 +491,8 @@ - machine = 'fat64' - elif archs == ('i386', 'ppc', 'ppc64', 'x86_64'): - machine = 'universal' -+ elif archs == ('arm64', 'x86_64'): -+ machine = 'universal2' - else: - raise ValueError( - "Don't know machine value for archs=%r" % (archs,)) -diff -r -u Python-3.7.15.orig/Modules/_ctypes/libffi_osx/include/ffi.h Python-3.7.15/Modules/_ctypes/libffi_osx/include/ffi.h ---- Python-3.7.15.orig/Modules/_ctypes/libffi_osx/include/ffi.h 2022-10-10 05:34:53.000000000 -0700 -+++ Python-3.7.15/Modules/_ctypes/libffi_osx/include/ffi.h 2022-10-14 08:54:33.000000000 -0700 -@@ -62,7 +62,8 @@ - # elif defined(__ppc__) || defined(__ppc64__) - # define POWERPC_DARWIN - # else --# error "Unsupported MacOS X CPU type" -+/* Temp fix to allow for universal2 to build through */ -+# define X86_DARWIN - # endif - #else - #error "Unsupported OS type" -@@ -199,9 +200,9 @@ - - void - ffi_raw_call( --/*@dependent@*/ ffi_cif* cif, -- void (*fn)(void), --/*@out@*/ void* rvalue, -+/*@dependent@*/ ffi_cif* cif, -+ void (*fn)(void), -+/*@out@*/ void* rvalue, - /*@dependent@*/ ffi_raw* avalue); - - void -@@ -225,9 +226,9 @@ - longs and doubles are followed by an empty 64-bit word. */ - void - ffi_java_raw_call( --/*@dependent@*/ ffi_cif* cif, -- void (*fn)(void), --/*@out@*/ void* rvalue, -+/*@dependent@*/ ffi_cif* cif, -+ void (*fn)(void), -+/*@out@*/ void* rvalue, - /*@dependent@*/ ffi_raw* avalue); - - void -@@ -272,8 +273,8 @@ - ffi_cif* cif; - - #if !FFI_NATIVE_RAW_API -- /* if this is enabled, then a raw closure has the same layout -- as a regular closure. We use this to install an intermediate -+ /* if this is enabled, then a raw closure has the same layout -+ as a regular closure. We use this to install an intermediate - handler to do the transaltion, void** -> ffi_raw*. */ - void (*translate_args)(ffi_cif*,void*,void**,void*); - void* this_closure; -@@ -303,17 +304,17 @@ - - ffi_status - ffi_prep_cif( --/*@out@*/ /*@partial@*/ ffi_cif* cif, -+/*@out@*/ /*@partial@*/ ffi_cif* cif, - ffi_abi abi, -- unsigned int nargs, --/*@dependent@*/ /*@out@*/ /*@partial@*/ ffi_type* rtype, -+ unsigned int nargs, -+/*@dependent@*/ /*@out@*/ /*@partial@*/ ffi_type* rtype, - /*@dependent@*/ ffi_type** atypes); - - void - ffi_call( --/*@dependent@*/ ffi_cif* cif, -- void (*fn)(void), --/*@out@*/ void* rvalue, -+/*@dependent@*/ ffi_cif* cif, -+ void (*fn)(void), -+/*@out@*/ void* rvalue, - /*@dependent@*/ void** avalue); - - /* Useful for eliminating compiler warnings */ -diff -r -u Python-3.7.15.orig/Modules/_ctypes/libffi_osx/include/fficonfig.h Python-3.7.15/Modules/_ctypes/libffi_osx/include/fficonfig.h ---- Python-3.7.15.orig/Modules/_ctypes/libffi_osx/include/fficonfig.h 2022-10-10 05:34:53.000000000 -0700 -+++ Python-3.7.15/Modules/_ctypes/libffi_osx/include/fficonfig.h 2022-10-14 13:19:13.000000000 -0700 -@@ -1,4 +1,4 @@ --/* Manually created fficonfig.h for Darwin on PowerPC or Intel -+/* Manually created fficonfig.h for Darwin on PowerPC or Intel - - This file is manually generated to do away with the need for autoconf and - therefore make it easier to cross-compile and build fat binaries. -@@ -18,7 +18,7 @@ - # define HAVE_LONG_DOUBLE 1 - # define SIZEOF_LONG_DOUBLE 16 - --#elif defined(__x86_64__) -+#elif defined(__x86_64__) || defined(__arm64__) - # define BYTEORDER 1234 - # undef HOST_WORDS_BIG_ENDIAN - # undef WORDS_BIGENDIAN -@@ -33,10 +33,10 @@ - # define SIZEOF_DOUBLE 8 - # if __GNUC__ >= 4 - # define HAVE_LONG_DOUBLE 1 --# define SIZEOF_LONG_DOUBLE 16 -+# define SIZEOF_LONG_DOUBLE 16 - # else - # undef HAVE_LONG_DOUBLE --# define SIZEOF_LONG_DOUBLE 8 -+# define SIZEOF_LONG_DOUBLE 8 - # endif - - #elif defined(__ppc64__) -@@ -147,4 +147,4 @@ - # else - # define FFI_HIDDEN - # endif --#endif -\ No newline at end of file -+#endif -diff -r -u Python-3.7.15.orig/Modules/_ctypes/libffi_osx/include/ffitarget.h Python-3.7.15/Modules/_ctypes/libffi_osx/include/ffitarget.h ---- Python-3.7.15.orig/Modules/_ctypes/libffi_osx/include/ffitarget.h 2022-10-10 05:34:53.000000000 -0700 -+++ Python-3.7.15/Modules/_ctypes/libffi_osx/include/ffitarget.h 2022-10-14 09:35:48.000000000 -0700 -@@ -4,10 +4,10 @@ - make building fat binaries harder. - */ - --#if defined(__i386__) || defined(__x86_64__) -+#if defined(__i386__) || defined(__x86_64__) || defined(__arm64__) - #include "x86-ffitarget.h" - #elif defined(__ppc__) || defined(__ppc64__) - #include "ppc-ffitarget.h" - #else - #error "Unsupported CPU type" --#endif -\ No newline at end of file -+#endif -diff -r -u Python-3.7.15.orig/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h Python-3.7.15/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h ---- Python-3.7.15.orig/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h 2022-10-10 05:34:53.000000000 -0700 -+++ Python-3.7.15/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h 2022-10-14 11:58:41.000000000 -0700 -@@ -33,7 +33,7 @@ - # define X86 - #endif - --#if defined(__x86_64__) -+#if defined(__x86_64__) || defined(__arm64__) - # ifndef X86_64 - # define X86_64 - # endif -@@ -58,7 +58,7 @@ - #endif - - /* ---- Intel x86 and AMD x86-64 - */ --#if !defined(X86_WIN32) && (defined(__i386__) || defined(__x86_64__)) -+#if !defined(X86_WIN32) && (defined(__i386__) || defined(__x86_64__) || defined(__arm64__)) - FFI_SYSV, - FFI_UNIX64, /* Unix variants all use the same ABI for x86-64 */ - # ifdef __i386__ -@@ -85,4 +85,4 @@ - # define FFI_NATIVE_RAW_API 1 /* x86 has native raw api support */ - #endif - --#endif // #ifndef LIBFFI_TARGET_H -\ No newline at end of file -+#endif // #ifndef LIBFFI_TARGET_H -Only in Python-3.7.15: config.log -diff -r -u Python-3.7.15.orig/configure Python-3.7.15/configure ---- Python-3.7.15.orig/configure 2022-10-10 05:34:53.000000000 -0700 -+++ Python-3.7.15/configure 2022-10-14 08:10:34.000000000 -0700 -@@ -7449,6 +7449,11 @@ - LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" - ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" - ;; -+ universal2) -+ UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" -+ LIPO_32BIT_FLAGS="" -+ ARCH_RUN_32BIT="" -+ ;; - *) - as_fn_error $? "proper usage is --with-universal-arch=32-bit|64-bit|all|intel|3-way" "$LINENO" 5 - ;; diff --git a/pkg/manylinux2014/Dockerfile b/pkg/manylinux2014/Dockerfile index c5c4cf0..d59ed3f 100644 --- a/pkg/manylinux2014/Dockerfile +++ b/pkg/manylinux2014/Dockerfile @@ -24,89 +24,11 @@ ARG PYTHON_VERSION ARG PYTHON_SPEC ARG ARCH +ENV CPP_BINARY_TYPE=rpm ENV PYTHON_VERSION=${PYTHON_VERSION} ENV PYTHON_SPEC=${PYTHON_SPEC} ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}" +ENV ARCH=${ARCH} -ENV PYTHON_INCLUDE_DIR /opt/python/${PYTHON_SPEC}/include -ENV PYTHON_LIBRARIES /opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION} - -RUN pip3 install pyyaml setuptools - -ADD .build/dependencies.yaml / -ADD .build/dep-version.py /usr/local/bin -ADD .build/dep-url.sh / - -# Download and install boost -RUN BOOST_VERSION=$(dep-version.py boost) && \ - BOOST_VERSION_UNDESRSCORE=$(echo $BOOST_VERSION | sed 's/\./_/g') && \ - . /dep-url.sh && download_dependency /dependencies.yaml boost && \ - cp -r boost_${BOOST_VERSION_UNDESRSCORE}/boost /usr/include/ && \ - rm -rf /boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz /boost_${BOOST_VERSION_UNDESRSCORE} - -RUN CMAKE_VERSION=$(dep-version.py cmake) && \ - . /dep-url.sh && ARCH=${ARCH} download_dependency /dependencies.yaml cmake && \ - cp cmake-${CMAKE_VERSION}-linux-${ARCH}/bin/* /usr/bin/ && \ - cp -r cmake-${CMAKE_VERSION}-linux-${ARCH}/share/cmake-* /usr/share/ && \ - rm -rf cmake-${CMAKE_VERSION}-linux-${ARCH} cmake-${CMAKE_VERSION}-linux-${ARCH}.tar.gz - -# Download and compile protobuf -RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \ - . /dep-url.sh && download_dependency /dependencies.yaml protobuf && \ - cd protobuf-${PROTOBUF_VERSION}/ && \ - CXXFLAGS=-fPIC ./configure && \ - make -j8 && make install && ldconfig && \ - rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION} - -# ZLib -RUN ZLIB_VERSION=$(dep-version.py zlib) && \ - . /dep-url.sh && download_dependency /dependencies.yaml zlib && \ - cd zlib-${ZLIB_VERSION} && \ - CFLAGS="-fPIC -O3" ./configure && \ - make -j8 && make install && \ - rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION} - -# Zstandard -RUN ZSTD_VERSION=$(dep-version.py zstd) && \ - . /dep-url.sh && download_dependency /dependencies.yaml zstd && \ - cd zstd-${ZSTD_VERSION} && \ - CFLAGS="-fPIC -O3" make -j8 && \ - make install && \ - rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz - -# Snappy -RUN SNAPPY_VERSION=$(dep-version.py snappy) && \ - . /dep-url.sh && download_dependency /dependencies.yaml snappy && \ - cd snappy-${SNAPPY_VERSION} && \ - CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \ - make -j8 && make install && \ - rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz - -RUN OPENSSL_VERSION=$(dep-version.py openssl) && \ - OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \ - . /dep-url.sh && download_dependency /dependencies.yaml openssl && \ - cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \ - ./config -fPIC --prefix=/usr/local/ssl/ && \ - make -j8 && make install && \ - rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} - -ENV LD_LIBRARY_PATH /usr/local/ssl/lib/:/usr/local/lib -ENV OPENSSL_ROOT_DIR /usr/local/ssl/ - -# LibCurl -RUN CURL_VERSION=$(dep-version.py curl) && \ - . /dep-url.sh && download_dependency /dependencies.yaml curl && \ - cd curl-${CURL_VERSION} && \ - CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \ - make -j8 && make install && \ - rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION} - -# Pulsar client C++ -RUN PULSAR_CPP_VERSION=$(dep-version.py pulsar-cpp) && \ - . /dep-url.sh && download_dependency /dependencies.yaml pulsar-cpp && \ - cd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} && \ - cmake . -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=OFF -DBUILD_STATIC_LIB=OFF -DLINK_STATIC=ON && \ - make -j8 && \ - make install && \ - rm -rf apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz +RUN pip3 install setuptools diff --git a/pkg/manylinux_musl/Dockerfile b/pkg/manylinux_musl/Dockerfile index 3002194..2681b88 100644 --- a/pkg/manylinux_musl/Dockerfile +++ b/pkg/manylinux_musl/Dockerfile @@ -30,80 +30,6 @@ ARG ARCH ENV ARCH=${ARCH} ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}" -ENV PYTHON_INCLUDE_DIR /opt/python/${PYTHON_SPEC}/include -ENV PYTHON_LIBRARIES /opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION} -RUN pip install pyyaml setuptools - -RUN apk add cmake - -ADD .build/dependencies.yaml / -ADD .build/dep-version.py /usr/local/bin -ADD .build/dep-url.sh / - -# Download and install boost -RUN BOOST_VERSION=$(dep-version.py boost) && \ - BOOST_VERSION_UNDESRSCORE=$(echo $BOOST_VERSION | sed 's/\./_/g') && \ - . /dep-url.sh && download_dependency /dependencies.yaml boost && \ - cp -r boost_${BOOST_VERSION_UNDESRSCORE}/boost /usr/include/ && \ - rm -rf /boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz /boost_${BOOST_VERSION_UNDESRSCORE} - -# Download and compile protobuf -RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \ - . /dep-url.sh && download_dependency /dependencies.yaml protobuf && \ - cd protobuf-${PROTOBUF_VERSION}/ && \ - CXXFLAGS=-fPIC ./configure && \ - make -j8 && make install && \ - rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION} - -# ZLib -RUN ZLIB_VERSION=$(dep-version.py zlib) && \ - . /dep-url.sh && download_dependency /dependencies.yaml zlib && \ - cd zlib-${ZLIB_VERSION} && \ - CFLAGS="-fPIC -O3" ./configure && \ - make -j8 && make install && \ - rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION} - -# Zstandard -RUN ZSTD_VERSION=$(dep-version.py zstd) && \ - . /dep-url.sh && download_dependency /dependencies.yaml zstd && \ - cd zstd-${ZSTD_VERSION} && \ - CFLAGS="-fPIC -O3" make -j8 && \ - make install && \ - rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz - -# Snappy -RUN SNAPPY_VERSION=$(dep-version.py snappy) && \ - . /dep-url.sh && download_dependency /dependencies.yaml snappy && \ - cd snappy-${SNAPPY_VERSION} && \ - CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \ - make -j8 && make install && \ - rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz - -RUN OPENSSL_VERSION=$(dep-version.py openssl) && \ - OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \ - . /dep-url.sh && download_dependency /dependencies.yaml openssl && \ - cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \ - ./config -fPIC --prefix=/usr/local/ssl/ && \ - make -j8 && make install && \ - rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} - -ENV LD_LIBRARY_PATH /usr/local/ssl/lib/:/usr/local/lib -ENV OPENSSL_ROOT_DIR /usr/local/ssl/ - -# LibCurl -RUN CURL_VERSION=$(dep-version.py curl) && \ - . /dep-url.sh && download_dependency /dependencies.yaml curl && \ - cd curl-${CURL_VERSION} && \ - CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \ - make -j8 && make install && \ - rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION} - -# Pulsar client C++ -RUN PULSAR_CPP_VERSION=$(dep-version.py pulsar-cpp) && \ - . /dep-url.sh && download_dependency /dependencies.yaml pulsar-cpp && \ - cd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} && \ - cmake . -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=OFF -DBUILD_STATIC_LIB=OFF -DLINK_STATIC=ON && \ - make -j8 && \ - make install && \ - rm -rf apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz +RUN pip install setuptools +ENV CPP_BINARY_TYPE="apk"