diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000..4976823 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,192 @@ +name: Build and test wheels + +on: + workflow_dispatch: + push: + tags: + - v* + +jobs: + wheels: + name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04, windows-2019, macos-11] + arch: ["x86_64", "arm64"] + include: + - os: windows-2019 + arch: win_amd64 + - os: windows-2019 + arch: win32 + - os: ubuntu-20.04 + arch: i686 + - os: ubuntu-20.04 + arch: aarch64 + - os: ubuntu-20.04 + arch: ppc64le + - os: ubuntu-20.04 + arch: s390x + exclude: + - os: windows-2019 + arch: "x86_64" + - os: ubuntu-20.04 + arch: "arm64" + steps: + - name: Checkout repo + uses: actions/checkout@v3 + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v2 + with: + platforms: all + - name: Build and test wheels + uses: pypa/cibuildwheel@v2.12.3 + env: + CIBW_BUILD: "*${{ matrix.arch }}" + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: ${{ runner.os }}-${{ matrix.arch }} + path: wheelhouse/*.whl + if-no-files-found: error + + wheels-old: + name: Build old wheels on ${{ matrix.os }} for ${{ matrix.arch }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04, windows-2019, macos-11] + arch: ["x86_64"] + include: + - os: windows-2019 + arch: win_amd64 + - os: windows-2019 + arch: win32 + - os: ubuntu-20.04 + arch: i686 + - os: ubuntu-20.04 + arch: aarch64 + - os: ubuntu-20.04 + arch: ppc64le + - os: ubuntu-20.04 + arch: s390x + exclude: + - os: windows-2019 + arch: "x86_64" + steps: + - name: Checkout repo + uses: actions/checkout@v3 + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v2 + with: + platforms: all + - name: Build and test wheels + uses: pypa/cibuildwheel@v1.12.0 + env: + CIBW_ARCHS: all + CIBW_BUILD_VERBOSITY: 3 + CIBW_BUILD: "*${{ matrix.arch }}" + CIBW_SKIP: cp36-* cp37-* cp38-* cp39-* pp37-* pp*-macosx* cp35-macosx_x86_64 + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: pytest {package}/tests + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: ${{ runner.os }}-${{ matrix.arch }} + path: wheelhouse/*.whl + if-no-files-found: error + + wheels-win27: + name: Build Python 2.7 wheels on Windows ${{ matrix.arch }} + runs-on: windows-2019 + strategy: + fail-fast: false + matrix: + arch: ["amd64", "win32"] + steps: + - name: Checkout repo + uses: actions/checkout@v3 + - name: Prepare compiler environment for Windows + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: ${{ matrix.arch }} + - name: Set Windows environment variables + if: runner.os == 'Windows' + shell: bash + run: | + echo "DISTUTILS_USE_SDK=1" >> $GITHUB_ENV + echo "MSSdk=1" >> $GITHUB_ENV + - name: Build and test wheels + uses: pypa/cibuildwheel@v1.12.0 + env: + CIBW_BUILD_VERBOSITY: 3 + CIBW_BUILD: "*27*${{ matrix.arch }}" + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: pytest {package}/tests + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: ${{ runner.os }}-${{ matrix.arch }} + path: wheelhouse/*.whl + if-no-files-found: error + + sdist: + name: Build sdist + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v3 + - name: Setup Python + uses: actions/setup-python@v4 + with: + # Build sdist on lowest supported Python. + # 3.8 is used because 3.6 has issues with pyproject.toml + # and 3.7 has bpo-41316. + python-version: '3.8' + - name: Build sdist + run: | + python -m pip install -U pip + python -m pip install -U build + python -m build -s + - name: Test sdist + run : python -m pip install dist/*.gz + - name: Upload sdist + uses: actions/upload-artifact@v3 + with: + name: sdist + path: dist/* + if-no-files-found: error + + publish: + name: Publish builds on PyPI + runs-on: ubuntu-latest + needs: [sdist, wheels, wheels-old, wheels-win27] + if: ${{ always() && (needs.sdist.result == 'success' || needs.wheels.result == 'success' || needs.wheels-old.result == 'success' || needs.wheels-win27.result == 'success') }} + environment: + name: release + url: https://test.pypi.org/p/python-lzo + permissions: + id-token: write + steps: + - name: Download builds + uses: actions/download-artifact@v3 + with: + path: dist + - name: Organise + working-directory: dist + run: | + mv **/*.whl . + mv **/*.gz . + find . -type d -delete + - name: Display structure of downloaded files + run: ls -R + - name: Publish package distributions to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + print-hash: true + repository-url: https://test.pypi.org/legacy/ \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index 7c63cd4..578bd8f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,11 +5,7 @@ # written by Markus F. X. J. Oberhumer # include COPYING -include README -include INSTALL -include MANIFEST.in include Makefile include NEWS -include lzomodule.c -include setup.py include tests/*.py +graft lzo-2.10 \ No newline at end of file diff --git a/PKG-INFO b/PKG-INFO deleted file mode 100644 index 817486a..0000000 --- a/PKG-INFO +++ /dev/null @@ -1,19 +0,0 @@ -Metadata-Version: 1.0 -Name: python-lzo -Version: 1.08 -Summary: Python bindings for the LZO data compression library -Home-page: http://www.oberhumer.com/opensource/lzo/ -Author: Markus F.X.J. Oberhumer -Author-email: markus@oberhumer.com -License: GNU General Public License (GPL) -Description: - This module provides Python bindings for the LZO data compression library. - - LZO is a portable lossless data compression library written in ANSI C. - It offers pretty fast compression and *very* fast decompression. - Decompression requires no memory. - - In addition there are slower compression levels achieving a quite - competitive compression ratio while still decompressing at - this very high speed. -Platform: All diff --git a/README.md b/README.md index 5b6b546..ad15709 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,4 @@ -![example workflow](https://github.com/jd-boyd/python-lzo/actions/workflows/tests.yaml/badge.svg) - -[![Appveyor Build Status](https://ci.appveyor.com/api/projects/status/github/jd-boyd/python-lzo?svg=true)](https://ci.appveyor.com/project/jd-boyd/python-lzo/branch/master) +[![Build and tests](https://github.com/jd-boyd/python-lzo/actions/workflows/wheels.yml/badge.svg)](https://github.com/jd-boyd/python-lzo/actions/workflows/wheels.yml) ``` Python-LZO -- Python bindings for LZO @@ -34,70 +32,57 @@ Python strings. # Installation -## Ubuntu/Debian Linux - -You need the following dependencies installed: -* `zlib1g-dev` -* `liblzo2-dev` -* `python-pip` or `python3-pip` - -Then, just `pip install python-lzo`. - -## CentOS/Red Hat Enterprise Linux - -You need the following dependencies installed: - -* `yum install epel-release` -* `yum install python-pip` -* `yum install lzo-devel` -* `yum install lzo-minilzo` - -Then, just `pip install python-lzo`. - -## macOS - -You need XCode installed, or something else that provides suitable C -compiler. Then just: - -`pip install python-lzo` +``` +pip install python-lzo +``` +Or explicitly from source, +either from a specific release or from the repo (requires build tools): +``` +pip install python-lzo-x.y.tar.gz +pip install https://[...]/python-lzo-x.y.tar.gz +pip install git+https://github.com/jd-boyd/python-lzo +``` -## Windows +# Building from source -There is an issue filed to provide a pre-compiled wheel to greatly -simplify this. +Building from source requires build tools. On most Linux distributions +they are probably already installed. On Windows you need +[Microsoft C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/) +(which should already be installed if you have Visual Studio). +On macOS you need XCode installed, or something else that provides a suitable C +compiler. Then either `git clone`, or download a source distribution and untar it. +Once you are in the root of the project directory where `pyproject.toml` is located, +run `python -m build -w`. This should build a wheel in the `dist` directory. +You might need to install `build` with `pip install build`. -Until then, here we go: -* Download http://www.oberhumer.com/opensource/lzo/download/lzo-2.09.tar.gz -* extract that file to `c:\src\` (should produce `c:\src\lzo-2.09` ) -* Install a visual studio compatible with your python build. If you are using - python 2.7, MS has a package just for you at: https://www.microsoft.com/en-us/download/details.aspx?id=44266 - Current automated tests are done with Visual Studio 2013, but newer should work as well. -* `python.exe setup.py install` +If you really want to build a wheel for Python 2.7 on Windows you'll need the +[Microsoft Visual C++ Compiler for Python 2.7](https://web.archive.org/web/20210116063838/https://www.microsoft.com/en-us/download/details.aspx?id=44266). # Where's the documentation ? -Python-LZO comes with built-in documentation which is accessible -using `lzo.__doc__` and `lzo.func__doc__`. See ["Chapter 3: Data Model" -in the Python Reference Manual](https://docs.python.org/3.6/reference/datamodel.html) for more information. - +Python-LZO comes with built-in documentation which is accessible using +```py +>>> import lzo +>>> help(lzo) +``` Additionally you should read the docs and study the example programs that ship with the LZO library. # Python 2 support statement -As of 1.15, Python 2.x is unsupported. To continue using Python 2.6 or -Python 2.7, please stick with version 1.14. +Python 2.7 is still supported but without being a priority. +Support will be dropped soon. # Notes -The Windows version is tested. - -It is not currently continuously tested on OSX, but that is coming. +Wheels are built with [cibuildwheel](https://cibuildwheel.readthedocs.io/) +on GitHub Actions. Tests are run for all combinations of platform and +Python version that it can run tests for. # Releasing -1. Update version in `setup.py` and the `MODULE_VERSION` define in - `lzomodule.c`. +1. Update version in `pyproject.toml`, `setup.py` and the `MODULE_VERSION` + define in `lzomodule.c`. 1. Update NEWS. 1. Tag with new release. 1. wheels (download from github actions) diff --git a/lzomodule.c b/lzomodule.c index f418ad1..9a12143 100644 --- a/lzomodule.c +++ b/lzomodule.c @@ -113,7 +113,11 @@ compress(PyObject *dummy, PyObject *args, PyObject *kwds) /* init */ UNUSED(dummy); +#if PY_VERSION_HEX >= 0x03030000 if (!PyArg_ParseTupleAndKeywords(args, kwds, "s#|ii$s", argnames, &in, &len, &level, &header, &algorithm)) +#else + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s#|iis", argnames, &in, &len, &level, &header, &algorithm)) +#endif return NULL; if (len < 0) return NULL; @@ -309,7 +313,11 @@ decompress(PyObject *dummy, PyObject *args, PyObject *kwds) /* init */ UNUSED(dummy); +#if PY_VERSION_HEX >= 0x03030000 if (!PyArg_ParseTupleAndKeywords(args, kwds, "s#|ii$s", argnames, &in, &len, &header, &buflen, &algorithm)) +#else + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s#|iis", argnames, &in, &len, &header, &buflen, &algorithm)) +#endif return NULL; if (header) { if (len < 5 + 3 || in[0] < 0xf0 || in[0] > 0xf1) diff --git a/pyproject.toml b/pyproject.toml index de11f32..437e56c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,3 +5,37 @@ requires = [ ] build-backend = "setuptools.build_meta" +[project] +name = "python-lzo" +version = "1.15" +description = "Python bindings for the LZO data compression library" +readme = "README.md" +requires-python = ">=2.6" +authors = [ + {name = "Markus F.X.J. Oberhumer", email = "markus@oberhumer.com"}, +] +maintainers = [ + {name = "Joshua D. Boyd", email = "jdboyd@jdboyd.net"}, +] +keywords = ["lzo", "compression"] +license = {text = "GNU General Public License v2 (GPLv2)"} +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", + "Topic :: Software Development :: Libraries", + "Topic :: System :: Archiving :: Compression", +] + +[project.optional-dependencies] +test = ["pytest"] + +[project.urls] +Homepage = "https://github.com/jd-boyd/python-lzo" + +[tool.cibuildwheel] +archs = ["all"] +build-verbosity = 3 +test-requires = "pytest" +test-command = "pytest {package}/tests" +test-skip = "*-win_arm64 *-macosx_arm64 *-macosx_universal2:arm64" \ No newline at end of file diff --git a/setup.py b/setup.py index 4897930..de012e8 100644 --- a/setup.py +++ b/setup.py @@ -1,24 +1,14 @@ -#! /usr/bin/env python +#!/usr/bin/env python # vi:ts=4:et from __future__ import print_function -import os, sys -import setuptools -import distutils -from distutils.cmd import Command -from distutils.core import setup -from distutils.extension import Extension -from distutils.util import split_quoted +import os +import subprocess +import sys +from glob import glob +from setuptools import Command, Extension, setup -include_dirs = [] -define_macros = [] -library_dirs = [] -libraries = [] -runtime_library_dirs = [] -extra_objects = [] -extra_compile_args = [] -extra_link_args = [] class TestCommand(Command): user_options = [] @@ -30,74 +20,15 @@ def finalize_options(self): pass def run(self): - import sys, subprocess raise SystemExit( subprocess.call([sys.executable, - '-m', - 'pytest'])) + "-m", + "pytest"])) -if sys.platform == "win32": - # Windows users have to configure the LZO_DIR path parameter to match - # their LZO source installation. The path set here is just an example - # and thus unlikely to match your installation. +lzo_dir = os.environ.get("LZO_DIR", "lzo-2.10") # Relative path. - LZO_DIR = os.environ.get('LZO_DIR', r".\lzo-2.10") - print("LZO_DIR:", LZO_DIR) - if not os.path.exists(LZO_DIR): - raise Exception("please set LZO_DIR to where the lzo source lives") - include_dirs.append(os.path.join(LZO_DIR, r"include\lzo")) - include_dirs.append(os.path.join(LZO_DIR, "include")) - lib1_file = os.path.join(LZO_DIR, "lzo.lib") - lib2_file = os.path.join(LZO_DIR, "lzo2.lib") - if os.path.exists(lib2_file): - lib_file = lib2_file - else: - lib_file = lib1_file - extra_objects.append(lib_file) - -elif sys.platform == "darwin": - libraries = ["lzo2"] - - LZO_DIR = os.environ.get('LZO_DIR', "/usr") - - if not os.path.exists(LZO_DIR+"/include/lzo"): - LZO_DIR = os. getcwd()+"/lzo-2.10" - - print("LZO_DIR:", LZO_DIR) - - print("include dir:", LZO_DIR+"/include") - - include_dirs.append(LZO_DIR+"/include") - library_dirs.append(LZO_DIR+"/lib") - - extra_link_args.append("-flat_namespace") - -else: - libraries = ["lzo2"] - include_dirs.append(os.environ.get("PREFIX", "/usr")+"/include") - ##library_dirs.append("/usr/local/lib") - ##runtime_library_dirs.append("/usr/local/lib") - - -############################################################################### - -def get_kw(**kw): return kw - -ext = Extension( - name="lzo", - sources=["lzomodule.c"], - include_dirs=include_dirs, - define_macros=define_macros, - library_dirs=library_dirs, - libraries=libraries, - runtime_library_dirs=runtime_library_dirs, - extra_objects=extra_objects, - extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args, -) - -setup_args = get_kw( +setup( name="python-lzo", version="1.15", description="Python bindings for the LZO data compression library", @@ -109,22 +40,22 @@ def get_kw(**kw): return kw license="GNU General Public License (GPL)", tests_require=['pytest'], cmdclass={ - 'test': TestCommand + "test": TestCommand }, - ext_modules=[ext], + ext_modules=[ + Extension( + name="lzo", + sources=["lzomodule.c"] + glob(os.path.join(lzo_dir, "src/*.c")), + include_dirs=[os.path.join(lzo_dir, "include")], + extra_link_args=["-flat_namespace"] if sys.platform == "darwin" else [], + ) + ], long_description=""" This module provides Python bindings for the LZO data compression library. - LZO is a portable lossless data compression library written in ANSI C. It offers pretty fast compression and *very* fast decompression. Decompression requires no memory. - In addition there are slower compression levels achieving a quite competitive compression ratio while still decompressing at this very high speed.""", ) - -if distutils.__version__ >= "1.0.2": - setup_args["platforms"] = "All" - -setup(**setup_args) diff --git a/tests/test_lzo.py b/tests/test_lzo.py index 12512de..585168d 100644 --- a/tests/test_lzo.py +++ b/tests/test_lzo.py @@ -86,7 +86,7 @@ def gen_all(src): raise lzo.error("internal error 1: %r %r", src, u1) if a0 != a1: raise lzo.error("internal error 2") - print(f"compressed using {algo} {len(src): 6} -> {len(c): 6}") + print("compressed using {} {:6} -> {:6}".format(algo, len(src), len(c))) def gen_raw(src, level=1): @@ -144,8 +144,16 @@ def test_lzo_big_raw(): gen_raw(b" " * 131072) -if sys.maxsize > 1<<32: +def is_pypy(): + if sys.version_info >= (3, 3): + return sys.implementation.name == "pypy" + else: + return "pypy" in sys.version.lower() + + +if sys.maxsize > 1<<32 and not is_pypy(): # This test raises OverflowError on 32-bit Pythons. Compressing # this much data requires a 64-bit system. + # On PyPy it raises MemoryError. def test_lzo_compress_extremely_big(): b = lzo.compress(bytes(bytearray((1024**3)*2)))