Merge pull request #509 from zodb/py-updates

Add Python 3.13; drop Python 3.8

.github/workflows/tests.yml

@@ -87,7 +87,13 @@ jobs:
       matrix:
         # 3.12.1 breaks zope.testrunner.
         # See https://github.com/zopefoundation/zope.testrunner/issues/157
-        python-version: ['pypy-3.10', '3.12.0', 3.8, 3.9, '3.10', '3.11']
+        python-version:
+          - '3.9'
+          - '3.10'
+          - '3.11'
+          - "3.12"
+          - "3.13"
+          - "pypy-3.10-v7.3.17"
         os: [ubuntu-latest, macos-latest]
         exclude:
           # Can't build persistent on 3.12 with -UNDEBUG because it
@@ -96,14 +102,14 @@ jobs:
           # persistent wheel, we have to exclude that one.
           # The same thing goes for PyPy.
           - os: ubuntu-latest
-            python-version: pypy-3.10
+            python-version: "pypy-3.10-v7.3.17"
 
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: 'pip'
@@ -166,7 +172,7 @@ jobs:
           ls -l dist
           twine check dist/*
       - name: Upload RelStorage wheel
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: RelStorage-${{ runner.os }}-${{ matrix.python-version }}.whl
           path: dist/*whl
@@ -250,26 +256,26 @@ jobs:
     name: ${{ matrix.image }}
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
         with:
           platforms: all
       - name: Build RelStorage
         env:
           DOCKER_IMAGE: quay.io/pypa/${{ matrix.image }}
         run: bash ./scripts/releases/make-manylinux
       - name: Store RelStorage wheels
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           path: wheelhouse/*whl
           name: ${{ matrix.image }}_wheels.zip
       - name: Publish package to PyPI
-        uses: pypa/gh-action-pypi-publish@v1.4.1
+        uses: pypa/gh-action-pypi-publish@v1.10.3
         if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
         with:
           user: __token__

.pylintrc

@@ -75,7 +75,7 @@ init-hook =
 # comments at the end of the line does the same thing (though Py3 supports
 # mixing)
 
-
+# too-many-positional-arguments: we inherit most of these and can't do anything about them.
 # invalid-name, ; We get lots of these, especially in scripts. should fix many of them
 # protected-access, ; We have many cases of this; legit ones need to be examinid and commented, then this removed
 # no-self-use, ; common in superclasses with extension points
@@ -146,7 +146,8 @@ disable=wrong-import-position,
     missing-param-doc,
     differing-param-doc,
     differing-type-doc,
-    compare-to-zero
+    compare-to-zero,
+    too-many-positional-arguments
 
 enable=consider-using-augmented-assign
 

CHANGES.rst

@@ -9,7 +9,10 @@
   to support newer compilers like GCC 13.
 - Compile in C++ 11 mode instead of whatever the compiler default was
   (sometimes C++ 03), because the latter is deprecated by Boost.
-
+- Stop relying on an undeclared dependency on ``six``. See
+  :issue:`504`.
+- Drop support for Python 3.8.
+- Add support for Python 3.13.
 
 4.0.0 (2023-12-11)
 ==================

appveyor.yml

@@ -37,8 +37,13 @@ environment:
     # Fully supported 64-bit versions, with testing. This should be
     # all the current (non EOL) versions.
 
+    - PYTHON: "C:\\Python313-x64"
+      PYTHON_VERSION: "3.13.0"
+      PYTHON_ARCH: "64"
+      PYTHON_EXE: python
+
     - PYTHON: "C:\\Python312-x64"
-      PYTHON_VERSION: "3.12.0b4"
+      PYTHON_VERSION: "3.12.0"
       PYTHON_ARCH: "64"
       PYTHON_EXE: python
 
@@ -57,30 +62,21 @@ environment:
       PYTHON_ARCH: "64"
       PYTHON_EXE: python
 
-    - PYTHON: "C:\\Python38-x64"
-      PYTHON_VERSION: "3.8.x"
-      PYTHON_ARCH: "64"
-      PYTHON_EXE: python
-
-    ## 32-bit, wheel only (no testing)
 
-    - PYTHON: "C:\\Python38"
-      PYTHON_VERSION: "3.8.x"
-      PYTHON_ARCH: "32"
-      PYTHON_EXE: python
-      GWHEEL_ONLY: true
 
 # The images have moved to MySQL8 only, and that cannot
 # currently (Dec 2023) be started with the services: syntax.
 # See https://github.com/appveyor/ci/issues/3834
 services:
-  - postgresql101
+  - postgresql13
+
 
 install:
   - ECHO "Filesystem root:"
   - ps: "ls \"C:/\""
   - ps: "ls \"C:/Program Files/\""
   - ps: "ls \"C:/Program Files/MySQL\""
+  - ps: "ls \"C:/Program Files/PostgreSQL\""
   - ps: "ls \"C:/Program Files (x86)/\""
 
   - ECHO "Installed SDKs:"
@@ -104,7 +100,8 @@ install:
         & scripts\install.ps1;
       }
 
-  - "SET PATH=C:\\Program Files\\PostgreSQL\\10\\bin;C:\\Program Files\\MySql\\MySQL Server 8.0\\bin;%PATH%"
+  # If commands start failing, like psql isn't found, tweak the PATH here.
+  - "SET PATH=C:\\Program Files\\PostgreSQL\\13\\bin;C:\\Program Files\\MySql\\MySQL Server 8.0\\bin;%PATH%"
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PYTHON%\\bin;%PATH%"
   - "SET PYEXE=%PYTHON%\\%PYTHON_EXE%.exe"
 

scripts/releases/make-manylinux

@@ -22,10 +22,10 @@ export PIP_NO_WARN_SCRIPT_LOCATION=1
 
 # Build configuration.
 
-export CC="ccache gcc"
-export CXX="ccache g++"
-export LDCXXSHARED="ccache g++ -shared"
-export LDSHARED="ccache gcc -shared"
+export CC="ccache $(which cc)"
+export CXX="ccache $(which c++)"
+export LDCXXSHARED="ccache $(which c++) -shared"
+export LDSHARED="ccache $(which cc) -shared"
 export CCACHE_NOCPP2=true
 export CCACHE_SLOPPINESS=file_macro,time_macros,include_file_ctime,include_file_mtime
 export CCACHE_NOHASHDIR=true
@@ -91,19 +91,29 @@ if [ -d /RelStorage -a -d /opt/python ]; then
             echo Installing epel
             rpm -Uvh https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/e/epel-release-7-14.noarch.rpm || true
         fi
-        yum -y install ccache || export CC=gcc LDSHARED="gcc -shared" LDCXXSHARED="gcc -shared"
+        yum -y install ccache || export CC=gcc CXX=c++ LDSHARED="gcc -shared" LDCXXSHARED="gcc -shared"
     fi
     if [ -e /sbin/apk ]; then
         # musllinux
         apk add ccache
     fi
 
+    # Ahh, overprotective security. Disable it.
+    echo "Fixing git's paranoia"
+    git config --global --add safe.directory /RelStorage/.git
+
     cd /RelStorage
     rm -rf wheelhouse
     mkdir wheelhouse
     ls -l /opt/python
-    for variant in `ls -d /opt/python/cp{312,38,39,310,311}*`; do
+    for variant in `ls -d /opt/python/cp{312,39,310,311,313}*`; do
         echo "Building $variant"
+        if [ "$variant" = "/opt/python/cp313-cp313t" ]; then
+            # It appears that Cython 3.0.11 cannot produce code that
+            # works here. Lots of compiler errors.
+            echo "Unable to build without gil"
+            continue
+        fi
         mkdir /tmp/build
         cd /tmp/build
         git clone /RelStorage RelStorage

setup.py

@@ -169,17 +169,17 @@ def read_file(*path):
     platforms=["any"],
     description="A backend for ZODB that stores pickles in a relational database.",
     # 3.8: importlib.metadata
-    python_requires=">=3.8",
+    python_requires=">=3.9",
     classifiers=[
         "Intended Audience :: Developers",
         "License :: OSI Approved :: Zope Public License",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
         "Topic :: Database",
@@ -298,16 +298,18 @@ def read_file(*path):
         # and the authors specifically request that other modules not depend on
         # psycopg2-binary.
         # See http://initd.org/psycopg/docs/install.html#binary-packages
-        'postgresql: platform_python_implementation == "CPython"': [
+        'postgresql: platform_python_implementation == "CPython" and python_version != "3.13"' : [
             # 2.4.1+ is required for proper bytea handling;
             # 2.6+ is needed for 64-bit lobject support;
             # 2.7+ is needed for Python 3.7 support and PostgreSQL 10+;
             # 2.7.6+ is needed for PostgreSQL 11;
             # 2.8 is needed for conn.info
+            # 2.9.10 will be needed for Python 3.13, but it's not out yet.
             'psycopg2 >= 2.8.3',
         ],
-        'postgresql: platform_python_implementation == "CPython" and python_version == "3.12"': [
-            # psycopg2 isn't importing on Appveyor/3.12b4.
+        'postgresql: platform_python_implementation == "CPython" and python_version == "3.13"': [
+            # psycopg2 2.9.10 is needed, but not available yet.
+            # See also 'all tested drivers'
             'pg8000',
         ],
         'postgresql: platform_python_implementation == "PyPy"': [
@@ -353,11 +355,11 @@ def read_file(*path):
             # pure-python
             # pg8000
             # This requirement is repeated in the driver class.
-            'pg8000 >= 1.29.0; python_version == "3.11"',
+            'pg8000 >= 1.29.0; python_version == "3.11" or python_version == "3.13"',
             # CFFI, runs on all implementations.
             'psycopg2cffi >= 2.7.4; python_version == "3.11" or platform_python_implementation == "PyPy"',
             # Psycopg2 on all CPython, it's the default
-            'psycopg2 >= 2.8.3; platform_python_implementation == "CPython"',
+            'psycopg2 >= 2.8.3; platform_python_implementation == "CPython" and python_version != "3.13"',
         ],
     },
     entry_points={

src/relstorage/adapters/batch.py

@@ -197,8 +197,7 @@ def select_from(self, columns, table, suffix='', timeout=None, **kw):
                 timeout,
                 kw
         ):
-            for row in cursor.fetchall():
-                yield row
+            yield from cursor.fetchall()
 
     def update_set_static(self, update_set, timeout=None,
                           batch_done_callback=lambda total_count: None,

src/relstorage/adapters/locker.py

@@ -20,7 +20,6 @@
 
 import abc
 import sys
-import six
 
 
 from relstorage._compat import ABC
@@ -277,10 +276,7 @@ def reraise_commit_lock_error(self, cursor, lock_stmt, kind):
         val = kind(message)
         val.__relstorage_cause__ = v
         del v
-        six.reraise(
-            kind,
-            val,
-            sys.exc_info()[2])
+        raise val.with_traceback(sys.exc_info()[2])
 
     # MySQL allows aggregates in the top level to use FOR UPDATE,
     # but PostgreSQL does not, so we have to use the second form.

src/relstorage/adapters/mysql/drivers/__init__.py

@@ -135,8 +135,7 @@ def __iter__(self):
         sleep = self.sleep
         batch = fetch()
         while batch:
-            for row in batch:
-                yield row
+            yield from batch
             if sleep is not None:
                 sleep() # pylint:disable=not-callable
             batch = fetch()

src/relstorage/adapters/oracle/adapter.py

@@ -86,6 +86,7 @@ def _create(self):
         twophase = self._twophase
         options = self.options
         dsn = self._dsn
+        inputsizes = None
 
         batcher_factory = lambda cursor, row_limit=None: OracleRowBatcher(
             cursor, inputsizes, row_limit

src/relstorage/adapters/oracle/batch.py

@@ -49,7 +49,7 @@ def replace_var(match):
             new_name = '%s_%d' % (name, rownum) # pylint:disable=used-before-assignment
             if name in self.inputsizes:
                 stmt_inputsizes[new_name] = self.inputsizes[name]
-            params[new_name] = row[name]
+            params[new_name] = row[name] # pylint:disable=possibly-used-before-assignment
             return ':%s' % new_name
 
         items = sorted(self.inserts.items())

src/relstorage/adapters/tests/test_replica.py

@@ -129,7 +129,7 @@ def test_next_with_new_conf(self):
 
 def test_suite():
     suite = unittest.TestSuite()
-    suite.addTest(unittest.makeSuite(ReplicaSelectorTests))
+    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(ReplicaSelectorTests))
     return suite
 
 if __name__ == '__main__':

src/relstorage/adapters/tests/test_util.py

@@ -103,7 +103,7 @@ def thing(self, arg1, arg2=False):
 
 def test_suite():
     suite = unittest.TestSuite()
-    suite.addTest(unittest.makeSuite(TestNoOp))
+    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestNoOp))
     return suite
 
 if __name__ == '__main__':

src/relstorage/cache/tests/test_cache_stats.py

@@ -162,7 +162,7 @@ class NoTest(unittest.TestCase):
             @unittest.skip("ZEO not installed")
             def test_cache_trace_analysis(self):
                 "Does nothing"
-        suite.addTest(unittest.makeSuite(NoTest))
+        suite.addTest(unittest.TestLoader().loadTestsFromTestCase(NoTest))
     else:
         suite.addTest(
             doctest.DocFileSuite(

src/relstorage/cache/tests/test_mvcc.py

@@ -459,8 +459,7 @@ class MockRowBatcher(RowBatcher):
             def select_from(self, *args, **kwargs):
                 # pylint:disable=signature-differs
                 try:
-                    for x in RowBatcher.select_from(self, *args, **kwargs):
-                        yield x
+                    yield from RowBatcher.select_from(self, *args, **kwargs)
                 except AggregateOperationTimeoutError as ex:
                     MockRowBatcher.ex = ex
                     raise

src/relstorage/storage/__init__.py

@@ -594,8 +594,7 @@ def __record_iternext_gen(self, start_oid_int):
         # at least so it seems. So here we make sure to have the cursor open.
         getattr(self._load_connection, 'cursor')
         with self._load_connection.server_side_cursor() as ss_cursor:
-            for record in self._adapter.dbiter.iter_current_records(ss_cursor, start_oid_int):
-                yield record
+            yield from self._adapter.dbiter.iter_current_records(ss_cursor, start_oid_int)
 
     def record_iternext(self, next=None): # pylint:disable=redefined-builtin
         """

src/relstorage/storage/tpc/vote.py

@@ -534,6 +534,7 @@ def tpc_finish(self, storage, transaction, f=None, _time=time.time):
             raise StorageTransactionError(
                 "tpc_finish called with wrong transaction")
         try:
+            locks_released = 0
             finish_entry = _time()
             # Handle the finishing. We cannot/must not fail now.
             # TODO: Move most of this into the Finish class/module.

src/relstorage/tests/RecoveryStorage.py

@@ -212,7 +212,7 @@ def checkPackWithGCOnDestinationAfterRestore(self):
 
         # Now pack the destination.
         from ZODB.utils import u64 as bytes8_to_int64
-        if IRelStorage.providedBy(self._dst):
+        if IRelStorage.providedBy(self._dst): # pylint:disable=no-value-for-parameter
             packtime = bytes8_to_int64(storage_last_tid)
         else:
             from persistent.timestamp import TimeStamp

src/relstorage/tests/__init__.py

@@ -276,8 +276,7 @@ def close(self):
         self.closed = True
 
     def __iter__(self):
-        for row in self.results:
-            yield row
+        yield from self.results
 
 class MockOptions(Options):
     cache_module_name = '' # disable