From 856d43dcfd7b472e696a37af39d273274ae72097 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 20 Aug 2024 08:40:18 -0700 Subject: [PATCH 01/69] [DEVOPS-198] Run cleanup script on Mac M1 runners when job starts (#657) - Build the Mac arm64 wheels using the Github runners instead of the self-hosted Mac M1 runners. See comment in build-wheels.yml for an explanation --- .github/actions/run-ee-server/action.yml | 1 - .github/workflows/build-wheels.yml | 152 ++++++++++++++--------- 2 files changed, 91 insertions(+), 62 deletions(-) diff --git a/.github/actions/run-ee-server/action.yml b/.github/actions/run-ee-server/action.yml index 5f73224e2..5bc57ff06 100644 --- a/.github/actions/run-ee-server/action.yml +++ b/.github/actions/run-ee-server/action.yml @@ -22,7 +22,6 @@ runs: steps: - name: Install crudini to manipulate config.conf # This will only work on the Github hosted runners. - # TODO: mac m1 self hosted runners do not have pipx installed by default run: pipx install crudini --pip-args "-c ${{ github.workspace }}/.github/workflows/requirements.txt" working-directory: .github/workflows shell: bash diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 7697546d6..febcb03d8 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -290,7 +290,79 @@ jobs: sha: ${{ github.sha }} context: "Build wheels (${{ matrix.python }}-macosx_x86_64)" + # TODO: this needs refactoring. But for now, just reuse code from the macos x86 job + # There's already an open PR to refactor all this code for building wheels and the sdist macOS-m1: + strategy: + fail-fast: false + matrix: + python: [ + "cp38", + "cp39", + "cp310", + "cp311", + "cp312" + ] + runs-on: macos-14 + steps: + - name: Show job status for commit + uses: myrotvorets/set-commit-status-action@v2.0.0 + if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} + with: + sha: ${{ github.sha }} + context: "Build wheels (${{ matrix.python }}-macosx_arm64)" + + - uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{ inputs.ref }} + fetch-depth: 0 + + - name: Set unoptimize flag + if: ${{ inputs.apply-no-optimizations }} + run: echo "UNOPTIMIZED=1" >> $GITHUB_ENV + + - name: Set include dsym flag + if: ${{ inputs.include-debug-info-for-macos }} + run: echo "INCLUDE_DSYM=1" >> $GITHUB_ENV + + # Install native version of Python 3.8 on arm64 + # The default Python 3.8 install used by cibuildwheel is for x86_64 + # https://cibuildwheel.pypa.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64 + - uses: actions/setup-python@v5 + with: + python-version: 3.8 + if: runner.os == 'macOS' && runner.arch == 'ARM64' && matrix.python == 'cp38' + + - name: Build wheel + uses: pypa/cibuildwheel@v2.19.2 + env: + CIBW_BUILD: ${{ matrix.python }}-macosx_arm64 + CIBW_BUILD_FRONTEND: build + CIBW_ENVIRONMENT: SSL_LIB_PATH="$(brew --prefix openssl@1.1)/lib/" CPATH="$(brew --prefix openssl@1.1)/include/" STATIC_SSL=1 + CIBW_ARCHS: "arm64" + + - name: Save macOS wheel + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: ${{ matrix.python }}-macosx_arm64.build + path: wheelhouse/*.whl + + - name: Set final commit status + uses: myrotvorets/set-commit-status-action@v2.0.0 + if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} + with: + status: ${{ job.status }} + sha: ${{ github.sha }} + context: "Build wheels (${{ matrix.python }}-macosx_arm64)" + + # If we build the wheels on the self hosted M1 runners using setup-python, it creates universal wheels + # This did not happen when we used the Python choco installs + # We only want to produce arm64 specific wheels for macOS + macOS-m1-self-hosted: + if: ${{ inputs.run_tests }} + needs: macOS-m1 runs-on: [ self-hosted, macOS, @@ -312,7 +384,7 @@ jobs: if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} with: sha: ${{ github.sha }} - context: "Build wheels (${{ matrix.python-version[0] }}-macosx_arm64)" + context: "Test self-hosted (${{ matrix.python-version[0] }}-macosx_arm64)" - uses: actions/checkout@v4 with: @@ -320,87 +392,44 @@ jobs: ref: ${{ inputs.ref }} fetch-depth: 0 - # Update dependencies if needed - - name: Add brew to path - run: echo PATH=$PATH:/opt/homebrew/bin/ >> $GITHUB_ENV - - - name: Install or upgrade Python - run: brew install python@${{ matrix.python-version[1] }} - - - name: Install or upgrade OpenSSL 1.1 - run: brew install openssl@1.1 + - name: Download wheel + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.python-version[0] }}-macosx_arm64.build - - name: Set environment variables for building - run: | - openssl_path=$(brew --prefix openssl@1.1) - echo SSL_LIB_PATH="$openssl_path/lib/" >> $GITHUB_ENV - echo CPATH="$openssl_path/include/" >> $GITHUB_ENV - echo STATIC_SSL=1 >> $GITHUB_ENV + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version[1] }} - - name: Install pip build packages - run: python${{ matrix.python-version[1] }} -m pip install --break-system-packages --force-reinstall -r requirements.txt + - name: Install wheel + run: python3 -m pip install aerospike --force-reinstall --no-index --find-links=./ - # Self-hosted runner only - # Need to be able to save Docker Hub credentials to keychain - - run: security unlock-keychain -p ${{ secrets.MAC_M1_SELF_HOSTED_RUNNER_PW }} - if: ${{ inputs.run_tests && inputs.use-server-rc }} + # The mac m1 self hosted runners use Docker Desktop which uses Keychain by default to store `docker login` credentials + # It's also required for pulling certain public images for some reason, like moby buildx + # There's currently no way to configure a different credential store (AFAIK), so we just need to make sure Keychain is unlocked + - name: Allow saving Docker Hub credentials using keychain and pulling moby buildx Docker image + run: security unlock-keychain -p ${{ secrets.MAC_M1_SELF_HOSTED_RUNNER_PW }} - uses: ./.github/actions/run-ee-server - if: ${{ inputs.run_tests }} with: use-server-rc: ${{ inputs.use-server-rc }} server-tag: ${{ inputs.server-tag }} docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} - - name: Set unoptimize flag - if: ${{ inputs.apply-no-optimizations }} - run: echo "UNOPTIMIZED=1" >> $GITHUB_ENV - - - name: Set include dsym flag - if: ${{ inputs.include-debug-info-for-macos }} - run: echo "INCLUDE_DSYM=1" >> $GITHUB_ENV - - - run: python${{ matrix.python-version[1] }} -m build - - - name: Install delocate - run: python${{ matrix.python-version[1] }} -m pip install --break-system-packages --force-reinstall delocate -c ./requirements.txt - working-directory: .github/workflows - - - run: delocate-wheel --require-archs "arm64" -w wheelhouse/ -v dist/*.whl - - run: python${{ matrix.python-version[1] }} -m pip install --break-system-packages --find-links=wheelhouse/ --no-index --force-reinstall aerospike - - - run: python${{ matrix.python-version[1] }} -m pip install --break-system-packages --force-reinstall -r requirements.txt - if: ${{ inputs.run_tests }} + - run: python3 -m pip install -r requirements.txt working-directory: test - - run: python${{ matrix.python-version[1] }} -m pytest new_tests/ - if: ${{ inputs.run_tests }} + - run: python3 -m pytest new_tests/ working-directory: test - - run: python${{ matrix.python-version[1] }} -c "import aerospike" - if: ${{ !inputs.run_tests }} - - - name: Save macOS wheel - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: ${{ matrix.python-version[0] }}-macosx_arm64.build - path: wheelhouse/*.whl - - - name: Stop server - if: ${{ always() && inputs.run_tests }} - run: | - docker container stop aerospike - docker container prune -f - - name: Set final commit status uses: myrotvorets/set-commit-status-action@v2.0.0 if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} with: sha: ${{ github.sha }} status: ${{ job.status }} - context: "Build wheels (${{ matrix.python-version[0] }}-macosx_arm64)" + context: "Test self-hosted (${{ matrix.python-version[0] }}-macosx_arm64)" windows-build: strategy: @@ -497,6 +526,7 @@ jobs: - run: python3 -m pip install pytest -c requirements.txt working-directory: test + # TODO: bug: inputs are not passed in. Just fix this in the refactoring PR - uses: ./.github/actions/run-ee-server - name: Connect to Docker container on remote machine with Docker daemon From b4b68380b1fd15b922c91e6663478136aea670d2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:40:59 +0000 Subject: [PATCH 02/69] Auto-bump version to 15.0.1rc4.dev11 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 5461fdb89..e4fd5bb09 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.0.1rc4.dev10 +15.0.1rc4.dev11 From 09f2eb46915bce8e0b8d23de9afb704a2fe5d48d Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 22 Aug 2024 09:10:46 -0700 Subject: [PATCH 03/69] [DEVOPS-256] Clean up after each job in Windows self-hosted runners (#660) - Fix bug where Windows doesn't test the right server version when specifying to use a server RC or server tag --- .github/workflows/build-wheels.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index febcb03d8..6c22731ba 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -521,13 +521,18 @@ jobs: python-version: ${{ matrix.python[1] }} - name: Install wheel - run: python3 -m pip install aerospike --force-reinstall --no-index --find-links=./ + run: python3 -m pip install aerospike --no-index --find-links=./ - run: python3 -m pip install pytest -c requirements.txt working-directory: test # TODO: bug: inputs are not passed in. Just fix this in the refactoring PR - uses: ./.github/actions/run-ee-server + with: + use-server-rc: ${{ inputs.use-server-rc }} + server-tag: ${{ inputs.server-tag }} + docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} + docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} - name: Connect to Docker container on remote machine with Docker daemon # DOCKER_HOST contains the IP address of the remote machine @@ -539,12 +544,6 @@ jobs: - run: python3 -m pytest new_tests/ working-directory: test - - name: Cleanup - if: ${{ always() }} - run: | - docker stop aerospike - docker container rm aerospike - - name: Show job status for commit if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} uses: myrotvorets/set-commit-status-action@v2.0.0 From 27d9a524df9ee388b72587dd4d0501db593a3901 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 22 Aug 2024 16:13:07 +0000 Subject: [PATCH 04/69] Auto-bump version to 15.0.1rc4.dev12 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e4fd5bb09..980f0cc91 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.0.1rc4.dev11 +15.0.1rc4.dev12 From 329e94ea80e5256462c7ecfc886d45d358a7eac5 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:16:38 -0700 Subject: [PATCH 05/69] [CLIENT-3057] Add missing aerospike_helpers.expressions.resources.ResultType constant NIL (#658) --- aerospike_helpers/expressions/resources.py | 2 +- test/new_tests/test_expressions_map.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/aerospike_helpers/expressions/resources.py b/aerospike_helpers/expressions/resources.py index d54d62787..a004de30c 100644 --- a/aerospike_helpers/expressions/resources.py +++ b/aerospike_helpers/expressions/resources.py @@ -116,7 +116,7 @@ class ResultType: """ Flags used to indicate expression value_type. """ - + NIL = 0 BOOLEAN = 1 INTEGER = 2 STRING = 3 diff --git a/test/new_tests/test_expressions_map.py b/test/new_tests/test_expressions_map.py index 31b5c43fa..6356957e7 100644 --- a/test/new_tests/test_expressions_map.py +++ b/test/new_tests/test_expressions_map.py @@ -163,7 +163,7 @@ class TestExpressions(TestBaseClass): def setup(self, request, as_connection): self.test_ns = "test" self.test_set = "demo" - + self.first_key = (self.test_ns, self.test_set, 0) for i in range(_NUM_RECORDS): key = ("test", "demo", i) rec = { @@ -828,7 +828,15 @@ def test_map_expr_inverted(self, bin_name: str, expr, expected): ops = [ expr_ops.expression_read(bin_name, expr.compile()) ] - key = (self.test_ns, self.test_set, 0) - _, _, bins = self.as_connection.operate(key, ops) + _, _, bins = self.as_connection.operate(self.first_key, ops) assert bins[bin_name] == expected + + def test_map_get_nil_value_type(self): + bin_name = "nmap_bin" + exp = MapGetByKey(None, aerospike.MAP_RETURN_VALUE, ResultType.NIL, 2, bin_name).compile() + ops = [ + expr_ops.expression_read(bin_name, exp) + ] + _, _, bins = self.as_connection.operate(self.first_key, ops) + assert bins[bin_name] is None From f4d69053fd350e3b551ea26d7062dc618ff26b84 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 21:17:16 +0000 Subject: [PATCH 06/69] Auto-bump version to 15.0.1rc4.dev13 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 980f0cc91..4cf4730ea 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.0.1rc4.dev12 +15.0.1rc4.dev13 From a55129caebfb62107dbe8146c942fe9488e7dc23 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:26:11 -0700 Subject: [PATCH 07/69] [CLIENT-2897] Show class name when printing a aerospike_helpers.HyperLogLog class instance (#659) --- aerospike_helpers/__init__.py | 10 ++++++++++ test/new_tests/test_hll.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/aerospike_helpers/__init__.py b/aerospike_helpers/__init__.py index a2e4c962e..bad706b07 100644 --- a/aerospike_helpers/__init__.py +++ b/aerospike_helpers/__init__.py @@ -25,3 +25,13 @@ class HyperLogLog(bytes): """ def __new__(cls, o) -> "HyperLogLog": return super().__new__(cls, o) + + # We need to implement repr() and str() ourselves + # Otherwise, this class will inherit these methods from bytes + # making it indistinguishable from bytes objects when printed + def __repr__(self) -> str: + bytes_str = super().__repr__() + return f"{self.__class__.__name__}({bytes_str})" + + def __str__(self) -> str: + return self.__repr__() diff --git a/test/new_tests/test_hll.py b/test/new_tests/test_hll.py index c2634886e..fa7528512 100644 --- a/test/new_tests/test_hll.py +++ b/test/new_tests/test_hll.py @@ -489,3 +489,19 @@ def test_put_get_hll_list(self): def test_hll_superclass(self): assert issubclass(HyperLogLog, bytes) + + def test_hll_str_repr(self): + bytes_obj = b'asdf' + hll = HyperLogLog(bytes_obj) + + expected_repr = f"{hll.__class__.__name__}({bytes_obj.__repr__()})" + assert str(hll) == expected_repr + assert repr(hll) == expected_repr + + hll_from_eval = eval(expected_repr) + # We compare HLL instances by comparing their bytes values + assert hll == hll_from_eval + + # Negative test for comparing HLL values + different_hll = HyperLogLog(b'asdff') + assert different_hll != hll_from_eval From e88629375d067ebe9cefc52eb116a4a88542cacb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 21:26:55 +0000 Subject: [PATCH 08/69] Auto-bump version to 15.0.1rc4.dev14 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4cf4730ea..de62022d2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.0.1rc4.dev13 +15.0.1rc4.dev14 From 1ffc37d5700b37758688d0612c468e5ea22c246f Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 29 Aug 2024 10:09:16 -0700 Subject: [PATCH 09/69] [CLIENT-2544] query.apply(): remove policy parameter (#514) --- src/main/query/apply.c | 8 +++----- test/new_tests/test_aggregate.py | 7 +++++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/main/query/apply.c b/src/main/query/apply.c index aae7e8f5c..9218927ed 100644 --- a/src/main/query/apply.c +++ b/src/main/query/apply.c @@ -37,16 +37,14 @@ AerospikeQuery *AerospikeQuery_Apply(AerospikeQuery *self, PyObject *args, PyObject *py_module = NULL; PyObject *py_function = NULL; PyObject *py_args = NULL; - PyObject *py_policy = NULL; PyObject *py_umodule = NULL; PyObject *py_ufunction = NULL; // Python function keyword arguments - static char *kwlist[] = {"module", "function", "arguments", "policy", NULL}; + static char *kwlist[] = {"module", "function", "arguments", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO:apply", kwlist, - &py_module, &py_function, &py_args, - &py_policy)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O:apply", kwlist, + &py_module, &py_function, &py_args)) { return NULL; } diff --git a/test/new_tests/test_aggregate.py b/test/new_tests/test_aggregate.py index 7529e089a..6109c0869 100644 --- a/test/new_tests/test_aggregate.py +++ b/test/new_tests/test_aggregate.py @@ -387,3 +387,10 @@ def user_callback(value): except e.ParamError as exception: assert exception.code == -2 + + # We can't use the inspect library to check the keyword args of a method defined using the C-API + # It doesn't work, so just check that passing in an invalid arg fails + def test_signature_invalid_arg(self): + query: aerospike.Query = self.as_connection.query("test", "demo") + with pytest.raises(TypeError): + query.apply("stream_example", "count", policy=None) From ee94c9ef6ff84a3e2efd83f79cde2a89c915e2be Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 17:09:50 +0000 Subject: [PATCH 10/69] Auto-bump version to 15.0.1rc4.dev15 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index de62022d2..614656fed 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.0.1rc4.dev14 +15.0.1rc4.dev15 From a598fb260e0993a6c3d9e167d2cddf97daa46ab5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 23:17:57 +0000 Subject: [PATCH 11/69] Auto-bump version to 15.0.1rc4 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 614656fed..88d95bb3f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.0.1rc4.dev15 +15.0.1rc4 From 6ab14fcd7ccc8df0f53154bafe174f78b024f40b Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 30 Aug 2024 09:38:26 -0700 Subject: [PATCH 12/69] [CLIENT-1837] CI/CD: Refactor build wheels workflow (#622) - Change build-wheels.yml to only build for one platform - Create another workflow to build all wheels and source distribution (main purpose is to build for CI/CD stages) Extra changes: - Create composite action to setup Docker on macOS x86 - Fixes a bug where build-wheels.yml tests against a server RC by default instead of server release for workflow_call events --- .../actions/setup-docker-on-macos/action.yml | 17 + .../workflows/build-and-run-stage-tests.yml | 24 +- .../build-and-upload-wheels-for-qe.yml | 13 +- .github/workflows/build-artifacts.yml | 110 +++ .github/workflows/build-sdist.yml | 60 ++ .github/workflows/build-wheels.yml | 650 ++++++------------ .../bump-stage-and-upload-to-jfrog.yml | 5 +- .github/workflows/bump-version.yml | 1 + .github/workflows/dev-workflow-p1.yml | 3 +- .github/workflows/dev-workflow-p2.yml | 4 +- .github/workflows/requirements.txt | 1 + .github/workflows/stage-tests.yml | 9 +- .github/workflows/stage-to-master.yml | 6 +- .github/workflows/test-server-rc.yml | 9 +- .github/workflows/update-version.yml | 1 + .github/workflows/valgrind.yml | 9 +- 16 files changed, 467 insertions(+), 455 deletions(-) create mode 100644 .github/actions/setup-docker-on-macos/action.yml create mode 100644 .github/workflows/build-artifacts.yml create mode 100644 .github/workflows/build-sdist.yml diff --git a/.github/actions/setup-docker-on-macos/action.yml b/.github/actions/setup-docker-on-macos/action.yml new file mode 100644 index 000000000..aa8e1da0b --- /dev/null +++ b/.github/actions/setup-docker-on-macos/action.yml @@ -0,0 +1,17 @@ +name: 'Install Docker on macOS runner' +description: 'Install Docker using colima' + +runs: + using: "composite" + steps: + - name: Install Docker Engine + run: brew install colima + shell: bash + + - name: Install Docker client + run: brew install docker + shell: bash + + - name: Start Docker Engine + run: colima start + shell: bash diff --git a/.github/workflows/build-and-run-stage-tests.yml b/.github/workflows/build-and-run-stage-tests.yml index 7b9a2bd6e..8f1ce4467 100644 --- a/.github/workflows/build-and-run-stage-tests.yml +++ b/.github/workflows/build-and-run-stage-tests.yml @@ -21,12 +21,32 @@ on: description: 'Test macOS x86 wheels (unstable)' jobs: - build-wheels: + build-select-wheels: + strategy: + matrix: + platform-tag: [ + "manylinux_x86_64", + "manylinux_aarch64", + "macosx_x86_64" + ] + # Need all the artifacts to run all the stage tests, so fail fast uses: ./.github/workflows/build-wheels.yml + with: + platform-tag: ${{ matrix.platform-tag }} + sha-to-build-and-test: ${{ github.sha }} + secrets: inherit + + build-sdist: + uses: ./.github/workflows/build-sdist.yml + with: + sha_to_build: ${{ github.sha }} run-stage-tests: uses: ./.github/workflows/stage-tests.yml - needs: build-wheels + needs: [ + build-select-wheels, + build-sdist + ] secrets: inherit with: use_jfrog_builds: false diff --git a/.github/workflows/build-and-upload-wheels-for-qe.yml b/.github/workflows/build-and-upload-wheels-for-qe.yml index 63fd09843..86de85074 100644 --- a/.github/workflows/build-and-upload-wheels-for-qe.yml +++ b/.github/workflows/build-and-upload-wheels-for-qe.yml @@ -17,11 +17,18 @@ on: jobs: build-artifacts: + strategy: + matrix: + platform-tag: [ + "manylinux_x86_64", + "manylinux_aarch64" + ] uses: ./.github/workflows/build-wheels.yml with: - # In a push event, any input values default to '' - # https://github.com/orgs/community/discussions/29242#discussioncomment-5063461 - apply-no-optimizations: ${{ github.event_name == 'workflow_dispatch' && inputs.disable-optimizations || false }} + platform-tag: ${{ matrix.platform-tag }} + unoptimized: ${{ github.event_name == 'workflow_dispatch' && inputs.disable-optimizations || false }} + sha-to-build-and-test: ${{ github.sha }} + secrets: inherit upload-to-jfrog: needs: build-artifacts diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 000000000..88fc4f159 --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,110 @@ +name: Build artifacts +run-name: Build artifacts (run_tests=${{ inputs.run_tests }}, use-server-rc=${{ inputs.use-server-rc }}, server-tag=${{ inputs.server-tag }}) + +# Builds manylinux wheels and source distribution +# Optionally run tests on manylinux wheels +# Then upload artifacts to Github + +on: + workflow_dispatch: + inputs: + # There may be a case where we want to test these build debug flags on all platforms that support them + unoptimized: + description: 'macOS or Linux: Apply -O0 flag?' + type: boolean + required: false + default: false + include-debug-info-for-macos: + description: 'macOS: Build wheels for debugging?' + type: boolean + required: false + default: false + run_tests: + description: "Run integration tests with the wheels after building them" + required: true + type: boolean + default: false + use-server-rc: + type: boolean + required: true + default: false + description: 'Test against server release candidate? (e.g to test new server features)' + server-tag: + type: string + required: true + default: 'latest' + description: 'Server docker image tag (e.g to test a client backport version)' + + workflow_call: + inputs: + # The "dev" tests test the artifacts against a server + # The dev-to-stage and stage-to-master workflow only need to build the artifacts, not test them + run_tests: + required: false + type: boolean + default: false + # workflow_call hack + is_workflow_call: + type: boolean + default: true + required: false + # This input is only used in workflow_call events + sha-to-build-and-test: + description: A calling workflow may want to run this workflow on a different ref than the calling workflow's ref + type: string + # Make it required to make things simple + required: true + # A calling workflow doesn't actually set values to the inputs below + # But that workflow needs to have default values for these inputs + unoptimized: + type: boolean + required: false + default: false + include-debug-info-for-macos: + type: boolean + required: false + default: false + use-server-rc: + required: false + default: false + type: boolean + server-tag: + type: string + required: false + default: 'latest' + secrets: + DOCKER_HUB_BOT_USERNAME: + required: true + DOCKER_HUB_BOT_PW: + required: true + MAC_M1_SELF_HOSTED_RUNNER_PW: + required: true + +jobs: + build-sdist: + uses: ./.github/workflows/build-sdist.yml + with: + sha_to_build: ${{ inputs.is_workflow_call == true && inputs.sha-to-build-and-test || github.sha }} + + build-wheels: + strategy: + matrix: + platform-tag: [ + "manylinux_x86_64", + "manylinux_aarch64", + "macosx_x86_64", + "macosx_arm64", + "win_amd64" + ] + fail-fast: false + uses: ./.github/workflows/build-wheels.yml + with: + platform-tag: ${{ matrix.platform-tag }} + # Can't use env context here, so just copy from build-sdist env var + sha-to-build-and-test: ${{ inputs.is_workflow_call == true && inputs.sha-to-build-and-test || github.sha }} + unoptimized: ${{ inputs.unoptimized }} + include-debug-info-for-macos: ${{ inputs.include-debug-info-for-macos }} + run_tests: ${{ inputs.run_tests }} + use-server-rc: ${{ inputs.use-server-rc }} + server-tag: ${{ inputs.server-tag }} + secrets: inherit diff --git a/.github/workflows/build-sdist.yml b/.github/workflows/build-sdist.yml new file mode 100644 index 000000000..fb6926701 --- /dev/null +++ b/.github/workflows/build-sdist.yml @@ -0,0 +1,60 @@ +on: + workflow_dispatch: + workflow_call: + inputs: + is_workflow_call: + type: boolean + default: true + required: false + sha_to_build: + type: string + required: true + +env: + STATUS_CHECK_MESSAGE: "Build source distribution" + COMMIT_SHA_TO_BUILD: ${{ inputs.is_workflow_call == true && inputs.sha_to_build || github.sha }} + +jobs: + build-sdist: + name: Build source distribution + runs-on: ubuntu-22.04 + steps: + - name: Show job status for commit + # Commit status will already be shown by the calling workflow for push and pull request events, but not + # for any other event like workflow_dispatch. so we have to do it manually + # If workflow_call triggered this job, github.event_name will inherit the event of the calling workflow + # The calling workflow can be triggered by push or pull request events, so there's that + # https://github.com/actions/runner/issues/3146#issuecomment-2000017097 + if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} + uses: myrotvorets/set-commit-status-action@v2.0.0 + with: + sha: ${{ env.COMMIT_SHA_TO_BUILD }} + context: ${{ env.STATUS_CHECK_MESSAGE }} + + - uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{ env.COMMIT_SHA_TO_BUILD }} + fetch-depth: 0 + + - name: Install build dependencies (pip packages) + run: python3 -m pip install -r requirements.txt + + - name: Build source distribution + run: python3 -m build --sdist + + - name: Upload source distribution to GitHub + uses: actions/upload-artifact@v4 + with: + path: ./dist/*.tar.gz + name: sdist.build + + - name: Set final commit status + uses: myrotvorets/set-commit-status-action@v2.0.0 + # Always run even if job failed or is cancelled + # But we don't want to show anything if the calling workflow was triggered by these events + if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} + with: + sha: ${{ env.COMMIT_SHA_TO_BUILD }} + status: ${{ job.status }} + context: ${{ env.STATUS_CHECK_MESSAGE }} diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 6c22731ba..be0f8f342 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -1,553 +1,353 @@ -name: Build wheels -run-name: Build wheels (run_tests=${{ inputs.run_tests }}, use-server-rc=${{ inputs.use-server-rc }}, server-tag=${{ inputs.server-tag }}, test-macos-x86=${{ inputs.test-macos-x86 }}, apply-no-optimizations=${{ inputs.apply-no-optimizations }}, include-debug-info-for-macos=${{ inputs.include-debug-info-for-macos }}) +name: 'Build wheels' +run-name: 'Build wheels (python-tags=${{ inputs.python-tags }}, platform-tag=${{ inputs.platform-tag }}, unoptimized=${{ inputs.unoptimized }}, include-debug-info-for-macos=${{ inputs.include-debug-info-for-macos }}, run_tests=${{ inputs.run_tests }}, use-server-rc=${{ inputs.use-server-rc }}, server-tag=${{ inputs.server-tag }})' -# Builds manylinux wheels and source distribution -# If running tests, publish results in commit status -# For each build, upload to Github as an artifact if it passes testing or does not need to run tests +# Build wheels on all (or select) Python versions supported by the Python client for a specific platform on: workflow_dispatch: inputs: - # If we only want to check that the builds pass on an arbitrary branch - run_tests: - description: "Run integration tests" + # These are the usual cases for building wheels: + # + # 1. One wheel for *one* supported Python version. This is for running specialized tests that only need one Python version + # like valgrind or a failing QE test. And usually, we only need one wheel for debugging purposes. + # 2. Wheels for *all* supported Python versions for *one* supported platform. This is useful for testing workflow changes for a + # single OS or CPU architecture (e.g testing that changes to debugging modes work on all Python versions) + # 3. Wheels for *all* supported Python versions and *all* supported platforms. This is for building wheels for different + # CI/CD stages (e.g dev, stage, or master). We can also test debugging modes for all platforms that support them + # + # We're able to combine case 1 and 2 into one workflow by creating an input that takes in a JSON list of strings (Python tags) + # to build wheels for. Actual list inputs aren't supported yet, so it's actually a JSON list encoded as a string. + # + # However, it's harder to combine this workflow (case 1 + 2) with case 3, because matrix outputs don't exist yet + # in Github Actions. So all jobs in the cibuildwheel job would have to pass for a self hosted job to run. + # We want each platform to be tested independently of each other, + # so there is a wrapper workflow that has a list of platforms to test and reuses this workflow for each platform. + # If one platform fails, it will not affect the building and testing of another platform (we disable fail fast mode) + python-tags: + type: string + description: Valid JSON list of Python tags to build the client for + required: false + default: '["cp38", "cp39", "cp310", "cp311", "cp312"]' + platform-tag: + description: Platform to build the client for. + type: choice required: true + options: + - manylinux_x86_64 + - manylinux_aarch64 + - macosx_x86_64 + - macosx_arm64 + - win_amd64 + # Makes debugging via gh cli easier. + default: manylinux_x86_64 + unoptimized: + description: 'macOS or Linux: Apply -O0 flag?' + # Windows supports a different flag to disable optimizations, but we haven't added support for it yet + type: boolean + required: false + default: false + include-debug-info-for-macos: + description: 'macOS: Build wheels for debugging?' type: boolean + required: false + default: false + run_tests: + description: 'Run Aerospike server and run tests using built wheels?' + type: boolean + required: false default: false use-server-rc: type: boolean required: true default: false description: 'Test against server release candidate?' - # If we are creating a backport and want to test an arbitrary branch against an older server version server-tag: required: true default: 'latest' description: 'Server docker image tag' - test-macos-x86: - required: true + + workflow_call: + inputs: + # See workflow call hack in update-version.yml + is_workflow_call: type: boolean - default: false - description: 'Test macOS x86 wheels (unstable)' - apply-no-optimizations: + default: true + required: false + python-tags: + type: string + required: false + default: '["cp38", "cp39", "cp310", "cp311", "cp312"]' + platform-tag: + type: string required: true + # Only used in workflow_call event + sha-to-build-and-test: + type: string + required: true + unoptimized: type: boolean + required: false default: false - description: 'Linux and macOS: apply -O0 when building C and Python client?' include-debug-info-for-macos: - required: true type: boolean + required: false default: false - description: 'macOS: include source files and line numbers for both C and Python client for debugging?' - - workflow_call: - inputs: - # The "dev" tests test the artifacts against a server release - # The "stage" tests and release workflow only need to build the artifacts, not test them run_tests: - description: "Run integration tests" - required: false type: boolean - default: false - ref: - type: string required: false - # Calling workflow doesn't actually use the options below - # But we need to set default values for workflow calls to use + default: false use-server-rc: required: false - default: true type: boolean + default: false + description: 'Test against server release candidate?' server-tag: - type: string required: false + type: string default: 'latest' - test-macos-x86: - required: false - type: boolean - default: false - apply-no-optimizations: - required: false - type: boolean - default: false - include-debug-info-for-macos: - required: false - type: boolean - default: false + description: 'Server docker image tag' secrets: + # Just make all the secrets required to make things simpler... DOCKER_HUB_BOT_USERNAME: - required: false + required: true DOCKER_HUB_BOT_PW: - required: false + required: true MAC_M1_SELF_HOSTED_RUNNER_PW: - required: false + required: true + +env: + COMMIT_SHA_TO_BUILD_AND_TEST: ${{ inputs.is_workflow_call == true && inputs.sha-to-build-and-test || github.sha }} + # Note that environment variables in Github are all strings + # Github mac m1 and windows runners don't support Docker / nested virtualization + # so we need to use self-hosted runners to test wheels for these platforms + RUN_INTEGRATION_TESTS_IN_CIBW: ${{ inputs.run_tests && (startsWith(inputs.platform-tag, 'manylinux') || inputs.platform-tag == 'macosx_x86_64') }} jobs: - build-sdist: - name: Build source distribution + # Maps don't exist in Github Actions, so we have to store the map using a script and fetch it in a job + # This uses up more billing minutes (rounded up to 1 minute for each job run), + # but this should be ok based on the minutes usage data for the aerospike organization + get-runner-os: + outputs: + runner-os: ${{ steps.get-runner-os.outputs.runner_os }} runs-on: ubuntu-22.04 steps: - - name: Show job status for commit - # If workflow_call triggered this job, github.event_name will inherit the event of the calling workflow - # https://github.com/actions/runner/issues/3146#issuecomment-2000017097 - # Commit status will already be shown by the calling workflow for these events - if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} - uses: myrotvorets/set-commit-status-action@v2.0.0 - with: - sha: ${{ github.sha }} - context: "Build wheels (sdist)" - - - uses: actions/checkout@v4 - with: - submodules: recursive - ref: ${{ inputs.ref }} - fetch-depth: 0 - - - name: Install build dependencies (pip packages) - run: python3 -m pip install -r requirements.txt - - - name: Build source distribution - run: python3 -m build --sdist - - - name: Upload source distribution to GitHub - uses: actions/upload-artifact@v4 - with: - path: ./dist/*.tar.gz - name: sdist.build - - - name: Set final commit status - uses: myrotvorets/set-commit-status-action@v2.0.0 - # Always run even if job failed or is cancelled - # But we don't want to show anything if the calling workflow was triggered by these events - if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} - with: - sha: ${{ github.sha }} - status: ${{ job.status }} - context: "Build wheels (sdist)" - - manylinux: + - id: get-runner-os + # Single source of truth for which runner OS to use for each platform tag + run: | + declare -A hashmap + hashmap[manylinux_x86_64]="ubuntu-22.04" + hashmap[manylinux_aarch64]="aerospike_arm_runners_2" + hashmap[macosx_x86_64]="macos-12-large" + hashmap[macosx_arm64]="macos-14" + hashmap[win_amd64]="windows-2022" + echo runner_os=${hashmap[${{ inputs.platform-tag }}]} >> $GITHUB_OUTPUT + # Bash >= 4 supports hashmaps + shell: bash + + cibuildwheel: + needs: get-runner-os strategy: - fail-fast: false matrix: - # Python versions to build wheels on - python: [ - "cp38", - "cp39", - "cp310", - "cp311", - "cp312" - ] - platform: [ - ["x86_64", "ubuntu-22.04"], - ["aarch64", "aerospike_arm_runners_2"] - ] - runs-on: ${{ matrix.platform[1] }} + python-tag: ${{ fromJSON(inputs.python-tags) }} + fail-fast: false + runs-on: ${{ needs.get-runner-os.outputs.runner-os }} + env: + BUILD_IDENTIFIER: "${{ matrix.python-tag }}-${{ inputs.platform-tag }}" steps: + - name: Create status check message + run: echo STATUS_CHECK_MESSAGE="cibuildwheel (${{ env.BUILD_IDENTIFIER }})" >> $GITHUB_ENV + shell: bash + - name: Show job status for commit uses: myrotvorets/set-commit-status-action@v2.0.0 if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} with: - sha: ${{ github.sha }} - context: "Build wheels (${{ matrix.python }}-manylinux_${{ matrix.platform[0] }})" + sha: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} + context: ${{ env.STATUS_CHECK_MESSAGE }} - uses: actions/checkout@v4 with: submodules: recursive - ref: ${{ inputs.ref }} + ref: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} + # We need the last tag before the ref, so we can relabel the version if needed fetch-depth: 0 - - uses: ./.github/actions/run-ee-server-for-ext-container - if: ${{ inputs.run_tests }} + - name: 'macOS arm64: Install experimental Python 3.8 macOS arm64 build' + # By default, cibuildwheel installs and uses Python 3.8 x86_64 to cross compile macOS arm64 wheels + # There is a bug that builds macOS x86_64 wheels instead, so we install this Python 3.8 native ARM build to ensure + # the wheel is compiled for macOS arm64 + # https://cibuildwheel.pypa.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64 + if: ${{ matrix.python-tag == 'cp38' && inputs.platform-tag == 'macosx_arm64' }} + run: | + curl -o /tmp/Python38.pkg https://www.python.org/ftp/python/3.8.10/python-3.8.10-macos11.pkg + sudo installer -pkg /tmp/Python38.pkg -target / + sh "/Applications/Python 3.8/Install Certificates.command" + + - name: 'Windows: Add msbuild to PATH' + if: ${{ inputs.platform-tag == 'win_amd64' }} + uses: microsoft/setup-msbuild@v1.1 + + - name: 'Windows: Install C client deps' + if: ${{ inputs.platform-tag == 'win_amd64' }} + run: nuget restore + working-directory: aerospike-client-c/vs + + - name: 'macOS x86: Setup Docker using colima for testing' + if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' && inputs.platform-tag == 'macosx_x86_64' }} + uses: ./.github/actions/setup-docker-on-macos + + - name: 'macOS x86: run Aerospike server in Docker container and connect via localhost' + if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' && inputs.platform-tag == 'macosx_x86_64' }} + uses: ./.github/actions/run-ee-server with: use-server-rc: ${{ inputs.use-server-rc }} server-tag: ${{ inputs.server-tag }} docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} - - name: Enable tests - if: ${{ inputs.run_tests }} - run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest new_tests/" >> $GITHUB_ENV - - - name: Disable tests (only run basic import test) - if: ${{ !inputs.run_tests }} - run: echo "TEST_COMMAND=python -c 'import aerospike'" >> $GITHUB_ENV - - - name: Set unoptimize flag - if: ${{ inputs.apply-no-optimizations }} - run: echo "UNOPTIMIZED=1" >> $GITHUB_ENV - - - name: Build wheel - uses: pypa/cibuildwheel@v2.19.2 - env: - CIBW_ENVIRONMENT_PASS_LINUX: ${{ inputs.apply-no-optimizations && 'UNOPTIMIZED' || '' }} - CIBW_BUILD: ${{ matrix.python }}-manylinux_${{ matrix.platform[0] }} - CIBW_BUILD_FRONTEND: build - CIBW_BEFORE_ALL_LINUX: > - yum install openssl-devel -y && - yum install python-devel -y && - yum install python-setuptools -y - CIBW_ARCHS: "${{ matrix.platform[0] }}" - CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }} - - - name: Upload wheels to GitHub - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - path: ./wheelhouse/*.whl - name: ${{ matrix.python }}-manylinux_${{ matrix.platform[0] }}.build - - - name: Set final commit status - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} - with: - sha: ${{ github.sha }} - status: ${{ job.status }} - context: "Build wheels (${{ matrix.python }}-manylinux_${{ matrix.platform[0] }})" - - macOS-x86: - strategy: - fail-fast: false - matrix: - python: [ - "cp38", - "cp39", - "cp310", - "cp311", - "cp312" - ] - runs-on: macos-12-large - steps: - - name: Show job status for commit - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} - with: - sha: ${{ github.sha }} - context: "Build wheels (${{ matrix.python }}-macosx_x86_64)" - - - uses: actions/checkout@v4 - with: - submodules: recursive - ref: ${{ inputs.ref }} - fetch-depth: 0 - - - name: Install Docker Engine - if: ${{ inputs.run_tests }} - run: brew install colima - - - name: Install Docker client - if: ${{ inputs.run_tests }} - run: brew install docker - - - name: Start Docker Engine - if: ${{ inputs.run_tests }} - run: colima start - - - uses: ./.github/actions/run-ee-server - if: ${{ inputs.run_tests }} + # TODO: combine this composite action and the above into one + - name: "Linux: run Aerospike server in Docker container and configure config.conf to connect to the server container's Docker IP address" + if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' && startsWith(inputs.platform-tag, 'manylinux') }} + uses: ./.github/actions/run-ee-server-for-ext-container with: use-server-rc: ${{ inputs.use-server-rc }} server-tag: ${{ inputs.server-tag }} docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} - - name: Enable tests - if: ${{ inputs.run_tests && inputs.test-macos-x86 }} + - name: If not running tests against server, only run basic import test + if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'false' }} + # Use double quotes otherwise Windows will throw this error in cibuildwheel + # 'import + # ^ + # SyntaxError: EOL while scanning string literal + run: echo "TEST_COMMAND=python -c \"import aerospike\"" >> $GITHUB_ENV + shell: bash + + - name: Otherwise, enable integration tests + if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }} run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest new_tests/" >> $GITHUB_ENV - - - name: Disable tests (only run basic import test) - if: ${{ !inputs.run_tests || !inputs.test-macos-x86 }} - run: echo "TEST_COMMAND=python -c 'import aerospike'" >> $GITHUB_ENV + shell: bash - name: Set unoptimize flag - if: ${{ inputs.apply-no-optimizations }} + if: ${{ inputs.unoptimized && (startsWith(inputs.platform-tag, 'manylinux') || startsWith(inputs.platform-tag, 'macosx')) }} run: echo "UNOPTIMIZED=1" >> $GITHUB_ENV - name: Set include dsym flag - if: ${{ inputs.include-debug-info-for-macos }} + if: ${{ inputs.include-debug-info-for-macos && startsWith(inputs.platform-tag, 'macosx') }} run: echo "INCLUDE_DSYM=1" >> $GITHUB_ENV - name: Build wheel - uses: pypa/cibuildwheel@v2.19.2 + uses: pypa/cibuildwheel@v2.20.0 env: - CIBW_BUILD: ${{ matrix.python }}-macosx_x86_64 + CIBW_ENVIRONMENT_PASS_LINUX: ${{ inputs.unoptimized && 'UNOPTIMIZED' || '' }} + CIBW_ENVIRONMENT_MACOS: SSL_LIB_PATH="$(brew --prefix openssl@1.1)/lib/" CPATH="$(brew --prefix openssl@1.1)/include/" STATIC_SSL=1 + CIBW_BUILD: ${{ env.BUILD_IDENTIFIER }} CIBW_BUILD_FRONTEND: build - CIBW_ENVIRONMENT: SSL_LIB_PATH="$(brew --prefix openssl@1.1)/lib/" CPATH="$(brew --prefix openssl@1.1)/include/" STATIC_SSL=1 - CIBW_ARCHS: "x86_64" + CIBW_BEFORE_ALL_LINUX: > + yum install openssl-devel -y && + yum install python-devel -y && + yum install python-setuptools -y + # delvewheel is not enabled by default but we do need to repair the wheel + CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel" + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair --add-path ./aerospike-client-c/vs/x64/Release -w {dest_dir} {wheel}" CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }} - - name: Save macOS wheel + - name: Upload wheels to GitHub uses: actions/upload-artifact@v4 - if: ${{ always() }} + if: ${{ !cancelled() }} with: - name: ${{ matrix.python }}-macosx_x86_64.build - path: wheelhouse/*.whl + path: ./wheelhouse/*.whl + name: ${{ env.BUILD_IDENTIFIER }}.build - name: Set final commit status uses: myrotvorets/set-commit-status-action@v2.0.0 if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} with: + sha: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} status: ${{ job.status }} - sha: ${{ github.sha }} - context: "Build wheels (${{ matrix.python }}-macosx_x86_64)" + context: ${{ env.STATUS_CHECK_MESSAGE }} - # TODO: this needs refactoring. But for now, just reuse code from the macos x86 job - # There's already an open PR to refactor all this code for building wheels and the sdist - macOS-m1: + test-self-hosted: + needs: cibuildwheel + # There's a top-level env variable for this but we can't use it here, unfortunately + if: ${{ inputs.run_tests && (inputs.platform-tag == 'macosx_arm64' || inputs.platform-tag == 'win_amd64') }} strategy: fail-fast: false matrix: - python: [ - "cp38", - "cp39", - "cp310", - "cp311", - "cp312" - ] - runs-on: macos-14 - steps: - - name: Show job status for commit - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} - with: - sha: ${{ github.sha }} - context: "Build wheels (${{ matrix.python }}-macosx_arm64)" - - - uses: actions/checkout@v4 - with: - submodules: recursive - ref: ${{ inputs.ref }} - fetch-depth: 0 - - - name: Set unoptimize flag - if: ${{ inputs.apply-no-optimizations }} - run: echo "UNOPTIMIZED=1" >> $GITHUB_ENV - - - name: Set include dsym flag - if: ${{ inputs.include-debug-info-for-macos }} - run: echo "INCLUDE_DSYM=1" >> $GITHUB_ENV - - # Install native version of Python 3.8 on arm64 - # The default Python 3.8 install used by cibuildwheel is for x86_64 - # https://cibuildwheel.pypa.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64 - - uses: actions/setup-python@v5 - with: - python-version: 3.8 - if: runner.os == 'macOS' && runner.arch == 'ARM64' && matrix.python == 'cp38' - - - name: Build wheel - uses: pypa/cibuildwheel@v2.19.2 - env: - CIBW_BUILD: ${{ matrix.python }}-macosx_arm64 - CIBW_BUILD_FRONTEND: build - CIBW_ENVIRONMENT: SSL_LIB_PATH="$(brew --prefix openssl@1.1)/lib/" CPATH="$(brew --prefix openssl@1.1)/include/" STATIC_SSL=1 - CIBW_ARCHS: "arm64" - - - name: Save macOS wheel - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: ${{ matrix.python }}-macosx_arm64.build - path: wheelhouse/*.whl - - - name: Set final commit status - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} - with: - status: ${{ job.status }} - sha: ${{ github.sha }} - context: "Build wheels (${{ matrix.python }}-macosx_arm64)" - - # If we build the wheels on the self hosted M1 runners using setup-python, it creates universal wheels - # This did not happen when we used the Python choco installs - # We only want to produce arm64 specific wheels for macOS - macOS-m1-self-hosted: - if: ${{ inputs.run_tests }} - needs: macOS-m1 - runs-on: [ - self-hosted, - macOS, - ARM64 - ] - strategy: - matrix: - python-version: [ - ["cp38", "3.8"], - ["cp39", "3.9"], - ["cp310", "3.10"], - ["cp311", "3.11"], - ["cp312", "3.12"] - ] - fail-fast: false + python-tag: ${{ fromJSON(inputs.python-tags) }} + runs-on: ${{ inputs.platform-tag == 'macosx_arm64' && fromJSON('["self-hosted", "macOS", "ARM64"]') || fromJSON('["self-hosted", "Windows", "X64"]') }} + env: + BUILD_IDENTIFIER: "${{ matrix.python-tag }}-${{ inputs.platform-tag }}" steps: - - name: Show job status for commit - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} - with: - sha: ${{ github.sha }} - context: "Test self-hosted (${{ matrix.python-version[0] }}-macosx_arm64)" - - - uses: actions/checkout@v4 - with: - submodules: recursive - ref: ${{ inputs.ref }} - fetch-depth: 0 + - name: Create status check message + run: echo STATUS_CHECK_MESSAGE="Test on self hosted (${{ env.BUILD_IDENTIFIER }})" >> $GITHUB_ENV + shell: bash - - name: Download wheel - uses: actions/download-artifact@v4 - with: - name: ${{ matrix.python-version[0] }}-macosx_arm64.build - - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version[1] }} - - - name: Install wheel - run: python3 -m pip install aerospike --force-reinstall --no-index --find-links=./ - - # The mac m1 self hosted runners use Docker Desktop which uses Keychain by default to store `docker login` credentials - # It's also required for pulling certain public images for some reason, like moby buildx - # There's currently no way to configure a different credential store (AFAIK), so we just need to make sure Keychain is unlocked - - name: Allow saving Docker Hub credentials using keychain and pulling moby buildx Docker image - run: security unlock-keychain -p ${{ secrets.MAC_M1_SELF_HOSTED_RUNNER_PW }} - - - uses: ./.github/actions/run-ee-server - with: - use-server-rc: ${{ inputs.use-server-rc }} - server-tag: ${{ inputs.server-tag }} - docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} - docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} - - - run: python3 -m pip install -r requirements.txt - working-directory: test - - - run: python3 -m pytest new_tests/ - working-directory: test - - - name: Set final commit status - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} - with: - sha: ${{ github.sha }} - status: ${{ job.status }} - context: "Test self-hosted (${{ matrix.python-version[0] }}-macosx_arm64)" - - windows-build: - strategy: - fail-fast: false - matrix: - python: [ - ["cp38", "3.8"], - ["cp39", "3.9"], - ["cp310", "3.10"], - ["cp311", "3.11"], - ["cp312", "3.12"] - ] - runs-on: windows-2022 - steps: - name: Show job status for commit uses: myrotvorets/set-commit-status-action@v2.0.0 if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} with: - sha: ${{ github.sha }} - context: "Build wheels (${{ matrix.python[0] }}-win_amd64)" + sha: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} + context: ${{ env.STATUS_CHECK_MESSAGE }} - uses: actions/checkout@v4 with: - submodules: recursive - ref: ${{ inputs.ref }} - fetch-depth: 0 - - - name: Add msbuild to PATH - uses: microsoft/setup-msbuild@v1.1 - - - name: Install C client deps - run: nuget restore - working-directory: aerospike-client-c/vs - - - name: Build wheel - uses: pypa/cibuildwheel@v2.19.2 - env: - CIBW_BUILD: ${{ matrix.python[0] }}-win_amd64 - CIBW_BUILD_FRONTEND: build - CIBW_ARCHS: auto64 - CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel" - CIBW_REPAIR_WHEEL_COMMAND: "delvewheel repair --add-path ./aerospike-client-c/vs/x64/Release -w {dest_dir} {wheel}" - - - uses: actions/upload-artifact@v4 - with: - path: ./wheelhouse/*.whl - name: ${{ matrix.python[0] }}-win_amd64.build + ref: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} - - name: Set final commit status - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} - with: - sha: ${{ github.sha }} - status: ${{ job.status }} - context: "Build wheels (${{ matrix.python[0] }}-win_amd64)" + # Need to be able to save Docker Hub credentials to keychain + - if: ${{ inputs.platform-tag == 'macosx_arm64' && inputs.use-server-rc }} + run: security unlock-keychain -p ${{ secrets.MAC_M1_SELF_HOSTED_RUNNER_PW }} - test-windows: - needs: windows-build - if: ${{ inputs.run_tests }} - strategy: - fail-fast: false - matrix: - python: [ - ["cp38", "3.8"], - ["cp39", "3.9"], - ["cp310", "3.10"], - ["cp311", "3.11"], - ["cp312", "3.12"] - ] - runs-on: [self-hosted, Windows, X64] - steps: - - name: Show job status for commit - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} - with: - sha: ${{ github.sha }} - context: "Test Windows (${{ matrix.python[0] }}-win_amd64)" - - uses: actions/checkout@v4 + - uses: ./.github/actions/run-ee-server with: - ref: ${{ inputs.ref }} + use-server-rc: ${{ inputs.use-server-rc }} + server-tag: ${{ inputs.server-tag }} + docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} + docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} - name: Download wheel uses: actions/download-artifact@v4 with: - name: ${{ matrix.python[0] }}-win_amd64.build + name: ${{ env.BUILD_IDENTIFIER }}.build + + - name: Convert Python tag to Python version + # Don't use sed because we want this command to work on both mac and windows + # The command used in GNU sed is different than in macOS sed + run: | + PYTHON_TAG=${{ matrix.python-tag }} + PYTHON_VERSION="${PYTHON_TAG/cp/}" + echo PYTHON_VERSION="${PYTHON_VERSION/3/3.}" >> $GITHUB_ENV + shell: bash - uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python[1] }} + python-version: ${{ env.PYTHON_VERSION }} - name: Install wheel - run: python3 -m pip install aerospike --no-index --find-links=./ - - - run: python3 -m pip install pytest -c requirements.txt - working-directory: test - - # TODO: bug: inputs are not passed in. Just fix this in the refactoring PR - - uses: ./.github/actions/run-ee-server - with: - use-server-rc: ${{ inputs.use-server-rc }} - server-tag: ${{ inputs.server-tag }} - docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} - docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + run: python3 -m pip install aerospike --force-reinstall --no-index --find-links=./ + shell: bash - name: Connect to Docker container on remote machine with Docker daemon + if: ${{ inputs.platform-tag == 'win_amd64' }} # DOCKER_HOST contains the IP address of the remote machine run: | $env:DOCKER_HOST_IP = $env:DOCKER_HOST | foreach {$_.replace("tcp://","")} | foreach {$_.replace(":2375", "")} crudini --set config.conf enterprise-edition hosts ${env:DOCKER_HOST_IP}:3000 working-directory: test + - run: python3 -m pip install pytest -c requirements.txt + working-directory: test + shell: bash + - run: python3 -m pytest new_tests/ working-directory: test + shell: bash - name: Show job status for commit if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} uses: myrotvorets/set-commit-status-action@v2.0.0 with: - sha: ${{ github.sha }} + sha: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} status: ${{ job.status }} - context: "Test Windows (${{ matrix.python[0] }}-win_amd64)" + context: ${{ env.STATUS_CHECK_MESSAGE }} diff --git a/.github/workflows/bump-stage-and-upload-to-jfrog.yml b/.github/workflows/bump-stage-and-upload-to-jfrog.yml index b797b9ffb..922c94de9 100644 --- a/.github/workflows/bump-stage-and-upload-to-jfrog.yml +++ b/.github/workflows/bump-stage-and-upload-to-jfrog.yml @@ -35,9 +35,10 @@ jobs: rebuild-artifacts-with-rc-version: needs: promote-dev-build-to-rc - uses: ./.github/workflows/build-wheels.yml + uses: ./.github/workflows/build-artifacts.yml with: - ref: ${{ needs.promote-dev-build-to-rc.outputs.bump_sha }} + sha-to-build-and-test: ${{ needs.promote-dev-build-to-rc.outputs.bump_sha }} + secrets: inherit upload-rc-artifacts-to-jfrog: needs: [ diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index 9552c1891..7ec371644 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -26,6 +26,7 @@ on: required: false description: Commit to bump off of type: string + # See workflow call hack in update-version.yml is_workflow_call: type: boolean default: true diff --git a/.github/workflows/dev-workflow-p1.yml b/.github/workflows/dev-workflow-p1.yml index 6acfd96f9..74044da6c 100644 --- a/.github/workflows/dev-workflow-p1.yml +++ b/.github/workflows/dev-workflow-p1.yml @@ -24,9 +24,10 @@ on: jobs: test-with-server-release: - uses: ./.github/workflows/build-wheels.yml + uses: ./.github/workflows/build-artifacts.yml with: run_tests: ${{ github.event_name == 'pull_request' && true || inputs.run_server_release_tests }} + sha-to-build-and-test: ${{ github.sha }} secrets: inherit test-with-server-rc: diff --git a/.github/workflows/dev-workflow-p2.yml b/.github/workflows/dev-workflow-p2.yml index 2234ce10f..7427ec970 100644 --- a/.github/workflows/dev-workflow-p2.yml +++ b/.github/workflows/dev-workflow-p2.yml @@ -19,11 +19,11 @@ jobs: rebuild-artifacts-with-new-dev-num: needs: bump-dev-number name: Rebuild artifacts with new dev number - uses: ./.github/workflows/build-wheels.yml + uses: ./.github/workflows/build-artifacts.yml with: # On pull_request_target, the bump version commit will be ignored # So we must pass it manually to the workflow - ref: ${{ needs.bump-dev-number.outputs.bump_sha }} + sha-to-build-and-test: ${{ needs.bump-dev-number.outputs.bump_sha }} secrets: inherit upload-to-jfrog: diff --git a/.github/workflows/requirements.txt b/.github/workflows/requirements.txt index 5e6bb447f..5099e0d6f 100644 --- a/.github/workflows/requirements.txt +++ b/.github/workflows/requirements.txt @@ -1,3 +1,4 @@ parver==0.5 crudini==0.9.4 delocate==0.10.4 +mypy==1.10.0 diff --git a/.github/workflows/stage-tests.yml b/.github/workflows/stage-tests.yml index 29a3a5164..a7664a3e0 100644 --- a/.github/workflows/stage-tests.yml +++ b/.github/workflows/stage-tests.yml @@ -186,14 +186,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install Docker Engine - run: brew install colima - - - name: Install Docker client - run: brew install docker - - - name: Start Docker Engine - run: colima start + - uses: ./.github/actions/setup-docker-on-macos - uses: ./.github/actions/run-ee-server with: diff --git a/.github/workflows/stage-to-master.yml b/.github/workflows/stage-to-master.yml index fc69f256e..154dc4dec 100644 --- a/.github/workflows/stage-to-master.yml +++ b/.github/workflows/stage-to-master.yml @@ -20,9 +20,10 @@ jobs: build-artifacts: needs: promote-rc-build-to-release - uses: ./.github/workflows/build-wheels.yml + uses: ./.github/workflows/build-artifacts.yml with: - ref: ${{ needs.promote-rc-build-to-release.outputs.bump_sha }} + sha-to-build-and-test: ${{ needs.promote-rc-build-to-release.outputs.bump_sha }} + secrets: inherit upload-to-jfrog: name: Upload artifacts to JFrog @@ -45,6 +46,7 @@ jobs: path: artifacts merge-multiple: true + # TODO: fix - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/test-server-rc.yml b/.github/workflows/test-server-rc.yml index 4a65c06bc..d0ccbd0e2 100644 --- a/.github/workflows/test-server-rc.yml +++ b/.github/workflows/test-server-rc.yml @@ -76,14 +76,7 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install Docker Engine - run: brew install colima - - - name: Install Docker client - run: brew install docker - - - name: Start Docker Engine - run: colima start + - uses: ./.github/actions/setup-docker-on-macos - uses: ./.github/actions/run-ee-server with: diff --git a/.github/workflows/update-version.yml b/.github/workflows/update-version.yml index 4900ba0dc..ce155ed1e 100644 --- a/.github/workflows/update-version.yml +++ b/.github/workflows/update-version.yml @@ -22,6 +22,7 @@ on: # A hack to tell if workflow is triggered by workflow_call or not # Calling workflows should not set this input # If workflow is triggered by workflow_dispatch, this should be set to the default boolean value: false + # https://github.com/actions/runner/discussions/1884#discussioncomment-6377587 is_workflow_call: type: boolean default: true diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 011539d2c..f79dea4a3 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -14,11 +14,16 @@ on: default: false jobs: - build-manylinux-wheels: + build-manylinux-wheel: uses: ./.github/workflows/build-wheels.yml + with: + python-tags: '["cp38"]' + platform-tag: manylinux_x86_64 + sha-to-build-and-test: ${{ github.sha }} + secrets: inherit valgrind: - needs: build-manylinux-wheels + needs: build-manylinux-wheel runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 From fbca0f69709370a96e5d167b7d2c153440ff82bf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 30 Aug 2024 16:39:04 +0000 Subject: [PATCH 13/69] Auto-bump version to 15.0.1rc5.dev1 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 88d95bb3f..fd8a91316 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.0.1rc4 +15.0.1rc5.dev1 From e0737219b37c8a1c6339b49fb92005f2e6d184f7 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:47:59 -0700 Subject: [PATCH 14/69] [CLIENT-2239] CI/CD: Use massif to get Python client memory usage over time while running integration tests (#664) - CI/CD: Removed pytest-memray test, since it's not used for CI/CD regression testing; only for debugging. I rarely check the results of this test on Github Actions and I can just run it locally when I need to debug the memory usage of the Python client code. --- .github/workflows/tests.yml | 43 ---------------------------------- .github/workflows/valgrind.yml | 20 +++++++++++++++- test/requirements.txt | 2 -- 3 files changed, 19 insertions(+), 46 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5aa87602d..c8bb7801a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -75,49 +75,6 @@ jobs: name: wheel-${{ matrix.py-version }} path: ./dist/*.whl - test-memray: - needs: build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-python@v2 - with: - python-version: "3.8" - architecture: 'x64' - - - uses: actions/download-artifact@v3 - with: - name: wheel-3.8 - - - name: Install client - run: pip install *.whl - - - name: Install test dependencies - run: pip install -r test/requirements.txt - - - name: Run Aerospike server - run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server - - - name: Create config.conf - run: cp config.conf.template config.conf - working-directory: test - - - uses: ./.github/actions/wait-for-as-server-to-start - with: - container-name: aerospike - - - name: Get number of tests - run: echo "NUM_TESTS=$(python3 -m pytest new_tests/ --collect-only -q | tail -n 1 | awk '{print $1;}')" >> $GITHUB_ENV - working-directory: test - - - name: Run tests - # Get number of tests since setting to 0 doesn't work properly - # pytest-memray currently throws a ZeroDivision error due to having a bug - # We ignore this for now - run: python -m pytest ./new_tests --memray --memray-bin-path=./ --most-allocations=${{ env.NUM_TESTS }} || true - working-directory: test - generate-coverage-report: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index f79dea4a3..15de27cee 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -12,6 +12,11 @@ on: description: 'Use server release candidate?' required: true default: false + massif: + type: boolean + description: 'Use massif for testing memory usage' + required: false + default: false jobs: build-manylinux-wheel: @@ -23,6 +28,8 @@ jobs: secrets: inherit valgrind: + env: + MASSIF_REPORT_FILE_NAME: massif.out needs: build-manylinux-wheel runs-on: ubuntu-22.04 steps: @@ -55,5 +62,16 @@ jobs: - run: sudo apt update - run: sudo apt install valgrind -y - - run: PYTHONMALLOC=malloc valgrind --leak-check=full --error-exitcode=1 python3 -m pytest -v new_tests/${{ github.event.inputs.test-file }} + + - run: echo VALGRIND_ARGS="--tool=massif --massif-out-file=./${{ env.MASSIF_REPORT_FILE_NAME }}" >> $GITHUB_ENV + if: ${{ inputs.massif }} + + - run: echo VALGRIND_ARGS="--leak-check=full" >> $GITHUB_ENV + if: ${{ !inputs.massif }} + + - run: PYTHONMALLOC=malloc valgrind --error-exitcode=1 ${{ env.VALGRIND_ARGS }} python3 -m pytest -v new_tests/${{ github.event.inputs.test-file }} + working-directory: test + + - run: ms_print ./${{ env.MASSIF_REPORT_FILE_NAME }} + if: ${{ !cancelled() && inputs.massif }} working-directory: test diff --git a/test/requirements.txt b/test/requirements.txt index 7e1acd0ec..e9043f2e7 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,3 @@ pytest==7.4.0 # To generate coverage reports in the Github Actions pipeline pytest-cov==4.1.0 -# Memory profiling -pytest-memray==1.5.0 From f06a2bbb1a6ed753842e4d2a46c0d1f1a128164b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 4 Sep 2024 18:48:37 +0000 Subject: [PATCH 15/69] Auto-bump version to 15.0.1rc5.dev2 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index fd8a91316..18dcf57b3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.0.1rc5.dev1 +15.0.1rc5.dev2 From d1e7bc22c89ccf1bc3987cd5bbfebbea0cb91f9b Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 9 Sep 2024 13:40:25 -0700 Subject: [PATCH 16/69] [CLIENT-3106] Refactor aerospike.exception initialization code and check if error indicator is set after calling a C-API method (#661) - Doc: fix base class for QueryAbortError, ClusterError, AdminError, and QueryError - Add regression tests for Aerospike exception classes - Renamed test_error_code.py to test_exceptions.py --- doc/exception.rst | 22 +- src/main/aerospike.c | 4 + src/main/exception.c | 826 +++++++++++------------------ test/new_tests/test_error_codes.py | 44 -- test/new_tests/test_exceptions.py | 165 ++++++ 5 files changed, 495 insertions(+), 566 deletions(-) delete mode 100644 test/new_tests/test_error_codes.py create mode 100644 test/new_tests/test_exceptions.py diff --git a/doc/exception.rst b/doc/exception.rst index 42cedf180..4ad2d547b 100644 --- a/doc/exception.rst +++ b/doc/exception.rst @@ -298,6 +298,14 @@ Server Errors Subclass of :py:exc:`~aerospike.exception.ServerError`. +.. py:exception:: QueryAbortedError + + Query was aborted. + + Error code: ``210`` + + Subclass of :py:exc:`~aerospike.exception.ServerError`. + Record Errors ------------- @@ -397,14 +405,6 @@ Record Errors Subclass of :py:exc:`~aerospike.exception.RecordError`. -.. py:exception:: QueryAbortedError - - Query was aborted. - - Error code: ``210`` - - Subclass of :py:exc:`~aerospike.exception.ClientError`. - Index Errors ------------ @@ -487,7 +487,7 @@ Query Errors Error code: ``213`` - Subclass of :py:exc:`~aerospike.exception.AerospikeError`. + Subclass of :py:exc:`~aerospike.exception.ServerError`. Cluster Errors -------------- @@ -507,7 +507,7 @@ Cluster Errors Error code: ``11`` - Subclass of :py:exc:`~aerospike.exception.AerospikeError`. + Subclass of :py:exc:`~aerospike.exception.ServerError`. Admin Errors ------------ @@ -516,6 +516,8 @@ Admin Errors The parent class for exceptions of the security API. + Subclass of :py:exc:`~aerospike.exception.ServerError`. + .. py:exception:: SecurityNotSupported Security functionality not supported by connected server. diff --git a/src/main/aerospike.c b/src/main/aerospike.c index ea517248b..2dc02816b 100644 --- a/src/main/aerospike.c +++ b/src/main/aerospike.c @@ -160,6 +160,10 @@ PyMODINIT_FUNC PyInit_aerospike(void) py_global_hosts = PyDict_New(); PyObject *exception = AerospikeException_New(); + if (exception == NULL) { + return NULL; + } + Py_INCREF(exception); int retval = PyModule_AddObject(aerospike, "exception", exception); if (retval == -1) { diff --git a/src/main/exception.c b/src/main/exception.c index b5eb55288..d1dcd5ec7 100644 --- a/src/main/exception.c +++ b/src/main/exception.c @@ -26,550 +26,352 @@ #include "exception_types.h" #include "macros.h" -static PyObject *module; +static PyObject *py_module; + +#define SUBMODULE_NAME "exception" + +// Used to create a Python Exception class +struct exception_def { + // When adding the exception to the module, we only need the class name + // Example: AerospikeError + const char *class_name; + // When creating an exception, we need to specify the module name + class name + // Example: exception.AerospikeError + const char *fully_qualified_class_name; + // If NULL, there is no base class + const char *base_class_name; + enum as_status_e code; + // Only applies to base exception classes that need their own fields + // NULL if this doesn't apply + const char *const *list_of_attrs; +}; + +// Used to create instances of the above struct +#define EXCEPTION_DEF(class_name, base_class_name, err_code, attrs) \ + { \ + class_name, SUBMODULE_NAME "." class_name, base_class_name, err_code, \ + attrs \ + } +// Base exception names +#define AEROSPIKE_ERR_EXCEPTION_NAME "AerospikeError" +#define CLIENT_ERR_EXCEPTION_NAME "ClientError" +#define SERVER_ERR_EXCEPTION_NAME "ServerError" +#define CLUSTER_ERR_EXCEPTION_NAME "ClusterError" +#define RECORD_ERR_EXCEPTION_NAME "RecordError" +#define INDEX_ERR_EXCEPTION_NAME "IndexError" +#define UDF_ERR_EXCEPTION_NAME "UDFError" +#define ADMIN_ERR_EXCEPTION_NAME "AdminError" +#define QUERY_ERR_EXCEPTION_NAME "QueryError" + +// If a base exception doesn't have an error code +// No exception should have an error code of 0, so this should be ok +#define NO_ERROR_CODE 0 + +const char *const aerospike_err_attrs[] = {"code", "file", "msg", "line", NULL}; +const char *const record_err_attrs[] = {"key", "bin", NULL}; +const char *const index_err_attrs[] = {"name", NULL}; +const char *const udf_err_attrs[] = {"module", "func", NULL}; + +// TODO: idea. define this as a list of tuples in python? +// Base classes must be defined before classes that inherit from them (topological sorting) +struct exception_def exception_defs[] = { + EXCEPTION_DEF(AEROSPIKE_ERR_EXCEPTION_NAME, NULL, NO_ERROR_CODE, + aerospike_err_attrs), + EXCEPTION_DEF(CLIENT_ERR_EXCEPTION_NAME, AEROSPIKE_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_CLIENT, NULL), + EXCEPTION_DEF(SERVER_ERR_EXCEPTION_NAME, AEROSPIKE_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_SERVER, NULL), + EXCEPTION_DEF("TimeoutError", AEROSPIKE_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_TIMEOUT, NULL), + // Client errors + EXCEPTION_DEF("ParamError", CLIENT_ERR_EXCEPTION_NAME, AEROSPIKE_ERR_PARAM, + NULL), + EXCEPTION_DEF("InvalidHostError", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_INVALID_HOST, NULL), + EXCEPTION_DEF("ConnectionError", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_CONNECTION, NULL), + EXCEPTION_DEF("TLSError", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_TLS_ERROR, NULL), + EXCEPTION_DEF("BatchFailed", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_BATCH_FAILED, NULL), + EXCEPTION_DEF("NoResponse", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_NO_RESPONSE, NULL), + EXCEPTION_DEF("MaxErrorRateExceeded", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_MAX_ERROR_RATE, NULL), + EXCEPTION_DEF("MaxRetriesExceeded", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_MAX_RETRIES_EXCEEDED, NULL), + EXCEPTION_DEF("InvalidNodeError", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_INVALID_NODE, NULL), + EXCEPTION_DEF("NoMoreConnectionsError", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_NO_MORE_CONNECTIONS, NULL), + EXCEPTION_DEF("AsyncConnectionError", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_ASYNC_CONNECTION, NULL), + EXCEPTION_DEF("ClientAbortError", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_CLIENT_ABORT, NULL), + // Server errors + EXCEPTION_DEF("InvalidRequest", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_REQUEST_INVALID, NULL), + EXCEPTION_DEF("ServerFull", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_SERVER_FULL, NULL), + EXCEPTION_DEF("AlwaysForbidden", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_ALWAYS_FORBIDDEN, NULL), + EXCEPTION_DEF("UnsupportedFeature", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_UNSUPPORTED_FEATURE, NULL), + EXCEPTION_DEF("DeviceOverload", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_DEVICE_OVERLOAD, NULL), + EXCEPTION_DEF("NamespaceNotFound", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_NAMESPACE_NOT_FOUND, NULL), + EXCEPTION_DEF("ForbiddenError", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_FAIL_FORBIDDEN, NULL), + EXCEPTION_DEF(QUERY_ERR_EXCEPTION_NAME, SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_QUERY, NULL), + EXCEPTION_DEF(CLUSTER_ERR_EXCEPTION_NAME, SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_CLUSTER, NULL), + EXCEPTION_DEF("InvalidGeoJSON", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_GEO_INVALID_GEOJSON, NULL), + EXCEPTION_DEF("OpNotApplicable", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_OP_NOT_APPLICABLE, NULL), + EXCEPTION_DEF("FilteredOut", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_FILTERED_OUT, NULL), + EXCEPTION_DEF("LostConflict", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_LOST_CONFLICT, NULL), + EXCEPTION_DEF("ScanAbortedError", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_SCAN_ABORTED, NULL), + EXCEPTION_DEF("ElementNotFoundError", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_FAIL_ELEMENT_NOT_FOUND, NULL), + EXCEPTION_DEF("ElementExistsError", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_FAIL_ELEMENT_EXISTS, NULL), + EXCEPTION_DEF("BatchDisabledError", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_BATCH_DISABLED, NULL), + EXCEPTION_DEF("BatchMaxRequestError", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_BATCH_MAX_REQUESTS_EXCEEDED, NULL), + EXCEPTION_DEF("BatchQueueFullError", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_BATCH_QUEUES_FULL, NULL), + EXCEPTION_DEF("QueryAbortedError", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_QUERY_ABORTED, NULL), + // Cluster errors + EXCEPTION_DEF("ClusterChangeError", CLUSTER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_CLUSTER_CHANGE, NULL), + // Record errors + // RecordError doesn't have an error code. It will be ignored in this case + EXCEPTION_DEF(RECORD_ERR_EXCEPTION_NAME, SERVER_ERR_EXCEPTION_NAME, + NO_ERROR_CODE, record_err_attrs), + EXCEPTION_DEF("RecordKeyMismatch", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_RECORD_KEY_MISMATCH, NULL), + EXCEPTION_DEF("RecordNotFound", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_RECORD_NOT_FOUND, NULL), + EXCEPTION_DEF("RecordGenerationError", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_RECORD_GENERATION, NULL), + EXCEPTION_DEF("RecordExistsError", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_RECORD_EXISTS, NULL), + EXCEPTION_DEF("RecordTooBig", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_RECORD_TOO_BIG, NULL), + EXCEPTION_DEF("RecordBusy", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_RECORD_BUSY, NULL), + EXCEPTION_DEF("BinNameError", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_BIN_NAME, NULL), + EXCEPTION_DEF("BinIncompatibleType", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_BIN_INCOMPATIBLE_TYPE, NULL), + EXCEPTION_DEF("BinExistsError", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_BIN_EXISTS, NULL), + EXCEPTION_DEF("BinNotFound", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_BIN_NOT_FOUND, NULL), + // Index errors + EXCEPTION_DEF(INDEX_ERR_EXCEPTION_NAME, SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_INDEX, index_err_attrs), + EXCEPTION_DEF("IndexNotFound", INDEX_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_INDEX_NOT_FOUND, NULL), + EXCEPTION_DEF("IndexFoundError", INDEX_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_INDEX_FOUND, NULL), + EXCEPTION_DEF("IndexOOM", INDEX_ERR_EXCEPTION_NAME, AEROSPIKE_ERR_INDEX_OOM, + NULL), + EXCEPTION_DEF("IndexNotReadable", INDEX_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_INDEX_NOT_READABLE, NULL), + EXCEPTION_DEF("IndexNameMaxLen", INDEX_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_INDEX_NAME_MAXLEN, NULL), + EXCEPTION_DEF("IndexNameMaxCount", INDEX_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_INDEX_MAXCOUNT, NULL), + // UDF errors + EXCEPTION_DEF(UDF_ERR_EXCEPTION_NAME, SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_UDF, udf_err_attrs), + EXCEPTION_DEF("UDFNotFound", UDF_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_UDF_NOT_FOUND, NULL), + EXCEPTION_DEF("LuaFileNotFound", UDF_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_LUA_FILE_NOT_FOUND, NULL), + // Admin errors + EXCEPTION_DEF(ADMIN_ERR_EXCEPTION_NAME, SERVER_ERR_EXCEPTION_NAME, + NO_ERROR_CODE, NULL), + EXCEPTION_DEF("SecurityNotSupported", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_SECURITY_NOT_SUPPORTED, NULL), + EXCEPTION_DEF("SecurityNotEnabled", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_SECURITY_NOT_ENABLED, NULL), + EXCEPTION_DEF("SecuritySchemeNotSupported", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_SECURITY_SCHEME_NOT_SUPPORTED, NULL), + EXCEPTION_DEF("InvalidCommand", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_COMMAND, NULL), + EXCEPTION_DEF("InvalidField", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_FIELD, NULL), + EXCEPTION_DEF("IllegalState", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_ILLEGAL_STATE, NULL), + EXCEPTION_DEF("InvalidUser", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_USER, NULL), + EXCEPTION_DEF("UserExistsError", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_USER_ALREADY_EXISTS, NULL), + EXCEPTION_DEF("InvalidPassword", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_PASSWORD, NULL), + EXCEPTION_DEF("ExpiredPassword", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_EXPIRED_PASSWORD, NULL), + EXCEPTION_DEF("ForbiddenPassword", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_FORBIDDEN_PASSWORD, NULL), + EXCEPTION_DEF("InvalidCredential", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_CREDENTIAL, NULL), + EXCEPTION_DEF("InvalidRole", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_ROLE, NULL), + EXCEPTION_DEF("RoleExistsError", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_ROLE_ALREADY_EXISTS, NULL), + EXCEPTION_DEF("RoleViolation", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_ROLE_VIOLATION, NULL), + EXCEPTION_DEF("InvalidPrivilege", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_PRIVILEGE, NULL), + EXCEPTION_DEF("NotAuthenticated", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_NOT_AUTHENTICATED, NULL), + EXCEPTION_DEF("InvalidWhitelist", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_WHITELIST, NULL), + EXCEPTION_DEF("NotWhitelisted", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_NOT_WHITELISTED, NULL), + EXCEPTION_DEF("QuotasNotEnabled", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_QUOTAS_NOT_ENABLED, NULL), + EXCEPTION_DEF("InvalidQuota", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_QUOTA, NULL), + EXCEPTION_DEF("QuotaExceeded", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_QUOTA_EXCEEDED, NULL), + // Query errors + EXCEPTION_DEF("QueryQueueFull", QUERY_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_QUERY_QUEUE_FULL, NULL), + EXCEPTION_DEF("QueryTimeout", QUERY_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_QUERY_TIMEOUT, NULL)}; + +// TODO: define aerospike module name somewhere else +#define FULLY_QUALIFIED_MODULE_NAME "aerospike." SUBMODULE_NAME + +// Returns NULL if an error occurred PyObject *AerospikeException_New(void) { static struct PyModuleDef moduledef = {PyModuleDef_HEAD_INIT, - "aerospike.exception", + FULLY_QUALIFIED_MODULE_NAME, "Exception objects", -1, NULL, NULL, NULL, NULL}; - module = PyModule_Create(&moduledef); - - struct exceptions exceptions_array; - - memset(&exceptions_array, 0, sizeof(exceptions_array)); - - struct server_exceptions_struct server_array = { - {&exceptions_array.InvalidRequest, &exceptions_array.ServerFull, - &exceptions_array.AlwaysForbidden, - &exceptions_array.UnsupportedFeature, &exceptions_array.DeviceOverload, - &exceptions_array.NamespaceNotFound, &exceptions_array.ForbiddenError, - &exceptions_array.QueryError, &exceptions_array.ClusterError, - &exceptions_array.InvalidGeoJSON, &exceptions_array.OpNotApplicable, - &exceptions_array.FilteredOut, &exceptions_array.LostConflict}, - {"InvalidRequest", "ServerFull", "AlwaysForbidden", - "UnsupportedFeature", "DeviceOverload", "NamespaceNotFound", - "ForbiddenError", "QueryError", "ClusterError", "InvalidGeoJSON", - "OpNotApplicable", "FilteredOut", "LostConflict"}, - {AEROSPIKE_ERR_REQUEST_INVALID, AEROSPIKE_ERR_SERVER_FULL, - AEROSPIKE_ERR_ALWAYS_FORBIDDEN, AEROSPIKE_ERR_UNSUPPORTED_FEATURE, - AEROSPIKE_ERR_DEVICE_OVERLOAD, AEROSPIKE_ERR_NAMESPACE_NOT_FOUND, - AEROSPIKE_ERR_FAIL_FORBIDDEN, AEROSPIKE_ERR_QUERY, - AEROSPIKE_ERR_CLUSTER, AEROSPIKE_ERR_GEO_INVALID_GEOJSON, - AEROSPIKE_ERR_OP_NOT_APPLICABLE, AEROSPIKE_FILTERED_OUT, - AEROSPIKE_LOST_CONFLICT}}; - - struct record_exceptions_struct record_array = { - {&exceptions_array.RecordKeyMismatch, &exceptions_array.RecordNotFound, - &exceptions_array.RecordGenerationError, - &exceptions_array.RecordExistsError, &exceptions_array.RecordTooBig, - &exceptions_array.RecordBusy, &exceptions_array.BinNameError, - &exceptions_array.BinIncompatibleType, - &exceptions_array.BinExistsError, &exceptions_array.BinNotFound}, - {"RecordKeyMismatch", "RecordNotFound", "RecordGenerationError", - "RecordExistsError", "RecordTooBig", "RecordBusy", "BinNameError", - "BinIncompatibleType", "BinExistsError", "BinNotFound"}, - {AEROSPIKE_ERR_RECORD_KEY_MISMATCH, AEROSPIKE_ERR_RECORD_NOT_FOUND, - AEROSPIKE_ERR_RECORD_GENERATION, AEROSPIKE_ERR_RECORD_EXISTS, - AEROSPIKE_ERR_RECORD_TOO_BIG, AEROSPIKE_ERR_RECORD_BUSY, - AEROSPIKE_ERR_BIN_NAME, AEROSPIKE_ERR_BIN_INCOMPATIBLE_TYPE, - AEROSPIKE_ERR_BIN_EXISTS, AEROSPIKE_ERR_BIN_NOT_FOUND}}; - - struct index_exceptions_struct index_array = { - {&exceptions_array.IndexNotFound, &exceptions_array.IndexFoundError, - &exceptions_array.IndexOOM, &exceptions_array.IndexNotReadable, - &exceptions_array.IndexNameMaxLen, - &exceptions_array.IndexNameMaxCount}, - {"IndexNotFound", "IndexFoundError", "IndexOOM", "IndexNotReadable", - "IndexNameMaxLen", "IndexNameMaxCount"}, - {AEROSPIKE_ERR_INDEX_NOT_FOUND, AEROSPIKE_ERR_INDEX_FOUND, - AEROSPIKE_ERR_INDEX_OOM, AEROSPIKE_ERR_INDEX_NOT_READABLE, - AEROSPIKE_ERR_INDEX_NAME_MAXLEN, AEROSPIKE_ERR_INDEX_MAXCOUNT}}; - - struct admin_exceptions_struct admin_array = { - {&exceptions_array.SecurityNotSupported, - &exceptions_array.SecurityNotEnabled, - &exceptions_array.SecuritySchemeNotSupported, - &exceptions_array.InvalidCommand, - &exceptions_array.InvalidField, - &exceptions_array.IllegalState, - &exceptions_array.InvalidUser, - &exceptions_array.UserExistsError, - &exceptions_array.InvalidPassword, - &exceptions_array.ExpiredPassword, - &exceptions_array.ForbiddenPassword, - &exceptions_array.InvalidCredential, - &exceptions_array.InvalidRole, - &exceptions_array.RoleExistsError, - &exceptions_array.RoleViolation, - &exceptions_array.InvalidPrivilege, - &exceptions_array.NotAuthenticated, - &exceptions_array.InvalidWhitelist, - &exceptions_array.NotWhitelisted, - &exceptions_array.QuotasNotEnabled, - &exceptions_array.InvalidQuota, - &exceptions_array.QuotaExceeded}, - {"SecurityNotSupported", - "SecurityNotEnabled", - "SecuritySchemeNotSupported", - "InvalidCommand", - "InvalidField", - "IllegalState", - "InvalidUser", - "UserExistsError", - "InvalidPassword", - "ExpiredPassword", - "ForbiddenPassword", - "InvalidCredential", - "InvalidRole", - "RoleExistsError", - "RoleViolation", - "InvalidPrivilege", - "NotAuthenticated", - "InvalidWhitelist", - "NotWhitelisted", - "QuotasNotEnabled", - "InvalidQuota", - "QuotaExceeded"}, - {AEROSPIKE_SECURITY_NOT_SUPPORTED, - AEROSPIKE_SECURITY_NOT_ENABLED, - AEROSPIKE_SECURITY_SCHEME_NOT_SUPPORTED, - AEROSPIKE_INVALID_COMMAND, - AEROSPIKE_INVALID_FIELD, - AEROSPIKE_ILLEGAL_STATE, - AEROSPIKE_INVALID_USER, - AEROSPIKE_USER_ALREADY_EXISTS, - AEROSPIKE_INVALID_PASSWORD, - AEROSPIKE_EXPIRED_PASSWORD, - AEROSPIKE_FORBIDDEN_PASSWORD, - AEROSPIKE_INVALID_CREDENTIAL, - AEROSPIKE_INVALID_ROLE, - AEROSPIKE_ROLE_ALREADY_EXISTS, - AEROSPIKE_ROLE_VIOLATION, - AEROSPIKE_INVALID_PRIVILEGE, - AEROSPIKE_NOT_AUTHENTICATED, - AEROSPIKE_INVALID_WHITELIST, - AEROSPIKE_NOT_WHITELISTED, - AEROSPIKE_QUOTAS_NOT_ENABLED, - AEROSPIKE_INVALID_QUOTA, - AEROSPIKE_QUOTA_EXCEEDED}}; - - PyObject *py_code = NULL; - PyObject *py_dict = PyDict_New(); - PyDict_SetItemString(py_dict, "code", Py_None); - PyDict_SetItemString(py_dict, "file", Py_None); - PyDict_SetItemString(py_dict, "msg", Py_None); - PyDict_SetItemString(py_dict, "line", Py_None); - - exceptions_array.AerospikeError = - PyErr_NewException("exception.AerospikeError", NULL, py_dict); - Py_INCREF(exceptions_array.AerospikeError); - Py_DECREF(py_dict); - PyModule_AddObject(module, "AerospikeError", - exceptions_array.AerospikeError); - PyObject_SetAttrString(exceptions_array.AerospikeError, "code", Py_None); - - exceptions_array.ClientError = PyErr_NewException( - "exception.ClientError", exceptions_array.AerospikeError, NULL); - Py_INCREF(exceptions_array.ClientError); - PyModule_AddObject(module, "ClientError", exceptions_array.ClientError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_CLIENT); - PyObject_SetAttrString(exceptions_array.ClientError, "code", py_code); - Py_DECREF(py_code); - - exceptions_array.ServerError = PyErr_NewException( - "exception.ServerError", exceptions_array.AerospikeError, NULL); - Py_INCREF(exceptions_array.ServerError); - PyModule_AddObject(module, "ServerError", exceptions_array.ServerError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_SERVER); - PyObject_SetAttrString(exceptions_array.ServerError, "code", py_code); - Py_DECREF(py_code); - - exceptions_array.TimeoutError = PyErr_NewException( - "exception.TimeoutError", exceptions_array.AerospikeError, NULL); - Py_INCREF(exceptions_array.TimeoutError); - PyModule_AddObject(module, "TimeoutError", exceptions_array.TimeoutError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_TIMEOUT); - PyObject_SetAttrString(exceptions_array.TimeoutError, "code", py_code); - Py_DECREF(py_code); - - //Client Exceptions - exceptions_array.ParamError = PyErr_NewException( - "exception.ParamError", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.ParamError); - PyModule_AddObject(module, "ParamError", exceptions_array.ParamError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_PARAM); - PyObject_SetAttrString(exceptions_array.ParamError, "code", py_code); - Py_DECREF(py_code); - - exceptions_array.InvalidHostError = PyErr_NewException( - "exception.InvalidHostError", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.InvalidHostError); - PyModule_AddObject(module, "InvalidHostError", - exceptions_array.InvalidHostError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_INVALID_HOST); - PyObject_SetAttrString(exceptions_array.InvalidHostError, "code", py_code); - Py_DECREF(py_code); - - exceptions_array.ConnectionError = PyErr_NewException( - "exception.ConnectionError", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.ConnectionError); - PyModule_AddObject(module, "ConnectionError", - exceptions_array.ConnectionError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_CONNECTION); - PyObject_SetAttrString(exceptions_array.ConnectionError, "code", py_code); - Py_DECREF(py_code); - - // TLSError, AEROSPIKE_ERR_TLS_ERROR, -9 - exceptions_array.TLSError = PyErr_NewException( - "exception.TLSError", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.TLSError); - PyModule_AddObject(module, "TLSError", exceptions_array.TLSError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_TLS_ERROR); - PyObject_SetAttrString(exceptions_array.TLSError, "code", py_code); - Py_DECREF(py_code); - - // BatchFailed, AEROSPIKE_BATCH_FAILED, -16 - exceptions_array.BatchFailed = PyErr_NewException( - "exception.BatchFailed", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.BatchFailed); - PyModule_AddObject(module, "BatchFailed", exceptions_array.BatchFailed); - py_code = PyLong_FromLong(AEROSPIKE_BATCH_FAILED); - PyObject_SetAttrString(exceptions_array.BatchFailed, "code", py_code); - Py_DECREF(py_code); - - // NoResponse, AEROSPIKE_NO_RESPONSE, -15 - exceptions_array.NoResponse = PyErr_NewException( - "exception.NoResponse", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.NoResponse); - PyModule_AddObject(module, "NoResponse", exceptions_array.NoResponse); - py_code = PyLong_FromLong(AEROSPIKE_NO_RESPONSE); - PyObject_SetAttrString(exceptions_array.NoResponse, "code", py_code); - Py_DECREF(py_code); - - // max errors limit reached, AEROSPIKE_MAX_ERROR_RATE, -14 - exceptions_array.MaxErrorRateExceeded = PyErr_NewException( - "exception.MaxErrorRateExceeded", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.MaxErrorRateExceeded); - PyModule_AddObject(module, "MaxErrorRateExceeded", - exceptions_array.MaxErrorRateExceeded); - py_code = PyLong_FromLong(AEROSPIKE_MAX_ERROR_RATE); - PyObject_SetAttrString(exceptions_array.MaxErrorRateExceeded, "code", - py_code); - Py_DECREF(py_code); - - // max retries exceeded, AEROSPIKE_ERR_MAX_RETRIES_EXCEEDED, -12 - exceptions_array.MaxRetriesExceeded = PyErr_NewException( - "exception.MaxRetriesExceeded", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.MaxRetriesExceeded); - PyModule_AddObject(module, "MaxRetriesExceeded", - exceptions_array.MaxRetriesExceeded); - py_code = PyLong_FromLong(AEROSPIKE_ERR_MAX_RETRIES_EXCEEDED); - PyObject_SetAttrString(exceptions_array.MaxRetriesExceeded, "code", - py_code); - Py_DECREF(py_code); - - // InvalidNodeError, AEROSPIKE_ERR_INVALID_NODE, -8 - exceptions_array.InvalidNodeError = PyErr_NewException( - "exception.InvalidNodeError", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.InvalidNodeError); - PyModule_AddObject(module, "InvalidNodeError", - exceptions_array.InvalidNodeError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_INVALID_NODE); - PyObject_SetAttrString(exceptions_array.InvalidNodeError, "code", py_code); - Py_DECREF(py_code); - - // NoMoreConnectionsError, AEROSPIKE_ERR_NO_MORE_CONNECTIONS, -7 - exceptions_array.NoMoreConnectionsError = PyErr_NewException( - "exception.NoMoreConnectionsError", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.NoMoreConnectionsError); - PyModule_AddObject(module, "NoMoreConnectionsError", - exceptions_array.NoMoreConnectionsError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_NO_MORE_CONNECTIONS); - PyObject_SetAttrString(exceptions_array.NoMoreConnectionsError, "code", - py_code); - Py_DECREF(py_code); - - // AsyncConnectionError, AEROSPIKE_ERR_ASYNC_CONNECTION, -6 - exceptions_array.AsyncConnectionError = PyErr_NewException( - "exception.AsyncConnectionError", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.AsyncConnectionError); - PyModule_AddObject(module, "AsyncConnectionError", - exceptions_array.AsyncConnectionError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_ASYNC_CONNECTION); - PyObject_SetAttrString(exceptions_array.AsyncConnectionError, "code", - py_code); - Py_DECREF(py_code); - - // ClientAbortError, AEROSPIKE_ERR_CLIENT_ABORT, -5 - exceptions_array.ClientAbortError = PyErr_NewException( - "exception.ClientAbortError", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.ClientAbortError); - PyModule_AddObject(module, "ClientAbortError", - exceptions_array.ClientAbortError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_CLIENT_ABORT); - PyObject_SetAttrString(exceptions_array.ClientAbortError, "code", py_code); - Py_DECREF(py_code); - - //Server Exceptions - int count = sizeof(server_array.server_exceptions) / - sizeof(server_array.server_exceptions[0]); - int i; - PyObject **current_exception; - for (i = 0; i < count; i++) { - current_exception = server_array.server_exceptions[i]; - char *name = server_array.server_exceptions_name[i]; - char prefix[40] = "exception."; - *current_exception = PyErr_NewException( - strcat(prefix, name), exceptions_array.ServerError, NULL); - Py_INCREF(*current_exception); - PyModule_AddObject(module, name, *current_exception); - PyObject *py_code = - PyLong_FromLong(server_array.server_exceptions_codes[i]); - PyObject_SetAttrString(*current_exception, "code", py_code); - Py_DECREF(py_code); + py_module = PyModule_Create(&moduledef); + if (py_module == NULL) { + return NULL; } - exceptions_array.ClusterChangeError = PyErr_NewException( - "exception.ClusterChangeError", exceptions_array.ClusterError, NULL); - Py_INCREF(exceptions_array.ClusterChangeError); - PyModule_AddObject(module, "ClusterChangeError", - exceptions_array.ClusterChangeError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_CLUSTER_CHANGE); - PyObject_SetAttrString(exceptions_array.ClusterChangeError, "code", - py_code); - Py_DECREF(py_code); - - //Extra Server Errors - // ScanAbortedError , AEROSPIKE_ERR_SCAN_ABORTED, 15 - exceptions_array.ScanAbortedError = PyErr_NewException( - "exception.ScanAbortedError", exceptions_array.ServerError, NULL); - Py_INCREF(exceptions_array.ScanAbortedError); - PyModule_AddObject(module, "ScanAbortedError", - exceptions_array.ScanAbortedError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_SCAN_ABORTED); - PyObject_SetAttrString(exceptions_array.ScanAbortedError, "code", py_code); - Py_DECREF(py_code); - - // ElementNotFoundError , AEROSPIKE_ERR_FAIL_ELEMENT_NOT_FOUND, 23 - exceptions_array.ElementNotFoundError = PyErr_NewException( - "exception.ElementNotFoundError", exceptions_array.ServerError, NULL); - Py_INCREF(exceptions_array.ElementNotFoundError); - PyModule_AddObject(module, "ElementNotFoundError", - exceptions_array.ElementNotFoundError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_FAIL_ELEMENT_NOT_FOUND); - PyObject_SetAttrString(exceptions_array.ElementNotFoundError, "code", - py_code); - Py_DECREF(py_code); - - // ElementExistsError , AEROSPIKE_ERR_FAIL_ELEMENT_EXISTS, 24 - exceptions_array.ElementExistsError = PyErr_NewException( - "exception.ElementExistsError", exceptions_array.ServerError, NULL); - Py_INCREF(exceptions_array.ElementExistsError); - PyModule_AddObject(module, "ElementExistsError", - exceptions_array.ElementExistsError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_FAIL_ELEMENT_EXISTS); - PyObject_SetAttrString(exceptions_array.ElementExistsError, "code", - py_code); - Py_DECREF(py_code); - - // BatchDisabledError , AEROSPIKE_ERR_BATCH_DISABLED, 150 - exceptions_array.BatchDisabledError = PyErr_NewException( - "exception.BatchDisabledError", exceptions_array.ServerError, NULL); - Py_INCREF(exceptions_array.BatchDisabledError); - PyModule_AddObject(module, "BatchDisabledError", - exceptions_array.BatchDisabledError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_BATCH_DISABLED); - PyObject_SetAttrString(exceptions_array.BatchDisabledError, "code", - py_code); - Py_DECREF(py_code); - - // BatchMaxRequestError , AEROSPIKE_ERR_BATCH_MAX_REQUESTS_EXCEEDED, 151 - exceptions_array.BatchMaxRequestError = PyErr_NewException( - "exception.BatchMaxRequestError", exceptions_array.ServerError, NULL); - Py_INCREF(exceptions_array.BatchMaxRequestError); - PyModule_AddObject(module, "BatchMaxRequestError", - exceptions_array.BatchMaxRequestError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_BATCH_MAX_REQUESTS_EXCEEDED); - PyObject_SetAttrString(exceptions_array.BatchMaxRequestError, "code", - py_code); - Py_DECREF(py_code); - - // BatchQueueFullError , AEROSPIKE_ERR_BATCH_QUEUES_FULL, 152 - exceptions_array.BatchQueueFullError = PyErr_NewException( - "exception.BatchQueueFullError", exceptions_array.ServerError, NULL); - Py_INCREF(exceptions_array.BatchQueueFullError); - PyModule_AddObject(module, "BatchQueueFullError", - exceptions_array.BatchQueueFullError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_BATCH_QUEUES_FULL); - PyObject_SetAttrString(exceptions_array.BatchQueueFullError, "code", - py_code); - Py_DECREF(py_code); - - // QueryAbortedError , AEROSPIKE_ERR_QUERY_ABORTED, 210 - exceptions_array.QueryAbortedError = PyErr_NewException( - "exception.QueryAbortedError", exceptions_array.ServerError, NULL); - Py_INCREF(exceptions_array.QueryAbortedError); - PyModule_AddObject(module, "QueryAbortedError", - exceptions_array.QueryAbortedError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_QUERY_ABORTED); - PyObject_SetAttrString(exceptions_array.QueryAbortedError, "code", py_code); - Py_DECREF(py_code); - - //Record exceptions - PyObject *py_record_dict = PyDict_New(); - PyDict_SetItemString(py_record_dict, "key", Py_None); - PyDict_SetItemString(py_record_dict, "bin", Py_None); - - exceptions_array.RecordError = PyErr_NewException( - "exception.RecordError", exceptions_array.ServerError, py_record_dict); - Py_INCREF(exceptions_array.RecordError); - Py_DECREF(py_record_dict); - PyObject_SetAttrString(exceptions_array.RecordError, "code", Py_None); - PyModule_AddObject(module, "RecordError", exceptions_array.RecordError); - - //int count = sizeof(record_exceptions)/sizeof(record_exceptions[0]); - count = sizeof(record_array.record_exceptions) / - sizeof(record_array.record_exceptions[0]); - for (i = 0; i < count; i++) { - current_exception = record_array.record_exceptions[i]; - char *name = record_array.record_exceptions_name[i]; - char prefix[40] = "exception."; - *current_exception = PyErr_NewException( - strcat(prefix, name), exceptions_array.RecordError, NULL); - Py_INCREF(*current_exception); - PyModule_AddObject(module, name, *current_exception); - PyObject *py_code = - PyLong_FromLong(record_array.record_exceptions_codes[i]); - PyObject_SetAttrString(*current_exception, "code", py_code); - Py_DECREF(py_code); - } + unsigned long exception_count = + sizeof(exception_defs) / sizeof(exception_defs[0]); + for (unsigned long i = 0; i < exception_count; i++) { + struct exception_def exception_def = exception_defs[i]; + + // TODO: if fetching base class is too slow, cache them using variables + // This only runs once when `import aerospike` is called, though + // When a module is loaded once through an import, it won't be loaded again + PyObject *py_base_class = NULL; + if (exception_def.base_class_name != NULL) { + py_base_class = PyObject_GetAttrString( + py_module, exception_def.base_class_name); + if (py_base_class == NULL) { + goto MODULE_CLEANUP_ON_ERROR; + } + } - //Index exceptions - PyObject *py_index_dict = PyDict_New(); - PyDict_SetItemString(py_index_dict, "name", Py_None); - - exceptions_array.IndexError = PyErr_NewException( - "exception.IndexError", exceptions_array.ServerError, py_index_dict); - Py_INCREF(exceptions_array.IndexError); - Py_DECREF(py_index_dict); - py_code = PyLong_FromLong(AEROSPIKE_ERR_INDEX); - PyObject_SetAttrString(exceptions_array.IndexError, "code", py_code); - Py_DECREF(py_code); - PyModule_AddObject(module, "IndexError", exceptions_array.IndexError); - - count = sizeof(index_array.index_exceptions) / - sizeof(index_array.index_exceptions[0]); - for (i = 0; i < count; i++) { - current_exception = index_array.index_exceptions[i]; - char *name = index_array.index_exceptions_name[i]; - char prefix[40] = "exception."; - *current_exception = PyErr_NewException( - strcat(prefix, name), exceptions_array.IndexError, NULL); - Py_INCREF(*current_exception); - PyModule_AddObject(module, name, *current_exception); - PyObject *py_code = - PyLong_FromLong(index_array.index_exceptions_codes[i]); - PyObject_SetAttrString(*current_exception, "code", py_code); - Py_DECREF(py_code); - } + // Set up class attributes + PyObject *py_exc_dict = NULL; + if (exception_def.list_of_attrs != NULL) { + py_exc_dict = PyDict_New(); + if (py_exc_dict == NULL) { + Py_XDECREF(py_base_class); + goto MODULE_CLEANUP_ON_ERROR; + } - //UDF exceptions - PyObject *py_udf_dict = PyDict_New(); - PyDict_SetItemString(py_udf_dict, "module", Py_None); - PyDict_SetItemString(py_udf_dict, "func", Py_None); - - exceptions_array.UDFError = PyErr_NewException( - "exception.UDFError", exceptions_array.ServerError, py_udf_dict); - Py_INCREF(exceptions_array.UDFError); - Py_DECREF(py_udf_dict); - PyModule_AddObject(module, "UDFError", exceptions_array.UDFError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_UDF); - PyObject_SetAttrString(exceptions_array.UDFError, "code", py_code); - Py_DECREF(py_code); - - exceptions_array.UDFNotFound = PyErr_NewException( - "exception.UDFNotFound", exceptions_array.UDFError, NULL); - Py_INCREF(exceptions_array.UDFNotFound); - PyModule_AddObject(module, "UDFNotFound", exceptions_array.UDFNotFound); - py_code = PyLong_FromLong(AEROSPIKE_ERR_UDF_NOT_FOUND); - PyObject_SetAttrString(exceptions_array.UDFNotFound, "code", py_code); - Py_DECREF(py_code); - - exceptions_array.LuaFileNotFound = PyErr_NewException( - "exception.LuaFileNotFound", exceptions_array.UDFError, NULL); - Py_INCREF(exceptions_array.LuaFileNotFound); - PyModule_AddObject(module, "LuaFileNotFound", - exceptions_array.LuaFileNotFound); - py_code = PyLong_FromLong(AEROSPIKE_ERR_LUA_FILE_NOT_FOUND); - PyObject_SetAttrString(exceptions_array.LuaFileNotFound, "code", py_code); - Py_DECREF(py_code); - - //Admin exceptions - exceptions_array.AdminError = PyErr_NewException( - "exception.AdminError", exceptions_array.ServerError, NULL); - Py_INCREF(exceptions_array.AdminError); - PyObject_SetAttrString(exceptions_array.AdminError, "code", Py_None); - PyModule_AddObject(module, "AdminError", exceptions_array.AdminError); - - count = sizeof(admin_array.admin_exceptions) / - sizeof(admin_array.admin_exceptions[0]); - for (i = 0; i < count; i++) { - current_exception = admin_array.admin_exceptions[i]; - char *name = admin_array.admin_exceptions_name[i]; - char prefix[40] = "exception."; - *current_exception = PyErr_NewException( - strcat(prefix, name), exceptions_array.AdminError, NULL); - Py_INCREF(*current_exception); - PyModule_AddObject(module, name, *current_exception); - PyObject *py_code = - PyLong_FromLong(admin_array.admin_exceptions_codes[i]); - PyObject_SetAttrString(*current_exception, "code", py_code); + const char *const *curr_attr_ref = exception_def.list_of_attrs; + while (*curr_attr_ref != NULL) { + int retval = + PyDict_SetItemString(py_exc_dict, *curr_attr_ref, Py_None); + if (retval == -1) { + Py_DECREF(py_exc_dict); + Py_XDECREF(py_base_class); + goto MODULE_CLEANUP_ON_ERROR; + } + curr_attr_ref++; + } + } + + PyObject *py_exception_class = + PyErr_NewException(exception_def.fully_qualified_class_name, + py_base_class, py_exc_dict); + Py_XDECREF(py_base_class); + Py_XDECREF(py_exc_dict); + if (py_exception_class == NULL) { + goto MODULE_CLEANUP_ON_ERROR; + } + + PyObject *py_code = NULL; + if (exception_def.code == NO_ERROR_CODE) { + py_code = Py_None; + } + else { + py_code = PyLong_FromLong(exception_def.code); + if (py_code == NULL) { + goto EXC_CLASS_CLEANUP_ON_ERROR; + } + } + int retval = + PyObject_SetAttrString(py_exception_class, "code", py_code); Py_DECREF(py_code); + if (retval == -1) { + goto EXC_CLASS_CLEANUP_ON_ERROR; + } + + retval = PyModule_AddObject(py_module, exception_def.class_name, + py_exception_class); + if (retval == -1) { + goto EXC_CLASS_CLEANUP_ON_ERROR; + } + continue; + + EXC_CLASS_CLEANUP_ON_ERROR: + Py_DECREF(py_exception_class); + goto MODULE_CLEANUP_ON_ERROR; } - //Query exceptions - exceptions_array.QueryQueueFull = PyErr_NewException( - "exception.QueryQueueFull", exceptions_array.QueryError, NULL); - Py_INCREF(exceptions_array.QueryQueueFull); - PyModule_AddObject(module, "QueryQueueFull", - exceptions_array.QueryQueueFull); - py_code = PyLong_FromLong(AEROSPIKE_ERR_QUERY_QUEUE_FULL); - PyObject_SetAttrString(exceptions_array.QueryQueueFull, "code", py_code); - Py_DECREF(py_code); - - exceptions_array.QueryTimeout = PyErr_NewException( - "exception.QueryTimeout", exceptions_array.QueryError, NULL); - Py_INCREF(exceptions_array.QueryTimeout); - PyModule_AddObject(module, "QueryTimeout", exceptions_array.QueryTimeout); - py_code = PyLong_FromLong(AEROSPIKE_ERR_QUERY_TIMEOUT); - PyObject_SetAttrString(exceptions_array.QueryTimeout, "code", py_code); - Py_DECREF(py_code); - - return module; + return py_module; + +MODULE_CLEANUP_ON_ERROR: + Py_DECREF(py_module); + return NULL; } void remove_exception(as_error *err) { PyObject *py_key = NULL, *py_value = NULL; Py_ssize_t pos = 0; - PyObject *py_module_dict = PyModule_GetDict(module); + PyObject *py_module_dict = PyModule_GetDict(py_module); while (PyDict_Next(py_module_dict, &pos, &py_key, &py_value)) { Py_DECREF(py_value); } } +// TODO: idea. Use python dict to map error code to exception void raise_exception(as_error *err) { PyObject *py_key = NULL, *py_value = NULL; Py_ssize_t pos = 0; - PyObject *py_module_dict = PyModule_GetDict(module); + PyObject *py_module_dict = PyModule_GetDict(py_module); bool found = false; while (PyDict_Next(py_module_dict, &pos, &py_key, &py_value)) { @@ -640,7 +442,7 @@ PyObject *raise_exception_old(as_error *err) { PyObject *py_key = NULL, *py_value = NULL; Py_ssize_t pos = 0; - PyObject *py_module_dict = PyModule_GetDict(module); + PyObject *py_module_dict = PyModule_GetDict(py_module); bool found = false; while (PyDict_Next(py_module_dict, &pos, &py_key, &py_value)) { diff --git a/test/new_tests/test_error_codes.py b/test/new_tests/test_error_codes.py deleted file mode 100644 index 6b2002934..000000000 --- a/test/new_tests/test_error_codes.py +++ /dev/null @@ -1,44 +0,0 @@ -import pytest -from aerospike import exception as e -from .as_errors import ( - AEROSPIKE_ERR_ASYNC_CONNECTION, - AEROSPIKE_ERR_BATCH_DISABLED, - AEROSPIKE_ERR_BATCH_MAX_REQUESTS_EXCEEDED, - AEROSPIKE_ERR_BATCH_QUEUES_FULL, - AEROSPIKE_ERR_CLIENT_ABORT, - AEROSPIKE_ERR_FAIL_ELEMENT_EXISTS, - AEROSPIKE_ERR_FAIL_ELEMENT_NOT_FOUND, - AEROSPIKE_ERR_INVALID_NODE, - AEROSPIKE_ERR_NO_MORE_CONNECTIONS, - AEROSPIKE_ERR_QUERY_ABORTED, - AEROSPIKE_ERR_SCAN_ABORTED, - AEROSPIKE_ERR_TLS_ERROR, -) - - -@pytest.mark.parametrize( - "error, error_name, error_code, base", - ( - (e.TLSError, "TLSError", AEROSPIKE_ERR_TLS_ERROR, e.ClientError), - (e.InvalidNodeError, "InvalidNodeError", AEROSPIKE_ERR_INVALID_NODE, e.ClientError), - (e.NoMoreConnectionsError, "NoMoreConnectionsError", AEROSPIKE_ERR_NO_MORE_CONNECTIONS, e.ClientError), - (e.AsyncConnectionError, "AsyncConnectionError", AEROSPIKE_ERR_ASYNC_CONNECTION, e.ClientError), - (e.ClientAbortError, "ClientAbortError", AEROSPIKE_ERR_CLIENT_ABORT, e.ClientError), - (e.ScanAbortedError, "ScanAbortedError", AEROSPIKE_ERR_SCAN_ABORTED, e.ServerError), - (e.ElementNotFoundError, "ElementNotFoundError", AEROSPIKE_ERR_FAIL_ELEMENT_NOT_FOUND, e.ServerError), - (e.ElementExistsError, "ElementExistsError", AEROSPIKE_ERR_FAIL_ELEMENT_EXISTS, e.ServerError), - (e.BatchDisabledError, "BatchDisabledError", AEROSPIKE_ERR_BATCH_DISABLED, e.ServerError), - (e.BatchMaxRequestError, "BatchMaxRequestError", AEROSPIKE_ERR_BATCH_MAX_REQUESTS_EXCEEDED, e.ServerError), - (e.BatchQueueFullError, "BatchQueueFullError", AEROSPIKE_ERR_BATCH_QUEUES_FULL, e.ServerError), - (e.QueryAbortedError, "QueryAbortedError", AEROSPIKE_ERR_QUERY_ABORTED, e.ServerError), - ), -) -def test_error_codes(error, error_name, error_code, base): - with pytest.raises(error) as test_error: - raise error - - test_error = test_error.value - - assert test_error.code == error_code - assert type(test_error).__name__ == error_name - assert issubclass(type(test_error), base) diff --git a/test/new_tests/test_exceptions.py b/test/new_tests/test_exceptions.py new file mode 100644 index 000000000..931a7f831 --- /dev/null +++ b/test/new_tests/test_exceptions.py @@ -0,0 +1,165 @@ +import pytest +from aerospike import exception as e +from .as_errors import ( + AEROSPIKE_ERR_ASYNC_CONNECTION, + AEROSPIKE_ERR_BATCH_DISABLED, + AEROSPIKE_ERR_BATCH_MAX_REQUESTS_EXCEEDED, + AEROSPIKE_ERR_BATCH_QUEUES_FULL, + AEROSPIKE_ERR_CLIENT_ABORT, + AEROSPIKE_ERR_FAIL_ELEMENT_EXISTS, + AEROSPIKE_ERR_FAIL_ELEMENT_NOT_FOUND, + AEROSPIKE_ERR_INVALID_NODE, + AEROSPIKE_ERR_NO_MORE_CONNECTIONS, + AEROSPIKE_ERR_QUERY_ABORTED, + AEROSPIKE_ERR_SCAN_ABORTED, + AEROSPIKE_ERR_TLS_ERROR, +) +from typing import Optional + +# Used to test inherited attributes (can be from indirect parent) +base_class_to_attrs = { + e.AerospikeError: [ + "code", + "msg", + "file", + "line", + # in_doubt is only added when an AerospikeError or subclass of it is raised by the client + # This attribute is not set when initializing the exception classes in `aerospike.exception` + # "in_doubt" + ], + e.RecordError: [ + "key", + "bin" + ], + e.IndexError: [ + "name" + ], + e.UDFError: [ + "module", + "func" + ] +} + + +# TODO: add missing type stubs +# TODO: make sure other places in the tests aren't doing the same thing as here. +# We'll do this by cleaning up the test code. But it's nice to test the API in one place +# TODO: add documentation for the tests in a README +@pytest.mark.parametrize( + "exc_class, expected_exc_name, expected_error_code, expected_exc_base_class", + ( + # Don't test error_name fields that are set to None + # The exception name should be the class name anyways... + (e.AerospikeError, None, None, Exception), + (e.ClientError, None, -1, e.AerospikeError), + # Client errors + (e.InvalidHostError, None, -4, e.ClientError), + (e.ParamError, None, -2, e.ClientError), + (e.MaxErrorRateExceeded, None, -14, e.ClientError), + (e.MaxRetriesExceeded, None, -12, e.ClientError), + (e.NoResponse, None, -15, e.ClientError), + (e.BatchFailed, None, -16, e.ClientError), + (e.ConnectionError, None, -10, e.ClientError), + (e.TLSError, "TLSError", AEROSPIKE_ERR_TLS_ERROR, e.ClientError), + (e.InvalidNodeError, "InvalidNodeError", AEROSPIKE_ERR_INVALID_NODE, e.ClientError), + (e.NoMoreConnectionsError, "NoMoreConnectionsError", AEROSPIKE_ERR_NO_MORE_CONNECTIONS, e.ClientError), + (e.AsyncConnectionError, "AsyncConnectionError", AEROSPIKE_ERR_ASYNC_CONNECTION, e.ClientError), + (e.ClientAbortError, "ClientAbortError", AEROSPIKE_ERR_CLIENT_ABORT, e.ClientError), + # Server errors + (e.ServerError, None, 1, e.AerospikeError), + (e.InvalidRequest, None, 4, e.ServerError), + (e.OpNotApplicable, None, 26, e.ServerError), + (e.FilteredOut, None, 27, e.ServerError), + (e.ServerFull, None, 8, e.ServerError), + (e.AlwaysForbidden, None, 10, e.ServerError), + (e.UnsupportedFeature, None, 16, e.ServerError), + (e.DeviceOverload, None, 18, e.ServerError), + (e.NamespaceNotFound, None, 20, e.ServerError), + (e.ForbiddenError, None, 22, e.ServerError), + (e.LostConflict, None, 28, e.ServerError), + (e.InvalidGeoJSON, None, 160, e.ServerError), + (e.ScanAbortedError, "ScanAbortedError", AEROSPIKE_ERR_SCAN_ABORTED, e.ServerError), + (e.ElementNotFoundError, "ElementNotFoundError", AEROSPIKE_ERR_FAIL_ELEMENT_NOT_FOUND, e.ServerError), + (e.ElementExistsError, "ElementExistsError", AEROSPIKE_ERR_FAIL_ELEMENT_EXISTS, e.ServerError), + (e.BatchDisabledError, "BatchDisabledError", AEROSPIKE_ERR_BATCH_DISABLED, e.ServerError), + (e.BatchMaxRequestError, "BatchMaxRequestError", AEROSPIKE_ERR_BATCH_MAX_REQUESTS_EXCEEDED, e.ServerError), + (e.BatchQueueFullError, "BatchQueueFullError", AEROSPIKE_ERR_BATCH_QUEUES_FULL, e.ServerError), + (e.QueryAbortedError, "QueryAbortedError", AEROSPIKE_ERR_QUERY_ABORTED, e.ServerError), + # Record errors + (e.RecordError, None, None, e.ServerError), + (e.RecordKeyMismatch, None, 19, e.RecordError), + (e.RecordNotFound, None, 2, e.RecordError), + (e.RecordGenerationError, None, 3, e.RecordError), + (e.RecordExistsError, None, 5, e.RecordError), + (e.RecordBusy, None, 14, e.RecordError), + (e.RecordTooBig, None, 13, e.RecordError), + (e.BinNameError, None, 21, e.RecordError), + (e.BinIncompatibleType, None, 12, e.RecordError), + (e.BinNotFound, None, 17, e.RecordError), + (e.BinExistsError, None, 6, e.RecordError), + # Index errors + (e.IndexError, None, 204, e.ServerError), + (e.IndexNotFound, None, 201, e.IndexError), + (e.IndexFoundError, None, 200, e.IndexError), + (e.IndexOOM, None, 202, e.IndexError), + (e.IndexNotReadable, None, 203, e.IndexError), + (e.IndexNameMaxLen, None, 205, e.IndexError), + (e.IndexNameMaxCount, None, 206, e.IndexError), + # Query errors + (e.QueryError, None, 213, e.ServerError), + (e.QueryQueueFull, None, 211, e.QueryError), + (e.QueryTimeout, None, 212, e.QueryError), + # Cluster errors + (e.ClusterError, None, 11, e.ServerError), + (e.ClusterChangeError, None, 7, e.ClusterError), + # Admin errors + (e.AdminError, None, None, e.ServerError), + (e.ExpiredPassword, None, 63, e.AdminError), + (e.ForbiddenPassword, None, 64, e.AdminError), + (e.IllegalState, None, 56, e.AdminError), + (e.InvalidCommand, None, 54, e.AdminError), + (e.InvalidCredential, None, 65, e.AdminError), + (e.InvalidField, None, 55, e.AdminError), + (e.InvalidPassword, None, 62, e.AdminError), + (e.InvalidPrivilege, None, 72, e.AdminError), + (e.InvalidRole, None, 70, e.AdminError), + (e.InvalidUser, None, 60, e.AdminError), + (e.QuotasNotEnabled, None, 74, e.AdminError), + (e.QuotaExceeded, None, 83, e.AdminError), + (e.InvalidQuota, None, 75, e.AdminError), + (e.NotWhitelisted, None, 82, e.AdminError), + (e.InvalidWhitelist, None, 73, e.AdminError), + (e.NotAuthenticated, None, 80, e.AdminError), + (e.RoleExistsError, None, 71, e.AdminError), + (e.RoleViolation, None, 81, e.AdminError), + (e.SecurityNotEnabled, None, 52, e.AdminError), + (e.SecurityNotSupported, None, 51, e.AdminError), + (e.SecuritySchemeNotSupported, None, 53, e.AdminError), + (e.UserExistsError, None, 61, e.AdminError), + # UDF errors + (e.UDFError, None, 100, e.ServerError), + (e.UDFNotFound, None, 1301, e.UDFError), + (e.LuaFileNotFound, None, 1302, e.UDFError), + ), +) +def test_aerospike_exceptions( + exc_class: type, + expected_exc_name: Optional[str], + expected_error_code: Optional[int], + expected_exc_base_class: type +): + with pytest.raises(exc_class) as excinfo: + raise exc_class + + assert excinfo.value.code == expected_error_code + + if expected_exc_name is not None: + assert excinfo.type.__name__ == expected_exc_name + + # Test directly inherited class + assert expected_exc_base_class in excinfo.type.__bases__ + + for base_class in base_class_to_attrs: + if issubclass(excinfo.type, base_class): + for attr in base_class_to_attrs[base_class]: + assert hasattr(excinfo.value, attr) From 79509c5bb00a74589ac612102c48d1416377de6d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 20:41:03 +0000 Subject: [PATCH 17/69] Auto-bump version to 15.0.1rc5.dev3 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 18dcf57b3..38abacc1f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.0.1rc5.dev2 +15.0.1rc5.dev3 From 0dad411e57b0b1fbaa7c2b2934600b4eefd53bd9 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 16 Sep 2024 09:50:56 -0700 Subject: [PATCH 18/69] [CLIENT-3118] Fix failing tests when running against server 7.2 (#671) --- test/new_tests/test_index.py | 7 +++++-- test/new_tests/test_mapkeys_index.py | 9 ++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/test/new_tests/test_index.py b/test/new_tests/test_index.py index abcef7283..c8c8fc19d 100644 --- a/test/new_tests/test_index.py +++ b/test/new_tests/test_index.py @@ -245,11 +245,14 @@ def test_create_string_index_with_correct_parameters_ns_length_extra(self): ns_name = "a" * 50 policy = {} - with pytest.raises(e.InvalidRequest) as err_info: + with pytest.raises((e.InvalidRequest, e.NamespaceNotFound)) as err_info: self.as_connection.index_string_create(ns_name, "demo", "name", "name_index", policy) err_code = err_info.value.code - assert err_code is AerospikeStatus.AEROSPIKE_ERR_REQUEST_INVALID + if (TestBaseClass.major_ver, TestBaseClass.minor_ver) >= (7, 2): + assert err_code is AerospikeStatus.AEROSPIKE_ERR_NAMESPACE_NOT_FOUND + else: + assert err_code is AerospikeStatus.AEROSPIKE_ERR_REQUEST_INVALID def test_create_string_index_with_incorrect_namespace(self): """ diff --git a/test/new_tests/test_mapkeys_index.py b/test/new_tests/test_mapkeys_index.py index 469e42f72..51cf307c3 100644 --- a/test/new_tests/test_mapkeys_index.py +++ b/test/new_tests/test_mapkeys_index.py @@ -121,16 +121,19 @@ def test_mapkeysindex_with_correct_parameters_string_on_numerickeys(self): ), ids=("ns too long", "set too long", "bin too long", "index name too long"), ) - def test_mapkeys_with_parameters_too_long(self, ns, test_set, test_bin, index_name): + def test_mapkeys_with_parameters_too_long(self, ns, test_set, test_bin, index_name, request): # Invoke index_map_keys_create() with correct arguments and set # length extra policy = {} - with pytest.raises(e.InvalidRequest) as err_info: + with pytest.raises((e.InvalidRequest, e.NamespaceNotFound)) as err_info: self.as_connection.index_map_keys_create(ns, test_set, test_bin, aerospike.INDEX_STRING, index_name, policy) err_code = err_info.value.code - assert err_code == AerospikeStatus.AEROSPIKE_ERR_REQUEST_INVALID + if request.node.callspec.id == "ns too long" and (TestBaseClass.major_ver, TestBaseClass.minor_ver) >= (7, 2): + assert err_code is AerospikeStatus.AEROSPIKE_ERR_NAMESPACE_NOT_FOUND + else: + assert err_code is AerospikeStatus.AEROSPIKE_ERR_REQUEST_INVALID def test_mapkeysindex_with_incorrect_namespace(self): """ From 5afa75e2491ea8c177f3892711bb06a50292320c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 16:51:31 +0000 Subject: [PATCH 19/69] Auto-bump version to 15.0.1rc5.dev4 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 38abacc1f..f1f3dc8bb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.0.1rc5.dev3 +15.0.1rc5.dev4 From 45ad62a493c8739053996a4a314533768c93c924 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 16 Sep 2024 10:57:32 -0700 Subject: [PATCH 20/69] [CLIENT-3106] Fix reference count error caused by not properly incrementing reference count for Py_None object (#672) --- src/main/exception.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/exception.c b/src/main/exception.c index d1dcd5ec7..4f9a7ab0d 100644 --- a/src/main/exception.c +++ b/src/main/exception.c @@ -321,6 +321,7 @@ PyObject *AerospikeException_New(void) PyObject *py_code = NULL; if (exception_def.code == NO_ERROR_CODE) { + Py_INCREF(Py_None); py_code = Py_None; } else { From bcd9f46cc92b6f5155fef9439a3f026ebc3c3164 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 17:58:10 +0000 Subject: [PATCH 21/69] Auto-bump version to 15.0.1rc5.dev5 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f1f3dc8bb..b7034f93d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.0.1rc5.dev4 +15.0.1rc5.dev5 From 25ac48919fdd170800086f342114db32a1721b93 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 16 Sep 2024 16:01:50 -0700 Subject: [PATCH 22/69] [CLIENT-3099] Add support for error code 32 (XDR key busy) (#662) Fix test for metrics node close listener --- .github/workflows/tests.yml | 2 +- aerospike-client-c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c8bb7801a..c308d2301 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -422,7 +422,7 @@ jobs: python-version: ${{ env.LOWEST_SUPPORTED_PY_VERSION }} architecture: 'x64' - name: Download aerolab - run: wget https://github.com/aerospike/aerolab/releases/download/7.6.0-7b5bbde/aerolab-linux-amd64-7.6.0.deb + run: wget https://github.com/aerospike/aerolab/releases/download/7.5.2/aerolab-linux-amd64-7.5.2.deb - name: Install aerolab run: sudo dpkg -i *.deb - name: Tell aerolab to use Docker diff --git a/aerospike-client-c b/aerospike-client-c index 54e250756..9161e5a22 160000 --- a/aerospike-client-c +++ b/aerospike-client-c @@ -1 +1 @@ -Subproject commit 54e25075646e815f023b5923603d35fa9d9b8f00 +Subproject commit 9161e5a22b2bf5aa7736a7533434a2a03e9ebe50 From 6b013db1570e71f64c5932c34e3e6a3a7bcf8f9e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 23:02:26 +0000 Subject: [PATCH 23/69] Auto-bump version to 15.0.1rc5.dev6 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b7034f93d..eae2fc2f9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.0.1rc5.dev5 +15.0.1rc5.dev6 From d200c6dd3d97bc36e102140db47d0957702d267c Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 17 Sep 2024 09:02:33 -0700 Subject: [PATCH 24/69] [CLIENT-3117] Replace an existing node in the cluster when a new peer has the same node name, but a different IP address (#673) --- aerospike-client-c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aerospike-client-c b/aerospike-client-c index 9161e5a22..0fabfa7f5 160000 --- a/aerospike-client-c +++ b/aerospike-client-c @@ -1 +1 @@ -Subproject commit 9161e5a22b2bf5aa7736a7533434a2a03e9ebe50 +Subproject commit 0fabfa7f52d7ee28b7ae80a621b791b9ec9e2fe8 From cebc1522174ffa6be66862ea806058e3cf9f0412 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 16:03:14 +0000 Subject: [PATCH 25/69] Auto-bump version to 15.0.1rc5.dev7 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index eae2fc2f9..c8805c101 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.0.1rc5.dev6 +15.0.1rc5.dev7 From e05acbb94838cf4376df85cc8e5dc97b2dd09a66 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:22:52 +0000 Subject: [PATCH 26/69] Auto-bump version to 15.0.1rc5 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c8805c101..718cc0025 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.0.1rc5.dev7 +15.0.1rc5 From 6cc80c3b5bae0164fd76eb8d6c5734e2363d4536 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:34:52 +0000 Subject: [PATCH 27/69] Auto-bump version to 15.1.0rc0 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 718cc0025..bc16935e5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.0.1rc5 +15.1.0rc0 From 50a2c88754b596818bc1cb5e357683c01bab7455 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:35:50 +0000 Subject: [PATCH 28/69] Auto-bump version to 15.1.0rc1.dev0 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index bc16935e5..379dc31a1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.1.0rc0 +15.1.0rc1.dev0 From f0bc93bdee70da54366d370ef241826c1bebe5a6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:37:56 +0000 Subject: [PATCH 29/69] Auto-bump version to 15.1.0rc1 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 379dc31a1..e844ba5f5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.1.0rc1.dev0 +15.1.0rc1 From c0c805ae938b40b21001df192a1fb3efbd614457 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 16:33:33 +0000 Subject: [PATCH 30/69] Auto-bump version to 15.1.0 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e844ba5f5..d14dfbac3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.1.0rc1 +15.1.0 From 8ff03277e308396ff1cfeb27593256b7645edda6 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 25 Sep 2024 15:36:32 -0700 Subject: [PATCH 31/69] [CLIENT-3106] Refactor aerospike module initialization code and check if error indicator is set after every C-API call (#667) - Consolidate the code that adds the aerospike's module constants into the same file and array - Remove aerospike_log_level enum because an enum already exists in C client's "common" submodule - Remove aerospike module state since the module itself already has the same members as Python attributes. There is no reason to store them twice as Python attributes and in the module state --- src/include/log.h | 20 - src/include/policy.h | 24 -- src/include/types.h | 3 + src/main/aerospike.c | 677 +++++++++++++++++++++++-------- src/main/client/type.c | 37 +- src/main/geospatial/type.c | 7 +- src/main/key_ordered_dict/type.c | 3 +- src/main/log.c | 28 +- src/main/nullobject/type.c | 6 +- src/main/policy.c | 363 ----------------- src/main/query/type.c | 7 +- src/main/scan/type.c | 6 +- 12 files changed, 554 insertions(+), 627 deletions(-) diff --git a/src/include/log.h b/src/include/log.h index 9e8c8209f..fa87b13cd 100644 --- a/src/include/log.h +++ b/src/include/log.h @@ -19,20 +19,6 @@ #include #include -/* - * Enum to declare log level constants - */ -typedef enum Aerospike_log_level_e { - - LOG_LEVEL_OFF = -1, - LOG_LEVEL_ERROR, - LOG_LEVEL_WARN, - LOG_LEVEL_INFO, - LOG_LEVEL_DEBUG, - LOG_LEVEL_TRACE - -} aerospike_log_level; - /* * Structure to hold user's log_callback object */ @@ -40,12 +26,6 @@ typedef struct Aerospike_log_callback { PyObject *callback; } AerospikeLogCallback; -/** - * Add log level constants to aerospike module - * aerospike.set_log_level(aerospike.LOG_LEVEL_DEBUG) - */ -as_status declare_log_constants(PyObject *aerospike); - /** * Set log level for C-SDK * aerospike.set_log_level( aerospike.LOG_LEVEL_WARN ) diff --git a/src/include/policy.h b/src/include/policy.h index 3f594f43d..0ba3135ad 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -29,14 +29,6 @@ #include #include -#define MAX_CONSTANT_STR_SIZE 512 - -/* - ******************************************************************************************************* - *Structure to map constant number to constant name string for Aerospike constants. - ******************************************************************************************************* - */ - enum Aerospike_serializer_values { SERIALIZER_NONE, /* default handler for serializer type */ SERIALIZER_PYTHON, @@ -193,20 +185,6 @@ enum aerospike_cdt_ctx_identifiers { CDT_CTX_MAP_KEY_CREATE = 0x24 }; -typedef struct Aerospike_Constants { - long constantno; - char constant_str[MAX_CONSTANT_STR_SIZE]; -} AerospikeConstants; - -typedef struct Aerospike_JobConstants { - char job_str[MAX_CONSTANT_STR_SIZE]; - char exposed_job_str[MAX_CONSTANT_STR_SIZE]; -} AerospikeJobConstants; -#define AEROSPIKE_CONSTANTS_ARR_SIZE \ - (sizeof(aerospike_constants) / sizeof(AerospikeConstants)) -#define AEROSPIKE_JOB_CONSTANTS_ARR_SIZE \ - (sizeof(aerospike_job_constants) / sizeof(AerospikeJobConstants)) - as_status pyobject_to_policy_admin(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_admin *policy, as_policy_admin **policy_p, @@ -270,8 +248,6 @@ as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, as_status pyobject_to_map_policy(as_error *err, PyObject *py_policy, as_map_policy *policy); -as_status declare_policy_constants(PyObject *aerospike); - void set_scan_options(as_error *err, as_scan *scan_p, PyObject *py_options); as_status set_query_options(as_error *err, PyObject *query_options, diff --git a/src/include/types.h b/src/include/types.h index 125dd3e6f..33640b3f9 100644 --- a/src/include/types.h +++ b/src/include/types.h @@ -27,6 +27,9 @@ #include #include "pool.h" +#define AEROSPIKE_MODULE_NAME "aerospike" +#define FULLY_QUALIFIED_TYPE_NAME(name) AEROSPIKE_MODULE_NAME "." name + // Bin names can be of type Unicode in Python // DB supports 32767 maximum number of bins #define MAX_UNICODE_OBJECTS 32767 diff --git a/src/main/aerospike.c b/src/main/aerospike.c index 2dc02816b..360b4318c 100644 --- a/src/main/aerospike.c +++ b/src/main/aerospike.c @@ -28,12 +28,17 @@ #include "exceptions.h" #include "policy.h" #include "log.h" -#include #include "serializer.h" #include "module_functions.h" #include "nullobject.h" #include "cdt_types.h" + +#include #include +#include +#include +#include +#include PyObject *py_global_hosts; int counter = 0xA8000000; @@ -50,7 +55,7 @@ config = {\n\ }\n\ client = aerospike.client(config)"); -static PyMethodDef Aerospike_Methods[] = { +static PyMethodDef aerospike_methods[] = { //Serialization {"set_serializer", (PyCFunction)AerospikeClient_Set_Serializer, @@ -83,187 +88,535 @@ static PyMethodDef Aerospike_Methods[] = { {NULL}}; -static AerospikeConstants operator_constants[] = { - {AS_OPERATOR_READ, "OPERATOR_READ"}, - {AS_OPERATOR_WRITE, "OPERATOR_WRITE"}, - {AS_OPERATOR_INCR, "OPERATOR_INCR"}, - {AS_OPERATOR_APPEND, "OPERATOR_APPEND"}, - {AS_OPERATOR_PREPEND, "OPERATOR_PREPEND"}, - {AS_OPERATOR_TOUCH, "OPERATOR_TOUCH"}, - {AS_OPERATOR_DELETE, "OPERATOR_DELETE"}}; - -#define OPERATOR_CONSTANTS_ARR_SIZE \ - (sizeof(operator_constants) / sizeof(AerospikeConstants)) - -static AerospikeConstants auth_mode_constants[] = { - {AS_AUTH_INTERNAL, "AUTH_INTERNAL"}, - {AS_AUTH_EXTERNAL, "AUTH_EXTERNAL"}, - {AS_AUTH_EXTERNAL_INSECURE, "AUTH_EXTERNAL_INSECURE"}, - {AS_AUTH_PKI, "AUTH_PKI"}}; - -#define AUTH_MODE_CONSTANTS_ARR_SIZE \ - (sizeof(auth_mode_constants) / sizeof(AerospikeConstants)) - -struct Aerospike_State { - PyObject *exception; - PyTypeObject *client; - PyTypeObject *query; - PyTypeObject *scan; - PyTypeObject *kdict; - PyObject *predicates; - PyTypeObject *geospatial; - PyTypeObject *null_object; - PyTypeObject *wildcard_object; - PyTypeObject *infinite_object; +struct module_constant_name_to_value { + const char *name; + // If false, is int value + bool is_str_value; + union value { + long integer; + const char *string; + } value; }; -#define Aerospike_State(o) ((struct Aerospike_State *)PyModule_GetState(o)) - -static int Aerospike_Clear(PyObject *aerospike) -{ - Py_CLEAR(Aerospike_State(aerospike)->exception); - Py_CLEAR(Aerospike_State(aerospike)->client); - Py_CLEAR(Aerospike_State(aerospike)->query); - Py_CLEAR(Aerospike_State(aerospike)->scan); - Py_CLEAR(Aerospike_State(aerospike)->kdict); - Py_CLEAR(Aerospike_State(aerospike)->predicates); - Py_CLEAR(Aerospike_State(aerospike)->geospatial); - Py_CLEAR(Aerospike_State(aerospike)->null_object); - Py_CLEAR(Aerospike_State(aerospike)->wildcard_object); - Py_CLEAR(Aerospike_State(aerospike)->infinite_object); - - return 0; -} - -PyMODINIT_FUNC PyInit_aerospike(void) -{ - // Makes things "thread-safe" - Py_Initialize(); - int i = 0; - - static struct PyModuleDef moduledef = {PyModuleDef_HEAD_INIT, - "aerospike", - "Aerospike Python Client", - sizeof(struct Aerospike_State), - Aerospike_Methods, - NULL, - NULL, - Aerospike_Clear}; - - PyObject *aerospike = PyModule_Create(&moduledef); +// TODO: many of these names are the same as the enum name +// Is there a way to generate this code? +// TODO: regression tests for all these constants +static struct module_constant_name_to_value module_constants[] = { + {"OPERATOR_READ", .value.integer = AS_OPERATOR_READ}, + {"OPERATOR_WRITE", .value.integer = AS_OPERATOR_WRITE}, + {"OPERATOR_INCR", .value.integer = AS_OPERATOR_INCR}, + {"OPERATOR_APPEND", .value.integer = AS_OPERATOR_APPEND}, + {"OPERATOR_PREPEND", .value.integer = AS_OPERATOR_PREPEND}, + {"OPERATOR_TOUCH", .value.integer = AS_OPERATOR_TOUCH}, + {"OPERATOR_DELETE", .value.integer = AS_OPERATOR_DELETE}, + + {"AUTH_INTERNAL", .value.integer = AS_AUTH_INTERNAL}, + {"AUTH_EXTERNAL", .value.integer = AS_AUTH_EXTERNAL}, + {"AUTH_EXTERNAL_INSECURE", .value.integer = AS_AUTH_EXTERNAL_INSECURE}, + {"AUTH_PKI", .value.integer = AS_AUTH_PKI}, + + {"POLICY_RETRY_NONE", .value.integer = AS_POLICY_RETRY_NONE}, + {"POLICY_RETRY_ONCE", .value.integer = AS_POLICY_RETRY_ONCE}, + + {"POLICY_EXISTS_IGNORE", .value.integer = AS_POLICY_EXISTS_IGNORE}, + {"POLICY_EXISTS_CREATE", .value.integer = AS_POLICY_EXISTS_CREATE}, + {"POLICY_EXISTS_UPDATE", .value.integer = AS_POLICY_EXISTS_UPDATE}, + {"POLICY_EXISTS_REPLACE", .value.integer = AS_POLICY_EXISTS_REPLACE}, + {"POLICY_EXISTS_CREATE_OR_REPLACE", + .value.integer = AS_POLICY_EXISTS_CREATE_OR_REPLACE}, + + {"UDF_TYPE_LUA", .value.integer = AS_UDF_TYPE_LUA}, + + {"POLICY_KEY_DIGEST", .value.integer = AS_POLICY_KEY_DIGEST}, + {"POLICY_KEY_SEND", .value.integer = AS_POLICY_KEY_SEND}, + {"POLICY_GEN_IGNORE", .value.integer = AS_POLICY_GEN_IGNORE}, + {"POLICY_GEN_EQ", .value.integer = AS_POLICY_GEN_EQ}, + {"POLICY_GEN_GT", .value.integer = AS_POLICY_GEN_GT}, + + {"JOB_STATUS_COMPLETED", .value.integer = AS_JOB_STATUS_COMPLETED}, + {"JOB_STATUS_UNDEF", .value.integer = AS_JOB_STATUS_UNDEF}, + {"JOB_STATUS_INPROGRESS", .value.integer = AS_JOB_STATUS_INPROGRESS}, + + {"POLICY_REPLICA_MASTER", .value.integer = AS_POLICY_REPLICA_MASTER}, + {"POLICY_REPLICA_ANY", .value.integer = AS_POLICY_REPLICA_ANY}, + {"POLICY_REPLICA_SEQUENCE", .value.integer = AS_POLICY_REPLICA_SEQUENCE}, + {"POLICY_REPLICA_PREFER_RACK", + .value.integer = AS_POLICY_REPLICA_PREFER_RACK}, + + {"POLICY_COMMIT_LEVEL_ALL", .value.integer = AS_POLICY_COMMIT_LEVEL_ALL}, + {"POLICY_COMMIT_LEVEL_MASTER", + .value.integer = AS_POLICY_COMMIT_LEVEL_MASTER}, + + {"SERIALIZER_USER", .value.integer = SERIALIZER_USER}, + {"SERIALIZER_JSON", .value.integer = SERIALIZER_JSON}, + {"SERIALIZER_NONE", .value.integer = SERIALIZER_NONE}, + + {"INTEGER", .value.integer = SEND_BOOL_AS_INTEGER}, + {"AS_BOOL", .value.integer = SEND_BOOL_AS_AS_BOOL}, + + {"INDEX_STRING", .value.integer = AS_INDEX_STRING}, + {"INDEX_NUMERIC", .value.integer = AS_INDEX_NUMERIC}, + {"INDEX_GEO2DSPHERE", .value.integer = AS_INDEX_GEO2DSPHERE}, + {"INDEX_BLOB", .value.integer = AS_INDEX_BLOB}, + {"INDEX_TYPE_DEFAULT", .value.integer = AS_INDEX_TYPE_DEFAULT}, + {"INDEX_TYPE_LIST", .value.integer = AS_INDEX_TYPE_LIST}, + {"INDEX_TYPE_MAPKEYS", .value.integer = AS_INDEX_TYPE_MAPKEYS}, + {"INDEX_TYPE_MAPVALUES", .value.integer = AS_INDEX_TYPE_MAPVALUES}, + + {"PRIV_USER_ADMIN", .value.integer = AS_PRIVILEGE_USER_ADMIN}, + {"PRIV_SYS_ADMIN", .value.integer = AS_PRIVILEGE_SYS_ADMIN}, + {"PRIV_DATA_ADMIN", .value.integer = AS_PRIVILEGE_DATA_ADMIN}, + {"PRIV_READ", .value.integer = AS_PRIVILEGE_READ}, + {"PRIV_WRITE", .value.integer = AS_PRIVILEGE_WRITE}, + {"PRIV_READ_WRITE", .value.integer = AS_PRIVILEGE_READ_WRITE}, + {"PRIV_READ_WRITE_UDF", .value.integer = AS_PRIVILEGE_READ_WRITE_UDF}, + {"PRIV_TRUNCATE", .value.integer = AS_PRIVILEGE_TRUNCATE}, + {"PRIV_UDF_ADMIN", .value.integer = AS_PRIVILEGE_UDF_ADMIN}, + {"PRIV_SINDEX_ADMIN", .value.integer = AS_PRIVILEGE_SINDEX_ADMIN}, + + // TODO: If only aerospike_helpers relies on these constants, + // maybe move these constants to aerospike_helpers package + {"OP_LIST_APPEND", .value.integer = OP_LIST_APPEND}, + {"OP_LIST_APPEND_ITEMS", .value.integer = OP_LIST_APPEND_ITEMS}, + {"OP_LIST_INSERT", .value.integer = OP_LIST_INSERT}, + {"OP_LIST_INSERT_ITEMS", .value.integer = OP_LIST_INSERT_ITEMS}, + {"OP_LIST_POP", .value.integer = OP_LIST_POP}, + {"OP_LIST_POP_RANGE", .value.integer = OP_LIST_POP_RANGE}, + {"OP_LIST_REMOVE", .value.integer = OP_LIST_REMOVE}, + {"OP_LIST_REMOVE_RANGE", .value.integer = OP_LIST_REMOVE_RANGE}, + {"OP_LIST_CLEAR", .value.integer = OP_LIST_CLEAR}, + {"OP_LIST_SET", .value.integer = OP_LIST_SET}, + {"OP_LIST_GET", .value.integer = OP_LIST_GET}, + {"OP_LIST_GET_RANGE", .value.integer = OP_LIST_GET_RANGE}, + {"OP_LIST_TRIM", .value.integer = OP_LIST_TRIM}, + {"OP_LIST_SIZE", .value.integer = OP_LIST_SIZE}, + {"OP_LIST_INCREMENT", .value.integer = OP_LIST_INCREMENT}, + /* New CDT Operations, post 3.16.0.1 */ + {"OP_LIST_GET_BY_INDEX", .value.integer = OP_LIST_GET_BY_INDEX}, + {"OP_LIST_GET_BY_INDEX_RANGE", .value.integer = OP_LIST_GET_BY_INDEX_RANGE}, + {"OP_LIST_GET_BY_RANK", .value.integer = OP_LIST_GET_BY_RANK}, + {"OP_LIST_GET_BY_RANK_RANGE", .value.integer = OP_LIST_GET_BY_RANK_RANGE}, + {"OP_LIST_GET_BY_VALUE", .value.integer = OP_LIST_GET_BY_VALUE}, + {"OP_LIST_GET_BY_VALUE_LIST", .value.integer = OP_LIST_GET_BY_VALUE_LIST}, + {"OP_LIST_GET_BY_VALUE_RANGE", .value.integer = OP_LIST_GET_BY_VALUE_RANGE}, + {"OP_LIST_REMOVE_BY_INDEX", .value.integer = OP_LIST_REMOVE_BY_INDEX}, + {"OP_LIST_REMOVE_BY_INDEX_RANGE", + .value.integer = OP_LIST_REMOVE_BY_INDEX_RANGE}, + {"OP_LIST_REMOVE_BY_RANK", .value.integer = OP_LIST_REMOVE_BY_RANK}, + {"OP_LIST_REMOVE_BY_RANK_RANGE", + .value.integer = OP_LIST_REMOVE_BY_RANK_RANGE}, + {"OP_LIST_REMOVE_BY_VALUE", .value.integer = OP_LIST_REMOVE_BY_VALUE}, + {"OP_LIST_REMOVE_BY_VALUE_LIST", + .value.integer = OP_LIST_REMOVE_BY_VALUE_LIST}, + {"OP_LIST_REMOVE_BY_VALUE_RANGE", + .value.integer = OP_LIST_REMOVE_BY_VALUE_RANGE}, + {"OP_LIST_SET_ORDER", .value.integer = OP_LIST_SET_ORDER}, + {"OP_LIST_SORT", .value.integer = OP_LIST_SORT}, + { + "OP_LIST_REMOVE_BY_VALUE_RANK_RANGE_REL", + + .value.integer = OP_LIST_REMOVE_BY_VALUE_RANK_RANGE_REL, + }, + { + "OP_LIST_GET_BY_VALUE_RANK_RANGE_REL", + + .value.integer = OP_LIST_GET_BY_VALUE_RANK_RANGE_REL, + }, + {"OP_LIST_CREATE", .value.integer = OP_LIST_CREATE}, + { + "OP_LIST_GET_BY_VALUE_RANK_RANGE_REL_TO_END", + + .value.integer = OP_LIST_GET_BY_VALUE_RANK_RANGE_REL_TO_END, + }, + {"OP_LIST_GET_BY_INDEX_RANGE_TO_END", + .value.integer = OP_LIST_GET_BY_INDEX_RANGE_TO_END}, + {"OP_LIST_GET_BY_RANK_RANGE_TO_END", + .value.integer = OP_LIST_GET_BY_RANK_RANGE_TO_END}, + {"OP_LIST_REMOVE_BY_REL_RANK_RANGE_TO_END", + .value.integer = OP_LIST_REMOVE_BY_REL_RANK_RANGE_TO_END}, + {"OP_LIST_REMOVE_BY_REL_RANK_RANGE", + .value.integer = OP_LIST_REMOVE_BY_REL_RANK_RANGE}, + {"OP_LIST_REMOVE_BY_INDEX_RANGE_TO_END", + .value.integer = OP_LIST_REMOVE_BY_INDEX_RANGE_TO_END}, + {"OP_LIST_REMOVE_BY_RANK_RANGE_TO_END", + .value.integer = OP_LIST_REMOVE_BY_RANK_RANGE_TO_END}, + + {"OP_MAP_SET_POLICY", .value.integer = OP_MAP_SET_POLICY}, + {"OP_MAP_CREATE", .value.integer = OP_MAP_CREATE}, + {"OP_MAP_PUT", .value.integer = OP_MAP_PUT}, + {"OP_MAP_PUT_ITEMS", .value.integer = OP_MAP_PUT_ITEMS}, + {"OP_MAP_INCREMENT", .value.integer = OP_MAP_INCREMENT}, + {"OP_MAP_DECREMENT", .value.integer = OP_MAP_DECREMENT}, + {"OP_MAP_SIZE", .value.integer = OP_MAP_SIZE}, + {"OP_MAP_CLEAR", .value.integer = OP_MAP_CLEAR}, + {"OP_MAP_REMOVE_BY_KEY", .value.integer = OP_MAP_REMOVE_BY_KEY}, + {"OP_MAP_REMOVE_BY_KEY_LIST", .value.integer = OP_MAP_REMOVE_BY_KEY_LIST}, + {"OP_MAP_REMOVE_BY_KEY_RANGE", .value.integer = OP_MAP_REMOVE_BY_KEY_RANGE}, + {"OP_MAP_REMOVE_BY_VALUE", .value.integer = OP_MAP_REMOVE_BY_VALUE}, + {"OP_MAP_REMOVE_BY_VALUE_LIST", + .value.integer = OP_MAP_REMOVE_BY_VALUE_LIST}, + {"OP_MAP_REMOVE_BY_VALUE_RANGE", + .value.integer = OP_MAP_REMOVE_BY_VALUE_RANGE}, + {"OP_MAP_REMOVE_BY_INDEX", .value.integer = OP_MAP_REMOVE_BY_INDEX}, + {"OP_MAP_REMOVE_BY_INDEX_RANGE", + .value.integer = OP_MAP_REMOVE_BY_INDEX_RANGE}, + {"OP_MAP_REMOVE_BY_RANK", .value.integer = OP_MAP_REMOVE_BY_RANK}, + {"OP_MAP_REMOVE_BY_RANK_RANGE", + .value.integer = OP_MAP_REMOVE_BY_RANK_RANGE}, + {"OP_MAP_GET_BY_KEY", .value.integer = OP_MAP_GET_BY_KEY}, + {"OP_MAP_GET_BY_KEY_RANGE", .value.integer = OP_MAP_GET_BY_KEY_RANGE}, + {"OP_MAP_GET_BY_VALUE", .value.integer = OP_MAP_GET_BY_VALUE}, + {"OP_MAP_GET_BY_VALUE_RANGE", .value.integer = OP_MAP_GET_BY_VALUE_RANGE}, + {"OP_MAP_GET_BY_INDEX", .value.integer = OP_MAP_GET_BY_INDEX}, + {"OP_MAP_GET_BY_INDEX_RANGE", .value.integer = OP_MAP_GET_BY_INDEX_RANGE}, + {"OP_MAP_GET_BY_RANK", .value.integer = OP_MAP_GET_BY_RANK}, + {"OP_MAP_GET_BY_RANK_RANGE", .value.integer = OP_MAP_GET_BY_RANK_RANGE}, + {"OP_MAP_GET_BY_VALUE_LIST", .value.integer = OP_MAP_GET_BY_VALUE_LIST}, + {"OP_MAP_GET_BY_KEY_LIST", .value.integer = OP_MAP_GET_BY_KEY_LIST}, + /* CDT operations for use with expressions, new in 5.0 */ + {"OP_MAP_REMOVE_BY_VALUE_RANK_RANGE_REL", + .value.integer = OP_MAP_REMOVE_BY_VALUE_RANK_RANGE_REL}, + {"OP_MAP_REMOVE_BY_KEY_INDEX_RANGE_REL", + .value.integer = OP_MAP_REMOVE_BY_KEY_INDEX_RANGE_REL}, + {"OP_MAP_GET_BY_VALUE_RANK_RANGE_REL", + .value.integer = OP_MAP_GET_BY_VALUE_RANK_RANGE_REL}, + {"OP_MAP_GET_BY_KEY_INDEX_RANGE_REL", + .value.integer = OP_MAP_GET_BY_KEY_INDEX_RANGE_REL}, + {"OP_MAP_REMOVE_BY_KEY_REL_INDEX_RANGE_TO_END", + .value.integer = OP_MAP_REMOVE_BY_KEY_REL_INDEX_RANGE_TO_END}, + {"OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE_TO_END", + .value.integer = OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE_TO_END}, + {"OP_MAP_REMOVE_BY_INDEX_RANGE_TO_END", + .value.integer = OP_MAP_REMOVE_BY_INDEX_RANGE_TO_END}, + {"OP_MAP_REMOVE_BY_RANK_RANGE_TO_END", + .value.integer = OP_MAP_REMOVE_BY_RANK_RANGE_TO_END}, + {"OP_MAP_GET_BY_KEY_REL_INDEX_RANGE_TO_END", + .value.integer = OP_MAP_GET_BY_KEY_REL_INDEX_RANGE_TO_END}, + {"OP_MAP_REMOVE_BY_KEY_REL_INDEX_RANGE", + .value.integer = OP_MAP_REMOVE_BY_KEY_REL_INDEX_RANGE}, + {"OP_MAP_REMOVE_BY_VALUE_REL_INDEX_RANGE", + .value.integer = OP_MAP_REMOVE_BY_VALUE_REL_INDEX_RANGE}, + {"OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE", + .value.integer = OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE}, + {"OP_MAP_GET_BY_KEY_REL_INDEX_RANGE", + .value.integer = OP_MAP_GET_BY_KEY_REL_INDEX_RANGE}, + {"OP_MAP_GET_BY_VALUE_RANK_RANGE_REL_TO_END", + .value.integer = OP_MAP_GET_BY_VALUE_RANK_RANGE_REL_TO_END}, + {"OP_MAP_GET_BY_INDEX_RANGE_TO_END", + .value.integer = OP_MAP_GET_BY_INDEX_RANGE_TO_END}, + {"OP_MAP_GET_BY_RANK_RANGE_TO_END", + .value.integer = OP_MAP_GET_BY_RANK_RANGE_TO_END}, + + {"MAP_UNORDERED", .value.integer = AS_MAP_UNORDERED}, + {"MAP_KEY_ORDERED", .value.integer = AS_MAP_KEY_ORDERED}, + {"MAP_KEY_VALUE_ORDERED", .value.integer = AS_MAP_KEY_VALUE_ORDERED}, + + {"MAP_RETURN_NONE", .value.integer = AS_MAP_RETURN_NONE}, + {"MAP_RETURN_INDEX", .value.integer = AS_MAP_RETURN_INDEX}, + {"MAP_RETURN_REVERSE_INDEX", .value.integer = AS_MAP_RETURN_REVERSE_INDEX}, + {"MAP_RETURN_RANK", .value.integer = AS_MAP_RETURN_RANK}, + {"MAP_RETURN_REVERSE_RANK", .value.integer = AS_MAP_RETURN_REVERSE_RANK}, + {"MAP_RETURN_COUNT", .value.integer = AS_MAP_RETURN_COUNT}, + {"MAP_RETURN_KEY", .value.integer = AS_MAP_RETURN_KEY}, + {"MAP_RETURN_VALUE", .value.integer = AS_MAP_RETURN_VALUE}, + {"MAP_RETURN_KEY_VALUE", .value.integer = AS_MAP_RETURN_KEY_VALUE}, + {"MAP_RETURN_EXISTS", .value.integer = AS_MAP_RETURN_EXISTS}, + {"MAP_RETURN_ORDERED_MAP", .value.integer = AS_MAP_RETURN_ORDERED_MAP}, + {"MAP_RETURN_UNORDERED_MAP", .value.integer = AS_MAP_RETURN_UNORDERED_MAP}, + + {"TTL_NAMESPACE_DEFAULT", .value.integer = AS_RECORD_DEFAULT_TTL}, + {"TTL_NEVER_EXPIRE", .value.integer = AS_RECORD_NO_EXPIRE_TTL}, + {"TTL_DONT_UPDATE", .value.integer = AS_RECORD_NO_CHANGE_TTL}, + {"TTL_CLIENT_DEFAULT", .value.integer = AS_RECORD_CLIENT_DEFAULT_TTL}, + + {"LIST_RETURN_NONE", .value.integer = AS_LIST_RETURN_NONE}, + {"LIST_RETURN_INDEX", .value.integer = AS_LIST_RETURN_INDEX}, + {"LIST_RETURN_REVERSE_INDEX", + .value.integer = AS_LIST_RETURN_REVERSE_INDEX}, + {"LIST_RETURN_RANK", .value.integer = AS_LIST_RETURN_RANK}, + {"LIST_RETURN_REVERSE_RANK", .value.integer = AS_LIST_RETURN_REVERSE_RANK}, + {"LIST_RETURN_COUNT", .value.integer = AS_LIST_RETURN_COUNT}, + {"LIST_RETURN_VALUE", .value.integer = AS_LIST_RETURN_VALUE}, + {"LIST_RETURN_EXISTS", .value.integer = AS_LIST_RETURN_EXISTS}, + + {"LIST_SORT_DROP_DUPLICATES", + .value.integer = AS_LIST_SORT_DROP_DUPLICATES}, + {"LIST_SORT_DEFAULT", .value.integer = AS_LIST_SORT_DEFAULT}, + + {"LIST_WRITE_DEFAULT", .value.integer = AS_LIST_WRITE_DEFAULT}, + {"LIST_WRITE_ADD_UNIQUE", .value.integer = AS_LIST_WRITE_ADD_UNIQUE}, + {"LIST_WRITE_INSERT_BOUNDED", + .value.integer = AS_LIST_WRITE_INSERT_BOUNDED}, + + {"LIST_ORDERED", .value.integer = AS_LIST_ORDERED}, + {"LIST_UNORDERED", .value.integer = AS_LIST_UNORDERED}, + + {"MAP_WRITE_NO_FAIL", .value.integer = AS_MAP_WRITE_NO_FAIL}, + {"MAP_WRITE_PARTIAL", .value.integer = AS_MAP_WRITE_PARTIAL}, + + {"LIST_WRITE_NO_FAIL", .value.integer = AS_LIST_WRITE_NO_FAIL}, + {"LIST_WRITE_PARTIAL", .value.integer = AS_LIST_WRITE_PARTIAL}, + + /* Map write flags post 3.5.0 */ + {"MAP_WRITE_FLAGS_DEFAULT", .value.integer = AS_MAP_WRITE_DEFAULT}, + {"MAP_WRITE_FLAGS_CREATE_ONLY", .value.integer = AS_MAP_WRITE_CREATE_ONLY}, + {"MAP_WRITE_FLAGS_UPDATE_ONLY", .value.integer = AS_MAP_WRITE_UPDATE_ONLY}, + {"MAP_WRITE_FLAGS_NO_FAIL", .value.integer = AS_MAP_WRITE_NO_FAIL}, + {"MAP_WRITE_FLAGS_PARTIAL", .value.integer = AS_MAP_WRITE_PARTIAL}, + + /* READ Mode constants 4.0.0 */ + + // AP Read Mode + {"POLICY_READ_MODE_AP_ONE", .value.integer = AS_POLICY_READ_MODE_AP_ONE}, + {"POLICY_READ_MODE_AP_ALL", .value.integer = AS_POLICY_READ_MODE_AP_ALL}, + + // SC Read Mode + {"POLICY_READ_MODE_SC_SESSION", + .value.integer = AS_POLICY_READ_MODE_SC_SESSION}, + {"POLICY_READ_MODE_SC_LINEARIZE", + .value.integer = AS_POLICY_READ_MODE_SC_LINEARIZE}, + {"POLICY_READ_MODE_SC_ALLOW_REPLICA", + .value.integer = AS_POLICY_READ_MODE_SC_ALLOW_REPLICA}, + {"POLICY_READ_MODE_SC_ALLOW_UNAVAILABLE", + .value.integer = AS_POLICY_READ_MODE_SC_ALLOW_UNAVAILABLE}, + + /* Bitwise constants: 3.9.0 */ + {"BIT_WRITE_DEFAULT", .value.integer = AS_BIT_WRITE_DEFAULT}, + {"BIT_WRITE_CREATE_ONLY", .value.integer = AS_BIT_WRITE_CREATE_ONLY}, + {"BIT_WRITE_UPDATE_ONLY", .value.integer = AS_BIT_WRITE_UPDATE_ONLY}, + {"BIT_WRITE_NO_FAIL", .value.integer = AS_BIT_WRITE_NO_FAIL}, + {"BIT_WRITE_PARTIAL", .value.integer = AS_BIT_WRITE_PARTIAL}, + + {"BIT_RESIZE_DEFAULT", .value.integer = AS_BIT_RESIZE_DEFAULT}, + {"BIT_RESIZE_FROM_FRONT", .value.integer = AS_BIT_RESIZE_FROM_FRONT}, + {"BIT_RESIZE_GROW_ONLY", .value.integer = AS_BIT_RESIZE_GROW_ONLY}, + {"BIT_RESIZE_SHRINK_ONLY", .value.integer = AS_BIT_RESIZE_SHRINK_ONLY}, + + {"BIT_OVERFLOW_FAIL", .value.integer = AS_BIT_OVERFLOW_FAIL}, + {"BIT_OVERFLOW_SATURATE", .value.integer = AS_BIT_OVERFLOW_SATURATE}, + {"BIT_OVERFLOW_WRAP", .value.integer = AS_BIT_OVERFLOW_WRAP}, + + /* BITWISE OPS: 3.9.0 */ + {"OP_BIT_INSERT", .value.integer = OP_BIT_INSERT}, + {"OP_BIT_RESIZE", .value.integer = OP_BIT_RESIZE}, + {"OP_BIT_REMOVE", .value.integer = OP_BIT_REMOVE}, + {"OP_BIT_SET", .value.integer = OP_BIT_SET}, + {"OP_BIT_OR", .value.integer = OP_BIT_OR}, + {"OP_BIT_XOR", .value.integer = OP_BIT_XOR}, + {"OP_BIT_AND", .value.integer = OP_BIT_AND}, + {"OP_BIT_NOT", .value.integer = OP_BIT_NOT}, + {"OP_BIT_LSHIFT", .value.integer = OP_BIT_LSHIFT}, + {"OP_BIT_RSHIFT", .value.integer = OP_BIT_RSHIFT}, + {"OP_BIT_ADD", .value.integer = OP_BIT_ADD}, + {"OP_BIT_SUBTRACT", .value.integer = OP_BIT_SUBTRACT}, + {"OP_BIT_GET_INT", .value.integer = OP_BIT_GET_INT}, + {"OP_BIT_SET_INT", .value.integer = OP_BIT_SET_INT}, + {"OP_BIT_GET", .value.integer = OP_BIT_GET}, + {"OP_BIT_COUNT", .value.integer = OP_BIT_COUNT}, + {"OP_BIT_LSCAN", .value.integer = OP_BIT_LSCAN}, + {"OP_BIT_RSCAN", .value.integer = OP_BIT_RSCAN}, + + /* Nested CDT constants: 3.9.0 */ + {"CDT_CTX_LIST_INDEX", .value.integer = AS_CDT_CTX_LIST_INDEX}, + {"CDT_CTX_LIST_RANK", .value.integer = AS_CDT_CTX_LIST_RANK}, + {"CDT_CTX_LIST_VALUE", .value.integer = AS_CDT_CTX_LIST_VALUE}, + {"CDT_CTX_LIST_INDEX_CREATE", .value.integer = CDT_CTX_LIST_INDEX_CREATE}, + {"CDT_CTX_MAP_INDEX", .value.integer = AS_CDT_CTX_MAP_INDEX}, + {"CDT_CTX_MAP_RANK", .value.integer = AS_CDT_CTX_MAP_RANK}, + {"CDT_CTX_MAP_KEY", .value.integer = AS_CDT_CTX_MAP_KEY}, + {"CDT_CTX_MAP_VALUE", .value.integer = AS_CDT_CTX_MAP_VALUE}, + {"CDT_CTX_MAP_KEY_CREATE", .value.integer = CDT_CTX_MAP_KEY_CREATE}, + + /* HLL constants 3.11.0 */ + {"OP_HLL_ADD", .value.integer = OP_HLL_ADD}, + {"OP_HLL_DESCRIBE", .value.integer = OP_HLL_DESCRIBE}, + {"OP_HLL_FOLD", .value.integer = OP_HLL_FOLD}, + {"OP_HLL_GET_COUNT", .value.integer = OP_HLL_GET_COUNT}, + {"OP_HLL_GET_INTERSECT_COUNT", .value.integer = OP_HLL_GET_INTERSECT_COUNT}, + {"OP_HLL_GET_SIMILARITY", .value.integer = OP_HLL_GET_SIMILARITY}, + {"OP_HLL_GET_UNION", .value.integer = OP_HLL_GET_UNION}, + {"OP_HLL_GET_UNION_COUNT", .value.integer = OP_HLL_GET_UNION_COUNT}, + {"OP_HLL_GET_SIMILARITY", .value.integer = OP_HLL_GET_SIMILARITY}, + {"OP_HLL_INIT", .value.integer = OP_HLL_INIT}, + {"OP_HLL_REFRESH_COUNT", .value.integer = OP_HLL_REFRESH_COUNT}, + {"OP_HLL_SET_UNION", .value.integer = OP_HLL_SET_UNION}, + {"OP_HLL_MAY_CONTAIN", + .value.integer = OP_HLL_MAY_CONTAIN}, // for expression filters + + {"HLL_WRITE_DEFAULT", .value.integer = AS_HLL_WRITE_DEFAULT}, + {"HLL_WRITE_CREATE_ONLY", .value.integer = AS_HLL_WRITE_CREATE_ONLY}, + {"HLL_WRITE_UPDATE_ONLY", .value.integer = AS_HLL_WRITE_UPDATE_ONLY}, + {"HLL_WRITE_NO_FAIL", .value.integer = AS_HLL_WRITE_NO_FAIL}, + {"HLL_WRITE_ALLOW_FOLD", .value.integer = AS_HLL_WRITE_ALLOW_FOLD}, + + /* Expression operation constants 5.1.0 */ + {"OP_EXPR_READ", .value.integer = OP_EXPR_READ}, + {"OP_EXPR_WRITE", .value.integer = OP_EXPR_WRITE}, + {"EXP_WRITE_DEFAULT", .value.integer = AS_EXP_WRITE_DEFAULT}, + {"EXP_WRITE_CREATE_ONLY", .value.integer = AS_EXP_WRITE_CREATE_ONLY}, + {"EXP_WRITE_UPDATE_ONLY", .value.integer = AS_EXP_WRITE_UPDATE_ONLY}, + {"EXP_WRITE_ALLOW_DELETE", .value.integer = AS_EXP_WRITE_ALLOW_DELETE}, + {"EXP_WRITE_POLICY_NO_FAIL", .value.integer = AS_EXP_WRITE_POLICY_NO_FAIL}, + {"EXP_WRITE_EVAL_NO_FAIL", .value.integer = AS_EXP_WRITE_EVAL_NO_FAIL}, + {"EXP_READ_DEFAULT", .value.integer = AS_EXP_READ_DEFAULT}, + {"EXP_READ_EVAL_NO_FAIL", .value.integer = AS_EXP_READ_EVAL_NO_FAIL}, + + /* For BinType expression, as_bytes_type */ + {"AS_BYTES_UNDEF", .value.integer = AS_BYTES_UNDEF}, + {"AS_BYTES_INTEGER", .value.integer = AS_BYTES_INTEGER}, + {"AS_BYTES_DOUBLE", .value.integer = AS_BYTES_DOUBLE}, + {"AS_BYTES_STRING", .value.integer = AS_BYTES_STRING}, + {"AS_BYTES_BLOB", .value.integer = AS_BYTES_BLOB}, + {"AS_BYTES_JAVA", .value.integer = AS_BYTES_JAVA}, + {"AS_BYTES_CSHARP", .value.integer = AS_BYTES_CSHARP}, + {"AS_BYTES_PYTHON", .value.integer = AS_BYTES_PYTHON}, + {"AS_BYTES_RUBY", .value.integer = AS_BYTES_RUBY}, + {"AS_BYTES_PHP", .value.integer = AS_BYTES_PHP}, + {"AS_BYTES_ERLANG", .value.integer = AS_BYTES_ERLANG}, + {"AS_BYTES_BOOL", .value.integer = AS_BYTES_BOOL}, + {"AS_BYTES_HLL", .value.integer = AS_BYTES_HLL}, + {"AS_BYTES_MAP", .value.integer = AS_BYTES_MAP}, + {"AS_BYTES_LIST", .value.integer = AS_BYTES_LIST}, + {"AS_BYTES_GEOJSON", .value.integer = AS_BYTES_GEOJSON}, + {"AS_BYTES_TYPE_MAX", .value.integer = AS_BYTES_TYPE_MAX}, + + /* Regex constants from predexp, still used by expressions */ + {"REGEX_NONE", .value.integer = REGEX_NONE}, + {"REGEX_EXTENDED", .value.integer = REGEX_EXTENDED}, + {"REGEX_ICASE", .value.integer = REGEX_ICASE}, + {"REGEX_NOSUB", .value.integer = REGEX_NOSUB}, + {"REGEX_NEWLINE", .value.integer = REGEX_NEWLINE}, + + {"QUERY_DURATION_LONG", .value.integer = AS_QUERY_DURATION_LONG}, + {"QUERY_DURATION_LONG_RELAX_AP", + .value.integer = AS_QUERY_DURATION_LONG_RELAX_AP}, + {"QUERY_DURATION_SHORT", .value.integer = AS_QUERY_DURATION_SHORT}, + + {"LOG_LEVEL_OFF", .value.integer = -1}, + {"LOG_LEVEL_ERROR", .value.integer = AS_LOG_LEVEL_ERROR}, + {"LOG_LEVEL_WARN", .value.integer = AS_LOG_LEVEL_WARN}, + {"LOG_LEVEL_INFO", .value.integer = AS_LOG_LEVEL_INFO}, + {"LOG_LEVEL_DEBUG", .value.integer = AS_LOG_LEVEL_DEBUG}, + {"LOG_LEVEL_TRACE", .value.integer = AS_LOG_LEVEL_TRACE}, + + {"JOB_SCAN", .is_str_value = true, .value.string = "scan"}, + {"JOB_QUERY", .is_str_value = true, .value.string = "query"}}; + +struct submodule_name_to_creation_method { + const char *name; + PyObject *(*pyobject_creation_method)(void); +}; - // In case adding objects to module fails, we can properly deallocate the module state later - memset(Aerospike_State(aerospike), 0, sizeof(struct Aerospike_State)); +static struct submodule_name_to_creation_method py_submodules[] = { + // We don't use module's __name__ attribute + // because the modules' __name__ is the fully qualified name which includes the package name + {"exception", AerospikeException_New}, + {"predicates", AerospikePredicates_New}, +}; - Aerospike_Enable_Default_Logging(); +struct type_name_to_creation_method { + const char *name; + PyTypeObject *(*pytype_ready_method)(void); +}; - py_global_hosts = PyDict_New(); +static struct type_name_to_creation_method py_module_types[] = { + // We also don't retrieve the type's __name__ because: + // 1. Some of the objects have names different from the class name when accessed from the package + // 2. We don't want to deal with extracting an object's __name__ from a Unicode object. + // We have to make sure the Unicode object lives as long as we need its internal buffer + // It's easier to just use a C string directly + {"Client", AerospikeClient_Ready}, + {"Query", AerospikeQuery_Ready}, + {"Scan", AerospikeScan_Ready}, + {"KeyOrderedDict", AerospikeKeyOrderedDict_Ready}, + {"GeoJSON", AerospikeGeospatial_Ready}, + {"null", AerospikeNullObject_Ready}, + {"CDTWildcard", AerospikeWildcardObject_Ready}, + {"CDTInfinite", AerospikeInfiniteObject_Ready}, +}; - PyObject *exception = AerospikeException_New(); - if (exception == NULL) { +PyMODINIT_FUNC PyInit_aerospike(void) +{ + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + .m_name = AEROSPIKE_MODULE_NAME, + .m_doc = "Aerospike Python Client", + .m_methods = aerospike_methods, + }; + + PyObject *py_aerospike_module = PyModule_Create(&moduledef); + if (py_aerospike_module == NULL) { return NULL; } - Py_INCREF(exception); - int retval = PyModule_AddObject(aerospike, "exception", exception); - if (retval == -1) { - goto CLEANUP; - } - Aerospike_State(aerospike)->exception = exception; - - PyTypeObject *client = AerospikeClient_Ready(); - Py_INCREF(client); - retval = PyModule_AddObject(aerospike, "Client", (PyObject *)client); - if (retval == -1) { - goto CLEANUP; - } - Aerospike_State(aerospike)->client = client; + Aerospike_Enable_Default_Logging(); - PyTypeObject *query = AerospikeQuery_Ready(); - Py_INCREF(query); - retval = PyModule_AddObject(aerospike, "Query", (PyObject *)query); - if (retval == -1) { - goto CLEANUP; + py_global_hosts = PyDict_New(); + if (py_global_hosts == NULL) { + goto MODULE_CLEANUP_ON_ERROR; } - Aerospike_State(aerospike)->query = query; - PyTypeObject *scan = AerospikeScan_Ready(); - Py_INCREF(scan); - retval = PyModule_AddObject(aerospike, "Scan", (PyObject *)scan); - if (retval == -1) { - goto CLEANUP; + unsigned long i = 0; + int retval; + for (i = 0; i < sizeof(py_submodules) / sizeof(py_submodules[0]); i++) { + PyObject *(*create_py_submodule)(void) = + py_submodules[i].pyobject_creation_method; + PyObject *py_submodule = create_py_submodule(); + if (py_submodule == NULL) { + goto GLOBAL_HOSTS_CLEANUP_ON_ERROR; + } + + retval = PyModule_AddObject(py_aerospike_module, py_submodules[i].name, + py_submodule); + if (retval == -1) { + Py_DECREF(py_submodule); + goto GLOBAL_HOSTS_CLEANUP_ON_ERROR; + } } - Aerospike_State(aerospike)->scan = scan; - PyTypeObject *kdict = AerospikeKeyOrderedDict_Ready(); - Py_INCREF(kdict); - retval = PyModule_AddObject(aerospike, "KeyOrderedDict", (PyObject *)kdict); - if (retval == -1) { - goto CLEANUP; + for (i = 0; i < sizeof(py_module_types) / sizeof(py_module_types[0]); i++) { + PyTypeObject *(*py_type_ready_func)(void) = + py_module_types[i].pytype_ready_method; + PyTypeObject *py_type = py_type_ready_func(); + if (py_type == NULL) { + goto GLOBAL_HOSTS_CLEANUP_ON_ERROR; + } + + Py_INCREF(py_type); + retval = PyModule_AddObject( + py_aerospike_module, py_module_types[i].name, (PyObject *)py_type); + if (retval == -1) { + Py_DECREF(py_type); + goto GLOBAL_HOSTS_CLEANUP_ON_ERROR; + } } - Aerospike_State(aerospike)->kdict = kdict; /* * Add constants to module. */ - for (i = 0; i < (int)OPERATOR_CONSTANTS_ARR_SIZE; i++) { - PyModule_AddIntConstant(aerospike, operator_constants[i].constant_str, - operator_constants[i].constantno); - } - - for (i = 0; i < (int)AUTH_MODE_CONSTANTS_ARR_SIZE; i++) { - PyModule_AddIntConstant(aerospike, auth_mode_constants[i].constant_str, - auth_mode_constants[i].constantno); - } - - declare_policy_constants(aerospike); - declare_log_constants(aerospike); - - PyObject *predicates = AerospikePredicates_New(); - Py_INCREF(predicates); - retval = PyModule_AddObject(aerospike, "predicates", predicates); - if (retval == -1) { - goto CLEANUP; - } - Aerospike_State(aerospike)->predicates = predicates; - - PyTypeObject *geospatial = AerospikeGeospatial_Ready(); - Py_INCREF(geospatial); - retval = PyModule_AddObject(aerospike, "GeoJSON", (PyObject *)geospatial); - if (retval == -1) { - goto CLEANUP; - } - Aerospike_State(aerospike)->geospatial = geospatial; - - PyTypeObject *null_object = AerospikeNullObject_Ready(); - Py_INCREF(null_object); - retval = PyModule_AddObject(aerospike, "null", (PyObject *)null_object); - if (retval == -1) { - goto CLEANUP; - } - Aerospike_State(aerospike)->null_object = null_object; - - PyTypeObject *wildcard_object = AerospikeWildcardObject_Ready(); - Py_INCREF(wildcard_object); - retval = PyModule_AddObject(aerospike, "CDTWildcard", - (PyObject *)wildcard_object); - if (retval == -1) { - goto CLEANUP; - } - Aerospike_State(aerospike)->wildcard_object = wildcard_object; - - PyTypeObject *infinite_object = AerospikeInfiniteObject_Ready(); - Py_INCREF(infinite_object); - retval = PyModule_AddObject(aerospike, "CDTInfinite", - (PyObject *)infinite_object); - if (retval == -1) { - goto CLEANUP; + for (i = 0; i < sizeof(module_constants) / sizeof(module_constants[0]); + i++) { + if (module_constants[i].is_str_value == false) { + retval = PyModule_AddIntConstant(py_aerospike_module, + module_constants[i].name, + module_constants[i].value.integer); + } + else { + retval = PyModule_AddStringConstant( + py_aerospike_module, module_constants[i].name, + module_constants[i].value.string); + } + + if (retval == -1) { + goto GLOBAL_HOSTS_CLEANUP_ON_ERROR; + } } - Aerospike_State(aerospike)->infinite_object = infinite_object; - return aerospike; + return py_aerospike_module; -CLEANUP: - Aerospike_Clear(aerospike); +GLOBAL_HOSTS_CLEANUP_ON_ERROR: + Py_DECREF(py_global_hosts); +MODULE_CLEANUP_ON_ERROR: + Py_DECREF(py_aerospike_module); return NULL; } diff --git a/src/main/client/type.c b/src/main/client/type.c index bea8f279f..ccd174b77 100644 --- a/src/main/client/type.c +++ b/src/main/client/type.c @@ -1290,24 +1290,25 @@ static void AerospikeClient_Type_Dealloc(PyObject *self) ******************************************************************************/ static PyTypeObject AerospikeClient_Type = { - PyVarObject_HEAD_INIT(NULL, 0) "aerospike.Client", // tp_name - sizeof(AerospikeClient), // tp_basicsize - 0, // tp_itemsize - (destructor)AerospikeClient_Type_Dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - 0, // tp_as_mapping - 0, // tp_hash - 0, // tp_call - 0, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer + PyVarObject_HEAD_INIT(NULL, 0) + FULLY_QUALIFIED_TYPE_NAME("Client"), // tp_name + sizeof(AerospikeClient), // tp_basicsize + 0, // tp_itemsize + (destructor)AerospikeClient_Type_Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, // tp_flags "The Client class manages the connections and trasactions against\n" diff --git a/src/main/geospatial/type.c b/src/main/geospatial/type.c index 880ee9a56..e2ebb0616 100644 --- a/src/main/geospatial/type.c +++ b/src/main/geospatial/type.c @@ -219,9 +219,10 @@ static void AerospikeGeospatial_Type_Dealloc(AerospikeGeospatial *self) * PYTHON TYPE DESCRIPTOR ******************************************************************************/ static PyTypeObject AerospikeGeospatial_Type = { - PyVarObject_HEAD_INIT(NULL, 0) "aerospike.Geospatial", // tp_name - sizeof(AerospikeGeospatial), // tp_basicsize - 0, // tp_itemsize + PyVarObject_HEAD_INIT(NULL, 0) + FULLY_QUALIFIED_TYPE_NAME("Geospatial"), // tp_name + sizeof(AerospikeGeospatial), // tp_basicsize + 0, // tp_itemsize (destructor)AerospikeGeospatial_Type_Dealloc, // tp_dealloc 0, // tp_print diff --git a/src/main/key_ordered_dict/type.c b/src/main/key_ordered_dict/type.c index 66dc4807e..1037ba05f 100644 --- a/src/main/key_ordered_dict/type.c +++ b/src/main/key_ordered_dict/type.c @@ -40,7 +40,8 @@ static int AerospikeKeyOrderedDict_Type_Init(PyObject *self, PyObject *args, ******************************************************************************/ static PyTypeObject AerospikeKeyOrderedDict_Type = { - PyVarObject_HEAD_INIT(NULL, 0).tp_name = "aerospike.KeyOrderedDict", + PyVarObject_HEAD_INIT(NULL, 0).tp_name = + FULLY_QUALIFIED_TYPE_NAME("KeyOrderedDict"), .tp_basicsize = sizeof(AerospikeKeyOrderedDict), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, .tp_doc = "The KeyOrderedDict class is a dictionary that directly maps\n" diff --git a/src/main/log.c b/src/main/log.c index 03b0e44bf..fc2090c6d 100644 --- a/src/main/log.c +++ b/src/main/log.c @@ -31,32 +31,6 @@ static AerospikeLogCallback user_callback; -/* - * Declare's log level constants. - */ -as_status declare_log_constants(PyObject *aerospike) -{ - - // Status to be returned. - as_status status = AEROSPIKE_OK; - - // Check if aerospike object is present or no. - if (!aerospike) { - status = AEROSPIKE_ERR; - goto exit; - } - - // Add incidividual constants to aerospike module. - PyModule_AddIntConstant(aerospike, "LOG_LEVEL_OFF", LOG_LEVEL_OFF); - PyModule_AddIntConstant(aerospike, "LOG_LEVEL_ERROR", LOG_LEVEL_ERROR); - PyModule_AddIntConstant(aerospike, "LOG_LEVEL_WARN", LOG_LEVEL_WARN); - PyModule_AddIntConstant(aerospike, "LOG_LEVEL_INFO", LOG_LEVEL_INFO); - PyModule_AddIntConstant(aerospike, "LOG_LEVEL_DEBUG", LOG_LEVEL_DEBUG); - PyModule_AddIntConstant(aerospike, "LOG_LEVEL_TRACE", LOG_LEVEL_TRACE); -exit: - return status; -} - PyObject *Aerospike_Set_Log_Level(PyObject *parent, PyObject *args, PyObject *kwds) { @@ -208,7 +182,7 @@ PyObject *Aerospike_Set_Log_Handler(PyObject *parent, PyObject *args, void Aerospike_Enable_Default_Logging() { // Invoke C API to set log level - as_log_set_level((as_log_level)LOG_LEVEL_ERROR); + as_log_set_level((as_log_level)AS_LOG_LEVEL_ERROR); // Register callback to C-SDK as_log_set_callback((as_log_callback)console_log_cb); diff --git a/src/main/nullobject/type.c b/src/main/nullobject/type.c index b250873a3..9a290ed65 100644 --- a/src/main/nullobject/type.c +++ b/src/main/nullobject/type.c @@ -33,9 +33,9 @@ static void AerospikeNullObject_Type_Dealloc(AerospikeNullObject *self) ******************************************************************************/ static PyTypeObject AerospikeNullObject_Type = { - PyVarObject_HEAD_INIT(NULL, 0) "aerospike.null", // tp_name - sizeof(AerospikeNullObject), // tp_basicsize - 0, // tp_itemsize + PyVarObject_HEAD_INIT(NULL, 0) FULLY_QUALIFIED_TYPE_NAME("null"), // tp_name + sizeof(AerospikeNullObject), // tp_basicsize + 0, // tp_itemsize (destructor)AerospikeNullObject_Type_Dealloc, // tp_dealloc 0, // tp_print diff --git a/src/main/policy.c b/src/main/policy.c index 4670f65f9..0cb2b1491 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -122,344 +122,6 @@ } \ } -/* - ******************************************************************************************************* - * Mapping of constant number to constant name string. - ******************************************************************************************************* - */ -static AerospikeConstants aerospike_constants[] = { - {AS_POLICY_RETRY_NONE, "POLICY_RETRY_NONE"}, - {AS_POLICY_RETRY_ONCE, "POLICY_RETRY_ONCE"}, - {AS_POLICY_EXISTS_IGNORE, "POLICY_EXISTS_IGNORE"}, - {AS_POLICY_EXISTS_CREATE, "POLICY_EXISTS_CREATE"}, - {AS_POLICY_EXISTS_UPDATE, "POLICY_EXISTS_UPDATE"}, - {AS_POLICY_EXISTS_REPLACE, "POLICY_EXISTS_REPLACE"}, - {AS_POLICY_EXISTS_CREATE_OR_REPLACE, "POLICY_EXISTS_CREATE_OR_REPLACE"}, - {AS_UDF_TYPE_LUA, "UDF_TYPE_LUA"}, - {AS_POLICY_KEY_DIGEST, "POLICY_KEY_DIGEST"}, - {AS_POLICY_KEY_SEND, "POLICY_KEY_SEND"}, - {AS_POLICY_GEN_IGNORE, "POLICY_GEN_IGNORE"}, - {AS_POLICY_GEN_EQ, "POLICY_GEN_EQ"}, - {AS_POLICY_GEN_GT, "POLICY_GEN_GT"}, - {AS_JOB_STATUS_COMPLETED, "JOB_STATUS_COMPLETED"}, - {AS_JOB_STATUS_UNDEF, "JOB_STATUS_UNDEF"}, - {AS_JOB_STATUS_INPROGRESS, "JOB_STATUS_INPROGRESS"}, - {AS_POLICY_REPLICA_MASTER, "POLICY_REPLICA_MASTER"}, - {AS_POLICY_REPLICA_ANY, "POLICY_REPLICA_ANY"}, - {AS_POLICY_REPLICA_SEQUENCE, "POLICY_REPLICA_SEQUENCE"}, - {AS_POLICY_REPLICA_PREFER_RACK, "POLICY_REPLICA_PREFER_RACK"}, - {AS_POLICY_COMMIT_LEVEL_ALL, "POLICY_COMMIT_LEVEL_ALL"}, - {AS_POLICY_COMMIT_LEVEL_MASTER, "POLICY_COMMIT_LEVEL_MASTER"}, - {SERIALIZER_USER, "SERIALIZER_USER"}, - {SERIALIZER_JSON, "SERIALIZER_JSON"}, - {SERIALIZER_NONE, "SERIALIZER_NONE"}, - {SEND_BOOL_AS_INTEGER, "INTEGER"}, - {SEND_BOOL_AS_AS_BOOL, "AS_BOOL"}, - {AS_INDEX_STRING, "INDEX_STRING"}, - {AS_INDEX_NUMERIC, "INDEX_NUMERIC"}, - {AS_INDEX_GEO2DSPHERE, "INDEX_GEO2DSPHERE"}, - {AS_INDEX_BLOB, "INDEX_BLOB"}, - {AS_INDEX_TYPE_DEFAULT, "INDEX_TYPE_DEFAULT"}, - {AS_INDEX_TYPE_LIST, "INDEX_TYPE_LIST"}, - {AS_INDEX_TYPE_MAPKEYS, "INDEX_TYPE_MAPKEYS"}, - {AS_INDEX_TYPE_MAPVALUES, "INDEX_TYPE_MAPVALUES"}, - {AS_PRIVILEGE_USER_ADMIN, "PRIV_USER_ADMIN"}, - {AS_PRIVILEGE_SYS_ADMIN, "PRIV_SYS_ADMIN"}, - {AS_PRIVILEGE_DATA_ADMIN, "PRIV_DATA_ADMIN"}, - {AS_PRIVILEGE_READ, "PRIV_READ"}, - {AS_PRIVILEGE_WRITE, "PRIV_WRITE"}, - {AS_PRIVILEGE_READ_WRITE, "PRIV_READ_WRITE"}, - {AS_PRIVILEGE_READ_WRITE_UDF, "PRIV_READ_WRITE_UDF"}, - {AS_PRIVILEGE_TRUNCATE, "PRIV_TRUNCATE"}, - {AS_PRIVILEGE_UDF_ADMIN, "PRIV_UDF_ADMIN"}, - {AS_PRIVILEGE_SINDEX_ADMIN, "PRIV_SINDEX_ADMIN"}, - - {OP_LIST_APPEND, "OP_LIST_APPEND"}, - {OP_LIST_APPEND_ITEMS, "OP_LIST_APPEND_ITEMS"}, - {OP_LIST_INSERT, "OP_LIST_INSERT"}, - {OP_LIST_INSERT_ITEMS, "OP_LIST_INSERT_ITEMS"}, - {OP_LIST_POP, "OP_LIST_POP"}, - {OP_LIST_POP_RANGE, "OP_LIST_POP_RANGE"}, - {OP_LIST_REMOVE, "OP_LIST_REMOVE"}, - {OP_LIST_REMOVE_RANGE, "OP_LIST_REMOVE_RANGE"}, - {OP_LIST_CLEAR, "OP_LIST_CLEAR"}, - {OP_LIST_SET, "OP_LIST_SET"}, - {OP_LIST_GET, "OP_LIST_GET"}, - {OP_LIST_GET_RANGE, "OP_LIST_GET_RANGE"}, - {OP_LIST_TRIM, "OP_LIST_TRIM"}, - {OP_LIST_SIZE, "OP_LIST_SIZE"}, - {OP_LIST_INCREMENT, "OP_LIST_INCREMENT"}, - - {OP_MAP_SET_POLICY, "OP_MAP_SET_POLICY"}, - {OP_MAP_CREATE, "OP_MAP_CREATE"}, - {OP_MAP_PUT, "OP_MAP_PUT"}, - {OP_MAP_PUT_ITEMS, "OP_MAP_PUT_ITEMS"}, - {OP_MAP_INCREMENT, "OP_MAP_INCREMENT"}, - {OP_MAP_DECREMENT, "OP_MAP_DECREMENT"}, - {OP_MAP_SIZE, "OP_MAP_SIZE"}, - {OP_MAP_CLEAR, "OP_MAP_CLEAR"}, - {OP_MAP_REMOVE_BY_KEY, "OP_MAP_REMOVE_BY_KEY"}, - {OP_MAP_REMOVE_BY_KEY_LIST, "OP_MAP_REMOVE_BY_KEY_LIST"}, - {OP_MAP_REMOVE_BY_KEY_RANGE, "OP_MAP_REMOVE_BY_KEY_RANGE"}, - {OP_MAP_REMOVE_BY_VALUE, "OP_MAP_REMOVE_BY_VALUE"}, - {OP_MAP_REMOVE_BY_VALUE_LIST, "OP_MAP_REMOVE_BY_VALUE_LIST"}, - {OP_MAP_REMOVE_BY_VALUE_RANGE, "OP_MAP_REMOVE_BY_VALUE_RANGE"}, - {OP_MAP_REMOVE_BY_INDEX, "OP_MAP_REMOVE_BY_INDEX"}, - {OP_MAP_REMOVE_BY_INDEX_RANGE, "OP_MAP_REMOVE_BY_INDEX_RANGE"}, - {OP_MAP_REMOVE_BY_RANK, "OP_MAP_REMOVE_BY_RANK"}, - {OP_MAP_REMOVE_BY_RANK_RANGE, "OP_MAP_REMOVE_BY_RANK_RANGE"}, - {OP_MAP_GET_BY_KEY, "OP_MAP_GET_BY_KEY"}, - {OP_MAP_GET_BY_KEY_RANGE, "OP_MAP_GET_BY_KEY_RANGE"}, - {OP_MAP_GET_BY_VALUE, "OP_MAP_GET_BY_VALUE"}, - {OP_MAP_GET_BY_VALUE_RANGE, "OP_MAP_GET_BY_VALUE_RANGE"}, - {OP_MAP_GET_BY_INDEX, "OP_MAP_GET_BY_INDEX"}, - {OP_MAP_GET_BY_INDEX_RANGE, "OP_MAP_GET_BY_INDEX_RANGE"}, - {OP_MAP_GET_BY_RANK, "OP_MAP_GET_BY_RANK"}, - {OP_MAP_GET_BY_RANK_RANGE, "OP_MAP_GET_BY_RANK_RANGE"}, - {OP_MAP_GET_BY_VALUE_LIST, "OP_MAP_GET_BY_VALUE_LIST"}, - {OP_MAP_GET_BY_KEY_LIST, "OP_MAP_GET_BY_KEY_LIST"}, - - {AS_MAP_UNORDERED, "MAP_UNORDERED"}, - {AS_MAP_KEY_ORDERED, "MAP_KEY_ORDERED"}, - {AS_MAP_KEY_VALUE_ORDERED, "MAP_KEY_VALUE_ORDERED"}, - - {AS_MAP_RETURN_NONE, "MAP_RETURN_NONE"}, - {AS_MAP_RETURN_INDEX, "MAP_RETURN_INDEX"}, - {AS_MAP_RETURN_REVERSE_INDEX, "MAP_RETURN_REVERSE_INDEX"}, - {AS_MAP_RETURN_RANK, "MAP_RETURN_RANK"}, - {AS_MAP_RETURN_REVERSE_RANK, "MAP_RETURN_REVERSE_RANK"}, - {AS_MAP_RETURN_COUNT, "MAP_RETURN_COUNT"}, - {AS_MAP_RETURN_KEY, "MAP_RETURN_KEY"}, - {AS_MAP_RETURN_VALUE, "MAP_RETURN_VALUE"}, - {AS_MAP_RETURN_KEY_VALUE, "MAP_RETURN_KEY_VALUE"}, - {AS_MAP_RETURN_EXISTS, "MAP_RETURN_EXISTS"}, - {AS_MAP_RETURN_ORDERED_MAP, "MAP_RETURN_ORDERED_MAP"}, - {AS_MAP_RETURN_UNORDERED_MAP, "MAP_RETURN_UNORDERED_MAP"}, - - {AS_RECORD_DEFAULT_TTL, "TTL_NAMESPACE_DEFAULT"}, - {AS_RECORD_NO_EXPIRE_TTL, "TTL_NEVER_EXPIRE"}, - {AS_RECORD_NO_CHANGE_TTL, "TTL_DONT_UPDATE"}, - {AS_RECORD_CLIENT_DEFAULT_TTL, "TTL_CLIENT_DEFAULT"}, - {AS_AUTH_INTERNAL, "AUTH_INTERNAL"}, - {AS_AUTH_EXTERNAL, "AUTH_EXTERNAL"}, - {AS_AUTH_EXTERNAL_INSECURE, "AUTH_EXTERNAL_INSECURE"}, - {AS_AUTH_PKI, "AUTH_PKI"}, - /* New CDT Operations, post 3.16.0.1 */ - {OP_LIST_GET_BY_INDEX, "OP_LIST_GET_BY_INDEX"}, - {OP_LIST_GET_BY_INDEX_RANGE, "OP_LIST_GET_BY_INDEX_RANGE"}, - {OP_LIST_GET_BY_RANK, "OP_LIST_GET_BY_RANK"}, - {OP_LIST_GET_BY_RANK_RANGE, "OP_LIST_GET_BY_RANK_RANGE"}, - {OP_LIST_GET_BY_VALUE, "OP_LIST_GET_BY_VALUE"}, - {OP_LIST_GET_BY_VALUE_LIST, "OP_LIST_GET_BY_VALUE_LIST"}, - {OP_LIST_GET_BY_VALUE_RANGE, "OP_LIST_GET_BY_VALUE_RANGE"}, - {OP_LIST_REMOVE_BY_INDEX, "OP_LIST_REMOVE_BY_INDEX"}, - {OP_LIST_REMOVE_BY_INDEX_RANGE, "OP_LIST_REMOVE_BY_INDEX_RANGE"}, - {OP_LIST_REMOVE_BY_RANK, "OP_LIST_REMOVE_BY_RANK"}, - {OP_LIST_REMOVE_BY_RANK_RANGE, "OP_LIST_REMOVE_BY_RANK_RANGE"}, - {OP_LIST_REMOVE_BY_VALUE, "OP_LIST_REMOVE_BY_VALUE"}, - {OP_LIST_REMOVE_BY_VALUE_LIST, "OP_LIST_REMOVE_BY_VALUE_LIST"}, - {OP_LIST_REMOVE_BY_VALUE_RANGE, "OP_LIST_REMOVE_BY_VALUE_RANGE"}, - {OP_LIST_SET_ORDER, "OP_LIST_SET_ORDER"}, - {OP_LIST_SORT, "OP_LIST_SORT"}, - {AS_LIST_RETURN_NONE, "LIST_RETURN_NONE"}, - {AS_LIST_RETURN_INDEX, "LIST_RETURN_INDEX"}, - {AS_LIST_RETURN_REVERSE_INDEX, "LIST_RETURN_REVERSE_INDEX"}, - {AS_LIST_RETURN_RANK, "LIST_RETURN_RANK"}, - {AS_LIST_RETURN_REVERSE_RANK, "LIST_RETURN_REVERSE_RANK"}, - {AS_LIST_RETURN_COUNT, "LIST_RETURN_COUNT"}, - {AS_LIST_RETURN_VALUE, "LIST_RETURN_VALUE"}, - {AS_LIST_RETURN_EXISTS, "LIST_RETURN_EXISTS"}, - {AS_LIST_SORT_DROP_DUPLICATES, "LIST_SORT_DROP_DUPLICATES"}, - {AS_LIST_SORT_DEFAULT, "LIST_SORT_DEFAULT"}, - {AS_LIST_WRITE_DEFAULT, "LIST_WRITE_DEFAULT"}, - {AS_LIST_WRITE_ADD_UNIQUE, "LIST_WRITE_ADD_UNIQUE"}, - {AS_LIST_WRITE_INSERT_BOUNDED, "LIST_WRITE_INSERT_BOUNDED"}, - {AS_LIST_ORDERED, "LIST_ORDERED"}, - {AS_LIST_UNORDERED, "LIST_UNORDERED"}, - {OP_LIST_REMOVE_BY_VALUE_RANK_RANGE_REL, - "OP_LIST_REMOVE_BY_VALUE_RANK_RANGE_REL"}, - {OP_LIST_GET_BY_VALUE_RANK_RANGE_REL, - "OP_LIST_GET_BY_VALUE_RANK_RANGE_REL"}, - {OP_LIST_CREATE, "OP_LIST_CREATE"}, - - /* CDT operations for use with expressions, new in 5.0 */ - {OP_MAP_REMOVE_BY_VALUE_RANK_RANGE_REL, - "OP_MAP_REMOVE_BY_VALUE_RANK_RANGE_REL"}, - {OP_MAP_REMOVE_BY_KEY_INDEX_RANGE_REL, - "OP_MAP_REMOVE_BY_KEY_INDEX_RANGE_REL"}, - {OP_MAP_GET_BY_VALUE_RANK_RANGE_REL, "OP_MAP_GET_BY_VALUE_RANK_RANGE_REL"}, - {OP_MAP_GET_BY_KEY_INDEX_RANGE_REL, "OP_MAP_GET_BY_KEY_INDEX_RANGE_REL"}, - - {OP_LIST_GET_BY_VALUE_RANK_RANGE_REL_TO_END, - "OP_LIST_GET_BY_VALUE_RANK_RANGE_REL_TO_END"}, - {OP_LIST_GET_BY_INDEX_RANGE_TO_END, "OP_LIST_GET_BY_INDEX_RANGE_TO_END"}, - {OP_LIST_GET_BY_RANK_RANGE_TO_END, "OP_LIST_GET_BY_RANK_RANGE_TO_END"}, - {OP_LIST_REMOVE_BY_REL_RANK_RANGE_TO_END, - "OP_LIST_REMOVE_BY_REL_RANK_RANGE_TO_END"}, - {OP_LIST_REMOVE_BY_REL_RANK_RANGE, "OP_LIST_REMOVE_BY_REL_RANK_RANGE"}, - {OP_LIST_REMOVE_BY_INDEX_RANGE_TO_END, - "OP_LIST_REMOVE_BY_INDEX_RANGE_TO_END"}, - {OP_LIST_REMOVE_BY_RANK_RANGE_TO_END, - "OP_LIST_REMOVE_BY_RANK_RANGE_TO_END"}, - - {AS_MAP_WRITE_NO_FAIL, "MAP_WRITE_NO_FAIL"}, - {AS_MAP_WRITE_PARTIAL, "MAP_WRITE_PARTIAL"}, - - {AS_LIST_WRITE_NO_FAIL, "LIST_WRITE_NO_FAIL"}, - {AS_LIST_WRITE_PARTIAL, "LIST_WRITE_PARTIAL"}, - - /* Map write flags post 3.5.0 */ - {AS_MAP_WRITE_DEFAULT, "MAP_WRITE_FLAGS_DEFAULT"}, - {AS_MAP_WRITE_CREATE_ONLY, "MAP_WRITE_FLAGS_CREATE_ONLY"}, - {AS_MAP_WRITE_UPDATE_ONLY, "MAP_WRITE_FLAGS_UPDATE_ONLY"}, - {AS_MAP_WRITE_NO_FAIL, "MAP_WRITE_FLAGS_NO_FAIL"}, - {AS_MAP_WRITE_PARTIAL, "MAP_WRITE_FLAGS_PARTIAL"}, - - /* READ Mode constants 4.0.0 */ - - // AP Read Mode - {AS_POLICY_READ_MODE_AP_ONE, "POLICY_READ_MODE_AP_ONE"}, - {AS_POLICY_READ_MODE_AP_ALL, "POLICY_READ_MODE_AP_ALL"}, - // SC Read Mode - {AS_POLICY_READ_MODE_SC_SESSION, "POLICY_READ_MODE_SC_SESSION"}, - {AS_POLICY_READ_MODE_SC_LINEARIZE, "POLICY_READ_MODE_SC_LINEARIZE"}, - {AS_POLICY_READ_MODE_SC_ALLOW_REPLICA, "POLICY_READ_MODE_SC_ALLOW_REPLICA"}, - {AS_POLICY_READ_MODE_SC_ALLOW_UNAVAILABLE, - "POLICY_READ_MODE_SC_ALLOW_UNAVAILABLE"}, - - /* Bitwise constants: 3.9.0 */ - {AS_BIT_WRITE_DEFAULT, "BIT_WRITE_DEFAULT"}, - {AS_BIT_WRITE_CREATE_ONLY, "BIT_WRITE_CREATE_ONLY"}, - {AS_BIT_WRITE_UPDATE_ONLY, "BIT_WRITE_UPDATE_ONLY"}, - {AS_BIT_WRITE_NO_FAIL, "BIT_WRITE_NO_FAIL"}, - {AS_BIT_WRITE_PARTIAL, "BIT_WRITE_PARTIAL"}, - - {AS_BIT_RESIZE_DEFAULT, "BIT_RESIZE_DEFAULT"}, - {AS_BIT_RESIZE_FROM_FRONT, "BIT_RESIZE_FROM_FRONT"}, - {AS_BIT_RESIZE_GROW_ONLY, "BIT_RESIZE_GROW_ONLY"}, - {AS_BIT_RESIZE_SHRINK_ONLY, "BIT_RESIZE_SHRINK_ONLY"}, - - {AS_BIT_OVERFLOW_FAIL, "BIT_OVERFLOW_FAIL"}, - {AS_BIT_OVERFLOW_SATURATE, "BIT_OVERFLOW_SATURATE"}, - {AS_BIT_OVERFLOW_WRAP, "BIT_OVERFLOW_WRAP"}, - - /* BITWISE OPS: 3.9.0 */ - {OP_BIT_INSERT, "OP_BIT_INSERT"}, - {OP_BIT_RESIZE, "OP_BIT_RESIZE"}, - {OP_BIT_REMOVE, "OP_BIT_REMOVE"}, - {OP_BIT_SET, "OP_BIT_SET"}, - {OP_BIT_OR, "OP_BIT_OR"}, - {OP_BIT_XOR, "OP_BIT_XOR"}, - {OP_BIT_AND, "OP_BIT_AND"}, - {OP_BIT_NOT, "OP_BIT_NOT"}, - {OP_BIT_LSHIFT, "OP_BIT_LSHIFT"}, - {OP_BIT_RSHIFT, "OP_BIT_RSHIFT"}, - {OP_BIT_ADD, "OP_BIT_ADD"}, - {OP_BIT_SUBTRACT, "OP_BIT_SUBTRACT"}, - {OP_BIT_GET_INT, "OP_BIT_GET_INT"}, - {OP_BIT_SET_INT, "OP_BIT_SET_INT"}, - {OP_BIT_GET, "OP_BIT_GET"}, - {OP_BIT_COUNT, "OP_BIT_COUNT"}, - {OP_BIT_LSCAN, "OP_BIT_LSCAN"}, - {OP_BIT_RSCAN, "OP_BIT_RSCAN"}, - - /* Nested CDT constants: 3.9.0 */ - {AS_CDT_CTX_LIST_INDEX, "CDT_CTX_LIST_INDEX"}, - {AS_CDT_CTX_LIST_RANK, "CDT_CTX_LIST_RANK"}, - {AS_CDT_CTX_LIST_VALUE, "CDT_CTX_LIST_VALUE"}, - {CDT_CTX_LIST_INDEX_CREATE, "CDT_CTX_LIST_INDEX_CREATE"}, - {AS_CDT_CTX_MAP_INDEX, "CDT_CTX_MAP_INDEX"}, - {AS_CDT_CTX_MAP_RANK, "CDT_CTX_MAP_RANK"}, - {AS_CDT_CTX_MAP_KEY, "CDT_CTX_MAP_KEY"}, - {AS_CDT_CTX_MAP_VALUE, "CDT_CTX_MAP_VALUE"}, - {CDT_CTX_MAP_KEY_CREATE, "CDT_CTX_MAP_KEY_CREATE"}, - - /* HLL constants 3.11.0 */ - {OP_HLL_ADD, "OP_HLL_ADD"}, - {OP_HLL_DESCRIBE, "OP_HLL_DESCRIBE"}, - {OP_HLL_FOLD, "OP_HLL_FOLD"}, - {OP_HLL_GET_COUNT, "OP_HLL_GET_COUNT"}, - {OP_HLL_GET_INTERSECT_COUNT, "OP_HLL_GET_INTERSECT_COUNT"}, - {OP_HLL_GET_SIMILARITY, "OP_HLL_GET_SIMILARITY"}, - {OP_HLL_GET_UNION, "OP_HLL_GET_UNION"}, - {OP_HLL_GET_UNION_COUNT, "OP_HLL_GET_UNION_COUNT"}, - {OP_HLL_GET_SIMILARITY, "OP_HLL_GET_SIMILARITY"}, - {OP_HLL_INIT, "OP_HLL_INIT"}, - {OP_HLL_REFRESH_COUNT, "OP_HLL_REFRESH_COUNT"}, - {OP_HLL_SET_UNION, "OP_HLL_SET_UNION"}, - {OP_HLL_MAY_CONTAIN, "OP_HLL_MAY_CONTAIN"}, // for expression filters - - {AS_HLL_WRITE_DEFAULT, "HLL_WRITE_DEFAULT"}, - {AS_HLL_WRITE_CREATE_ONLY, "HLL_WRITE_CREATE_ONLY"}, - {AS_HLL_WRITE_UPDATE_ONLY, "HLL_WRITE_UPDATE_ONLY"}, - {AS_HLL_WRITE_NO_FAIL, "HLL_WRITE_NO_FAIL"}, - {AS_HLL_WRITE_ALLOW_FOLD, "HLL_WRITE_ALLOW_FOLD"}, - - {OP_MAP_REMOVE_BY_KEY_REL_INDEX_RANGE_TO_END, - "OP_MAP_REMOVE_BY_KEY_REL_INDEX_RANGE_TO_END"}, - {OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE_TO_END, - "OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE_TO_END"}, - {OP_MAP_REMOVE_BY_INDEX_RANGE_TO_END, - "OP_MAP_REMOVE_BY_INDEX_RANGE_TO_END"}, - {OP_MAP_REMOVE_BY_RANK_RANGE_TO_END, "OP_MAP_REMOVE_BY_RANK_RANGE_TO_END"}, - {OP_MAP_GET_BY_KEY_REL_INDEX_RANGE_TO_END, - "OP_MAP_GET_BY_KEY_REL_INDEX_RANGE_TO_END"}, - {OP_MAP_REMOVE_BY_KEY_REL_INDEX_RANGE, - "OP_MAP_REMOVE_BY_KEY_REL_INDEX_RANGE"}, - {OP_MAP_REMOVE_BY_VALUE_REL_INDEX_RANGE, - "OP_MAP_REMOVE_BY_VALUE_REL_INDEX_RANGE"}, - {OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE, - "OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE"}, - {OP_MAP_GET_BY_KEY_REL_INDEX_RANGE, "OP_MAP_GET_BY_KEY_REL_INDEX_RANGE"}, - {OP_MAP_GET_BY_VALUE_RANK_RANGE_REL_TO_END, - "OP_MAP_GET_BY_VALUE_RANK_RANGE_REL_TO_END"}, - {OP_MAP_GET_BY_INDEX_RANGE_TO_END, "OP_MAP_GET_BY_INDEX_RANGE_TO_END"}, - {OP_MAP_GET_BY_RANK_RANGE_TO_END, "OP_MAP_GET_BY_RANK_RANGE_TO_END"}, - - /* Expression operation constants 5.1.0 */ - {OP_EXPR_READ, "OP_EXPR_READ"}, - {OP_EXPR_WRITE, "OP_EXPR_WRITE"}, - {AS_EXP_WRITE_DEFAULT, "EXP_WRITE_DEFAULT"}, - {AS_EXP_WRITE_CREATE_ONLY, "EXP_WRITE_CREATE_ONLY"}, - {AS_EXP_WRITE_UPDATE_ONLY, "EXP_WRITE_UPDATE_ONLY"}, - {AS_EXP_WRITE_ALLOW_DELETE, "EXP_WRITE_ALLOW_DELETE"}, - {AS_EXP_WRITE_POLICY_NO_FAIL, "EXP_WRITE_POLICY_NO_FAIL"}, - {AS_EXP_WRITE_EVAL_NO_FAIL, "EXP_WRITE_EVAL_NO_FAIL"}, - {AS_EXP_READ_DEFAULT, "EXP_READ_DEFAULT"}, - {AS_EXP_READ_EVAL_NO_FAIL, "EXP_READ_EVAL_NO_FAIL"}, - - /* For BinType expression, as_bytes_type */ - {AS_BYTES_UNDEF, "AS_BYTES_UNDEF"}, - {AS_BYTES_INTEGER, "AS_BYTES_INTEGER"}, - {AS_BYTES_DOUBLE, "AS_BYTES_DOUBLE"}, - {AS_BYTES_STRING, "AS_BYTES_STRING"}, - {AS_BYTES_BLOB, "AS_BYTES_BLOB"}, - {AS_BYTES_JAVA, "AS_BYTES_JAVA"}, - {AS_BYTES_CSHARP, "AS_BYTES_CSHARP"}, - {AS_BYTES_PYTHON, "AS_BYTES_PYTHON"}, - {AS_BYTES_RUBY, "AS_BYTES_RUBY"}, - {AS_BYTES_PHP, "AS_BYTES_PHP"}, - {AS_BYTES_ERLANG, "AS_BYTES_ERLANG"}, - {AS_BYTES_BOOL, "AS_BYTES_BOOL"}, - {AS_BYTES_HLL, "AS_BYTES_HLL"}, - {AS_BYTES_MAP, "AS_BYTES_MAP"}, - {AS_BYTES_LIST, "AS_BYTES_LIST"}, - {AS_BYTES_GEOJSON, "AS_BYTES_GEOJSON"}, - {AS_BYTES_TYPE_MAX, "AS_BYTES_TYPE_MAX"}, - - /* Regex constants from predexp, still used by expressions */ - {REGEX_NONE, "REGEX_NONE"}, - {REGEX_EXTENDED, "REGEX_EXTENDED"}, - {REGEX_ICASE, "REGEX_ICASE"}, - {REGEX_NOSUB, "REGEX_NOSUB"}, - {REGEX_NEWLINE, "REGEX_NEWLINE"}, - - {AS_QUERY_DURATION_LONG, "QUERY_DURATION_LONG"}, - {AS_QUERY_DURATION_LONG_RELAX_AP, "QUERY_DURATION_LONG_RELAX_AP"}, - {AS_QUERY_DURATION_SHORT, "QUERY_DURATION_SHORT"}}; - -static AerospikeJobConstants aerospike_job_constants[] = { - {"scan", "JOB_SCAN"}, {"query", "JOB_QUERY"}}; /** * Function for setting scan parameters in scan. * Like Percentage, Concurrent, Nobins @@ -549,31 +211,6 @@ as_status set_query_options(as_error *err, PyObject *query_options, } return AEROSPIKE_OK; } -/** - * Declares policy constants. - */ -as_status declare_policy_constants(PyObject *aerospike) -{ - as_status status = AEROSPIKE_OK; - int i; - - if (!aerospike) { - status = AEROSPIKE_ERR; - goto exit; - } - for (i = 0; i < (int)AEROSPIKE_CONSTANTS_ARR_SIZE; i++) { - PyModule_AddIntConstant(aerospike, aerospike_constants[i].constant_str, - aerospike_constants[i].constantno); - } - - for (i = 0; i < (int)AEROSPIKE_JOB_CONSTANTS_ARR_SIZE; i++) { - PyModule_AddStringConstant(aerospike, - aerospike_job_constants[i].exposed_job_str, - aerospike_job_constants[i].job_str); - } -exit: - return status; -} /** * Converts a PyObject into an as_policy_admin object. diff --git a/src/main/query/type.c b/src/main/query/type.c index 263487f9e..6c4758ddc 100644 --- a/src/main/query/type.c +++ b/src/main/query/type.c @@ -245,9 +245,10 @@ static void AerospikeQuery_Type_Dealloc(AerospikeQuery *self) * PYTHON TYPE DESCRIPTOR ******************************************************************************/ static PyTypeObject AerospikeQuery_Type = { - PyVarObject_HEAD_INIT(NULL, 0) "aerospike.Query", // tp_name - sizeof(AerospikeQuery), // tp_basicsize - 0, // tp_itemsize + PyVarObject_HEAD_INIT(NULL, 0) + FULLY_QUALIFIED_TYPE_NAME("Query"), // tp_name + sizeof(AerospikeQuery), // tp_basicsize + 0, // tp_itemsize (destructor)AerospikeQuery_Type_Dealloc, // tp_dealloc 0, // tp_print diff --git a/src/main/scan/type.c b/src/main/scan/type.c index 62084e319..bdbbcc0f9 100644 --- a/src/main/scan/type.c +++ b/src/main/scan/type.c @@ -188,9 +188,9 @@ static void AerospikeScan_Type_Dealloc(AerospikeScan *self) ******************************************************************************/ static PyTypeObject AerospikeScan_Type = { - PyVarObject_HEAD_INIT(NULL, 0) "aerospike.Scan", // tp_name - sizeof(AerospikeScan), // tp_basicsize - 0, // tp_itemsize + PyVarObject_HEAD_INIT(NULL, 0) FULLY_QUALIFIED_TYPE_NAME("Scan"), // tp_name + sizeof(AerospikeScan), // tp_basicsize + 0, // tp_itemsize (destructor)AerospikeScan_Type_Dealloc, // tp_dealloc 0, // tp_print From 796109c6e8919fbcc964d51d6a48f028d8fffd32 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 25 Sep 2024 22:37:10 +0000 Subject: [PATCH 32/69] Auto-bump version to 15.2.0rc1.dev1 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d14dfbac3..ee7c25585 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.1.0 +15.2.0rc1.dev1 From 318f3fbc70f67d8ddf47e3b059b790d369264381 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 01:14:17 +0000 Subject: [PATCH 33/69] Auto-bump version to 15.2.0rc1 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ee7c25585..c147f1ce5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.2.0rc1.dev1 +15.2.0rc1 From b6616ffff1c7e50f357853973c2f756c2f96b506 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 2 Oct 2024 16:19:10 -0700 Subject: [PATCH 34/69] [CLIENT-3106] Refactor base policy code and check if error indicator was set while parsing policy dictionaries (#678) - Consolidate code for parsing base policy fields into a helper function. This reduces the number of codepaths for parsing each policy type - Revert aerospike module's m_size to -1 since that was the set value before the module state code was added. We don't want to support running the aerospike module multiple times in one process because it has some global variables in the process like py_global_hosts --- src/main/aerospike.c | 1 + src/main/policy.c | 208 ++++++++++++++++++++++--------------------- 2 files changed, 109 insertions(+), 100 deletions(-) diff --git a/src/main/aerospike.c b/src/main/aerospike.c index 360b4318c..c94bc0fea 100644 --- a/src/main/aerospike.c +++ b/src/main/aerospike.c @@ -542,6 +542,7 @@ PyMODINIT_FUNC PyInit_aerospike(void) .m_name = AEROSPIKE_MODULE_NAME, .m_doc = "Aerospike Python Client", .m_methods = aerospike_methods, + .m_size = -1, }; PyObject *py_aerospike_module = PyModule_Create(&moduledef); diff --git a/src/main/policy.c b/src/main/policy.c index 0cb2b1491..e27d57a6e 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -52,26 +52,42 @@ #define POLICY_UPDATE() *policy_p = policy; +// TODO: Python exceptions should be propagated up instead of being cleared +// but the policy helper functions don't handle this case and they only populate +// an as_error object and return a status code. +// That will take too much time to refactor, so just clear the exception and +// populate the as_error object instead. This currently makes it harder to +// debug why a C-API call failed though, because we don't have the exact +// exception that was thrown #define POLICY_SET_FIELD(__field, __type) \ { \ - PyObject *py_field = PyDict_GetItemString(py_policy, #__field); \ - if (py_field) { \ - if (PyLong_Check(py_field)) { \ - policy->__field = (__type)PyLong_AsLong(py_field); \ - } \ - else { \ - return as_error_update(err, AEROSPIKE_ERR_PARAM, \ - "%s is invalid", #__field); \ - } \ + PyObject *py_field_name = PyUnicode_FromString(#__field); \ + if (py_field_name == NULL) { \ + PyErr_Clear(); \ + return as_error_update(err, AEROSPIKE_ERR_CLIENT, \ + "Unable to create Python unicode object"); \ } \ - } - -#define POLICY_SET_BASE_FIELD(__field, __type) \ - { \ - PyObject *py_field = PyDict_GetItemString(py_policy, #__field); \ + PyObject *py_field = \ + PyDict_GetItemWithError(py_policy, py_field_name); \ + if (py_field == NULL && PyErr_Occurred()) { \ + PyErr_Clear(); \ + Py_DECREF(py_field_name); \ + return as_error_update( \ + err, AEROSPIKE_ERR_CLIENT, \ + "Unable to fetch field from policy dictionary"); \ + } \ + Py_DECREF(py_field_name); \ + \ if (py_field) { \ if (PyLong_Check(py_field)) { \ - policy->base.__field = (__type)PyLong_AsLong(py_field); \ + long field_val = PyLong_AsLong(py_field); \ + if (field_val == -1 && PyErr_Occurred()) { \ + PyErr_Clear(); \ + return as_error_update( \ + err, AEROSPIKE_ERR_CLIENT, \ + "Unable to fetch long value from policy field"); \ + } \ + policy->__field = (__type)field_val; \ } \ else { \ return as_error_update(err, AEROSPIKE_ERR_PARAM, \ @@ -80,30 +96,35 @@ } \ } -#define POLICY_SET_EXPRESSIONS_BASE_FIELD() \ +#define POLICY_SET_EXPRESSIONS_FIELD() \ { \ if (exp_list) { \ + PyObject *py_field_name = PyUnicode_FromString("expressions"); \ + if (py_field_name == NULL) { \ + PyErr_Clear(); \ + return as_error_update( \ + err, AEROSPIKE_ERR_CLIENT, \ + "Unable to create Python unicode object"); \ + } \ PyObject *py_exp_list = \ - PyDict_GetItemString(py_policy, "expressions"); \ + PyDict_GetItemWithError(py_policy, py_field_name); \ + if (py_exp_list == NULL && PyErr_Occurred()) { \ + PyErr_Clear(); \ + Py_DECREF(py_field_name); \ + return as_error_update(err, AEROSPIKE_ERR_CLIENT, \ + "Unable to fetch expressions field " \ + "from policy dictionary"); \ + } \ + Py_DECREF(py_field_name); \ if (py_exp_list) { \ if (convert_exp_list(self, py_exp_list, &exp_list, err) == \ AEROSPIKE_OK) { \ - policy->base.filter_exp = exp_list; \ + policy->filter_exp = exp_list; \ *exp_list_p = exp_list; \ } \ - } \ - } \ - } - -#define POLICY_SET_EXPRESSIONS_FIELD() \ - { \ - PyObject *py_exp_list = \ - PyDict_GetItemString(py_policy, "expressions"); \ - if (py_exp_list) { \ - if (convert_exp_list(self, py_exp_list, &exp_list, err) == \ - AEROSPIKE_OK) { \ - policy->filter_exp = exp_list; \ - *exp_list_p = exp_list; \ + else { \ + return err->code; \ + } \ } \ } \ } @@ -219,8 +240,7 @@ as_status set_query_options(as_error *err, PyObject *query_options, * and initialized (although, we do reset the error object here). */ as_status pyobject_to_policy_admin(AerospikeClient *self, as_error *err, - PyObject *py_policy, // remove self - as_policy_admin *policy, + PyObject *py_policy, as_policy_admin *policy, as_policy_admin **policy_p, as_policy_admin *config_admin_policy) { @@ -242,6 +262,21 @@ as_status pyobject_to_policy_admin(AerospikeClient *self, as_error *err, return err->code; } +static inline as_status +pyobject_to_policy_base(AerospikeClient *self, as_error *err, + PyObject *py_policy, as_policy_base *policy, + as_exp *exp_list, as_exp **exp_list_p) +{ + POLICY_SET_FIELD(total_timeout, uint32_t); + POLICY_SET_FIELD(socket_timeout, uint32_t); + POLICY_SET_FIELD(max_retries, uint32_t); + POLICY_SET_FIELD(sleep_between_retries, uint32_t); + POLICY_SET_FIELD(compress, bool); + + POLICY_SET_EXPRESSIONS_FIELD(); + return AEROSPIKE_OK; +} + /** * Converts a PyObject into an as_policy_apply object. * Returns AEROSPIKE_OK on success. On error, the err argument is populated. @@ -263,11 +298,11 @@ as_status pyobject_to_policy_apply(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - POLICY_SET_BASE_FIELD(total_timeout, uint32_t); - POLICY_SET_BASE_FIELD(socket_timeout, uint32_t); - POLICY_SET_BASE_FIELD(max_retries, uint32_t); - POLICY_SET_BASE_FIELD(sleep_between_retries, uint32_t); - POLICY_SET_BASE_FIELD(compress, bool); + as_status retval = pyobject_to_policy_base( + self, err, py_policy, &policy->base, exp_list, exp_list_p); + if (retval != AEROSPIKE_OK) { + return retval; + } POLICY_SET_FIELD(key, as_policy_key); POLICY_SET_FIELD(replica, as_policy_replica); @@ -275,9 +310,6 @@ as_status pyobject_to_policy_apply(AerospikeClient *self, as_error *err, POLICY_SET_FIELD(commit_level, as_policy_commit_level); POLICY_SET_FIELD(durable_delete, bool); POLICY_SET_FIELD(ttl, uint32_t); - - // C client 5.0 new expressions - POLICY_SET_EXPRESSIONS_BASE_FIELD(); } // Update the policy @@ -337,19 +369,14 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, as_policy_query_copy(config_query_policy, policy); if (py_policy && py_policy != Py_None) { - // Set policy fields - POLICY_SET_BASE_FIELD(total_timeout, uint32_t); - POLICY_SET_BASE_FIELD(socket_timeout, uint32_t); - POLICY_SET_BASE_FIELD(max_retries, uint32_t); - POLICY_SET_BASE_FIELD(sleep_between_retries, uint32_t); - POLICY_SET_BASE_FIELD(compress, bool); - + as_status retval = pyobject_to_policy_base( + self, err, py_policy, &policy->base, exp_list, exp_list_p); + if (retval != AEROSPIKE_OK) { + return retval; + } POLICY_SET_FIELD(deserialize, bool); POLICY_SET_FIELD(replica, as_policy_replica); - // C client 5.0 new expressions - POLICY_SET_EXPRESSIONS_BASE_FIELD(); - // C client 6.0.0 POLICY_SET_FIELD(short_query, bool); @@ -384,11 +411,11 @@ as_status pyobject_to_policy_read(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - POLICY_SET_BASE_FIELD(total_timeout, uint32_t); - POLICY_SET_BASE_FIELD(socket_timeout, uint32_t); - POLICY_SET_BASE_FIELD(max_retries, uint32_t); - POLICY_SET_BASE_FIELD(sleep_between_retries, uint32_t); - POLICY_SET_BASE_FIELD(compress, bool); + as_status retval = pyobject_to_policy_base( + self, err, py_policy, &policy->base, exp_list, exp_list_p); + if (retval != AEROSPIKE_OK) { + return retval; + } POLICY_SET_FIELD(key, as_policy_key); POLICY_SET_FIELD(replica, as_policy_replica); @@ -398,9 +425,6 @@ as_status pyobject_to_policy_read(AerospikeClient *self, as_error *err, // 4.0.0 new policies POLICY_SET_FIELD(read_mode_ap, as_policy_read_mode_ap); POLICY_SET_FIELD(read_mode_sc, as_policy_read_mode_sc); - - // C client 5.0 new expressions - POLICY_SET_EXPRESSIONS_BASE_FIELD(); } // Update the policy @@ -431,11 +455,11 @@ as_status pyobject_to_policy_remove(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - POLICY_SET_BASE_FIELD(total_timeout, uint32_t); - POLICY_SET_BASE_FIELD(socket_timeout, uint32_t); - POLICY_SET_BASE_FIELD(max_retries, uint32_t); - POLICY_SET_BASE_FIELD(sleep_between_retries, uint32_t); - POLICY_SET_BASE_FIELD(compress, bool); + as_status retval = pyobject_to_policy_base( + self, err, py_policy, &policy->base, exp_list, exp_list_p); + if (retval != AEROSPIKE_OK) { + return retval; + } POLICY_SET_FIELD(generation, uint16_t); @@ -444,9 +468,6 @@ as_status pyobject_to_policy_remove(AerospikeClient *self, as_error *err, POLICY_SET_FIELD(commit_level, as_policy_commit_level); POLICY_SET_FIELD(replica, as_policy_replica); POLICY_SET_FIELD(durable_delete, bool); - - // C client 5.0 new expressions - POLICY_SET_EXPRESSIONS_BASE_FIELD(); } // Update the policy @@ -476,19 +497,16 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - POLICY_SET_BASE_FIELD(total_timeout, uint32_t); - POLICY_SET_BASE_FIELD(socket_timeout, uint32_t); - POLICY_SET_BASE_FIELD(max_retries, uint32_t); - POLICY_SET_BASE_FIELD(sleep_between_retries, uint32_t); - POLICY_SET_BASE_FIELD(compress, bool); + as_status retval = pyobject_to_policy_base( + self, err, py_policy, &policy->base, exp_list, exp_list_p); + if (retval != AEROSPIKE_OK) { + return retval; + } POLICY_SET_FIELD(durable_delete, bool); POLICY_SET_FIELD(records_per_second, uint32_t); POLICY_SET_FIELD(max_records, uint64_t); POLICY_SET_FIELD(replica, as_policy_replica); - - // C client 5.0 new expressions - POLICY_SET_EXPRESSIONS_BASE_FIELD(); } // Update the policy @@ -518,12 +536,11 @@ as_status pyobject_to_policy_write(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - // Base policy_fields - POLICY_SET_BASE_FIELD(total_timeout, uint32_t); - POLICY_SET_BASE_FIELD(socket_timeout, uint32_t); - POLICY_SET_BASE_FIELD(max_retries, uint32_t); - POLICY_SET_BASE_FIELD(sleep_between_retries, uint32_t); - POLICY_SET_BASE_FIELD(compress, bool); + as_status retval = pyobject_to_policy_base( + self, err, py_policy, &policy->base, exp_list, exp_list_p); + if (retval != AEROSPIKE_OK) { + return retval; + } POLICY_SET_FIELD(key, as_policy_key); POLICY_SET_FIELD(gen, as_policy_gen); @@ -532,9 +549,6 @@ as_status pyobject_to_policy_write(AerospikeClient *self, as_error *err, POLICY_SET_FIELD(durable_delete, bool); POLICY_SET_FIELD(replica, as_policy_replica); POLICY_SET_FIELD(compression_threshold, uint32_t); - - // C client 5.0 new expressions - POLICY_SET_EXPRESSIONS_BASE_FIELD(); } // Update the policy @@ -565,11 +579,11 @@ as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - POLICY_SET_BASE_FIELD(total_timeout, uint32_t); - POLICY_SET_BASE_FIELD(socket_timeout, uint32_t); - POLICY_SET_BASE_FIELD(max_retries, uint32_t); - POLICY_SET_BASE_FIELD(sleep_between_retries, uint32_t); - POLICY_SET_BASE_FIELD(compress, bool); + as_status retval = pyobject_to_policy_base( + self, err, py_policy, &policy->base, exp_list, exp_list_p); + if (retval != AEROSPIKE_OK) { + return retval; + } POLICY_SET_FIELD(key, as_policy_key); POLICY_SET_FIELD(gen, as_policy_gen); @@ -583,9 +597,6 @@ as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, // 4.0.0 new policies POLICY_SET_FIELD(read_mode_ap, as_policy_read_mode_ap); POLICY_SET_FIELD(read_mode_sc, as_policy_read_mode_sc); - - // C client 5.0 new expressions - POLICY_SET_EXPRESSIONS_BASE_FIELD(); } // Update the policy @@ -615,11 +626,11 @@ as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - POLICY_SET_BASE_FIELD(total_timeout, uint32_t); - POLICY_SET_BASE_FIELD(socket_timeout, uint32_t); - POLICY_SET_BASE_FIELD(max_retries, uint32_t); - POLICY_SET_BASE_FIELD(sleep_between_retries, uint32_t); - POLICY_SET_BASE_FIELD(compress, bool); + as_status retval = pyobject_to_policy_base( + self, err, py_policy, &policy->base, exp_list, exp_list_p); + if (retval != AEROSPIKE_OK) { + return retval; + } POLICY_SET_FIELD(concurrent, bool); POLICY_SET_FIELD(allow_inline, bool); @@ -631,9 +642,6 @@ as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, POLICY_SET_FIELD(read_mode_ap, as_policy_read_mode_ap); POLICY_SET_FIELD(read_mode_sc, as_policy_read_mode_sc); - // C client 5.0 new expressions - POLICY_SET_EXPRESSIONS_BASE_FIELD(); - // C client 6.0.0 (batch writes) POLICY_SET_FIELD(allow_inline_ssd, bool); POLICY_SET_FIELD(respond_all_keys, bool); From a1c9e421eb1683d8811c1b3bca04fe2f73a8dbc5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 2 Oct 2024 23:19:44 +0000 Subject: [PATCH 35/69] Auto-bump version to 15.2.0rc2.dev1 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c147f1ce5..657281240 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.2.0rc1 +15.2.0rc2.dev1 From fb6a10cc51fc0f99b8799189962be6896a59adee Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 21 Oct 2024 12:01:24 -0700 Subject: [PATCH 36/69] [CLIENT-3148] CI/CD: Fix failing client tests due to changed default aerospike.conf in server 7.2.0.1_2 Docker image (#685) --- .github/actions/run-ee-server/action.yml | 2 +- .github/workflows/test-server-rc.yml | 2 +- .github/workflows/tests.yml | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/run-ee-server/action.yml b/.github/actions/run-ee-server/action.yml index 5bc57ff06..bf17a1756 100644 --- a/.github/actions/run-ee-server/action.yml +++ b/.github/actions/run-ee-server/action.yml @@ -72,7 +72,7 @@ runs: # so we have to manually enable it load: true - - run: docker run -d --name aerospike -p 3000:3000 ${{ env.SECURITY_IMAGE_NAME }} + - run: docker run -d --name aerospike -p 3000:3000 -e DEFAULT_TTL=2592000 ${{ env.SECURITY_IMAGE_NAME }} shell: bash - uses: ./.github/actions/wait-for-as-server-to-start diff --git a/.github/workflows/test-server-rc.yml b/.github/workflows/test-server-rc.yml index d0ccbd0e2..e03a6cb8a 100644 --- a/.github/workflows/test-server-rc.yml +++ b/.github/workflows/test-server-rc.yml @@ -145,7 +145,7 @@ jobs: password: ${{ secrets.DOCKER_HUB_BOT_PW }} - name: Run server RC - run: docker run -d -p 3000:3000 --name aerospike ${{ vars.SERVER_RC_REPO_LINK }} + run: docker run -d -p 3000:3000 --name aerospike -e DEFAULT_TTL=2592000 ${{ vars.SERVER_RC_REPO_LINK }} - name: Create config.conf run: cp config.conf.template config.conf diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c308d2301..f5c703780 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -230,7 +230,7 @@ jobs: run: pip install -r test/requirements.txt - name: Run Aerospike server - run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server + run: docker run -d --name aerospike -p 3000-3002:3000-3002 -e DEFAULT_TTL=2592000 aerospike/aerospike-server - name: Create config.conf run: cp config.conf.template config.conf @@ -286,11 +286,11 @@ jobs: - name: Run Aerospike server release candidate with latest tag if: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }} - run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server-rc:latest + run: docker run -d --name aerospike -p 3000-3002:3000-3002 -e DEFAULT_TTL=2592000 aerospike/aerospike-server-rc:latest - name: Run Aerospike server if: ${{ !contains(github.event.pull_request.labels.*.name, 'new-server-features') }} - run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server + run: docker run -d --name aerospike -p 3000-3002:3000-3002 -e DEFAULT_TTL=2592000 aerospike/aerospike-server - name: Create config.conf run: cp config.conf.template config.conf @@ -328,7 +328,7 @@ jobs: run: pip install -r test/requirements.txt - name: Run lowest supported server - run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server:${{ vars.LOWEST_SUPPORTED_SERVER_VERSION }} + run: docker run -d --name aerospike -p 3000-3002:3000-3002 -e DEFAULT_TTL=2592000 aerospike/aerospike-server:${{ vars.LOWEST_SUPPORTED_SERVER_VERSION }} - name: Create config.conf run: cp config.conf.template config.conf From 14d97bd81ba31efe4713806b5499bb562178aa10 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 19:01:57 +0000 Subject: [PATCH 37/69] Auto-bump version to 15.2.0rc2.dev2 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 657281240..c647cf0f8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.2.0rc2.dev1 +15.2.0rc2.dev2 From c1c9264e12c080ba42a35843cc1c272d0ad1e8e7 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Fri, 1 Nov 2024 11:46:06 -0700 Subject: [PATCH 38/69] [CLIENT-2217] macOS wheels: update OpenSSL dependency from 1.1.1 to 3 (#471) --- .github/workflows/build-wheels.yml | 3 ++- BUILD.md | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index be0f8f342..df4d04d41 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -153,6 +153,7 @@ jobs: runs-on: ${{ needs.get-runner-os.outputs.runner-os }} env: BUILD_IDENTIFIER: "${{ matrix.python-tag }}-${{ inputs.platform-tag }}" + MACOS_OPENSSL_VERSION: 3 steps: - name: Create status check message run: echo STATUS_CHECK_MESSAGE="cibuildwheel (${{ env.BUILD_IDENTIFIER }})" >> $GITHUB_ENV @@ -241,7 +242,7 @@ jobs: uses: pypa/cibuildwheel@v2.20.0 env: CIBW_ENVIRONMENT_PASS_LINUX: ${{ inputs.unoptimized && 'UNOPTIMIZED' || '' }} - CIBW_ENVIRONMENT_MACOS: SSL_LIB_PATH="$(brew --prefix openssl@1.1)/lib/" CPATH="$(brew --prefix openssl@1.1)/include/" STATIC_SSL=1 + CIBW_ENVIRONMENT_MACOS: SSL_LIB_PATH="$(brew --prefix openssl@${{ env.MACOS_OPENSSL_VERSION }})/lib/" CPATH="$(brew --prefix openssl@${{ env.MACOS_OPENSSL_VERSION }})/include/" STATIC_SSL=1 CIBW_BUILD: ${{ env.BUILD_IDENTIFIER }} CIBW_BUILD_FRONTEND: build CIBW_BEFORE_ALL_LINUX: > diff --git a/BUILD.md b/BUILD.md index 0b882e5fe..61504b29a 100644 --- a/BUILD.md +++ b/BUILD.md @@ -87,8 +87,7 @@ By default macOS will be missing command line tools. The dependencies can be installed through the macOS package manager [Homebrew](http://brew.sh/). - brew install openssl@1 - # brew uninstall openssl@3 + brew install openssl@3 ### All distros @@ -109,8 +108,8 @@ using the wrong version of the C client. This can causes strange issues when bui Also, for macOS or any other operating system that doesn't have OpenSSL installed by default, you must install it and specify its location when building the wheel. In macOS, you would run these commands: ``` -export SSL_LIB_PATH="$(brew --prefix openssl@1.1)/lib/" -export CPATH="$(brew --prefix openssl@1.1)/include/" +export SSL_LIB_PATH="$(brew --prefix openssl@3)/lib/" +export CPATH="$(brew --prefix openssl@3)/include/" export STATIC_SSL=1 ``` From a11ed94421688deb74afb778bb23717078f96f97 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 1 Nov 2024 18:46:42 +0000 Subject: [PATCH 39/69] Auto-bump version to 15.2.0rc2.dev3 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c647cf0f8..03b42ed2a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.2.0rc2.dev2 +15.2.0rc2.dev3 From d8eff6eac1ec95e1f6ca3bf65b1903ed27a5047f Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Nov 2024 14:03:49 -0800 Subject: [PATCH 40/69] [CLIENT-3121] Support multi-record transactions (#674) CI/CD: - Always deploy enterprise edition server with strong consistency enabled along with security features - Create separate folder for Dockerfile build context when deploying an enterprise edition server - Allow running valgrind test on specific server version. This allows running against a server alpha version for MRTs when the latest tag for a server release candidate points to the next upcoming server release preceding MRTs - Add option to run build wheels with specific test files for quick smoke testing - Show exact test names that failed while tests are running during build wheels workflow - Fix documentation spell check workflow failing - Fix test node close listener for metrics - Set default Docker image name for Dockerfile that deploys an EE server for testing Misc: - Remove commented out list and map operations in type stubs that were removed in a prior release - Cluster API breaking change: rename tran_count to command_count --- .github/actions/run-ee-server/action.yml | 49 ++++- .../wait-for-as-server-to-start/action.yml | 5 +- .github/workflows/Dockerfile | 10 - .github/workflows/build-wheels.yml | 12 +- .../workflows/docker-build-context/Dockerfile | 49 +++++ .../workflows/docker-build-context/roster.smd | 12 ++ .../{ => docker-build-context}/security.smd | 0 .github/workflows/tests.yml | 36 +--- .github/workflows/valgrind.yml | 4 + .../wait-for-as-server-to-start.bash | 10 +- .gitmodules | 2 +- aerospike-client-c | 2 +- aerospike-stubs/aerospike.pyi | 73 +++---- aerospike_helpers/expressions/base.py | 6 +- aerospike_helpers/metrics/__init__.py | 14 +- doc/aerospike.rst | 74 +++++++ doc/client.rst | 184 +++++++++++------ doc/conf.py | 2 +- doc/data_mapping.rst | 2 +- doc/index.rst | 2 + doc/spelling_wordlist.txt | 11 + doc/transaction.rst | 55 +++++ setup.py | 4 +- src/include/client.h | 7 + src/include/transaction.h | 3 + src/include/types.h | 9 + src/main/aerospike.c | 27 +++ src/main/client/mrt.c | 74 +++++++ src/main/client/type.c | 5 + src/main/conversions.c | 6 +- src/main/exception.c | 2 + src/main/policy.c | 44 ++++ src/main/transaction/type.c | 190 ++++++++++++++++++ test/metrics/test_node_close_listener.py | 63 +++--- test/new_tests/conftest.py | 27 +++ test/new_tests/test_base_class.py | 1 - test/new_tests/test_metrics.py | 2 +- test/new_tests/test_mrt_api.py | 87 ++++++++ test/new_tests/test_mrt_functionality.py | 104 ++++++++++ test/new_tests/test_query.py | 2 + 40 files changed, 1053 insertions(+), 218 deletions(-) delete mode 100644 .github/workflows/Dockerfile create mode 100644 .github/workflows/docker-build-context/Dockerfile create mode 100644 .github/workflows/docker-build-context/roster.smd rename .github/workflows/{ => docker-build-context}/security.smd (100%) create mode 100644 doc/transaction.rst create mode 100644 src/include/transaction.h create mode 100644 src/main/client/mrt.c create mode 100644 src/main/transaction/type.c create mode 100644 test/new_tests/test_mrt_api.py create mode 100644 test/new_tests/test_mrt_functionality.py diff --git a/.github/actions/run-ee-server/action.yml b/.github/actions/run-ee-server/action.yml index bf17a1756..d83d18e4d 100644 --- a/.github/actions/run-ee-server/action.yml +++ b/.github/actions/run-ee-server/action.yml @@ -6,15 +6,19 @@ inputs: # All inputs in composite actions are strings use-server-rc: required: true - default: false + description: Deploy server release candidate? + default: 'false' server-tag: required: true + description: Specify Docker tag default: 'latest' # Github Composite Actions can't access secrets # so we need to pass them in as inputs docker-hub-username: + description: Required for using release candidates required: false docker-hub-password: + description: Required for using release candidates required: false runs: @@ -35,13 +39,17 @@ runs: run: | crudini --existing=param --set config.conf enterprise-edition hosts '' crudini --existing=param --set config.conf enterprise-edition hosts 127.0.0.1:3000 - crudini --existing=param --set config.conf enterprise-edition user superuser - crudini --existing=param --set config.conf enterprise-edition password superuser working-directory: test shell: bash - - name: Create config folder to store configs in - run: mkdir configs + - run: echo SUPERUSER_NAME_AND_PASSWORD="superuser" >> $GITHUB_ENV + shell: bash + + - name: Set credentials in config file + run: | + crudini --existing=param --set config.conf enterprise-edition user ${{ env.SUPERUSER_NAME_AND_PASSWORD }} + crudini --existing=param --set config.conf enterprise-edition password ${{ env.SUPERUSER_NAME_AND_PASSWORD }} + working-directory: test shell: bash - name: Log into Docker Hub to get server RC @@ -52,7 +60,7 @@ runs: - run: echo IMAGE_NAME=aerospike/aerospike-server-enterprise${{ inputs.use-server-rc == 'true' && '-rc' || '' }}:${{ inputs.server-tag }} >> $GITHUB_ENV shell: bash - - run: echo SECURITY_IMAGE_NAME=${{ env.IMAGE_NAME }}-security >> $GITHUB_ENV + - run: echo NEW_IMAGE_NAME=${{ env.IMAGE_NAME }}-security-and-sc >> $GITHUB_ENV shell: bash # macOS Github runners and Windows self-hosted runners don't have buildx installed by default @@ -63,19 +71,38 @@ runs: uses: docker/build-push-action@v6 with: # Don't want to use default Git context or else it will clone the whole Python client repo again - context: .github/workflows + context: .github/workflows/docker-build-context build-args: | - image=${{ env.IMAGE_NAME }} - tags: ${{ env.SECURITY_IMAGE_NAME }} + server_image=${{ env.IMAGE_NAME }} + tags: ${{ env.NEW_IMAGE_NAME }} # setup-buildx-action configures Docker to use the docker-container build driver # This driver doesn't publish an image locally by default # so we have to manually enable it load: true - - run: docker run -d --name aerospike -p 3000:3000 -e DEFAULT_TTL=2592000 ${{ env.SECURITY_IMAGE_NAME }} + - run: echo SERVER_CONTAINER_NAME="aerospike" >> $GITHUB_ENV + shell: bash + + - run: docker run -d --name ${{ env.SERVER_CONTAINER_NAME }} -e DEFAULT_TTL=2592000 -p 3000:3000 ${{ env.NEW_IMAGE_NAME }} shell: bash - uses: ./.github/actions/wait-for-as-server-to-start with: - container-name: aerospike + container-name: ${{ env.SERVER_CONTAINER_NAME }} is-security-enabled: true + is-strong-consistency-enabled: true + + - run: echo ASADM_AUTH_FLAGS="--user=${{ env.SUPERUSER_NAME_AND_PASSWORD }} --password=${{ env.SUPERUSER_NAME_AND_PASSWORD }}" >> $GITHUB_ENV + shell: bash + + # All the partitions are assumed to be dead when reusing a roster file + - run: docker exec ${{ env.SERVER_CONTAINER_NAME }} asadm $ASADM_AUTH_FLAGS --enable --execute "manage revive ns test" + shell: bash + + # Apply changes + - run: docker exec ${{ env.SERVER_CONTAINER_NAME }} asadm $ASADM_AUTH_FLAGS --enable --execute "manage recluster" + shell: bash + + # For debugging + - run: docker logs aerospike + shell: bash diff --git a/.github/actions/wait-for-as-server-to-start/action.yml b/.github/actions/wait-for-as-server-to-start/action.yml index 26841102b..373c26970 100644 --- a/.github/actions/wait-for-as-server-to-start/action.yml +++ b/.github/actions/wait-for-as-server-to-start/action.yml @@ -6,6 +6,9 @@ inputs: is-security-enabled: required: false default: 'false' + is-strong-consistency-enabled: + required: false + default: 'false' runs: using: "composite" @@ -21,5 +24,5 @@ runs: # Also, we don't want to fail if we timeout in case the server *did* finish starting up but the script couldn't detect it due to a bug # Effectively, this composite action is like calling "sleep" that is optimized to exit early when it detects an ok from the server - name: Wait for EE server to start - run: timeout 30 bash ./.github/workflows/wait-for-as-server-to-start.bash ${{ inputs.container-name }} ${{ inputs.is-security-enabled }} || true + run: timeout 30 bash ./.github/workflows/wait-for-as-server-to-start.bash ${{ inputs.container-name }} ${{ inputs.is-security-enabled }} ${{ inputs.is-strong-consistency-enabled }} || true shell: bash diff --git a/.github/workflows/Dockerfile b/.github/workflows/Dockerfile deleted file mode 100644 index e15848240..000000000 --- a/.github/workflows/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG image -FROM $image -RUN echo -e "security {\n\tenable-quotas true\n}\n" >> /etc/aerospike/aerospike.template.conf -# security.smd was generated manually by -# 1. Starting a new Aerospike EE server using Docker -# 2. Creating the superuser user -# 3. Copying /opt/aerospike/smd/security.smd from the container and committing it to this repo -# This file should always work -# TODO: generate this automatically, somehow -COPY security.smd /opt/aerospike/smd/ diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index df4d04d41..5d99c7917 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -65,6 +65,10 @@ on: required: true default: 'latest' description: 'Server docker image tag' + test-file: + required: false + default: '' + description: 'new_tests/' workflow_call: inputs: @@ -106,6 +110,10 @@ on: type: string default: 'latest' description: 'Server docker image tag' + test-file: + required: false + type: string + default: '' secrets: # Just make all the secrets required to make things simpler... DOCKER_HUB_BOT_USERNAME: @@ -227,7 +235,7 @@ jobs: - name: Otherwise, enable integration tests if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }} - run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest new_tests/" >> $GITHUB_ENV + run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest -vv new_tests/${{ inputs.test-file }}" >> $GITHUB_ENV shell: bash - name: Set unoptimize flag @@ -341,7 +349,7 @@ jobs: working-directory: test shell: bash - - run: python3 -m pytest new_tests/ + - run: python3 -m pytest -vv new_tests/${{ inputs.test-file }} working-directory: test shell: bash diff --git a/.github/workflows/docker-build-context/Dockerfile b/.github/workflows/docker-build-context/Dockerfile new file mode 100644 index 000000000..9ac06a152 --- /dev/null +++ b/.github/workflows/docker-build-context/Dockerfile @@ -0,0 +1,49 @@ +ARG server_image=aerospike/aerospike-server-enterprise +ARG ROSTER_FILE_NAME=roster.smd +# Temp file for passing node id from one build stage to another +# Docker doesn't support command substitution for setting values for ARG variables, so we have to do this +ARG NODE_ID_FILE_NAME=node_id + +FROM $server_image as configure-server + +WORKDIR /opt/aerospike/smd + +# Enable authentication + +ARG AEROSPIKE_CONF_TEMPLATE_PATH=/etc/aerospike/aerospike.template.conf + +# Not using asconfig to edit config because we are working with a template file, which may not have valid values yet +RUN echo -e "security {\n\tenable-quotas true\n}\n" >> $AEROSPIKE_CONF_TEMPLATE_PATH +# security.smd was generated manually by +# 1. Starting a new Aerospike EE server using Docker +# 2. Creating the superuser user +# 3. Copying /opt/aerospike/smd/security.smd from the container and committing it to this repo +# This file should always work +# TODO: generate this automatically, somehow. +COPY security.smd . + +# Enable strong consistency +RUN sed -i "s/\(namespace.*{\)/\1\n\tstrong-consistency true/" $AEROSPIKE_CONF_TEMPLATE_PATH +RUN sed -i "s/\(namespace.*{\)/\1\n\tstrong-consistency-allow-expunge true/" $AEROSPIKE_CONF_TEMPLATE_PATH +ARG ROSTER_FILE_NAME +COPY $ROSTER_FILE_NAME . + +# Fetch node id from roster.smd + +# There's no tag for the latest major version to prevent breaking changes in jq +# This is the next best thing +FROM ghcr.io/jqlang/jq:1.7 as get-jq +# jq docker image doesn't have a shell +# We need a shell to fetch and pass the node id to the next build stage +FROM busybox as get-node-id +COPY --from=get-jq /jq /bin/ +ARG ROSTER_FILE_NAME +COPY $ROSTER_FILE_NAME . +ARG NODE_ID_FILE_NAME +RUN jq --raw-output '.[1].value' $ROSTER_FILE_NAME > $NODE_ID_FILE_NAME + +FROM configure-server as set-node-id +ARG NODE_ID_FILE_NAME +COPY --from=get-node-id $NODE_ID_FILE_NAME . +RUN sed -i "s/\(^service {\)/\1\n\tnode-id $(cat $NODE_ID_FILE_NAME)/" $AEROSPIKE_CONF_TEMPLATE_PATH +RUN rm $NODE_ID_FILE_NAME diff --git a/.github/workflows/docker-build-context/roster.smd b/.github/workflows/docker-build-context/roster.smd new file mode 100644 index 000000000..66daed5f6 --- /dev/null +++ b/.github/workflows/docker-build-context/roster.smd @@ -0,0 +1,12 @@ +[ + [ + 97107025374203, + 1 + ], + { + "key": "test", + "value": "a1", + "generation": 1, + "timestamp": 465602976982 + } +] diff --git a/.github/workflows/security.smd b/.github/workflows/docker-build-context/security.smd similarity index 100% rename from .github/workflows/security.smd rename to .github/workflows/docker-build-context/security.smd diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f5c703780..4279dc17e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -127,6 +127,7 @@ jobs: mkdir -p nullobject/src/main/nullobject mkdir -p query/src/main/query mkdir -p scan/src/main/scan + mkdir -p transaction/src/main/transaction cd ../../../../ cp src/main/*.c build/temp*/src/main/src/main @@ -138,6 +139,7 @@ jobs: cp src/main/nullobject/*.c build/temp*/src/main/nullobject/src/main/nullobject/ cp src/main/query/*.c build/temp*/src/main/query/src/main/query/ cp src/main/scan/*.c build/temp*/src/main/scan/src/main/scan/ + cp src/main/transaction/*.c build/temp*/src/main/transaction/src/main/transaction/ - name: Generate coverage report for all object files if: ${{ !cancelled() }} @@ -372,7 +374,7 @@ jobs: docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} - name: Run tests - run: python -m pytest ./new_tests/test_admin_*.py + run: python -m pytest ./new_tests/test_{mrt_functionality,admin_*}.py working-directory: test - name: Show logs if failed @@ -412,30 +414,14 @@ jobs: run: sphinx-build -b linkcheck . links working-directory: doc - test-metrics-node-close-listener: - needs: build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ env.LOWEST_SUPPORTED_PY_VERSION }} - architecture: 'x64' - - name: Download aerolab - run: wget https://github.com/aerospike/aerolab/releases/download/7.5.2/aerolab-linux-amd64-7.5.2.deb - - name: Install aerolab - run: sudo dpkg -i *.deb - - name: Tell aerolab to use Docker - run: aerolab config backend -t docker - - uses: actions/download-artifact@v3 - with: - name: wheel-${{ env.LOWEST_SUPPORTED_PY_VERSION }} - - run: python3 -m pip install *.whl - - run: python3 test_node_close_listener.py - working-directory: test/metrics - - test-metrics-cluster-name: + test-metrics: needs: build + strategy: + matrix: + suffix: + - node_close_listener + - cluster_name + fail-fast: false runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -449,5 +435,5 @@ jobs: - run: python3 -m pip install *.whl - run: python3 -m pip install -r requirements.txt working-directory: test/metrics - - run: python3 test_cluster_name.py + - run: python3 test_${{ matrix.suffix }}.py working-directory: test/metrics diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 15de27cee..1eff9db25 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -12,6 +12,9 @@ on: description: 'Use server release candidate?' required: true default: false + server-tag: + required: false + default: latest massif: type: boolean description: 'Use massif for testing memory usage' @@ -57,6 +60,7 @@ jobs: uses: ./.github/actions/run-ee-server with: use-server-rc: ${{ inputs.use-server-rc }} + server-tag: ${{ inputs.server-tag }} docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} diff --git a/.github/workflows/wait-for-as-server-to-start.bash b/.github/workflows/wait-for-as-server-to-start.bash index c43e17da5..18189f193 100755 --- a/.github/workflows/wait-for-as-server-to-start.bash +++ b/.github/workflows/wait-for-as-server-to-start.bash @@ -6,6 +6,7 @@ set -o pipefail container_name=$1 is_security_enabled=$2 +is_strong_consistency_enabled=$3 if [[ $is_security_enabled == true ]]; then # We need to pass credentials to asinfo if server requires it @@ -38,7 +39,14 @@ done while true; do echo "Waiting for server to stabilize (i.e return a cluster key)..." # We assume that when an ERROR is returned, the cluster is not stable yet (i.e not fully initialized) - if docker exec "$container_name" asinfo $user_credentials -v cluster-stable 2>&1 | (! grep -qE "^ERROR"); then + cluster_stable_info_cmd="cluster-stable" + if [[ $is_strong_consistency_enabled == true ]]; then + # The Dockerfile uses a roster from a previously running Aerospike server in a Docker container + # When we reuse this roster, the server assumes all of its partitions are dead because it's running on a new + # storage device. + cluster_stable_info_cmd="$cluster_stable_info_cmd:ignore-migrations=true" + fi + if docker exec "$container_name" asinfo $user_credentials -v $cluster_stable_info_cmd 2>&1 | (! grep -qE "^ERROR"); then echo "Server is in a stable state." break fi diff --git a/.gitmodules b/.gitmodules index 136ba68cb..05ae9fd87 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,4 +2,4 @@ path = aerospike-client-c # url = git@github.com:aerospike/aerospike-client-c.git url = https://github.com/aerospike/aerospike-client-c.git - branch = stage + branch = CLIENT-2294 diff --git a/aerospike-client-c b/aerospike-client-c index 0fabfa7f5..c6e67c3a2 160000 --- a/aerospike-client-c +++ b/aerospike-client-c @@ -1 +1 @@ -Subproject commit 0fabfa7f52d7ee28b7ae80a621b791b9ec9e2fe8 +Subproject commit c6e67c3a2f37fc8ef85ea7fdd335fa05dfd5195e diff --git a/aerospike-stubs/aerospike.pyi b/aerospike-stubs/aerospike.pyi index 04589c5ba..8d4476f65 100644 --- a/aerospike-stubs/aerospike.pyi +++ b/aerospike-stubs/aerospike.pyi @@ -1,4 +1,4 @@ -from typing import Any, Callable, Union, final, Literal, Optional +from typing import Any, Callable, Union, final, Literal, Optional, Final from aerospike_helpers.batch.records import BatchRecords from aerospike_helpers.metrics import MetricsPolicy @@ -293,6 +293,25 @@ QUERY_DURATION_LONG: Literal[0] QUERY_DURATION_SHORT: Literal[1] QUERY_DURATION_LONG_RELAX_AP: Literal[2] +MRT_COMMIT_OK: Literal[0] +MRT_COMMIT_ALREADY_COMMITTED: Literal[1] +MRT_COMMIT_ALREADY_ABORTED: Literal[2] +MRT_COMMIT_VERIFY_FAILED: Literal[3] +MRT_COMMIT_MARK_ROLL_FORWARD_ABANDONED: Literal[4] +MRT_COMMIT_ROLL_FORWARD_ABANDONED: Literal[5] +MRT_COMMIT_CLOSE_ABANDONED: Literal[6] + +MRT_ABORT_OK: Literal[0] +MRT_ABORT_ALREADY_COMMITTED: Literal[1] +MRT_ABORT_ALREADY_ABORTED: Literal[2] +MRT_ABORT_ROLL_BACK_ABANDONED: Literal[3] +MRT_ABORT_CLOSE_ABANDONED: Literal[4] + +MRT_STATE_OPEN: Literal[0] +MRT_STATE_VERIFIED: Literal[1] +MRT_STATE_COMMITTED: Literal[2] +MRT_STATE_ABORTED: Literal[3] + @final class CDTInfinite: def __init__(self, *args, **kwargs) -> None: ... @@ -301,6 +320,14 @@ class CDTInfinite: class CDTWildcard: def __init__(self, *args, **kwargs) -> None: ... +@final +class Transaction: + def __init__(self, reads_capacity: int = 128, writes_capacity: int = 128) -> None: ... + id: int + in_doubt: bool + state: int + timeout: int + class Client: def __init__(self, *args, **kwargs) -> None: ... def admin_change_password(self, username: str, password: str, policy: dict = ...) -> None: ... @@ -359,48 +386,6 @@ class Client: def job_info(self, job_id: int, module, policy: dict = ...) -> dict: ... def enable_metrics(self, policy: Optional[MetricsPolicy] = None) -> None: ... def disable_metrics(self) -> None: ... - # List and map operations in the aerospike module are deprecated and undocumented - # def list_append(self, *args, **kwargs) -> Any: ... - # def list_clear(self, *args, **kwargs) -> Any: ... - # def list_extend(self, *args, **kwargs) -> Any: ... - # def list_get(self, *args, **kwargs) -> Any: ... - # def list_get_range(self, *args, **kwargs) -> Any: ... - # def list_insert(self, *args, **kwargs) -> Any: ... - # def list_insert_items(self, *args, **kwargs) -> Any: ... - # def list_pop(self, *args, **kwargs) -> Any: ... - # def list_pop_range(self, *args, **kwargs) -> Any: ... - # def list_remove(self, *args, **kwargs) -> Any: ... - # def list_remove_range(self, *args, **kwargs) -> Any: ... - # def list_set(self, *args, **kwargs) -> Any: ... - # def list_size(self, *args, **kwargs) -> Any: ... - # def list_trim(self, *args, **kwargs) -> Any: ... - # def map_clear(self, *args, **kwargs) -> Any: ... - # def map_decrement(self, *args, **kwargs) -> Any: ... - # def map_get_by_index(self, *args, **kwargs) -> Any: ... - # def map_get_by_index_range(self, *args, **kwargs) -> Any: ... - # def map_get_by_key(self, *args, **kwargs) -> Any: ... - # def map_get_by_key_list(self, *args, **kwargs) -> Any: ... - # def map_get_by_key_range(self, *args, **kwargs) -> Any: ... - # def map_get_by_rank(self, *args, **kwargs) -> Any: ... - # def map_get_by_rank_range(self, *args, **kwargs) -> Any: ... - # def map_get_by_value(self, *args, **kwargs) -> Any: ... - # def map_get_by_value_list(self, *args, **kwargs) -> Any: ... - # def map_get_by_value_range(self, *args, **kwargs) -> Any: ... - # def map_increment(self, *args, **kwargs) -> Any: ... - # def map_put(self, *args, **kwargs) -> Any: ... - # def map_put_items(self, *args, **kwargs) -> Any: ... - # def map_remove_by_index(self, *args, **kwargs) -> Any: ... - # def map_remove_by_index_range(self, *args, **kwargs) -> Any: ... - # def map_remove_by_key(self, *args, **kwargs) -> Any: ... - # def map_remove_by_key_list(self, *args, **kwargs) -> Any: ... - # def map_remove_by_key_range(self, *args, **kwargs) -> Any: ... - # def map_remove_by_rank(self, *args, **kwargs) -> Any: ... - # def map_remove_by_rank_range(self, *args, **kwargs) -> Any: ... - # def map_remove_by_value(self, *args, **kwargs) -> Any: ... - # def map_remove_by_value_list(self, *args, **kwargs) -> Any: ... - # def map_remove_by_value_range(self, *args, **kwargs) -> Any: ... - # def map_set_policy(self, key, bin, map_policy) -> Any: ... - # def map_size(self, *args, **kwargs) -> Any: ... def operate(self, key: tuple, list: list, meta: dict = ..., policy: dict = ...) -> tuple: ... def operate_ordered(self, key: tuple, list: list, meta: dict = ..., policy: dict = ...) -> list: ... def prepend(self, key: tuple, bin: str, val: str, meta: dict = ..., policy: dict = ...) -> None: ... @@ -421,6 +406,8 @@ class Client: def udf_list(self, policy: dict = ...) -> list: ... def udf_put(self, filename: str, udf_type = ..., policy: dict = ...) -> None: ... def udf_remove(self, module: str, policy: dict = ...) -> None: ... + def commit(self, transaction: Transaction) -> int: ... + def abort(self, transaction: Transaction) -> int: ... class GeoJSON: geo_data: Any diff --git a/aerospike_helpers/expressions/base.py b/aerospike_helpers/expressions/base.py index 870b6f437..c5b2ba765 100644 --- a/aerospike_helpers/expressions/base.py +++ b/aerospike_helpers/expressions/base.py @@ -301,8 +301,8 @@ def __init__(self, bin: str): class GeoBin(_BaseExpr): - """Create an expression that returns a bin as a geojson. Returns the unknown-value - if the bin is not a geojson. + """Create an expression that returns a bin as a GeoJSON. Returns the unknown-value + if the bin is not a GeoJSON. """ _op = _ExprOp.BIN @@ -312,7 +312,7 @@ def __init__(self, bin: str): """Args: bin (str): Bin name. - :return: (geojson bin) + :return: (GeoJSON bin) Example:: diff --git a/aerospike_helpers/metrics/__init__.py b/aerospike_helpers/metrics/__init__.py index a564149e9..e0339f25c 100644 --- a/aerospike_helpers/metrics/__init__.py +++ b/aerospike_helpers/metrics/__init__.py @@ -28,7 +28,7 @@ class ConnectionStats: """Connection statistics. Attributes: - in_use (int): Connections actively being used in database transactions on this node. + in_use (int): Connections actively being used in database commands on this node. There can be multiple pools per node. This value is a summary of those pools on this node. in_pool (int): Connections residing in pool(s) on this node. There can be multiple pools per node. This value is a summary of those pools on this node. @@ -62,10 +62,10 @@ class Node: address (str): The IP address / host name of the node (not including the port number). port (int): Port number of the node's address. conns (:py:class:`ConnectionStats`): Synchronous connection stats on this node. - error_count (int): Transaction error count since node was initialized. If the error is retryable, - multiple errors per transaction may occur. - timeout_count (int): Transaction timeout count since node was initialized. - If the timeout is retryable (ie socketTimeout), multiple timeouts per transaction may occur. + error_count (int): Command error count since node was initialized. If the error is retryable, + multiple errors per command may occur. + timeout_count (int): Command timeout count since node was initialized. + If the timeout is retryable (i.e socketTimeout), multiple timeouts per command may occur. metrics (:py:class:`NodeMetrics`): Node metrics """ pass @@ -77,8 +77,8 @@ class Cluster: Attributes: cluster_name (Optional[str]): Expected cluster name for all nodes. May be :py:obj:`None`. invalid_node_count (int): Count of add node failures in the most recent cluster tend iteration. - tran_count (int): Transaction count. The value is cumulative and not reset per metrics interval. - retry_count (int): Transaction retry count. There can be multiple retries for a single transaction. + command_count (int): Command count. The value is cumulative and not reset per metrics interval. + retry_count (int): Command retry count. There can be multiple retries for a single command. The value is cumulative and not reset per metrics interval. nodes (list[:py:class:`Node`]): Active nodes in cluster. """ diff --git a/doc/aerospike.rst b/doc/aerospike.rst index 49e4cc8e9..dbbeffc2a 100644 --- a/doc/aerospike.rst +++ b/doc/aerospike.rst @@ -1575,3 +1575,77 @@ Query Duration Treat query as a LONG query, but relax read consistency for AP namespaces. This value is treated exactly like :data:`aerospike.QUERY_DURATION_LONG` for server versions < 7.1. + +.. _mrt_commit_status_constants: + +MRT Commit Status +----------------- + +.. data:: MRT_COMMIT_OK + + Commit succeeded. + +.. data:: MRT_COMMIT_ALREADY_COMMITTED + + Transaction has already been committed. + +.. data:: MRT_COMMIT_ALREADY_ABORTED + + Transaction has already been aborted. + +.. data:: MRT_COMMIT_VERIFY_FAILED + + Transaction verify failed. Transaction will be aborted. + +.. data:: MRT_COMMIT_MARK_ROLL_FORWARD_ABANDONED + + Transaction mark roll forward abandoned. Transaction will be aborted when error is not in doubt. + If the error is in doubt (usually timeout), the commit is in doubt. + +.. data:: MRT_COMMIT_ROLL_FORWARD_ABANDONED + + Client roll forward abandoned. Server will eventually commit the transaction. + +.. data:: MRT_COMMIT_CLOSE_ABANDONED + + Transaction has been rolled forward, but client transaction close was abandoned. + Server will eventually close the transaction. + +.. _mrt_abort_status_constants: + +MRT Abort Status +---------------- + +.. data:: MRT_ABORT_OK + + Abort succeeded. + +.. data:: MRT_ABORT_ALREADY_COMMITTED + + Transaction has already been committed. + +.. data:: MRT_ABORT_ALREADY_ABORTED + + Transaction has already been aborted. + +.. data:: MRT_ABORT_ROLL_BACK_ABANDONED + + Client roll back abandoned. Server will eventually abort the transaction. + +.. data:: MRT_ABORT_CLOSE_ABANDONED + + Transaction has been rolled back, but client transaction close was abandoned. + Server will eventually close the transaction. + +.. _mrt_state: + +Multi-record Transaction State +------------------------------ + +.. data:: MRT_STATE_OPEN + +.. data:: MRT_STATE_VERIFIED + +.. data:: MRT_STATE_COMMITTED + +.. data:: MRT_STATE_ABORTED diff --git a/doc/client.rst b/doc/client.rst index bce635bd3..05c551a33 100755 --- a/doc/client.rst +++ b/doc/client.rst @@ -305,8 +305,8 @@ Batch Operations The following batch methods will return a :class:`~aerospike_helpers.batch.records.BatchRecords` object with a ``result`` value of ``0`` if one of the following is true: - * All transactions are successful. - * One or more transactions failed because: + * All commands are successful. + * One or more commands failed because: - A record was filtered out by an expression - The record was not found @@ -317,7 +317,7 @@ Batch Operations * If the underlying C client throws an error, the returned :class:`~aerospike_helpers.batch.records.BatchRecords` object will have a ``result`` value equal to an `as_status `_ error code. In this case, the :class:`~aerospike_helpers.batch.records.BatchRecords` object has a list of batch records called ``batch_records``, - and each batch record contains the result of that transaction. + and each batch record contains the result of that command. .. method:: batch_write(batch_records: BatchRecords, [policy_batch: dict]) -> BatchRecords @@ -364,7 +364,7 @@ Batch Operations .. method:: batch_operate(keys: list, ops: list, [policy_batch: dict], [policy_batch_write: dict], [ttl: int]) -> BatchRecords - Perform the same read/write transactions on multiple keys. + Perform the same read/write commands on multiple keys. .. note:: Prior to Python client 14.0.0, using the :meth:`~batch_operate()` method with only read operations caused an error. This bug was fixed in version 14.0.0. @@ -538,7 +538,7 @@ Single-Record Transactions .. method:: operate(key, list: list[, meta: dict[, policy: dict]]) -> (key, meta, bins) - Performs an atomic transaction, with multiple bin operations, against a single record with a given *key*. + Performs an atomic command, with multiple bin operations, against a single record with a given *key*. Starting with Aerospike server version 3.6.0, non-existent bins are not present in the returned :ref:`aerospike_record_tuple`. \ The returned record tuple will only contain one element per bin, even if multiple operations were performed on the bin. \ @@ -564,7 +564,7 @@ Single-Record Transactions .. method:: operate_ordered(key, list: list[, meta: dict[, policy: dict]]) -> (key, meta, bins) - Performs an atomic transaction, with multiple bin operations, against a single record with a given *key*. \ + Performs an atomic command, with multiple bin operations, against a single record with a given *key*. \ The results will be returned as a list of (bin-name, result) tuples. The order of the \ elements in the list will correspond to the order of the operations \ from the input parameters. @@ -587,6 +587,34 @@ Single-Record Transactions .. index:: single: User Defined Functions +Multi-Record Transactions +-------------------------- + +.. class:: Client + :noindex: + + .. method:: commit(transaction: aerospike.Transaction) -> int: + + Attempt to commit the given multi-record transaction. First, the expected record versions are + sent to the server nodes for verification. If all nodes return success, the transaction is + committed. Otherwise, the transaction is aborted. + + Requires server version 8.0+ + + :param transaction: Multi-record transaction. + :type transaction: :py:class:`aerospike.Transaction` + :return: The status of the commit. One of :ref:`mrt_commit_status_constants`. + + .. method:: abort(transaction: aerospike.Transaction) -> int: + + Abort and rollback the given multi-record transaction. + + Requires server version 8.0+ + + :param transaction: Multi-record transaction. + :type transaction: :py:class:`aerospike.Transaction` + :return: The status of the abort. One of :ref:`mrt_abort_status_constants`. + .. _aerospike_udf_operations: User Defined Functions @@ -1324,7 +1352,7 @@ The user dictionary has the following key-value pairs: * 0: read quota in records per second - * 1: single record read transaction rate (TPS) + * 1: single record read command rate (TPS) * 2: read scan/query record per second rate (RPS) @@ -1337,7 +1365,7 @@ The user dictionary has the following key-value pairs: * 0: write quota in records per second - * 1: single record write transaction rate (TPS) + * 1: single record write command rate (TPS) * 2: write scan/query record per second rate (RPS) @@ -1557,14 +1585,14 @@ Write Policies :columns: 1 * **max_retries** (:class:`int`) - | Maximum number of retries before aborting the current transaction. The initial attempt is not counted as a retry. + | Maximum number of retries before aborting the current command. The initial attempt is not counted as a retry. | - | If max_retries is exceeded, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. + | If max_retries is exceeded, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. | | Default: ``0`` .. warning:: Database writes that are not idempotent (such as "add") should not be retried because the write operation may be performed multiple times \ - if the client timed out previous transaction attempts. It's important to use a distinct write policy for non-idempotent writes, which sets max_retries = `0`; + if the client timed out previous command attempts. It's important to use a distinct write policy for non-idempotent writes, which sets max_retries = `0`; * **sleep_between_retries** (:class:`int`) | Milliseconds to sleep between retries. Enter ``0`` to skip sleep. @@ -1573,18 +1601,18 @@ Write Policies * **socket_timeout** (:class:`int`) | Socket idle timeout in milliseconds when processing a database command. | - | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the transaction is retried. + | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the command is retried. | | If both ``socket_timeout`` and ``total_timeout`` are non-zero and ``socket_timeout`` > ``total_timeout``, then ``socket_timeout`` will be set to ``total_timeout``. \ If ``socket_timeout`` is ``0``, there will be no socket idle limit. | | Default: ``30000`` * **total_timeout** (:class:`int`) - | Total transaction timeout in milliseconds. + | Total command timeout in milliseconds. | - | The total_timeout is tracked on the client and sent to the server along with the transaction in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the transaction. + | The total_timeout is tracked on the client and sent to the server along with the command in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the command. | - | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the transaction completes, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. + | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the command completes, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. | | Default: ``1000`` * **compress** (:class:`bool`) @@ -1605,7 +1633,7 @@ Write Policies | Default: :data:`aerospike.POLICY_EXISTS_IGNORE` * **ttl** The default time-to-live (expiration) of the record in seconds. This field will only be used if - the write transaction: + the write command: 1. Doesn't contain a metadata dictionary with a ``ttl`` value. 2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`. @@ -1624,7 +1652,7 @@ Write Policies | | Default: ``False`` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None @@ -1639,6 +1667,11 @@ Write Policies Default: :data:`aerospike.POLICY_REPLICA_SEQUENCE` + * **txn** :class:`aerospike.Transaction` + Multi-record command identifier. + + Default: :py:obj:`None` + .. _aerospike_read_policies: Read Policies @@ -1652,9 +1685,9 @@ Read Policies :columns: 1 * **max_retries** (:class:`int`) - | Maximum number of retries before aborting the current transaction. The initial attempt is not counted as a retry. + | Maximum number of retries before aborting the current command. The initial attempt is not counted as a retry. | - | If max_retries is exceeded, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. + | If max_retries is exceeded, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. | | Default: ``2`` * **sleep_between_retries** (:class:`int`) @@ -1664,17 +1697,17 @@ Read Policies * **socket_timeout** (:class:`int`) | Socket idle timeout in milliseconds when processing a database command. | - | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the transaction is retried. + | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the command is retried. | | If both ``socket_timeout`` and ``total_timeout`` are non-zero and ``socket_timeout`` > ``total_timeout``, then ``socket_timeout`` will be set to ``total_timeout``. If ``socket_timeout`` is ``0``, there will be no socket idle limit. | | Default: ``30000`` * **total_timeout** (:class:`int`) - | Total transaction timeout in milliseconds. + | Total command timeout in milliseconds. | - | The total_timeout is tracked on the client and sent to the server along with the transaction in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the transaction. + | The total_timeout is tracked on the client and sent to the server along with the command in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the command. | - | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the transaction completes, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. + | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the command completes, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. | | Default: ``1000`` * **compress** (:class:`bool`) @@ -1732,12 +1765,17 @@ Read Policies | | Default: ``aerospike.POLICY_REPLICA_SEQUENCE`` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None .. note:: Requires Aerospike server version >= 5.2. + * **txn** :class:`aerospike.Transaction` + Multi-record command identifier. + + Default: :py:obj:`None` + .. _aerospike_operate_policies: Operate Policies @@ -1751,14 +1789,14 @@ Operate Policies :columns: 1 * **max_retries** (:class:`int`) - | Maximum number of retries before aborting the current transaction. The initial attempt is not counted as a retry. + | Maximum number of retries before aborting the current command. The initial attempt is not counted as a retry. | - | If max_retries is exceeded, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. + | If max_retries is exceeded, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. | | Default: ``0`` .. warning:: Database writes that are not idempotent (such as "add") should not be retried because the write operation may be performed multiple times \ - if the client timed out previous transaction attempts. It's important to use a distinct write policy for non-idempotent writes, which sets max_retries = `0`; + if the client timed out previous command attempts. It's important to use a distinct write policy for non-idempotent writes, which sets max_retries = `0`; * **sleep_between_retries** (:class:`int`) | Milliseconds to sleep between retries. Enter ``0`` to skip sleep. @@ -1767,17 +1805,17 @@ Operate Policies * **socket_timeout** (:class:`int`) | Socket idle timeout in milliseconds when processing a database command. | - | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the transaction is retried. + | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the command is retried. | | If both ``socket_timeout`` and ``total_timeout`` are non-zero and ``socket_timeout`` > ``total_timeout``, then ``socket_timeout`` will be set to ``total_timeout``. If ``socket_timeout`` is ``0``, there will be no socket idle limit. | | Default: ``30000`` * **total_timeout** (:class:`int`) - | Total transaction timeout in milliseconds. + | Total command timeout in milliseconds. | - | The total_timeout is tracked on the client and sent to the server along with the transaction in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the transaction. + | The total_timeout is tracked on the client and sent to the server along with the command in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the command. | - | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the transaction completes, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. + | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the command completes, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. | | Default: ``1000`` * **compress** (:class:`bool`) @@ -1798,7 +1836,7 @@ Operate Policies | Default: :data:`aerospike.POLICY_GEN_IGNORE` * **ttl** (:class:`int`) The default time-to-live (expiration) of the record in seconds. This field will only be used if an - operate transaction contains a write operation and either: + operate command contains a write operation and either: 1. Doesn't contain a metadata dictionary with a ``ttl`` value. 2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`. @@ -1860,11 +1898,15 @@ Operate Policies | | Default: :py:obj:`True` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None .. note:: Requires Aerospike server version >= 5.2. + * **txn** :class:`aerospike.Transaction` + Multi-record command identifier. + + Default: :py:obj:`None` .. _aerospike_apply_policies: @@ -1879,14 +1921,14 @@ Apply Policies :columns: 1 * **max_retries** (:class:`int`) - | Maximum number of retries before aborting the current transaction. The initial attempt is not counted as a retry. + | Maximum number of retries before aborting the current command. The initial attempt is not counted as a retry. | - | If max_retries is exceeded, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. + | If max_retries is exceeded, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. | | Default: ``0`` .. warning:: Database writes that are not idempotent (such as "add") should not be retried because the write operation may be performed multiple times \ - if the client timed out previous transaction attempts. It's important to use a distinct write policy for non-idempotent writes, which sets max_retries = `0`; + if the client timed out previous command attempts. It's important to use a distinct write policy for non-idempotent writes, which sets max_retries = `0`; * **sleep_between_retries** (:class:`int`) | Milliseconds to sleep between retries. Enter ``0`` to skip sleep. @@ -1895,17 +1937,17 @@ Apply Policies * **socket_timeout** (:class:`int`) | Socket idle timeout in milliseconds when processing a database command. | - | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the transaction is retried. + | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the command is retried. | | If both ``socket_timeout`` and ``total_timeout`` are non-zero and ``socket_timeout`` > ``total_timeout``, then ``socket_timeout`` will be set to ``total_timeout``. If ``socket_timeout`` is ``0``, there will be no socket idle limit. | | Default: ``30000`` * **total_timeout** (:class:`int`) - | Total transaction timeout in milliseconds. + | Total command timeout in milliseconds. | - | The total_timeout is tracked on the client and sent to the server along with the transaction in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the transaction. + | The total_timeout is tracked on the client and sent to the server along with the command in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the command. | - | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the transaction completes, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. + | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the command completes, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. | | Default: ``1000`` * **compress** (:class:`bool`) @@ -1930,7 +1972,7 @@ Apply Policies | Default: :data:`aerospike.POLICY_COMMIT_LEVEL_ALL` * **ttl** (:class:`int`) The default time-to-live (expiration) of the record in seconds. This field will only be used if an apply - transaction doesn't have an apply policy with a ``ttl`` value that overrides this field. + command doesn't have an apply policy with a ``ttl`` value that overrides this field. There are also special values that can be set for this field. See :ref:`TTL_CONSTANTS`. * **durable_delete** (:class:`bool`) @@ -1938,11 +1980,15 @@ Apply Policies | | Default: ``False`` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None .. note:: Requires Aerospike server version >= 5.2. + * **txn** :class:`aerospike.Transaction` + Multi-record command identifier. + + Default: :py:obj:`None` .. _aerospike_remove_policies: @@ -1958,14 +2004,14 @@ Remove Policies :columns: 1 * **max_retries** (:class:`int`) - | Maximum number of retries before aborting the current transaction. The initial attempt is not counted as a retry. + | Maximum number of retries before aborting the current command. The initial attempt is not counted as a retry. | - | If max_retries is exceeded, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. + | If max_retries is exceeded, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. | | Default: ``0`` .. warning:: Database writes that are not idempotent (such as "add") should not be retried because the write operation may be performed multiple times \ - if the client timed out previous transaction attempts. It's important to use a distinct write policy for non-idempotent writes, which sets max_retries = `0`; + if the client timed out previous command attempts. It's important to use a distinct write policy for non-idempotent writes, which sets max_retries = `0`; * **sleep_between_retries** (:class:`int`) | Milliseconds to sleep between retries. Enter ``0`` to skip sleep. @@ -1973,17 +2019,17 @@ Remove Policies * **socket_timeout** (:class:`int`) | Socket idle timeout in milliseconds when processing a database command. | - | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the transaction is retried. + | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the command is retried. | | If both ``socket_timeout`` and ``total_timeout`` are non-zero and ``socket_timeout`` > ``total_timeout``, then ``socket_timeout`` will be set to ``total_timeout``. If ``socket_timeout`` is ``0``, there will be no socket idle limit. | | Default: ``30000`` * **total_timeout** (:class:`int`) - | Total transaction timeout in milliseconds. + | Total command timeout in milliseconds. | - | The total_timeout is tracked on the client and sent to the server along with the transaction in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the transaction. + | The total_timeout is tracked on the client and sent to the server along with the command in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the command. | - | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the transaction completes, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. + | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the command completes, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. | | Default: ``1000`` * **compress** (:class:`bool`) @@ -2021,11 +2067,15 @@ Remove Policies | Default: ``aerospike.POLICY_REPLICA_SEQUENCE`` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None .. note:: Requires Aerospike server version >= 5.2. + * **txn** :class:`aerospike.Transaction` + Multi-record command identifier. + + Default: :py:obj:`None` .. _aerospike_batch_policies: @@ -2040,9 +2090,9 @@ Batch Policies :columns: 1 * **max_retries** (:class:`int`) - | Maximum number of retries before aborting the current transaction. The initial attempt is not counted as a retry. + | Maximum number of retries before aborting the current command. The initial attempt is not counted as a retry. | - | If max_retries is exceeded, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. + | If max_retries is exceeded, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. | | Default: ``2`` @@ -2055,17 +2105,17 @@ Batch Policies * **socket_timeout** (:class:`int`) | Socket idle timeout in milliseconds when processing a database command. | - | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the transaction is retried. + | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the command is retried. | | If both ``socket_timeout`` and ``total_timeout`` are non-zero and ``socket_timeout`` > ``total_timeout``, then ``socket_timeout`` will be set to ``total_timeout``. If ``socket_timeout`` is ``0``, there will be no socket idle limit. | | Default: ``30000`` * **total_timeout** (:class:`int`) - | Total transaction timeout in milliseconds. + | Total command timeout in milliseconds. | - | The total_timeout is tracked on the client and sent to the server along with the transaction in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the transaction. + | The total_timeout is tracked on the client and sent to the server along with the command in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the command. | - | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the transaction completes, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. + | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the command completes, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. | | Default: ``1000`` * **compress** (:class:`bool`) @@ -2119,7 +2169,7 @@ Batch Policies | | Default ``False`` * **allow_inline** (:class:`bool`) - | Allow batch to be processed immediately in the server's receiving thread when the server deems it to be appropriate. If `False`, the batch will always be processed in separate transaction threads. This field is only relevant for the new batch index protocol. + | Allow batch to be processed immediately in the server's receiving thread when the server deems it to be appropriate. If `False`, the batch will always be processed in separate service threads. This field is only relevant for the new batch index protocol. | | Default ``True`` * **allow_inline_ssd** (:class:`bool`) @@ -2136,7 +2186,7 @@ Batch Policies | | Default: ``True`` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None @@ -2158,6 +2208,10 @@ Batch Policies Server versions < 6.0 do not support this field and treat this value as false for key specific errors. Default: ``True`` + * **txn** :class:`aerospike.Transaction` + Multi-record command identifier. + + Default: :py:obj:`None` .. _aerospike_batch_write_policies: @@ -2192,7 +2246,7 @@ Batch Write Policies | | Default: ``False`` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None * **ttl** :class:`int` @@ -2241,11 +2295,11 @@ Batch Apply Policies | | Default: ``0`` * **durable_delete** :class:`bool` - | If the transaction results in a record deletion, leave a tombstone for the record. This prevents deleted records from reappearing after node failures. Valid for Aerospike Server Enterprise Edition only. + | If the command results in a record deletion, leave a tombstone for the record. This prevents deleted records from reappearing after node failures. Valid for Aerospike Server Enterprise Edition only. | | Default: :py:obj:`False` (do not tombstone deleted records). * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None @@ -2282,7 +2336,7 @@ Batch Remove Policies | | Default: ``False`` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None @@ -2307,7 +2361,7 @@ Batch Read Policies | | Default: :data:`aerospike.POLICY_READ_MODE_SC_SESSION` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None * **read_touch_ttl_percent** @@ -2520,8 +2574,8 @@ Role Objects * ``"privileges"``: a :class:`list` of :ref:`aerospike_privilege_dict`. * ``"whitelist"``: a :class:`list` of IP address strings. - * ``"read_quota"``: a :class:`int` representing the allowed read transactions per second. - * ``"write_quota"``: a :class:`int` representing the allowed write transactions per second. + * ``"read_quota"``: a :class:`int` representing the allowed read commands per second. + * ``"write_quota"``: a :class:`int` representing the allowed write commands per second. .. _aerospike_privilege_dict: diff --git a/doc/conf.py b/doc/conf.py index 0560d2c30..7bae041fa 100755 --- a/doc/conf.py +++ b/doc/conf.py @@ -255,7 +255,7 @@ def __getattr__(cls, name): # Spelling check -spelling_ignore_pypi_package_names = True +spelling_ignore_pypi_package_names = False linkcheck_anchors_ignore = [ 'truncate', diff --git a/doc/data_mapping.rst b/doc/data_mapping.rst index dfd43a50c..e38a1cae7 100644 --- a/doc/data_mapping.rst +++ b/doc/data_mapping.rst @@ -69,7 +69,7 @@ For server 7.1 and higher, map keys can only be of type string, bytes, and integ :ref:`KeyOrderedDict ` is a special case. Like :class:`dict`, :class:`~aerospike.KeyOrderedDict` maps to the Aerospike map data type. \ However, the map will be sorted in key order before being sent to the server (see :ref:`aerospike_map_order`). -It is possible to nest these datatypes. For example a list may contain a dictionary, or a dictionary may contain a list +It is possible to nest these data types. For example a list may contain a dictionary, or a dictionary may contain a list as a value. .. _integer: https://aerospike.com/docs/server/guide/data-types/scalar-data-types#integer diff --git a/doc/index.rst b/doc/index.rst index d12c2d389..406223815 100755 --- a/doc/index.rst +++ b/doc/index.rst @@ -59,6 +59,7 @@ Class Description :ref:`aerospike.Query` Handles queries over secondary indexes. :ref:`aerospike.geojson` Handles GeoJSON type data. :ref:`aerospike.KeyOrderedDict` Key ordered dictionary +:ref:`aerospike.Transaction` Multi-record transaction ================================= =========== In addition, the :ref:`Data_Mapping` page explains how **Python** types map to **Aerospike Server** types. @@ -79,6 +80,7 @@ Content query geojson key_ordered_dict + transaction predicates exception aerospike_helpers diff --git a/doc/spelling_wordlist.txt b/doc/spelling_wordlist.txt index 6a03b1afe..19d2c4ba2 100644 --- a/doc/spelling_wordlist.txt +++ b/doc/spelling_wordlist.txt @@ -3,6 +3,7 @@ lua namespace geospatial serialized +serializers Serializers deserialized Aerospike @@ -76,3 +77,13 @@ subtransactions socketTimeout ttl inlined +hashmaps +config +deserialize +async +retryable +backoff +msg +func +bcrypt +namespaces diff --git a/doc/transaction.rst b/doc/transaction.rst new file mode 100644 index 000000000..34dda0331 --- /dev/null +++ b/doc/transaction.rst @@ -0,0 +1,55 @@ +.. _aerospike.Transaction: + +.. currentmodule:: aerospike + +================================================================= +:class:`aerospike.Transaction` --- Multi Record Transaction Class +================================================================= + +In the whole documentation, "commands" are all database commands that can be sent to the server (e.g single-record +operations, batch, scans, or queries). + +"Transactions" link individual commands (except for scan and query) together so they can be rolled forward or +backward consistently. + +Methods +======= + +.. class:: Transaction + + Initialize multi-record transaction (MRT), assign random transaction id and initialize + reads/writes hashmaps with default capacities. The default MRT timeout is 10 seconds. + + For both parameters, an unsigned 32-bit integer must be passed and the minimum value should be 16. + + :param reads_capacity: expected number of record reads in the MRT. Defaults to ``128``. + :type reads_capacity: int, optional + :param writes_capacity: expected number of record writes in the MRT. Defaults to ``128``. + :type writes_capacity: int, optional + + .. py:attribute:: id + + Get the random transaction id that was generated on class instance creation. + + The value is an unsigned 64-bit integer. + + This attribute is read-only. + + :type: int + .. py:attribute:: in_doubt + + This attribute is read-only. + + :type: bool + .. py:attribute:: state + + One of the :ref:`mrt_state` constants. + + This attribute is read-only. + + :type: int + .. py:attribute:: timeout + + This attribute can be read and written to. + + :type: int diff --git a/setup.py b/setup.py index a486c06e3..78513f45d 100644 --- a/setup.py +++ b/setup.py @@ -366,7 +366,9 @@ def clean(): 'src/main/client/batch_remove.c', 'src/main/client/batch_apply.c', 'src/main/client/batch_read.c', - 'src/main/client/metrics.c' + 'src/main/client/metrics.c', + 'src/main/transaction/type.c', + 'src/main/client/mrt.c' ], # Compile diff --git a/src/include/client.h b/src/include/client.h index 3f442e014..d4f3b140a 100644 --- a/src/include/client.h +++ b/src/include/client.h @@ -601,3 +601,10 @@ int check_type(AerospikeClient *self, PyObject *py_value, int op, PyObject *AerospikeClient_Truncate(AerospikeClient *self, PyObject *args, PyObject *kwds); + +// MRT + +PyObject *AerospikeClient_Commit(AerospikeClient *self, PyObject *args, + PyObject *kwds); +PyObject *AerospikeClient_Abort(AerospikeClient *self, PyObject *args, + PyObject *kwds); diff --git a/src/include/transaction.h b/src/include/transaction.h new file mode 100644 index 000000000..d4dc15543 --- /dev/null +++ b/src/include/transaction.h @@ -0,0 +1,3 @@ +#include + +PyTypeObject *AerospikeTransaction_Ready(); diff --git a/src/include/types.h b/src/include/types.h index 33640b3f9..319794a93 100644 --- a/src/include/types.h +++ b/src/include/types.h @@ -25,6 +25,7 @@ #include #include #include +#include #include "pool.h" #define AEROSPIKE_MODULE_NAME "aerospike" @@ -99,3 +100,11 @@ typedef struct { typedef struct { PyDictObject dict; } AerospikeKeyOrderedDict; + +typedef struct { + PyObject_HEAD + /* Type-specific fields go here. */ + as_txn *txn; +} AerospikeTransaction; + +extern PyTypeObject AerospikeTransaction_Type; diff --git a/src/main/aerospike.c b/src/main/aerospike.c index c94bc0fea..e8d2fef03 100644 --- a/src/main/aerospike.c +++ b/src/main/aerospike.c @@ -32,6 +32,7 @@ #include "module_functions.h" #include "nullobject.h" #include "cdt_types.h" +#include "transaction.h" #include #include @@ -39,6 +40,7 @@ #include #include #include +#include PyObject *py_global_hosts; int counter = 0xA8000000; @@ -499,6 +501,30 @@ static struct module_constant_name_to_value module_constants[] = { {"LOG_LEVEL_DEBUG", .value.integer = AS_LOG_LEVEL_DEBUG}, {"LOG_LEVEL_TRACE", .value.integer = AS_LOG_LEVEL_TRACE}, + {"MRT_COMMIT_OK", .value.integer = AS_COMMIT_OK}, + {"MRT_COMMIT_ALREADY_COMMITTED", + .value.integer = AS_COMMIT_ALREADY_COMMITTED}, + {"MRT_COMMIT_ALREADY_ABORTED", .value.integer = AS_COMMIT_ALREADY_ABORTED}, + {"MRT_COMMIT_VERIFY_FAILED", .value.integer = AS_COMMIT_VERIFY_FAILED}, + {"MRT_COMMIT_MARK_ROLL_FORWARD_ABANDONED", + .value.integer = AS_COMMIT_MARK_ROLL_FORWARD_ABANDONED}, + {"MRT_COMMIT_ROLL_FORWARD_ABANDONED", + .value.integer = AS_COMMIT_ROLL_FORWARD_ABANDONED}, + {"MRT_COMMIT_CLOSE_ABANDONED", .value.integer = AS_COMMIT_CLOSE_ABANDONED}, + + {"MRT_ABORT_OK", .value.integer = AS_ABORT_OK}, + {"MRT_ABORT_ALREADY_COMMITTED", + .value.integer = AS_ABORT_ALREADY_COMMITTED}, + {"MRT_ABORT_ALREADY_ABORTED", .value.integer = AS_ABORT_ALREADY_ABORTED}, + {"MRT_ABORT_ROLL_BACK_ABANDONED", + .value.integer = AS_ABORT_ROLL_BACK_ABANDONED}, + {"MRT_ABORT_CLOSE_ABANDONED", .value.integer = AS_ABORT_CLOSE_ABANDONED}, + + {"MRT_STATE_OPEN", .value.integer = AS_TXN_STATE_OPEN}, + {"MRT_STATE_VERIFIED", .value.integer = AS_TXN_STATE_VERIFIED}, + {"MRT_STATE_COMMITTED", .value.integer = AS_TXN_STATE_COMMITTED}, + {"MRT_STATE_ABORTED", .value.integer = AS_TXN_STATE_ABORTED}, + {"JOB_SCAN", .is_str_value = true, .value.string = "scan"}, {"JOB_QUERY", .is_str_value = true, .value.string = "query"}}; @@ -533,6 +559,7 @@ static struct type_name_to_creation_method py_module_types[] = { {"null", AerospikeNullObject_Ready}, {"CDTWildcard", AerospikeWildcardObject_Ready}, {"CDTInfinite", AerospikeInfiniteObject_Ready}, + {"Transaction", AerospikeTransaction_Ready}, }; PyMODINIT_FUNC PyInit_aerospike(void) diff --git a/src/main/client/mrt.c b/src/main/client/mrt.c new file mode 100644 index 000000000..d49bd43c4 --- /dev/null +++ b/src/main/client/mrt.c @@ -0,0 +1,74 @@ +#include + +#include +#include "exceptions.h" +#include "types.h" +#include "client.h" + +PyObject *AerospikeClient_Commit(AerospikeClient *self, PyObject *args, + PyObject *kwds) +{ + AerospikeTransaction *py_transaction = NULL; + + static char *kwlist[] = {"transaction", NULL}; + + if (PyArg_ParseTupleAndKeywords(args, kwds, "O!:commit", kwlist, + &AerospikeTransaction_Type, + (PyObject **)(&py_transaction)) == false) { + return NULL; + } + + as_error err; + as_error_init(&err); + + as_commit_status status; + + Py_BEGIN_ALLOW_THREADS + aerospike_commit(self->as, &err, py_transaction->txn, &status); + Py_END_ALLOW_THREADS + + if (err.code != AEROSPIKE_OK) { + raise_exception(&err); + return NULL; + } + + PyObject *py_status = PyLong_FromUnsignedLong((unsigned long)status); + if (py_status == NULL) { + return NULL; + } + return py_status; +} + +PyObject *AerospikeClient_Abort(AerospikeClient *self, PyObject *args, + PyObject *kwds) +{ + AerospikeTransaction *py_transaction = NULL; + + static char *kwlist[] = {"transaction", NULL}; + + if (PyArg_ParseTupleAndKeywords(args, kwds, "O!|p:abort", kwlist, + &AerospikeTransaction_Type, + (PyObject **)(&py_transaction)) == false) { + return NULL; + } + + as_error err; + as_error_init(&err); + + as_abort_status status; + + Py_BEGIN_ALLOW_THREADS + aerospike_abort(self->as, &err, py_transaction->txn, &status); + Py_END_ALLOW_THREADS + + if (err.code != AEROSPIKE_OK) { + raise_exception(&err); + return NULL; + } + + PyObject *py_status = PyLong_FromUnsignedLong((unsigned long)status); + if (py_status == NULL) { + return NULL; + } + return py_status; +} diff --git a/src/main/client/type.c b/src/main/client/type.c index ccd174b77..aaacbcac2 100644 --- a/src/main/client/type.c +++ b/src/main/client/type.c @@ -529,6 +529,11 @@ static PyMethodDef AerospikeClient_Type_Methods[] = { {"truncate", (PyCFunction)AerospikeClient_Truncate, METH_VARARGS | METH_KEYWORDS, truncate_doc}, + // Multi record transactions + {"commit", (PyCFunction)AerospikeClient_Commit, + METH_VARARGS | METH_KEYWORDS}, + {"abort", (PyCFunction)AerospikeClient_Abort, METH_VARARGS | METH_KEYWORDS}, + {NULL}}; /******************************************************************************* diff --git a/src/main/conversions.c b/src/main/conversions.c index 210c9994e..37f55fbc8 100644 --- a/src/main/conversions.c +++ b/src/main/conversions.c @@ -974,9 +974,9 @@ PyObject *create_py_cluster_from_as_cluster(as_error *error_p, py_invalid_node_count); Py_DECREF(py_invalid_node_count); - PyObject *py_transaction_count = PyLong_FromLong(cluster->tran_count); - PyObject_SetAttrString(py_cluster, "tran_count", py_transaction_count); - Py_DECREF(py_transaction_count); + PyObject *py_command_count = PyLong_FromLong(cluster->command_count); + PyObject_SetAttrString(py_cluster, "command_count", py_command_count); + Py_DECREF(py_command_count); PyObject *py_retry_count = PyLong_FromLong(cluster->retry_count); PyObject_SetAttrString(py_cluster, "retry_count", py_retry_count); diff --git a/src/main/exception.c b/src/main/exception.c index 4f9a7ab0d..3702edf54 100644 --- a/src/main/exception.c +++ b/src/main/exception.c @@ -109,6 +109,8 @@ struct exception_def exception_defs[] = { AEROSPIKE_ERR_ASYNC_CONNECTION, NULL), EXCEPTION_DEF("ClientAbortError", CLIENT_ERR_EXCEPTION_NAME, AEROSPIKE_ERR_CLIENT_ABORT, NULL), + EXCEPTION_DEF("RollAlreadyAttempted", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_ROLL_ALREADY_ATTEMPTED, NULL), // Server errors EXCEPTION_DEF("InvalidRequest", SERVER_ERR_EXCEPTION_NAME, AEROSPIKE_ERR_REQUEST_INVALID, NULL), diff --git a/src/main/policy.c b/src/main/policy.c index e27d57a6e..40d2d3ed9 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -262,6 +262,43 @@ as_status pyobject_to_policy_admin(AerospikeClient *self, as_error *err, return err->code; } +static inline void check_and_set_txn_field(as_error *err, + as_policy_base *policy_base, + PyObject *py_policy) +{ + PyObject *py_txn_field_name = PyUnicode_FromString("txn"); + if (py_txn_field_name == NULL) { + as_error_update(err, AEROSPIKE_ERR_CLIENT, + "Unable to create Python string \"txn\""); + return; + } + PyObject *py_obj_txn = + PyDict_GetItemWithError(py_policy, py_txn_field_name); + Py_DECREF(py_txn_field_name); + if (py_obj_txn == NULL) { + if (PyErr_Occurred()) { + PyErr_Clear(); + as_error_update(err, AEROSPIKE_ERR_CLIENT, + "Getting the transaction field from Python policy " + "dictionary returned a non-KeyError exception"); + } + // Whether or not a key error was raised + return; + } + + PyTypeObject *py_expected_field_type = &AerospikeTransaction_Type; + if (Py_TYPE(py_obj_txn) != py_expected_field_type) { + // TypeError should be set here, + // but there is no Aerospike exception to represent that error + as_error_update(err, AEROSPIKE_ERR_PARAM, "txn is not of type %s", + py_expected_field_type->tp_name); + return; + } + + AerospikeTransaction *py_txn = (AerospikeTransaction *)py_obj_txn; + policy_base->txn = py_txn->txn; +} + static inline as_status pyobject_to_policy_base(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_base *policy, @@ -273,6 +310,13 @@ pyobject_to_policy_base(AerospikeClient *self, as_error *err, POLICY_SET_FIELD(sleep_between_retries, uint32_t); POLICY_SET_FIELD(compress, bool); + // Setting txn field to a non-NULL value in a query or scan policy is a no-op, + // so this is safe to call for a scan/query policy's base policy + check_and_set_txn_field(err, policy, py_policy); + if (err->code != AEROSPIKE_OK) { + return err->code; + } + POLICY_SET_EXPRESSIONS_FIELD(); return AEROSPIKE_OK; } diff --git a/src/main/transaction/type.c b/src/main/transaction/type.c new file mode 100644 index 000000000..66b820489 --- /dev/null +++ b/src/main/transaction/type.c @@ -0,0 +1,190 @@ +#include + +#include "types.h" + +static void AerospikeTransaction_dealloc(AerospikeTransaction *self) +{ + // Transaction object can be created but not initialized, so need to check + if (self->txn != NULL) { + as_txn_destroy(self->txn); + } + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyObject *AerospikeTransaction_new(PyTypeObject *type, PyObject *args, + PyObject *kwds) +{ + AerospikeTransaction *self = + (AerospikeTransaction *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } + return (PyObject *)self; +} + +// Error indicator must always be checked after this call +// Constructor parameter name needed for constructing error message +static uint32_t convert_pyobject_to_uint32_t(PyObject *pyobject, + const char *param_name_of_pyobj) +{ + if (!PyLong_Check(pyobject)) { + PyErr_Format(PyExc_TypeError, "%s must be an integer", + param_name_of_pyobj); + goto error; + } + unsigned long long_value = PyLong_AsUnsignedLong(pyobject); + if (PyErr_Occurred()) { + goto error; + } + + if (long_value > UINT32_MAX) { + PyErr_Format(PyExc_ValueError, + "%s is too large for an unsigned 32-bit integer", + param_name_of_pyobj); + goto error; + } + + uint32_t value = (uint32_t)long_value; + return value; + +error: + return 0; +} + +// We don't initialize in __new__ because it's not documented how to raise +// exceptions in __new__ +// We can raise an exception and fail out in __init__ though +static int AerospikeTransaction_init(AerospikeTransaction *self, PyObject *args, + PyObject *kwds) +{ + static char *kwlist[] = {"reads_capacity", "writes_capacity", NULL}; + // We could use unsigned longs directly in the format string + // But then we can't tell if they were set or not by the user + // So we just use PyObjects for the optional args instead + PyObject *py_reads_capacity = NULL; + PyObject *py_writes_capacity = NULL; + + if (PyArg_ParseTupleAndKeywords(args, kwds, "|OO", kwlist, + &py_reads_capacity, + &py_writes_capacity) == false) { + goto error; + } + + as_txn *txn; + uint32_t reads_capacity, writes_capacity; + if (py_reads_capacity) { + reads_capacity = + convert_pyobject_to_uint32_t(py_reads_capacity, kwlist[0]); + if (PyErr_Occurred()) { + goto error; + } + } + else { + reads_capacity = AS_TXN_READ_CAPACITY_DEFAULT; + } + + if (py_writes_capacity) { + writes_capacity = + convert_pyobject_to_uint32_t(py_writes_capacity, kwlist[1]); + if (PyErr_Occurred()) { + goto error; + } + } + else { + writes_capacity = AS_TXN_WRITE_CAPACITY_DEFAULT; + } + + txn = as_txn_create_capacity(reads_capacity, writes_capacity); + + // If this transaction object was already initialized before, reinitialize it + if (self->txn) { + as_txn_destroy(self->txn); + } + self->txn = txn; + + return 0; +error: + return -1; +} + +static PyObject *AerospikeTransaction_get_in_doubt(AerospikeTransaction *self, + void *closure) +{ + PyObject *py_in_doubt = PyBool_FromLong(self->txn->in_doubt); + if (py_in_doubt == NULL) { + return NULL; + } + return py_in_doubt; +} + +static PyObject *AerospikeTransaction_get_state(AerospikeTransaction *self, + void *closure) +{ + PyObject *py_state = PyLong_FromLong((long)self->txn->state); + if (py_state == NULL) { + return NULL; + } + return py_state; +} + +static PyObject *AerospikeTransaction_get_timeout(AerospikeTransaction *self, + void *closure) +{ + PyObject *py_timeout = + PyLong_FromUnsignedLong((unsigned long)self->txn->timeout); + if (py_timeout == NULL) { + return NULL; + } + return py_timeout; +} + +static int AerospikeTransaction_set_timeout(AerospikeTransaction *self, + PyObject *py_value, void *closure) +{ + uint32_t timeout = convert_pyobject_to_uint32_t(py_value, "timeout"); + if (PyErr_Occurred()) { + return -1; + } + + self->txn->timeout = timeout; + return 0; +} + +static PyObject *AerospikeTransaction_get_id(AerospikeTransaction *self, + void *closure) +{ + PyObject *py_id = + PyLong_FromUnsignedLongLong((unsigned long long)self->txn->id); + if (py_id == NULL) { + return NULL; + } + return py_id; +} + +static PyGetSetDef AerospikeTransaction_getsetters[] = { + {.name = "timeout", + .get = (getter)AerospikeTransaction_get_timeout, + .set = (setter)AerospikeTransaction_set_timeout}, + {.name = "in_doubt", .get = (getter)AerospikeTransaction_get_in_doubt}, + {.name = "state", .get = (getter)AerospikeTransaction_get_state}, + {.name = "id", .get = (getter)AerospikeTransaction_get_id}, + {NULL} /* Sentinel */ +}; + +PyTypeObject AerospikeTransaction_Type = { + .ob_base = PyVarObject_HEAD_INIT(NULL, 0).tp_name = + FULLY_QUALIFIED_TYPE_NAME("Transaction"), + .tp_basicsize = sizeof(AerospikeTransaction), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = AerospikeTransaction_new, + .tp_init = (initproc)AerospikeTransaction_init, + .tp_dealloc = (destructor)AerospikeTransaction_dealloc, + .tp_getset = AerospikeTransaction_getsetters}; + +PyTypeObject *AerospikeTransaction_Ready() +{ + return PyType_Ready(&AerospikeTransaction_Type) == 0 + ? &AerospikeTransaction_Type + : NULL; +} diff --git a/test/metrics/test_node_close_listener.py b/test/metrics/test_node_close_listener.py index fe9e3dad0..32d81001e 100644 --- a/test/metrics/test_node_close_listener.py +++ b/test/metrics/test_node_close_listener.py @@ -1,6 +1,5 @@ -import subprocess import time -import json +import docker import aerospike from aerospike_helpers.metrics import MetricsListeners, MetricsPolicy, Cluster, Node, ConnectionStats, NodeMetrics @@ -65,38 +64,22 @@ def snapshot(_: Cluster): pass -NODE_COUNT = 1 -print(f"Creating {NODE_COUNT} node cluster...") -subprocess.run(["aerolab", "cluster", "create", f"--count={NODE_COUNT}"], check=True) +docker_client = docker.from_env() +print("Running server container...") +SERVER_PORT_NUMBER = 3000 +container = docker_client.containers.run("aerospike/aerospike-server", detach=True, + ports={"3000/tcp": SERVER_PORT_NUMBER}) +print("Waiting for server to initialize...") +time.sleep(5) try: - print("Wait for server to fully start up...") - time.sleep(5) - - # Connect to the first node - completed_process = subprocess.run(["aerolab", "cluster", "list", "--json"], check=True, capture_output=True) - list_of_nodes = json.loads(completed_process.stdout) - - def get_first_node(node_info: dict): - return node_info["NodeNo"] == "1" - - filtered_list_of_nodes = filter(get_first_node, list_of_nodes) - first_node = list(filtered_list_of_nodes)[0] - first_node_port = int(first_node["DockerExposePorts"]) - HOST_NAME = "127.0.0.1" - + print("Connecting to server...") config = { "hosts": [ - (HOST_NAME, first_node_port) + ("127.0.0.1", SERVER_PORT_NUMBER) ], - # The nodes use internal Docker IP addresses as their access addresses - # But we cannot connect to those from the host - # But the nodes use localhost as the alternate address - # So we can connect to that instead - "use_services_alternate": True } - print(f"Connecting to {HOST_NAME}:{first_node_port} using Python client...") - client = aerospike.client(config) + as_client = aerospike.client(config) try: # Show logs to confirm that node is removed from the client's perspective aerospike.set_log_level(aerospike.LOG_LEVEL_DEBUG) @@ -111,17 +94,20 @@ def get_first_node(node_info: dict): ) policy = MetricsPolicy(metrics_listeners=listeners) print("Enabling metrics...") - client.enable_metrics(policy=policy) + as_client.enable_metrics(policy=policy) # Close the last node - NODE_TO_CLOSE = NODE_COUNT - print(f"Closing node {NODE_TO_CLOSE}...") - subprocess.run(["aerolab", "cluster", "stop", f"--nodes={NODE_TO_CLOSE}"], check=True) - # Run with --force or else it will ask to confirm - subprocess.run(["aerolab", "cluster", "destroy", f"--nodes={NODE_TO_CLOSE}", "--force"], check=True) + print("Closing node...") + container.stop() + container.remove() print("Giving client time to run the node_close listener...") - time.sleep(10) + elapsed_secs = 0 + while elapsed_secs < 10: + if node_close_called: + break + time.sleep(1) + elapsed_secs += 1 assert node_close_called is True # Need to print assert result somehow @@ -130,9 +116,6 @@ def get_first_node(node_info: dict): # Calling close() after we lose connection to the whole cluster is safe. It will be a no-op # It is not explicitly documented for the Python client or C client, but this behavior was verified with C # client developer - client.close() + as_client.close() finally: - print("Cleaning up...") - if NODE_COUNT > 1: - subprocess.run(["aerolab", "cluster", "stop"], check=True) - subprocess.run(["aerolab", "cluster", "destroy", "--force"], check=True) + docker_client.close() diff --git a/test/new_tests/conftest.py b/test/new_tests/conftest.py index 6bd4c466d..f0a13b3a3 100644 --- a/test/new_tests/conftest.py +++ b/test/new_tests/conftest.py @@ -106,6 +106,33 @@ def as_connection(request): request.cls.as_connection = as_client + # Check that strong consistency is enabled for all nodes + if TestBaseClass.enterprise_in_use(): + ns_info = as_client.info_all("get-config:context=namespace;namespace=test") + are_all_nodes_sc_enabled = False + for i, (error, result) in enumerate(ns_info.values()): + if error: + # If we can't determine SC is enabled, just assume it isn't + # We don't want to break the tests if this code fails + print("Node returned error while getting config for namespace test") + break + ns_properties = result.split(";") + ns_properties = filter(lambda prop: "strong-consistency=" in prop, ns_properties) + ns_properties = list(ns_properties) + if len(ns_properties) == 0: + print("Strong consistency not found in node properties, so assuming it's disabled by default") + break + elif len(ns_properties) > 1: + print("Only one strong-consistency property should be present") + break + _, sc_enabled = ns_properties[0].split("=") + if sc_enabled == 'false': + print("One of the nodes is not SC enabled") + break + if i == len(ns_info) - 1: + are_all_nodes_sc_enabled = True + TestBaseClass.strong_consistency_enabled = TestBaseClass.enterprise_in_use() and are_all_nodes_sc_enabled + def close_connection(): as_client.close() diff --git a/test/new_tests/test_base_class.py b/test/new_tests/test_base_class.py index 4c5acc7dd..e596b1732 100644 --- a/test/new_tests/test_base_class.py +++ b/test/new_tests/test_base_class.py @@ -181,7 +181,6 @@ def get_new_connection(add_config=None): # major_ver = res[0] # minor_ver = res[1] # print("major_ver:", major_ver, "minor_ver:", minor_ver) - return client @staticmethod diff --git a/test/new_tests/test_metrics.py b/test/new_tests/test_metrics.py index 54a3e7ed9..0fc4201f5 100644 --- a/test/new_tests/test_metrics.py +++ b/test/new_tests/test_metrics.py @@ -158,7 +158,7 @@ def test_setting_metrics_policy_custom_settings(self): assert type(cluster) == Cluster assert cluster.cluster_name is None or type(cluster.cluster_name) == str assert type(cluster.invalid_node_count) == int - assert type(cluster.tran_count) == int + assert type(cluster.command_count) == int assert type(cluster.retry_count) == int assert type(cluster.nodes) == list # Also check the Node and ConnectionStats objects in the Cluster object were populated diff --git a/test/new_tests/test_mrt_api.py b/test/new_tests/test_mrt_api.py new file mode 100644 index 000000000..352e9621c --- /dev/null +++ b/test/new_tests/test_mrt_api.py @@ -0,0 +1,87 @@ +import aerospike +from aerospike import exception as e +import pytest +from contextlib import nullcontext +from typing import Optional, Callable + + +@pytest.mark.usefixtures("as_connection") +class TestMRTAPI: + @pytest.mark.parametrize( + "kwargs, context, err_msg", + [ + ({}, nullcontext(), None), + ({"reads_capacity": 256, "writes_capacity": 256}, nullcontext(), None), + ( + {"reads_capacity": 256, "writes_capacity": 256, "invalid_arg": 1}, + pytest.raises((TypeError)), "function takes at most 2 keyword arguments (3 given)" + ), + ( + {"reads_capacity": "256", "writes_capacity": 256}, + pytest.raises((TypeError)), "reads_capacity must be an integer" + ), + ( + {"reads_capacity": 256, "writes_capacity": "256"}, + pytest.raises((TypeError)), "writes_capacity must be an integer" + ), + # Only need to test codepath once for uint32_t conversion helper function + ( + {"reads_capacity": 2**32, "writes_capacity": 256}, + # Linux x64's unsigned long is 8 bytes long at most + # but Windows x64's unsigned long is 4 bytes long at most + # Python in Windows x64 will throw an internal error (OverflowError) when trying to convert a Python + # int that is larger than 4 bytes into an unsigned long. + # That error doesn't happen in Linux for that same scenario, so we throw our own error + pytest.raises((ValueError, OverflowError)), + "reads_capacity is too large for an unsigned 32-bit integer" + ) + ], + ) + def test_transaction_class(self, kwargs: dict, context, err_msg: Optional[str]): + with context as excinfo: + mrt = aerospike.Transaction(**kwargs) + if type(context) == nullcontext: + assert type(mrt.id) == int + assert type(mrt.timeout) == int + assert type(mrt.state) == int + assert type(mrt.in_doubt) == bool + mrt.timeout = 10 + else: + # Just use kwargs to id the test case + if kwargs == {"reads_capacity": 2**32, "writes_capacity": 256} and excinfo.type == OverflowError: + # Internal Python error thrown in Windows + assert str(excinfo.value) == "Python int too large to convert to C unsigned long" + else: + # Custom error thrown by Python client for other platforms + assert str(excinfo.value) == err_msg + + # Even though this is an unlikely use case, this should not cause problems. + def test_transaction_reinit(self): + mrt = aerospike.Transaction() + # Create a new transaction object using the same Python class instance + mrt.__init__() + + @pytest.mark.parametrize( + "args", + [ + [], + ["string"] + ] + ) + @pytest.mark.parametrize( + "api_call", + [ + aerospike.Client.commit, + aerospike.Client.abort + ] + ) + def test_mrt_invalid_args(self, args: list, api_call: Callable): + with pytest.raises(TypeError): + api_call(self.as_connection, *args) + + def test_invalid_txn_in_policy(self): + policy = {"txn": True} + key = ("test", "demo", 1) + with pytest.raises(e.ParamError) as excinfo: + self.as_connection.get(key, policy) + assert excinfo.value.msg == "txn is not of type aerospike.Transaction" diff --git a/test/new_tests/test_mrt_functionality.py b/test/new_tests/test_mrt_functionality.py new file mode 100644 index 000000000..d395fe090 --- /dev/null +++ b/test/new_tests/test_mrt_functionality.py @@ -0,0 +1,104 @@ +import pytest +from aerospike import exception as e +import aerospike +from .test_base_class import TestBaseClass +import time + + +class TestMRTBasicFunctionality: + def setup_class(cls): + cls.keys = [] + NUM_RECORDS = 2 + for i in range(NUM_RECORDS): + key = ("test", "demo", i) + cls.keys.append(key) + cls.bin_name = "a" + + @pytest.fixture(autouse=True) + def insert_or_update_records(self, as_connection): + if (TestBaseClass.major_ver, TestBaseClass.minor_ver) < (8, 0): + pytest.skip("MRT is only supported in server version 8.0 or higher") + if TestBaseClass.strong_consistency_enabled is False: + pytest.skip("Strong consistency is not enabled") + + for i, key in enumerate(self.keys): + self.as_connection.put(key, {self.bin_name: i}) + + # Test case 1: Execute a simple MRT with multiple SRTs(Read and Write) in any sequence (P3) + # Validate that all operations complete successfully. + def test_commit_api_and_functionality(self): + mrt = aerospike.Transaction() + policy = { + "txn": mrt + } + self.as_connection.put(self.keys[0], {self.bin_name: 1}, policy=policy) + # Reads in an MRT should read the intermediate values of the MRT + _, _, bins = self.as_connection.get(self.keys[0], policy) + # Check that original value was overwritten + assert bins == {self.bin_name: 1} + self.as_connection.put(self.keys[1], {self.bin_name: 2}, policy=policy) + + retval = self.as_connection.commit(transaction=mrt) + assert retval == aerospike.MRT_COMMIT_OK + + # Were the writes committed? + for i in range(len(self.keys)): + _, _, bins = self.as_connection.get(self.keys[i]) + assert bins == {self.bin_name: i + 1} + + def test_timeout_mrt(self): + mrt = aerospike.Transaction() + mrt.timeout = 1 + policy = { + "txn": mrt + } + self.as_connection.put(self.keys[0], {self.bin_name: 1}, policy=policy) + time.sleep(3) + # Server should indicate that MRT has expired + with pytest.raises(e.AerospikeError): + self.as_connection.put(self.keys[1], {self.bin_name: 2}, policy=policy) + + # Cleanup MRT on server side before continuing to run test + self.as_connection.abort(mrt) + + # Test case 57: "Execute the MRT. Before issuing commit, give abort request using abort API" (P1) + def test_abort_api_and_functionality(self): + mrt = aerospike.Transaction() + policy = { + "txn": mrt + } + self.as_connection.put(self.keys[0], {self.bin_name: 1}, policy=policy) + # Should return intermediate overwritten value from MRT + _, _, bins = self.as_connection.get(self.keys[0], policy) + assert bins == {self.bin_name: 1} + self.as_connection.put(self.keys[1], {self.bin_name: 2}, policy=policy) + + retval = self.as_connection.abort(transaction=mrt) + assert retval == aerospike.MRT_ABORT_OK + + # Test that MRT didn't go through + # i.e write commands were rolled back + for i in range(len(self.keys)): + _, _, bins = self.as_connection.get(self.keys[i]) + assert bins == {self.bin_name: i} + + def test_commit_fail(self): + mrt = aerospike.Transaction() + policy = { + "txn": mrt + } + self.as_connection.put(self.keys[0], {self.bin_name: 1}, policy=policy) + self.as_connection.abort(mrt) + status = self.as_connection.commit(mrt) + assert status == aerospike.MRT_COMMIT_ALREADY_ABORTED + + # Test case 10: Issue abort after issung commit. (P1) + def test_abort_fail(self): + mrt = aerospike.Transaction() + policy = { + "txn": mrt + } + self.as_connection.put(self.keys[0], {self.bin_name: 1}, policy=policy) + self.as_connection.commit(mrt) + status = self.as_connection.abort(mrt) + assert status == aerospike.MRT_ABORT_ALREADY_COMMITTED diff --git a/test/new_tests/test_query.py b/test/new_tests/test_query.py index 57337c36c..c7c43eba0 100644 --- a/test/new_tests/test_query.py +++ b/test/new_tests/test_query.py @@ -1148,6 +1148,8 @@ def callback(input_tuple): ] ) def test_query_expected_duration(self, duration: int): + if duration == aerospike.QUERY_DURATION_LONG_RELAX_AP and TestBaseClass.strong_consistency_enabled: + pytest.skip("Using aerospike.QUERY_DURATION_LONG_RELAX_AP will fail if server is in SC mode") query: aerospike.Query = self.as_connection.query("test", "demo") policy = { "expected_duration": duration From a40e5ee88274f04f246347e89fdf569471cf0be1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 6 Nov 2024 22:04:26 +0000 Subject: [PATCH 41/69] Auto-bump version to 15.2.0rc2.dev4 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 03b42ed2a..bd2694b10 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.2.0rc2.dev3 +15.2.0rc2.dev4 From 6f2abe7f23b9dfbc5e7604859019a39e0fcc759b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 6 Nov 2024 23:10:19 +0000 Subject: [PATCH 42/69] Auto-bump version to 15.2.0rc2 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index bd2694b10..02f5e9a2c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.2.0rc2.dev4 +15.2.0rc2 From a46024d01c158ed04cfe74d424aafa139836a3be Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:35:19 -0800 Subject: [PATCH 43/69] [CLIENT-3121] Multi record transactions: raise error if using transaction not in open state. Remove unused client exception "RollAlreadyAttempted" (#693) --- aerospike-client-c | 2 +- src/main/exception.c | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/aerospike-client-c b/aerospike-client-c index c6e67c3a2..10e144219 160000 --- a/aerospike-client-c +++ b/aerospike-client-c @@ -1 +1 @@ -Subproject commit c6e67c3a2f37fc8ef85ea7fdd335fa05dfd5195e +Subproject commit 10e144219248fc7983d352e9024fc8194dc2b69e diff --git a/src/main/exception.c b/src/main/exception.c index 3702edf54..4f9a7ab0d 100644 --- a/src/main/exception.c +++ b/src/main/exception.c @@ -109,8 +109,6 @@ struct exception_def exception_defs[] = { AEROSPIKE_ERR_ASYNC_CONNECTION, NULL), EXCEPTION_DEF("ClientAbortError", CLIENT_ERR_EXCEPTION_NAME, AEROSPIKE_ERR_CLIENT_ABORT, NULL), - EXCEPTION_DEF("RollAlreadyAttempted", CLIENT_ERR_EXCEPTION_NAME, - AEROSPIKE_ROLL_ALREADY_ATTEMPTED, NULL), // Server errors EXCEPTION_DEF("InvalidRequest", SERVER_ERR_EXCEPTION_NAME, AEROSPIKE_ERR_REQUEST_INVALID, NULL), From 1ce51aaecb274b545e444d3e070dd2845fb90993 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 18 Nov 2024 20:35:57 +0000 Subject: [PATCH 44/69] Auto-bump version to 15.2.0rc3.dev1 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 02f5e9a2c..8886457a0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.2.0rc2 +15.2.0rc3.dev1 From 3561eeea28a2acc87679760a5387bee976d747f3 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:25:38 -0800 Subject: [PATCH 45/69] [CLIENT-3169] CI/CD: Lock delvewheel to latest minor version to prevent breaking changes from upstream (#695) --- .github/workflows/build-wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 5d99c7917..efad71185 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -258,7 +258,7 @@ jobs: yum install python-devel -y && yum install python-setuptools -y # delvewheel is not enabled by default but we do need to repair the wheel - CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel" + CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel==1.*" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair --add-path ./aerospike-client-c/vs/x64/Release -w {dest_dir} {wheel}" CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }} From e57b18d23d9108b6b439c5d79fae285c4e5dbf2a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 20 Nov 2024 19:26:12 +0000 Subject: [PATCH 46/69] Auto-bump version to 15.2.0rc3.dev2 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8886457a0..63427784a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.2.0rc3.dev1 +15.2.0rc3.dev2 From d95dfd5048c889f9b7c8f5c6dded0e111ddb572e Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:19:19 -0800 Subject: [PATCH 47/69] [DEVOPS-234] CI/CD: Reuse existing feature branch builds for valgrind runs (#670) It will be a while to clean up all the memory leaks while running the Python client regression tests, and we normally run valgrind manually. We won't be running valgrind as a CI/CD check for a while because of this. Often times, we run valgrind twice, where the first run uses a subset of the full test suite, and the second run uses the whole test suite. Both runs are on the same commit, so this change forces valgrind to reuse a wheel from JFrog if it already exists, like in the second run. Known issue: running the valgrind workflow on central branches (e.g dev) will build and upload the wheels to the generic JFrog repo. This shouldn't cause any issues but this isn't intended behavior. --- .github/workflows/valgrind.yml | 108 +++++++++++++++++++++++++++++++-- 1 file changed, 104 insertions(+), 4 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 1eff9db25..f01175f2e 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -21,19 +21,103 @@ on: required: false default: false +env: + PYTHON_TAG: cp38 + jobs: + look-for-wheel-in-jfrog: + outputs: + num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} + # So we can pass the python tag to a reusable workflow + python-tag: ${{ env.PYTHON_TAG }} + runs-on: ubuntu-22.04 + env: + JF_SEARCH_RESULTS_FILE_NAME: wheel_commit_matches.txt + steps: + - uses: jfrog/setup-jfrog-cli@v4 + env: + JF_URL: ${{ secrets.JFROG_PLATFORM_URL }} + JF_ACCESS_TOKEN: ${{ secrets.JFROG_ACCESS_TOKEN }} + + - name: Get shortened commit hash of this workflow run + # versioningit commit sha is always 8 chars long it seems + run: echo SHORT_GITHUB_SHA=$(echo ${{ github.sha }} | cut -c1-8) >> $GITHUB_ENV + + - name: Look for wheel built with default settings in JFrog + # AQL has the option to exclude patterns from search results + # but it doesn't allow regex, so we can't filter out any type of label in a wheel name + # Example: we want to filter out "unoptimized" and "dsym" but in case we add more labels, we want to use regex + # to handle those new labels without updating the regex. + run: jf rt search "${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/*${{ env.SHORT_GITHUB_SHA }}*${{ env.PYTHON_TAG }}*manylinux*x86_64*.whl" > ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + + - name: Show unfiltered results + run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + + - name: Install sponge + run: sudo apt install -y moreutils + + - name: Filter out wheels with labels in results + run: jq 'map(select(.path | test("${{ env.SHORT_GITHUB_SHA }}\\.") | not))' ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | sponge ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + shell: bash + + - name: Check if artifacts with labels were filtered out + run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + + - name: Count artifacts + id: count_num_artifacts_found + run: echo num_artifacts=$(jq length ${{ env.JF_SEARCH_RESULTS_FILE_NAME }}) >> $GITHUB_OUTPUT + + - name: Multiple artifacts found, not sure which one to use. Fail out + if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts > 1 }} + run: exit 1 + + - name: Found the exact artifact in JFrog. Get the artifact name + if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} + run: echo ARTIFACT_PATH=$(jq -r .[0].path ${{ env.JF_SEARCH_RESULTS_FILE_NAME }}) >> $GITHUB_ENV + + - name: Then download artifact from JFrog + if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} + run: jf rt download --flat --fail-no-op ${{ env.ARTIFACT_PATH }} + + - name: Pass to valgrind job + if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} + uses: actions/upload-artifact@v4 + with: + # Artifact name doesn't matter. Valgrind job downloads all artifacts to get the one wheel + if-no-files-found: error + path: './*.whl' + build-manylinux-wheel: + needs: look-for-wheel-in-jfrog + if: ${{ needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 }} uses: ./.github/workflows/build-wheels.yml with: - python-tags: '["cp38"]' + python-tags: '["${{ needs.look-for-wheel-in-jfrog.outputs.python-tag }}"]' platform-tag: manylinux_x86_64 sha-to-build-and-test: ${{ github.sha }} secrets: inherit + upload-built-wheel-to-jfrog: + needs: build-manylinux-wheel + # TODO: this job should skip when this workflow is run on central branches + # We already have artifacts available for central branches in the PyPI-type JFrog repo + # The problem is we have to conditionally skip this job, but using the github context to get the branch name + # doesn't work for some reason. Just leave this alone for now. + uses: ./.github/workflows/upload-to-jfrog.yml + with: + jfrog-repo-name: ${{ vars.JFROG_GENERIC_REPO_NAME }} + secrets: inherit + valgrind: env: MASSIF_REPORT_FILE_NAME: massif.out - needs: build-manylinux-wheel + needs: [ + look-for-wheel-in-jfrog, + build-manylinux-wheel + ] + # Case 1: Found artifact in JFrog + # Case 2: Did not find artifact in JFrog, had to build it in GHA + if: ${{ !cancelled() && (needs.look-for-wheel-in-jfrog.result == 'success' && (needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 1) || (needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 && needs.build-manylinux-wheel.result == 'success')) }} runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 @@ -41,14 +125,18 @@ jobs: submodules: recursive fetch-depth: 0 + - name: Convert Python tag to Python version + run: echo PYTHON_VERSION=$(echo ${{ env.PYTHON_TAG }} | sed -e "s/cp3/cp3./" -e "s/cp//") >> $GITHUB_ENV + shell: bash + - uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '${{ env.PYTHON_VERSION }}' architecture: 'x64' - uses: actions/download-artifact@v4 with: - name: cp38-manylinux_x86_64.build + merge-multiple: true - name: Install client run: pip install ./*.whl @@ -76,6 +164,18 @@ jobs: - run: PYTHONMALLOC=malloc valgrind --error-exitcode=1 ${{ env.VALGRIND_ARGS }} python3 -m pytest -v new_tests/${{ github.event.inputs.test-file }} working-directory: test + # TODO: upload report as artifact - run: ms_print ./${{ env.MASSIF_REPORT_FILE_NAME }} if: ${{ !cancelled() && inputs.massif }} working-directory: test + + # See reason for deleting artifacts in dev-workflow-p2.yml + delete-artifacts: + needs: [ + # These jobs must have downloaded the artifact from Github before we can delete it + upload-built-wheel-to-jfrog, + valgrind + ] + # Workflow run must clean up after itself even if cancelled + if: ${{ always() }} + uses: ./.github/workflows/delete-artifacts.yml From 625236c51d89a43a083cf3221db6e4037d1f476f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 20 Nov 2024 20:19:53 +0000 Subject: [PATCH 48/69] Auto-bump version to 15.2.0rc3.dev3 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 63427784a..a40a53b2d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.2.0rc3.dev2 +15.2.0rc3.dev3 From 1448f14550852e53412bf806f26f00c54b24051d Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 21 Nov 2024 07:19:08 -0800 Subject: [PATCH 49/69] [CLIENT-3027] CI/CD: Refactor composite action that deploys an EE server for testing (#682) Combine all test configuration code into one composite action --- .../action.yml | 37 -------- .github/actions/run-ee-server/action.yml | 92 +++++++++++++------ .github/workflows/build-wheels.yml | 24 +---- .github/workflows/stage-tests.yml | 2 + .github/workflows/test-server-rc.yml | 3 +- 5 files changed, 71 insertions(+), 87 deletions(-) delete mode 100644 .github/actions/run-ee-server-for-ext-container/action.yml diff --git a/.github/actions/run-ee-server-for-ext-container/action.yml b/.github/actions/run-ee-server-for-ext-container/action.yml deleted file mode 100644 index 42e8b53a9..000000000 --- a/.github/actions/run-ee-server-for-ext-container/action.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: 'Run EE server for another Docker container' -description: 'Run EE server and configure tests to connect to it from another Docker container' -inputs: - # All inputs in composite actions are strings - use-server-rc: - required: true - default: false - server-tag: - required: true - default: 'latest' - # Github Composite Actions can't access secrets - # so we need to pass them in as inputs - docker-hub-username: - required: false - docker-hub-password: - required: false - -runs: - using: "composite" - steps: - - name: Run EE server - uses: ./.github/actions/run-ee-server - with: - use-server-rc: ${{ inputs.use-server-rc }} - server-tag: ${{ inputs.server-tag }} - docker-hub-username: ${{ inputs.docker-hub-username }} - docker-hub-password: ${{ inputs.docker-hub-password }} - - - name: Get IP address of Docker container hosting server - id: get-server-ip-address - run: echo server-ip=$(docker container inspect -f '{{ .NetworkSettings.IPAddress }}' aerospike) >> $GITHUB_OUTPUT - shell: bash - - - name: Configure tests to connect to that Docker container - run: crudini --existing=param --set config.conf enterprise-edition hosts ${{ steps.get-server-ip-address.outputs.server-ip }}:3000 - working-directory: test - shell: bash diff --git a/.github/actions/run-ee-server/action.yml b/.github/actions/run-ee-server/action.yml index d83d18e4d..4a359b6f6 100644 --- a/.github/actions/run-ee-server/action.yml +++ b/.github/actions/run-ee-server/action.yml @@ -1,4 +1,4 @@ -name: 'Run EE Server' +name: 'Run EE Server in a Docker container' description: 'Run EE server. Returns once server is ready. Only tested on Linux and macOS' # NOTE: do not share this server container with others # since it's using the default admin / admin credentials @@ -20,37 +20,15 @@ inputs: docker-hub-password: description: Required for using release candidates required: false + where-is-client-connecting-from: + required: false + description: 'docker-host, separate-docker-container, "remote-connection" via DOCKER_HOST' + default: 'docker-host' runs: using: "composite" steps: - - name: Install crudini to manipulate config.conf - # This will only work on the Github hosted runners. - run: pipx install crudini --pip-args "-c ${{ github.workspace }}/.github/workflows/requirements.txt" - working-directory: .github/workflows - shell: bash - - - name: Create config.conf - run: cp config.conf.template config.conf - working-directory: test - shell: bash - - - name: Use enterprise edition instead of community edition in config.conf - run: | - crudini --existing=param --set config.conf enterprise-edition hosts '' - crudini --existing=param --set config.conf enterprise-edition hosts 127.0.0.1:3000 - working-directory: test - shell: bash - - - run: echo SUPERUSER_NAME_AND_PASSWORD="superuser" >> $GITHUB_ENV - shell: bash - - - name: Set credentials in config file - run: | - crudini --existing=param --set config.conf enterprise-edition user ${{ env.SUPERUSER_NAME_AND_PASSWORD }} - crudini --existing=param --set config.conf enterprise-edition password ${{ env.SUPERUSER_NAME_AND_PASSWORD }} - working-directory: test - shell: bash + # Start up server - name: Log into Docker Hub to get server RC if: ${{ inputs.use-server-rc == 'true' }} @@ -60,7 +38,7 @@ runs: - run: echo IMAGE_NAME=aerospike/aerospike-server-enterprise${{ inputs.use-server-rc == 'true' && '-rc' || '' }}:${{ inputs.server-tag }} >> $GITHUB_ENV shell: bash - - run: echo NEW_IMAGE_NAME=${{ env.IMAGE_NAME }}-security-and-sc >> $GITHUB_ENV + - run: echo NEW_IMAGE_NAME=${{ env.IMAGE_NAME }}-python-client-testing >> $GITHUB_ENV shell: bash # macOS Github runners and Windows self-hosted runners don't have buildx installed by default @@ -92,6 +70,9 @@ runs: is-security-enabled: true is-strong-consistency-enabled: true + - run: echo SUPERUSER_NAME_AND_PASSWORD="superuser" >> $GITHUB_ENV + shell: bash + - run: echo ASADM_AUTH_FLAGS="--user=${{ env.SUPERUSER_NAME_AND_PASSWORD }} --password=${{ env.SUPERUSER_NAME_AND_PASSWORD }}" >> $GITHUB_ENV shell: bash @@ -106,3 +87,56 @@ runs: # For debugging - run: docker logs aerospike shell: bash + + # Configure tests + + - name: Install crudini to manipulate config.conf + run: pipx install crudini --pip-args "-c ${{ github.workspace }}/.github/workflows/requirements.txt" + working-directory: .github/workflows + shell: bash + + - name: Create config.conf + run: cp config.conf.template config.conf + working-directory: test + shell: bash + + - name: Disable community edition connection + run: crudini --existing=param --set config.conf community-edition hosts '' + working-directory: test + shell: bash + + - name: Set credentials in config file + run: | + crudini --existing=param --set config.conf enterprise-edition user ${{ env.SUPERUSER_NAME_AND_PASSWORD }} + crudini --existing=param --set config.conf enterprise-edition password ${{ env.SUPERUSER_NAME_AND_PASSWORD }} + working-directory: test + shell: bash + + - name: Set IP address to localhost + if: ${{ inputs.where-is-client-connecting-from == 'docker-host' }} + run: echo SERVER_IP=127.0.0.1 >> $GITHUB_ENV + working-directory: test + shell: bash + + - name: Set IP address to remote machine running the Docker daemon + if: ${{ inputs.where-is-client-connecting-from == 'remote-connection' }} + run: | + SERVER_IP=${DOCKER_HOST/tcp:\/\//} + echo SERVER_IP=${SERVER_IP/:2375/} >> $GITHUB_ENV + working-directory: test + shell: bash + + - name: Set IP address to Docker container for the server + if: ${{ inputs.where-is-client-connecting-from == 'separate-docker-container' }} + run: echo SERVER_IP=$(docker container inspect -f '{{ .NetworkSettings.IPAddress }}' aerospike) >> $GITHUB_ENV + shell: bash + + - name: Invalid input + if: ${{ env.SERVER_IP == '' }} + run: exit 1 + shell: bash + + - name: Set EE server's IP address + run: crudini --existing=param --set config.conf enterprise-edition hosts ${{ env.SERVER_IP }}:3000 + working-directory: test + shell: bash diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index efad71185..8fdd99fd9 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -205,24 +205,15 @@ jobs: if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' && inputs.platform-tag == 'macosx_x86_64' }} uses: ./.github/actions/setup-docker-on-macos - - name: 'macOS x86: run Aerospike server in Docker container and connect via localhost' - if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' && inputs.platform-tag == 'macosx_x86_64' }} + - name: 'Run Aerospike server in Docker container and configure tests accordingly' + if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }} uses: ./.github/actions/run-ee-server with: use-server-rc: ${{ inputs.use-server-rc }} server-tag: ${{ inputs.server-tag }} docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} - - # TODO: combine this composite action and the above into one - - name: "Linux: run Aerospike server in Docker container and configure config.conf to connect to the server container's Docker IP address" - if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' && startsWith(inputs.platform-tag, 'manylinux') }} - uses: ./.github/actions/run-ee-server-for-ext-container - with: - use-server-rc: ${{ inputs.use-server-rc }} - server-tag: ${{ inputs.server-tag }} - docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} - docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + where-is-client-connecting-from: ${{ inputs.platform-tag == 'macosx_x86_64' && 'docker-host' || 'separate-docker-container' }} - name: If not running tests against server, only run basic import test if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'false' }} @@ -314,6 +305,7 @@ jobs: server-tag: ${{ inputs.server-tag }} docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + where-is-client-connecting-from: ${{ inputs.platform-tag == 'win_amd64' && 'remote-connection' || 'docker-host' }} - name: Download wheel uses: actions/download-artifact@v4 @@ -337,14 +329,6 @@ jobs: run: python3 -m pip install aerospike --force-reinstall --no-index --find-links=./ shell: bash - - name: Connect to Docker container on remote machine with Docker daemon - if: ${{ inputs.platform-tag == 'win_amd64' }} - # DOCKER_HOST contains the IP address of the remote machine - run: | - $env:DOCKER_HOST_IP = $env:DOCKER_HOST | foreach {$_.replace("tcp://","")} | foreach {$_.replace(":2375", "")} - crudini --set config.conf enterprise-edition hosts ${env:DOCKER_HOST_IP}:3000 - working-directory: test - - run: python3 -m pip install pytest -c requirements.txt working-directory: test shell: bash diff --git a/.github/workflows/stage-tests.yml b/.github/workflows/stage-tests.yml index a7664a3e0..ba8f96384 100644 --- a/.github/workflows/stage-tests.yml +++ b/.github/workflows/stage-tests.yml @@ -102,6 +102,7 @@ jobs: server-tag: ${{ inputs.server-tag }} docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + where-is-client-connecting-from: 'separate-docker-container' - name: Run distro container # Run distro container on host network to access the Aerospike server using localhost (without having to change config.conf) @@ -194,6 +195,7 @@ jobs: server-tag: ${{ inputs.server-tag }} docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + where-is-client-connecting-from: 'docker-host' - name: Install wheel run: python3 -m pip install *.whl diff --git a/.github/workflows/test-server-rc.yml b/.github/workflows/test-server-rc.yml index e03a6cb8a..1d11d5702 100644 --- a/.github/workflows/test-server-rc.yml +++ b/.github/workflows/test-server-rc.yml @@ -34,12 +34,13 @@ jobs: - run: docker run -d --name manylinux quay.io/pypa/manylinux2014_${{ matrix.platform[0] }} tail -f /dev/null - - uses: ./.github/actions/run-ee-server-for-ext-container + - uses: ./.github/actions/run-ee-server with: use-server-rc: true server-tag: latest docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + where-is-client-connecting-from: 'docker-container' - uses: actions/download-artifact@v4 with: From a2c35b5463d31510a082b7ad8e98b99429aa88f9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:19:51 +0000 Subject: [PATCH 50/69] Auto-bump version to 15.2.0rc3.dev4 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index a40a53b2d..045439162 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.2.0rc3.dev3 +15.2.0rc3.dev4 From b1b8ed0ea360bc8b7615d75dde2f5ba9d7fab688 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 21 Nov 2024 10:50:09 -0800 Subject: [PATCH 51/69] [CLIENT-2992] Add Python 3.13 support (#629) --- .build.yml | 2 +- .github/workflows/build-wheels.yml | 6 +++--- .github/workflows/stage-tests.yml | 3 +++ .github/workflows/tests.yml | 5 +++-- README.rst | 2 +- pyproject.toml | 1 + test/tox.ini | 2 +- 7 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.build.yml b/.build.yml index 7be220ecf..f3ad3e625 100644 --- a/.build.yml +++ b/.build.yml @@ -8,7 +8,7 @@ container: build: - name: build environment: - PYTHONS: /opt/python/cp38-cp38/bin,/opt/python/cp39-cp39/bin,/opt/python/cp310-cp310/bin,/opt/python/cp311-cp311/bin,/opt/python/cp312-cp312/bin + PYTHONS: /opt/python/cp38-cp38/bin,/opt/python/cp39-cp39/bin,/opt/python/cp310-cp310/bin,/opt/python/cp311-cp311/bin,/opt/python/cp312-cp312/bin,/opt/python/cp313-cp313/bin script: - scripts/manylinux2014build.sh artifact: diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 8fdd99fd9..13ded5ca0 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -27,7 +27,7 @@ on: type: string description: Valid JSON list of Python tags to build the client for required: false - default: '["cp38", "cp39", "cp310", "cp311", "cp312"]' + default: '["cp38", "cp39", "cp310", "cp311", "cp312", "cp313"]' platform-tag: description: Platform to build the client for. type: choice @@ -80,7 +80,7 @@ on: python-tags: type: string required: false - default: '["cp38", "cp39", "cp310", "cp311", "cp312"]' + default: '["cp38", "cp39", "cp310", "cp311", "cp312", "cp313"]' platform-tag: type: string required: true @@ -238,7 +238,7 @@ jobs: run: echo "INCLUDE_DSYM=1" >> $GITHUB_ENV - name: Build wheel - uses: pypa/cibuildwheel@v2.20.0 + uses: pypa/cibuildwheel@v2.21.3 env: CIBW_ENVIRONMENT_PASS_LINUX: ${{ inputs.unoptimized && 'UNOPTIMIZED' || '' }} CIBW_ENVIRONMENT_MACOS: SSL_LIB_PATH="$(brew --prefix openssl@${{ env.MACOS_OPENSSL_VERSION }})/lib/" CPATH="$(brew --prefix openssl@${{ env.MACOS_OPENSSL_VERSION }})/include/" STATIC_SSL=1 diff --git a/.github/workflows/stage-tests.yml b/.github/workflows/stage-tests.yml index ba8f96384..7523ee7d2 100644 --- a/.github/workflows/stage-tests.yml +++ b/.github/workflows/stage-tests.yml @@ -58,6 +58,8 @@ jobs: ["python:3.11-bookworm", 2, "linux/arm64", "3.11"], ["python:3.12-bookworm", 2, "linux/amd64", "3.12"], ["python:3.12-bookworm", 2, "linux/arm64", "3.12"], + ["python:3.13-bookworm", 2, "linux/amd64", "3.13"], + ["python:3.13-bookworm", 2, "linux/arm64", "3.13"], ["amazonlinux:2023", 1, "linux/amd64", "3.9"], ["redhat/ubi9", 1, "linux/amd64", "3.9"], ] @@ -158,6 +160,7 @@ jobs: "3.10", "3.11", "3.12", + "3.13", ] fail-fast: false runs-on: ${{ matrix.runner-os }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4279dc17e..901924f1d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -44,7 +44,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - py-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + py-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] fail-fast: false steps: @@ -256,7 +256,8 @@ jobs: "3.9", "3.10", "3.11", - "3.12" + "3.12", + "3.13" ] fail-fast: false diff --git a/README.rst b/README.rst index 5096c484c..3261ac834 100644 --- a/README.rst +++ b/README.rst @@ -11,7 +11,7 @@ Aerospike Python Client Compatibility ------------- -The Python client for Aerospike works with Python 3.8 - 3.12 and supports the following OS'es: +The Python client for Aerospike works with Python 3.8 - 3.13 and supports the following OS'es: * macOS 12 - 14 * CentOS 7 Linux diff --git a/pyproject.toml b/pyproject.toml index b4d871c69..0b9e0238b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Database" ] diff --git a/test/tox.ini b/test/tox.ini index e3ab21558..e1cb1b973 100644 --- a/test/tox.ini +++ b/test/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] skipsdist = True -envlist = py38, py39, py310, py311, py312 +envlist = py38, py39, py310, py311, py312, py313 [testenv] commands = pip install --find-links=local/wheels --no-index aerospike From da46a153a9d1e2db77af711230e0037cfa825685 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:50:49 +0000 Subject: [PATCH 52/69] Auto-bump version to 15.2.0rc3.dev5 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 045439162..10ba07ce6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.2.0rc3.dev4 +15.2.0rc3.dev5 From 8d941e54fb308f964e8049c6fd4c0f0572ec2fdc Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:14:04 -0800 Subject: [PATCH 53/69] [CLIENT-3166] Fix AerospikeError attributes not being assigned when an error code without a matching Exception is raised (#694) AerospikeError.in_doubt should always be set as an attribute, even if an AerospikeError was not raised by the client yet --- src/include/exceptions.h | 2 ++ src/main/client/apply.c | 1 + src/main/client/exists.c | 1 + src/main/client/get.c | 1 + src/main/client/info.c | 3 ++ src/main/client/operate.c | 2 ++ src/main/client/put.c | 1 + src/main/client/remove.c | 1 + src/main/client/remove_bin.c | 2 ++ src/main/client/sec_index.c | 2 ++ src/main/client/select.c | 1 + src/main/client/udf.c | 4 +++ src/main/exception.c | 53 ++++++++++++++----------------- src/main/query/apply.c | 1 + src/main/query/foreach.c | 1 + src/main/scan/apply.c | 1 + test/new_tests/test_exceptions.py | 4 +-- 17 files changed, 48 insertions(+), 33 deletions(-) diff --git a/src/include/exceptions.h b/src/include/exceptions.h index 2079018bb..f2778307d 100644 --- a/src/include/exceptions.h +++ b/src/include/exceptions.h @@ -22,3 +22,5 @@ PyObject *AerospikeException_New(void); void raise_exception(as_error *err); PyObject *raise_exception_old(as_error *err); void remove_exception(as_error *err); +void set_aerospike_exc_attrs_using_tuple_of_attrs(PyObject *py_exc, + PyObject *py_tuple); diff --git a/src/main/client/apply.c b/src/main/client/apply.c index 118140ac9..e342241cf 100644 --- a/src/main/client/apply.c +++ b/src/main/client/apply.c @@ -171,6 +171,7 @@ PyObject *AerospikeClient_Apply_Invoke(AerospikeClient *self, PyObject *py_key, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } diff --git a/src/main/client/exists.c b/src/main/client/exists.c index 383e3a430..d835b324a 100644 --- a/src/main/client/exists.c +++ b/src/main/client/exists.c @@ -139,6 +139,7 @@ extern PyObject *AerospikeClient_Exists_Invoke(AerospikeClient *self, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } diff --git a/src/main/client/get.c b/src/main/client/get.c index 88083795f..41a43f525 100644 --- a/src/main/client/get.c +++ b/src/main/client/get.c @@ -136,6 +136,7 @@ PyObject *AerospikeClient_Get_Invoke(AerospikeClient *self, PyObject *py_key, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } diff --git a/src/main/client/info.c b/src/main/client/info.c index 2cdc55687..be6095b60 100644 --- a/src/main/client/info.c +++ b/src/main/client/info.c @@ -99,6 +99,7 @@ static bool AerospikeClient_InfoAll_each(as_error *err, const as_node *node, PyObject *py_err = NULL; error_to_pyobject(&udata_ptr->error, &py_err); PyObject *exception_type = raise_exception_old(&udata_ptr->error); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); PyErr_SetObject(exception_type, py_err); Py_DECREF(py_err); PyGILState_Release(gil_state); @@ -108,6 +109,7 @@ static bool AerospikeClient_InfoAll_each(as_error *err, const as_node *node, PyObject *py_err = NULL; error_to_pyobject(err, &py_err); PyObject *exception_type = raise_exception_old(err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); PyErr_SetObject(exception_type, py_err); Py_DECREF(py_err); PyGILState_Release(gil_state); @@ -213,6 +215,7 @@ static PyObject *AerospikeClient_InfoAll_Invoke(AerospikeClient *self, error_to_pyobject(&info_callback_udata.error, &py_err); PyObject *exception_type = raise_exception_old(&info_callback_udata.error); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); PyErr_SetObject(exception_type, py_err); Py_DECREF(py_err); if (py_nodes) { diff --git a/src/main/client/operate.c b/src/main/client/operate.c index dbc2513e8..668ff6cf0 100644 --- a/src/main/client/operate.c +++ b/src/main/client/operate.c @@ -79,6 +79,7 @@ static inline bool isExprOp(int op); PyObject *py_err = NULL; \ error_to_pyobject(&err, &py_err); \ PyObject *exception_type = raise_exception_old(&err); \ + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); \ if (PyObject_HasAttrString(exception_type, "key")) { \ PyObject_SetAttrString(exception_type, "key", py_key); \ } \ @@ -1206,6 +1207,7 @@ PyObject *AerospikeClient_OperateOrdered(AerospikeClient *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } diff --git a/src/main/client/put.c b/src/main/client/put.c index d1d51b0df..a09453c13 100644 --- a/src/main/client/put.c +++ b/src/main/client/put.c @@ -133,6 +133,7 @@ PyObject *AerospikeClient_Put_Invoke(AerospikeClient *self, PyObject *py_key, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } diff --git a/src/main/client/remove.c b/src/main/client/remove.c index 1d15c4c5a..b7266742b 100644 --- a/src/main/client/remove.c +++ b/src/main/client/remove.c @@ -136,6 +136,7 @@ PyObject *AerospikeClient_Remove_Invoke(AerospikeClient *self, PyObject *py_key, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } diff --git a/src/main/client/remove_bin.c b/src/main/client/remove_bin.c index e4826960c..ca627841f 100644 --- a/src/main/client/remove_bin.c +++ b/src/main/client/remove_bin.c @@ -163,6 +163,7 @@ AerospikeClient_RemoveBin_Invoke(AerospikeClient *self, PyObject *py_key, PyObject *py_err = NULL; error_to_pyobject(err, &py_err); PyObject *exception_type = raise_exception_old(err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } @@ -239,6 +240,7 @@ PyObject *AerospikeClient_RemoveBin(AerospikeClient *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } diff --git a/src/main/client/sec_index.c b/src/main/client/sec_index.c index 0276148b6..d9129c23e 100644 --- a/src/main/client/sec_index.c +++ b/src/main/client/sec_index.c @@ -232,6 +232,7 @@ PyObject *AerospikeClient_Index_Cdt_Create(AerospikeClient *self, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "name")) { PyObject_SetAttrString(exception_type, "name", py_name); } @@ -333,6 +334,7 @@ PyObject *AerospikeClient_Index_Remove(AerospikeClient *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "name")) { PyObject_SetAttrString(exception_type, "name", py_name); } diff --git a/src/main/client/select.c b/src/main/client/select.c index bf4f70280..5e6f55995 100644 --- a/src/main/client/select.c +++ b/src/main/client/select.c @@ -185,6 +185,7 @@ PyObject *AerospikeClient_Select_Invoke(AerospikeClient *self, PyObject *py_key, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } diff --git a/src/main/client/udf.c b/src/main/client/udf.c index a31f086f3..d161b2726 100644 --- a/src/main/client/udf.c +++ b/src/main/client/udf.c @@ -276,6 +276,7 @@ PyObject *AerospikeClient_UDF_Put(AerospikeClient *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "module")) { PyObject_SetAttrString(exception_type, "module", Py_None); } @@ -372,6 +373,7 @@ PyObject *AerospikeClient_UDF_Remove(AerospikeClient *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "module")) { PyObject_SetAttrString(exception_type, "module", py_filename); } @@ -464,6 +466,7 @@ PyObject *AerospikeClient_UDF_List(AerospikeClient *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "module")) { PyObject_SetAttrString(exception_type, "module", Py_None); } @@ -579,6 +582,7 @@ PyObject *AerospikeClient_UDF_Get_UDF(AerospikeClient *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "module")) { PyObject_SetAttrString(exception_type, "module", py_module); } diff --git a/src/main/exception.c b/src/main/exception.c index 4f9a7ab0d..c0884ec81 100644 --- a/src/main/exception.c +++ b/src/main/exception.c @@ -68,7 +68,9 @@ struct exception_def { // No exception should have an error code of 0, so this should be ok #define NO_ERROR_CODE 0 -const char *const aerospike_err_attrs[] = {"code", "file", "msg", "line", NULL}; +// Same order as the tuple of args passed into the exception +const char *const aerospike_err_attrs[] = {"code", "msg", "file", + "line", "in_doubt", NULL}; const char *const record_err_attrs[] = {"key", "bin", NULL}; const char *const index_err_attrs[] = {"name", NULL}; const char *const udf_err_attrs[] = {"module", "func", NULL}; @@ -367,6 +369,25 @@ void remove_exception(as_error *err) } } +// We have this as a separate method because both raise_exception and raise_exception_old need to use it +void set_aerospike_exc_attrs_using_tuple_of_attrs(PyObject *py_exc, + PyObject *py_tuple) +{ + for (unsigned long i = 0; + i < sizeof(aerospike_err_attrs) / sizeof(aerospike_err_attrs[0]) - 1; + i++) { + // Here, we are assuming the number of attrs is the same as the number of tuple members + PyObject *py_arg = PyTuple_GetItem(py_tuple, i); + if (py_arg == NULL) { + // Don't fail out if number of attrs > number of tuple members + // This condition should never be true, though + PyErr_Clear(); + break; + } + PyObject_SetAttrString(py_exc, aerospike_err_attrs[i], py_arg); + } +} + // TODO: idea. Use python dict to map error code to exception void raise_exception(as_error *err) { @@ -383,37 +404,8 @@ void raise_exception(as_error *err) } if (err->code == PyLong_AsLong(py_code)) { found = true; - PyObject *py_attr = NULL; - py_attr = PyUnicode_FromString(err->message); - PyObject_SetAttrString(py_value, "msg", py_attr); - Py_DECREF(py_attr); - - // as_error.file is a char* so this may be null - if (err->file) { - py_attr = PyUnicode_FromString(err->file); - PyObject_SetAttrString(py_value, "file", py_attr); - Py_DECREF(py_attr); - } - else { - PyObject_SetAttrString(py_value, "file", Py_None); - } - // If the line is 0, set it as None - if (err->line > 0) { - py_attr = PyLong_FromLong(err->line); - PyObject_SetAttrString(py_value, "line", py_attr); - Py_DECREF(py_attr); - } - else { - PyObject_SetAttrString(py_value, "line", Py_None); - } - - py_attr = PyBool_FromLong(err->in_doubt); - PyObject_SetAttrString(py_value, "in_doubt", py_attr); - Py_DECREF(py_attr); - break; } - Py_DECREF(py_code); } } // We haven't found the right exception, just use AerospikeError @@ -431,6 +423,7 @@ void raise_exception(as_error *err) // Convert C error to Python exception PyObject *py_err = NULL; error_to_pyobject(err, &py_err); + set_aerospike_exc_attrs_using_tuple_of_attrs(py_value, py_err); // Raise exception PyErr_SetObject(py_value, py_err); diff --git a/src/main/query/apply.c b/src/main/query/apply.c index 9218927ed..f78c273cb 100644 --- a/src/main/query/apply.c +++ b/src/main/query/apply.c @@ -150,6 +150,7 @@ AerospikeQuery *AerospikeQuery_Apply(AerospikeQuery *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "module")) { PyObject_SetAttrString(exception_type, "module", py_module); } diff --git a/src/main/query/foreach.c b/src/main/query/foreach.c index 0842c76fd..db4e1d452 100644 --- a/src/main/query/foreach.c +++ b/src/main/query/foreach.c @@ -254,6 +254,7 @@ PyObject *AerospikeQuery_Foreach(AerospikeQuery *self, PyObject *args, error_to_pyobject(&data.error, &py_err); exception_type = raise_exception_old(&data.error); } + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "name")) { PyObject_SetAttrString(exception_type, "name", Py_None); } diff --git a/src/main/scan/apply.c b/src/main/scan/apply.c index 13e4e0ce4..ef1a2cfdf 100644 --- a/src/main/scan/apply.c +++ b/src/main/scan/apply.c @@ -148,6 +148,7 @@ AerospikeScan *AerospikeScan_Apply(AerospikeScan *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "module")) { PyObject_SetAttrString(exception_type, "module", py_module); } diff --git a/test/new_tests/test_exceptions.py b/test/new_tests/test_exceptions.py index 931a7f831..1276735e7 100644 --- a/test/new_tests/test_exceptions.py +++ b/test/new_tests/test_exceptions.py @@ -23,9 +23,7 @@ "msg", "file", "line", - # in_doubt is only added when an AerospikeError or subclass of it is raised by the client - # This attribute is not set when initializing the exception classes in `aerospike.exception` - # "in_doubt" + "in_doubt" ], e.RecordError: [ "key", From 7573e26d45a2880088818581366811ad00f5469d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 21 Nov 2024 20:14:39 +0000 Subject: [PATCH 54/69] Auto-bump version to 15.2.0rc3.dev6 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 10ba07ce6..d7b240953 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.2.0rc3.dev5 +15.2.0rc3.dev6 From 10884ec39e1612ae67298a48f2d8f5e2b6535517 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 21 Nov 2024 16:41:28 -0800 Subject: [PATCH 55/69] [CLIENT-3173] Multi-record transactions: Set default timeout to use the server configuration's value (#696) - Add support for server error code AEROSPIKE_MRT_TOO_MANY_WRITES - Improve documentation for MRT timeouts --- .gitmodules | 2 +- aerospike-client-c | 2 +- doc/transaction.rst | 10 +++++++++- test/new_tests/test_mrt_api.py | 2 ++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 05ae9fd87..136ba68cb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,4 +2,4 @@ path = aerospike-client-c # url = git@github.com:aerospike/aerospike-client-c.git url = https://github.com/aerospike/aerospike-client-c.git - branch = CLIENT-2294 + branch = stage diff --git a/aerospike-client-c b/aerospike-client-c index 10e144219..886306eee 160000 --- a/aerospike-client-c +++ b/aerospike-client-c @@ -1 +1 @@ -Subproject commit 10e144219248fc7983d352e9024fc8194dc2b69e +Subproject commit 886306eeea9869190cae9c0a7782eee83e1c345d diff --git a/doc/transaction.rst b/doc/transaction.rst index 34dda0331..220cacbac 100644 --- a/doc/transaction.rst +++ b/doc/transaction.rst @@ -18,7 +18,7 @@ Methods .. class:: Transaction Initialize multi-record transaction (MRT), assign random transaction id and initialize - reads/writes hashmaps with default capacities. The default MRT timeout is 10 seconds. + reads/writes hashmaps with default capacities. For both parameters, an unsigned 32-bit integer must be passed and the minimum value should be 16. @@ -50,6 +50,14 @@ Methods :type: int .. py:attribute:: timeout + MRT timeout in seconds. The timer starts when the MRT monitor record is created. + This occurs when the first command in the MRT is executed. If the timeout is reached before + :py:meth:`~aerospike.Client.commit` or :py:meth:`~aerospike.Client.abort` is called, the server will expire and + rollback the MRT. + + The default client MRT timeout is zero. This means use the server configuration ``mrt-duration`` + as the MRT timeout. The default ``mrt-duration`` is 10 seconds. + This attribute can be read and written to. :type: int diff --git a/test/new_tests/test_mrt_api.py b/test/new_tests/test_mrt_api.py index 352e9621c..e88e40539 100644 --- a/test/new_tests/test_mrt_api.py +++ b/test/new_tests/test_mrt_api.py @@ -41,10 +41,12 @@ def test_transaction_class(self, kwargs: dict, context, err_msg: Optional[str]): with context as excinfo: mrt = aerospike.Transaction(**kwargs) if type(context) == nullcontext: + # Can we read these fields assert type(mrt.id) == int assert type(mrt.timeout) == int assert type(mrt.state) == int assert type(mrt.in_doubt) == bool + # Can we change the timeout? mrt.timeout = 10 else: # Just use kwargs to id the test case From 6b3191e933c0b237011d74ea3dde4b36ff931098 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 22 Nov 2024 00:42:06 +0000 Subject: [PATCH 56/69] Auto-bump version to 15.2.0rc3.dev7 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d7b240953..2f938c4c5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.2.0rc3.dev6 +15.2.0rc3.dev7 From ca2f4be813e1625a3bb6af930860e780347e7c74 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 21 Nov 2024 16:48:44 -0800 Subject: [PATCH 57/69] [CLIENT-3160] Enable TLS when deploying an enterprise edition server for testing (#692) We enable TLS standard authentication to verify that the OpenSSL library bundled with the wheel works --- .github/actions/run-ee-server/action.yml | 40 ++++++- .github/workflows/build-wheels.yml | 6 + .../workflows/docker-build-context/Dockerfile | 103 ++++++++++++++---- 3 files changed, 123 insertions(+), 26 deletions(-) diff --git a/.github/actions/run-ee-server/action.yml b/.github/actions/run-ee-server/action.yml index 4a359b6f6..898ce018c 100644 --- a/.github/actions/run-ee-server/action.yml +++ b/.github/actions/run-ee-server/action.yml @@ -45,13 +45,35 @@ runs: - if: ${{ runner.os == 'Windows' || runner.os == 'macOS' }} uses: docker/setup-buildx-action@v3 - - name: Build and push + - run: echo CA_CERT_FILE_NAME="ca.cer" >> $GITHUB_ENV + shell: bash + + - run: echo CA_KEY_FILE_NAME="ca.pem" >> $GITHUB_ENV + shell: bash + + - name: Create a certificate authority + run: openssl req -x509 -newkey rsa:2048 -keyout ${{ env.CA_KEY_FILE_NAME }} -out ${{ env.CA_CERT_FILE_NAME }} -nodes -subj '/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=mydummyca' + working-directory: .github/workflows/docker-build-context + shell: bash + env: + # Makes sure that the subject isn't interpreted as a path + MSYS_NO_PATHCONV: 1 + + - run: echo TLS_PORT="4333" >> $GITHUB_ENV + shell: bash + + - name: Build Aerospike server Docker image for testing + # We enable TLS standard authentication to verify that the OpenSSL library bundled with the wheel works + # You can manually verify this by enabling debug logging in the client and checking that the server certificate was verified uses: docker/build-push-action@v6 with: # Don't want to use default Git context or else it will clone the whole Python client repo again context: .github/workflows/docker-build-context build-args: | - server_image=${{ env.IMAGE_NAME }} + SERVER_IMAGE=${{ env.IMAGE_NAME }} + CA_KEY_FILE_NAME=${{ env.CA_KEY_FILE_NAME }} + CA_CERT_FILE_NAME=${{ env.CA_CERT_FILE_NAME }} + TLS_PORT=${{ env.TLS_PORT }} tags: ${{ env.NEW_IMAGE_NAME }} # setup-buildx-action configures Docker to use the docker-container build driver # This driver doesn't publish an image locally by default @@ -61,7 +83,7 @@ runs: - run: echo SERVER_CONTAINER_NAME="aerospike" >> $GITHUB_ENV shell: bash - - run: docker run -d --name ${{ env.SERVER_CONTAINER_NAME }} -e DEFAULT_TTL=2592000 -p 3000:3000 ${{ env.NEW_IMAGE_NAME }} + - run: docker run -d --name ${{ env.SERVER_CONTAINER_NAME }} -e DEFAULT_TTL=2592000 -p 3000:3000 -p ${{ env.TLS_PORT }}:${{ env.TLS_PORT }} ${{ env.NEW_IMAGE_NAME }} shell: bash - uses: ./.github/actions/wait-for-as-server-to-start @@ -85,7 +107,7 @@ runs: shell: bash # For debugging - - run: docker logs aerospike + - run: docker logs ${{ env.SERVER_CONTAINER_NAME }} shell: bash # Configure tests @@ -109,6 +131,8 @@ runs: run: | crudini --existing=param --set config.conf enterprise-edition user ${{ env.SUPERUSER_NAME_AND_PASSWORD }} crudini --existing=param --set config.conf enterprise-edition password ${{ env.SUPERUSER_NAME_AND_PASSWORD }} + crudini --set config.conf tls enable true + crudini --set config.conf tls cafile ../.github/workflows/docker-build-context/${{ env.CA_CERT_FILE_NAME }} working-directory: test shell: bash @@ -128,7 +152,7 @@ runs: - name: Set IP address to Docker container for the server if: ${{ inputs.where-is-client-connecting-from == 'separate-docker-container' }} - run: echo SERVER_IP=$(docker container inspect -f '{{ .NetworkSettings.IPAddress }}' aerospike) >> $GITHUB_ENV + run: echo SERVER_IP=$(docker container inspect -f '{{ .NetworkSettings.IPAddress }}' ${{ env.SERVER_CONTAINER_NAME }}) >> $GITHUB_ENV shell: bash - name: Invalid input @@ -136,7 +160,11 @@ runs: run: exit 1 shell: bash + - name: Get cluster name + run: echo CLUSTER_NAME=$(docker exec ${{ env.SERVER_CONTAINER_NAME }} asinfo $ASADM_AUTH_FLAGS -v "get-config:context=service" -l | grep -i cluster-name | cut -d = -f 2) >> $GITHUB_ENV + shell: bash + - name: Set EE server's IP address - run: crudini --existing=param --set config.conf enterprise-edition hosts ${{ env.SERVER_IP }}:3000 + run: crudini --existing=param --set config.conf enterprise-edition hosts "${{ env.SERVER_IP }}:${{ env.TLS_PORT }}|${{ env.CLUSTER_NAME }}" working-directory: test shell: bash diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 13ded5ca0..5472e5e6a 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -226,6 +226,7 @@ jobs: - name: Otherwise, enable integration tests if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }} + # Run with capture output disabled to check that TLS works (i.e we are using the bundled openssl) run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest -vv new_tests/${{ inputs.test-file }}" >> $GITHUB_ENV shell: bash @@ -253,6 +254,11 @@ jobs: CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair --add-path ./aerospike-client-c/vs/x64/Release -w {dest_dir} {wheel}" CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }} + # For debugging + - run: docker logs aerospike + if: ${{ always() && env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }} + shell: bash + - name: Upload wheels to GitHub uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} diff --git a/.github/workflows/docker-build-context/Dockerfile b/.github/workflows/docker-build-context/Dockerfile index 9ac06a152..f40106410 100644 --- a/.github/workflows/docker-build-context/Dockerfile +++ b/.github/workflows/docker-build-context/Dockerfile @@ -1,18 +1,26 @@ -ARG server_image=aerospike/aerospike-server-enterprise +ARG SERVER_IMAGE=aerospike/aerospike-server-enterprise + +# Shared between build stages to get node ID from roster file and to build final image ARG ROSTER_FILE_NAME=roster.smd -# Temp file for passing node id from one build stage to another +# Temp file for passing node id from the two build stages mentioned above # Docker doesn't support command substitution for setting values for ARG variables, so we have to do this ARG NODE_ID_FILE_NAME=node_id -FROM $server_image as configure-server - -WORKDIR /opt/aerospike/smd +# Shared between build stages to generate the server certificate and to build final image +# aerospike.conf needed to get cluster name for certificate +ARG AEROSPIKE_CONF_TEMPLATE_PATH=/etc/aerospike/aerospike.template.conf +ARG SERVER_KEY_FILE_NAME=server.pem +ARG SERVER_CERT_FILE_NAME=server.cer +# Temp file +# Cluster name fetched from stage to generate cert also needs to be set in aerospike.conf in our final image +ARG CLUSTER_NAME_FILE_NAME=cluster_name -# Enable authentication +FROM $SERVER_IMAGE AS enable-security -ARG AEROSPIKE_CONF_TEMPLATE_PATH=/etc/aerospike/aerospike.template.conf +WORKDIR /opt/aerospike/smd # Not using asconfig to edit config because we are working with a template file, which may not have valid values yet +ARG AEROSPIKE_CONF_TEMPLATE_PATH RUN echo -e "security {\n\tenable-quotas true\n}\n" >> $AEROSPIKE_CONF_TEMPLATE_PATH # security.smd was generated manually by # 1. Starting a new Aerospike EE server using Docker @@ -22,28 +30,83 @@ RUN echo -e "security {\n\tenable-quotas true\n}\n" >> $AEROSPIKE_CONF_TEMPLATE_ # TODO: generate this automatically, somehow. COPY security.smd . -# Enable strong consistency -RUN sed -i "s/\(namespace.*{\)/\1\n\tstrong-consistency true/" $AEROSPIKE_CONF_TEMPLATE_PATH -RUN sed -i "s/\(namespace.*{\)/\1\n\tstrong-consistency-allow-expunge true/" $AEROSPIKE_CONF_TEMPLATE_PATH -ARG ROSTER_FILE_NAME -COPY $ROSTER_FILE_NAME . +# Strong consistency only: fetch node id from roster.smd (JSON file) -# Fetch node id from roster.smd +# jq docker image doesn't have a shell +# We need a shell to fetch and pass the node id to the next build stage +FROM busybox AS get-node-id-for-sc # There's no tag for the latest major version to prevent breaking changes in jq # This is the next best thing -FROM ghcr.io/jqlang/jq:1.7 as get-jq -# jq docker image doesn't have a shell -# We need a shell to fetch and pass the node id to the next build stage -FROM busybox as get-node-id -COPY --from=get-jq /jq /bin/ +COPY --from=ghcr.io/jqlang/jq:1.7 /jq /bin/ ARG ROSTER_FILE_NAME COPY $ROSTER_FILE_NAME . ARG NODE_ID_FILE_NAME RUN jq --raw-output '.[1].value' $ROSTER_FILE_NAME > $NODE_ID_FILE_NAME -FROM configure-server as set-node-id +FROM enable-security AS enable-strong-consistency + +RUN sed -i "s/\(namespace.*{\)/\1\n\tstrong-consistency true/" $AEROSPIKE_CONF_TEMPLATE_PATH +RUN sed -i "s/\(namespace.*{\)/\1\n\tstrong-consistency-allow-expunge true/" $AEROSPIKE_CONF_TEMPLATE_PATH +ARG ROSTER_FILE_NAME +COPY $ROSTER_FILE_NAME . + ARG NODE_ID_FILE_NAME -COPY --from=get-node-id $NODE_ID_FILE_NAME . +COPY --from=get-node-id-for-sc $NODE_ID_FILE_NAME . RUN sed -i "s/\(^service {\)/\1\n\tnode-id $(cat $NODE_ID_FILE_NAME)/" $AEROSPIKE_CONF_TEMPLATE_PATH +# We don't want to clutter final image with temp files (junk) RUN rm $NODE_ID_FILE_NAME + +# Use a separate build stage to generate certs since we don't want openssl bundled in the final image + +FROM $SERVER_IMAGE AS generate-server-cert-for-tls + +RUN apt update +RUN apt install -y openssl + +ARG AEROSPIKE_CONF_TEMPLATE_PATH +ARG CLUSTER_NAME_FILE_NAME +RUN grep -Eo "cluster-name [a-z]+" $AEROSPIKE_CONF_TEMPLATE_PATH | awk '{print $2}' > $CLUSTER_NAME_FILE_NAME + +# Generate server private key and CSR + +ARG SERVER_CSR_FILE_NAME=server.csr +ARG SERVER_KEY_FILE_NAME +RUN openssl req -newkey rsa:4096 -keyout $SERVER_KEY_FILE_NAME -nodes -new -out $SERVER_CSR_FILE_NAME -subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=$(cat $CLUSTER_NAME_FILE_NAME)" + +# Send CSR to CA and get server certificate +# We use an external CA because we want the client to use that same CA to verify the server certificate upon connecting +ARG CA_KEY_FILE_NAME +ARG CA_CERT_FILE_NAME +COPY $CA_KEY_FILE_NAME . +COPY $CA_CERT_FILE_NAME . +ARG SERVER_CERT_FILE_NAME +RUN openssl x509 -req -in $SERVER_CSR_FILE_NAME -CA $CA_CERT_FILE_NAME -CAkey $CA_KEY_FILE_NAME -out $SERVER_CERT_FILE_NAME + +FROM enable-strong-consistency AS enable-tls + +ARG SSL_WORKING_DIR=/etc/ssl +WORKDIR $SSL_WORKING_DIR +ARG SERVER_KEY_FILE_NAME +ARG SERVER_CERT_FILE_NAME +ARG SERVER_KEY_INSTALL_PATH=$SSL_WORKING_DIR/private/$SERVER_KEY_FILE_NAME +ARG SERVER_CERT_INSTALL_PATH=$SSL_WORKING_DIR/certs/$SERVER_CERT_FILE_NAME + +COPY --from=generate-server-cert-for-tls $SERVER_KEY_FILE_NAME $SERVER_KEY_INSTALL_PATH +COPY --from=generate-server-cert-for-tls $SERVER_CERT_FILE_NAME $SERVER_CERT_INSTALL_PATH + +ARG CLUSTER_NAME_FILE_NAME +COPY --from=generate-server-cert-for-tls $CLUSTER_NAME_FILE_NAME . +# Service sub-stanza under network stanza +RUN sed -i "s|\(^\tservice {\)|\1\n\t\ttls-name $(cat $CLUSTER_NAME_FILE_NAME)|" $AEROSPIKE_CONF_TEMPLATE_PATH +RUN sed -i "s|\(^\tservice {\)|\1\n\t\ttls-authenticate-client false|" $AEROSPIKE_CONF_TEMPLATE_PATH +ARG TLS_PORT=4333 +RUN sed -i "s|\(^\tservice {\)|\1\n\t\ttls-port $TLS_PORT|" $AEROSPIKE_CONF_TEMPLATE_PATH + +RUN sed -i "s|\(^network {\)|\1\n\ttls $(cat $CLUSTER_NAME_FILE_NAME) {\n\t\tcert-file $SERVER_CERT_INSTALL_PATH\n\t}|" $AEROSPIKE_CONF_TEMPLATE_PATH +RUN sed -i "s|\(^\ttls $(cat $CLUSTER_NAME_FILE_NAME) {\)|\1\n\t\tkey-file $SERVER_KEY_INSTALL_PATH|" $AEROSPIKE_CONF_TEMPLATE_PATH + +EXPOSE $TLS_PORT + +# Cleanup +RUN rm $CLUSTER_NAME_FILE_NAME From 3618777a94bad632563d5ad07942fd815f3e66f2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 22 Nov 2024 00:49:20 +0000 Subject: [PATCH 58/69] Auto-bump version to 15.2.0rc3.dev8 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2f938c4c5..9fa4f495c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.2.0rc3.dev7 +15.2.0rc3.dev8 From bf7d9f7193a8933d44642a5d9e74c7751cd1fb7c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 22 Nov 2024 15:58:31 +0000 Subject: [PATCH 59/69] Auto-bump version to 15.2.0rc3 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9fa4f495c..686d39c91 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.2.0rc3.dev8 +15.2.0rc3 From 18c09f95ab86ca7865f10ad32f695bc91cd8507c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:29:08 +0000 Subject: [PATCH 60/69] Auto-bump version to 16.0.0rc1.dev1 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 686d39c91..4a16a4b3e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.2.0rc3 +16.0.0rc1.dev1 From ba127bf450dc31e567a37d2d8f02171e51060305 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:33:12 +0000 Subject: [PATCH 61/69] Auto-bump version to 16.0.0rc1 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4a16a4b3e..0e8c7a637 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -16.0.0rc1.dev1 +16.0.0rc1 From 63032f25e14f6c634e97a92cd7f465caaf75a7eb Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 2 Dec 2024 14:42:29 -0800 Subject: [PATCH 62/69] [CLIENT-2217] Bundle manylinux2014 wheels with OpenSSL 3.0 (#688) CI/CD: Add workflow to build and install OpenSSL 3 with the manylinux2014 images --- .github/workflows/build-wheels.yml | 20 +++++- .../manylinux2014-openssl.Dockerfile | 24 +++++++ .../update-manylinux-openssl-image.yml | 62 +++++++++++++++++++ 3 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/manylinux2014-openssl.Dockerfile create mode 100644 .github/workflows/update-manylinux-openssl-image.yml diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 5472e5e6a..b48bfe597 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -162,6 +162,7 @@ jobs: env: BUILD_IDENTIFIER: "${{ matrix.python-tag }}-${{ inputs.platform-tag }}" MACOS_OPENSSL_VERSION: 3 + CUSTOM_IMAGE_NAME: ghcr.io/aerospike/manylinux2014_{0}:latest steps: - name: Create status check message run: echo STATUS_CHECK_MESSAGE="cibuildwheel (${{ env.BUILD_IDENTIFIER }})" >> $GITHUB_ENV @@ -227,7 +228,7 @@ jobs: - name: Otherwise, enable integration tests if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }} # Run with capture output disabled to check that TLS works (i.e we are using the bundled openssl) - run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest -vv new_tests/${{ inputs.test-file }}" >> $GITHUB_ENV + run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest -vvs new_tests/${{ inputs.test-file }}" >> $GITHUB_ENV shell: bash - name: Set unoptimize flag @@ -238,6 +239,19 @@ jobs: if: ${{ inputs.include-debug-info-for-macos && startsWith(inputs.platform-tag, 'macosx') }} run: echo "INCLUDE_DSYM=1" >> $GITHUB_ENV + - if: ${{ startsWith(inputs.platform-tag, 'manylinux') }} + run: echo CIBW_MANYLINUX_X86_64_IMAGE=${{ format(env.CUSTOM_IMAGE_NAME, 'x86_64') }} >> $GITHUB_ENV + + - if: ${{ startsWith(inputs.platform-tag, 'manylinux') }} + run: echo CIBW_MANYLINUX_AARCH64_IMAGE=${{ format(env.CUSTOM_IMAGE_NAME, 'aarch64') }} >> $GITHUB_ENV + + - uses: docker/login-action@v3 + if: ${{ startsWith(inputs.platform-tag, 'manylinux') }} + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build wheel uses: pypa/cibuildwheel@v2.21.3 env: @@ -246,11 +260,11 @@ jobs: CIBW_BUILD: ${{ env.BUILD_IDENTIFIER }} CIBW_BUILD_FRONTEND: build CIBW_BEFORE_ALL_LINUX: > - yum install openssl-devel -y && - yum install python-devel -y && yum install python-setuptools -y # delvewheel is not enabled by default but we do need to repair the wheel CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel==1.*" + # We want to check that our new openssl 3 install is used, not the system default + CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair -w {dest_dir} {wheel} && auditwheel show {dest_dir}/* CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair --add-path ./aerospike-client-c/vs/x64/Release -w {dest_dir} {wheel}" CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }} diff --git a/.github/workflows/manylinux2014-openssl.Dockerfile b/.github/workflows/manylinux2014-openssl.Dockerfile new file mode 100644 index 000000000..a807685ff --- /dev/null +++ b/.github/workflows/manylinux2014-openssl.Dockerfile @@ -0,0 +1,24 @@ +ARG CPU_ARCH=x86_64 +FROM quay.io/pypa/manylinux2014_$CPU_ARCH +ARG OPENSSL_VERSION +LABEL com.aerospike.clients.openssl-version=$OPENSSL_VERSION + +RUN yum install -y perl-core wget + +ARG OPENSSL_TAR_NAME=openssl-$OPENSSL_VERSION +RUN wget https://www.openssl.org/source/$OPENSSL_TAR_NAME.tar.gz +RUN tar xzvf $OPENSSL_TAR_NAME.tar.gz +WORKDIR $OPENSSL_TAR_NAME + +# The default folder pointed to by --prefix contains a default openssl installation +# But we're assuming it's fine to replace the default openssl that comes with the image +# We aren't going to use this image in production, anyways +RUN ./Configure +RUN make +# These tests are expected to fail because we are using a buggy version of nm +# https://github.com/openssl/openssl/issues/18953 +# devtoolset-11 contains a newer version of binutils 2.36, which contains a bug fix for nm +# We don't use it though because we want to make sure the compiled openssl 3 library is compatible with manylinux2014's +# default env +RUN make V=1 TESTS='-test_symbol_presence*' test +RUN make install diff --git a/.github/workflows/update-manylinux-openssl-image.yml b/.github/workflows/update-manylinux-openssl-image.yml new file mode 100644 index 000000000..17dfd9e92 --- /dev/null +++ b/.github/workflows/update-manylinux-openssl-image.yml @@ -0,0 +1,62 @@ +on: + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 17 * * 1-5' + workflow_dispatch: + +jobs: + main: + env: + # We want granular control over the openssl version bundled with our wheels + OPENSSL_VERSION: '3.0.15' + REGISTRY: ghcr.io + strategy: + matrix: + arch-and-runner-os: [ + [x86_64, ubuntu-24.04], + [aarch64, aerospike_arm_runners_2] + ] + fail-fast: false + + runs-on: ${{ matrix.arch-and-runner-os[1] }} + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/workflows + + - run: docker pull quay.io/pypa/manylinux2014_${{ matrix.arch-and-runner-os[0] }} + + - uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/metadata-action@v5 + id: meta + with: + images: ${{ env.REGISTRY }}/aerospike/manylinux2014_${{ matrix.arch-and-runner-os[0] }} + flavor: latest=true + + - name: Set up Docker Buildx so we can cache our Docker image layers + uses: docker/setup-buildx-action@v3 + + - name: Build and push + uses: docker/build-push-action@v6 + with: + # Don't want to use default Git context or else it will clone the whole Python client repo again + context: .github/workflows + file: .github/workflows/manylinux2014-openssl.Dockerfile + build-args: | + OPENSSL_VERSION=${{ env.OPENSSL_VERSION }} + CPU_ARCH=${{ matrix.arch-and-runner-os[0] }} + # setup-buildx-action configures Docker to use the docker-container build driver + # This driver doesn't publish an image locally by default + # so we have to manually enable it + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + # Also cache intermediate layers to make development faster + cache-from: type=gha + cache-to: type=gha,mode=max From ff0e9fd04b7d39f27b76df4eec394df23e89b7ad Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 22:43:08 +0000 Subject: [PATCH 63/69] Auto-bump version to 16.0.0rc2.dev1 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0e8c7a637..e283b9179 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -16.0.0rc1 +16.0.0rc2.dev1 From 366d798e838278052f90bd44bd73ee975fa98161 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 23:52:50 +0000 Subject: [PATCH 64/69] Auto-bump version to 16.0.0rc2 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e283b9179..361af3256 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -16.0.0rc2.dev1 +16.0.0rc2 From e12a2712b770eac231060db1d9f6a47812ea2840 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 4 Dec 2024 14:03:44 -0800 Subject: [PATCH 65/69] [CLIENT-2217] CI/CD: Verify we are linking the wheel with the correct OpenSSL version (#700) --- .github/workflows/build-wheels.yml | 13 +++++++++++-- .github/workflows/manylinux2014-openssl.Dockerfile | 1 + 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index b48bfe597..37f8b7ae0 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -263,8 +263,17 @@ jobs: yum install python-setuptools -y # delvewheel is not enabled by default but we do need to repair the wheel CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel==1.*" - # We want to check that our new openssl 3 install is used, not the system default - CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair -w {dest_dir} {wheel} && auditwheel show {dest_dir}/* + # We want to check that our wheel links to the new openssl 3 install, not the system default + # This assumes that ldd prints out the "soname" for the libraries + # We can also manually verify the repair worked by checking the repaired wheel's compatibility tag + CIBW_REPAIR_WHEEL_COMMAND_LINUX: > + WHEEL_DIR=wheel-contents && + unzip {wheel} -d $WHEEL_DIR && + ldd $WHEEL_DIR/*.so | awk '{print $1}' | grep libssl.so.3 && + ldd $WHEEL_DIR/*.so | awk '{print $1}' | grep libcrypto.so.3 && + auditwheel repair -w {dest_dir} {wheel} && + auditwheel show {dest_dir}/* && + rm -rf $WHEEL_DIR CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair --add-path ./aerospike-client-c/vs/x64/Release -w {dest_dir} {wheel}" CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }} diff --git a/.github/workflows/manylinux2014-openssl.Dockerfile b/.github/workflows/manylinux2014-openssl.Dockerfile index a807685ff..00db671fd 100644 --- a/.github/workflows/manylinux2014-openssl.Dockerfile +++ b/.github/workflows/manylinux2014-openssl.Dockerfile @@ -5,6 +5,7 @@ LABEL com.aerospike.clients.openssl-version=$OPENSSL_VERSION RUN yum install -y perl-core wget +WORKDIR / ARG OPENSSL_TAR_NAME=openssl-$OPENSSL_VERSION RUN wget https://www.openssl.org/source/$OPENSSL_TAR_NAME.tar.gz RUN tar xzvf $OPENSSL_TAR_NAME.tar.gz From 0ab340b0f3981707abcd0dab1c6585b99543d89b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 4 Dec 2024 22:04:29 +0000 Subject: [PATCH 66/69] Auto-bump version to 16.0.0rc3.dev1 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 361af3256..0a903f20d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -16.0.0rc2 +16.0.0rc3.dev1 From 11d64dbb7d8cbf466c5a65dffd816d79e591bab9 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 9 Dec 2024 09:08:58 -0800 Subject: [PATCH 67/69] [CLIENT-3205] Multi record transactions: support correct server error codes for MRTs (#701) - [CLIENT-3199] Do not close/delete MRT monitor record on abort/commit when a write command in that MRT fails and is in doubt - [CLIENT-3212] Raise TranactionFailed exception instead of BatchFailed exception when batch MRT verify fails. - Always unlock macOS keychain on self-hosted macOS runners to prevent workflow from failing --- .github/workflows/build-wheels.yml | 3 ++- aerospike-client-c | 2 +- src/main/exception.c | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 37f8b7ae0..334f82dc8 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -325,7 +325,8 @@ jobs: ref: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} # Need to be able to save Docker Hub credentials to keychain - - if: ${{ inputs.platform-tag == 'macosx_arm64' && inputs.use-server-rc }} + # Unable to do this via the ansible script for self hosted mac m1 runners + - if: ${{ inputs.platform-tag == 'macosx_arm64' }} run: security unlock-keychain -p ${{ secrets.MAC_M1_SELF_HOSTED_RUNNER_PW }} - uses: ./.github/actions/run-ee-server diff --git a/aerospike-client-c b/aerospike-client-c index 886306eee..5bffa81cc 160000 --- a/aerospike-client-c +++ b/aerospike-client-c @@ -1 +1 @@ -Subproject commit 886306eeea9869190cae9c0a7782eee83e1c345d +Subproject commit 5bffa81ccf98f36951cf3d638bacea463dd27580 diff --git a/src/main/exception.c b/src/main/exception.c index c0884ec81..4c681131e 100644 --- a/src/main/exception.c +++ b/src/main/exception.c @@ -111,6 +111,8 @@ struct exception_def exception_defs[] = { AEROSPIKE_ERR_ASYNC_CONNECTION, NULL), EXCEPTION_DEF("ClientAbortError", CLIENT_ERR_EXCEPTION_NAME, AEROSPIKE_ERR_CLIENT_ABORT, NULL), + EXCEPTION_DEF("TranactionFailed", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_TXN_FAILED, NULL), // Server errors EXCEPTION_DEF("InvalidRequest", SERVER_ERR_EXCEPTION_NAME, AEROSPIKE_ERR_REQUEST_INVALID, NULL), From d5bc2d691b39cf44d2fcdddb5171ed48b16ee25d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:09:40 +0000 Subject: [PATCH 68/69] Auto-bump version to 16.0.0rc3.dev2 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0a903f20d..e60378d5e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -16.0.0rc3.dev1 +16.0.0rc3.dev2 From 323f3a14afd77a2073a6c12686b58369cfc87b8e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:07:53 +0000 Subject: [PATCH 69/69] Auto-bump version to 16.0.0rc3 [skip ci] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e60378d5e..3dfcbc0dd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -16.0.0rc3.dev2 +16.0.0rc3