diff --git a/.build.yml b/.build.yml index 7be220ecf..f3ad3e625 100644 --- a/.build.yml +++ b/.build.yml @@ -8,7 +8,7 @@ container: build: - name: build environment: - PYTHONS: /opt/python/cp38-cp38/bin,/opt/python/cp39-cp39/bin,/opt/python/cp310-cp310/bin,/opt/python/cp311-cp311/bin,/opt/python/cp312-cp312/bin + PYTHONS: /opt/python/cp38-cp38/bin,/opt/python/cp39-cp39/bin,/opt/python/cp310-cp310/bin,/opt/python/cp311-cp311/bin,/opt/python/cp312-cp312/bin,/opt/python/cp313-cp313/bin script: - scripts/manylinux2014build.sh artifact: diff --git a/.github/actions/run-ee-server-for-ext-container/action.yml b/.github/actions/run-ee-server-for-ext-container/action.yml deleted file mode 100644 index 42e8b53a9..000000000 --- a/.github/actions/run-ee-server-for-ext-container/action.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: 'Run EE server for another Docker container' -description: 'Run EE server and configure tests to connect to it from another Docker container' -inputs: - # All inputs in composite actions are strings - use-server-rc: - required: true - default: false - server-tag: - required: true - default: 'latest' - # Github Composite Actions can't access secrets - # so we need to pass them in as inputs - docker-hub-username: - required: false - docker-hub-password: - required: false - -runs: - using: "composite" - steps: - - name: Run EE server - uses: ./.github/actions/run-ee-server - with: - use-server-rc: ${{ inputs.use-server-rc }} - server-tag: ${{ inputs.server-tag }} - docker-hub-username: ${{ inputs.docker-hub-username }} - docker-hub-password: ${{ inputs.docker-hub-password }} - - - name: Get IP address of Docker container hosting server - id: get-server-ip-address - run: echo server-ip=$(docker container inspect -f '{{ .NetworkSettings.IPAddress }}' aerospike) >> $GITHUB_OUTPUT - shell: bash - - - name: Configure tests to connect to that Docker container - run: crudini --existing=param --set config.conf enterprise-edition hosts ${{ steps.get-server-ip-address.outputs.server-ip }}:3000 - working-directory: test - shell: bash diff --git a/.github/actions/run-ee-server/action.yml b/.github/actions/run-ee-server/action.yml index 5f73224e2..898ce018c 100644 --- a/.github/actions/run-ee-server/action.yml +++ b/.github/actions/run-ee-server/action.yml @@ -1,4 +1,4 @@ -name: 'Run EE Server' +name: 'Run EE Server in a Docker container' description: 'Run EE server. Returns once server is ready. Only tested on Linux and macOS' # NOTE: do not share this server container with others # since it's using the default admin / admin credentials @@ -6,44 +6,29 @@ inputs: # All inputs in composite actions are strings use-server-rc: required: true - default: false + description: Deploy server release candidate? + default: 'false' server-tag: required: true + description: Specify Docker tag default: 'latest' # Github Composite Actions can't access secrets # so we need to pass them in as inputs docker-hub-username: + description: Required for using release candidates required: false docker-hub-password: + description: Required for using release candidates required: false + where-is-client-connecting-from: + required: false + description: 'docker-host, separate-docker-container, "remote-connection" via DOCKER_HOST' + default: 'docker-host' runs: using: "composite" steps: - - name: Install crudini to manipulate config.conf - # This will only work on the Github hosted runners. - # TODO: mac m1 self hosted runners do not have pipx installed by default - run: pipx install crudini --pip-args "-c ${{ github.workspace }}/.github/workflows/requirements.txt" - working-directory: .github/workflows - shell: bash - - - name: Create config.conf - run: cp config.conf.template config.conf - working-directory: test - shell: bash - - - name: Use enterprise edition instead of community edition in config.conf - run: | - crudini --existing=param --set config.conf enterprise-edition hosts '' - crudini --existing=param --set config.conf enterprise-edition hosts 127.0.0.1:3000 - crudini --existing=param --set config.conf enterprise-edition user superuser - crudini --existing=param --set config.conf enterprise-edition password superuser - working-directory: test - shell: bash - - - name: Create config folder to store configs in - run: mkdir configs - shell: bash + # Start up server - name: Log into Docker Hub to get server RC if: ${{ inputs.use-server-rc == 'true' }} @@ -53,30 +38,133 @@ runs: - run: echo IMAGE_NAME=aerospike/aerospike-server-enterprise${{ inputs.use-server-rc == 'true' && '-rc' || '' }}:${{ inputs.server-tag }} >> $GITHUB_ENV shell: bash - - run: echo SECURITY_IMAGE_NAME=${{ env.IMAGE_NAME }}-security >> $GITHUB_ENV + - run: echo NEW_IMAGE_NAME=${{ env.IMAGE_NAME }}-python-client-testing >> $GITHUB_ENV shell: bash # macOS Github runners and Windows self-hosted runners don't have buildx installed by default - if: ${{ runner.os == 'Windows' || runner.os == 'macOS' }} uses: docker/setup-buildx-action@v3 - - name: Build and push + - run: echo CA_CERT_FILE_NAME="ca.cer" >> $GITHUB_ENV + shell: bash + + - run: echo CA_KEY_FILE_NAME="ca.pem" >> $GITHUB_ENV + shell: bash + + - name: Create a certificate authority + run: openssl req -x509 -newkey rsa:2048 -keyout ${{ env.CA_KEY_FILE_NAME }} -out ${{ env.CA_CERT_FILE_NAME }} -nodes -subj '/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=mydummyca' + working-directory: .github/workflows/docker-build-context + shell: bash + env: + # Makes sure that the subject isn't interpreted as a path + MSYS_NO_PATHCONV: 1 + + - run: echo TLS_PORT="4333" >> $GITHUB_ENV + shell: bash + + - name: Build Aerospike server Docker image for testing + # We enable TLS standard authentication to verify that the OpenSSL library bundled with the wheel works + # You can manually verify this by enabling debug logging in the client and checking that the server certificate was verified uses: docker/build-push-action@v6 with: # Don't want to use default Git context or else it will clone the whole Python client repo again - context: .github/workflows + context: .github/workflows/docker-build-context build-args: | - image=${{ env.IMAGE_NAME }} - tags: ${{ env.SECURITY_IMAGE_NAME }} + SERVER_IMAGE=${{ env.IMAGE_NAME }} + CA_KEY_FILE_NAME=${{ env.CA_KEY_FILE_NAME }} + CA_CERT_FILE_NAME=${{ env.CA_CERT_FILE_NAME }} + TLS_PORT=${{ env.TLS_PORT }} + tags: ${{ env.NEW_IMAGE_NAME }} # setup-buildx-action configures Docker to use the docker-container build driver # This driver doesn't publish an image locally by default # so we have to manually enable it load: true - - run: docker run -d --name aerospike -p 3000:3000 ${{ env.SECURITY_IMAGE_NAME }} + - run: echo SERVER_CONTAINER_NAME="aerospike" >> $GITHUB_ENV + shell: bash + + - run: docker run -d --name ${{ env.SERVER_CONTAINER_NAME }} -e DEFAULT_TTL=2592000 -p 3000:3000 -p ${{ env.TLS_PORT }}:${{ env.TLS_PORT }} ${{ env.NEW_IMAGE_NAME }} shell: bash - uses: ./.github/actions/wait-for-as-server-to-start with: - container-name: aerospike + container-name: ${{ env.SERVER_CONTAINER_NAME }} is-security-enabled: true + is-strong-consistency-enabled: true + + - run: echo SUPERUSER_NAME_AND_PASSWORD="superuser" >> $GITHUB_ENV + shell: bash + + - run: echo ASADM_AUTH_FLAGS="--user=${{ env.SUPERUSER_NAME_AND_PASSWORD }} --password=${{ env.SUPERUSER_NAME_AND_PASSWORD }}" >> $GITHUB_ENV + shell: bash + + # All the partitions are assumed to be dead when reusing a roster file + - run: docker exec ${{ env.SERVER_CONTAINER_NAME }} asadm $ASADM_AUTH_FLAGS --enable --execute "manage revive ns test" + shell: bash + + # Apply changes + - run: docker exec ${{ env.SERVER_CONTAINER_NAME }} asadm $ASADM_AUTH_FLAGS --enable --execute "manage recluster" + shell: bash + + # For debugging + - run: docker logs ${{ env.SERVER_CONTAINER_NAME }} + shell: bash + + # Configure tests + + - name: Install crudini to manipulate config.conf + run: pipx install crudini --pip-args "-c ${{ github.workspace }}/.github/workflows/requirements.txt" + working-directory: .github/workflows + shell: bash + + - name: Create config.conf + run: cp config.conf.template config.conf + working-directory: test + shell: bash + + - name: Disable community edition connection + run: crudini --existing=param --set config.conf community-edition hosts '' + working-directory: test + shell: bash + + - name: Set credentials in config file + run: | + crudini --existing=param --set config.conf enterprise-edition user ${{ env.SUPERUSER_NAME_AND_PASSWORD }} + crudini --existing=param --set config.conf enterprise-edition password ${{ env.SUPERUSER_NAME_AND_PASSWORD }} + crudini --set config.conf tls enable true + crudini --set config.conf tls cafile ../.github/workflows/docker-build-context/${{ env.CA_CERT_FILE_NAME }} + working-directory: test + shell: bash + + - name: Set IP address to localhost + if: ${{ inputs.where-is-client-connecting-from == 'docker-host' }} + run: echo SERVER_IP=127.0.0.1 >> $GITHUB_ENV + working-directory: test + shell: bash + + - name: Set IP address to remote machine running the Docker daemon + if: ${{ inputs.where-is-client-connecting-from == 'remote-connection' }} + run: | + SERVER_IP=${DOCKER_HOST/tcp:\/\//} + echo SERVER_IP=${SERVER_IP/:2375/} >> $GITHUB_ENV + working-directory: test + shell: bash + + - name: Set IP address to Docker container for the server + if: ${{ inputs.where-is-client-connecting-from == 'separate-docker-container' }} + run: echo SERVER_IP=$(docker container inspect -f '{{ .NetworkSettings.IPAddress }}' ${{ env.SERVER_CONTAINER_NAME }}) >> $GITHUB_ENV + shell: bash + + - name: Invalid input + if: ${{ env.SERVER_IP == '' }} + run: exit 1 + shell: bash + + - name: Get cluster name + run: echo CLUSTER_NAME=$(docker exec ${{ env.SERVER_CONTAINER_NAME }} asinfo $ASADM_AUTH_FLAGS -v "get-config:context=service" -l | grep -i cluster-name | cut -d = -f 2) >> $GITHUB_ENV + shell: bash + + - name: Set EE server's IP address + run: crudini --existing=param --set config.conf enterprise-edition hosts "${{ env.SERVER_IP }}:${{ env.TLS_PORT }}|${{ env.CLUSTER_NAME }}" + working-directory: test + shell: bash diff --git a/.github/actions/setup-docker-on-macos/action.yml b/.github/actions/setup-docker-on-macos/action.yml new file mode 100644 index 000000000..aa8e1da0b --- /dev/null +++ b/.github/actions/setup-docker-on-macos/action.yml @@ -0,0 +1,17 @@ +name: 'Install Docker on macOS runner' +description: 'Install Docker using colima' + +runs: + using: "composite" + steps: + - name: Install Docker Engine + run: brew install colima + shell: bash + + - name: Install Docker client + run: brew install docker + shell: bash + + - name: Start Docker Engine + run: colima start + shell: bash diff --git a/.github/actions/wait-for-as-server-to-start/action.yml b/.github/actions/wait-for-as-server-to-start/action.yml index 26841102b..373c26970 100644 --- a/.github/actions/wait-for-as-server-to-start/action.yml +++ b/.github/actions/wait-for-as-server-to-start/action.yml @@ -6,6 +6,9 @@ inputs: is-security-enabled: required: false default: 'false' + is-strong-consistency-enabled: + required: false + default: 'false' runs: using: "composite" @@ -21,5 +24,5 @@ runs: # Also, we don't want to fail if we timeout in case the server *did* finish starting up but the script couldn't detect it due to a bug # Effectively, this composite action is like calling "sleep" that is optimized to exit early when it detects an ok from the server - name: Wait for EE server to start - run: timeout 30 bash ./.github/workflows/wait-for-as-server-to-start.bash ${{ inputs.container-name }} ${{ inputs.is-security-enabled }} || true + run: timeout 30 bash ./.github/workflows/wait-for-as-server-to-start.bash ${{ inputs.container-name }} ${{ inputs.is-security-enabled }} ${{ inputs.is-strong-consistency-enabled }} || true shell: bash diff --git a/.github/workflows/Dockerfile b/.github/workflows/Dockerfile deleted file mode 100644 index e15848240..000000000 --- a/.github/workflows/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG image -FROM $image -RUN echo -e "security {\n\tenable-quotas true\n}\n" >> /etc/aerospike/aerospike.template.conf -# security.smd was generated manually by -# 1. Starting a new Aerospike EE server using Docker -# 2. Creating the superuser user -# 3. Copying /opt/aerospike/smd/security.smd from the container and committing it to this repo -# This file should always work -# TODO: generate this automatically, somehow -COPY security.smd /opt/aerospike/smd/ diff --git a/.github/workflows/build-and-run-stage-tests.yml b/.github/workflows/build-and-run-stage-tests.yml index 7b9a2bd6e..8f1ce4467 100644 --- a/.github/workflows/build-and-run-stage-tests.yml +++ b/.github/workflows/build-and-run-stage-tests.yml @@ -21,12 +21,32 @@ on: description: 'Test macOS x86 wheels (unstable)' jobs: - build-wheels: + build-select-wheels: + strategy: + matrix: + platform-tag: [ + "manylinux_x86_64", + "manylinux_aarch64", + "macosx_x86_64" + ] + # Need all the artifacts to run all the stage tests, so fail fast uses: ./.github/workflows/build-wheels.yml + with: + platform-tag: ${{ matrix.platform-tag }} + sha-to-build-and-test: ${{ github.sha }} + secrets: inherit + + build-sdist: + uses: ./.github/workflows/build-sdist.yml + with: + sha_to_build: ${{ github.sha }} run-stage-tests: uses: ./.github/workflows/stage-tests.yml - needs: build-wheels + needs: [ + build-select-wheels, + build-sdist + ] secrets: inherit with: use_jfrog_builds: false diff --git a/.github/workflows/build-and-upload-wheels-for-qe.yml b/.github/workflows/build-and-upload-wheels-for-qe.yml index 63fd09843..86de85074 100644 --- a/.github/workflows/build-and-upload-wheels-for-qe.yml +++ b/.github/workflows/build-and-upload-wheels-for-qe.yml @@ -17,11 +17,18 @@ on: jobs: build-artifacts: + strategy: + matrix: + platform-tag: [ + "manylinux_x86_64", + "manylinux_aarch64" + ] uses: ./.github/workflows/build-wheels.yml with: - # In a push event, any input values default to '' - # https://github.com/orgs/community/discussions/29242#discussioncomment-5063461 - apply-no-optimizations: ${{ github.event_name == 'workflow_dispatch' && inputs.disable-optimizations || false }} + platform-tag: ${{ matrix.platform-tag }} + unoptimized: ${{ github.event_name == 'workflow_dispatch' && inputs.disable-optimizations || false }} + sha-to-build-and-test: ${{ github.sha }} + secrets: inherit upload-to-jfrog: needs: build-artifacts diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 000000000..88fc4f159 --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,110 @@ +name: Build artifacts +run-name: Build artifacts (run_tests=${{ inputs.run_tests }}, use-server-rc=${{ inputs.use-server-rc }}, server-tag=${{ inputs.server-tag }}) + +# Builds manylinux wheels and source distribution +# Optionally run tests on manylinux wheels +# Then upload artifacts to Github + +on: + workflow_dispatch: + inputs: + # There may be a case where we want to test these build debug flags on all platforms that support them + unoptimized: + description: 'macOS or Linux: Apply -O0 flag?' + type: boolean + required: false + default: false + include-debug-info-for-macos: + description: 'macOS: Build wheels for debugging?' + type: boolean + required: false + default: false + run_tests: + description: "Run integration tests with the wheels after building them" + required: true + type: boolean + default: false + use-server-rc: + type: boolean + required: true + default: false + description: 'Test against server release candidate? (e.g to test new server features)' + server-tag: + type: string + required: true + default: 'latest' + description: 'Server docker image tag (e.g to test a client backport version)' + + workflow_call: + inputs: + # The "dev" tests test the artifacts against a server + # The dev-to-stage and stage-to-master workflow only need to build the artifacts, not test them + run_tests: + required: false + type: boolean + default: false + # workflow_call hack + is_workflow_call: + type: boolean + default: true + required: false + # This input is only used in workflow_call events + sha-to-build-and-test: + description: A calling workflow may want to run this workflow on a different ref than the calling workflow's ref + type: string + # Make it required to make things simple + required: true + # A calling workflow doesn't actually set values to the inputs below + # But that workflow needs to have default values for these inputs + unoptimized: + type: boolean + required: false + default: false + include-debug-info-for-macos: + type: boolean + required: false + default: false + use-server-rc: + required: false + default: false + type: boolean + server-tag: + type: string + required: false + default: 'latest' + secrets: + DOCKER_HUB_BOT_USERNAME: + required: true + DOCKER_HUB_BOT_PW: + required: true + MAC_M1_SELF_HOSTED_RUNNER_PW: + required: true + +jobs: + build-sdist: + uses: ./.github/workflows/build-sdist.yml + with: + sha_to_build: ${{ inputs.is_workflow_call == true && inputs.sha-to-build-and-test || github.sha }} + + build-wheels: + strategy: + matrix: + platform-tag: [ + "manylinux_x86_64", + "manylinux_aarch64", + "macosx_x86_64", + "macosx_arm64", + "win_amd64" + ] + fail-fast: false + uses: ./.github/workflows/build-wheels.yml + with: + platform-tag: ${{ matrix.platform-tag }} + # Can't use env context here, so just copy from build-sdist env var + sha-to-build-and-test: ${{ inputs.is_workflow_call == true && inputs.sha-to-build-and-test || github.sha }} + unoptimized: ${{ inputs.unoptimized }} + include-debug-info-for-macos: ${{ inputs.include-debug-info-for-macos }} + run_tests: ${{ inputs.run_tests }} + use-server-rc: ${{ inputs.use-server-rc }} + server-tag: ${{ inputs.server-tag }} + secrets: inherit diff --git a/.github/workflows/build-sdist.yml b/.github/workflows/build-sdist.yml new file mode 100644 index 000000000..fb6926701 --- /dev/null +++ b/.github/workflows/build-sdist.yml @@ -0,0 +1,60 @@ +on: + workflow_dispatch: + workflow_call: + inputs: + is_workflow_call: + type: boolean + default: true + required: false + sha_to_build: + type: string + required: true + +env: + STATUS_CHECK_MESSAGE: "Build source distribution" + COMMIT_SHA_TO_BUILD: ${{ inputs.is_workflow_call == true && inputs.sha_to_build || github.sha }} + +jobs: + build-sdist: + name: Build source distribution + runs-on: ubuntu-22.04 + steps: + - name: Show job status for commit + # Commit status will already be shown by the calling workflow for push and pull request events, but not + # for any other event like workflow_dispatch. so we have to do it manually + # If workflow_call triggered this job, github.event_name will inherit the event of the calling workflow + # The calling workflow can be triggered by push or pull request events, so there's that + # https://github.com/actions/runner/issues/3146#issuecomment-2000017097 + if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} + uses: myrotvorets/set-commit-status-action@v2.0.0 + with: + sha: ${{ env.COMMIT_SHA_TO_BUILD }} + context: ${{ env.STATUS_CHECK_MESSAGE }} + + - uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{ env.COMMIT_SHA_TO_BUILD }} + fetch-depth: 0 + + - name: Install build dependencies (pip packages) + run: python3 -m pip install -r requirements.txt + + - name: Build source distribution + run: python3 -m build --sdist + + - name: Upload source distribution to GitHub + uses: actions/upload-artifact@v4 + with: + path: ./dist/*.tar.gz + name: sdist.build + + - name: Set final commit status + uses: myrotvorets/set-commit-status-action@v2.0.0 + # Always run even if job failed or is cancelled + # But we don't want to show anything if the calling workflow was triggered by these events + if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} + with: + sha: ${{ env.COMMIT_SHA_TO_BUILD }} + status: ${{ job.status }} + context: ${{ env.STATUS_CHECK_MESSAGE }} diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 7697546d6..334f82dc8 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -1,524 +1,376 @@ -name: Build wheels -run-name: Build wheels (run_tests=${{ inputs.run_tests }}, use-server-rc=${{ inputs.use-server-rc }}, server-tag=${{ inputs.server-tag }}, test-macos-x86=${{ inputs.test-macos-x86 }}, apply-no-optimizations=${{ inputs.apply-no-optimizations }}, include-debug-info-for-macos=${{ inputs.include-debug-info-for-macos }}) +name: 'Build wheels' +run-name: 'Build wheels (python-tags=${{ inputs.python-tags }}, platform-tag=${{ inputs.platform-tag }}, unoptimized=${{ inputs.unoptimized }}, include-debug-info-for-macos=${{ inputs.include-debug-info-for-macos }}, run_tests=${{ inputs.run_tests }}, use-server-rc=${{ inputs.use-server-rc }}, server-tag=${{ inputs.server-tag }})' -# Builds manylinux wheels and source distribution -# If running tests, publish results in commit status -# For each build, upload to Github as an artifact if it passes testing or does not need to run tests +# Build wheels on all (or select) Python versions supported by the Python client for a specific platform on: workflow_dispatch: inputs: - # If we only want to check that the builds pass on an arbitrary branch - run_tests: - description: "Run integration tests" + # These are the usual cases for building wheels: + # + # 1. One wheel for *one* supported Python version. This is for running specialized tests that only need one Python version + # like valgrind or a failing QE test. And usually, we only need one wheel for debugging purposes. + # 2. Wheels for *all* supported Python versions for *one* supported platform. This is useful for testing workflow changes for a + # single OS or CPU architecture (e.g testing that changes to debugging modes work on all Python versions) + # 3. Wheels for *all* supported Python versions and *all* supported platforms. This is for building wheels for different + # CI/CD stages (e.g dev, stage, or master). We can also test debugging modes for all platforms that support them + # + # We're able to combine case 1 and 2 into one workflow by creating an input that takes in a JSON list of strings (Python tags) + # to build wheels for. Actual list inputs aren't supported yet, so it's actually a JSON list encoded as a string. + # + # However, it's harder to combine this workflow (case 1 + 2) with case 3, because matrix outputs don't exist yet + # in Github Actions. So all jobs in the cibuildwheel job would have to pass for a self hosted job to run. + # We want each platform to be tested independently of each other, + # so there is a wrapper workflow that has a list of platforms to test and reuses this workflow for each platform. + # If one platform fails, it will not affect the building and testing of another platform (we disable fail fast mode) + python-tags: + type: string + description: Valid JSON list of Python tags to build the client for + required: false + default: '["cp38", "cp39", "cp310", "cp311", "cp312", "cp313"]' + platform-tag: + description: Platform to build the client for. + type: choice required: true + options: + - manylinux_x86_64 + - manylinux_aarch64 + - macosx_x86_64 + - macosx_arm64 + - win_amd64 + # Makes debugging via gh cli easier. + default: manylinux_x86_64 + unoptimized: + description: 'macOS or Linux: Apply -O0 flag?' + # Windows supports a different flag to disable optimizations, but we haven't added support for it yet + type: boolean + required: false + default: false + include-debug-info-for-macos: + description: 'macOS: Build wheels for debugging?' type: boolean + required: false + default: false + run_tests: + description: 'Run Aerospike server and run tests using built wheels?' + type: boolean + required: false default: false use-server-rc: type: boolean required: true default: false description: 'Test against server release candidate?' - # If we are creating a backport and want to test an arbitrary branch against an older server version server-tag: required: true default: 'latest' description: 'Server docker image tag' - test-macos-x86: - required: true + test-file: + required: false + default: '' + description: 'new_tests/' + + workflow_call: + inputs: + # See workflow call hack in update-version.yml + is_workflow_call: type: boolean - default: false - description: 'Test macOS x86 wheels (unstable)' - apply-no-optimizations: + default: true + required: false + python-tags: + type: string + required: false + default: '["cp38", "cp39", "cp310", "cp311", "cp312", "cp313"]' + platform-tag: + type: string + required: true + # Only used in workflow_call event + sha-to-build-and-test: + type: string required: true + unoptimized: type: boolean + required: false default: false - description: 'Linux and macOS: apply -O0 when building C and Python client?' include-debug-info-for-macos: - required: true type: boolean + required: false default: false - description: 'macOS: include source files and line numbers for both C and Python client for debugging?' - - workflow_call: - inputs: - # The "dev" tests test the artifacts against a server release - # The "stage" tests and release workflow only need to build the artifacts, not test them run_tests: - description: "Run integration tests" - required: false type: boolean - default: false - ref: - type: string required: false - # Calling workflow doesn't actually use the options below - # But we need to set default values for workflow calls to use + default: false use-server-rc: required: false - default: true type: boolean + default: false + description: 'Test against server release candidate?' server-tag: - type: string required: false + type: string default: 'latest' - test-macos-x86: - required: false - type: boolean - default: false - apply-no-optimizations: - required: false - type: boolean - default: false - include-debug-info-for-macos: + description: 'Server docker image tag' + test-file: required: false - type: boolean - default: false + type: string + default: '' secrets: + # Just make all the secrets required to make things simpler... DOCKER_HUB_BOT_USERNAME: - required: false + required: true DOCKER_HUB_BOT_PW: - required: false + required: true MAC_M1_SELF_HOSTED_RUNNER_PW: - required: false + required: true + +env: + COMMIT_SHA_TO_BUILD_AND_TEST: ${{ inputs.is_workflow_call == true && inputs.sha-to-build-and-test || github.sha }} + # Note that environment variables in Github are all strings + # Github mac m1 and windows runners don't support Docker / nested virtualization + # so we need to use self-hosted runners to test wheels for these platforms + RUN_INTEGRATION_TESTS_IN_CIBW: ${{ inputs.run_tests && (startsWith(inputs.platform-tag, 'manylinux') || inputs.platform-tag == 'macosx_x86_64') }} jobs: - build-sdist: - name: Build source distribution + # Maps don't exist in Github Actions, so we have to store the map using a script and fetch it in a job + # This uses up more billing minutes (rounded up to 1 minute for each job run), + # but this should be ok based on the minutes usage data for the aerospike organization + get-runner-os: + outputs: + runner-os: ${{ steps.get-runner-os.outputs.runner_os }} runs-on: ubuntu-22.04 steps: - - name: Show job status for commit - # If workflow_call triggered this job, github.event_name will inherit the event of the calling workflow - # https://github.com/actions/runner/issues/3146#issuecomment-2000017097 - # Commit status will already be shown by the calling workflow for these events - if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} - uses: myrotvorets/set-commit-status-action@v2.0.0 - with: - sha: ${{ github.sha }} - context: "Build wheels (sdist)" - - - uses: actions/checkout@v4 - with: - submodules: recursive - ref: ${{ inputs.ref }} - fetch-depth: 0 - - - name: Install build dependencies (pip packages) - run: python3 -m pip install -r requirements.txt - - - name: Build source distribution - run: python3 -m build --sdist - - - name: Upload source distribution to GitHub - uses: actions/upload-artifact@v4 - with: - path: ./dist/*.tar.gz - name: sdist.build - - - name: Set final commit status - uses: myrotvorets/set-commit-status-action@v2.0.0 - # Always run even if job failed or is cancelled - # But we don't want to show anything if the calling workflow was triggered by these events - if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} - with: - sha: ${{ github.sha }} - status: ${{ job.status }} - context: "Build wheels (sdist)" - - manylinux: + - id: get-runner-os + # Single source of truth for which runner OS to use for each platform tag + run: | + declare -A hashmap + hashmap[manylinux_x86_64]="ubuntu-22.04" + hashmap[manylinux_aarch64]="aerospike_arm_runners_2" + hashmap[macosx_x86_64]="macos-12-large" + hashmap[macosx_arm64]="macos-14" + hashmap[win_amd64]="windows-2022" + echo runner_os=${hashmap[${{ inputs.platform-tag }}]} >> $GITHUB_OUTPUT + # Bash >= 4 supports hashmaps + shell: bash + + cibuildwheel: + needs: get-runner-os strategy: - fail-fast: false matrix: - # Python versions to build wheels on - python: [ - "cp38", - "cp39", - "cp310", - "cp311", - "cp312" - ] - platform: [ - ["x86_64", "ubuntu-22.04"], - ["aarch64", "aerospike_arm_runners_2"] - ] - runs-on: ${{ matrix.platform[1] }} - steps: - - name: Show job status for commit - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} - with: - sha: ${{ github.sha }} - context: "Build wheels (${{ matrix.python }}-manylinux_${{ matrix.platform[0] }})" - - - uses: actions/checkout@v4 - with: - submodules: recursive - ref: ${{ inputs.ref }} - fetch-depth: 0 - - - uses: ./.github/actions/run-ee-server-for-ext-container - if: ${{ inputs.run_tests }} - with: - use-server-rc: ${{ inputs.use-server-rc }} - server-tag: ${{ inputs.server-tag }} - docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} - docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} - - - name: Enable tests - if: ${{ inputs.run_tests }} - run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest new_tests/" >> $GITHUB_ENV - - - name: Disable tests (only run basic import test) - if: ${{ !inputs.run_tests }} - run: echo "TEST_COMMAND=python -c 'import aerospike'" >> $GITHUB_ENV - - - name: Set unoptimize flag - if: ${{ inputs.apply-no-optimizations }} - run: echo "UNOPTIMIZED=1" >> $GITHUB_ENV - - - name: Build wheel - uses: pypa/cibuildwheel@v2.19.2 - env: - CIBW_ENVIRONMENT_PASS_LINUX: ${{ inputs.apply-no-optimizations && 'UNOPTIMIZED' || '' }} - CIBW_BUILD: ${{ matrix.python }}-manylinux_${{ matrix.platform[0] }} - CIBW_BUILD_FRONTEND: build - CIBW_BEFORE_ALL_LINUX: > - yum install openssl-devel -y && - yum install python-devel -y && - yum install python-setuptools -y - CIBW_ARCHS: "${{ matrix.platform[0] }}" - CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }} - - - name: Upload wheels to GitHub - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - path: ./wheelhouse/*.whl - name: ${{ matrix.python }}-manylinux_${{ matrix.platform[0] }}.build - - - name: Set final commit status - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} - with: - sha: ${{ github.sha }} - status: ${{ job.status }} - context: "Build wheels (${{ matrix.python }}-manylinux_${{ matrix.platform[0] }})" - - macOS-x86: - strategy: + python-tag: ${{ fromJSON(inputs.python-tags) }} fail-fast: false - matrix: - python: [ - "cp38", - "cp39", - "cp310", - "cp311", - "cp312" - ] - runs-on: macos-12-large + runs-on: ${{ needs.get-runner-os.outputs.runner-os }} + env: + BUILD_IDENTIFIER: "${{ matrix.python-tag }}-${{ inputs.platform-tag }}" + MACOS_OPENSSL_VERSION: 3 + CUSTOM_IMAGE_NAME: ghcr.io/aerospike/manylinux2014_{0}:latest steps: - - name: Show job status for commit - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} - with: - sha: ${{ github.sha }} - context: "Build wheels (${{ matrix.python }}-macosx_x86_64)" - - - uses: actions/checkout@v4 - with: - submodules: recursive - ref: ${{ inputs.ref }} - fetch-depth: 0 + - name: Create status check message + run: echo STATUS_CHECK_MESSAGE="cibuildwheel (${{ env.BUILD_IDENTIFIER }})" >> $GITHUB_ENV + shell: bash - - name: Install Docker Engine - if: ${{ inputs.run_tests }} - run: brew install colima - - - name: Install Docker client - if: ${{ inputs.run_tests }} - run: brew install docker - - - name: Start Docker Engine - if: ${{ inputs.run_tests }} - run: colima start - - - uses: ./.github/actions/run-ee-server - if: ${{ inputs.run_tests }} - with: - use-server-rc: ${{ inputs.use-server-rc }} - server-tag: ${{ inputs.server-tag }} - docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} - docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} - - - name: Enable tests - if: ${{ inputs.run_tests && inputs.test-macos-x86 }} - run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest new_tests/" >> $GITHUB_ENV - - - name: Disable tests (only run basic import test) - if: ${{ !inputs.run_tests || !inputs.test-macos-x86 }} - run: echo "TEST_COMMAND=python -c 'import aerospike'" >> $GITHUB_ENV - - - name: Set unoptimize flag - if: ${{ inputs.apply-no-optimizations }} - run: echo "UNOPTIMIZED=1" >> $GITHUB_ENV - - - name: Set include dsym flag - if: ${{ inputs.include-debug-info-for-macos }} - run: echo "INCLUDE_DSYM=1" >> $GITHUB_ENV - - - name: Build wheel - uses: pypa/cibuildwheel@v2.19.2 - env: - CIBW_BUILD: ${{ matrix.python }}-macosx_x86_64 - CIBW_BUILD_FRONTEND: build - CIBW_ENVIRONMENT: SSL_LIB_PATH="$(brew --prefix openssl@1.1)/lib/" CPATH="$(brew --prefix openssl@1.1)/include/" STATIC_SSL=1 - CIBW_ARCHS: "x86_64" - CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }} - - - name: Save macOS wheel - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: ${{ matrix.python }}-macosx_x86_64.build - path: wheelhouse/*.whl - - - name: Set final commit status - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} - with: - status: ${{ job.status }} - sha: ${{ github.sha }} - context: "Build wheels (${{ matrix.python }}-macosx_x86_64)" - - macOS-m1: - runs-on: [ - self-hosted, - macOS, - ARM64 - ] - strategy: - matrix: - python-version: [ - ["cp38", "3.8"], - ["cp39", "3.9"], - ["cp310", "3.10"], - ["cp311", "3.11"], - ["cp312", "3.12"] - ] - fail-fast: false - steps: - name: Show job status for commit uses: myrotvorets/set-commit-status-action@v2.0.0 if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} with: - sha: ${{ github.sha }} - context: "Build wheels (${{ matrix.python-version[0] }}-macosx_arm64)" + sha: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} + context: ${{ env.STATUS_CHECK_MESSAGE }} - uses: actions/checkout@v4 with: submodules: recursive - ref: ${{ inputs.ref }} + ref: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} + # We need the last tag before the ref, so we can relabel the version if needed fetch-depth: 0 - # Update dependencies if needed - - name: Add brew to path - run: echo PATH=$PATH:/opt/homebrew/bin/ >> $GITHUB_ENV - - - name: Install or upgrade Python - run: brew install python@${{ matrix.python-version[1] }} - - - name: Install or upgrade OpenSSL 1.1 - run: brew install openssl@1.1 - - - name: Set environment variables for building + - name: 'macOS arm64: Install experimental Python 3.8 macOS arm64 build' + # By default, cibuildwheel installs and uses Python 3.8 x86_64 to cross compile macOS arm64 wheels + # There is a bug that builds macOS x86_64 wheels instead, so we install this Python 3.8 native ARM build to ensure + # the wheel is compiled for macOS arm64 + # https://cibuildwheel.pypa.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64 + if: ${{ matrix.python-tag == 'cp38' && inputs.platform-tag == 'macosx_arm64' }} run: | - openssl_path=$(brew --prefix openssl@1.1) - echo SSL_LIB_PATH="$openssl_path/lib/" >> $GITHUB_ENV - echo CPATH="$openssl_path/include/" >> $GITHUB_ENV - echo STATIC_SSL=1 >> $GITHUB_ENV - - - name: Install pip build packages - run: python${{ matrix.python-version[1] }} -m pip install --break-system-packages --force-reinstall -r requirements.txt - - # Self-hosted runner only - # Need to be able to save Docker Hub credentials to keychain - - run: security unlock-keychain -p ${{ secrets.MAC_M1_SELF_HOSTED_RUNNER_PW }} - if: ${{ inputs.run_tests && inputs.use-server-rc }} - - - uses: ./.github/actions/run-ee-server - if: ${{ inputs.run_tests }} + curl -o /tmp/Python38.pkg https://www.python.org/ftp/python/3.8.10/python-3.8.10-macos11.pkg + sudo installer -pkg /tmp/Python38.pkg -target / + sh "/Applications/Python 3.8/Install Certificates.command" + + - name: 'Windows: Add msbuild to PATH' + if: ${{ inputs.platform-tag == 'win_amd64' }} + uses: microsoft/setup-msbuild@v1.1 + + - name: 'Windows: Install C client deps' + if: ${{ inputs.platform-tag == 'win_amd64' }} + run: nuget restore + working-directory: aerospike-client-c/vs + + - name: 'macOS x86: Setup Docker using colima for testing' + if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' && inputs.platform-tag == 'macosx_x86_64' }} + uses: ./.github/actions/setup-docker-on-macos + + - name: 'Run Aerospike server in Docker container and configure tests accordingly' + if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }} + uses: ./.github/actions/run-ee-server with: use-server-rc: ${{ inputs.use-server-rc }} server-tag: ${{ inputs.server-tag }} docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + where-is-client-connecting-from: ${{ inputs.platform-tag == 'macosx_x86_64' && 'docker-host' || 'separate-docker-container' }} + + - name: If not running tests against server, only run basic import test + if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'false' }} + # Use double quotes otherwise Windows will throw this error in cibuildwheel + # 'import + # ^ + # SyntaxError: EOL while scanning string literal + run: echo "TEST_COMMAND=python -c \"import aerospike\"" >> $GITHUB_ENV + shell: bash + + - name: Otherwise, enable integration tests + if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }} + # Run with capture output disabled to check that TLS works (i.e we are using the bundled openssl) + run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest -vvs new_tests/${{ inputs.test-file }}" >> $GITHUB_ENV + shell: bash - name: Set unoptimize flag - if: ${{ inputs.apply-no-optimizations }} + if: ${{ inputs.unoptimized && (startsWith(inputs.platform-tag, 'manylinux') || startsWith(inputs.platform-tag, 'macosx')) }} run: echo "UNOPTIMIZED=1" >> $GITHUB_ENV - name: Set include dsym flag - if: ${{ inputs.include-debug-info-for-macos }} + if: ${{ inputs.include-debug-info-for-macos && startsWith(inputs.platform-tag, 'macosx') }} run: echo "INCLUDE_DSYM=1" >> $GITHUB_ENV - - run: python${{ matrix.python-version[1] }} -m build - - - name: Install delocate - run: python${{ matrix.python-version[1] }} -m pip install --break-system-packages --force-reinstall delocate -c ./requirements.txt - working-directory: .github/workflows + - if: ${{ startsWith(inputs.platform-tag, 'manylinux') }} + run: echo CIBW_MANYLINUX_X86_64_IMAGE=${{ format(env.CUSTOM_IMAGE_NAME, 'x86_64') }} >> $GITHUB_ENV + + - if: ${{ startsWith(inputs.platform-tag, 'manylinux') }} + run: echo CIBW_MANYLINUX_AARCH64_IMAGE=${{ format(env.CUSTOM_IMAGE_NAME, 'aarch64') }} >> $GITHUB_ENV - - run: delocate-wheel --require-archs "arm64" -w wheelhouse/ -v dist/*.whl - - run: python${{ matrix.python-version[1] }} -m pip install --break-system-packages --find-links=wheelhouse/ --no-index --force-reinstall aerospike - - - run: python${{ matrix.python-version[1] }} -m pip install --break-system-packages --force-reinstall -r requirements.txt - if: ${{ inputs.run_tests }} - working-directory: test + - uses: docker/login-action@v3 + if: ${{ startsWith(inputs.platform-tag, 'manylinux') }} + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - run: python${{ matrix.python-version[1] }} -m pytest new_tests/ - if: ${{ inputs.run_tests }} - working-directory: test + - name: Build wheel + uses: pypa/cibuildwheel@v2.21.3 + env: + CIBW_ENVIRONMENT_PASS_LINUX: ${{ inputs.unoptimized && 'UNOPTIMIZED' || '' }} + CIBW_ENVIRONMENT_MACOS: SSL_LIB_PATH="$(brew --prefix openssl@${{ env.MACOS_OPENSSL_VERSION }})/lib/" CPATH="$(brew --prefix openssl@${{ env.MACOS_OPENSSL_VERSION }})/include/" STATIC_SSL=1 + CIBW_BUILD: ${{ env.BUILD_IDENTIFIER }} + CIBW_BUILD_FRONTEND: build + CIBW_BEFORE_ALL_LINUX: > + yum install python-setuptools -y + # delvewheel is not enabled by default but we do need to repair the wheel + CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel==1.*" + # We want to check that our wheel links to the new openssl 3 install, not the system default + # This assumes that ldd prints out the "soname" for the libraries + # We can also manually verify the repair worked by checking the repaired wheel's compatibility tag + CIBW_REPAIR_WHEEL_COMMAND_LINUX: > + WHEEL_DIR=wheel-contents && + unzip {wheel} -d $WHEEL_DIR && + ldd $WHEEL_DIR/*.so | awk '{print $1}' | grep libssl.so.3 && + ldd $WHEEL_DIR/*.so | awk '{print $1}' | grep libcrypto.so.3 && + auditwheel repair -w {dest_dir} {wheel} && + auditwheel show {dest_dir}/* && + rm -rf $WHEEL_DIR + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair --add-path ./aerospike-client-c/vs/x64/Release -w {dest_dir} {wheel}" + CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }} - - run: python${{ matrix.python-version[1] }} -c "import aerospike" - if: ${{ !inputs.run_tests }} + # For debugging + - run: docker logs aerospike + if: ${{ always() && env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }} + shell: bash - - name: Save macOS wheel + - name: Upload wheels to GitHub uses: actions/upload-artifact@v4 - if: ${{ always() }} + if: ${{ !cancelled() }} with: - name: ${{ matrix.python-version[0] }}-macosx_arm64.build - path: wheelhouse/*.whl - - - name: Stop server - if: ${{ always() && inputs.run_tests }} - run: | - docker container stop aerospike - docker container prune -f + path: ./wheelhouse/*.whl + name: ${{ env.BUILD_IDENTIFIER }}.build - name: Set final commit status uses: myrotvorets/set-commit-status-action@v2.0.0 if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} with: - sha: ${{ github.sha }} + sha: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} status: ${{ job.status }} - context: "Build wheels (${{ matrix.python-version[0] }}-macosx_arm64)" + context: ${{ env.STATUS_CHECK_MESSAGE }} - windows-build: + test-self-hosted: + needs: cibuildwheel + # There's a top-level env variable for this but we can't use it here, unfortunately + if: ${{ inputs.run_tests && (inputs.platform-tag == 'macosx_arm64' || inputs.platform-tag == 'win_amd64') }} strategy: fail-fast: false matrix: - python: [ - ["cp38", "3.8"], - ["cp39", "3.9"], - ["cp310", "3.10"], - ["cp311", "3.11"], - ["cp312", "3.12"] - ] - runs-on: windows-2022 + python-tag: ${{ fromJSON(inputs.python-tags) }} + runs-on: ${{ inputs.platform-tag == 'macosx_arm64' && fromJSON('["self-hosted", "macOS", "ARM64"]') || fromJSON('["self-hosted", "Windows", "X64"]') }} + env: + BUILD_IDENTIFIER: "${{ matrix.python-tag }}-${{ inputs.platform-tag }}" steps: + - name: Create status check message + run: echo STATUS_CHECK_MESSAGE="Test on self hosted (${{ env.BUILD_IDENTIFIER }})" >> $GITHUB_ENV + shell: bash + - name: Show job status for commit uses: myrotvorets/set-commit-status-action@v2.0.0 if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} with: - sha: ${{ github.sha }} - context: "Build wheels (${{ matrix.python[0] }}-win_amd64)" + sha: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} + context: ${{ env.STATUS_CHECK_MESSAGE }} - uses: actions/checkout@v4 with: - submodules: recursive - ref: ${{ inputs.ref }} - fetch-depth: 0 - - - name: Add msbuild to PATH - uses: microsoft/setup-msbuild@v1.1 - - - name: Install C client deps - run: nuget restore - working-directory: aerospike-client-c/vs - - - name: Build wheel - uses: pypa/cibuildwheel@v2.19.2 - env: - CIBW_BUILD: ${{ matrix.python[0] }}-win_amd64 - CIBW_BUILD_FRONTEND: build - CIBW_ARCHS: auto64 - CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel" - CIBW_REPAIR_WHEEL_COMMAND: "delvewheel repair --add-path ./aerospike-client-c/vs/x64/Release -w {dest_dir} {wheel}" - - - uses: actions/upload-artifact@v4 - with: - path: ./wheelhouse/*.whl - name: ${{ matrix.python[0] }}-win_amd64.build + ref: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} - - name: Set final commit status - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} - with: - sha: ${{ github.sha }} - status: ${{ job.status }} - context: "Build wheels (${{ matrix.python[0] }}-win_amd64)" + # Need to be able to save Docker Hub credentials to keychain + # Unable to do this via the ansible script for self hosted mac m1 runners + - if: ${{ inputs.platform-tag == 'macosx_arm64' }} + run: security unlock-keychain -p ${{ secrets.MAC_M1_SELF_HOSTED_RUNNER_PW }} - test-windows: - needs: windows-build - if: ${{ inputs.run_tests }} - strategy: - fail-fast: false - matrix: - python: [ - ["cp38", "3.8"], - ["cp39", "3.9"], - ["cp310", "3.10"], - ["cp311", "3.11"], - ["cp312", "3.12"] - ] - runs-on: [self-hosted, Windows, X64] - steps: - - name: Show job status for commit - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }} - with: - sha: ${{ github.sha }} - context: "Test Windows (${{ matrix.python[0] }}-win_amd64)" - - uses: actions/checkout@v4 + - uses: ./.github/actions/run-ee-server with: - ref: ${{ inputs.ref }} + use-server-rc: ${{ inputs.use-server-rc }} + server-tag: ${{ inputs.server-tag }} + docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} + docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + where-is-client-connecting-from: ${{ inputs.platform-tag == 'win_amd64' && 'remote-connection' || 'docker-host' }} - name: Download wheel uses: actions/download-artifact@v4 with: - name: ${{ matrix.python[0] }}-win_amd64.build + name: ${{ env.BUILD_IDENTIFIER }}.build + + - name: Convert Python tag to Python version + # Don't use sed because we want this command to work on both mac and windows + # The command used in GNU sed is different than in macOS sed + run: | + PYTHON_TAG=${{ matrix.python-tag }} + PYTHON_VERSION="${PYTHON_TAG/cp/}" + echo PYTHON_VERSION="${PYTHON_VERSION/3/3.}" >> $GITHUB_ENV + shell: bash - uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python[1] }} + python-version: ${{ env.PYTHON_VERSION }} - name: Install wheel run: python3 -m pip install aerospike --force-reinstall --no-index --find-links=./ + shell: bash - run: python3 -m pip install pytest -c requirements.txt working-directory: test + shell: bash - - uses: ./.github/actions/run-ee-server - - - name: Connect to Docker container on remote machine with Docker daemon - # DOCKER_HOST contains the IP address of the remote machine - run: | - $env:DOCKER_HOST_IP = $env:DOCKER_HOST | foreach {$_.replace("tcp://","")} | foreach {$_.replace(":2375", "")} - crudini --set config.conf enterprise-edition hosts ${env:DOCKER_HOST_IP}:3000 - working-directory: test - - - run: python3 -m pytest new_tests/ + - run: python3 -m pytest -vv new_tests/${{ inputs.test-file }} working-directory: test - - - name: Cleanup - if: ${{ always() }} - run: | - docker stop aerospike - docker container rm aerospike + shell: bash - name: Show job status for commit if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }} uses: myrotvorets/set-commit-status-action@v2.0.0 with: - sha: ${{ github.sha }} + sha: ${{ env.COMMIT_SHA_TO_BUILD_AND_TEST }} status: ${{ job.status }} - context: "Test Windows (${{ matrix.python[0] }}-win_amd64)" + context: ${{ env.STATUS_CHECK_MESSAGE }} diff --git a/.github/workflows/bump-stage-and-upload-to-jfrog.yml b/.github/workflows/bump-stage-and-upload-to-jfrog.yml index b797b9ffb..922c94de9 100644 --- a/.github/workflows/bump-stage-and-upload-to-jfrog.yml +++ b/.github/workflows/bump-stage-and-upload-to-jfrog.yml @@ -35,9 +35,10 @@ jobs: rebuild-artifacts-with-rc-version: needs: promote-dev-build-to-rc - uses: ./.github/workflows/build-wheels.yml + uses: ./.github/workflows/build-artifacts.yml with: - ref: ${{ needs.promote-dev-build-to-rc.outputs.bump_sha }} + sha-to-build-and-test: ${{ needs.promote-dev-build-to-rc.outputs.bump_sha }} + secrets: inherit upload-rc-artifacts-to-jfrog: needs: [ diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index 9552c1891..7ec371644 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -26,6 +26,7 @@ on: required: false description: Commit to bump off of type: string + # See workflow call hack in update-version.yml is_workflow_call: type: boolean default: true diff --git a/.github/workflows/dev-workflow-p1.yml b/.github/workflows/dev-workflow-p1.yml index 6acfd96f9..74044da6c 100644 --- a/.github/workflows/dev-workflow-p1.yml +++ b/.github/workflows/dev-workflow-p1.yml @@ -24,9 +24,10 @@ on: jobs: test-with-server-release: - uses: ./.github/workflows/build-wheels.yml + uses: ./.github/workflows/build-artifacts.yml with: run_tests: ${{ github.event_name == 'pull_request' && true || inputs.run_server_release_tests }} + sha-to-build-and-test: ${{ github.sha }} secrets: inherit test-with-server-rc: diff --git a/.github/workflows/dev-workflow-p2.yml b/.github/workflows/dev-workflow-p2.yml index 2234ce10f..7427ec970 100644 --- a/.github/workflows/dev-workflow-p2.yml +++ b/.github/workflows/dev-workflow-p2.yml @@ -19,11 +19,11 @@ jobs: rebuild-artifacts-with-new-dev-num: needs: bump-dev-number name: Rebuild artifacts with new dev number - uses: ./.github/workflows/build-wheels.yml + uses: ./.github/workflows/build-artifacts.yml with: # On pull_request_target, the bump version commit will be ignored # So we must pass it manually to the workflow - ref: ${{ needs.bump-dev-number.outputs.bump_sha }} + sha-to-build-and-test: ${{ needs.bump-dev-number.outputs.bump_sha }} secrets: inherit upload-to-jfrog: diff --git a/.github/workflows/docker-build-context/Dockerfile b/.github/workflows/docker-build-context/Dockerfile new file mode 100644 index 000000000..f40106410 --- /dev/null +++ b/.github/workflows/docker-build-context/Dockerfile @@ -0,0 +1,112 @@ +ARG SERVER_IMAGE=aerospike/aerospike-server-enterprise + +# Shared between build stages to get node ID from roster file and to build final image +ARG ROSTER_FILE_NAME=roster.smd +# Temp file for passing node id from the two build stages mentioned above +# Docker doesn't support command substitution for setting values for ARG variables, so we have to do this +ARG NODE_ID_FILE_NAME=node_id + +# Shared between build stages to generate the server certificate and to build final image +# aerospike.conf needed to get cluster name for certificate +ARG AEROSPIKE_CONF_TEMPLATE_PATH=/etc/aerospike/aerospike.template.conf +ARG SERVER_KEY_FILE_NAME=server.pem +ARG SERVER_CERT_FILE_NAME=server.cer +# Temp file +# Cluster name fetched from stage to generate cert also needs to be set in aerospike.conf in our final image +ARG CLUSTER_NAME_FILE_NAME=cluster_name + +FROM $SERVER_IMAGE AS enable-security + +WORKDIR /opt/aerospike/smd + +# Not using asconfig to edit config because we are working with a template file, which may not have valid values yet +ARG AEROSPIKE_CONF_TEMPLATE_PATH +RUN echo -e "security {\n\tenable-quotas true\n}\n" >> $AEROSPIKE_CONF_TEMPLATE_PATH +# security.smd was generated manually by +# 1. Starting a new Aerospike EE server using Docker +# 2. Creating the superuser user +# 3. Copying /opt/aerospike/smd/security.smd from the container and committing it to this repo +# This file should always work +# TODO: generate this automatically, somehow. +COPY security.smd . + +# Strong consistency only: fetch node id from roster.smd (JSON file) + +# jq docker image doesn't have a shell +# We need a shell to fetch and pass the node id to the next build stage +FROM busybox AS get-node-id-for-sc + +# There's no tag for the latest major version to prevent breaking changes in jq +# This is the next best thing +COPY --from=ghcr.io/jqlang/jq:1.7 /jq /bin/ +ARG ROSTER_FILE_NAME +COPY $ROSTER_FILE_NAME . +ARG NODE_ID_FILE_NAME +RUN jq --raw-output '.[1].value' $ROSTER_FILE_NAME > $NODE_ID_FILE_NAME + +FROM enable-security AS enable-strong-consistency + +RUN sed -i "s/\(namespace.*{\)/\1\n\tstrong-consistency true/" $AEROSPIKE_CONF_TEMPLATE_PATH +RUN sed -i "s/\(namespace.*{\)/\1\n\tstrong-consistency-allow-expunge true/" $AEROSPIKE_CONF_TEMPLATE_PATH +ARG ROSTER_FILE_NAME +COPY $ROSTER_FILE_NAME . + +ARG NODE_ID_FILE_NAME +COPY --from=get-node-id-for-sc $NODE_ID_FILE_NAME . +RUN sed -i "s/\(^service {\)/\1\n\tnode-id $(cat $NODE_ID_FILE_NAME)/" $AEROSPIKE_CONF_TEMPLATE_PATH +# We don't want to clutter final image with temp files (junk) +RUN rm $NODE_ID_FILE_NAME + +# Use a separate build stage to generate certs since we don't want openssl bundled in the final image + +FROM $SERVER_IMAGE AS generate-server-cert-for-tls + +RUN apt update +RUN apt install -y openssl + +ARG AEROSPIKE_CONF_TEMPLATE_PATH +ARG CLUSTER_NAME_FILE_NAME +RUN grep -Eo "cluster-name [a-z]+" $AEROSPIKE_CONF_TEMPLATE_PATH | awk '{print $2}' > $CLUSTER_NAME_FILE_NAME + +# Generate server private key and CSR + +ARG SERVER_CSR_FILE_NAME=server.csr +ARG SERVER_KEY_FILE_NAME +RUN openssl req -newkey rsa:4096 -keyout $SERVER_KEY_FILE_NAME -nodes -new -out $SERVER_CSR_FILE_NAME -subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=$(cat $CLUSTER_NAME_FILE_NAME)" + +# Send CSR to CA and get server certificate +# We use an external CA because we want the client to use that same CA to verify the server certificate upon connecting +ARG CA_KEY_FILE_NAME +ARG CA_CERT_FILE_NAME +COPY $CA_KEY_FILE_NAME . +COPY $CA_CERT_FILE_NAME . +ARG SERVER_CERT_FILE_NAME +RUN openssl x509 -req -in $SERVER_CSR_FILE_NAME -CA $CA_CERT_FILE_NAME -CAkey $CA_KEY_FILE_NAME -out $SERVER_CERT_FILE_NAME + +FROM enable-strong-consistency AS enable-tls + +ARG SSL_WORKING_DIR=/etc/ssl +WORKDIR $SSL_WORKING_DIR +ARG SERVER_KEY_FILE_NAME +ARG SERVER_CERT_FILE_NAME +ARG SERVER_KEY_INSTALL_PATH=$SSL_WORKING_DIR/private/$SERVER_KEY_FILE_NAME +ARG SERVER_CERT_INSTALL_PATH=$SSL_WORKING_DIR/certs/$SERVER_CERT_FILE_NAME + +COPY --from=generate-server-cert-for-tls $SERVER_KEY_FILE_NAME $SERVER_KEY_INSTALL_PATH +COPY --from=generate-server-cert-for-tls $SERVER_CERT_FILE_NAME $SERVER_CERT_INSTALL_PATH + +ARG CLUSTER_NAME_FILE_NAME +COPY --from=generate-server-cert-for-tls $CLUSTER_NAME_FILE_NAME . +# Service sub-stanza under network stanza +RUN sed -i "s|\(^\tservice {\)|\1\n\t\ttls-name $(cat $CLUSTER_NAME_FILE_NAME)|" $AEROSPIKE_CONF_TEMPLATE_PATH +RUN sed -i "s|\(^\tservice {\)|\1\n\t\ttls-authenticate-client false|" $AEROSPIKE_CONF_TEMPLATE_PATH +ARG TLS_PORT=4333 +RUN sed -i "s|\(^\tservice {\)|\1\n\t\ttls-port $TLS_PORT|" $AEROSPIKE_CONF_TEMPLATE_PATH + +RUN sed -i "s|\(^network {\)|\1\n\ttls $(cat $CLUSTER_NAME_FILE_NAME) {\n\t\tcert-file $SERVER_CERT_INSTALL_PATH\n\t}|" $AEROSPIKE_CONF_TEMPLATE_PATH +RUN sed -i "s|\(^\ttls $(cat $CLUSTER_NAME_FILE_NAME) {\)|\1\n\t\tkey-file $SERVER_KEY_INSTALL_PATH|" $AEROSPIKE_CONF_TEMPLATE_PATH + +EXPOSE $TLS_PORT + +# Cleanup +RUN rm $CLUSTER_NAME_FILE_NAME diff --git a/.github/workflows/docker-build-context/roster.smd b/.github/workflows/docker-build-context/roster.smd new file mode 100644 index 000000000..66daed5f6 --- /dev/null +++ b/.github/workflows/docker-build-context/roster.smd @@ -0,0 +1,12 @@ +[ + [ + 97107025374203, + 1 + ], + { + "key": "test", + "value": "a1", + "generation": 1, + "timestamp": 465602976982 + } +] diff --git a/.github/workflows/security.smd b/.github/workflows/docker-build-context/security.smd similarity index 100% rename from .github/workflows/security.smd rename to .github/workflows/docker-build-context/security.smd diff --git a/.github/workflows/manylinux2014-openssl.Dockerfile b/.github/workflows/manylinux2014-openssl.Dockerfile new file mode 100644 index 000000000..00db671fd --- /dev/null +++ b/.github/workflows/manylinux2014-openssl.Dockerfile @@ -0,0 +1,25 @@ +ARG CPU_ARCH=x86_64 +FROM quay.io/pypa/manylinux2014_$CPU_ARCH +ARG OPENSSL_VERSION +LABEL com.aerospike.clients.openssl-version=$OPENSSL_VERSION + +RUN yum install -y perl-core wget + +WORKDIR / +ARG OPENSSL_TAR_NAME=openssl-$OPENSSL_VERSION +RUN wget https://www.openssl.org/source/$OPENSSL_TAR_NAME.tar.gz +RUN tar xzvf $OPENSSL_TAR_NAME.tar.gz +WORKDIR $OPENSSL_TAR_NAME + +# The default folder pointed to by --prefix contains a default openssl installation +# But we're assuming it's fine to replace the default openssl that comes with the image +# We aren't going to use this image in production, anyways +RUN ./Configure +RUN make +# These tests are expected to fail because we are using a buggy version of nm +# https://github.com/openssl/openssl/issues/18953 +# devtoolset-11 contains a newer version of binutils 2.36, which contains a bug fix for nm +# We don't use it though because we want to make sure the compiled openssl 3 library is compatible with manylinux2014's +# default env +RUN make V=1 TESTS='-test_symbol_presence*' test +RUN make install diff --git a/.github/workflows/requirements.txt b/.github/workflows/requirements.txt index 5e6bb447f..5099e0d6f 100644 --- a/.github/workflows/requirements.txt +++ b/.github/workflows/requirements.txt @@ -1,3 +1,4 @@ parver==0.5 crudini==0.9.4 delocate==0.10.4 +mypy==1.10.0 diff --git a/.github/workflows/stage-tests.yml b/.github/workflows/stage-tests.yml index 29a3a5164..7523ee7d2 100644 --- a/.github/workflows/stage-tests.yml +++ b/.github/workflows/stage-tests.yml @@ -58,6 +58,8 @@ jobs: ["python:3.11-bookworm", 2, "linux/arm64", "3.11"], ["python:3.12-bookworm", 2, "linux/amd64", "3.12"], ["python:3.12-bookworm", 2, "linux/arm64", "3.12"], + ["python:3.13-bookworm", 2, "linux/amd64", "3.13"], + ["python:3.13-bookworm", 2, "linux/arm64", "3.13"], ["amazonlinux:2023", 1, "linux/amd64", "3.9"], ["redhat/ubi9", 1, "linux/amd64", "3.9"], ] @@ -102,6 +104,7 @@ jobs: server-tag: ${{ inputs.server-tag }} docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + where-is-client-connecting-from: 'separate-docker-container' - name: Run distro container # Run distro container on host network to access the Aerospike server using localhost (without having to change config.conf) @@ -157,6 +160,7 @@ jobs: "3.10", "3.11", "3.12", + "3.13", ] fail-fast: false runs-on: ${{ matrix.runner-os }} @@ -186,14 +190,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install Docker Engine - run: brew install colima - - - name: Install Docker client - run: brew install docker - - - name: Start Docker Engine - run: colima start + - uses: ./.github/actions/setup-docker-on-macos - uses: ./.github/actions/run-ee-server with: @@ -201,6 +198,7 @@ jobs: server-tag: ${{ inputs.server-tag }} docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + where-is-client-connecting-from: 'docker-host' - name: Install wheel run: python3 -m pip install *.whl diff --git a/.github/workflows/stage-to-master.yml b/.github/workflows/stage-to-master.yml index fc69f256e..154dc4dec 100644 --- a/.github/workflows/stage-to-master.yml +++ b/.github/workflows/stage-to-master.yml @@ -20,9 +20,10 @@ jobs: build-artifacts: needs: promote-rc-build-to-release - uses: ./.github/workflows/build-wheels.yml + uses: ./.github/workflows/build-artifacts.yml with: - ref: ${{ needs.promote-rc-build-to-release.outputs.bump_sha }} + sha-to-build-and-test: ${{ needs.promote-rc-build-to-release.outputs.bump_sha }} + secrets: inherit upload-to-jfrog: name: Upload artifacts to JFrog @@ -45,6 +46,7 @@ jobs: path: artifacts merge-multiple: true + # TODO: fix - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/test-server-rc.yml b/.github/workflows/test-server-rc.yml index 4a65c06bc..1d11d5702 100644 --- a/.github/workflows/test-server-rc.yml +++ b/.github/workflows/test-server-rc.yml @@ -34,12 +34,13 @@ jobs: - run: docker run -d --name manylinux quay.io/pypa/manylinux2014_${{ matrix.platform[0] }} tail -f /dev/null - - uses: ./.github/actions/run-ee-server-for-ext-container + - uses: ./.github/actions/run-ee-server with: use-server-rc: true server-tag: latest docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} + where-is-client-connecting-from: 'docker-container' - uses: actions/download-artifact@v4 with: @@ -76,14 +77,7 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install Docker Engine - run: brew install colima - - - name: Install Docker client - run: brew install docker - - - name: Start Docker Engine - run: colima start + - uses: ./.github/actions/setup-docker-on-macos - uses: ./.github/actions/run-ee-server with: @@ -152,7 +146,7 @@ jobs: password: ${{ secrets.DOCKER_HUB_BOT_PW }} - name: Run server RC - run: docker run -d -p 3000:3000 --name aerospike ${{ vars.SERVER_RC_REPO_LINK }} + run: docker run -d -p 3000:3000 --name aerospike -e DEFAULT_TTL=2592000 ${{ vars.SERVER_RC_REPO_LINK }} - name: Create config.conf run: cp config.conf.template config.conf diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8fe05ac2a..662b7a74f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -44,7 +44,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - py-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + py-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] # Make sure we can build and run tests on an instrumented build that uses libasan # We aren't necessarily checking for memory errors / leaks in this test sanitizer: [false] @@ -89,49 +89,6 @@ jobs: name: ${{ env.WHEEL_GH_ARTIFACT_NAME }} path: ./dist/*.whl - test-memray: - needs: build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-python@v2 - with: - python-version: "3.8" - architecture: 'x64' - - - uses: actions/download-artifact@v3 - with: - name: wheel-3.8 - - - name: Install client - run: pip install *.whl - - - name: Install test dependencies - run: pip install -r test/requirements.txt - - - name: Run Aerospike server - run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server - - - name: Create config.conf - run: cp config.conf.template config.conf - working-directory: test - - - uses: ./.github/actions/wait-for-as-server-to-start - with: - container-name: aerospike - - - name: Get number of tests - run: echo "NUM_TESTS=$(python3 -m pytest new_tests/ --collect-only -q | tail -n 1 | awk '{print $1;}')" >> $GITHUB_ENV - working-directory: test - - - name: Run tests - # Get number of tests since setting to 0 doesn't work properly - # pytest-memray currently throws a ZeroDivision error due to having a bug - # We ignore this for now - run: python -m pytest ./new_tests --memray --memray-bin-path=./ --most-allocations=${{ env.NUM_TESTS }} || true - working-directory: test - generate-coverage-report: runs-on: ubuntu-latest steps: @@ -184,6 +141,7 @@ jobs: mkdir -p nullobject/src/main/nullobject mkdir -p query/src/main/query mkdir -p scan/src/main/scan + mkdir -p transaction/src/main/transaction cd ../../../../ cp src/main/*.c build/temp*/src/main/src/main @@ -195,6 +153,7 @@ jobs: cp src/main/nullobject/*.c build/temp*/src/main/nullobject/src/main/nullobject/ cp src/main/query/*.c build/temp*/src/main/query/src/main/query/ cp src/main/scan/*.c build/temp*/src/main/scan/src/main/scan/ + cp src/main/transaction/*.c build/temp*/src/main/transaction/src/main/transaction/ - name: Generate coverage report for all object files if: ${{ !cancelled() }} @@ -287,7 +246,7 @@ jobs: run: pip install -r test/requirements.txt - name: Run Aerospike server - run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server + run: docker run -d --name aerospike -p 3000-3002:3000-3002 -e DEFAULT_TTL=2592000 aerospike/aerospike-server - name: Create config.conf run: cp config.conf.template config.conf @@ -311,7 +270,8 @@ jobs: "3.9", "3.10", "3.11", - "3.12" + "3.12", + "3.13" ] sanitizer: [false] include: @@ -352,11 +312,11 @@ jobs: - name: Run Aerospike server release candidate with latest tag if: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }} - run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server-rc:latest + run: docker run -d --name aerospike -p 3000-3002:3000-3002 -e DEFAULT_TTL=2592000 aerospike/aerospike-server-rc:latest - name: Run Aerospike server if: ${{ !contains(github.event.pull_request.labels.*.name, 'new-server-features') }} - run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server + run: docker run -d --name aerospike -p 3000-3002:3000-3002 -e DEFAULT_TTL=2592000 aerospike/aerospike-server - name: Create config.conf run: cp config.conf.template config.conf @@ -399,7 +359,7 @@ jobs: run: pip install -r test/requirements.txt - name: Run lowest supported server - run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server:${{ vars.LOWEST_SUPPORTED_SERVER_VERSION }} + run: docker run -d --name aerospike -p 3000-3002:3000-3002 -e DEFAULT_TTL=2592000 aerospike/aerospike-server:${{ vars.LOWEST_SUPPORTED_SERVER_VERSION }} - name: Create config.conf run: cp config.conf.template config.conf @@ -443,7 +403,7 @@ jobs: docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} - name: Run tests - run: python -m pytest ./new_tests/test_admin_*.py + run: python -m pytest ./new_tests/test_{mrt_functionality,admin_*}.py working-directory: test - name: Show logs if failed @@ -483,30 +443,14 @@ jobs: run: sphinx-build -b linkcheck . links working-directory: doc - test-metrics-node-close-listener: - needs: build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ env.LOWEST_SUPPORTED_PY_VERSION }} - architecture: 'x64' - - name: Download aerolab - run: wget https://github.com/aerospike/aerolab/releases/download/7.6.0-7b5bbde/aerolab-linux-amd64-7.6.0.deb - - name: Install aerolab - run: sudo dpkg -i *.deb - - name: Tell aerolab to use Docker - run: aerolab config backend -t docker - - uses: actions/download-artifact@v3 - with: - name: wheel-${{ env.LOWEST_SUPPORTED_PY_VERSION }} - - run: python3 -m pip install *.whl - - run: python3 test_node_close_listener.py - working-directory: test/metrics - - test-metrics-cluster-name: + test-metrics: needs: build + strategy: + matrix: + suffix: + - node_close_listener + - cluster_name + fail-fast: false runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -520,5 +464,5 @@ jobs: - run: python3 -m pip install *.whl - run: python3 -m pip install -r requirements.txt working-directory: test/metrics - - run: python3 test_cluster_name.py + - run: python3 test_${{ matrix.suffix }}.py working-directory: test/metrics diff --git a/.github/workflows/update-manylinux-openssl-image.yml b/.github/workflows/update-manylinux-openssl-image.yml new file mode 100644 index 000000000..17dfd9e92 --- /dev/null +++ b/.github/workflows/update-manylinux-openssl-image.yml @@ -0,0 +1,62 @@ +on: + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 17 * * 1-5' + workflow_dispatch: + +jobs: + main: + env: + # We want granular control over the openssl version bundled with our wheels + OPENSSL_VERSION: '3.0.15' + REGISTRY: ghcr.io + strategy: + matrix: + arch-and-runner-os: [ + [x86_64, ubuntu-24.04], + [aarch64, aerospike_arm_runners_2] + ] + fail-fast: false + + runs-on: ${{ matrix.arch-and-runner-os[1] }} + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/workflows + + - run: docker pull quay.io/pypa/manylinux2014_${{ matrix.arch-and-runner-os[0] }} + + - uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/metadata-action@v5 + id: meta + with: + images: ${{ env.REGISTRY }}/aerospike/manylinux2014_${{ matrix.arch-and-runner-os[0] }} + flavor: latest=true + + - name: Set up Docker Buildx so we can cache our Docker image layers + uses: docker/setup-buildx-action@v3 + + - name: Build and push + uses: docker/build-push-action@v6 + with: + # Don't want to use default Git context or else it will clone the whole Python client repo again + context: .github/workflows + file: .github/workflows/manylinux2014-openssl.Dockerfile + build-args: | + OPENSSL_VERSION=${{ env.OPENSSL_VERSION }} + CPU_ARCH=${{ matrix.arch-and-runner-os[0] }} + # setup-buildx-action configures Docker to use the docker-container build driver + # This driver doesn't publish an image locally by default + # so we have to manually enable it + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + # Also cache intermediate layers to make development faster + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/update-version.yml b/.github/workflows/update-version.yml index 4900ba0dc..ce155ed1e 100644 --- a/.github/workflows/update-version.yml +++ b/.github/workflows/update-version.yml @@ -22,6 +22,7 @@ on: # A hack to tell if workflow is triggered by workflow_call or not # Calling workflows should not set this input # If workflow is triggered by workflow_dispatch, this should be set to the default boolean value: false + # https://github.com/actions/runner/discussions/1884#discussioncomment-6377587 is_workflow_call: type: boolean default: true diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 011539d2c..f01175f2e 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -12,13 +12,112 @@ on: description: 'Use server release candidate?' required: true default: false + server-tag: + required: false + default: latest + massif: + type: boolean + description: 'Use massif for testing memory usage' + required: false + default: false + +env: + PYTHON_TAG: cp38 jobs: - build-manylinux-wheels: + look-for-wheel-in-jfrog: + outputs: + num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} + # So we can pass the python tag to a reusable workflow + python-tag: ${{ env.PYTHON_TAG }} + runs-on: ubuntu-22.04 + env: + JF_SEARCH_RESULTS_FILE_NAME: wheel_commit_matches.txt + steps: + - uses: jfrog/setup-jfrog-cli@v4 + env: + JF_URL: ${{ secrets.JFROG_PLATFORM_URL }} + JF_ACCESS_TOKEN: ${{ secrets.JFROG_ACCESS_TOKEN }} + + - name: Get shortened commit hash of this workflow run + # versioningit commit sha is always 8 chars long it seems + run: echo SHORT_GITHUB_SHA=$(echo ${{ github.sha }} | cut -c1-8) >> $GITHUB_ENV + + - name: Look for wheel built with default settings in JFrog + # AQL has the option to exclude patterns from search results + # but it doesn't allow regex, so we can't filter out any type of label in a wheel name + # Example: we want to filter out "unoptimized" and "dsym" but in case we add more labels, we want to use regex + # to handle those new labels without updating the regex. + run: jf rt search "${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/*${{ env.SHORT_GITHUB_SHA }}*${{ env.PYTHON_TAG }}*manylinux*x86_64*.whl" > ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + + - name: Show unfiltered results + run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + + - name: Install sponge + run: sudo apt install -y moreutils + + - name: Filter out wheels with labels in results + run: jq 'map(select(.path | test("${{ env.SHORT_GITHUB_SHA }}\\.") | not))' ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | sponge ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + shell: bash + + - name: Check if artifacts with labels were filtered out + run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + + - name: Count artifacts + id: count_num_artifacts_found + run: echo num_artifacts=$(jq length ${{ env.JF_SEARCH_RESULTS_FILE_NAME }}) >> $GITHUB_OUTPUT + + - name: Multiple artifacts found, not sure which one to use. Fail out + if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts > 1 }} + run: exit 1 + + - name: Found the exact artifact in JFrog. Get the artifact name + if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} + run: echo ARTIFACT_PATH=$(jq -r .[0].path ${{ env.JF_SEARCH_RESULTS_FILE_NAME }}) >> $GITHUB_ENV + + - name: Then download artifact from JFrog + if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} + run: jf rt download --flat --fail-no-op ${{ env.ARTIFACT_PATH }} + + - name: Pass to valgrind job + if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} + uses: actions/upload-artifact@v4 + with: + # Artifact name doesn't matter. Valgrind job downloads all artifacts to get the one wheel + if-no-files-found: error + path: './*.whl' + + build-manylinux-wheel: + needs: look-for-wheel-in-jfrog + if: ${{ needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 }} uses: ./.github/workflows/build-wheels.yml + with: + python-tags: '["${{ needs.look-for-wheel-in-jfrog.outputs.python-tag }}"]' + platform-tag: manylinux_x86_64 + sha-to-build-and-test: ${{ github.sha }} + secrets: inherit + + upload-built-wheel-to-jfrog: + needs: build-manylinux-wheel + # TODO: this job should skip when this workflow is run on central branches + # We already have artifacts available for central branches in the PyPI-type JFrog repo + # The problem is we have to conditionally skip this job, but using the github context to get the branch name + # doesn't work for some reason. Just leave this alone for now. + uses: ./.github/workflows/upload-to-jfrog.yml + with: + jfrog-repo-name: ${{ vars.JFROG_GENERIC_REPO_NAME }} + secrets: inherit valgrind: - needs: build-manylinux-wheels + env: + MASSIF_REPORT_FILE_NAME: massif.out + needs: [ + look-for-wheel-in-jfrog, + build-manylinux-wheel + ] + # Case 1: Found artifact in JFrog + # Case 2: Did not find artifact in JFrog, had to build it in GHA + if: ${{ !cancelled() && (needs.look-for-wheel-in-jfrog.result == 'success' && (needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 1) || (needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 && needs.build-manylinux-wheel.result == 'success')) }} runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 @@ -26,14 +125,18 @@ jobs: submodules: recursive fetch-depth: 0 + - name: Convert Python tag to Python version + run: echo PYTHON_VERSION=$(echo ${{ env.PYTHON_TAG }} | sed -e "s/cp3/cp3./" -e "s/cp//") >> $GITHUB_ENV + shell: bash + - uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '${{ env.PYTHON_VERSION }}' architecture: 'x64' - uses: actions/download-artifact@v4 with: - name: cp38-manylinux_x86_64.build + merge-multiple: true - name: Install client run: pip install ./*.whl @@ -45,10 +148,34 @@ jobs: uses: ./.github/actions/run-ee-server with: use-server-rc: ${{ inputs.use-server-rc }} + server-tag: ${{ inputs.server-tag }} docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} - run: sudo apt update - run: sudo apt install valgrind -y - - run: PYTHONMALLOC=malloc valgrind --leak-check=full --error-exitcode=1 python3 -m pytest -v new_tests/${{ github.event.inputs.test-file }} + + - run: echo VALGRIND_ARGS="--tool=massif --massif-out-file=./${{ env.MASSIF_REPORT_FILE_NAME }}" >> $GITHUB_ENV + if: ${{ inputs.massif }} + + - run: echo VALGRIND_ARGS="--leak-check=full" >> $GITHUB_ENV + if: ${{ !inputs.massif }} + + - run: PYTHONMALLOC=malloc valgrind --error-exitcode=1 ${{ env.VALGRIND_ARGS }} python3 -m pytest -v new_tests/${{ github.event.inputs.test-file }} working-directory: test + + # TODO: upload report as artifact + - run: ms_print ./${{ env.MASSIF_REPORT_FILE_NAME }} + if: ${{ !cancelled() && inputs.massif }} + working-directory: test + + # See reason for deleting artifacts in dev-workflow-p2.yml + delete-artifacts: + needs: [ + # These jobs must have downloaded the artifact from Github before we can delete it + upload-built-wheel-to-jfrog, + valgrind + ] + # Workflow run must clean up after itself even if cancelled + if: ${{ always() }} + uses: ./.github/workflows/delete-artifacts.yml diff --git a/.github/workflows/wait-for-as-server-to-start.bash b/.github/workflows/wait-for-as-server-to-start.bash index c43e17da5..18189f193 100755 --- a/.github/workflows/wait-for-as-server-to-start.bash +++ b/.github/workflows/wait-for-as-server-to-start.bash @@ -6,6 +6,7 @@ set -o pipefail container_name=$1 is_security_enabled=$2 +is_strong_consistency_enabled=$3 if [[ $is_security_enabled == true ]]; then # We need to pass credentials to asinfo if server requires it @@ -38,7 +39,14 @@ done while true; do echo "Waiting for server to stabilize (i.e return a cluster key)..." # We assume that when an ERROR is returned, the cluster is not stable yet (i.e not fully initialized) - if docker exec "$container_name" asinfo $user_credentials -v cluster-stable 2>&1 | (! grep -qE "^ERROR"); then + cluster_stable_info_cmd="cluster-stable" + if [[ $is_strong_consistency_enabled == true ]]; then + # The Dockerfile uses a roster from a previously running Aerospike server in a Docker container + # When we reuse this roster, the server assumes all of its partitions are dead because it's running on a new + # storage device. + cluster_stable_info_cmd="$cluster_stable_info_cmd:ignore-migrations=true" + fi + if docker exec "$container_name" asinfo $user_credentials -v $cluster_stable_info_cmd 2>&1 | (! grep -qE "^ERROR"); then echo "Server is in a stable state." break fi diff --git a/BUILD.md b/BUILD.md index 52e7cd96a..baa740de6 100644 --- a/BUILD.md +++ b/BUILD.md @@ -87,8 +87,7 @@ By default macOS will be missing command line tools. The dependencies can be installed through the macOS package manager [Homebrew](http://brew.sh/). - brew install openssl@1 - # brew uninstall openssl@3 + brew install openssl@3 ### All distros @@ -109,8 +108,8 @@ using the wrong version of the C client. This can causes strange issues when bui Also, for macOS or any other operating system that doesn't have OpenSSL installed by default, you must install it and specify its location when building the wheel. In macOS, you would run these commands: ``` -export SSL_LIB_PATH="$(brew --prefix openssl@1.1)/lib/" -export CPATH="$(brew --prefix openssl@1.1)/include/" +export SSL_LIB_PATH="$(brew --prefix openssl@3)/lib/" +export CPATH="$(brew --prefix openssl@3)/include/" export STATIC_SSL=1 ``` diff --git a/README.rst b/README.rst index 5096c484c..3261ac834 100644 --- a/README.rst +++ b/README.rst @@ -11,7 +11,7 @@ Aerospike Python Client Compatibility ------------- -The Python client for Aerospike works with Python 3.8 - 3.12 and supports the following OS'es: +The Python client for Aerospike works with Python 3.8 - 3.13 and supports the following OS'es: * macOS 12 - 14 * CentOS 7 Linux diff --git a/VERSION b/VERSION index 5461fdb89..3dfcbc0dd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -15.0.1rc4.dev10 +16.0.0rc3 diff --git a/aerospike-client-c b/aerospike-client-c index 54e250756..5bffa81cc 160000 --- a/aerospike-client-c +++ b/aerospike-client-c @@ -1 +1 @@ -Subproject commit 54e25075646e815f023b5923603d35fa9d9b8f00 +Subproject commit 5bffa81ccf98f36951cf3d638bacea463dd27580 diff --git a/aerospike-stubs/aerospike.pyi b/aerospike-stubs/aerospike.pyi index 04589c5ba..8d4476f65 100644 --- a/aerospike-stubs/aerospike.pyi +++ b/aerospike-stubs/aerospike.pyi @@ -1,4 +1,4 @@ -from typing import Any, Callable, Union, final, Literal, Optional +from typing import Any, Callable, Union, final, Literal, Optional, Final from aerospike_helpers.batch.records import BatchRecords from aerospike_helpers.metrics import MetricsPolicy @@ -293,6 +293,25 @@ QUERY_DURATION_LONG: Literal[0] QUERY_DURATION_SHORT: Literal[1] QUERY_DURATION_LONG_RELAX_AP: Literal[2] +MRT_COMMIT_OK: Literal[0] +MRT_COMMIT_ALREADY_COMMITTED: Literal[1] +MRT_COMMIT_ALREADY_ABORTED: Literal[2] +MRT_COMMIT_VERIFY_FAILED: Literal[3] +MRT_COMMIT_MARK_ROLL_FORWARD_ABANDONED: Literal[4] +MRT_COMMIT_ROLL_FORWARD_ABANDONED: Literal[5] +MRT_COMMIT_CLOSE_ABANDONED: Literal[6] + +MRT_ABORT_OK: Literal[0] +MRT_ABORT_ALREADY_COMMITTED: Literal[1] +MRT_ABORT_ALREADY_ABORTED: Literal[2] +MRT_ABORT_ROLL_BACK_ABANDONED: Literal[3] +MRT_ABORT_CLOSE_ABANDONED: Literal[4] + +MRT_STATE_OPEN: Literal[0] +MRT_STATE_VERIFIED: Literal[1] +MRT_STATE_COMMITTED: Literal[2] +MRT_STATE_ABORTED: Literal[3] + @final class CDTInfinite: def __init__(self, *args, **kwargs) -> None: ... @@ -301,6 +320,14 @@ class CDTInfinite: class CDTWildcard: def __init__(self, *args, **kwargs) -> None: ... +@final +class Transaction: + def __init__(self, reads_capacity: int = 128, writes_capacity: int = 128) -> None: ... + id: int + in_doubt: bool + state: int + timeout: int + class Client: def __init__(self, *args, **kwargs) -> None: ... def admin_change_password(self, username: str, password: str, policy: dict = ...) -> None: ... @@ -359,48 +386,6 @@ class Client: def job_info(self, job_id: int, module, policy: dict = ...) -> dict: ... def enable_metrics(self, policy: Optional[MetricsPolicy] = None) -> None: ... def disable_metrics(self) -> None: ... - # List and map operations in the aerospike module are deprecated and undocumented - # def list_append(self, *args, **kwargs) -> Any: ... - # def list_clear(self, *args, **kwargs) -> Any: ... - # def list_extend(self, *args, **kwargs) -> Any: ... - # def list_get(self, *args, **kwargs) -> Any: ... - # def list_get_range(self, *args, **kwargs) -> Any: ... - # def list_insert(self, *args, **kwargs) -> Any: ... - # def list_insert_items(self, *args, **kwargs) -> Any: ... - # def list_pop(self, *args, **kwargs) -> Any: ... - # def list_pop_range(self, *args, **kwargs) -> Any: ... - # def list_remove(self, *args, **kwargs) -> Any: ... - # def list_remove_range(self, *args, **kwargs) -> Any: ... - # def list_set(self, *args, **kwargs) -> Any: ... - # def list_size(self, *args, **kwargs) -> Any: ... - # def list_trim(self, *args, **kwargs) -> Any: ... - # def map_clear(self, *args, **kwargs) -> Any: ... - # def map_decrement(self, *args, **kwargs) -> Any: ... - # def map_get_by_index(self, *args, **kwargs) -> Any: ... - # def map_get_by_index_range(self, *args, **kwargs) -> Any: ... - # def map_get_by_key(self, *args, **kwargs) -> Any: ... - # def map_get_by_key_list(self, *args, **kwargs) -> Any: ... - # def map_get_by_key_range(self, *args, **kwargs) -> Any: ... - # def map_get_by_rank(self, *args, **kwargs) -> Any: ... - # def map_get_by_rank_range(self, *args, **kwargs) -> Any: ... - # def map_get_by_value(self, *args, **kwargs) -> Any: ... - # def map_get_by_value_list(self, *args, **kwargs) -> Any: ... - # def map_get_by_value_range(self, *args, **kwargs) -> Any: ... - # def map_increment(self, *args, **kwargs) -> Any: ... - # def map_put(self, *args, **kwargs) -> Any: ... - # def map_put_items(self, *args, **kwargs) -> Any: ... - # def map_remove_by_index(self, *args, **kwargs) -> Any: ... - # def map_remove_by_index_range(self, *args, **kwargs) -> Any: ... - # def map_remove_by_key(self, *args, **kwargs) -> Any: ... - # def map_remove_by_key_list(self, *args, **kwargs) -> Any: ... - # def map_remove_by_key_range(self, *args, **kwargs) -> Any: ... - # def map_remove_by_rank(self, *args, **kwargs) -> Any: ... - # def map_remove_by_rank_range(self, *args, **kwargs) -> Any: ... - # def map_remove_by_value(self, *args, **kwargs) -> Any: ... - # def map_remove_by_value_list(self, *args, **kwargs) -> Any: ... - # def map_remove_by_value_range(self, *args, **kwargs) -> Any: ... - # def map_set_policy(self, key, bin, map_policy) -> Any: ... - # def map_size(self, *args, **kwargs) -> Any: ... def operate(self, key: tuple, list: list, meta: dict = ..., policy: dict = ...) -> tuple: ... def operate_ordered(self, key: tuple, list: list, meta: dict = ..., policy: dict = ...) -> list: ... def prepend(self, key: tuple, bin: str, val: str, meta: dict = ..., policy: dict = ...) -> None: ... @@ -421,6 +406,8 @@ class Client: def udf_list(self, policy: dict = ...) -> list: ... def udf_put(self, filename: str, udf_type = ..., policy: dict = ...) -> None: ... def udf_remove(self, module: str, policy: dict = ...) -> None: ... + def commit(self, transaction: Transaction) -> int: ... + def abort(self, transaction: Transaction) -> int: ... class GeoJSON: geo_data: Any diff --git a/aerospike_helpers/__init__.py b/aerospike_helpers/__init__.py index a2e4c962e..bad706b07 100644 --- a/aerospike_helpers/__init__.py +++ b/aerospike_helpers/__init__.py @@ -25,3 +25,13 @@ class HyperLogLog(bytes): """ def __new__(cls, o) -> "HyperLogLog": return super().__new__(cls, o) + + # We need to implement repr() and str() ourselves + # Otherwise, this class will inherit these methods from bytes + # making it indistinguishable from bytes objects when printed + def __repr__(self) -> str: + bytes_str = super().__repr__() + return f"{self.__class__.__name__}({bytes_str})" + + def __str__(self) -> str: + return self.__repr__() diff --git a/aerospike_helpers/expressions/base.py b/aerospike_helpers/expressions/base.py index 870b6f437..c5b2ba765 100644 --- a/aerospike_helpers/expressions/base.py +++ b/aerospike_helpers/expressions/base.py @@ -301,8 +301,8 @@ def __init__(self, bin: str): class GeoBin(_BaseExpr): - """Create an expression that returns a bin as a geojson. Returns the unknown-value - if the bin is not a geojson. + """Create an expression that returns a bin as a GeoJSON. Returns the unknown-value + if the bin is not a GeoJSON. """ _op = _ExprOp.BIN @@ -312,7 +312,7 @@ def __init__(self, bin: str): """Args: bin (str): Bin name. - :return: (geojson bin) + :return: (GeoJSON bin) Example:: diff --git a/aerospike_helpers/expressions/resources.py b/aerospike_helpers/expressions/resources.py index d54d62787..a004de30c 100644 --- a/aerospike_helpers/expressions/resources.py +++ b/aerospike_helpers/expressions/resources.py @@ -116,7 +116,7 @@ class ResultType: """ Flags used to indicate expression value_type. """ - + NIL = 0 BOOLEAN = 1 INTEGER = 2 STRING = 3 diff --git a/aerospike_helpers/metrics/__init__.py b/aerospike_helpers/metrics/__init__.py index a564149e9..e0339f25c 100644 --- a/aerospike_helpers/metrics/__init__.py +++ b/aerospike_helpers/metrics/__init__.py @@ -28,7 +28,7 @@ class ConnectionStats: """Connection statistics. Attributes: - in_use (int): Connections actively being used in database transactions on this node. + in_use (int): Connections actively being used in database commands on this node. There can be multiple pools per node. This value is a summary of those pools on this node. in_pool (int): Connections residing in pool(s) on this node. There can be multiple pools per node. This value is a summary of those pools on this node. @@ -62,10 +62,10 @@ class Node: address (str): The IP address / host name of the node (not including the port number). port (int): Port number of the node's address. conns (:py:class:`ConnectionStats`): Synchronous connection stats on this node. - error_count (int): Transaction error count since node was initialized. If the error is retryable, - multiple errors per transaction may occur. - timeout_count (int): Transaction timeout count since node was initialized. - If the timeout is retryable (ie socketTimeout), multiple timeouts per transaction may occur. + error_count (int): Command error count since node was initialized. If the error is retryable, + multiple errors per command may occur. + timeout_count (int): Command timeout count since node was initialized. + If the timeout is retryable (i.e socketTimeout), multiple timeouts per command may occur. metrics (:py:class:`NodeMetrics`): Node metrics """ pass @@ -77,8 +77,8 @@ class Cluster: Attributes: cluster_name (Optional[str]): Expected cluster name for all nodes. May be :py:obj:`None`. invalid_node_count (int): Count of add node failures in the most recent cluster tend iteration. - tran_count (int): Transaction count. The value is cumulative and not reset per metrics interval. - retry_count (int): Transaction retry count. There can be multiple retries for a single transaction. + command_count (int): Command count. The value is cumulative and not reset per metrics interval. + retry_count (int): Command retry count. There can be multiple retries for a single command. The value is cumulative and not reset per metrics interval. nodes (list[:py:class:`Node`]): Active nodes in cluster. """ diff --git a/doc/aerospike.rst b/doc/aerospike.rst index 49e4cc8e9..dbbeffc2a 100644 --- a/doc/aerospike.rst +++ b/doc/aerospike.rst @@ -1575,3 +1575,77 @@ Query Duration Treat query as a LONG query, but relax read consistency for AP namespaces. This value is treated exactly like :data:`aerospike.QUERY_DURATION_LONG` for server versions < 7.1. + +.. _mrt_commit_status_constants: + +MRT Commit Status +----------------- + +.. data:: MRT_COMMIT_OK + + Commit succeeded. + +.. data:: MRT_COMMIT_ALREADY_COMMITTED + + Transaction has already been committed. + +.. data:: MRT_COMMIT_ALREADY_ABORTED + + Transaction has already been aborted. + +.. data:: MRT_COMMIT_VERIFY_FAILED + + Transaction verify failed. Transaction will be aborted. + +.. data:: MRT_COMMIT_MARK_ROLL_FORWARD_ABANDONED + + Transaction mark roll forward abandoned. Transaction will be aborted when error is not in doubt. + If the error is in doubt (usually timeout), the commit is in doubt. + +.. data:: MRT_COMMIT_ROLL_FORWARD_ABANDONED + + Client roll forward abandoned. Server will eventually commit the transaction. + +.. data:: MRT_COMMIT_CLOSE_ABANDONED + + Transaction has been rolled forward, but client transaction close was abandoned. + Server will eventually close the transaction. + +.. _mrt_abort_status_constants: + +MRT Abort Status +---------------- + +.. data:: MRT_ABORT_OK + + Abort succeeded. + +.. data:: MRT_ABORT_ALREADY_COMMITTED + + Transaction has already been committed. + +.. data:: MRT_ABORT_ALREADY_ABORTED + + Transaction has already been aborted. + +.. data:: MRT_ABORT_ROLL_BACK_ABANDONED + + Client roll back abandoned. Server will eventually abort the transaction. + +.. data:: MRT_ABORT_CLOSE_ABANDONED + + Transaction has been rolled back, but client transaction close was abandoned. + Server will eventually close the transaction. + +.. _mrt_state: + +Multi-record Transaction State +------------------------------ + +.. data:: MRT_STATE_OPEN + +.. data:: MRT_STATE_VERIFIED + +.. data:: MRT_STATE_COMMITTED + +.. data:: MRT_STATE_ABORTED diff --git a/doc/client.rst b/doc/client.rst index bce635bd3..05c551a33 100755 --- a/doc/client.rst +++ b/doc/client.rst @@ -305,8 +305,8 @@ Batch Operations The following batch methods will return a :class:`~aerospike_helpers.batch.records.BatchRecords` object with a ``result`` value of ``0`` if one of the following is true: - * All transactions are successful. - * One or more transactions failed because: + * All commands are successful. + * One or more commands failed because: - A record was filtered out by an expression - The record was not found @@ -317,7 +317,7 @@ Batch Operations * If the underlying C client throws an error, the returned :class:`~aerospike_helpers.batch.records.BatchRecords` object will have a ``result`` value equal to an `as_status `_ error code. In this case, the :class:`~aerospike_helpers.batch.records.BatchRecords` object has a list of batch records called ``batch_records``, - and each batch record contains the result of that transaction. + and each batch record contains the result of that command. .. method:: batch_write(batch_records: BatchRecords, [policy_batch: dict]) -> BatchRecords @@ -364,7 +364,7 @@ Batch Operations .. method:: batch_operate(keys: list, ops: list, [policy_batch: dict], [policy_batch_write: dict], [ttl: int]) -> BatchRecords - Perform the same read/write transactions on multiple keys. + Perform the same read/write commands on multiple keys. .. note:: Prior to Python client 14.0.0, using the :meth:`~batch_operate()` method with only read operations caused an error. This bug was fixed in version 14.0.0. @@ -538,7 +538,7 @@ Single-Record Transactions .. method:: operate(key, list: list[, meta: dict[, policy: dict]]) -> (key, meta, bins) - Performs an atomic transaction, with multiple bin operations, against a single record with a given *key*. + Performs an atomic command, with multiple bin operations, against a single record with a given *key*. Starting with Aerospike server version 3.6.0, non-existent bins are not present in the returned :ref:`aerospike_record_tuple`. \ The returned record tuple will only contain one element per bin, even if multiple operations were performed on the bin. \ @@ -564,7 +564,7 @@ Single-Record Transactions .. method:: operate_ordered(key, list: list[, meta: dict[, policy: dict]]) -> (key, meta, bins) - Performs an atomic transaction, with multiple bin operations, against a single record with a given *key*. \ + Performs an atomic command, with multiple bin operations, against a single record with a given *key*. \ The results will be returned as a list of (bin-name, result) tuples. The order of the \ elements in the list will correspond to the order of the operations \ from the input parameters. @@ -587,6 +587,34 @@ Single-Record Transactions .. index:: single: User Defined Functions +Multi-Record Transactions +-------------------------- + +.. class:: Client + :noindex: + + .. method:: commit(transaction: aerospike.Transaction) -> int: + + Attempt to commit the given multi-record transaction. First, the expected record versions are + sent to the server nodes for verification. If all nodes return success, the transaction is + committed. Otherwise, the transaction is aborted. + + Requires server version 8.0+ + + :param transaction: Multi-record transaction. + :type transaction: :py:class:`aerospike.Transaction` + :return: The status of the commit. One of :ref:`mrt_commit_status_constants`. + + .. method:: abort(transaction: aerospike.Transaction) -> int: + + Abort and rollback the given multi-record transaction. + + Requires server version 8.0+ + + :param transaction: Multi-record transaction. + :type transaction: :py:class:`aerospike.Transaction` + :return: The status of the abort. One of :ref:`mrt_abort_status_constants`. + .. _aerospike_udf_operations: User Defined Functions @@ -1324,7 +1352,7 @@ The user dictionary has the following key-value pairs: * 0: read quota in records per second - * 1: single record read transaction rate (TPS) + * 1: single record read command rate (TPS) * 2: read scan/query record per second rate (RPS) @@ -1337,7 +1365,7 @@ The user dictionary has the following key-value pairs: * 0: write quota in records per second - * 1: single record write transaction rate (TPS) + * 1: single record write command rate (TPS) * 2: write scan/query record per second rate (RPS) @@ -1557,14 +1585,14 @@ Write Policies :columns: 1 * **max_retries** (:class:`int`) - | Maximum number of retries before aborting the current transaction. The initial attempt is not counted as a retry. + | Maximum number of retries before aborting the current command. The initial attempt is not counted as a retry. | - | If max_retries is exceeded, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. + | If max_retries is exceeded, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. | | Default: ``0`` .. warning:: Database writes that are not idempotent (such as "add") should not be retried because the write operation may be performed multiple times \ - if the client timed out previous transaction attempts. It's important to use a distinct write policy for non-idempotent writes, which sets max_retries = `0`; + if the client timed out previous command attempts. It's important to use a distinct write policy for non-idempotent writes, which sets max_retries = `0`; * **sleep_between_retries** (:class:`int`) | Milliseconds to sleep between retries. Enter ``0`` to skip sleep. @@ -1573,18 +1601,18 @@ Write Policies * **socket_timeout** (:class:`int`) | Socket idle timeout in milliseconds when processing a database command. | - | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the transaction is retried. + | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the command is retried. | | If both ``socket_timeout`` and ``total_timeout`` are non-zero and ``socket_timeout`` > ``total_timeout``, then ``socket_timeout`` will be set to ``total_timeout``. \ If ``socket_timeout`` is ``0``, there will be no socket idle limit. | | Default: ``30000`` * **total_timeout** (:class:`int`) - | Total transaction timeout in milliseconds. + | Total command timeout in milliseconds. | - | The total_timeout is tracked on the client and sent to the server along with the transaction in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the transaction. + | The total_timeout is tracked on the client and sent to the server along with the command in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the command. | - | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the transaction completes, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. + | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the command completes, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. | | Default: ``1000`` * **compress** (:class:`bool`) @@ -1605,7 +1633,7 @@ Write Policies | Default: :data:`aerospike.POLICY_EXISTS_IGNORE` * **ttl** The default time-to-live (expiration) of the record in seconds. This field will only be used if - the write transaction: + the write command: 1. Doesn't contain a metadata dictionary with a ``ttl`` value. 2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`. @@ -1624,7 +1652,7 @@ Write Policies | | Default: ``False`` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None @@ -1639,6 +1667,11 @@ Write Policies Default: :data:`aerospike.POLICY_REPLICA_SEQUENCE` + * **txn** :class:`aerospike.Transaction` + Multi-record command identifier. + + Default: :py:obj:`None` + .. _aerospike_read_policies: Read Policies @@ -1652,9 +1685,9 @@ Read Policies :columns: 1 * **max_retries** (:class:`int`) - | Maximum number of retries before aborting the current transaction. The initial attempt is not counted as a retry. + | Maximum number of retries before aborting the current command. The initial attempt is not counted as a retry. | - | If max_retries is exceeded, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. + | If max_retries is exceeded, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. | | Default: ``2`` * **sleep_between_retries** (:class:`int`) @@ -1664,17 +1697,17 @@ Read Policies * **socket_timeout** (:class:`int`) | Socket idle timeout in milliseconds when processing a database command. | - | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the transaction is retried. + | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the command is retried. | | If both ``socket_timeout`` and ``total_timeout`` are non-zero and ``socket_timeout`` > ``total_timeout``, then ``socket_timeout`` will be set to ``total_timeout``. If ``socket_timeout`` is ``0``, there will be no socket idle limit. | | Default: ``30000`` * **total_timeout** (:class:`int`) - | Total transaction timeout in milliseconds. + | Total command timeout in milliseconds. | - | The total_timeout is tracked on the client and sent to the server along with the transaction in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the transaction. + | The total_timeout is tracked on the client and sent to the server along with the command in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the command. | - | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the transaction completes, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. + | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the command completes, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. | | Default: ``1000`` * **compress** (:class:`bool`) @@ -1732,12 +1765,17 @@ Read Policies | | Default: ``aerospike.POLICY_REPLICA_SEQUENCE`` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None .. note:: Requires Aerospike server version >= 5.2. + * **txn** :class:`aerospike.Transaction` + Multi-record command identifier. + + Default: :py:obj:`None` + .. _aerospike_operate_policies: Operate Policies @@ -1751,14 +1789,14 @@ Operate Policies :columns: 1 * **max_retries** (:class:`int`) - | Maximum number of retries before aborting the current transaction. The initial attempt is not counted as a retry. + | Maximum number of retries before aborting the current command. The initial attempt is not counted as a retry. | - | If max_retries is exceeded, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. + | If max_retries is exceeded, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. | | Default: ``0`` .. warning:: Database writes that are not idempotent (such as "add") should not be retried because the write operation may be performed multiple times \ - if the client timed out previous transaction attempts. It's important to use a distinct write policy for non-idempotent writes, which sets max_retries = `0`; + if the client timed out previous command attempts. It's important to use a distinct write policy for non-idempotent writes, which sets max_retries = `0`; * **sleep_between_retries** (:class:`int`) | Milliseconds to sleep between retries. Enter ``0`` to skip sleep. @@ -1767,17 +1805,17 @@ Operate Policies * **socket_timeout** (:class:`int`) | Socket idle timeout in milliseconds when processing a database command. | - | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the transaction is retried. + | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the command is retried. | | If both ``socket_timeout`` and ``total_timeout`` are non-zero and ``socket_timeout`` > ``total_timeout``, then ``socket_timeout`` will be set to ``total_timeout``. If ``socket_timeout`` is ``0``, there will be no socket idle limit. | | Default: ``30000`` * **total_timeout** (:class:`int`) - | Total transaction timeout in milliseconds. + | Total command timeout in milliseconds. | - | The total_timeout is tracked on the client and sent to the server along with the transaction in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the transaction. + | The total_timeout is tracked on the client and sent to the server along with the command in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the command. | - | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the transaction completes, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. + | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the command completes, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. | | Default: ``1000`` * **compress** (:class:`bool`) @@ -1798,7 +1836,7 @@ Operate Policies | Default: :data:`aerospike.POLICY_GEN_IGNORE` * **ttl** (:class:`int`) The default time-to-live (expiration) of the record in seconds. This field will only be used if an - operate transaction contains a write operation and either: + operate command contains a write operation and either: 1. Doesn't contain a metadata dictionary with a ``ttl`` value. 2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`. @@ -1860,11 +1898,15 @@ Operate Policies | | Default: :py:obj:`True` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None .. note:: Requires Aerospike server version >= 5.2. + * **txn** :class:`aerospike.Transaction` + Multi-record command identifier. + + Default: :py:obj:`None` .. _aerospike_apply_policies: @@ -1879,14 +1921,14 @@ Apply Policies :columns: 1 * **max_retries** (:class:`int`) - | Maximum number of retries before aborting the current transaction. The initial attempt is not counted as a retry. + | Maximum number of retries before aborting the current command. The initial attempt is not counted as a retry. | - | If max_retries is exceeded, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. + | If max_retries is exceeded, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. | | Default: ``0`` .. warning:: Database writes that are not idempotent (such as "add") should not be retried because the write operation may be performed multiple times \ - if the client timed out previous transaction attempts. It's important to use a distinct write policy for non-idempotent writes, which sets max_retries = `0`; + if the client timed out previous command attempts. It's important to use a distinct write policy for non-idempotent writes, which sets max_retries = `0`; * **sleep_between_retries** (:class:`int`) | Milliseconds to sleep between retries. Enter ``0`` to skip sleep. @@ -1895,17 +1937,17 @@ Apply Policies * **socket_timeout** (:class:`int`) | Socket idle timeout in milliseconds when processing a database command. | - | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the transaction is retried. + | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the command is retried. | | If both ``socket_timeout`` and ``total_timeout`` are non-zero and ``socket_timeout`` > ``total_timeout``, then ``socket_timeout`` will be set to ``total_timeout``. If ``socket_timeout`` is ``0``, there will be no socket idle limit. | | Default: ``30000`` * **total_timeout** (:class:`int`) - | Total transaction timeout in milliseconds. + | Total command timeout in milliseconds. | - | The total_timeout is tracked on the client and sent to the server along with the transaction in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the transaction. + | The total_timeout is tracked on the client and sent to the server along with the command in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the command. | - | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the transaction completes, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. + | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the command completes, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. | | Default: ``1000`` * **compress** (:class:`bool`) @@ -1930,7 +1972,7 @@ Apply Policies | Default: :data:`aerospike.POLICY_COMMIT_LEVEL_ALL` * **ttl** (:class:`int`) The default time-to-live (expiration) of the record in seconds. This field will only be used if an apply - transaction doesn't have an apply policy with a ``ttl`` value that overrides this field. + command doesn't have an apply policy with a ``ttl`` value that overrides this field. There are also special values that can be set for this field. See :ref:`TTL_CONSTANTS`. * **durable_delete** (:class:`bool`) @@ -1938,11 +1980,15 @@ Apply Policies | | Default: ``False`` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None .. note:: Requires Aerospike server version >= 5.2. + * **txn** :class:`aerospike.Transaction` + Multi-record command identifier. + + Default: :py:obj:`None` .. _aerospike_remove_policies: @@ -1958,14 +2004,14 @@ Remove Policies :columns: 1 * **max_retries** (:class:`int`) - | Maximum number of retries before aborting the current transaction. The initial attempt is not counted as a retry. + | Maximum number of retries before aborting the current command. The initial attempt is not counted as a retry. | - | If max_retries is exceeded, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. + | If max_retries is exceeded, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. | | Default: ``0`` .. warning:: Database writes that are not idempotent (such as "add") should not be retried because the write operation may be performed multiple times \ - if the client timed out previous transaction attempts. It's important to use a distinct write policy for non-idempotent writes, which sets max_retries = `0`; + if the client timed out previous command attempts. It's important to use a distinct write policy for non-idempotent writes, which sets max_retries = `0`; * **sleep_between_retries** (:class:`int`) | Milliseconds to sleep between retries. Enter ``0`` to skip sleep. @@ -1973,17 +2019,17 @@ Remove Policies * **socket_timeout** (:class:`int`) | Socket idle timeout in milliseconds when processing a database command. | - | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the transaction is retried. + | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the command is retried. | | If both ``socket_timeout`` and ``total_timeout`` are non-zero and ``socket_timeout`` > ``total_timeout``, then ``socket_timeout`` will be set to ``total_timeout``. If ``socket_timeout`` is ``0``, there will be no socket idle limit. | | Default: ``30000`` * **total_timeout** (:class:`int`) - | Total transaction timeout in milliseconds. + | Total command timeout in milliseconds. | - | The total_timeout is tracked on the client and sent to the server along with the transaction in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the transaction. + | The total_timeout is tracked on the client and sent to the server along with the command in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the command. | - | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the transaction completes, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. + | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the command completes, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. | | Default: ``1000`` * **compress** (:class:`bool`) @@ -2021,11 +2067,15 @@ Remove Policies | Default: ``aerospike.POLICY_REPLICA_SEQUENCE`` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None .. note:: Requires Aerospike server version >= 5.2. + * **txn** :class:`aerospike.Transaction` + Multi-record command identifier. + + Default: :py:obj:`None` .. _aerospike_batch_policies: @@ -2040,9 +2090,9 @@ Batch Policies :columns: 1 * **max_retries** (:class:`int`) - | Maximum number of retries before aborting the current transaction. The initial attempt is not counted as a retry. + | Maximum number of retries before aborting the current command. The initial attempt is not counted as a retry. | - | If max_retries is exceeded, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. + | If max_retries is exceeded, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. | | Default: ``2`` @@ -2055,17 +2105,17 @@ Batch Policies * **socket_timeout** (:class:`int`) | Socket idle timeout in milliseconds when processing a database command. | - | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the transaction is retried. + | If socket_timeout is not ``0`` and the socket has been idle for at least socket_timeout, both max_retries and total_timeout are checked. If max_retries and total_timeout are not exceeded, the command is retried. | | If both ``socket_timeout`` and ``total_timeout`` are non-zero and ``socket_timeout`` > ``total_timeout``, then ``socket_timeout`` will be set to ``total_timeout``. If ``socket_timeout`` is ``0``, there will be no socket idle limit. | | Default: ``30000`` * **total_timeout** (:class:`int`) - | Total transaction timeout in milliseconds. + | Total command timeout in milliseconds. | - | The total_timeout is tracked on the client and sent to the server along with the transaction in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the transaction. + | The total_timeout is tracked on the client and sent to the server along with the command in the wire protocol. The client will most likely timeout first, but the server also has the capability to timeout the command. | - | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the transaction completes, the transaction will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. + | If ``total_timeout`` is not ``0`` and ``total_timeout`` is reached before the command completes, the command will return error ``AEROSPIKE_ERR_TIMEOUT``. If ``total_timeout`` is ``0``, there will be no total time limit. | | Default: ``1000`` * **compress** (:class:`bool`) @@ -2119,7 +2169,7 @@ Batch Policies | | Default ``False`` * **allow_inline** (:class:`bool`) - | Allow batch to be processed immediately in the server's receiving thread when the server deems it to be appropriate. If `False`, the batch will always be processed in separate transaction threads. This field is only relevant for the new batch index protocol. + | Allow batch to be processed immediately in the server's receiving thread when the server deems it to be appropriate. If `False`, the batch will always be processed in separate service threads. This field is only relevant for the new batch index protocol. | | Default ``True`` * **allow_inline_ssd** (:class:`bool`) @@ -2136,7 +2186,7 @@ Batch Policies | | Default: ``True`` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None @@ -2158,6 +2208,10 @@ Batch Policies Server versions < 6.0 do not support this field and treat this value as false for key specific errors. Default: ``True`` + * **txn** :class:`aerospike.Transaction` + Multi-record command identifier. + + Default: :py:obj:`None` .. _aerospike_batch_write_policies: @@ -2192,7 +2246,7 @@ Batch Write Policies | | Default: ``False`` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None * **ttl** :class:`int` @@ -2241,11 +2295,11 @@ Batch Apply Policies | | Default: ``0`` * **durable_delete** :class:`bool` - | If the transaction results in a record deletion, leave a tombstone for the record. This prevents deleted records from reappearing after node failures. Valid for Aerospike Server Enterprise Edition only. + | If the command results in a record deletion, leave a tombstone for the record. This prevents deleted records from reappearing after node failures. Valid for Aerospike Server Enterprise Edition only. | | Default: :py:obj:`False` (do not tombstone deleted records). * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None @@ -2282,7 +2336,7 @@ Batch Remove Policies | | Default: ``False`` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None @@ -2307,7 +2361,7 @@ Batch Read Policies | | Default: :data:`aerospike.POLICY_READ_MODE_SC_SESSION` * **expressions** :class:`list` - | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a transaction. + | Compiled aerospike expressions :mod:`aerospike_helpers` used for filtering records within a command. | | Default: None * **read_touch_ttl_percent** @@ -2520,8 +2574,8 @@ Role Objects * ``"privileges"``: a :class:`list` of :ref:`aerospike_privilege_dict`. * ``"whitelist"``: a :class:`list` of IP address strings. - * ``"read_quota"``: a :class:`int` representing the allowed read transactions per second. - * ``"write_quota"``: a :class:`int` representing the allowed write transactions per second. + * ``"read_quota"``: a :class:`int` representing the allowed read commands per second. + * ``"write_quota"``: a :class:`int` representing the allowed write commands per second. .. _aerospike_privilege_dict: diff --git a/doc/conf.py b/doc/conf.py index 0560d2c30..7bae041fa 100755 --- a/doc/conf.py +++ b/doc/conf.py @@ -255,7 +255,7 @@ def __getattr__(cls, name): # Spelling check -spelling_ignore_pypi_package_names = True +spelling_ignore_pypi_package_names = False linkcheck_anchors_ignore = [ 'truncate', diff --git a/doc/data_mapping.rst b/doc/data_mapping.rst index dfd43a50c..e38a1cae7 100644 --- a/doc/data_mapping.rst +++ b/doc/data_mapping.rst @@ -69,7 +69,7 @@ For server 7.1 and higher, map keys can only be of type string, bytes, and integ :ref:`KeyOrderedDict ` is a special case. Like :class:`dict`, :class:`~aerospike.KeyOrderedDict` maps to the Aerospike map data type. \ However, the map will be sorted in key order before being sent to the server (see :ref:`aerospike_map_order`). -It is possible to nest these datatypes. For example a list may contain a dictionary, or a dictionary may contain a list +It is possible to nest these data types. For example a list may contain a dictionary, or a dictionary may contain a list as a value. .. _integer: https://aerospike.com/docs/server/guide/data-types/scalar-data-types#integer diff --git a/doc/exception.rst b/doc/exception.rst index 42cedf180..4ad2d547b 100644 --- a/doc/exception.rst +++ b/doc/exception.rst @@ -298,6 +298,14 @@ Server Errors Subclass of :py:exc:`~aerospike.exception.ServerError`. +.. py:exception:: QueryAbortedError + + Query was aborted. + + Error code: ``210`` + + Subclass of :py:exc:`~aerospike.exception.ServerError`. + Record Errors ------------- @@ -397,14 +405,6 @@ Record Errors Subclass of :py:exc:`~aerospike.exception.RecordError`. -.. py:exception:: QueryAbortedError - - Query was aborted. - - Error code: ``210`` - - Subclass of :py:exc:`~aerospike.exception.ClientError`. - Index Errors ------------ @@ -487,7 +487,7 @@ Query Errors Error code: ``213`` - Subclass of :py:exc:`~aerospike.exception.AerospikeError`. + Subclass of :py:exc:`~aerospike.exception.ServerError`. Cluster Errors -------------- @@ -507,7 +507,7 @@ Cluster Errors Error code: ``11`` - Subclass of :py:exc:`~aerospike.exception.AerospikeError`. + Subclass of :py:exc:`~aerospike.exception.ServerError`. Admin Errors ------------ @@ -516,6 +516,8 @@ Admin Errors The parent class for exceptions of the security API. + Subclass of :py:exc:`~aerospike.exception.ServerError`. + .. py:exception:: SecurityNotSupported Security functionality not supported by connected server. diff --git a/doc/index.rst b/doc/index.rst index d12c2d389..406223815 100755 --- a/doc/index.rst +++ b/doc/index.rst @@ -59,6 +59,7 @@ Class Description :ref:`aerospike.Query` Handles queries over secondary indexes. :ref:`aerospike.geojson` Handles GeoJSON type data. :ref:`aerospike.KeyOrderedDict` Key ordered dictionary +:ref:`aerospike.Transaction` Multi-record transaction ================================= =========== In addition, the :ref:`Data_Mapping` page explains how **Python** types map to **Aerospike Server** types. @@ -79,6 +80,7 @@ Content query geojson key_ordered_dict + transaction predicates exception aerospike_helpers diff --git a/doc/spelling_wordlist.txt b/doc/spelling_wordlist.txt index 6a03b1afe..19d2c4ba2 100644 --- a/doc/spelling_wordlist.txt +++ b/doc/spelling_wordlist.txt @@ -3,6 +3,7 @@ lua namespace geospatial serialized +serializers Serializers deserialized Aerospike @@ -76,3 +77,13 @@ subtransactions socketTimeout ttl inlined +hashmaps +config +deserialize +async +retryable +backoff +msg +func +bcrypt +namespaces diff --git a/doc/transaction.rst b/doc/transaction.rst new file mode 100644 index 000000000..220cacbac --- /dev/null +++ b/doc/transaction.rst @@ -0,0 +1,63 @@ +.. _aerospike.Transaction: + +.. currentmodule:: aerospike + +================================================================= +:class:`aerospike.Transaction` --- Multi Record Transaction Class +================================================================= + +In the whole documentation, "commands" are all database commands that can be sent to the server (e.g single-record +operations, batch, scans, or queries). + +"Transactions" link individual commands (except for scan and query) together so they can be rolled forward or +backward consistently. + +Methods +======= + +.. class:: Transaction + + Initialize multi-record transaction (MRT), assign random transaction id and initialize + reads/writes hashmaps with default capacities. + + For both parameters, an unsigned 32-bit integer must be passed and the minimum value should be 16. + + :param reads_capacity: expected number of record reads in the MRT. Defaults to ``128``. + :type reads_capacity: int, optional + :param writes_capacity: expected number of record writes in the MRT. Defaults to ``128``. + :type writes_capacity: int, optional + + .. py:attribute:: id + + Get the random transaction id that was generated on class instance creation. + + The value is an unsigned 64-bit integer. + + This attribute is read-only. + + :type: int + .. py:attribute:: in_doubt + + This attribute is read-only. + + :type: bool + .. py:attribute:: state + + One of the :ref:`mrt_state` constants. + + This attribute is read-only. + + :type: int + .. py:attribute:: timeout + + MRT timeout in seconds. The timer starts when the MRT monitor record is created. + This occurs when the first command in the MRT is executed. If the timeout is reached before + :py:meth:`~aerospike.Client.commit` or :py:meth:`~aerospike.Client.abort` is called, the server will expire and + rollback the MRT. + + The default client MRT timeout is zero. This means use the server configuration ``mrt-duration`` + as the MRT timeout. The default ``mrt-duration`` is 10 seconds. + + This attribute can be read and written to. + + :type: int diff --git a/pyproject.toml b/pyproject.toml index b4d871c69..0b9e0238b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Database" ] diff --git a/setup.py b/setup.py index 8e09dd151..567b41e60 100644 --- a/setup.py +++ b/setup.py @@ -373,7 +373,9 @@ def clean(): 'src/main/client/batch_remove.c', 'src/main/client/batch_apply.c', 'src/main/client/batch_read.c', - 'src/main/client/metrics.c' + 'src/main/client/metrics.c', + 'src/main/transaction/type.c', + 'src/main/client/mrt.c' ], # Compile diff --git a/src/include/client.h b/src/include/client.h index 3f442e014..d4f3b140a 100644 --- a/src/include/client.h +++ b/src/include/client.h @@ -601,3 +601,10 @@ int check_type(AerospikeClient *self, PyObject *py_value, int op, PyObject *AerospikeClient_Truncate(AerospikeClient *self, PyObject *args, PyObject *kwds); + +// MRT + +PyObject *AerospikeClient_Commit(AerospikeClient *self, PyObject *args, + PyObject *kwds); +PyObject *AerospikeClient_Abort(AerospikeClient *self, PyObject *args, + PyObject *kwds); diff --git a/src/include/exceptions.h b/src/include/exceptions.h index 2079018bb..f2778307d 100644 --- a/src/include/exceptions.h +++ b/src/include/exceptions.h @@ -22,3 +22,5 @@ PyObject *AerospikeException_New(void); void raise_exception(as_error *err); PyObject *raise_exception_old(as_error *err); void remove_exception(as_error *err); +void set_aerospike_exc_attrs_using_tuple_of_attrs(PyObject *py_exc, + PyObject *py_tuple); diff --git a/src/include/log.h b/src/include/log.h index 9e8c8209f..fa87b13cd 100644 --- a/src/include/log.h +++ b/src/include/log.h @@ -19,20 +19,6 @@ #include #include -/* - * Enum to declare log level constants - */ -typedef enum Aerospike_log_level_e { - - LOG_LEVEL_OFF = -1, - LOG_LEVEL_ERROR, - LOG_LEVEL_WARN, - LOG_LEVEL_INFO, - LOG_LEVEL_DEBUG, - LOG_LEVEL_TRACE - -} aerospike_log_level; - /* * Structure to hold user's log_callback object */ @@ -40,12 +26,6 @@ typedef struct Aerospike_log_callback { PyObject *callback; } AerospikeLogCallback; -/** - * Add log level constants to aerospike module - * aerospike.set_log_level(aerospike.LOG_LEVEL_DEBUG) - */ -as_status declare_log_constants(PyObject *aerospike); - /** * Set log level for C-SDK * aerospike.set_log_level( aerospike.LOG_LEVEL_WARN ) diff --git a/src/include/policy.h b/src/include/policy.h index 3f594f43d..0ba3135ad 100644 --- a/src/include/policy.h +++ b/src/include/policy.h @@ -29,14 +29,6 @@ #include #include -#define MAX_CONSTANT_STR_SIZE 512 - -/* - ******************************************************************************************************* - *Structure to map constant number to constant name string for Aerospike constants. - ******************************************************************************************************* - */ - enum Aerospike_serializer_values { SERIALIZER_NONE, /* default handler for serializer type */ SERIALIZER_PYTHON, @@ -193,20 +185,6 @@ enum aerospike_cdt_ctx_identifiers { CDT_CTX_MAP_KEY_CREATE = 0x24 }; -typedef struct Aerospike_Constants { - long constantno; - char constant_str[MAX_CONSTANT_STR_SIZE]; -} AerospikeConstants; - -typedef struct Aerospike_JobConstants { - char job_str[MAX_CONSTANT_STR_SIZE]; - char exposed_job_str[MAX_CONSTANT_STR_SIZE]; -} AerospikeJobConstants; -#define AEROSPIKE_CONSTANTS_ARR_SIZE \ - (sizeof(aerospike_constants) / sizeof(AerospikeConstants)) -#define AEROSPIKE_JOB_CONSTANTS_ARR_SIZE \ - (sizeof(aerospike_job_constants) / sizeof(AerospikeJobConstants)) - as_status pyobject_to_policy_admin(AerospikeClient *self, as_error *err, PyObject *py_policy, as_policy_admin *policy, as_policy_admin **policy_p, @@ -270,8 +248,6 @@ as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, as_status pyobject_to_map_policy(as_error *err, PyObject *py_policy, as_map_policy *policy); -as_status declare_policy_constants(PyObject *aerospike); - void set_scan_options(as_error *err, as_scan *scan_p, PyObject *py_options); as_status set_query_options(as_error *err, PyObject *query_options, diff --git a/src/include/transaction.h b/src/include/transaction.h new file mode 100644 index 000000000..d4dc15543 --- /dev/null +++ b/src/include/transaction.h @@ -0,0 +1,3 @@ +#include + +PyTypeObject *AerospikeTransaction_Ready(); diff --git a/src/include/types.h b/src/include/types.h index 125dd3e6f..319794a93 100644 --- a/src/include/types.h +++ b/src/include/types.h @@ -25,8 +25,12 @@ #include #include #include +#include #include "pool.h" +#define AEROSPIKE_MODULE_NAME "aerospike" +#define FULLY_QUALIFIED_TYPE_NAME(name) AEROSPIKE_MODULE_NAME "." name + // Bin names can be of type Unicode in Python // DB supports 32767 maximum number of bins #define MAX_UNICODE_OBJECTS 32767 @@ -96,3 +100,11 @@ typedef struct { typedef struct { PyDictObject dict; } AerospikeKeyOrderedDict; + +typedef struct { + PyObject_HEAD + /* Type-specific fields go here. */ + as_txn *txn; +} AerospikeTransaction; + +extern PyTypeObject AerospikeTransaction_Type; diff --git a/src/main/aerospike.c b/src/main/aerospike.c index ea517248b..e8d2fef03 100644 --- a/src/main/aerospike.c +++ b/src/main/aerospike.c @@ -28,12 +28,19 @@ #include "exceptions.h" #include "policy.h" #include "log.h" -#include #include "serializer.h" #include "module_functions.h" #include "nullobject.h" #include "cdt_types.h" +#include "transaction.h" + +#include #include +#include +#include +#include +#include +#include PyObject *py_global_hosts; int counter = 0xA8000000; @@ -50,7 +57,7 @@ config = {\n\ }\n\ client = aerospike.client(config)"); -static PyMethodDef Aerospike_Methods[] = { +static PyMethodDef aerospike_methods[] = { //Serialization {"set_serializer", (PyCFunction)AerospikeClient_Set_Serializer, @@ -83,183 +90,561 @@ static PyMethodDef Aerospike_Methods[] = { {NULL}}; -static AerospikeConstants operator_constants[] = { - {AS_OPERATOR_READ, "OPERATOR_READ"}, - {AS_OPERATOR_WRITE, "OPERATOR_WRITE"}, - {AS_OPERATOR_INCR, "OPERATOR_INCR"}, - {AS_OPERATOR_APPEND, "OPERATOR_APPEND"}, - {AS_OPERATOR_PREPEND, "OPERATOR_PREPEND"}, - {AS_OPERATOR_TOUCH, "OPERATOR_TOUCH"}, - {AS_OPERATOR_DELETE, "OPERATOR_DELETE"}}; - -#define OPERATOR_CONSTANTS_ARR_SIZE \ - (sizeof(operator_constants) / sizeof(AerospikeConstants)) - -static AerospikeConstants auth_mode_constants[] = { - {AS_AUTH_INTERNAL, "AUTH_INTERNAL"}, - {AS_AUTH_EXTERNAL, "AUTH_EXTERNAL"}, - {AS_AUTH_EXTERNAL_INSECURE, "AUTH_EXTERNAL_INSECURE"}, - {AS_AUTH_PKI, "AUTH_PKI"}}; - -#define AUTH_MODE_CONSTANTS_ARR_SIZE \ - (sizeof(auth_mode_constants) / sizeof(AerospikeConstants)) - -struct Aerospike_State { - PyObject *exception; - PyTypeObject *client; - PyTypeObject *query; - PyTypeObject *scan; - PyTypeObject *kdict; - PyObject *predicates; - PyTypeObject *geospatial; - PyTypeObject *null_object; - PyTypeObject *wildcard_object; - PyTypeObject *infinite_object; +struct module_constant_name_to_value { + const char *name; + // If false, is int value + bool is_str_value; + union value { + long integer; + const char *string; + } value; }; -#define Aerospike_State(o) ((struct Aerospike_State *)PyModule_GetState(o)) - -static int Aerospike_Clear(PyObject *aerospike) -{ - Py_CLEAR(Aerospike_State(aerospike)->exception); - Py_CLEAR(Aerospike_State(aerospike)->client); - Py_CLEAR(Aerospike_State(aerospike)->query); - Py_CLEAR(Aerospike_State(aerospike)->scan); - Py_CLEAR(Aerospike_State(aerospike)->kdict); - Py_CLEAR(Aerospike_State(aerospike)->predicates); - Py_CLEAR(Aerospike_State(aerospike)->geospatial); - Py_CLEAR(Aerospike_State(aerospike)->null_object); - Py_CLEAR(Aerospike_State(aerospike)->wildcard_object); - Py_CLEAR(Aerospike_State(aerospike)->infinite_object); - - return 0; -} +// TODO: many of these names are the same as the enum name +// Is there a way to generate this code? +// TODO: regression tests for all these constants +static struct module_constant_name_to_value module_constants[] = { + {"OPERATOR_READ", .value.integer = AS_OPERATOR_READ}, + {"OPERATOR_WRITE", .value.integer = AS_OPERATOR_WRITE}, + {"OPERATOR_INCR", .value.integer = AS_OPERATOR_INCR}, + {"OPERATOR_APPEND", .value.integer = AS_OPERATOR_APPEND}, + {"OPERATOR_PREPEND", .value.integer = AS_OPERATOR_PREPEND}, + {"OPERATOR_TOUCH", .value.integer = AS_OPERATOR_TOUCH}, + {"OPERATOR_DELETE", .value.integer = AS_OPERATOR_DELETE}, + + {"AUTH_INTERNAL", .value.integer = AS_AUTH_INTERNAL}, + {"AUTH_EXTERNAL", .value.integer = AS_AUTH_EXTERNAL}, + {"AUTH_EXTERNAL_INSECURE", .value.integer = AS_AUTH_EXTERNAL_INSECURE}, + {"AUTH_PKI", .value.integer = AS_AUTH_PKI}, + + {"POLICY_RETRY_NONE", .value.integer = AS_POLICY_RETRY_NONE}, + {"POLICY_RETRY_ONCE", .value.integer = AS_POLICY_RETRY_ONCE}, + + {"POLICY_EXISTS_IGNORE", .value.integer = AS_POLICY_EXISTS_IGNORE}, + {"POLICY_EXISTS_CREATE", .value.integer = AS_POLICY_EXISTS_CREATE}, + {"POLICY_EXISTS_UPDATE", .value.integer = AS_POLICY_EXISTS_UPDATE}, + {"POLICY_EXISTS_REPLACE", .value.integer = AS_POLICY_EXISTS_REPLACE}, + {"POLICY_EXISTS_CREATE_OR_REPLACE", + .value.integer = AS_POLICY_EXISTS_CREATE_OR_REPLACE}, + + {"UDF_TYPE_LUA", .value.integer = AS_UDF_TYPE_LUA}, + + {"POLICY_KEY_DIGEST", .value.integer = AS_POLICY_KEY_DIGEST}, + {"POLICY_KEY_SEND", .value.integer = AS_POLICY_KEY_SEND}, + {"POLICY_GEN_IGNORE", .value.integer = AS_POLICY_GEN_IGNORE}, + {"POLICY_GEN_EQ", .value.integer = AS_POLICY_GEN_EQ}, + {"POLICY_GEN_GT", .value.integer = AS_POLICY_GEN_GT}, + + {"JOB_STATUS_COMPLETED", .value.integer = AS_JOB_STATUS_COMPLETED}, + {"JOB_STATUS_UNDEF", .value.integer = AS_JOB_STATUS_UNDEF}, + {"JOB_STATUS_INPROGRESS", .value.integer = AS_JOB_STATUS_INPROGRESS}, + + {"POLICY_REPLICA_MASTER", .value.integer = AS_POLICY_REPLICA_MASTER}, + {"POLICY_REPLICA_ANY", .value.integer = AS_POLICY_REPLICA_ANY}, + {"POLICY_REPLICA_SEQUENCE", .value.integer = AS_POLICY_REPLICA_SEQUENCE}, + {"POLICY_REPLICA_PREFER_RACK", + .value.integer = AS_POLICY_REPLICA_PREFER_RACK}, + + {"POLICY_COMMIT_LEVEL_ALL", .value.integer = AS_POLICY_COMMIT_LEVEL_ALL}, + {"POLICY_COMMIT_LEVEL_MASTER", + .value.integer = AS_POLICY_COMMIT_LEVEL_MASTER}, + + {"SERIALIZER_USER", .value.integer = SERIALIZER_USER}, + {"SERIALIZER_JSON", .value.integer = SERIALIZER_JSON}, + {"SERIALIZER_NONE", .value.integer = SERIALIZER_NONE}, + + {"INTEGER", .value.integer = SEND_BOOL_AS_INTEGER}, + {"AS_BOOL", .value.integer = SEND_BOOL_AS_AS_BOOL}, + + {"INDEX_STRING", .value.integer = AS_INDEX_STRING}, + {"INDEX_NUMERIC", .value.integer = AS_INDEX_NUMERIC}, + {"INDEX_GEO2DSPHERE", .value.integer = AS_INDEX_GEO2DSPHERE}, + {"INDEX_BLOB", .value.integer = AS_INDEX_BLOB}, + {"INDEX_TYPE_DEFAULT", .value.integer = AS_INDEX_TYPE_DEFAULT}, + {"INDEX_TYPE_LIST", .value.integer = AS_INDEX_TYPE_LIST}, + {"INDEX_TYPE_MAPKEYS", .value.integer = AS_INDEX_TYPE_MAPKEYS}, + {"INDEX_TYPE_MAPVALUES", .value.integer = AS_INDEX_TYPE_MAPVALUES}, + + {"PRIV_USER_ADMIN", .value.integer = AS_PRIVILEGE_USER_ADMIN}, + {"PRIV_SYS_ADMIN", .value.integer = AS_PRIVILEGE_SYS_ADMIN}, + {"PRIV_DATA_ADMIN", .value.integer = AS_PRIVILEGE_DATA_ADMIN}, + {"PRIV_READ", .value.integer = AS_PRIVILEGE_READ}, + {"PRIV_WRITE", .value.integer = AS_PRIVILEGE_WRITE}, + {"PRIV_READ_WRITE", .value.integer = AS_PRIVILEGE_READ_WRITE}, + {"PRIV_READ_WRITE_UDF", .value.integer = AS_PRIVILEGE_READ_WRITE_UDF}, + {"PRIV_TRUNCATE", .value.integer = AS_PRIVILEGE_TRUNCATE}, + {"PRIV_UDF_ADMIN", .value.integer = AS_PRIVILEGE_UDF_ADMIN}, + {"PRIV_SINDEX_ADMIN", .value.integer = AS_PRIVILEGE_SINDEX_ADMIN}, + + // TODO: If only aerospike_helpers relies on these constants, + // maybe move these constants to aerospike_helpers package + {"OP_LIST_APPEND", .value.integer = OP_LIST_APPEND}, + {"OP_LIST_APPEND_ITEMS", .value.integer = OP_LIST_APPEND_ITEMS}, + {"OP_LIST_INSERT", .value.integer = OP_LIST_INSERT}, + {"OP_LIST_INSERT_ITEMS", .value.integer = OP_LIST_INSERT_ITEMS}, + {"OP_LIST_POP", .value.integer = OP_LIST_POP}, + {"OP_LIST_POP_RANGE", .value.integer = OP_LIST_POP_RANGE}, + {"OP_LIST_REMOVE", .value.integer = OP_LIST_REMOVE}, + {"OP_LIST_REMOVE_RANGE", .value.integer = OP_LIST_REMOVE_RANGE}, + {"OP_LIST_CLEAR", .value.integer = OP_LIST_CLEAR}, + {"OP_LIST_SET", .value.integer = OP_LIST_SET}, + {"OP_LIST_GET", .value.integer = OP_LIST_GET}, + {"OP_LIST_GET_RANGE", .value.integer = OP_LIST_GET_RANGE}, + {"OP_LIST_TRIM", .value.integer = OP_LIST_TRIM}, + {"OP_LIST_SIZE", .value.integer = OP_LIST_SIZE}, + {"OP_LIST_INCREMENT", .value.integer = OP_LIST_INCREMENT}, + /* New CDT Operations, post 3.16.0.1 */ + {"OP_LIST_GET_BY_INDEX", .value.integer = OP_LIST_GET_BY_INDEX}, + {"OP_LIST_GET_BY_INDEX_RANGE", .value.integer = OP_LIST_GET_BY_INDEX_RANGE}, + {"OP_LIST_GET_BY_RANK", .value.integer = OP_LIST_GET_BY_RANK}, + {"OP_LIST_GET_BY_RANK_RANGE", .value.integer = OP_LIST_GET_BY_RANK_RANGE}, + {"OP_LIST_GET_BY_VALUE", .value.integer = OP_LIST_GET_BY_VALUE}, + {"OP_LIST_GET_BY_VALUE_LIST", .value.integer = OP_LIST_GET_BY_VALUE_LIST}, + {"OP_LIST_GET_BY_VALUE_RANGE", .value.integer = OP_LIST_GET_BY_VALUE_RANGE}, + {"OP_LIST_REMOVE_BY_INDEX", .value.integer = OP_LIST_REMOVE_BY_INDEX}, + {"OP_LIST_REMOVE_BY_INDEX_RANGE", + .value.integer = OP_LIST_REMOVE_BY_INDEX_RANGE}, + {"OP_LIST_REMOVE_BY_RANK", .value.integer = OP_LIST_REMOVE_BY_RANK}, + {"OP_LIST_REMOVE_BY_RANK_RANGE", + .value.integer = OP_LIST_REMOVE_BY_RANK_RANGE}, + {"OP_LIST_REMOVE_BY_VALUE", .value.integer = OP_LIST_REMOVE_BY_VALUE}, + {"OP_LIST_REMOVE_BY_VALUE_LIST", + .value.integer = OP_LIST_REMOVE_BY_VALUE_LIST}, + {"OP_LIST_REMOVE_BY_VALUE_RANGE", + .value.integer = OP_LIST_REMOVE_BY_VALUE_RANGE}, + {"OP_LIST_SET_ORDER", .value.integer = OP_LIST_SET_ORDER}, + {"OP_LIST_SORT", .value.integer = OP_LIST_SORT}, + { + "OP_LIST_REMOVE_BY_VALUE_RANK_RANGE_REL", + + .value.integer = OP_LIST_REMOVE_BY_VALUE_RANK_RANGE_REL, + }, + { + "OP_LIST_GET_BY_VALUE_RANK_RANGE_REL", + + .value.integer = OP_LIST_GET_BY_VALUE_RANK_RANGE_REL, + }, + {"OP_LIST_CREATE", .value.integer = OP_LIST_CREATE}, + { + "OP_LIST_GET_BY_VALUE_RANK_RANGE_REL_TO_END", + + .value.integer = OP_LIST_GET_BY_VALUE_RANK_RANGE_REL_TO_END, + }, + {"OP_LIST_GET_BY_INDEX_RANGE_TO_END", + .value.integer = OP_LIST_GET_BY_INDEX_RANGE_TO_END}, + {"OP_LIST_GET_BY_RANK_RANGE_TO_END", + .value.integer = OP_LIST_GET_BY_RANK_RANGE_TO_END}, + {"OP_LIST_REMOVE_BY_REL_RANK_RANGE_TO_END", + .value.integer = OP_LIST_REMOVE_BY_REL_RANK_RANGE_TO_END}, + {"OP_LIST_REMOVE_BY_REL_RANK_RANGE", + .value.integer = OP_LIST_REMOVE_BY_REL_RANK_RANGE}, + {"OP_LIST_REMOVE_BY_INDEX_RANGE_TO_END", + .value.integer = OP_LIST_REMOVE_BY_INDEX_RANGE_TO_END}, + {"OP_LIST_REMOVE_BY_RANK_RANGE_TO_END", + .value.integer = OP_LIST_REMOVE_BY_RANK_RANGE_TO_END}, + + {"OP_MAP_SET_POLICY", .value.integer = OP_MAP_SET_POLICY}, + {"OP_MAP_CREATE", .value.integer = OP_MAP_CREATE}, + {"OP_MAP_PUT", .value.integer = OP_MAP_PUT}, + {"OP_MAP_PUT_ITEMS", .value.integer = OP_MAP_PUT_ITEMS}, + {"OP_MAP_INCREMENT", .value.integer = OP_MAP_INCREMENT}, + {"OP_MAP_DECREMENT", .value.integer = OP_MAP_DECREMENT}, + {"OP_MAP_SIZE", .value.integer = OP_MAP_SIZE}, + {"OP_MAP_CLEAR", .value.integer = OP_MAP_CLEAR}, + {"OP_MAP_REMOVE_BY_KEY", .value.integer = OP_MAP_REMOVE_BY_KEY}, + {"OP_MAP_REMOVE_BY_KEY_LIST", .value.integer = OP_MAP_REMOVE_BY_KEY_LIST}, + {"OP_MAP_REMOVE_BY_KEY_RANGE", .value.integer = OP_MAP_REMOVE_BY_KEY_RANGE}, + {"OP_MAP_REMOVE_BY_VALUE", .value.integer = OP_MAP_REMOVE_BY_VALUE}, + {"OP_MAP_REMOVE_BY_VALUE_LIST", + .value.integer = OP_MAP_REMOVE_BY_VALUE_LIST}, + {"OP_MAP_REMOVE_BY_VALUE_RANGE", + .value.integer = OP_MAP_REMOVE_BY_VALUE_RANGE}, + {"OP_MAP_REMOVE_BY_INDEX", .value.integer = OP_MAP_REMOVE_BY_INDEX}, + {"OP_MAP_REMOVE_BY_INDEX_RANGE", + .value.integer = OP_MAP_REMOVE_BY_INDEX_RANGE}, + {"OP_MAP_REMOVE_BY_RANK", .value.integer = OP_MAP_REMOVE_BY_RANK}, + {"OP_MAP_REMOVE_BY_RANK_RANGE", + .value.integer = OP_MAP_REMOVE_BY_RANK_RANGE}, + {"OP_MAP_GET_BY_KEY", .value.integer = OP_MAP_GET_BY_KEY}, + {"OP_MAP_GET_BY_KEY_RANGE", .value.integer = OP_MAP_GET_BY_KEY_RANGE}, + {"OP_MAP_GET_BY_VALUE", .value.integer = OP_MAP_GET_BY_VALUE}, + {"OP_MAP_GET_BY_VALUE_RANGE", .value.integer = OP_MAP_GET_BY_VALUE_RANGE}, + {"OP_MAP_GET_BY_INDEX", .value.integer = OP_MAP_GET_BY_INDEX}, + {"OP_MAP_GET_BY_INDEX_RANGE", .value.integer = OP_MAP_GET_BY_INDEX_RANGE}, + {"OP_MAP_GET_BY_RANK", .value.integer = OP_MAP_GET_BY_RANK}, + {"OP_MAP_GET_BY_RANK_RANGE", .value.integer = OP_MAP_GET_BY_RANK_RANGE}, + {"OP_MAP_GET_BY_VALUE_LIST", .value.integer = OP_MAP_GET_BY_VALUE_LIST}, + {"OP_MAP_GET_BY_KEY_LIST", .value.integer = OP_MAP_GET_BY_KEY_LIST}, + /* CDT operations for use with expressions, new in 5.0 */ + {"OP_MAP_REMOVE_BY_VALUE_RANK_RANGE_REL", + .value.integer = OP_MAP_REMOVE_BY_VALUE_RANK_RANGE_REL}, + {"OP_MAP_REMOVE_BY_KEY_INDEX_RANGE_REL", + .value.integer = OP_MAP_REMOVE_BY_KEY_INDEX_RANGE_REL}, + {"OP_MAP_GET_BY_VALUE_RANK_RANGE_REL", + .value.integer = OP_MAP_GET_BY_VALUE_RANK_RANGE_REL}, + {"OP_MAP_GET_BY_KEY_INDEX_RANGE_REL", + .value.integer = OP_MAP_GET_BY_KEY_INDEX_RANGE_REL}, + {"OP_MAP_REMOVE_BY_KEY_REL_INDEX_RANGE_TO_END", + .value.integer = OP_MAP_REMOVE_BY_KEY_REL_INDEX_RANGE_TO_END}, + {"OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE_TO_END", + .value.integer = OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE_TO_END}, + {"OP_MAP_REMOVE_BY_INDEX_RANGE_TO_END", + .value.integer = OP_MAP_REMOVE_BY_INDEX_RANGE_TO_END}, + {"OP_MAP_REMOVE_BY_RANK_RANGE_TO_END", + .value.integer = OP_MAP_REMOVE_BY_RANK_RANGE_TO_END}, + {"OP_MAP_GET_BY_KEY_REL_INDEX_RANGE_TO_END", + .value.integer = OP_MAP_GET_BY_KEY_REL_INDEX_RANGE_TO_END}, + {"OP_MAP_REMOVE_BY_KEY_REL_INDEX_RANGE", + .value.integer = OP_MAP_REMOVE_BY_KEY_REL_INDEX_RANGE}, + {"OP_MAP_REMOVE_BY_VALUE_REL_INDEX_RANGE", + .value.integer = OP_MAP_REMOVE_BY_VALUE_REL_INDEX_RANGE}, + {"OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE", + .value.integer = OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE}, + {"OP_MAP_GET_BY_KEY_REL_INDEX_RANGE", + .value.integer = OP_MAP_GET_BY_KEY_REL_INDEX_RANGE}, + {"OP_MAP_GET_BY_VALUE_RANK_RANGE_REL_TO_END", + .value.integer = OP_MAP_GET_BY_VALUE_RANK_RANGE_REL_TO_END}, + {"OP_MAP_GET_BY_INDEX_RANGE_TO_END", + .value.integer = OP_MAP_GET_BY_INDEX_RANGE_TO_END}, + {"OP_MAP_GET_BY_RANK_RANGE_TO_END", + .value.integer = OP_MAP_GET_BY_RANK_RANGE_TO_END}, + + {"MAP_UNORDERED", .value.integer = AS_MAP_UNORDERED}, + {"MAP_KEY_ORDERED", .value.integer = AS_MAP_KEY_ORDERED}, + {"MAP_KEY_VALUE_ORDERED", .value.integer = AS_MAP_KEY_VALUE_ORDERED}, + + {"MAP_RETURN_NONE", .value.integer = AS_MAP_RETURN_NONE}, + {"MAP_RETURN_INDEX", .value.integer = AS_MAP_RETURN_INDEX}, + {"MAP_RETURN_REVERSE_INDEX", .value.integer = AS_MAP_RETURN_REVERSE_INDEX}, + {"MAP_RETURN_RANK", .value.integer = AS_MAP_RETURN_RANK}, + {"MAP_RETURN_REVERSE_RANK", .value.integer = AS_MAP_RETURN_REVERSE_RANK}, + {"MAP_RETURN_COUNT", .value.integer = AS_MAP_RETURN_COUNT}, + {"MAP_RETURN_KEY", .value.integer = AS_MAP_RETURN_KEY}, + {"MAP_RETURN_VALUE", .value.integer = AS_MAP_RETURN_VALUE}, + {"MAP_RETURN_KEY_VALUE", .value.integer = AS_MAP_RETURN_KEY_VALUE}, + {"MAP_RETURN_EXISTS", .value.integer = AS_MAP_RETURN_EXISTS}, + {"MAP_RETURN_ORDERED_MAP", .value.integer = AS_MAP_RETURN_ORDERED_MAP}, + {"MAP_RETURN_UNORDERED_MAP", .value.integer = AS_MAP_RETURN_UNORDERED_MAP}, + + {"TTL_NAMESPACE_DEFAULT", .value.integer = AS_RECORD_DEFAULT_TTL}, + {"TTL_NEVER_EXPIRE", .value.integer = AS_RECORD_NO_EXPIRE_TTL}, + {"TTL_DONT_UPDATE", .value.integer = AS_RECORD_NO_CHANGE_TTL}, + {"TTL_CLIENT_DEFAULT", .value.integer = AS_RECORD_CLIENT_DEFAULT_TTL}, + + {"LIST_RETURN_NONE", .value.integer = AS_LIST_RETURN_NONE}, + {"LIST_RETURN_INDEX", .value.integer = AS_LIST_RETURN_INDEX}, + {"LIST_RETURN_REVERSE_INDEX", + .value.integer = AS_LIST_RETURN_REVERSE_INDEX}, + {"LIST_RETURN_RANK", .value.integer = AS_LIST_RETURN_RANK}, + {"LIST_RETURN_REVERSE_RANK", .value.integer = AS_LIST_RETURN_REVERSE_RANK}, + {"LIST_RETURN_COUNT", .value.integer = AS_LIST_RETURN_COUNT}, + {"LIST_RETURN_VALUE", .value.integer = AS_LIST_RETURN_VALUE}, + {"LIST_RETURN_EXISTS", .value.integer = AS_LIST_RETURN_EXISTS}, + + {"LIST_SORT_DROP_DUPLICATES", + .value.integer = AS_LIST_SORT_DROP_DUPLICATES}, + {"LIST_SORT_DEFAULT", .value.integer = AS_LIST_SORT_DEFAULT}, + + {"LIST_WRITE_DEFAULT", .value.integer = AS_LIST_WRITE_DEFAULT}, + {"LIST_WRITE_ADD_UNIQUE", .value.integer = AS_LIST_WRITE_ADD_UNIQUE}, + {"LIST_WRITE_INSERT_BOUNDED", + .value.integer = AS_LIST_WRITE_INSERT_BOUNDED}, + + {"LIST_ORDERED", .value.integer = AS_LIST_ORDERED}, + {"LIST_UNORDERED", .value.integer = AS_LIST_UNORDERED}, + + {"MAP_WRITE_NO_FAIL", .value.integer = AS_MAP_WRITE_NO_FAIL}, + {"MAP_WRITE_PARTIAL", .value.integer = AS_MAP_WRITE_PARTIAL}, + + {"LIST_WRITE_NO_FAIL", .value.integer = AS_LIST_WRITE_NO_FAIL}, + {"LIST_WRITE_PARTIAL", .value.integer = AS_LIST_WRITE_PARTIAL}, + + /* Map write flags post 3.5.0 */ + {"MAP_WRITE_FLAGS_DEFAULT", .value.integer = AS_MAP_WRITE_DEFAULT}, + {"MAP_WRITE_FLAGS_CREATE_ONLY", .value.integer = AS_MAP_WRITE_CREATE_ONLY}, + {"MAP_WRITE_FLAGS_UPDATE_ONLY", .value.integer = AS_MAP_WRITE_UPDATE_ONLY}, + {"MAP_WRITE_FLAGS_NO_FAIL", .value.integer = AS_MAP_WRITE_NO_FAIL}, + {"MAP_WRITE_FLAGS_PARTIAL", .value.integer = AS_MAP_WRITE_PARTIAL}, + + /* READ Mode constants 4.0.0 */ + + // AP Read Mode + {"POLICY_READ_MODE_AP_ONE", .value.integer = AS_POLICY_READ_MODE_AP_ONE}, + {"POLICY_READ_MODE_AP_ALL", .value.integer = AS_POLICY_READ_MODE_AP_ALL}, + + // SC Read Mode + {"POLICY_READ_MODE_SC_SESSION", + .value.integer = AS_POLICY_READ_MODE_SC_SESSION}, + {"POLICY_READ_MODE_SC_LINEARIZE", + .value.integer = AS_POLICY_READ_MODE_SC_LINEARIZE}, + {"POLICY_READ_MODE_SC_ALLOW_REPLICA", + .value.integer = AS_POLICY_READ_MODE_SC_ALLOW_REPLICA}, + {"POLICY_READ_MODE_SC_ALLOW_UNAVAILABLE", + .value.integer = AS_POLICY_READ_MODE_SC_ALLOW_UNAVAILABLE}, + + /* Bitwise constants: 3.9.0 */ + {"BIT_WRITE_DEFAULT", .value.integer = AS_BIT_WRITE_DEFAULT}, + {"BIT_WRITE_CREATE_ONLY", .value.integer = AS_BIT_WRITE_CREATE_ONLY}, + {"BIT_WRITE_UPDATE_ONLY", .value.integer = AS_BIT_WRITE_UPDATE_ONLY}, + {"BIT_WRITE_NO_FAIL", .value.integer = AS_BIT_WRITE_NO_FAIL}, + {"BIT_WRITE_PARTIAL", .value.integer = AS_BIT_WRITE_PARTIAL}, + + {"BIT_RESIZE_DEFAULT", .value.integer = AS_BIT_RESIZE_DEFAULT}, + {"BIT_RESIZE_FROM_FRONT", .value.integer = AS_BIT_RESIZE_FROM_FRONT}, + {"BIT_RESIZE_GROW_ONLY", .value.integer = AS_BIT_RESIZE_GROW_ONLY}, + {"BIT_RESIZE_SHRINK_ONLY", .value.integer = AS_BIT_RESIZE_SHRINK_ONLY}, + + {"BIT_OVERFLOW_FAIL", .value.integer = AS_BIT_OVERFLOW_FAIL}, + {"BIT_OVERFLOW_SATURATE", .value.integer = AS_BIT_OVERFLOW_SATURATE}, + {"BIT_OVERFLOW_WRAP", .value.integer = AS_BIT_OVERFLOW_WRAP}, + + /* BITWISE OPS: 3.9.0 */ + {"OP_BIT_INSERT", .value.integer = OP_BIT_INSERT}, + {"OP_BIT_RESIZE", .value.integer = OP_BIT_RESIZE}, + {"OP_BIT_REMOVE", .value.integer = OP_BIT_REMOVE}, + {"OP_BIT_SET", .value.integer = OP_BIT_SET}, + {"OP_BIT_OR", .value.integer = OP_BIT_OR}, + {"OP_BIT_XOR", .value.integer = OP_BIT_XOR}, + {"OP_BIT_AND", .value.integer = OP_BIT_AND}, + {"OP_BIT_NOT", .value.integer = OP_BIT_NOT}, + {"OP_BIT_LSHIFT", .value.integer = OP_BIT_LSHIFT}, + {"OP_BIT_RSHIFT", .value.integer = OP_BIT_RSHIFT}, + {"OP_BIT_ADD", .value.integer = OP_BIT_ADD}, + {"OP_BIT_SUBTRACT", .value.integer = OP_BIT_SUBTRACT}, + {"OP_BIT_GET_INT", .value.integer = OP_BIT_GET_INT}, + {"OP_BIT_SET_INT", .value.integer = OP_BIT_SET_INT}, + {"OP_BIT_GET", .value.integer = OP_BIT_GET}, + {"OP_BIT_COUNT", .value.integer = OP_BIT_COUNT}, + {"OP_BIT_LSCAN", .value.integer = OP_BIT_LSCAN}, + {"OP_BIT_RSCAN", .value.integer = OP_BIT_RSCAN}, + + /* Nested CDT constants: 3.9.0 */ + {"CDT_CTX_LIST_INDEX", .value.integer = AS_CDT_CTX_LIST_INDEX}, + {"CDT_CTX_LIST_RANK", .value.integer = AS_CDT_CTX_LIST_RANK}, + {"CDT_CTX_LIST_VALUE", .value.integer = AS_CDT_CTX_LIST_VALUE}, + {"CDT_CTX_LIST_INDEX_CREATE", .value.integer = CDT_CTX_LIST_INDEX_CREATE}, + {"CDT_CTX_MAP_INDEX", .value.integer = AS_CDT_CTX_MAP_INDEX}, + {"CDT_CTX_MAP_RANK", .value.integer = AS_CDT_CTX_MAP_RANK}, + {"CDT_CTX_MAP_KEY", .value.integer = AS_CDT_CTX_MAP_KEY}, + {"CDT_CTX_MAP_VALUE", .value.integer = AS_CDT_CTX_MAP_VALUE}, + {"CDT_CTX_MAP_KEY_CREATE", .value.integer = CDT_CTX_MAP_KEY_CREATE}, + + /* HLL constants 3.11.0 */ + {"OP_HLL_ADD", .value.integer = OP_HLL_ADD}, + {"OP_HLL_DESCRIBE", .value.integer = OP_HLL_DESCRIBE}, + {"OP_HLL_FOLD", .value.integer = OP_HLL_FOLD}, + {"OP_HLL_GET_COUNT", .value.integer = OP_HLL_GET_COUNT}, + {"OP_HLL_GET_INTERSECT_COUNT", .value.integer = OP_HLL_GET_INTERSECT_COUNT}, + {"OP_HLL_GET_SIMILARITY", .value.integer = OP_HLL_GET_SIMILARITY}, + {"OP_HLL_GET_UNION", .value.integer = OP_HLL_GET_UNION}, + {"OP_HLL_GET_UNION_COUNT", .value.integer = OP_HLL_GET_UNION_COUNT}, + {"OP_HLL_GET_SIMILARITY", .value.integer = OP_HLL_GET_SIMILARITY}, + {"OP_HLL_INIT", .value.integer = OP_HLL_INIT}, + {"OP_HLL_REFRESH_COUNT", .value.integer = OP_HLL_REFRESH_COUNT}, + {"OP_HLL_SET_UNION", .value.integer = OP_HLL_SET_UNION}, + {"OP_HLL_MAY_CONTAIN", + .value.integer = OP_HLL_MAY_CONTAIN}, // for expression filters + + {"HLL_WRITE_DEFAULT", .value.integer = AS_HLL_WRITE_DEFAULT}, + {"HLL_WRITE_CREATE_ONLY", .value.integer = AS_HLL_WRITE_CREATE_ONLY}, + {"HLL_WRITE_UPDATE_ONLY", .value.integer = AS_HLL_WRITE_UPDATE_ONLY}, + {"HLL_WRITE_NO_FAIL", .value.integer = AS_HLL_WRITE_NO_FAIL}, + {"HLL_WRITE_ALLOW_FOLD", .value.integer = AS_HLL_WRITE_ALLOW_FOLD}, + + /* Expression operation constants 5.1.0 */ + {"OP_EXPR_READ", .value.integer = OP_EXPR_READ}, + {"OP_EXPR_WRITE", .value.integer = OP_EXPR_WRITE}, + {"EXP_WRITE_DEFAULT", .value.integer = AS_EXP_WRITE_DEFAULT}, + {"EXP_WRITE_CREATE_ONLY", .value.integer = AS_EXP_WRITE_CREATE_ONLY}, + {"EXP_WRITE_UPDATE_ONLY", .value.integer = AS_EXP_WRITE_UPDATE_ONLY}, + {"EXP_WRITE_ALLOW_DELETE", .value.integer = AS_EXP_WRITE_ALLOW_DELETE}, + {"EXP_WRITE_POLICY_NO_FAIL", .value.integer = AS_EXP_WRITE_POLICY_NO_FAIL}, + {"EXP_WRITE_EVAL_NO_FAIL", .value.integer = AS_EXP_WRITE_EVAL_NO_FAIL}, + {"EXP_READ_DEFAULT", .value.integer = AS_EXP_READ_DEFAULT}, + {"EXP_READ_EVAL_NO_FAIL", .value.integer = AS_EXP_READ_EVAL_NO_FAIL}, + + /* For BinType expression, as_bytes_type */ + {"AS_BYTES_UNDEF", .value.integer = AS_BYTES_UNDEF}, + {"AS_BYTES_INTEGER", .value.integer = AS_BYTES_INTEGER}, + {"AS_BYTES_DOUBLE", .value.integer = AS_BYTES_DOUBLE}, + {"AS_BYTES_STRING", .value.integer = AS_BYTES_STRING}, + {"AS_BYTES_BLOB", .value.integer = AS_BYTES_BLOB}, + {"AS_BYTES_JAVA", .value.integer = AS_BYTES_JAVA}, + {"AS_BYTES_CSHARP", .value.integer = AS_BYTES_CSHARP}, + {"AS_BYTES_PYTHON", .value.integer = AS_BYTES_PYTHON}, + {"AS_BYTES_RUBY", .value.integer = AS_BYTES_RUBY}, + {"AS_BYTES_PHP", .value.integer = AS_BYTES_PHP}, + {"AS_BYTES_ERLANG", .value.integer = AS_BYTES_ERLANG}, + {"AS_BYTES_BOOL", .value.integer = AS_BYTES_BOOL}, + {"AS_BYTES_HLL", .value.integer = AS_BYTES_HLL}, + {"AS_BYTES_MAP", .value.integer = AS_BYTES_MAP}, + {"AS_BYTES_LIST", .value.integer = AS_BYTES_LIST}, + {"AS_BYTES_GEOJSON", .value.integer = AS_BYTES_GEOJSON}, + {"AS_BYTES_TYPE_MAX", .value.integer = AS_BYTES_TYPE_MAX}, + + /* Regex constants from predexp, still used by expressions */ + {"REGEX_NONE", .value.integer = REGEX_NONE}, + {"REGEX_EXTENDED", .value.integer = REGEX_EXTENDED}, + {"REGEX_ICASE", .value.integer = REGEX_ICASE}, + {"REGEX_NOSUB", .value.integer = REGEX_NOSUB}, + {"REGEX_NEWLINE", .value.integer = REGEX_NEWLINE}, + + {"QUERY_DURATION_LONG", .value.integer = AS_QUERY_DURATION_LONG}, + {"QUERY_DURATION_LONG_RELAX_AP", + .value.integer = AS_QUERY_DURATION_LONG_RELAX_AP}, + {"QUERY_DURATION_SHORT", .value.integer = AS_QUERY_DURATION_SHORT}, + + {"LOG_LEVEL_OFF", .value.integer = -1}, + {"LOG_LEVEL_ERROR", .value.integer = AS_LOG_LEVEL_ERROR}, + {"LOG_LEVEL_WARN", .value.integer = AS_LOG_LEVEL_WARN}, + {"LOG_LEVEL_INFO", .value.integer = AS_LOG_LEVEL_INFO}, + {"LOG_LEVEL_DEBUG", .value.integer = AS_LOG_LEVEL_DEBUG}, + {"LOG_LEVEL_TRACE", .value.integer = AS_LOG_LEVEL_TRACE}, + + {"MRT_COMMIT_OK", .value.integer = AS_COMMIT_OK}, + {"MRT_COMMIT_ALREADY_COMMITTED", + .value.integer = AS_COMMIT_ALREADY_COMMITTED}, + {"MRT_COMMIT_ALREADY_ABORTED", .value.integer = AS_COMMIT_ALREADY_ABORTED}, + {"MRT_COMMIT_VERIFY_FAILED", .value.integer = AS_COMMIT_VERIFY_FAILED}, + {"MRT_COMMIT_MARK_ROLL_FORWARD_ABANDONED", + .value.integer = AS_COMMIT_MARK_ROLL_FORWARD_ABANDONED}, + {"MRT_COMMIT_ROLL_FORWARD_ABANDONED", + .value.integer = AS_COMMIT_ROLL_FORWARD_ABANDONED}, + {"MRT_COMMIT_CLOSE_ABANDONED", .value.integer = AS_COMMIT_CLOSE_ABANDONED}, + + {"MRT_ABORT_OK", .value.integer = AS_ABORT_OK}, + {"MRT_ABORT_ALREADY_COMMITTED", + .value.integer = AS_ABORT_ALREADY_COMMITTED}, + {"MRT_ABORT_ALREADY_ABORTED", .value.integer = AS_ABORT_ALREADY_ABORTED}, + {"MRT_ABORT_ROLL_BACK_ABANDONED", + .value.integer = AS_ABORT_ROLL_BACK_ABANDONED}, + {"MRT_ABORT_CLOSE_ABANDONED", .value.integer = AS_ABORT_CLOSE_ABANDONED}, + + {"MRT_STATE_OPEN", .value.integer = AS_TXN_STATE_OPEN}, + {"MRT_STATE_VERIFIED", .value.integer = AS_TXN_STATE_VERIFIED}, + {"MRT_STATE_COMMITTED", .value.integer = AS_TXN_STATE_COMMITTED}, + {"MRT_STATE_ABORTED", .value.integer = AS_TXN_STATE_ABORTED}, + + {"JOB_SCAN", .is_str_value = true, .value.string = "scan"}, + {"JOB_QUERY", .is_str_value = true, .value.string = "query"}}; + +struct submodule_name_to_creation_method { + const char *name; + PyObject *(*pyobject_creation_method)(void); +}; -PyMODINIT_FUNC PyInit_aerospike(void) -{ - // Makes things "thread-safe" - Py_Initialize(); - int i = 0; +static struct submodule_name_to_creation_method py_submodules[] = { + // We don't use module's __name__ attribute + // because the modules' __name__ is the fully qualified name which includes the package name + {"exception", AerospikeException_New}, + {"predicates", AerospikePredicates_New}, +}; - static struct PyModuleDef moduledef = {PyModuleDef_HEAD_INIT, - "aerospike", - "Aerospike Python Client", - sizeof(struct Aerospike_State), - Aerospike_Methods, - NULL, - NULL, - Aerospike_Clear}; +struct type_name_to_creation_method { + const char *name; + PyTypeObject *(*pytype_ready_method)(void); +}; - PyObject *aerospike = PyModule_Create(&moduledef); +static struct type_name_to_creation_method py_module_types[] = { + // We also don't retrieve the type's __name__ because: + // 1. Some of the objects have names different from the class name when accessed from the package + // 2. We don't want to deal with extracting an object's __name__ from a Unicode object. + // We have to make sure the Unicode object lives as long as we need its internal buffer + // It's easier to just use a C string directly + {"Client", AerospikeClient_Ready}, + {"Query", AerospikeQuery_Ready}, + {"Scan", AerospikeScan_Ready}, + {"KeyOrderedDict", AerospikeKeyOrderedDict_Ready}, + {"GeoJSON", AerospikeGeospatial_Ready}, + {"null", AerospikeNullObject_Ready}, + {"CDTWildcard", AerospikeWildcardObject_Ready}, + {"CDTInfinite", AerospikeInfiniteObject_Ready}, + {"Transaction", AerospikeTransaction_Ready}, +}; - // In case adding objects to module fails, we can properly deallocate the module state later - memset(Aerospike_State(aerospike), 0, sizeof(struct Aerospike_State)); +PyMODINIT_FUNC PyInit_aerospike(void) +{ + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + .m_name = AEROSPIKE_MODULE_NAME, + .m_doc = "Aerospike Python Client", + .m_methods = aerospike_methods, + .m_size = -1, + }; + + PyObject *py_aerospike_module = PyModule_Create(&moduledef); + if (py_aerospike_module == NULL) { + return NULL; + } Aerospike_Enable_Default_Logging(); py_global_hosts = PyDict_New(); - - PyObject *exception = AerospikeException_New(); - Py_INCREF(exception); - int retval = PyModule_AddObject(aerospike, "exception", exception); - if (retval == -1) { - goto CLEANUP; + if (py_global_hosts == NULL) { + goto MODULE_CLEANUP_ON_ERROR; } - Aerospike_State(aerospike)->exception = exception; - PyTypeObject *client = AerospikeClient_Ready(); - Py_INCREF(client); - retval = PyModule_AddObject(aerospike, "Client", (PyObject *)client); - if (retval == -1) { - goto CLEANUP; + unsigned long i = 0; + int retval; + for (i = 0; i < sizeof(py_submodules) / sizeof(py_submodules[0]); i++) { + PyObject *(*create_py_submodule)(void) = + py_submodules[i].pyobject_creation_method; + PyObject *py_submodule = create_py_submodule(); + if (py_submodule == NULL) { + goto GLOBAL_HOSTS_CLEANUP_ON_ERROR; + } + + retval = PyModule_AddObject(py_aerospike_module, py_submodules[i].name, + py_submodule); + if (retval == -1) { + Py_DECREF(py_submodule); + goto GLOBAL_HOSTS_CLEANUP_ON_ERROR; + } } - Aerospike_State(aerospike)->client = client; - PyTypeObject *query = AerospikeQuery_Ready(); - Py_INCREF(query); - retval = PyModule_AddObject(aerospike, "Query", (PyObject *)query); - if (retval == -1) { - goto CLEANUP; + for (i = 0; i < sizeof(py_module_types) / sizeof(py_module_types[0]); i++) { + PyTypeObject *(*py_type_ready_func)(void) = + py_module_types[i].pytype_ready_method; + PyTypeObject *py_type = py_type_ready_func(); + if (py_type == NULL) { + goto GLOBAL_HOSTS_CLEANUP_ON_ERROR; + } + + Py_INCREF(py_type); + retval = PyModule_AddObject( + py_aerospike_module, py_module_types[i].name, (PyObject *)py_type); + if (retval == -1) { + Py_DECREF(py_type); + goto GLOBAL_HOSTS_CLEANUP_ON_ERROR; + } } - Aerospike_State(aerospike)->query = query; - - PyTypeObject *scan = AerospikeScan_Ready(); - Py_INCREF(scan); - retval = PyModule_AddObject(aerospike, "Scan", (PyObject *)scan); - if (retval == -1) { - goto CLEANUP; - } - Aerospike_State(aerospike)->scan = scan; - - PyTypeObject *kdict = AerospikeKeyOrderedDict_Ready(); - Py_INCREF(kdict); - retval = PyModule_AddObject(aerospike, "KeyOrderedDict", (PyObject *)kdict); - if (retval == -1) { - goto CLEANUP; - } - Aerospike_State(aerospike)->kdict = kdict; /* * Add constants to module. */ - for (i = 0; i < (int)OPERATOR_CONSTANTS_ARR_SIZE; i++) { - PyModule_AddIntConstant(aerospike, operator_constants[i].constant_str, - operator_constants[i].constantno); - } - - for (i = 0; i < (int)AUTH_MODE_CONSTANTS_ARR_SIZE; i++) { - PyModule_AddIntConstant(aerospike, auth_mode_constants[i].constant_str, - auth_mode_constants[i].constantno); - } - - declare_policy_constants(aerospike); - declare_log_constants(aerospike); - - PyObject *predicates = AerospikePredicates_New(); - Py_INCREF(predicates); - retval = PyModule_AddObject(aerospike, "predicates", predicates); - if (retval == -1) { - goto CLEANUP; - } - Aerospike_State(aerospike)->predicates = predicates; - - PyTypeObject *geospatial = AerospikeGeospatial_Ready(); - Py_INCREF(geospatial); - retval = PyModule_AddObject(aerospike, "GeoJSON", (PyObject *)geospatial); - if (retval == -1) { - goto CLEANUP; - } - Aerospike_State(aerospike)->geospatial = geospatial; - - PyTypeObject *null_object = AerospikeNullObject_Ready(); - Py_INCREF(null_object); - retval = PyModule_AddObject(aerospike, "null", (PyObject *)null_object); - if (retval == -1) { - goto CLEANUP; - } - Aerospike_State(aerospike)->null_object = null_object; - - PyTypeObject *wildcard_object = AerospikeWildcardObject_Ready(); - Py_INCREF(wildcard_object); - retval = PyModule_AddObject(aerospike, "CDTWildcard", - (PyObject *)wildcard_object); - if (retval == -1) { - goto CLEANUP; - } - Aerospike_State(aerospike)->wildcard_object = wildcard_object; - - PyTypeObject *infinite_object = AerospikeInfiniteObject_Ready(); - Py_INCREF(infinite_object); - retval = PyModule_AddObject(aerospike, "CDTInfinite", - (PyObject *)infinite_object); - if (retval == -1) { - goto CLEANUP; + for (i = 0; i < sizeof(module_constants) / sizeof(module_constants[0]); + i++) { + if (module_constants[i].is_str_value == false) { + retval = PyModule_AddIntConstant(py_aerospike_module, + module_constants[i].name, + module_constants[i].value.integer); + } + else { + retval = PyModule_AddStringConstant( + py_aerospike_module, module_constants[i].name, + module_constants[i].value.string); + } + + if (retval == -1) { + goto GLOBAL_HOSTS_CLEANUP_ON_ERROR; + } } - Aerospike_State(aerospike)->infinite_object = infinite_object; - return aerospike; + return py_aerospike_module; -CLEANUP: - Aerospike_Clear(aerospike); +GLOBAL_HOSTS_CLEANUP_ON_ERROR: + Py_DECREF(py_global_hosts); +MODULE_CLEANUP_ON_ERROR: + Py_DECREF(py_aerospike_module); return NULL; } diff --git a/src/main/client/apply.c b/src/main/client/apply.c index 118140ac9..e342241cf 100644 --- a/src/main/client/apply.c +++ b/src/main/client/apply.c @@ -171,6 +171,7 @@ PyObject *AerospikeClient_Apply_Invoke(AerospikeClient *self, PyObject *py_key, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } diff --git a/src/main/client/exists.c b/src/main/client/exists.c index 383e3a430..d835b324a 100644 --- a/src/main/client/exists.c +++ b/src/main/client/exists.c @@ -139,6 +139,7 @@ extern PyObject *AerospikeClient_Exists_Invoke(AerospikeClient *self, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } diff --git a/src/main/client/get.c b/src/main/client/get.c index 88083795f..41a43f525 100644 --- a/src/main/client/get.c +++ b/src/main/client/get.c @@ -136,6 +136,7 @@ PyObject *AerospikeClient_Get_Invoke(AerospikeClient *self, PyObject *py_key, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } diff --git a/src/main/client/info.c b/src/main/client/info.c index 2cdc55687..be6095b60 100644 --- a/src/main/client/info.c +++ b/src/main/client/info.c @@ -99,6 +99,7 @@ static bool AerospikeClient_InfoAll_each(as_error *err, const as_node *node, PyObject *py_err = NULL; error_to_pyobject(&udata_ptr->error, &py_err); PyObject *exception_type = raise_exception_old(&udata_ptr->error); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); PyErr_SetObject(exception_type, py_err); Py_DECREF(py_err); PyGILState_Release(gil_state); @@ -108,6 +109,7 @@ static bool AerospikeClient_InfoAll_each(as_error *err, const as_node *node, PyObject *py_err = NULL; error_to_pyobject(err, &py_err); PyObject *exception_type = raise_exception_old(err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); PyErr_SetObject(exception_type, py_err); Py_DECREF(py_err); PyGILState_Release(gil_state); @@ -213,6 +215,7 @@ static PyObject *AerospikeClient_InfoAll_Invoke(AerospikeClient *self, error_to_pyobject(&info_callback_udata.error, &py_err); PyObject *exception_type = raise_exception_old(&info_callback_udata.error); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); PyErr_SetObject(exception_type, py_err); Py_DECREF(py_err); if (py_nodes) { diff --git a/src/main/client/mrt.c b/src/main/client/mrt.c new file mode 100644 index 000000000..d49bd43c4 --- /dev/null +++ b/src/main/client/mrt.c @@ -0,0 +1,74 @@ +#include + +#include +#include "exceptions.h" +#include "types.h" +#include "client.h" + +PyObject *AerospikeClient_Commit(AerospikeClient *self, PyObject *args, + PyObject *kwds) +{ + AerospikeTransaction *py_transaction = NULL; + + static char *kwlist[] = {"transaction", NULL}; + + if (PyArg_ParseTupleAndKeywords(args, kwds, "O!:commit", kwlist, + &AerospikeTransaction_Type, + (PyObject **)(&py_transaction)) == false) { + return NULL; + } + + as_error err; + as_error_init(&err); + + as_commit_status status; + + Py_BEGIN_ALLOW_THREADS + aerospike_commit(self->as, &err, py_transaction->txn, &status); + Py_END_ALLOW_THREADS + + if (err.code != AEROSPIKE_OK) { + raise_exception(&err); + return NULL; + } + + PyObject *py_status = PyLong_FromUnsignedLong((unsigned long)status); + if (py_status == NULL) { + return NULL; + } + return py_status; +} + +PyObject *AerospikeClient_Abort(AerospikeClient *self, PyObject *args, + PyObject *kwds) +{ + AerospikeTransaction *py_transaction = NULL; + + static char *kwlist[] = {"transaction", NULL}; + + if (PyArg_ParseTupleAndKeywords(args, kwds, "O!|p:abort", kwlist, + &AerospikeTransaction_Type, + (PyObject **)(&py_transaction)) == false) { + return NULL; + } + + as_error err; + as_error_init(&err); + + as_abort_status status; + + Py_BEGIN_ALLOW_THREADS + aerospike_abort(self->as, &err, py_transaction->txn, &status); + Py_END_ALLOW_THREADS + + if (err.code != AEROSPIKE_OK) { + raise_exception(&err); + return NULL; + } + + PyObject *py_status = PyLong_FromUnsignedLong((unsigned long)status); + if (py_status == NULL) { + return NULL; + } + return py_status; +} diff --git a/src/main/client/operate.c b/src/main/client/operate.c index dbc2513e8..668ff6cf0 100644 --- a/src/main/client/operate.c +++ b/src/main/client/operate.c @@ -79,6 +79,7 @@ static inline bool isExprOp(int op); PyObject *py_err = NULL; \ error_to_pyobject(&err, &py_err); \ PyObject *exception_type = raise_exception_old(&err); \ + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); \ if (PyObject_HasAttrString(exception_type, "key")) { \ PyObject_SetAttrString(exception_type, "key", py_key); \ } \ @@ -1206,6 +1207,7 @@ PyObject *AerospikeClient_OperateOrdered(AerospikeClient *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } diff --git a/src/main/client/put.c b/src/main/client/put.c index d1d51b0df..a09453c13 100644 --- a/src/main/client/put.c +++ b/src/main/client/put.c @@ -133,6 +133,7 @@ PyObject *AerospikeClient_Put_Invoke(AerospikeClient *self, PyObject *py_key, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } diff --git a/src/main/client/remove.c b/src/main/client/remove.c index 1d15c4c5a..b7266742b 100644 --- a/src/main/client/remove.c +++ b/src/main/client/remove.c @@ -136,6 +136,7 @@ PyObject *AerospikeClient_Remove_Invoke(AerospikeClient *self, PyObject *py_key, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } diff --git a/src/main/client/remove_bin.c b/src/main/client/remove_bin.c index e4826960c..ca627841f 100644 --- a/src/main/client/remove_bin.c +++ b/src/main/client/remove_bin.c @@ -163,6 +163,7 @@ AerospikeClient_RemoveBin_Invoke(AerospikeClient *self, PyObject *py_key, PyObject *py_err = NULL; error_to_pyobject(err, &py_err); PyObject *exception_type = raise_exception_old(err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } @@ -239,6 +240,7 @@ PyObject *AerospikeClient_RemoveBin(AerospikeClient *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } diff --git a/src/main/client/sec_index.c b/src/main/client/sec_index.c index 0276148b6..d9129c23e 100644 --- a/src/main/client/sec_index.c +++ b/src/main/client/sec_index.c @@ -232,6 +232,7 @@ PyObject *AerospikeClient_Index_Cdt_Create(AerospikeClient *self, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "name")) { PyObject_SetAttrString(exception_type, "name", py_name); } @@ -333,6 +334,7 @@ PyObject *AerospikeClient_Index_Remove(AerospikeClient *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "name")) { PyObject_SetAttrString(exception_type, "name", py_name); } diff --git a/src/main/client/select.c b/src/main/client/select.c index bf4f70280..5e6f55995 100644 --- a/src/main/client/select.c +++ b/src/main/client/select.c @@ -185,6 +185,7 @@ PyObject *AerospikeClient_Select_Invoke(AerospikeClient *self, PyObject *py_key, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "key")) { PyObject_SetAttrString(exception_type, "key", py_key); } diff --git a/src/main/client/type.c b/src/main/client/type.c index bea8f279f..aaacbcac2 100644 --- a/src/main/client/type.c +++ b/src/main/client/type.c @@ -529,6 +529,11 @@ static PyMethodDef AerospikeClient_Type_Methods[] = { {"truncate", (PyCFunction)AerospikeClient_Truncate, METH_VARARGS | METH_KEYWORDS, truncate_doc}, + // Multi record transactions + {"commit", (PyCFunction)AerospikeClient_Commit, + METH_VARARGS | METH_KEYWORDS}, + {"abort", (PyCFunction)AerospikeClient_Abort, METH_VARARGS | METH_KEYWORDS}, + {NULL}}; /******************************************************************************* @@ -1290,24 +1295,25 @@ static void AerospikeClient_Type_Dealloc(PyObject *self) ******************************************************************************/ static PyTypeObject AerospikeClient_Type = { - PyVarObject_HEAD_INIT(NULL, 0) "aerospike.Client", // tp_name - sizeof(AerospikeClient), // tp_basicsize - 0, // tp_itemsize - (destructor)AerospikeClient_Type_Dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - 0, // tp_as_mapping - 0, // tp_hash - 0, // tp_call - 0, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer + PyVarObject_HEAD_INIT(NULL, 0) + FULLY_QUALIFIED_TYPE_NAME("Client"), // tp_name + sizeof(AerospikeClient), // tp_basicsize + 0, // tp_itemsize + (destructor)AerospikeClient_Type_Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, // tp_flags "The Client class manages the connections and trasactions against\n" diff --git a/src/main/client/udf.c b/src/main/client/udf.c index a31f086f3..d161b2726 100644 --- a/src/main/client/udf.c +++ b/src/main/client/udf.c @@ -276,6 +276,7 @@ PyObject *AerospikeClient_UDF_Put(AerospikeClient *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "module")) { PyObject_SetAttrString(exception_type, "module", Py_None); } @@ -372,6 +373,7 @@ PyObject *AerospikeClient_UDF_Remove(AerospikeClient *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "module")) { PyObject_SetAttrString(exception_type, "module", py_filename); } @@ -464,6 +466,7 @@ PyObject *AerospikeClient_UDF_List(AerospikeClient *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "module")) { PyObject_SetAttrString(exception_type, "module", Py_None); } @@ -579,6 +582,7 @@ PyObject *AerospikeClient_UDF_Get_UDF(AerospikeClient *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "module")) { PyObject_SetAttrString(exception_type, "module", py_module); } diff --git a/src/main/conversions.c b/src/main/conversions.c index 210c9994e..37f55fbc8 100644 --- a/src/main/conversions.c +++ b/src/main/conversions.c @@ -974,9 +974,9 @@ PyObject *create_py_cluster_from_as_cluster(as_error *error_p, py_invalid_node_count); Py_DECREF(py_invalid_node_count); - PyObject *py_transaction_count = PyLong_FromLong(cluster->tran_count); - PyObject_SetAttrString(py_cluster, "tran_count", py_transaction_count); - Py_DECREF(py_transaction_count); + PyObject *py_command_count = PyLong_FromLong(cluster->command_count); + PyObject_SetAttrString(py_cluster, "command_count", py_command_count); + Py_DECREF(py_command_count); PyObject *py_retry_count = PyLong_FromLong(cluster->retry_count); PyObject_SetAttrString(py_cluster, "retry_count", py_retry_count); diff --git a/src/main/exception.c b/src/main/exception.c index b5eb55288..4c681131e 100644 --- a/src/main/exception.c +++ b/src/main/exception.c @@ -26,550 +26,376 @@ #include "exception_types.h" #include "macros.h" -static PyObject *module; +static PyObject *py_module; + +#define SUBMODULE_NAME "exception" + +// Used to create a Python Exception class +struct exception_def { + // When adding the exception to the module, we only need the class name + // Example: AerospikeError + const char *class_name; + // When creating an exception, we need to specify the module name + class name + // Example: exception.AerospikeError + const char *fully_qualified_class_name; + // If NULL, there is no base class + const char *base_class_name; + enum as_status_e code; + // Only applies to base exception classes that need their own fields + // NULL if this doesn't apply + const char *const *list_of_attrs; +}; + +// Used to create instances of the above struct +#define EXCEPTION_DEF(class_name, base_class_name, err_code, attrs) \ + { \ + class_name, SUBMODULE_NAME "." class_name, base_class_name, err_code, \ + attrs \ + } +// Base exception names +#define AEROSPIKE_ERR_EXCEPTION_NAME "AerospikeError" +#define CLIENT_ERR_EXCEPTION_NAME "ClientError" +#define SERVER_ERR_EXCEPTION_NAME "ServerError" +#define CLUSTER_ERR_EXCEPTION_NAME "ClusterError" +#define RECORD_ERR_EXCEPTION_NAME "RecordError" +#define INDEX_ERR_EXCEPTION_NAME "IndexError" +#define UDF_ERR_EXCEPTION_NAME "UDFError" +#define ADMIN_ERR_EXCEPTION_NAME "AdminError" +#define QUERY_ERR_EXCEPTION_NAME "QueryError" + +// If a base exception doesn't have an error code +// No exception should have an error code of 0, so this should be ok +#define NO_ERROR_CODE 0 + +// Same order as the tuple of args passed into the exception +const char *const aerospike_err_attrs[] = {"code", "msg", "file", + "line", "in_doubt", NULL}; +const char *const record_err_attrs[] = {"key", "bin", NULL}; +const char *const index_err_attrs[] = {"name", NULL}; +const char *const udf_err_attrs[] = {"module", "func", NULL}; + +// TODO: idea. define this as a list of tuples in python? +// Base classes must be defined before classes that inherit from them (topological sorting) +struct exception_def exception_defs[] = { + EXCEPTION_DEF(AEROSPIKE_ERR_EXCEPTION_NAME, NULL, NO_ERROR_CODE, + aerospike_err_attrs), + EXCEPTION_DEF(CLIENT_ERR_EXCEPTION_NAME, AEROSPIKE_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_CLIENT, NULL), + EXCEPTION_DEF(SERVER_ERR_EXCEPTION_NAME, AEROSPIKE_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_SERVER, NULL), + EXCEPTION_DEF("TimeoutError", AEROSPIKE_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_TIMEOUT, NULL), + // Client errors + EXCEPTION_DEF("ParamError", CLIENT_ERR_EXCEPTION_NAME, AEROSPIKE_ERR_PARAM, + NULL), + EXCEPTION_DEF("InvalidHostError", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_INVALID_HOST, NULL), + EXCEPTION_DEF("ConnectionError", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_CONNECTION, NULL), + EXCEPTION_DEF("TLSError", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_TLS_ERROR, NULL), + EXCEPTION_DEF("BatchFailed", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_BATCH_FAILED, NULL), + EXCEPTION_DEF("NoResponse", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_NO_RESPONSE, NULL), + EXCEPTION_DEF("MaxErrorRateExceeded", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_MAX_ERROR_RATE, NULL), + EXCEPTION_DEF("MaxRetriesExceeded", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_MAX_RETRIES_EXCEEDED, NULL), + EXCEPTION_DEF("InvalidNodeError", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_INVALID_NODE, NULL), + EXCEPTION_DEF("NoMoreConnectionsError", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_NO_MORE_CONNECTIONS, NULL), + EXCEPTION_DEF("AsyncConnectionError", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_ASYNC_CONNECTION, NULL), + EXCEPTION_DEF("ClientAbortError", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_CLIENT_ABORT, NULL), + EXCEPTION_DEF("TranactionFailed", CLIENT_ERR_EXCEPTION_NAME, + AEROSPIKE_TXN_FAILED, NULL), + // Server errors + EXCEPTION_DEF("InvalidRequest", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_REQUEST_INVALID, NULL), + EXCEPTION_DEF("ServerFull", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_SERVER_FULL, NULL), + EXCEPTION_DEF("AlwaysForbidden", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_ALWAYS_FORBIDDEN, NULL), + EXCEPTION_DEF("UnsupportedFeature", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_UNSUPPORTED_FEATURE, NULL), + EXCEPTION_DEF("DeviceOverload", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_DEVICE_OVERLOAD, NULL), + EXCEPTION_DEF("NamespaceNotFound", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_NAMESPACE_NOT_FOUND, NULL), + EXCEPTION_DEF("ForbiddenError", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_FAIL_FORBIDDEN, NULL), + EXCEPTION_DEF(QUERY_ERR_EXCEPTION_NAME, SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_QUERY, NULL), + EXCEPTION_DEF(CLUSTER_ERR_EXCEPTION_NAME, SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_CLUSTER, NULL), + EXCEPTION_DEF("InvalidGeoJSON", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_GEO_INVALID_GEOJSON, NULL), + EXCEPTION_DEF("OpNotApplicable", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_OP_NOT_APPLICABLE, NULL), + EXCEPTION_DEF("FilteredOut", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_FILTERED_OUT, NULL), + EXCEPTION_DEF("LostConflict", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_LOST_CONFLICT, NULL), + EXCEPTION_DEF("ScanAbortedError", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_SCAN_ABORTED, NULL), + EXCEPTION_DEF("ElementNotFoundError", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_FAIL_ELEMENT_NOT_FOUND, NULL), + EXCEPTION_DEF("ElementExistsError", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_FAIL_ELEMENT_EXISTS, NULL), + EXCEPTION_DEF("BatchDisabledError", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_BATCH_DISABLED, NULL), + EXCEPTION_DEF("BatchMaxRequestError", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_BATCH_MAX_REQUESTS_EXCEEDED, NULL), + EXCEPTION_DEF("BatchQueueFullError", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_BATCH_QUEUES_FULL, NULL), + EXCEPTION_DEF("QueryAbortedError", SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_QUERY_ABORTED, NULL), + // Cluster errors + EXCEPTION_DEF("ClusterChangeError", CLUSTER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_CLUSTER_CHANGE, NULL), + // Record errors + // RecordError doesn't have an error code. It will be ignored in this case + EXCEPTION_DEF(RECORD_ERR_EXCEPTION_NAME, SERVER_ERR_EXCEPTION_NAME, + NO_ERROR_CODE, record_err_attrs), + EXCEPTION_DEF("RecordKeyMismatch", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_RECORD_KEY_MISMATCH, NULL), + EXCEPTION_DEF("RecordNotFound", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_RECORD_NOT_FOUND, NULL), + EXCEPTION_DEF("RecordGenerationError", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_RECORD_GENERATION, NULL), + EXCEPTION_DEF("RecordExistsError", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_RECORD_EXISTS, NULL), + EXCEPTION_DEF("RecordTooBig", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_RECORD_TOO_BIG, NULL), + EXCEPTION_DEF("RecordBusy", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_RECORD_BUSY, NULL), + EXCEPTION_DEF("BinNameError", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_BIN_NAME, NULL), + EXCEPTION_DEF("BinIncompatibleType", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_BIN_INCOMPATIBLE_TYPE, NULL), + EXCEPTION_DEF("BinExistsError", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_BIN_EXISTS, NULL), + EXCEPTION_DEF("BinNotFound", RECORD_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_BIN_NOT_FOUND, NULL), + // Index errors + EXCEPTION_DEF(INDEX_ERR_EXCEPTION_NAME, SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_INDEX, index_err_attrs), + EXCEPTION_DEF("IndexNotFound", INDEX_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_INDEX_NOT_FOUND, NULL), + EXCEPTION_DEF("IndexFoundError", INDEX_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_INDEX_FOUND, NULL), + EXCEPTION_DEF("IndexOOM", INDEX_ERR_EXCEPTION_NAME, AEROSPIKE_ERR_INDEX_OOM, + NULL), + EXCEPTION_DEF("IndexNotReadable", INDEX_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_INDEX_NOT_READABLE, NULL), + EXCEPTION_DEF("IndexNameMaxLen", INDEX_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_INDEX_NAME_MAXLEN, NULL), + EXCEPTION_DEF("IndexNameMaxCount", INDEX_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_INDEX_MAXCOUNT, NULL), + // UDF errors + EXCEPTION_DEF(UDF_ERR_EXCEPTION_NAME, SERVER_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_UDF, udf_err_attrs), + EXCEPTION_DEF("UDFNotFound", UDF_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_UDF_NOT_FOUND, NULL), + EXCEPTION_DEF("LuaFileNotFound", UDF_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_LUA_FILE_NOT_FOUND, NULL), + // Admin errors + EXCEPTION_DEF(ADMIN_ERR_EXCEPTION_NAME, SERVER_ERR_EXCEPTION_NAME, + NO_ERROR_CODE, NULL), + EXCEPTION_DEF("SecurityNotSupported", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_SECURITY_NOT_SUPPORTED, NULL), + EXCEPTION_DEF("SecurityNotEnabled", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_SECURITY_NOT_ENABLED, NULL), + EXCEPTION_DEF("SecuritySchemeNotSupported", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_SECURITY_SCHEME_NOT_SUPPORTED, NULL), + EXCEPTION_DEF("InvalidCommand", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_COMMAND, NULL), + EXCEPTION_DEF("InvalidField", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_FIELD, NULL), + EXCEPTION_DEF("IllegalState", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_ILLEGAL_STATE, NULL), + EXCEPTION_DEF("InvalidUser", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_USER, NULL), + EXCEPTION_DEF("UserExistsError", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_USER_ALREADY_EXISTS, NULL), + EXCEPTION_DEF("InvalidPassword", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_PASSWORD, NULL), + EXCEPTION_DEF("ExpiredPassword", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_EXPIRED_PASSWORD, NULL), + EXCEPTION_DEF("ForbiddenPassword", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_FORBIDDEN_PASSWORD, NULL), + EXCEPTION_DEF("InvalidCredential", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_CREDENTIAL, NULL), + EXCEPTION_DEF("InvalidRole", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_ROLE, NULL), + EXCEPTION_DEF("RoleExistsError", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_ROLE_ALREADY_EXISTS, NULL), + EXCEPTION_DEF("RoleViolation", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_ROLE_VIOLATION, NULL), + EXCEPTION_DEF("InvalidPrivilege", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_PRIVILEGE, NULL), + EXCEPTION_DEF("NotAuthenticated", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_NOT_AUTHENTICATED, NULL), + EXCEPTION_DEF("InvalidWhitelist", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_WHITELIST, NULL), + EXCEPTION_DEF("NotWhitelisted", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_NOT_WHITELISTED, NULL), + EXCEPTION_DEF("QuotasNotEnabled", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_QUOTAS_NOT_ENABLED, NULL), + EXCEPTION_DEF("InvalidQuota", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_INVALID_QUOTA, NULL), + EXCEPTION_DEF("QuotaExceeded", ADMIN_ERR_EXCEPTION_NAME, + AEROSPIKE_QUOTA_EXCEEDED, NULL), + // Query errors + EXCEPTION_DEF("QueryQueueFull", QUERY_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_QUERY_QUEUE_FULL, NULL), + EXCEPTION_DEF("QueryTimeout", QUERY_ERR_EXCEPTION_NAME, + AEROSPIKE_ERR_QUERY_TIMEOUT, NULL)}; + +// TODO: define aerospike module name somewhere else +#define FULLY_QUALIFIED_MODULE_NAME "aerospike." SUBMODULE_NAME + +// Returns NULL if an error occurred PyObject *AerospikeException_New(void) { static struct PyModuleDef moduledef = {PyModuleDef_HEAD_INIT, - "aerospike.exception", + FULLY_QUALIFIED_MODULE_NAME, "Exception objects", -1, NULL, NULL, NULL, NULL}; - module = PyModule_Create(&moduledef); - - struct exceptions exceptions_array; - - memset(&exceptions_array, 0, sizeof(exceptions_array)); - - struct server_exceptions_struct server_array = { - {&exceptions_array.InvalidRequest, &exceptions_array.ServerFull, - &exceptions_array.AlwaysForbidden, - &exceptions_array.UnsupportedFeature, &exceptions_array.DeviceOverload, - &exceptions_array.NamespaceNotFound, &exceptions_array.ForbiddenError, - &exceptions_array.QueryError, &exceptions_array.ClusterError, - &exceptions_array.InvalidGeoJSON, &exceptions_array.OpNotApplicable, - &exceptions_array.FilteredOut, &exceptions_array.LostConflict}, - {"InvalidRequest", "ServerFull", "AlwaysForbidden", - "UnsupportedFeature", "DeviceOverload", "NamespaceNotFound", - "ForbiddenError", "QueryError", "ClusterError", "InvalidGeoJSON", - "OpNotApplicable", "FilteredOut", "LostConflict"}, - {AEROSPIKE_ERR_REQUEST_INVALID, AEROSPIKE_ERR_SERVER_FULL, - AEROSPIKE_ERR_ALWAYS_FORBIDDEN, AEROSPIKE_ERR_UNSUPPORTED_FEATURE, - AEROSPIKE_ERR_DEVICE_OVERLOAD, AEROSPIKE_ERR_NAMESPACE_NOT_FOUND, - AEROSPIKE_ERR_FAIL_FORBIDDEN, AEROSPIKE_ERR_QUERY, - AEROSPIKE_ERR_CLUSTER, AEROSPIKE_ERR_GEO_INVALID_GEOJSON, - AEROSPIKE_ERR_OP_NOT_APPLICABLE, AEROSPIKE_FILTERED_OUT, - AEROSPIKE_LOST_CONFLICT}}; - - struct record_exceptions_struct record_array = { - {&exceptions_array.RecordKeyMismatch, &exceptions_array.RecordNotFound, - &exceptions_array.RecordGenerationError, - &exceptions_array.RecordExistsError, &exceptions_array.RecordTooBig, - &exceptions_array.RecordBusy, &exceptions_array.BinNameError, - &exceptions_array.BinIncompatibleType, - &exceptions_array.BinExistsError, &exceptions_array.BinNotFound}, - {"RecordKeyMismatch", "RecordNotFound", "RecordGenerationError", - "RecordExistsError", "RecordTooBig", "RecordBusy", "BinNameError", - "BinIncompatibleType", "BinExistsError", "BinNotFound"}, - {AEROSPIKE_ERR_RECORD_KEY_MISMATCH, AEROSPIKE_ERR_RECORD_NOT_FOUND, - AEROSPIKE_ERR_RECORD_GENERATION, AEROSPIKE_ERR_RECORD_EXISTS, - AEROSPIKE_ERR_RECORD_TOO_BIG, AEROSPIKE_ERR_RECORD_BUSY, - AEROSPIKE_ERR_BIN_NAME, AEROSPIKE_ERR_BIN_INCOMPATIBLE_TYPE, - AEROSPIKE_ERR_BIN_EXISTS, AEROSPIKE_ERR_BIN_NOT_FOUND}}; - - struct index_exceptions_struct index_array = { - {&exceptions_array.IndexNotFound, &exceptions_array.IndexFoundError, - &exceptions_array.IndexOOM, &exceptions_array.IndexNotReadable, - &exceptions_array.IndexNameMaxLen, - &exceptions_array.IndexNameMaxCount}, - {"IndexNotFound", "IndexFoundError", "IndexOOM", "IndexNotReadable", - "IndexNameMaxLen", "IndexNameMaxCount"}, - {AEROSPIKE_ERR_INDEX_NOT_FOUND, AEROSPIKE_ERR_INDEX_FOUND, - AEROSPIKE_ERR_INDEX_OOM, AEROSPIKE_ERR_INDEX_NOT_READABLE, - AEROSPIKE_ERR_INDEX_NAME_MAXLEN, AEROSPIKE_ERR_INDEX_MAXCOUNT}}; - - struct admin_exceptions_struct admin_array = { - {&exceptions_array.SecurityNotSupported, - &exceptions_array.SecurityNotEnabled, - &exceptions_array.SecuritySchemeNotSupported, - &exceptions_array.InvalidCommand, - &exceptions_array.InvalidField, - &exceptions_array.IllegalState, - &exceptions_array.InvalidUser, - &exceptions_array.UserExistsError, - &exceptions_array.InvalidPassword, - &exceptions_array.ExpiredPassword, - &exceptions_array.ForbiddenPassword, - &exceptions_array.InvalidCredential, - &exceptions_array.InvalidRole, - &exceptions_array.RoleExistsError, - &exceptions_array.RoleViolation, - &exceptions_array.InvalidPrivilege, - &exceptions_array.NotAuthenticated, - &exceptions_array.InvalidWhitelist, - &exceptions_array.NotWhitelisted, - &exceptions_array.QuotasNotEnabled, - &exceptions_array.InvalidQuota, - &exceptions_array.QuotaExceeded}, - {"SecurityNotSupported", - "SecurityNotEnabled", - "SecuritySchemeNotSupported", - "InvalidCommand", - "InvalidField", - "IllegalState", - "InvalidUser", - "UserExistsError", - "InvalidPassword", - "ExpiredPassword", - "ForbiddenPassword", - "InvalidCredential", - "InvalidRole", - "RoleExistsError", - "RoleViolation", - "InvalidPrivilege", - "NotAuthenticated", - "InvalidWhitelist", - "NotWhitelisted", - "QuotasNotEnabled", - "InvalidQuota", - "QuotaExceeded"}, - {AEROSPIKE_SECURITY_NOT_SUPPORTED, - AEROSPIKE_SECURITY_NOT_ENABLED, - AEROSPIKE_SECURITY_SCHEME_NOT_SUPPORTED, - AEROSPIKE_INVALID_COMMAND, - AEROSPIKE_INVALID_FIELD, - AEROSPIKE_ILLEGAL_STATE, - AEROSPIKE_INVALID_USER, - AEROSPIKE_USER_ALREADY_EXISTS, - AEROSPIKE_INVALID_PASSWORD, - AEROSPIKE_EXPIRED_PASSWORD, - AEROSPIKE_FORBIDDEN_PASSWORD, - AEROSPIKE_INVALID_CREDENTIAL, - AEROSPIKE_INVALID_ROLE, - AEROSPIKE_ROLE_ALREADY_EXISTS, - AEROSPIKE_ROLE_VIOLATION, - AEROSPIKE_INVALID_PRIVILEGE, - AEROSPIKE_NOT_AUTHENTICATED, - AEROSPIKE_INVALID_WHITELIST, - AEROSPIKE_NOT_WHITELISTED, - AEROSPIKE_QUOTAS_NOT_ENABLED, - AEROSPIKE_INVALID_QUOTA, - AEROSPIKE_QUOTA_EXCEEDED}}; - - PyObject *py_code = NULL; - PyObject *py_dict = PyDict_New(); - PyDict_SetItemString(py_dict, "code", Py_None); - PyDict_SetItemString(py_dict, "file", Py_None); - PyDict_SetItemString(py_dict, "msg", Py_None); - PyDict_SetItemString(py_dict, "line", Py_None); - - exceptions_array.AerospikeError = - PyErr_NewException("exception.AerospikeError", NULL, py_dict); - Py_INCREF(exceptions_array.AerospikeError); - Py_DECREF(py_dict); - PyModule_AddObject(module, "AerospikeError", - exceptions_array.AerospikeError); - PyObject_SetAttrString(exceptions_array.AerospikeError, "code", Py_None); - - exceptions_array.ClientError = PyErr_NewException( - "exception.ClientError", exceptions_array.AerospikeError, NULL); - Py_INCREF(exceptions_array.ClientError); - PyModule_AddObject(module, "ClientError", exceptions_array.ClientError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_CLIENT); - PyObject_SetAttrString(exceptions_array.ClientError, "code", py_code); - Py_DECREF(py_code); - - exceptions_array.ServerError = PyErr_NewException( - "exception.ServerError", exceptions_array.AerospikeError, NULL); - Py_INCREF(exceptions_array.ServerError); - PyModule_AddObject(module, "ServerError", exceptions_array.ServerError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_SERVER); - PyObject_SetAttrString(exceptions_array.ServerError, "code", py_code); - Py_DECREF(py_code); - - exceptions_array.TimeoutError = PyErr_NewException( - "exception.TimeoutError", exceptions_array.AerospikeError, NULL); - Py_INCREF(exceptions_array.TimeoutError); - PyModule_AddObject(module, "TimeoutError", exceptions_array.TimeoutError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_TIMEOUT); - PyObject_SetAttrString(exceptions_array.TimeoutError, "code", py_code); - Py_DECREF(py_code); - - //Client Exceptions - exceptions_array.ParamError = PyErr_NewException( - "exception.ParamError", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.ParamError); - PyModule_AddObject(module, "ParamError", exceptions_array.ParamError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_PARAM); - PyObject_SetAttrString(exceptions_array.ParamError, "code", py_code); - Py_DECREF(py_code); - - exceptions_array.InvalidHostError = PyErr_NewException( - "exception.InvalidHostError", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.InvalidHostError); - PyModule_AddObject(module, "InvalidHostError", - exceptions_array.InvalidHostError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_INVALID_HOST); - PyObject_SetAttrString(exceptions_array.InvalidHostError, "code", py_code); - Py_DECREF(py_code); - - exceptions_array.ConnectionError = PyErr_NewException( - "exception.ConnectionError", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.ConnectionError); - PyModule_AddObject(module, "ConnectionError", - exceptions_array.ConnectionError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_CONNECTION); - PyObject_SetAttrString(exceptions_array.ConnectionError, "code", py_code); - Py_DECREF(py_code); - - // TLSError, AEROSPIKE_ERR_TLS_ERROR, -9 - exceptions_array.TLSError = PyErr_NewException( - "exception.TLSError", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.TLSError); - PyModule_AddObject(module, "TLSError", exceptions_array.TLSError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_TLS_ERROR); - PyObject_SetAttrString(exceptions_array.TLSError, "code", py_code); - Py_DECREF(py_code); - - // BatchFailed, AEROSPIKE_BATCH_FAILED, -16 - exceptions_array.BatchFailed = PyErr_NewException( - "exception.BatchFailed", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.BatchFailed); - PyModule_AddObject(module, "BatchFailed", exceptions_array.BatchFailed); - py_code = PyLong_FromLong(AEROSPIKE_BATCH_FAILED); - PyObject_SetAttrString(exceptions_array.BatchFailed, "code", py_code); - Py_DECREF(py_code); - - // NoResponse, AEROSPIKE_NO_RESPONSE, -15 - exceptions_array.NoResponse = PyErr_NewException( - "exception.NoResponse", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.NoResponse); - PyModule_AddObject(module, "NoResponse", exceptions_array.NoResponse); - py_code = PyLong_FromLong(AEROSPIKE_NO_RESPONSE); - PyObject_SetAttrString(exceptions_array.NoResponse, "code", py_code); - Py_DECREF(py_code); - - // max errors limit reached, AEROSPIKE_MAX_ERROR_RATE, -14 - exceptions_array.MaxErrorRateExceeded = PyErr_NewException( - "exception.MaxErrorRateExceeded", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.MaxErrorRateExceeded); - PyModule_AddObject(module, "MaxErrorRateExceeded", - exceptions_array.MaxErrorRateExceeded); - py_code = PyLong_FromLong(AEROSPIKE_MAX_ERROR_RATE); - PyObject_SetAttrString(exceptions_array.MaxErrorRateExceeded, "code", - py_code); - Py_DECREF(py_code); - - // max retries exceeded, AEROSPIKE_ERR_MAX_RETRIES_EXCEEDED, -12 - exceptions_array.MaxRetriesExceeded = PyErr_NewException( - "exception.MaxRetriesExceeded", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.MaxRetriesExceeded); - PyModule_AddObject(module, "MaxRetriesExceeded", - exceptions_array.MaxRetriesExceeded); - py_code = PyLong_FromLong(AEROSPIKE_ERR_MAX_RETRIES_EXCEEDED); - PyObject_SetAttrString(exceptions_array.MaxRetriesExceeded, "code", - py_code); - Py_DECREF(py_code); - - // InvalidNodeError, AEROSPIKE_ERR_INVALID_NODE, -8 - exceptions_array.InvalidNodeError = PyErr_NewException( - "exception.InvalidNodeError", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.InvalidNodeError); - PyModule_AddObject(module, "InvalidNodeError", - exceptions_array.InvalidNodeError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_INVALID_NODE); - PyObject_SetAttrString(exceptions_array.InvalidNodeError, "code", py_code); - Py_DECREF(py_code); - - // NoMoreConnectionsError, AEROSPIKE_ERR_NO_MORE_CONNECTIONS, -7 - exceptions_array.NoMoreConnectionsError = PyErr_NewException( - "exception.NoMoreConnectionsError", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.NoMoreConnectionsError); - PyModule_AddObject(module, "NoMoreConnectionsError", - exceptions_array.NoMoreConnectionsError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_NO_MORE_CONNECTIONS); - PyObject_SetAttrString(exceptions_array.NoMoreConnectionsError, "code", - py_code); - Py_DECREF(py_code); - - // AsyncConnectionError, AEROSPIKE_ERR_ASYNC_CONNECTION, -6 - exceptions_array.AsyncConnectionError = PyErr_NewException( - "exception.AsyncConnectionError", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.AsyncConnectionError); - PyModule_AddObject(module, "AsyncConnectionError", - exceptions_array.AsyncConnectionError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_ASYNC_CONNECTION); - PyObject_SetAttrString(exceptions_array.AsyncConnectionError, "code", - py_code); - Py_DECREF(py_code); - - // ClientAbortError, AEROSPIKE_ERR_CLIENT_ABORT, -5 - exceptions_array.ClientAbortError = PyErr_NewException( - "exception.ClientAbortError", exceptions_array.ClientError, NULL); - Py_INCREF(exceptions_array.ClientAbortError); - PyModule_AddObject(module, "ClientAbortError", - exceptions_array.ClientAbortError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_CLIENT_ABORT); - PyObject_SetAttrString(exceptions_array.ClientAbortError, "code", py_code); - Py_DECREF(py_code); - - //Server Exceptions - int count = sizeof(server_array.server_exceptions) / - sizeof(server_array.server_exceptions[0]); - int i; - PyObject **current_exception; - for (i = 0; i < count; i++) { - current_exception = server_array.server_exceptions[i]; - char *name = server_array.server_exceptions_name[i]; - char prefix[40] = "exception."; - *current_exception = PyErr_NewException( - strcat(prefix, name), exceptions_array.ServerError, NULL); - Py_INCREF(*current_exception); - PyModule_AddObject(module, name, *current_exception); - PyObject *py_code = - PyLong_FromLong(server_array.server_exceptions_codes[i]); - PyObject_SetAttrString(*current_exception, "code", py_code); - Py_DECREF(py_code); + py_module = PyModule_Create(&moduledef); + if (py_module == NULL) { + return NULL; } - exceptions_array.ClusterChangeError = PyErr_NewException( - "exception.ClusterChangeError", exceptions_array.ClusterError, NULL); - Py_INCREF(exceptions_array.ClusterChangeError); - PyModule_AddObject(module, "ClusterChangeError", - exceptions_array.ClusterChangeError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_CLUSTER_CHANGE); - PyObject_SetAttrString(exceptions_array.ClusterChangeError, "code", - py_code); - Py_DECREF(py_code); - - //Extra Server Errors - // ScanAbortedError , AEROSPIKE_ERR_SCAN_ABORTED, 15 - exceptions_array.ScanAbortedError = PyErr_NewException( - "exception.ScanAbortedError", exceptions_array.ServerError, NULL); - Py_INCREF(exceptions_array.ScanAbortedError); - PyModule_AddObject(module, "ScanAbortedError", - exceptions_array.ScanAbortedError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_SCAN_ABORTED); - PyObject_SetAttrString(exceptions_array.ScanAbortedError, "code", py_code); - Py_DECREF(py_code); - - // ElementNotFoundError , AEROSPIKE_ERR_FAIL_ELEMENT_NOT_FOUND, 23 - exceptions_array.ElementNotFoundError = PyErr_NewException( - "exception.ElementNotFoundError", exceptions_array.ServerError, NULL); - Py_INCREF(exceptions_array.ElementNotFoundError); - PyModule_AddObject(module, "ElementNotFoundError", - exceptions_array.ElementNotFoundError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_FAIL_ELEMENT_NOT_FOUND); - PyObject_SetAttrString(exceptions_array.ElementNotFoundError, "code", - py_code); - Py_DECREF(py_code); - - // ElementExistsError , AEROSPIKE_ERR_FAIL_ELEMENT_EXISTS, 24 - exceptions_array.ElementExistsError = PyErr_NewException( - "exception.ElementExistsError", exceptions_array.ServerError, NULL); - Py_INCREF(exceptions_array.ElementExistsError); - PyModule_AddObject(module, "ElementExistsError", - exceptions_array.ElementExistsError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_FAIL_ELEMENT_EXISTS); - PyObject_SetAttrString(exceptions_array.ElementExistsError, "code", - py_code); - Py_DECREF(py_code); - - // BatchDisabledError , AEROSPIKE_ERR_BATCH_DISABLED, 150 - exceptions_array.BatchDisabledError = PyErr_NewException( - "exception.BatchDisabledError", exceptions_array.ServerError, NULL); - Py_INCREF(exceptions_array.BatchDisabledError); - PyModule_AddObject(module, "BatchDisabledError", - exceptions_array.BatchDisabledError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_BATCH_DISABLED); - PyObject_SetAttrString(exceptions_array.BatchDisabledError, "code", - py_code); - Py_DECREF(py_code); - - // BatchMaxRequestError , AEROSPIKE_ERR_BATCH_MAX_REQUESTS_EXCEEDED, 151 - exceptions_array.BatchMaxRequestError = PyErr_NewException( - "exception.BatchMaxRequestError", exceptions_array.ServerError, NULL); - Py_INCREF(exceptions_array.BatchMaxRequestError); - PyModule_AddObject(module, "BatchMaxRequestError", - exceptions_array.BatchMaxRequestError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_BATCH_MAX_REQUESTS_EXCEEDED); - PyObject_SetAttrString(exceptions_array.BatchMaxRequestError, "code", - py_code); - Py_DECREF(py_code); - - // BatchQueueFullError , AEROSPIKE_ERR_BATCH_QUEUES_FULL, 152 - exceptions_array.BatchQueueFullError = PyErr_NewException( - "exception.BatchQueueFullError", exceptions_array.ServerError, NULL); - Py_INCREF(exceptions_array.BatchQueueFullError); - PyModule_AddObject(module, "BatchQueueFullError", - exceptions_array.BatchQueueFullError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_BATCH_QUEUES_FULL); - PyObject_SetAttrString(exceptions_array.BatchQueueFullError, "code", - py_code); - Py_DECREF(py_code); - - // QueryAbortedError , AEROSPIKE_ERR_QUERY_ABORTED, 210 - exceptions_array.QueryAbortedError = PyErr_NewException( - "exception.QueryAbortedError", exceptions_array.ServerError, NULL); - Py_INCREF(exceptions_array.QueryAbortedError); - PyModule_AddObject(module, "QueryAbortedError", - exceptions_array.QueryAbortedError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_QUERY_ABORTED); - PyObject_SetAttrString(exceptions_array.QueryAbortedError, "code", py_code); - Py_DECREF(py_code); - - //Record exceptions - PyObject *py_record_dict = PyDict_New(); - PyDict_SetItemString(py_record_dict, "key", Py_None); - PyDict_SetItemString(py_record_dict, "bin", Py_None); - - exceptions_array.RecordError = PyErr_NewException( - "exception.RecordError", exceptions_array.ServerError, py_record_dict); - Py_INCREF(exceptions_array.RecordError); - Py_DECREF(py_record_dict); - PyObject_SetAttrString(exceptions_array.RecordError, "code", Py_None); - PyModule_AddObject(module, "RecordError", exceptions_array.RecordError); - - //int count = sizeof(record_exceptions)/sizeof(record_exceptions[0]); - count = sizeof(record_array.record_exceptions) / - sizeof(record_array.record_exceptions[0]); - for (i = 0; i < count; i++) { - current_exception = record_array.record_exceptions[i]; - char *name = record_array.record_exceptions_name[i]; - char prefix[40] = "exception."; - *current_exception = PyErr_NewException( - strcat(prefix, name), exceptions_array.RecordError, NULL); - Py_INCREF(*current_exception); - PyModule_AddObject(module, name, *current_exception); - PyObject *py_code = - PyLong_FromLong(record_array.record_exceptions_codes[i]); - PyObject_SetAttrString(*current_exception, "code", py_code); - Py_DECREF(py_code); - } + unsigned long exception_count = + sizeof(exception_defs) / sizeof(exception_defs[0]); + for (unsigned long i = 0; i < exception_count; i++) { + struct exception_def exception_def = exception_defs[i]; + + // TODO: if fetching base class is too slow, cache them using variables + // This only runs once when `import aerospike` is called, though + // When a module is loaded once through an import, it won't be loaded again + PyObject *py_base_class = NULL; + if (exception_def.base_class_name != NULL) { + py_base_class = PyObject_GetAttrString( + py_module, exception_def.base_class_name); + if (py_base_class == NULL) { + goto MODULE_CLEANUP_ON_ERROR; + } + } - //Index exceptions - PyObject *py_index_dict = PyDict_New(); - PyDict_SetItemString(py_index_dict, "name", Py_None); - - exceptions_array.IndexError = PyErr_NewException( - "exception.IndexError", exceptions_array.ServerError, py_index_dict); - Py_INCREF(exceptions_array.IndexError); - Py_DECREF(py_index_dict); - py_code = PyLong_FromLong(AEROSPIKE_ERR_INDEX); - PyObject_SetAttrString(exceptions_array.IndexError, "code", py_code); - Py_DECREF(py_code); - PyModule_AddObject(module, "IndexError", exceptions_array.IndexError); - - count = sizeof(index_array.index_exceptions) / - sizeof(index_array.index_exceptions[0]); - for (i = 0; i < count; i++) { - current_exception = index_array.index_exceptions[i]; - char *name = index_array.index_exceptions_name[i]; - char prefix[40] = "exception."; - *current_exception = PyErr_NewException( - strcat(prefix, name), exceptions_array.IndexError, NULL); - Py_INCREF(*current_exception); - PyModule_AddObject(module, name, *current_exception); - PyObject *py_code = - PyLong_FromLong(index_array.index_exceptions_codes[i]); - PyObject_SetAttrString(*current_exception, "code", py_code); - Py_DECREF(py_code); - } + // Set up class attributes + PyObject *py_exc_dict = NULL; + if (exception_def.list_of_attrs != NULL) { + py_exc_dict = PyDict_New(); + if (py_exc_dict == NULL) { + Py_XDECREF(py_base_class); + goto MODULE_CLEANUP_ON_ERROR; + } + + const char *const *curr_attr_ref = exception_def.list_of_attrs; + while (*curr_attr_ref != NULL) { + int retval = + PyDict_SetItemString(py_exc_dict, *curr_attr_ref, Py_None); + if (retval == -1) { + Py_DECREF(py_exc_dict); + Py_XDECREF(py_base_class); + goto MODULE_CLEANUP_ON_ERROR; + } + curr_attr_ref++; + } + } + + PyObject *py_exception_class = + PyErr_NewException(exception_def.fully_qualified_class_name, + py_base_class, py_exc_dict); + Py_XDECREF(py_base_class); + Py_XDECREF(py_exc_dict); + if (py_exception_class == NULL) { + goto MODULE_CLEANUP_ON_ERROR; + } - //UDF exceptions - PyObject *py_udf_dict = PyDict_New(); - PyDict_SetItemString(py_udf_dict, "module", Py_None); - PyDict_SetItemString(py_udf_dict, "func", Py_None); - - exceptions_array.UDFError = PyErr_NewException( - "exception.UDFError", exceptions_array.ServerError, py_udf_dict); - Py_INCREF(exceptions_array.UDFError); - Py_DECREF(py_udf_dict); - PyModule_AddObject(module, "UDFError", exceptions_array.UDFError); - py_code = PyLong_FromLong(AEROSPIKE_ERR_UDF); - PyObject_SetAttrString(exceptions_array.UDFError, "code", py_code); - Py_DECREF(py_code); - - exceptions_array.UDFNotFound = PyErr_NewException( - "exception.UDFNotFound", exceptions_array.UDFError, NULL); - Py_INCREF(exceptions_array.UDFNotFound); - PyModule_AddObject(module, "UDFNotFound", exceptions_array.UDFNotFound); - py_code = PyLong_FromLong(AEROSPIKE_ERR_UDF_NOT_FOUND); - PyObject_SetAttrString(exceptions_array.UDFNotFound, "code", py_code); - Py_DECREF(py_code); - - exceptions_array.LuaFileNotFound = PyErr_NewException( - "exception.LuaFileNotFound", exceptions_array.UDFError, NULL); - Py_INCREF(exceptions_array.LuaFileNotFound); - PyModule_AddObject(module, "LuaFileNotFound", - exceptions_array.LuaFileNotFound); - py_code = PyLong_FromLong(AEROSPIKE_ERR_LUA_FILE_NOT_FOUND); - PyObject_SetAttrString(exceptions_array.LuaFileNotFound, "code", py_code); - Py_DECREF(py_code); - - //Admin exceptions - exceptions_array.AdminError = PyErr_NewException( - "exception.AdminError", exceptions_array.ServerError, NULL); - Py_INCREF(exceptions_array.AdminError); - PyObject_SetAttrString(exceptions_array.AdminError, "code", Py_None); - PyModule_AddObject(module, "AdminError", exceptions_array.AdminError); - - count = sizeof(admin_array.admin_exceptions) / - sizeof(admin_array.admin_exceptions[0]); - for (i = 0; i < count; i++) { - current_exception = admin_array.admin_exceptions[i]; - char *name = admin_array.admin_exceptions_name[i]; - char prefix[40] = "exception."; - *current_exception = PyErr_NewException( - strcat(prefix, name), exceptions_array.AdminError, NULL); - Py_INCREF(*current_exception); - PyModule_AddObject(module, name, *current_exception); - PyObject *py_code = - PyLong_FromLong(admin_array.admin_exceptions_codes[i]); - PyObject_SetAttrString(*current_exception, "code", py_code); + PyObject *py_code = NULL; + if (exception_def.code == NO_ERROR_CODE) { + Py_INCREF(Py_None); + py_code = Py_None; + } + else { + py_code = PyLong_FromLong(exception_def.code); + if (py_code == NULL) { + goto EXC_CLASS_CLEANUP_ON_ERROR; + } + } + int retval = + PyObject_SetAttrString(py_exception_class, "code", py_code); Py_DECREF(py_code); + if (retval == -1) { + goto EXC_CLASS_CLEANUP_ON_ERROR; + } + + retval = PyModule_AddObject(py_module, exception_def.class_name, + py_exception_class); + if (retval == -1) { + goto EXC_CLASS_CLEANUP_ON_ERROR; + } + continue; + + EXC_CLASS_CLEANUP_ON_ERROR: + Py_DECREF(py_exception_class); + goto MODULE_CLEANUP_ON_ERROR; } - //Query exceptions - exceptions_array.QueryQueueFull = PyErr_NewException( - "exception.QueryQueueFull", exceptions_array.QueryError, NULL); - Py_INCREF(exceptions_array.QueryQueueFull); - PyModule_AddObject(module, "QueryQueueFull", - exceptions_array.QueryQueueFull); - py_code = PyLong_FromLong(AEROSPIKE_ERR_QUERY_QUEUE_FULL); - PyObject_SetAttrString(exceptions_array.QueryQueueFull, "code", py_code); - Py_DECREF(py_code); - - exceptions_array.QueryTimeout = PyErr_NewException( - "exception.QueryTimeout", exceptions_array.QueryError, NULL); - Py_INCREF(exceptions_array.QueryTimeout); - PyModule_AddObject(module, "QueryTimeout", exceptions_array.QueryTimeout); - py_code = PyLong_FromLong(AEROSPIKE_ERR_QUERY_TIMEOUT); - PyObject_SetAttrString(exceptions_array.QueryTimeout, "code", py_code); - Py_DECREF(py_code); - - return module; + return py_module; + +MODULE_CLEANUP_ON_ERROR: + Py_DECREF(py_module); + return NULL; } void remove_exception(as_error *err) { PyObject *py_key = NULL, *py_value = NULL; Py_ssize_t pos = 0; - PyObject *py_module_dict = PyModule_GetDict(module); + PyObject *py_module_dict = PyModule_GetDict(py_module); while (PyDict_Next(py_module_dict, &pos, &py_key, &py_value)) { Py_DECREF(py_value); } } +// We have this as a separate method because both raise_exception and raise_exception_old need to use it +void set_aerospike_exc_attrs_using_tuple_of_attrs(PyObject *py_exc, + PyObject *py_tuple) +{ + for (unsigned long i = 0; + i < sizeof(aerospike_err_attrs) / sizeof(aerospike_err_attrs[0]) - 1; + i++) { + // Here, we are assuming the number of attrs is the same as the number of tuple members + PyObject *py_arg = PyTuple_GetItem(py_tuple, i); + if (py_arg == NULL) { + // Don't fail out if number of attrs > number of tuple members + // This condition should never be true, though + PyErr_Clear(); + break; + } + PyObject_SetAttrString(py_exc, aerospike_err_attrs[i], py_arg); + } +} + +// TODO: idea. Use python dict to map error code to exception void raise_exception(as_error *err) { PyObject *py_key = NULL, *py_value = NULL; Py_ssize_t pos = 0; - PyObject *py_module_dict = PyModule_GetDict(module); + PyObject *py_module_dict = PyModule_GetDict(py_module); bool found = false; while (PyDict_Next(py_module_dict, &pos, &py_key, &py_value)) { @@ -580,37 +406,8 @@ void raise_exception(as_error *err) } if (err->code == PyLong_AsLong(py_code)) { found = true; - PyObject *py_attr = NULL; - py_attr = PyUnicode_FromString(err->message); - PyObject_SetAttrString(py_value, "msg", py_attr); - Py_DECREF(py_attr); - - // as_error.file is a char* so this may be null - if (err->file) { - py_attr = PyUnicode_FromString(err->file); - PyObject_SetAttrString(py_value, "file", py_attr); - Py_DECREF(py_attr); - } - else { - PyObject_SetAttrString(py_value, "file", Py_None); - } - // If the line is 0, set it as None - if (err->line > 0) { - py_attr = PyLong_FromLong(err->line); - PyObject_SetAttrString(py_value, "line", py_attr); - Py_DECREF(py_attr); - } - else { - PyObject_SetAttrString(py_value, "line", Py_None); - } - - py_attr = PyBool_FromLong(err->in_doubt); - PyObject_SetAttrString(py_value, "in_doubt", py_attr); - Py_DECREF(py_attr); - break; } - Py_DECREF(py_code); } } // We haven't found the right exception, just use AerospikeError @@ -628,6 +425,7 @@ void raise_exception(as_error *err) // Convert C error to Python exception PyObject *py_err = NULL; error_to_pyobject(err, &py_err); + set_aerospike_exc_attrs_using_tuple_of_attrs(py_value, py_err); // Raise exception PyErr_SetObject(py_value, py_err); @@ -640,7 +438,7 @@ PyObject *raise_exception_old(as_error *err) { PyObject *py_key = NULL, *py_value = NULL; Py_ssize_t pos = 0; - PyObject *py_module_dict = PyModule_GetDict(module); + PyObject *py_module_dict = PyModule_GetDict(py_module); bool found = false; while (PyDict_Next(py_module_dict, &pos, &py_key, &py_value)) { diff --git a/src/main/geospatial/type.c b/src/main/geospatial/type.c index 880ee9a56..e2ebb0616 100644 --- a/src/main/geospatial/type.c +++ b/src/main/geospatial/type.c @@ -219,9 +219,10 @@ static void AerospikeGeospatial_Type_Dealloc(AerospikeGeospatial *self) * PYTHON TYPE DESCRIPTOR ******************************************************************************/ static PyTypeObject AerospikeGeospatial_Type = { - PyVarObject_HEAD_INIT(NULL, 0) "aerospike.Geospatial", // tp_name - sizeof(AerospikeGeospatial), // tp_basicsize - 0, // tp_itemsize + PyVarObject_HEAD_INIT(NULL, 0) + FULLY_QUALIFIED_TYPE_NAME("Geospatial"), // tp_name + sizeof(AerospikeGeospatial), // tp_basicsize + 0, // tp_itemsize (destructor)AerospikeGeospatial_Type_Dealloc, // tp_dealloc 0, // tp_print diff --git a/src/main/key_ordered_dict/type.c b/src/main/key_ordered_dict/type.c index 66dc4807e..1037ba05f 100644 --- a/src/main/key_ordered_dict/type.c +++ b/src/main/key_ordered_dict/type.c @@ -40,7 +40,8 @@ static int AerospikeKeyOrderedDict_Type_Init(PyObject *self, PyObject *args, ******************************************************************************/ static PyTypeObject AerospikeKeyOrderedDict_Type = { - PyVarObject_HEAD_INIT(NULL, 0).tp_name = "aerospike.KeyOrderedDict", + PyVarObject_HEAD_INIT(NULL, 0).tp_name = + FULLY_QUALIFIED_TYPE_NAME("KeyOrderedDict"), .tp_basicsize = sizeof(AerospikeKeyOrderedDict), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, .tp_doc = "The KeyOrderedDict class is a dictionary that directly maps\n" diff --git a/src/main/log.c b/src/main/log.c index 03b0e44bf..fc2090c6d 100644 --- a/src/main/log.c +++ b/src/main/log.c @@ -31,32 +31,6 @@ static AerospikeLogCallback user_callback; -/* - * Declare's log level constants. - */ -as_status declare_log_constants(PyObject *aerospike) -{ - - // Status to be returned. - as_status status = AEROSPIKE_OK; - - // Check if aerospike object is present or no. - if (!aerospike) { - status = AEROSPIKE_ERR; - goto exit; - } - - // Add incidividual constants to aerospike module. - PyModule_AddIntConstant(aerospike, "LOG_LEVEL_OFF", LOG_LEVEL_OFF); - PyModule_AddIntConstant(aerospike, "LOG_LEVEL_ERROR", LOG_LEVEL_ERROR); - PyModule_AddIntConstant(aerospike, "LOG_LEVEL_WARN", LOG_LEVEL_WARN); - PyModule_AddIntConstant(aerospike, "LOG_LEVEL_INFO", LOG_LEVEL_INFO); - PyModule_AddIntConstant(aerospike, "LOG_LEVEL_DEBUG", LOG_LEVEL_DEBUG); - PyModule_AddIntConstant(aerospike, "LOG_LEVEL_TRACE", LOG_LEVEL_TRACE); -exit: - return status; -} - PyObject *Aerospike_Set_Log_Level(PyObject *parent, PyObject *args, PyObject *kwds) { @@ -208,7 +182,7 @@ PyObject *Aerospike_Set_Log_Handler(PyObject *parent, PyObject *args, void Aerospike_Enable_Default_Logging() { // Invoke C API to set log level - as_log_set_level((as_log_level)LOG_LEVEL_ERROR); + as_log_set_level((as_log_level)AS_LOG_LEVEL_ERROR); // Register callback to C-SDK as_log_set_callback((as_log_callback)console_log_cb); diff --git a/src/main/nullobject/type.c b/src/main/nullobject/type.c index b250873a3..9a290ed65 100644 --- a/src/main/nullobject/type.c +++ b/src/main/nullobject/type.c @@ -33,9 +33,9 @@ static void AerospikeNullObject_Type_Dealloc(AerospikeNullObject *self) ******************************************************************************/ static PyTypeObject AerospikeNullObject_Type = { - PyVarObject_HEAD_INIT(NULL, 0) "aerospike.null", // tp_name - sizeof(AerospikeNullObject), // tp_basicsize - 0, // tp_itemsize + PyVarObject_HEAD_INIT(NULL, 0) FULLY_QUALIFIED_TYPE_NAME("null"), // tp_name + sizeof(AerospikeNullObject), // tp_basicsize + 0, // tp_itemsize (destructor)AerospikeNullObject_Type_Dealloc, // tp_dealloc 0, // tp_print diff --git a/src/main/policy.c b/src/main/policy.c index 4670f65f9..40d2d3ed9 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -52,26 +52,42 @@ #define POLICY_UPDATE() *policy_p = policy; +// TODO: Python exceptions should be propagated up instead of being cleared +// but the policy helper functions don't handle this case and they only populate +// an as_error object and return a status code. +// That will take too much time to refactor, so just clear the exception and +// populate the as_error object instead. This currently makes it harder to +// debug why a C-API call failed though, because we don't have the exact +// exception that was thrown #define POLICY_SET_FIELD(__field, __type) \ { \ - PyObject *py_field = PyDict_GetItemString(py_policy, #__field); \ - if (py_field) { \ - if (PyLong_Check(py_field)) { \ - policy->__field = (__type)PyLong_AsLong(py_field); \ - } \ - else { \ - return as_error_update(err, AEROSPIKE_ERR_PARAM, \ - "%s is invalid", #__field); \ - } \ + PyObject *py_field_name = PyUnicode_FromString(#__field); \ + if (py_field_name == NULL) { \ + PyErr_Clear(); \ + return as_error_update(err, AEROSPIKE_ERR_CLIENT, \ + "Unable to create Python unicode object"); \ } \ - } - -#define POLICY_SET_BASE_FIELD(__field, __type) \ - { \ - PyObject *py_field = PyDict_GetItemString(py_policy, #__field); \ + PyObject *py_field = \ + PyDict_GetItemWithError(py_policy, py_field_name); \ + if (py_field == NULL && PyErr_Occurred()) { \ + PyErr_Clear(); \ + Py_DECREF(py_field_name); \ + return as_error_update( \ + err, AEROSPIKE_ERR_CLIENT, \ + "Unable to fetch field from policy dictionary"); \ + } \ + Py_DECREF(py_field_name); \ + \ if (py_field) { \ if (PyLong_Check(py_field)) { \ - policy->base.__field = (__type)PyLong_AsLong(py_field); \ + long field_val = PyLong_AsLong(py_field); \ + if (field_val == -1 && PyErr_Occurred()) { \ + PyErr_Clear(); \ + return as_error_update( \ + err, AEROSPIKE_ERR_CLIENT, \ + "Unable to fetch long value from policy field"); \ + } \ + policy->__field = (__type)field_val; \ } \ else { \ return as_error_update(err, AEROSPIKE_ERR_PARAM, \ @@ -80,30 +96,35 @@ } \ } -#define POLICY_SET_EXPRESSIONS_BASE_FIELD() \ +#define POLICY_SET_EXPRESSIONS_FIELD() \ { \ if (exp_list) { \ + PyObject *py_field_name = PyUnicode_FromString("expressions"); \ + if (py_field_name == NULL) { \ + PyErr_Clear(); \ + return as_error_update( \ + err, AEROSPIKE_ERR_CLIENT, \ + "Unable to create Python unicode object"); \ + } \ PyObject *py_exp_list = \ - PyDict_GetItemString(py_policy, "expressions"); \ + PyDict_GetItemWithError(py_policy, py_field_name); \ + if (py_exp_list == NULL && PyErr_Occurred()) { \ + PyErr_Clear(); \ + Py_DECREF(py_field_name); \ + return as_error_update(err, AEROSPIKE_ERR_CLIENT, \ + "Unable to fetch expressions field " \ + "from policy dictionary"); \ + } \ + Py_DECREF(py_field_name); \ if (py_exp_list) { \ if (convert_exp_list(self, py_exp_list, &exp_list, err) == \ AEROSPIKE_OK) { \ - policy->base.filter_exp = exp_list; \ + policy->filter_exp = exp_list; \ *exp_list_p = exp_list; \ } \ - } \ - } \ - } - -#define POLICY_SET_EXPRESSIONS_FIELD() \ - { \ - PyObject *py_exp_list = \ - PyDict_GetItemString(py_policy, "expressions"); \ - if (py_exp_list) { \ - if (convert_exp_list(self, py_exp_list, &exp_list, err) == \ - AEROSPIKE_OK) { \ - policy->filter_exp = exp_list; \ - *exp_list_p = exp_list; \ + else { \ + return err->code; \ + } \ } \ } \ } @@ -122,344 +143,6 @@ } \ } -/* - ******************************************************************************************************* - * Mapping of constant number to constant name string. - ******************************************************************************************************* - */ -static AerospikeConstants aerospike_constants[] = { - {AS_POLICY_RETRY_NONE, "POLICY_RETRY_NONE"}, - {AS_POLICY_RETRY_ONCE, "POLICY_RETRY_ONCE"}, - {AS_POLICY_EXISTS_IGNORE, "POLICY_EXISTS_IGNORE"}, - {AS_POLICY_EXISTS_CREATE, "POLICY_EXISTS_CREATE"}, - {AS_POLICY_EXISTS_UPDATE, "POLICY_EXISTS_UPDATE"}, - {AS_POLICY_EXISTS_REPLACE, "POLICY_EXISTS_REPLACE"}, - {AS_POLICY_EXISTS_CREATE_OR_REPLACE, "POLICY_EXISTS_CREATE_OR_REPLACE"}, - {AS_UDF_TYPE_LUA, "UDF_TYPE_LUA"}, - {AS_POLICY_KEY_DIGEST, "POLICY_KEY_DIGEST"}, - {AS_POLICY_KEY_SEND, "POLICY_KEY_SEND"}, - {AS_POLICY_GEN_IGNORE, "POLICY_GEN_IGNORE"}, - {AS_POLICY_GEN_EQ, "POLICY_GEN_EQ"}, - {AS_POLICY_GEN_GT, "POLICY_GEN_GT"}, - {AS_JOB_STATUS_COMPLETED, "JOB_STATUS_COMPLETED"}, - {AS_JOB_STATUS_UNDEF, "JOB_STATUS_UNDEF"}, - {AS_JOB_STATUS_INPROGRESS, "JOB_STATUS_INPROGRESS"}, - {AS_POLICY_REPLICA_MASTER, "POLICY_REPLICA_MASTER"}, - {AS_POLICY_REPLICA_ANY, "POLICY_REPLICA_ANY"}, - {AS_POLICY_REPLICA_SEQUENCE, "POLICY_REPLICA_SEQUENCE"}, - {AS_POLICY_REPLICA_PREFER_RACK, "POLICY_REPLICA_PREFER_RACK"}, - {AS_POLICY_COMMIT_LEVEL_ALL, "POLICY_COMMIT_LEVEL_ALL"}, - {AS_POLICY_COMMIT_LEVEL_MASTER, "POLICY_COMMIT_LEVEL_MASTER"}, - {SERIALIZER_USER, "SERIALIZER_USER"}, - {SERIALIZER_JSON, "SERIALIZER_JSON"}, - {SERIALIZER_NONE, "SERIALIZER_NONE"}, - {SEND_BOOL_AS_INTEGER, "INTEGER"}, - {SEND_BOOL_AS_AS_BOOL, "AS_BOOL"}, - {AS_INDEX_STRING, "INDEX_STRING"}, - {AS_INDEX_NUMERIC, "INDEX_NUMERIC"}, - {AS_INDEX_GEO2DSPHERE, "INDEX_GEO2DSPHERE"}, - {AS_INDEX_BLOB, "INDEX_BLOB"}, - {AS_INDEX_TYPE_DEFAULT, "INDEX_TYPE_DEFAULT"}, - {AS_INDEX_TYPE_LIST, "INDEX_TYPE_LIST"}, - {AS_INDEX_TYPE_MAPKEYS, "INDEX_TYPE_MAPKEYS"}, - {AS_INDEX_TYPE_MAPVALUES, "INDEX_TYPE_MAPVALUES"}, - {AS_PRIVILEGE_USER_ADMIN, "PRIV_USER_ADMIN"}, - {AS_PRIVILEGE_SYS_ADMIN, "PRIV_SYS_ADMIN"}, - {AS_PRIVILEGE_DATA_ADMIN, "PRIV_DATA_ADMIN"}, - {AS_PRIVILEGE_READ, "PRIV_READ"}, - {AS_PRIVILEGE_WRITE, "PRIV_WRITE"}, - {AS_PRIVILEGE_READ_WRITE, "PRIV_READ_WRITE"}, - {AS_PRIVILEGE_READ_WRITE_UDF, "PRIV_READ_WRITE_UDF"}, - {AS_PRIVILEGE_TRUNCATE, "PRIV_TRUNCATE"}, - {AS_PRIVILEGE_UDF_ADMIN, "PRIV_UDF_ADMIN"}, - {AS_PRIVILEGE_SINDEX_ADMIN, "PRIV_SINDEX_ADMIN"}, - - {OP_LIST_APPEND, "OP_LIST_APPEND"}, - {OP_LIST_APPEND_ITEMS, "OP_LIST_APPEND_ITEMS"}, - {OP_LIST_INSERT, "OP_LIST_INSERT"}, - {OP_LIST_INSERT_ITEMS, "OP_LIST_INSERT_ITEMS"}, - {OP_LIST_POP, "OP_LIST_POP"}, - {OP_LIST_POP_RANGE, "OP_LIST_POP_RANGE"}, - {OP_LIST_REMOVE, "OP_LIST_REMOVE"}, - {OP_LIST_REMOVE_RANGE, "OP_LIST_REMOVE_RANGE"}, - {OP_LIST_CLEAR, "OP_LIST_CLEAR"}, - {OP_LIST_SET, "OP_LIST_SET"}, - {OP_LIST_GET, "OP_LIST_GET"}, - {OP_LIST_GET_RANGE, "OP_LIST_GET_RANGE"}, - {OP_LIST_TRIM, "OP_LIST_TRIM"}, - {OP_LIST_SIZE, "OP_LIST_SIZE"}, - {OP_LIST_INCREMENT, "OP_LIST_INCREMENT"}, - - {OP_MAP_SET_POLICY, "OP_MAP_SET_POLICY"}, - {OP_MAP_CREATE, "OP_MAP_CREATE"}, - {OP_MAP_PUT, "OP_MAP_PUT"}, - {OP_MAP_PUT_ITEMS, "OP_MAP_PUT_ITEMS"}, - {OP_MAP_INCREMENT, "OP_MAP_INCREMENT"}, - {OP_MAP_DECREMENT, "OP_MAP_DECREMENT"}, - {OP_MAP_SIZE, "OP_MAP_SIZE"}, - {OP_MAP_CLEAR, "OP_MAP_CLEAR"}, - {OP_MAP_REMOVE_BY_KEY, "OP_MAP_REMOVE_BY_KEY"}, - {OP_MAP_REMOVE_BY_KEY_LIST, "OP_MAP_REMOVE_BY_KEY_LIST"}, - {OP_MAP_REMOVE_BY_KEY_RANGE, "OP_MAP_REMOVE_BY_KEY_RANGE"}, - {OP_MAP_REMOVE_BY_VALUE, "OP_MAP_REMOVE_BY_VALUE"}, - {OP_MAP_REMOVE_BY_VALUE_LIST, "OP_MAP_REMOVE_BY_VALUE_LIST"}, - {OP_MAP_REMOVE_BY_VALUE_RANGE, "OP_MAP_REMOVE_BY_VALUE_RANGE"}, - {OP_MAP_REMOVE_BY_INDEX, "OP_MAP_REMOVE_BY_INDEX"}, - {OP_MAP_REMOVE_BY_INDEX_RANGE, "OP_MAP_REMOVE_BY_INDEX_RANGE"}, - {OP_MAP_REMOVE_BY_RANK, "OP_MAP_REMOVE_BY_RANK"}, - {OP_MAP_REMOVE_BY_RANK_RANGE, "OP_MAP_REMOVE_BY_RANK_RANGE"}, - {OP_MAP_GET_BY_KEY, "OP_MAP_GET_BY_KEY"}, - {OP_MAP_GET_BY_KEY_RANGE, "OP_MAP_GET_BY_KEY_RANGE"}, - {OP_MAP_GET_BY_VALUE, "OP_MAP_GET_BY_VALUE"}, - {OP_MAP_GET_BY_VALUE_RANGE, "OP_MAP_GET_BY_VALUE_RANGE"}, - {OP_MAP_GET_BY_INDEX, "OP_MAP_GET_BY_INDEX"}, - {OP_MAP_GET_BY_INDEX_RANGE, "OP_MAP_GET_BY_INDEX_RANGE"}, - {OP_MAP_GET_BY_RANK, "OP_MAP_GET_BY_RANK"}, - {OP_MAP_GET_BY_RANK_RANGE, "OP_MAP_GET_BY_RANK_RANGE"}, - {OP_MAP_GET_BY_VALUE_LIST, "OP_MAP_GET_BY_VALUE_LIST"}, - {OP_MAP_GET_BY_KEY_LIST, "OP_MAP_GET_BY_KEY_LIST"}, - - {AS_MAP_UNORDERED, "MAP_UNORDERED"}, - {AS_MAP_KEY_ORDERED, "MAP_KEY_ORDERED"}, - {AS_MAP_KEY_VALUE_ORDERED, "MAP_KEY_VALUE_ORDERED"}, - - {AS_MAP_RETURN_NONE, "MAP_RETURN_NONE"}, - {AS_MAP_RETURN_INDEX, "MAP_RETURN_INDEX"}, - {AS_MAP_RETURN_REVERSE_INDEX, "MAP_RETURN_REVERSE_INDEX"}, - {AS_MAP_RETURN_RANK, "MAP_RETURN_RANK"}, - {AS_MAP_RETURN_REVERSE_RANK, "MAP_RETURN_REVERSE_RANK"}, - {AS_MAP_RETURN_COUNT, "MAP_RETURN_COUNT"}, - {AS_MAP_RETURN_KEY, "MAP_RETURN_KEY"}, - {AS_MAP_RETURN_VALUE, "MAP_RETURN_VALUE"}, - {AS_MAP_RETURN_KEY_VALUE, "MAP_RETURN_KEY_VALUE"}, - {AS_MAP_RETURN_EXISTS, "MAP_RETURN_EXISTS"}, - {AS_MAP_RETURN_ORDERED_MAP, "MAP_RETURN_ORDERED_MAP"}, - {AS_MAP_RETURN_UNORDERED_MAP, "MAP_RETURN_UNORDERED_MAP"}, - - {AS_RECORD_DEFAULT_TTL, "TTL_NAMESPACE_DEFAULT"}, - {AS_RECORD_NO_EXPIRE_TTL, "TTL_NEVER_EXPIRE"}, - {AS_RECORD_NO_CHANGE_TTL, "TTL_DONT_UPDATE"}, - {AS_RECORD_CLIENT_DEFAULT_TTL, "TTL_CLIENT_DEFAULT"}, - {AS_AUTH_INTERNAL, "AUTH_INTERNAL"}, - {AS_AUTH_EXTERNAL, "AUTH_EXTERNAL"}, - {AS_AUTH_EXTERNAL_INSECURE, "AUTH_EXTERNAL_INSECURE"}, - {AS_AUTH_PKI, "AUTH_PKI"}, - /* New CDT Operations, post 3.16.0.1 */ - {OP_LIST_GET_BY_INDEX, "OP_LIST_GET_BY_INDEX"}, - {OP_LIST_GET_BY_INDEX_RANGE, "OP_LIST_GET_BY_INDEX_RANGE"}, - {OP_LIST_GET_BY_RANK, "OP_LIST_GET_BY_RANK"}, - {OP_LIST_GET_BY_RANK_RANGE, "OP_LIST_GET_BY_RANK_RANGE"}, - {OP_LIST_GET_BY_VALUE, "OP_LIST_GET_BY_VALUE"}, - {OP_LIST_GET_BY_VALUE_LIST, "OP_LIST_GET_BY_VALUE_LIST"}, - {OP_LIST_GET_BY_VALUE_RANGE, "OP_LIST_GET_BY_VALUE_RANGE"}, - {OP_LIST_REMOVE_BY_INDEX, "OP_LIST_REMOVE_BY_INDEX"}, - {OP_LIST_REMOVE_BY_INDEX_RANGE, "OP_LIST_REMOVE_BY_INDEX_RANGE"}, - {OP_LIST_REMOVE_BY_RANK, "OP_LIST_REMOVE_BY_RANK"}, - {OP_LIST_REMOVE_BY_RANK_RANGE, "OP_LIST_REMOVE_BY_RANK_RANGE"}, - {OP_LIST_REMOVE_BY_VALUE, "OP_LIST_REMOVE_BY_VALUE"}, - {OP_LIST_REMOVE_BY_VALUE_LIST, "OP_LIST_REMOVE_BY_VALUE_LIST"}, - {OP_LIST_REMOVE_BY_VALUE_RANGE, "OP_LIST_REMOVE_BY_VALUE_RANGE"}, - {OP_LIST_SET_ORDER, "OP_LIST_SET_ORDER"}, - {OP_LIST_SORT, "OP_LIST_SORT"}, - {AS_LIST_RETURN_NONE, "LIST_RETURN_NONE"}, - {AS_LIST_RETURN_INDEX, "LIST_RETURN_INDEX"}, - {AS_LIST_RETURN_REVERSE_INDEX, "LIST_RETURN_REVERSE_INDEX"}, - {AS_LIST_RETURN_RANK, "LIST_RETURN_RANK"}, - {AS_LIST_RETURN_REVERSE_RANK, "LIST_RETURN_REVERSE_RANK"}, - {AS_LIST_RETURN_COUNT, "LIST_RETURN_COUNT"}, - {AS_LIST_RETURN_VALUE, "LIST_RETURN_VALUE"}, - {AS_LIST_RETURN_EXISTS, "LIST_RETURN_EXISTS"}, - {AS_LIST_SORT_DROP_DUPLICATES, "LIST_SORT_DROP_DUPLICATES"}, - {AS_LIST_SORT_DEFAULT, "LIST_SORT_DEFAULT"}, - {AS_LIST_WRITE_DEFAULT, "LIST_WRITE_DEFAULT"}, - {AS_LIST_WRITE_ADD_UNIQUE, "LIST_WRITE_ADD_UNIQUE"}, - {AS_LIST_WRITE_INSERT_BOUNDED, "LIST_WRITE_INSERT_BOUNDED"}, - {AS_LIST_ORDERED, "LIST_ORDERED"}, - {AS_LIST_UNORDERED, "LIST_UNORDERED"}, - {OP_LIST_REMOVE_BY_VALUE_RANK_RANGE_REL, - "OP_LIST_REMOVE_BY_VALUE_RANK_RANGE_REL"}, - {OP_LIST_GET_BY_VALUE_RANK_RANGE_REL, - "OP_LIST_GET_BY_VALUE_RANK_RANGE_REL"}, - {OP_LIST_CREATE, "OP_LIST_CREATE"}, - - /* CDT operations for use with expressions, new in 5.0 */ - {OP_MAP_REMOVE_BY_VALUE_RANK_RANGE_REL, - "OP_MAP_REMOVE_BY_VALUE_RANK_RANGE_REL"}, - {OP_MAP_REMOVE_BY_KEY_INDEX_RANGE_REL, - "OP_MAP_REMOVE_BY_KEY_INDEX_RANGE_REL"}, - {OP_MAP_GET_BY_VALUE_RANK_RANGE_REL, "OP_MAP_GET_BY_VALUE_RANK_RANGE_REL"}, - {OP_MAP_GET_BY_KEY_INDEX_RANGE_REL, "OP_MAP_GET_BY_KEY_INDEX_RANGE_REL"}, - - {OP_LIST_GET_BY_VALUE_RANK_RANGE_REL_TO_END, - "OP_LIST_GET_BY_VALUE_RANK_RANGE_REL_TO_END"}, - {OP_LIST_GET_BY_INDEX_RANGE_TO_END, "OP_LIST_GET_BY_INDEX_RANGE_TO_END"}, - {OP_LIST_GET_BY_RANK_RANGE_TO_END, "OP_LIST_GET_BY_RANK_RANGE_TO_END"}, - {OP_LIST_REMOVE_BY_REL_RANK_RANGE_TO_END, - "OP_LIST_REMOVE_BY_REL_RANK_RANGE_TO_END"}, - {OP_LIST_REMOVE_BY_REL_RANK_RANGE, "OP_LIST_REMOVE_BY_REL_RANK_RANGE"}, - {OP_LIST_REMOVE_BY_INDEX_RANGE_TO_END, - "OP_LIST_REMOVE_BY_INDEX_RANGE_TO_END"}, - {OP_LIST_REMOVE_BY_RANK_RANGE_TO_END, - "OP_LIST_REMOVE_BY_RANK_RANGE_TO_END"}, - - {AS_MAP_WRITE_NO_FAIL, "MAP_WRITE_NO_FAIL"}, - {AS_MAP_WRITE_PARTIAL, "MAP_WRITE_PARTIAL"}, - - {AS_LIST_WRITE_NO_FAIL, "LIST_WRITE_NO_FAIL"}, - {AS_LIST_WRITE_PARTIAL, "LIST_WRITE_PARTIAL"}, - - /* Map write flags post 3.5.0 */ - {AS_MAP_WRITE_DEFAULT, "MAP_WRITE_FLAGS_DEFAULT"}, - {AS_MAP_WRITE_CREATE_ONLY, "MAP_WRITE_FLAGS_CREATE_ONLY"}, - {AS_MAP_WRITE_UPDATE_ONLY, "MAP_WRITE_FLAGS_UPDATE_ONLY"}, - {AS_MAP_WRITE_NO_FAIL, "MAP_WRITE_FLAGS_NO_FAIL"}, - {AS_MAP_WRITE_PARTIAL, "MAP_WRITE_FLAGS_PARTIAL"}, - - /* READ Mode constants 4.0.0 */ - - // AP Read Mode - {AS_POLICY_READ_MODE_AP_ONE, "POLICY_READ_MODE_AP_ONE"}, - {AS_POLICY_READ_MODE_AP_ALL, "POLICY_READ_MODE_AP_ALL"}, - // SC Read Mode - {AS_POLICY_READ_MODE_SC_SESSION, "POLICY_READ_MODE_SC_SESSION"}, - {AS_POLICY_READ_MODE_SC_LINEARIZE, "POLICY_READ_MODE_SC_LINEARIZE"}, - {AS_POLICY_READ_MODE_SC_ALLOW_REPLICA, "POLICY_READ_MODE_SC_ALLOW_REPLICA"}, - {AS_POLICY_READ_MODE_SC_ALLOW_UNAVAILABLE, - "POLICY_READ_MODE_SC_ALLOW_UNAVAILABLE"}, - - /* Bitwise constants: 3.9.0 */ - {AS_BIT_WRITE_DEFAULT, "BIT_WRITE_DEFAULT"}, - {AS_BIT_WRITE_CREATE_ONLY, "BIT_WRITE_CREATE_ONLY"}, - {AS_BIT_WRITE_UPDATE_ONLY, "BIT_WRITE_UPDATE_ONLY"}, - {AS_BIT_WRITE_NO_FAIL, "BIT_WRITE_NO_FAIL"}, - {AS_BIT_WRITE_PARTIAL, "BIT_WRITE_PARTIAL"}, - - {AS_BIT_RESIZE_DEFAULT, "BIT_RESIZE_DEFAULT"}, - {AS_BIT_RESIZE_FROM_FRONT, "BIT_RESIZE_FROM_FRONT"}, - {AS_BIT_RESIZE_GROW_ONLY, "BIT_RESIZE_GROW_ONLY"}, - {AS_BIT_RESIZE_SHRINK_ONLY, "BIT_RESIZE_SHRINK_ONLY"}, - - {AS_BIT_OVERFLOW_FAIL, "BIT_OVERFLOW_FAIL"}, - {AS_BIT_OVERFLOW_SATURATE, "BIT_OVERFLOW_SATURATE"}, - {AS_BIT_OVERFLOW_WRAP, "BIT_OVERFLOW_WRAP"}, - - /* BITWISE OPS: 3.9.0 */ - {OP_BIT_INSERT, "OP_BIT_INSERT"}, - {OP_BIT_RESIZE, "OP_BIT_RESIZE"}, - {OP_BIT_REMOVE, "OP_BIT_REMOVE"}, - {OP_BIT_SET, "OP_BIT_SET"}, - {OP_BIT_OR, "OP_BIT_OR"}, - {OP_BIT_XOR, "OP_BIT_XOR"}, - {OP_BIT_AND, "OP_BIT_AND"}, - {OP_BIT_NOT, "OP_BIT_NOT"}, - {OP_BIT_LSHIFT, "OP_BIT_LSHIFT"}, - {OP_BIT_RSHIFT, "OP_BIT_RSHIFT"}, - {OP_BIT_ADD, "OP_BIT_ADD"}, - {OP_BIT_SUBTRACT, "OP_BIT_SUBTRACT"}, - {OP_BIT_GET_INT, "OP_BIT_GET_INT"}, - {OP_BIT_SET_INT, "OP_BIT_SET_INT"}, - {OP_BIT_GET, "OP_BIT_GET"}, - {OP_BIT_COUNT, "OP_BIT_COUNT"}, - {OP_BIT_LSCAN, "OP_BIT_LSCAN"}, - {OP_BIT_RSCAN, "OP_BIT_RSCAN"}, - - /* Nested CDT constants: 3.9.0 */ - {AS_CDT_CTX_LIST_INDEX, "CDT_CTX_LIST_INDEX"}, - {AS_CDT_CTX_LIST_RANK, "CDT_CTX_LIST_RANK"}, - {AS_CDT_CTX_LIST_VALUE, "CDT_CTX_LIST_VALUE"}, - {CDT_CTX_LIST_INDEX_CREATE, "CDT_CTX_LIST_INDEX_CREATE"}, - {AS_CDT_CTX_MAP_INDEX, "CDT_CTX_MAP_INDEX"}, - {AS_CDT_CTX_MAP_RANK, "CDT_CTX_MAP_RANK"}, - {AS_CDT_CTX_MAP_KEY, "CDT_CTX_MAP_KEY"}, - {AS_CDT_CTX_MAP_VALUE, "CDT_CTX_MAP_VALUE"}, - {CDT_CTX_MAP_KEY_CREATE, "CDT_CTX_MAP_KEY_CREATE"}, - - /* HLL constants 3.11.0 */ - {OP_HLL_ADD, "OP_HLL_ADD"}, - {OP_HLL_DESCRIBE, "OP_HLL_DESCRIBE"}, - {OP_HLL_FOLD, "OP_HLL_FOLD"}, - {OP_HLL_GET_COUNT, "OP_HLL_GET_COUNT"}, - {OP_HLL_GET_INTERSECT_COUNT, "OP_HLL_GET_INTERSECT_COUNT"}, - {OP_HLL_GET_SIMILARITY, "OP_HLL_GET_SIMILARITY"}, - {OP_HLL_GET_UNION, "OP_HLL_GET_UNION"}, - {OP_HLL_GET_UNION_COUNT, "OP_HLL_GET_UNION_COUNT"}, - {OP_HLL_GET_SIMILARITY, "OP_HLL_GET_SIMILARITY"}, - {OP_HLL_INIT, "OP_HLL_INIT"}, - {OP_HLL_REFRESH_COUNT, "OP_HLL_REFRESH_COUNT"}, - {OP_HLL_SET_UNION, "OP_HLL_SET_UNION"}, - {OP_HLL_MAY_CONTAIN, "OP_HLL_MAY_CONTAIN"}, // for expression filters - - {AS_HLL_WRITE_DEFAULT, "HLL_WRITE_DEFAULT"}, - {AS_HLL_WRITE_CREATE_ONLY, "HLL_WRITE_CREATE_ONLY"}, - {AS_HLL_WRITE_UPDATE_ONLY, "HLL_WRITE_UPDATE_ONLY"}, - {AS_HLL_WRITE_NO_FAIL, "HLL_WRITE_NO_FAIL"}, - {AS_HLL_WRITE_ALLOW_FOLD, "HLL_WRITE_ALLOW_FOLD"}, - - {OP_MAP_REMOVE_BY_KEY_REL_INDEX_RANGE_TO_END, - "OP_MAP_REMOVE_BY_KEY_REL_INDEX_RANGE_TO_END"}, - {OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE_TO_END, - "OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE_TO_END"}, - {OP_MAP_REMOVE_BY_INDEX_RANGE_TO_END, - "OP_MAP_REMOVE_BY_INDEX_RANGE_TO_END"}, - {OP_MAP_REMOVE_BY_RANK_RANGE_TO_END, "OP_MAP_REMOVE_BY_RANK_RANGE_TO_END"}, - {OP_MAP_GET_BY_KEY_REL_INDEX_RANGE_TO_END, - "OP_MAP_GET_BY_KEY_REL_INDEX_RANGE_TO_END"}, - {OP_MAP_REMOVE_BY_KEY_REL_INDEX_RANGE, - "OP_MAP_REMOVE_BY_KEY_REL_INDEX_RANGE"}, - {OP_MAP_REMOVE_BY_VALUE_REL_INDEX_RANGE, - "OP_MAP_REMOVE_BY_VALUE_REL_INDEX_RANGE"}, - {OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE, - "OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE"}, - {OP_MAP_GET_BY_KEY_REL_INDEX_RANGE, "OP_MAP_GET_BY_KEY_REL_INDEX_RANGE"}, - {OP_MAP_GET_BY_VALUE_RANK_RANGE_REL_TO_END, - "OP_MAP_GET_BY_VALUE_RANK_RANGE_REL_TO_END"}, - {OP_MAP_GET_BY_INDEX_RANGE_TO_END, "OP_MAP_GET_BY_INDEX_RANGE_TO_END"}, - {OP_MAP_GET_BY_RANK_RANGE_TO_END, "OP_MAP_GET_BY_RANK_RANGE_TO_END"}, - - /* Expression operation constants 5.1.0 */ - {OP_EXPR_READ, "OP_EXPR_READ"}, - {OP_EXPR_WRITE, "OP_EXPR_WRITE"}, - {AS_EXP_WRITE_DEFAULT, "EXP_WRITE_DEFAULT"}, - {AS_EXP_WRITE_CREATE_ONLY, "EXP_WRITE_CREATE_ONLY"}, - {AS_EXP_WRITE_UPDATE_ONLY, "EXP_WRITE_UPDATE_ONLY"}, - {AS_EXP_WRITE_ALLOW_DELETE, "EXP_WRITE_ALLOW_DELETE"}, - {AS_EXP_WRITE_POLICY_NO_FAIL, "EXP_WRITE_POLICY_NO_FAIL"}, - {AS_EXP_WRITE_EVAL_NO_FAIL, "EXP_WRITE_EVAL_NO_FAIL"}, - {AS_EXP_READ_DEFAULT, "EXP_READ_DEFAULT"}, - {AS_EXP_READ_EVAL_NO_FAIL, "EXP_READ_EVAL_NO_FAIL"}, - - /* For BinType expression, as_bytes_type */ - {AS_BYTES_UNDEF, "AS_BYTES_UNDEF"}, - {AS_BYTES_INTEGER, "AS_BYTES_INTEGER"}, - {AS_BYTES_DOUBLE, "AS_BYTES_DOUBLE"}, - {AS_BYTES_STRING, "AS_BYTES_STRING"}, - {AS_BYTES_BLOB, "AS_BYTES_BLOB"}, - {AS_BYTES_JAVA, "AS_BYTES_JAVA"}, - {AS_BYTES_CSHARP, "AS_BYTES_CSHARP"}, - {AS_BYTES_PYTHON, "AS_BYTES_PYTHON"}, - {AS_BYTES_RUBY, "AS_BYTES_RUBY"}, - {AS_BYTES_PHP, "AS_BYTES_PHP"}, - {AS_BYTES_ERLANG, "AS_BYTES_ERLANG"}, - {AS_BYTES_BOOL, "AS_BYTES_BOOL"}, - {AS_BYTES_HLL, "AS_BYTES_HLL"}, - {AS_BYTES_MAP, "AS_BYTES_MAP"}, - {AS_BYTES_LIST, "AS_BYTES_LIST"}, - {AS_BYTES_GEOJSON, "AS_BYTES_GEOJSON"}, - {AS_BYTES_TYPE_MAX, "AS_BYTES_TYPE_MAX"}, - - /* Regex constants from predexp, still used by expressions */ - {REGEX_NONE, "REGEX_NONE"}, - {REGEX_EXTENDED, "REGEX_EXTENDED"}, - {REGEX_ICASE, "REGEX_ICASE"}, - {REGEX_NOSUB, "REGEX_NOSUB"}, - {REGEX_NEWLINE, "REGEX_NEWLINE"}, - - {AS_QUERY_DURATION_LONG, "QUERY_DURATION_LONG"}, - {AS_QUERY_DURATION_LONG_RELAX_AP, "QUERY_DURATION_LONG_RELAX_AP"}, - {AS_QUERY_DURATION_SHORT, "QUERY_DURATION_SHORT"}}; - -static AerospikeJobConstants aerospike_job_constants[] = { - {"scan", "JOB_SCAN"}, {"query", "JOB_QUERY"}}; /** * Function for setting scan parameters in scan. * Like Percentage, Concurrent, Nobins @@ -549,31 +232,6 @@ as_status set_query_options(as_error *err, PyObject *query_options, } return AEROSPIKE_OK; } -/** - * Declares policy constants. - */ -as_status declare_policy_constants(PyObject *aerospike) -{ - as_status status = AEROSPIKE_OK; - int i; - - if (!aerospike) { - status = AEROSPIKE_ERR; - goto exit; - } - for (i = 0; i < (int)AEROSPIKE_CONSTANTS_ARR_SIZE; i++) { - PyModule_AddIntConstant(aerospike, aerospike_constants[i].constant_str, - aerospike_constants[i].constantno); - } - - for (i = 0; i < (int)AEROSPIKE_JOB_CONSTANTS_ARR_SIZE; i++) { - PyModule_AddStringConstant(aerospike, - aerospike_job_constants[i].exposed_job_str, - aerospike_job_constants[i].job_str); - } -exit: - return status; -} /** * Converts a PyObject into an as_policy_admin object. @@ -582,8 +240,7 @@ as_status declare_policy_constants(PyObject *aerospike) * and initialized (although, we do reset the error object here). */ as_status pyobject_to_policy_admin(AerospikeClient *self, as_error *err, - PyObject *py_policy, // remove self - as_policy_admin *policy, + PyObject *py_policy, as_policy_admin *policy, as_policy_admin **policy_p, as_policy_admin *config_admin_policy) { @@ -605,6 +262,65 @@ as_status pyobject_to_policy_admin(AerospikeClient *self, as_error *err, return err->code; } +static inline void check_and_set_txn_field(as_error *err, + as_policy_base *policy_base, + PyObject *py_policy) +{ + PyObject *py_txn_field_name = PyUnicode_FromString("txn"); + if (py_txn_field_name == NULL) { + as_error_update(err, AEROSPIKE_ERR_CLIENT, + "Unable to create Python string \"txn\""); + return; + } + PyObject *py_obj_txn = + PyDict_GetItemWithError(py_policy, py_txn_field_name); + Py_DECREF(py_txn_field_name); + if (py_obj_txn == NULL) { + if (PyErr_Occurred()) { + PyErr_Clear(); + as_error_update(err, AEROSPIKE_ERR_CLIENT, + "Getting the transaction field from Python policy " + "dictionary returned a non-KeyError exception"); + } + // Whether or not a key error was raised + return; + } + + PyTypeObject *py_expected_field_type = &AerospikeTransaction_Type; + if (Py_TYPE(py_obj_txn) != py_expected_field_type) { + // TypeError should be set here, + // but there is no Aerospike exception to represent that error + as_error_update(err, AEROSPIKE_ERR_PARAM, "txn is not of type %s", + py_expected_field_type->tp_name); + return; + } + + AerospikeTransaction *py_txn = (AerospikeTransaction *)py_obj_txn; + policy_base->txn = py_txn->txn; +} + +static inline as_status +pyobject_to_policy_base(AerospikeClient *self, as_error *err, + PyObject *py_policy, as_policy_base *policy, + as_exp *exp_list, as_exp **exp_list_p) +{ + POLICY_SET_FIELD(total_timeout, uint32_t); + POLICY_SET_FIELD(socket_timeout, uint32_t); + POLICY_SET_FIELD(max_retries, uint32_t); + POLICY_SET_FIELD(sleep_between_retries, uint32_t); + POLICY_SET_FIELD(compress, bool); + + // Setting txn field to a non-NULL value in a query or scan policy is a no-op, + // so this is safe to call for a scan/query policy's base policy + check_and_set_txn_field(err, policy, py_policy); + if (err->code != AEROSPIKE_OK) { + return err->code; + } + + POLICY_SET_EXPRESSIONS_FIELD(); + return AEROSPIKE_OK; +} + /** * Converts a PyObject into an as_policy_apply object. * Returns AEROSPIKE_OK on success. On error, the err argument is populated. @@ -626,11 +342,11 @@ as_status pyobject_to_policy_apply(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - POLICY_SET_BASE_FIELD(total_timeout, uint32_t); - POLICY_SET_BASE_FIELD(socket_timeout, uint32_t); - POLICY_SET_BASE_FIELD(max_retries, uint32_t); - POLICY_SET_BASE_FIELD(sleep_between_retries, uint32_t); - POLICY_SET_BASE_FIELD(compress, bool); + as_status retval = pyobject_to_policy_base( + self, err, py_policy, &policy->base, exp_list, exp_list_p); + if (retval != AEROSPIKE_OK) { + return retval; + } POLICY_SET_FIELD(key, as_policy_key); POLICY_SET_FIELD(replica, as_policy_replica); @@ -638,9 +354,6 @@ as_status pyobject_to_policy_apply(AerospikeClient *self, as_error *err, POLICY_SET_FIELD(commit_level, as_policy_commit_level); POLICY_SET_FIELD(durable_delete, bool); POLICY_SET_FIELD(ttl, uint32_t); - - // C client 5.0 new expressions - POLICY_SET_EXPRESSIONS_BASE_FIELD(); } // Update the policy @@ -700,19 +413,14 @@ as_status pyobject_to_policy_query(AerospikeClient *self, as_error *err, as_policy_query_copy(config_query_policy, policy); if (py_policy && py_policy != Py_None) { - // Set policy fields - POLICY_SET_BASE_FIELD(total_timeout, uint32_t); - POLICY_SET_BASE_FIELD(socket_timeout, uint32_t); - POLICY_SET_BASE_FIELD(max_retries, uint32_t); - POLICY_SET_BASE_FIELD(sleep_between_retries, uint32_t); - POLICY_SET_BASE_FIELD(compress, bool); - + as_status retval = pyobject_to_policy_base( + self, err, py_policy, &policy->base, exp_list, exp_list_p); + if (retval != AEROSPIKE_OK) { + return retval; + } POLICY_SET_FIELD(deserialize, bool); POLICY_SET_FIELD(replica, as_policy_replica); - // C client 5.0 new expressions - POLICY_SET_EXPRESSIONS_BASE_FIELD(); - // C client 6.0.0 POLICY_SET_FIELD(short_query, bool); @@ -747,11 +455,11 @@ as_status pyobject_to_policy_read(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - POLICY_SET_BASE_FIELD(total_timeout, uint32_t); - POLICY_SET_BASE_FIELD(socket_timeout, uint32_t); - POLICY_SET_BASE_FIELD(max_retries, uint32_t); - POLICY_SET_BASE_FIELD(sleep_between_retries, uint32_t); - POLICY_SET_BASE_FIELD(compress, bool); + as_status retval = pyobject_to_policy_base( + self, err, py_policy, &policy->base, exp_list, exp_list_p); + if (retval != AEROSPIKE_OK) { + return retval; + } POLICY_SET_FIELD(key, as_policy_key); POLICY_SET_FIELD(replica, as_policy_replica); @@ -761,9 +469,6 @@ as_status pyobject_to_policy_read(AerospikeClient *self, as_error *err, // 4.0.0 new policies POLICY_SET_FIELD(read_mode_ap, as_policy_read_mode_ap); POLICY_SET_FIELD(read_mode_sc, as_policy_read_mode_sc); - - // C client 5.0 new expressions - POLICY_SET_EXPRESSIONS_BASE_FIELD(); } // Update the policy @@ -794,11 +499,11 @@ as_status pyobject_to_policy_remove(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - POLICY_SET_BASE_FIELD(total_timeout, uint32_t); - POLICY_SET_BASE_FIELD(socket_timeout, uint32_t); - POLICY_SET_BASE_FIELD(max_retries, uint32_t); - POLICY_SET_BASE_FIELD(sleep_between_retries, uint32_t); - POLICY_SET_BASE_FIELD(compress, bool); + as_status retval = pyobject_to_policy_base( + self, err, py_policy, &policy->base, exp_list, exp_list_p); + if (retval != AEROSPIKE_OK) { + return retval; + } POLICY_SET_FIELD(generation, uint16_t); @@ -807,9 +512,6 @@ as_status pyobject_to_policy_remove(AerospikeClient *self, as_error *err, POLICY_SET_FIELD(commit_level, as_policy_commit_level); POLICY_SET_FIELD(replica, as_policy_replica); POLICY_SET_FIELD(durable_delete, bool); - - // C client 5.0 new expressions - POLICY_SET_EXPRESSIONS_BASE_FIELD(); } // Update the policy @@ -839,19 +541,16 @@ as_status pyobject_to_policy_scan(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - POLICY_SET_BASE_FIELD(total_timeout, uint32_t); - POLICY_SET_BASE_FIELD(socket_timeout, uint32_t); - POLICY_SET_BASE_FIELD(max_retries, uint32_t); - POLICY_SET_BASE_FIELD(sleep_between_retries, uint32_t); - POLICY_SET_BASE_FIELD(compress, bool); + as_status retval = pyobject_to_policy_base( + self, err, py_policy, &policy->base, exp_list, exp_list_p); + if (retval != AEROSPIKE_OK) { + return retval; + } POLICY_SET_FIELD(durable_delete, bool); POLICY_SET_FIELD(records_per_second, uint32_t); POLICY_SET_FIELD(max_records, uint64_t); POLICY_SET_FIELD(replica, as_policy_replica); - - // C client 5.0 new expressions - POLICY_SET_EXPRESSIONS_BASE_FIELD(); } // Update the policy @@ -881,12 +580,11 @@ as_status pyobject_to_policy_write(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - // Base policy_fields - POLICY_SET_BASE_FIELD(total_timeout, uint32_t); - POLICY_SET_BASE_FIELD(socket_timeout, uint32_t); - POLICY_SET_BASE_FIELD(max_retries, uint32_t); - POLICY_SET_BASE_FIELD(sleep_between_retries, uint32_t); - POLICY_SET_BASE_FIELD(compress, bool); + as_status retval = pyobject_to_policy_base( + self, err, py_policy, &policy->base, exp_list, exp_list_p); + if (retval != AEROSPIKE_OK) { + return retval; + } POLICY_SET_FIELD(key, as_policy_key); POLICY_SET_FIELD(gen, as_policy_gen); @@ -895,9 +593,6 @@ as_status pyobject_to_policy_write(AerospikeClient *self, as_error *err, POLICY_SET_FIELD(durable_delete, bool); POLICY_SET_FIELD(replica, as_policy_replica); POLICY_SET_FIELD(compression_threshold, uint32_t); - - // C client 5.0 new expressions - POLICY_SET_EXPRESSIONS_BASE_FIELD(); } // Update the policy @@ -928,11 +623,11 @@ as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - POLICY_SET_BASE_FIELD(total_timeout, uint32_t); - POLICY_SET_BASE_FIELD(socket_timeout, uint32_t); - POLICY_SET_BASE_FIELD(max_retries, uint32_t); - POLICY_SET_BASE_FIELD(sleep_between_retries, uint32_t); - POLICY_SET_BASE_FIELD(compress, bool); + as_status retval = pyobject_to_policy_base( + self, err, py_policy, &policy->base, exp_list, exp_list_p); + if (retval != AEROSPIKE_OK) { + return retval; + } POLICY_SET_FIELD(key, as_policy_key); POLICY_SET_FIELD(gen, as_policy_gen); @@ -946,9 +641,6 @@ as_status pyobject_to_policy_operate(AerospikeClient *self, as_error *err, // 4.0.0 new policies POLICY_SET_FIELD(read_mode_ap, as_policy_read_mode_ap); POLICY_SET_FIELD(read_mode_sc, as_policy_read_mode_sc); - - // C client 5.0 new expressions - POLICY_SET_EXPRESSIONS_BASE_FIELD(); } // Update the policy @@ -978,11 +670,11 @@ as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, if (py_policy && py_policy != Py_None) { // Set policy fields - POLICY_SET_BASE_FIELD(total_timeout, uint32_t); - POLICY_SET_BASE_FIELD(socket_timeout, uint32_t); - POLICY_SET_BASE_FIELD(max_retries, uint32_t); - POLICY_SET_BASE_FIELD(sleep_between_retries, uint32_t); - POLICY_SET_BASE_FIELD(compress, bool); + as_status retval = pyobject_to_policy_base( + self, err, py_policy, &policy->base, exp_list, exp_list_p); + if (retval != AEROSPIKE_OK) { + return retval; + } POLICY_SET_FIELD(concurrent, bool); POLICY_SET_FIELD(allow_inline, bool); @@ -994,9 +686,6 @@ as_status pyobject_to_policy_batch(AerospikeClient *self, as_error *err, POLICY_SET_FIELD(read_mode_ap, as_policy_read_mode_ap); POLICY_SET_FIELD(read_mode_sc, as_policy_read_mode_sc); - // C client 5.0 new expressions - POLICY_SET_EXPRESSIONS_BASE_FIELD(); - // C client 6.0.0 (batch writes) POLICY_SET_FIELD(allow_inline_ssd, bool); POLICY_SET_FIELD(respond_all_keys, bool); diff --git a/src/main/query/apply.c b/src/main/query/apply.c index aae7e8f5c..f78c273cb 100644 --- a/src/main/query/apply.c +++ b/src/main/query/apply.c @@ -37,16 +37,14 @@ AerospikeQuery *AerospikeQuery_Apply(AerospikeQuery *self, PyObject *args, PyObject *py_module = NULL; PyObject *py_function = NULL; PyObject *py_args = NULL; - PyObject *py_policy = NULL; PyObject *py_umodule = NULL; PyObject *py_ufunction = NULL; // Python function keyword arguments - static char *kwlist[] = {"module", "function", "arguments", "policy", NULL}; + static char *kwlist[] = {"module", "function", "arguments", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO:apply", kwlist, - &py_module, &py_function, &py_args, - &py_policy)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O:apply", kwlist, + &py_module, &py_function, &py_args)) { return NULL; } @@ -152,6 +150,7 @@ AerospikeQuery *AerospikeQuery_Apply(AerospikeQuery *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "module")) { PyObject_SetAttrString(exception_type, "module", py_module); } diff --git a/src/main/query/foreach.c b/src/main/query/foreach.c index 0842c76fd..db4e1d452 100644 --- a/src/main/query/foreach.c +++ b/src/main/query/foreach.c @@ -254,6 +254,7 @@ PyObject *AerospikeQuery_Foreach(AerospikeQuery *self, PyObject *args, error_to_pyobject(&data.error, &py_err); exception_type = raise_exception_old(&data.error); } + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "name")) { PyObject_SetAttrString(exception_type, "name", Py_None); } diff --git a/src/main/query/type.c b/src/main/query/type.c index 263487f9e..6c4758ddc 100644 --- a/src/main/query/type.c +++ b/src/main/query/type.c @@ -245,9 +245,10 @@ static void AerospikeQuery_Type_Dealloc(AerospikeQuery *self) * PYTHON TYPE DESCRIPTOR ******************************************************************************/ static PyTypeObject AerospikeQuery_Type = { - PyVarObject_HEAD_INIT(NULL, 0) "aerospike.Query", // tp_name - sizeof(AerospikeQuery), // tp_basicsize - 0, // tp_itemsize + PyVarObject_HEAD_INIT(NULL, 0) + FULLY_QUALIFIED_TYPE_NAME("Query"), // tp_name + sizeof(AerospikeQuery), // tp_basicsize + 0, // tp_itemsize (destructor)AerospikeQuery_Type_Dealloc, // tp_dealloc 0, // tp_print diff --git a/src/main/scan/apply.c b/src/main/scan/apply.c index 13e4e0ce4..ef1a2cfdf 100644 --- a/src/main/scan/apply.c +++ b/src/main/scan/apply.c @@ -148,6 +148,7 @@ AerospikeScan *AerospikeScan_Apply(AerospikeScan *self, PyObject *args, PyObject *py_err = NULL; error_to_pyobject(&err, &py_err); PyObject *exception_type = raise_exception_old(&err); + set_aerospike_exc_attrs_using_tuple_of_attrs(exception_type, py_err); if (PyObject_HasAttrString(exception_type, "module")) { PyObject_SetAttrString(exception_type, "module", py_module); } diff --git a/src/main/scan/type.c b/src/main/scan/type.c index 62084e319..bdbbcc0f9 100644 --- a/src/main/scan/type.c +++ b/src/main/scan/type.c @@ -188,9 +188,9 @@ static void AerospikeScan_Type_Dealloc(AerospikeScan *self) ******************************************************************************/ static PyTypeObject AerospikeScan_Type = { - PyVarObject_HEAD_INIT(NULL, 0) "aerospike.Scan", // tp_name - sizeof(AerospikeScan), // tp_basicsize - 0, // tp_itemsize + PyVarObject_HEAD_INIT(NULL, 0) FULLY_QUALIFIED_TYPE_NAME("Scan"), // tp_name + sizeof(AerospikeScan), // tp_basicsize + 0, // tp_itemsize (destructor)AerospikeScan_Type_Dealloc, // tp_dealloc 0, // tp_print diff --git a/src/main/transaction/type.c b/src/main/transaction/type.c new file mode 100644 index 000000000..66b820489 --- /dev/null +++ b/src/main/transaction/type.c @@ -0,0 +1,190 @@ +#include + +#include "types.h" + +static void AerospikeTransaction_dealloc(AerospikeTransaction *self) +{ + // Transaction object can be created but not initialized, so need to check + if (self->txn != NULL) { + as_txn_destroy(self->txn); + } + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyObject *AerospikeTransaction_new(PyTypeObject *type, PyObject *args, + PyObject *kwds) +{ + AerospikeTransaction *self = + (AerospikeTransaction *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } + return (PyObject *)self; +} + +// Error indicator must always be checked after this call +// Constructor parameter name needed for constructing error message +static uint32_t convert_pyobject_to_uint32_t(PyObject *pyobject, + const char *param_name_of_pyobj) +{ + if (!PyLong_Check(pyobject)) { + PyErr_Format(PyExc_TypeError, "%s must be an integer", + param_name_of_pyobj); + goto error; + } + unsigned long long_value = PyLong_AsUnsignedLong(pyobject); + if (PyErr_Occurred()) { + goto error; + } + + if (long_value > UINT32_MAX) { + PyErr_Format(PyExc_ValueError, + "%s is too large for an unsigned 32-bit integer", + param_name_of_pyobj); + goto error; + } + + uint32_t value = (uint32_t)long_value; + return value; + +error: + return 0; +} + +// We don't initialize in __new__ because it's not documented how to raise +// exceptions in __new__ +// We can raise an exception and fail out in __init__ though +static int AerospikeTransaction_init(AerospikeTransaction *self, PyObject *args, + PyObject *kwds) +{ + static char *kwlist[] = {"reads_capacity", "writes_capacity", NULL}; + // We could use unsigned longs directly in the format string + // But then we can't tell if they were set or not by the user + // So we just use PyObjects for the optional args instead + PyObject *py_reads_capacity = NULL; + PyObject *py_writes_capacity = NULL; + + if (PyArg_ParseTupleAndKeywords(args, kwds, "|OO", kwlist, + &py_reads_capacity, + &py_writes_capacity) == false) { + goto error; + } + + as_txn *txn; + uint32_t reads_capacity, writes_capacity; + if (py_reads_capacity) { + reads_capacity = + convert_pyobject_to_uint32_t(py_reads_capacity, kwlist[0]); + if (PyErr_Occurred()) { + goto error; + } + } + else { + reads_capacity = AS_TXN_READ_CAPACITY_DEFAULT; + } + + if (py_writes_capacity) { + writes_capacity = + convert_pyobject_to_uint32_t(py_writes_capacity, kwlist[1]); + if (PyErr_Occurred()) { + goto error; + } + } + else { + writes_capacity = AS_TXN_WRITE_CAPACITY_DEFAULT; + } + + txn = as_txn_create_capacity(reads_capacity, writes_capacity); + + // If this transaction object was already initialized before, reinitialize it + if (self->txn) { + as_txn_destroy(self->txn); + } + self->txn = txn; + + return 0; +error: + return -1; +} + +static PyObject *AerospikeTransaction_get_in_doubt(AerospikeTransaction *self, + void *closure) +{ + PyObject *py_in_doubt = PyBool_FromLong(self->txn->in_doubt); + if (py_in_doubt == NULL) { + return NULL; + } + return py_in_doubt; +} + +static PyObject *AerospikeTransaction_get_state(AerospikeTransaction *self, + void *closure) +{ + PyObject *py_state = PyLong_FromLong((long)self->txn->state); + if (py_state == NULL) { + return NULL; + } + return py_state; +} + +static PyObject *AerospikeTransaction_get_timeout(AerospikeTransaction *self, + void *closure) +{ + PyObject *py_timeout = + PyLong_FromUnsignedLong((unsigned long)self->txn->timeout); + if (py_timeout == NULL) { + return NULL; + } + return py_timeout; +} + +static int AerospikeTransaction_set_timeout(AerospikeTransaction *self, + PyObject *py_value, void *closure) +{ + uint32_t timeout = convert_pyobject_to_uint32_t(py_value, "timeout"); + if (PyErr_Occurred()) { + return -1; + } + + self->txn->timeout = timeout; + return 0; +} + +static PyObject *AerospikeTransaction_get_id(AerospikeTransaction *self, + void *closure) +{ + PyObject *py_id = + PyLong_FromUnsignedLongLong((unsigned long long)self->txn->id); + if (py_id == NULL) { + return NULL; + } + return py_id; +} + +static PyGetSetDef AerospikeTransaction_getsetters[] = { + {.name = "timeout", + .get = (getter)AerospikeTransaction_get_timeout, + .set = (setter)AerospikeTransaction_set_timeout}, + {.name = "in_doubt", .get = (getter)AerospikeTransaction_get_in_doubt}, + {.name = "state", .get = (getter)AerospikeTransaction_get_state}, + {.name = "id", .get = (getter)AerospikeTransaction_get_id}, + {NULL} /* Sentinel */ +}; + +PyTypeObject AerospikeTransaction_Type = { + .ob_base = PyVarObject_HEAD_INIT(NULL, 0).tp_name = + FULLY_QUALIFIED_TYPE_NAME("Transaction"), + .tp_basicsize = sizeof(AerospikeTransaction), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = AerospikeTransaction_new, + .tp_init = (initproc)AerospikeTransaction_init, + .tp_dealloc = (destructor)AerospikeTransaction_dealloc, + .tp_getset = AerospikeTransaction_getsetters}; + +PyTypeObject *AerospikeTransaction_Ready() +{ + return PyType_Ready(&AerospikeTransaction_Type) == 0 + ? &AerospikeTransaction_Type + : NULL; +} diff --git a/test/metrics/test_node_close_listener.py b/test/metrics/test_node_close_listener.py index fe9e3dad0..32d81001e 100644 --- a/test/metrics/test_node_close_listener.py +++ b/test/metrics/test_node_close_listener.py @@ -1,6 +1,5 @@ -import subprocess import time -import json +import docker import aerospike from aerospike_helpers.metrics import MetricsListeners, MetricsPolicy, Cluster, Node, ConnectionStats, NodeMetrics @@ -65,38 +64,22 @@ def snapshot(_: Cluster): pass -NODE_COUNT = 1 -print(f"Creating {NODE_COUNT} node cluster...") -subprocess.run(["aerolab", "cluster", "create", f"--count={NODE_COUNT}"], check=True) +docker_client = docker.from_env() +print("Running server container...") +SERVER_PORT_NUMBER = 3000 +container = docker_client.containers.run("aerospike/aerospike-server", detach=True, + ports={"3000/tcp": SERVER_PORT_NUMBER}) +print("Waiting for server to initialize...") +time.sleep(5) try: - print("Wait for server to fully start up...") - time.sleep(5) - - # Connect to the first node - completed_process = subprocess.run(["aerolab", "cluster", "list", "--json"], check=True, capture_output=True) - list_of_nodes = json.loads(completed_process.stdout) - - def get_first_node(node_info: dict): - return node_info["NodeNo"] == "1" - - filtered_list_of_nodes = filter(get_first_node, list_of_nodes) - first_node = list(filtered_list_of_nodes)[0] - first_node_port = int(first_node["DockerExposePorts"]) - HOST_NAME = "127.0.0.1" - + print("Connecting to server...") config = { "hosts": [ - (HOST_NAME, first_node_port) + ("127.0.0.1", SERVER_PORT_NUMBER) ], - # The nodes use internal Docker IP addresses as their access addresses - # But we cannot connect to those from the host - # But the nodes use localhost as the alternate address - # So we can connect to that instead - "use_services_alternate": True } - print(f"Connecting to {HOST_NAME}:{first_node_port} using Python client...") - client = aerospike.client(config) + as_client = aerospike.client(config) try: # Show logs to confirm that node is removed from the client's perspective aerospike.set_log_level(aerospike.LOG_LEVEL_DEBUG) @@ -111,17 +94,20 @@ def get_first_node(node_info: dict): ) policy = MetricsPolicy(metrics_listeners=listeners) print("Enabling metrics...") - client.enable_metrics(policy=policy) + as_client.enable_metrics(policy=policy) # Close the last node - NODE_TO_CLOSE = NODE_COUNT - print(f"Closing node {NODE_TO_CLOSE}...") - subprocess.run(["aerolab", "cluster", "stop", f"--nodes={NODE_TO_CLOSE}"], check=True) - # Run with --force or else it will ask to confirm - subprocess.run(["aerolab", "cluster", "destroy", f"--nodes={NODE_TO_CLOSE}", "--force"], check=True) + print("Closing node...") + container.stop() + container.remove() print("Giving client time to run the node_close listener...") - time.sleep(10) + elapsed_secs = 0 + while elapsed_secs < 10: + if node_close_called: + break + time.sleep(1) + elapsed_secs += 1 assert node_close_called is True # Need to print assert result somehow @@ -130,9 +116,6 @@ def get_first_node(node_info: dict): # Calling close() after we lose connection to the whole cluster is safe. It will be a no-op # It is not explicitly documented for the Python client or C client, but this behavior was verified with C # client developer - client.close() + as_client.close() finally: - print("Cleaning up...") - if NODE_COUNT > 1: - subprocess.run(["aerolab", "cluster", "stop"], check=True) - subprocess.run(["aerolab", "cluster", "destroy", "--force"], check=True) + docker_client.close() diff --git a/test/new_tests/conftest.py b/test/new_tests/conftest.py index 6bd4c466d..f0a13b3a3 100644 --- a/test/new_tests/conftest.py +++ b/test/new_tests/conftest.py @@ -106,6 +106,33 @@ def as_connection(request): request.cls.as_connection = as_client + # Check that strong consistency is enabled for all nodes + if TestBaseClass.enterprise_in_use(): + ns_info = as_client.info_all("get-config:context=namespace;namespace=test") + are_all_nodes_sc_enabled = False + for i, (error, result) in enumerate(ns_info.values()): + if error: + # If we can't determine SC is enabled, just assume it isn't + # We don't want to break the tests if this code fails + print("Node returned error while getting config for namespace test") + break + ns_properties = result.split(";") + ns_properties = filter(lambda prop: "strong-consistency=" in prop, ns_properties) + ns_properties = list(ns_properties) + if len(ns_properties) == 0: + print("Strong consistency not found in node properties, so assuming it's disabled by default") + break + elif len(ns_properties) > 1: + print("Only one strong-consistency property should be present") + break + _, sc_enabled = ns_properties[0].split("=") + if sc_enabled == 'false': + print("One of the nodes is not SC enabled") + break + if i == len(ns_info) - 1: + are_all_nodes_sc_enabled = True + TestBaseClass.strong_consistency_enabled = TestBaseClass.enterprise_in_use() and are_all_nodes_sc_enabled + def close_connection(): as_client.close() diff --git a/test/new_tests/test_aggregate.py b/test/new_tests/test_aggregate.py index 7529e089a..6109c0869 100644 --- a/test/new_tests/test_aggregate.py +++ b/test/new_tests/test_aggregate.py @@ -387,3 +387,10 @@ def user_callback(value): except e.ParamError as exception: assert exception.code == -2 + + # We can't use the inspect library to check the keyword args of a method defined using the C-API + # It doesn't work, so just check that passing in an invalid arg fails + def test_signature_invalid_arg(self): + query: aerospike.Query = self.as_connection.query("test", "demo") + with pytest.raises(TypeError): + query.apply("stream_example", "count", policy=None) diff --git a/test/new_tests/test_base_class.py b/test/new_tests/test_base_class.py index 4c5acc7dd..e596b1732 100644 --- a/test/new_tests/test_base_class.py +++ b/test/new_tests/test_base_class.py @@ -181,7 +181,6 @@ def get_new_connection(add_config=None): # major_ver = res[0] # minor_ver = res[1] # print("major_ver:", major_ver, "minor_ver:", minor_ver) - return client @staticmethod diff --git a/test/new_tests/test_error_codes.py b/test/new_tests/test_error_codes.py deleted file mode 100644 index 6b2002934..000000000 --- a/test/new_tests/test_error_codes.py +++ /dev/null @@ -1,44 +0,0 @@ -import pytest -from aerospike import exception as e -from .as_errors import ( - AEROSPIKE_ERR_ASYNC_CONNECTION, - AEROSPIKE_ERR_BATCH_DISABLED, - AEROSPIKE_ERR_BATCH_MAX_REQUESTS_EXCEEDED, - AEROSPIKE_ERR_BATCH_QUEUES_FULL, - AEROSPIKE_ERR_CLIENT_ABORT, - AEROSPIKE_ERR_FAIL_ELEMENT_EXISTS, - AEROSPIKE_ERR_FAIL_ELEMENT_NOT_FOUND, - AEROSPIKE_ERR_INVALID_NODE, - AEROSPIKE_ERR_NO_MORE_CONNECTIONS, - AEROSPIKE_ERR_QUERY_ABORTED, - AEROSPIKE_ERR_SCAN_ABORTED, - AEROSPIKE_ERR_TLS_ERROR, -) - - -@pytest.mark.parametrize( - "error, error_name, error_code, base", - ( - (e.TLSError, "TLSError", AEROSPIKE_ERR_TLS_ERROR, e.ClientError), - (e.InvalidNodeError, "InvalidNodeError", AEROSPIKE_ERR_INVALID_NODE, e.ClientError), - (e.NoMoreConnectionsError, "NoMoreConnectionsError", AEROSPIKE_ERR_NO_MORE_CONNECTIONS, e.ClientError), - (e.AsyncConnectionError, "AsyncConnectionError", AEROSPIKE_ERR_ASYNC_CONNECTION, e.ClientError), - (e.ClientAbortError, "ClientAbortError", AEROSPIKE_ERR_CLIENT_ABORT, e.ClientError), - (e.ScanAbortedError, "ScanAbortedError", AEROSPIKE_ERR_SCAN_ABORTED, e.ServerError), - (e.ElementNotFoundError, "ElementNotFoundError", AEROSPIKE_ERR_FAIL_ELEMENT_NOT_FOUND, e.ServerError), - (e.ElementExistsError, "ElementExistsError", AEROSPIKE_ERR_FAIL_ELEMENT_EXISTS, e.ServerError), - (e.BatchDisabledError, "BatchDisabledError", AEROSPIKE_ERR_BATCH_DISABLED, e.ServerError), - (e.BatchMaxRequestError, "BatchMaxRequestError", AEROSPIKE_ERR_BATCH_MAX_REQUESTS_EXCEEDED, e.ServerError), - (e.BatchQueueFullError, "BatchQueueFullError", AEROSPIKE_ERR_BATCH_QUEUES_FULL, e.ServerError), - (e.QueryAbortedError, "QueryAbortedError", AEROSPIKE_ERR_QUERY_ABORTED, e.ServerError), - ), -) -def test_error_codes(error, error_name, error_code, base): - with pytest.raises(error) as test_error: - raise error - - test_error = test_error.value - - assert test_error.code == error_code - assert type(test_error).__name__ == error_name - assert issubclass(type(test_error), base) diff --git a/test/new_tests/test_exceptions.py b/test/new_tests/test_exceptions.py new file mode 100644 index 000000000..1276735e7 --- /dev/null +++ b/test/new_tests/test_exceptions.py @@ -0,0 +1,163 @@ +import pytest +from aerospike import exception as e +from .as_errors import ( + AEROSPIKE_ERR_ASYNC_CONNECTION, + AEROSPIKE_ERR_BATCH_DISABLED, + AEROSPIKE_ERR_BATCH_MAX_REQUESTS_EXCEEDED, + AEROSPIKE_ERR_BATCH_QUEUES_FULL, + AEROSPIKE_ERR_CLIENT_ABORT, + AEROSPIKE_ERR_FAIL_ELEMENT_EXISTS, + AEROSPIKE_ERR_FAIL_ELEMENT_NOT_FOUND, + AEROSPIKE_ERR_INVALID_NODE, + AEROSPIKE_ERR_NO_MORE_CONNECTIONS, + AEROSPIKE_ERR_QUERY_ABORTED, + AEROSPIKE_ERR_SCAN_ABORTED, + AEROSPIKE_ERR_TLS_ERROR, +) +from typing import Optional + +# Used to test inherited attributes (can be from indirect parent) +base_class_to_attrs = { + e.AerospikeError: [ + "code", + "msg", + "file", + "line", + "in_doubt" + ], + e.RecordError: [ + "key", + "bin" + ], + e.IndexError: [ + "name" + ], + e.UDFError: [ + "module", + "func" + ] +} + + +# TODO: add missing type stubs +# TODO: make sure other places in the tests aren't doing the same thing as here. +# We'll do this by cleaning up the test code. But it's nice to test the API in one place +# TODO: add documentation for the tests in a README +@pytest.mark.parametrize( + "exc_class, expected_exc_name, expected_error_code, expected_exc_base_class", + ( + # Don't test error_name fields that are set to None + # The exception name should be the class name anyways... + (e.AerospikeError, None, None, Exception), + (e.ClientError, None, -1, e.AerospikeError), + # Client errors + (e.InvalidHostError, None, -4, e.ClientError), + (e.ParamError, None, -2, e.ClientError), + (e.MaxErrorRateExceeded, None, -14, e.ClientError), + (e.MaxRetriesExceeded, None, -12, e.ClientError), + (e.NoResponse, None, -15, e.ClientError), + (e.BatchFailed, None, -16, e.ClientError), + (e.ConnectionError, None, -10, e.ClientError), + (e.TLSError, "TLSError", AEROSPIKE_ERR_TLS_ERROR, e.ClientError), + (e.InvalidNodeError, "InvalidNodeError", AEROSPIKE_ERR_INVALID_NODE, e.ClientError), + (e.NoMoreConnectionsError, "NoMoreConnectionsError", AEROSPIKE_ERR_NO_MORE_CONNECTIONS, e.ClientError), + (e.AsyncConnectionError, "AsyncConnectionError", AEROSPIKE_ERR_ASYNC_CONNECTION, e.ClientError), + (e.ClientAbortError, "ClientAbortError", AEROSPIKE_ERR_CLIENT_ABORT, e.ClientError), + # Server errors + (e.ServerError, None, 1, e.AerospikeError), + (e.InvalidRequest, None, 4, e.ServerError), + (e.OpNotApplicable, None, 26, e.ServerError), + (e.FilteredOut, None, 27, e.ServerError), + (e.ServerFull, None, 8, e.ServerError), + (e.AlwaysForbidden, None, 10, e.ServerError), + (e.UnsupportedFeature, None, 16, e.ServerError), + (e.DeviceOverload, None, 18, e.ServerError), + (e.NamespaceNotFound, None, 20, e.ServerError), + (e.ForbiddenError, None, 22, e.ServerError), + (e.LostConflict, None, 28, e.ServerError), + (e.InvalidGeoJSON, None, 160, e.ServerError), + (e.ScanAbortedError, "ScanAbortedError", AEROSPIKE_ERR_SCAN_ABORTED, e.ServerError), + (e.ElementNotFoundError, "ElementNotFoundError", AEROSPIKE_ERR_FAIL_ELEMENT_NOT_FOUND, e.ServerError), + (e.ElementExistsError, "ElementExistsError", AEROSPIKE_ERR_FAIL_ELEMENT_EXISTS, e.ServerError), + (e.BatchDisabledError, "BatchDisabledError", AEROSPIKE_ERR_BATCH_DISABLED, e.ServerError), + (e.BatchMaxRequestError, "BatchMaxRequestError", AEROSPIKE_ERR_BATCH_MAX_REQUESTS_EXCEEDED, e.ServerError), + (e.BatchQueueFullError, "BatchQueueFullError", AEROSPIKE_ERR_BATCH_QUEUES_FULL, e.ServerError), + (e.QueryAbortedError, "QueryAbortedError", AEROSPIKE_ERR_QUERY_ABORTED, e.ServerError), + # Record errors + (e.RecordError, None, None, e.ServerError), + (e.RecordKeyMismatch, None, 19, e.RecordError), + (e.RecordNotFound, None, 2, e.RecordError), + (e.RecordGenerationError, None, 3, e.RecordError), + (e.RecordExistsError, None, 5, e.RecordError), + (e.RecordBusy, None, 14, e.RecordError), + (e.RecordTooBig, None, 13, e.RecordError), + (e.BinNameError, None, 21, e.RecordError), + (e.BinIncompatibleType, None, 12, e.RecordError), + (e.BinNotFound, None, 17, e.RecordError), + (e.BinExistsError, None, 6, e.RecordError), + # Index errors + (e.IndexError, None, 204, e.ServerError), + (e.IndexNotFound, None, 201, e.IndexError), + (e.IndexFoundError, None, 200, e.IndexError), + (e.IndexOOM, None, 202, e.IndexError), + (e.IndexNotReadable, None, 203, e.IndexError), + (e.IndexNameMaxLen, None, 205, e.IndexError), + (e.IndexNameMaxCount, None, 206, e.IndexError), + # Query errors + (e.QueryError, None, 213, e.ServerError), + (e.QueryQueueFull, None, 211, e.QueryError), + (e.QueryTimeout, None, 212, e.QueryError), + # Cluster errors + (e.ClusterError, None, 11, e.ServerError), + (e.ClusterChangeError, None, 7, e.ClusterError), + # Admin errors + (e.AdminError, None, None, e.ServerError), + (e.ExpiredPassword, None, 63, e.AdminError), + (e.ForbiddenPassword, None, 64, e.AdminError), + (e.IllegalState, None, 56, e.AdminError), + (e.InvalidCommand, None, 54, e.AdminError), + (e.InvalidCredential, None, 65, e.AdminError), + (e.InvalidField, None, 55, e.AdminError), + (e.InvalidPassword, None, 62, e.AdminError), + (e.InvalidPrivilege, None, 72, e.AdminError), + (e.InvalidRole, None, 70, e.AdminError), + (e.InvalidUser, None, 60, e.AdminError), + (e.QuotasNotEnabled, None, 74, e.AdminError), + (e.QuotaExceeded, None, 83, e.AdminError), + (e.InvalidQuota, None, 75, e.AdminError), + (e.NotWhitelisted, None, 82, e.AdminError), + (e.InvalidWhitelist, None, 73, e.AdminError), + (e.NotAuthenticated, None, 80, e.AdminError), + (e.RoleExistsError, None, 71, e.AdminError), + (e.RoleViolation, None, 81, e.AdminError), + (e.SecurityNotEnabled, None, 52, e.AdminError), + (e.SecurityNotSupported, None, 51, e.AdminError), + (e.SecuritySchemeNotSupported, None, 53, e.AdminError), + (e.UserExistsError, None, 61, e.AdminError), + # UDF errors + (e.UDFError, None, 100, e.ServerError), + (e.UDFNotFound, None, 1301, e.UDFError), + (e.LuaFileNotFound, None, 1302, e.UDFError), + ), +) +def test_aerospike_exceptions( + exc_class: type, + expected_exc_name: Optional[str], + expected_error_code: Optional[int], + expected_exc_base_class: type +): + with pytest.raises(exc_class) as excinfo: + raise exc_class + + assert excinfo.value.code == expected_error_code + + if expected_exc_name is not None: + assert excinfo.type.__name__ == expected_exc_name + + # Test directly inherited class + assert expected_exc_base_class in excinfo.type.__bases__ + + for base_class in base_class_to_attrs: + if issubclass(excinfo.type, base_class): + for attr in base_class_to_attrs[base_class]: + assert hasattr(excinfo.value, attr) diff --git a/test/new_tests/test_expressions_map.py b/test/new_tests/test_expressions_map.py index 31b5c43fa..6356957e7 100644 --- a/test/new_tests/test_expressions_map.py +++ b/test/new_tests/test_expressions_map.py @@ -163,7 +163,7 @@ class TestExpressions(TestBaseClass): def setup(self, request, as_connection): self.test_ns = "test" self.test_set = "demo" - + self.first_key = (self.test_ns, self.test_set, 0) for i in range(_NUM_RECORDS): key = ("test", "demo", i) rec = { @@ -828,7 +828,15 @@ def test_map_expr_inverted(self, bin_name: str, expr, expected): ops = [ expr_ops.expression_read(bin_name, expr.compile()) ] - key = (self.test_ns, self.test_set, 0) - _, _, bins = self.as_connection.operate(key, ops) + _, _, bins = self.as_connection.operate(self.first_key, ops) assert bins[bin_name] == expected + + def test_map_get_nil_value_type(self): + bin_name = "nmap_bin" + exp = MapGetByKey(None, aerospike.MAP_RETURN_VALUE, ResultType.NIL, 2, bin_name).compile() + ops = [ + expr_ops.expression_read(bin_name, exp) + ] + _, _, bins = self.as_connection.operate(self.first_key, ops) + assert bins[bin_name] is None diff --git a/test/new_tests/test_hll.py b/test/new_tests/test_hll.py index c2634886e..fa7528512 100644 --- a/test/new_tests/test_hll.py +++ b/test/new_tests/test_hll.py @@ -489,3 +489,19 @@ def test_put_get_hll_list(self): def test_hll_superclass(self): assert issubclass(HyperLogLog, bytes) + + def test_hll_str_repr(self): + bytes_obj = b'asdf' + hll = HyperLogLog(bytes_obj) + + expected_repr = f"{hll.__class__.__name__}({bytes_obj.__repr__()})" + assert str(hll) == expected_repr + assert repr(hll) == expected_repr + + hll_from_eval = eval(expected_repr) + # We compare HLL instances by comparing their bytes values + assert hll == hll_from_eval + + # Negative test for comparing HLL values + different_hll = HyperLogLog(b'asdff') + assert different_hll != hll_from_eval diff --git a/test/new_tests/test_index.py b/test/new_tests/test_index.py index abcef7283..c8c8fc19d 100644 --- a/test/new_tests/test_index.py +++ b/test/new_tests/test_index.py @@ -245,11 +245,14 @@ def test_create_string_index_with_correct_parameters_ns_length_extra(self): ns_name = "a" * 50 policy = {} - with pytest.raises(e.InvalidRequest) as err_info: + with pytest.raises((e.InvalidRequest, e.NamespaceNotFound)) as err_info: self.as_connection.index_string_create(ns_name, "demo", "name", "name_index", policy) err_code = err_info.value.code - assert err_code is AerospikeStatus.AEROSPIKE_ERR_REQUEST_INVALID + if (TestBaseClass.major_ver, TestBaseClass.minor_ver) >= (7, 2): + assert err_code is AerospikeStatus.AEROSPIKE_ERR_NAMESPACE_NOT_FOUND + else: + assert err_code is AerospikeStatus.AEROSPIKE_ERR_REQUEST_INVALID def test_create_string_index_with_incorrect_namespace(self): """ diff --git a/test/new_tests/test_mapkeys_index.py b/test/new_tests/test_mapkeys_index.py index 469e42f72..51cf307c3 100644 --- a/test/new_tests/test_mapkeys_index.py +++ b/test/new_tests/test_mapkeys_index.py @@ -121,16 +121,19 @@ def test_mapkeysindex_with_correct_parameters_string_on_numerickeys(self): ), ids=("ns too long", "set too long", "bin too long", "index name too long"), ) - def test_mapkeys_with_parameters_too_long(self, ns, test_set, test_bin, index_name): + def test_mapkeys_with_parameters_too_long(self, ns, test_set, test_bin, index_name, request): # Invoke index_map_keys_create() with correct arguments and set # length extra policy = {} - with pytest.raises(e.InvalidRequest) as err_info: + with pytest.raises((e.InvalidRequest, e.NamespaceNotFound)) as err_info: self.as_connection.index_map_keys_create(ns, test_set, test_bin, aerospike.INDEX_STRING, index_name, policy) err_code = err_info.value.code - assert err_code == AerospikeStatus.AEROSPIKE_ERR_REQUEST_INVALID + if request.node.callspec.id == "ns too long" and (TestBaseClass.major_ver, TestBaseClass.minor_ver) >= (7, 2): + assert err_code is AerospikeStatus.AEROSPIKE_ERR_NAMESPACE_NOT_FOUND + else: + assert err_code is AerospikeStatus.AEROSPIKE_ERR_REQUEST_INVALID def test_mapkeysindex_with_incorrect_namespace(self): """ diff --git a/test/new_tests/test_metrics.py b/test/new_tests/test_metrics.py index 54a3e7ed9..0fc4201f5 100644 --- a/test/new_tests/test_metrics.py +++ b/test/new_tests/test_metrics.py @@ -158,7 +158,7 @@ def test_setting_metrics_policy_custom_settings(self): assert type(cluster) == Cluster assert cluster.cluster_name is None or type(cluster.cluster_name) == str assert type(cluster.invalid_node_count) == int - assert type(cluster.tran_count) == int + assert type(cluster.command_count) == int assert type(cluster.retry_count) == int assert type(cluster.nodes) == list # Also check the Node and ConnectionStats objects in the Cluster object were populated diff --git a/test/new_tests/test_mrt_api.py b/test/new_tests/test_mrt_api.py new file mode 100644 index 000000000..e88e40539 --- /dev/null +++ b/test/new_tests/test_mrt_api.py @@ -0,0 +1,89 @@ +import aerospike +from aerospike import exception as e +import pytest +from contextlib import nullcontext +from typing import Optional, Callable + + +@pytest.mark.usefixtures("as_connection") +class TestMRTAPI: + @pytest.mark.parametrize( + "kwargs, context, err_msg", + [ + ({}, nullcontext(), None), + ({"reads_capacity": 256, "writes_capacity": 256}, nullcontext(), None), + ( + {"reads_capacity": 256, "writes_capacity": 256, "invalid_arg": 1}, + pytest.raises((TypeError)), "function takes at most 2 keyword arguments (3 given)" + ), + ( + {"reads_capacity": "256", "writes_capacity": 256}, + pytest.raises((TypeError)), "reads_capacity must be an integer" + ), + ( + {"reads_capacity": 256, "writes_capacity": "256"}, + pytest.raises((TypeError)), "writes_capacity must be an integer" + ), + # Only need to test codepath once for uint32_t conversion helper function + ( + {"reads_capacity": 2**32, "writes_capacity": 256}, + # Linux x64's unsigned long is 8 bytes long at most + # but Windows x64's unsigned long is 4 bytes long at most + # Python in Windows x64 will throw an internal error (OverflowError) when trying to convert a Python + # int that is larger than 4 bytes into an unsigned long. + # That error doesn't happen in Linux for that same scenario, so we throw our own error + pytest.raises((ValueError, OverflowError)), + "reads_capacity is too large for an unsigned 32-bit integer" + ) + ], + ) + def test_transaction_class(self, kwargs: dict, context, err_msg: Optional[str]): + with context as excinfo: + mrt = aerospike.Transaction(**kwargs) + if type(context) == nullcontext: + # Can we read these fields + assert type(mrt.id) == int + assert type(mrt.timeout) == int + assert type(mrt.state) == int + assert type(mrt.in_doubt) == bool + # Can we change the timeout? + mrt.timeout = 10 + else: + # Just use kwargs to id the test case + if kwargs == {"reads_capacity": 2**32, "writes_capacity": 256} and excinfo.type == OverflowError: + # Internal Python error thrown in Windows + assert str(excinfo.value) == "Python int too large to convert to C unsigned long" + else: + # Custom error thrown by Python client for other platforms + assert str(excinfo.value) == err_msg + + # Even though this is an unlikely use case, this should not cause problems. + def test_transaction_reinit(self): + mrt = aerospike.Transaction() + # Create a new transaction object using the same Python class instance + mrt.__init__() + + @pytest.mark.parametrize( + "args", + [ + [], + ["string"] + ] + ) + @pytest.mark.parametrize( + "api_call", + [ + aerospike.Client.commit, + aerospike.Client.abort + ] + ) + def test_mrt_invalid_args(self, args: list, api_call: Callable): + with pytest.raises(TypeError): + api_call(self.as_connection, *args) + + def test_invalid_txn_in_policy(self): + policy = {"txn": True} + key = ("test", "demo", 1) + with pytest.raises(e.ParamError) as excinfo: + self.as_connection.get(key, policy) + assert excinfo.value.msg == "txn is not of type aerospike.Transaction" diff --git a/test/new_tests/test_mrt_functionality.py b/test/new_tests/test_mrt_functionality.py new file mode 100644 index 000000000..d395fe090 --- /dev/null +++ b/test/new_tests/test_mrt_functionality.py @@ -0,0 +1,104 @@ +import pytest +from aerospike import exception as e +import aerospike +from .test_base_class import TestBaseClass +import time + + +class TestMRTBasicFunctionality: + def setup_class(cls): + cls.keys = [] + NUM_RECORDS = 2 + for i in range(NUM_RECORDS): + key = ("test", "demo", i) + cls.keys.append(key) + cls.bin_name = "a" + + @pytest.fixture(autouse=True) + def insert_or_update_records(self, as_connection): + if (TestBaseClass.major_ver, TestBaseClass.minor_ver) < (8, 0): + pytest.skip("MRT is only supported in server version 8.0 or higher") + if TestBaseClass.strong_consistency_enabled is False: + pytest.skip("Strong consistency is not enabled") + + for i, key in enumerate(self.keys): + self.as_connection.put(key, {self.bin_name: i}) + + # Test case 1: Execute a simple MRT with multiple SRTs(Read and Write) in any sequence (P3) + # Validate that all operations complete successfully. + def test_commit_api_and_functionality(self): + mrt = aerospike.Transaction() + policy = { + "txn": mrt + } + self.as_connection.put(self.keys[0], {self.bin_name: 1}, policy=policy) + # Reads in an MRT should read the intermediate values of the MRT + _, _, bins = self.as_connection.get(self.keys[0], policy) + # Check that original value was overwritten + assert bins == {self.bin_name: 1} + self.as_connection.put(self.keys[1], {self.bin_name: 2}, policy=policy) + + retval = self.as_connection.commit(transaction=mrt) + assert retval == aerospike.MRT_COMMIT_OK + + # Were the writes committed? + for i in range(len(self.keys)): + _, _, bins = self.as_connection.get(self.keys[i]) + assert bins == {self.bin_name: i + 1} + + def test_timeout_mrt(self): + mrt = aerospike.Transaction() + mrt.timeout = 1 + policy = { + "txn": mrt + } + self.as_connection.put(self.keys[0], {self.bin_name: 1}, policy=policy) + time.sleep(3) + # Server should indicate that MRT has expired + with pytest.raises(e.AerospikeError): + self.as_connection.put(self.keys[1], {self.bin_name: 2}, policy=policy) + + # Cleanup MRT on server side before continuing to run test + self.as_connection.abort(mrt) + + # Test case 57: "Execute the MRT. Before issuing commit, give abort request using abort API" (P1) + def test_abort_api_and_functionality(self): + mrt = aerospike.Transaction() + policy = { + "txn": mrt + } + self.as_connection.put(self.keys[0], {self.bin_name: 1}, policy=policy) + # Should return intermediate overwritten value from MRT + _, _, bins = self.as_connection.get(self.keys[0], policy) + assert bins == {self.bin_name: 1} + self.as_connection.put(self.keys[1], {self.bin_name: 2}, policy=policy) + + retval = self.as_connection.abort(transaction=mrt) + assert retval == aerospike.MRT_ABORT_OK + + # Test that MRT didn't go through + # i.e write commands were rolled back + for i in range(len(self.keys)): + _, _, bins = self.as_connection.get(self.keys[i]) + assert bins == {self.bin_name: i} + + def test_commit_fail(self): + mrt = aerospike.Transaction() + policy = { + "txn": mrt + } + self.as_connection.put(self.keys[0], {self.bin_name: 1}, policy=policy) + self.as_connection.abort(mrt) + status = self.as_connection.commit(mrt) + assert status == aerospike.MRT_COMMIT_ALREADY_ABORTED + + # Test case 10: Issue abort after issung commit. (P1) + def test_abort_fail(self): + mrt = aerospike.Transaction() + policy = { + "txn": mrt + } + self.as_connection.put(self.keys[0], {self.bin_name: 1}, policy=policy) + self.as_connection.commit(mrt) + status = self.as_connection.abort(mrt) + assert status == aerospike.MRT_ABORT_ALREADY_COMMITTED diff --git a/test/new_tests/test_query.py b/test/new_tests/test_query.py index 57337c36c..c7c43eba0 100644 --- a/test/new_tests/test_query.py +++ b/test/new_tests/test_query.py @@ -1148,6 +1148,8 @@ def callback(input_tuple): ] ) def test_query_expected_duration(self, duration: int): + if duration == aerospike.QUERY_DURATION_LONG_RELAX_AP and TestBaseClass.strong_consistency_enabled: + pytest.skip("Using aerospike.QUERY_DURATION_LONG_RELAX_AP will fail if server is in SC mode") query: aerospike.Query = self.as_connection.query("test", "demo") policy = { "expected_duration": duration diff --git a/test/requirements.txt b/test/requirements.txt index 7e1acd0ec..e9043f2e7 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,3 @@ pytest==7.4.0 # To generate coverage reports in the Github Actions pipeline pytest-cov==4.1.0 -# Memory profiling -pytest-memray==1.5.0 diff --git a/test/tox.ini b/test/tox.ini index e3ab21558..e1cb1b973 100644 --- a/test/tox.ini +++ b/test/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] skipsdist = True -envlist = py38, py39, py310, py311, py312 +envlist = py38, py39, py310, py311, py312, py313 [testenv] commands = pip install --find-links=local/wheels --no-index aerospike