[CLIENT-3121] Support multi-record transactions

.github/actions/run-ee-server/action.yml

@@ -6,15 +6,19 @@ inputs:
   # All inputs in composite actions are strings
   use-server-rc:
     required: true
-    default: false
+    description: Deploy server release candidate?
+    default: 'false'
   server-tag:
     required: true
+    description: Specify Docker tag
     default: 'latest'
   # Github Composite Actions can't access secrets
   # so we need to pass them in as inputs
   docker-hub-username:
+    description: Required for using release candidates
     required: false
   docker-hub-password:
+    description: Required for using release candidates
     required: false
 
 runs:
@@ -35,13 +39,17 @@ runs:
     run: |
       crudini --existing=param --set config.conf enterprise-edition hosts ''
       crudini --existing=param --set config.conf enterprise-edition hosts 127.0.0.1:3000
-      crudini --existing=param --set config.conf enterprise-edition user superuser
-      crudini --existing=param --set config.conf enterprise-edition password superuser
     working-directory: test
     shell: bash
 
-  - name: Create config folder to store configs in
-    run: mkdir configs
+  - run: echo SUPERUSER_NAME_AND_PASSWORD="superuser" >> $GITHUB_ENV
+    shell: bash
+
+  - name: Set credentials in config file
+    run: |
+      crudini --existing=param --set config.conf enterprise-edition user ${{ env.SUPERUSER_NAME_AND_PASSWORD }}
+      crudini --existing=param --set config.conf enterprise-edition password ${{ env.SUPERUSER_NAME_AND_PASSWORD }}
+    working-directory: test
     shell: bash
 
   - name: Log into Docker Hub to get server RC
@@ -52,7 +60,7 @@ runs:
   - run: echo IMAGE_NAME=aerospike/aerospike-server-enterprise${{ inputs.use-server-rc == 'true' && '-rc' || '' }}:${{ inputs.server-tag }} >> $GITHUB_ENV
     shell: bash
 
-  - run: echo SECURITY_IMAGE_NAME=${{ env.IMAGE_NAME }}-security >> $GITHUB_ENV
+  - run: echo NEW_IMAGE_NAME=${{ env.IMAGE_NAME }}-security-and-sc >> $GITHUB_ENV
     shell: bash
 
   # macOS Github runners and Windows self-hosted runners don't have buildx installed by default
@@ -63,19 +71,38 @@ runs:
     uses: docker/build-push-action@v6
     with:
       # Don't want to use default Git context or else it will clone the whole Python client repo again
-      context: .github/workflows
+      context: .github/workflows/docker-build-context
       build-args: |
-        image=${{ env.IMAGE_NAME }}
-      tags: ${{ env.SECURITY_IMAGE_NAME }}
+        server_image=${{ env.IMAGE_NAME }}
+      tags: ${{ env.NEW_IMAGE_NAME }}
       # setup-buildx-action configures Docker to use the docker-container build driver
       # This driver doesn't publish an image locally by default
       # so we have to manually enable it
       load: true
 
-  - run: docker run -d --name aerospike -p 3000:3000 -e DEFAULT_TTL=2592000 ${{ env.SECURITY_IMAGE_NAME }}
+  - run: echo SERVER_CONTAINER_NAME="aerospike" >> $GITHUB_ENV
+    shell: bash
+
+  - run: docker run -d --name ${{ env.SERVER_CONTAINER_NAME }} -e DEFAULT_TTL=2592000 -p 3000:3000 ${{ env.NEW_IMAGE_NAME }}
     shell: bash
 
   - uses: ./.github/actions/wait-for-as-server-to-start
     with:
-      container-name: aerospike
+      container-name: ${{ env.SERVER_CONTAINER_NAME }}
       is-security-enabled: true
+      is-strong-consistency-enabled: true
+
+  - run: echo ASADM_AUTH_FLAGS="--user=${{ env.SUPERUSER_NAME_AND_PASSWORD }} --password=${{ env.SUPERUSER_NAME_AND_PASSWORD }}" >> $GITHUB_ENV
+    shell: bash
+
+  # All the partitions are assumed to be dead when reusing a roster file
+  - run: docker exec ${{ env.SERVER_CONTAINER_NAME }} asadm $ASADM_AUTH_FLAGS --enable --execute "manage revive ns test"
+    shell: bash
+
+  # Apply changes
+  - run: docker exec ${{ env.SERVER_CONTAINER_NAME }} asadm $ASADM_AUTH_FLAGS --enable --execute "manage recluster"
+    shell: bash
+
+  # For debugging
+  - run: docker logs aerospike
+    shell: bash

.github/actions/wait-for-as-server-to-start/action.yml

@@ -6,6 +6,9 @@ inputs:
   is-security-enabled:
     required: false
     default: 'false'
+  is-strong-consistency-enabled:
+    required: false
+    default: 'false'
 
 runs:
   using: "composite"
@@ -21,5 +24,5 @@ runs:
   # Also, we don't want to fail if we timeout in case the server *did* finish starting up but the script couldn't detect it due to a bug
   # Effectively, this composite action is like calling "sleep" that is optimized to exit early when it detects an ok from the server
   - name: Wait for EE server to start
-    run: timeout 30 bash ./.github/workflows/wait-for-as-server-to-start.bash ${{ inputs.container-name }} ${{ inputs.is-security-enabled }} || true
+    run: timeout 30 bash ./.github/workflows/wait-for-as-server-to-start.bash ${{ inputs.container-name }} ${{ inputs.is-security-enabled }} ${{ inputs.is-strong-consistency-enabled }} || true
     shell: bash

.github/workflows/build-wheels.yml

@@ -65,6 +65,10 @@ on:
         required: true
         default: 'latest'
         description: 'Server docker image tag'
+      test-file:
+        required: false
+        default: ''
+        description: 'new_tests/<value>'
 
   workflow_call:
     inputs:
@@ -106,6 +110,10 @@ on:
         type: string
         default: 'latest'
         description: 'Server docker image tag'
+      test-file:
+        required: false
+        type: string
+        default: ''
     secrets:
       # Just make all the secrets required to make things simpler...
       DOCKER_HUB_BOT_USERNAME:
@@ -226,7 +234,7 @@ jobs:
 
     - name: Otherwise, enable integration tests
       if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }}
-      run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest new_tests/" >> $GITHUB_ENV
+      run: echo "TEST_COMMAND=cd {project}/test/ && pip install -r requirements.txt && python -m pytest -vv new_tests/${{ inputs.test-file }}" >> $GITHUB_ENV
       shell: bash
 
     - name: Set unoptimize flag
@@ -340,7 +348,7 @@ jobs:
         working-directory: test
         shell: bash
 
-      - run: python3 -m pytest new_tests/
+      - run: python3 -m pytest -vv new_tests/${{ inputs.test-file }}
         working-directory: test
         shell: bash
 

.github/workflows/docker-build-context/Dockerfile

@@ -0,0 +1,49 @@
+ARG server_image=aerospike/aerospike-server-enterprise
+ARG ROSTER_FILE_NAME=roster.smd
+# Temp file for passing node id from one build stage to another
+# Docker doesn't support command substitution for setting values for ARG variables, so we have to do this
+ARG NODE_ID_FILE_NAME=node_id
+
+FROM $server_image as configure-server
+
+WORKDIR /opt/aerospike/smd
+
+# Enable authentication
+
+ARG AEROSPIKE_CONF_TEMPLATE_PATH=/etc/aerospike/aerospike.template.conf
+
+# Not using asconfig to edit config because we are working with a template file, which may not have valid values yet
+RUN echo -e "security {\n\tenable-quotas true\n}\n" >> $AEROSPIKE_CONF_TEMPLATE_PATH
+# security.smd was generated manually by
+# 1. Starting a new Aerospike EE server using Docker
+# 2. Creating the superuser user
+# 3. Copying /opt/aerospike/smd/security.smd from the container and committing it to this repo
+# This file should always work
+# TODO: generate this automatically, somehow.
+COPY security.smd .
+
+# Enable strong consistency
+RUN sed -i "s/\(namespace.*{\)/\1\n\tstrong-consistency true/" $AEROSPIKE_CONF_TEMPLATE_PATH
+RUN sed -i "s/\(namespace.*{\)/\1\n\tstrong-consistency-allow-expunge true/" $AEROSPIKE_CONF_TEMPLATE_PATH
+ARG ROSTER_FILE_NAME
+COPY $ROSTER_FILE_NAME .
+
+# Fetch node id from roster.smd
+
+# There's no tag for the latest major version to prevent breaking changes in jq
+# This is the next best thing
+FROM ghcr.io/jqlang/jq:1.7 as get-jq
+# jq docker image doesn't have a shell
+# We need a shell to fetch and pass the node id to the next build stage
+FROM busybox as get-node-id
+COPY --from=get-jq /jq /bin/
+ARG ROSTER_FILE_NAME
+COPY $ROSTER_FILE_NAME .
+ARG NODE_ID_FILE_NAME
+RUN jq --raw-output '.[1].value' $ROSTER_FILE_NAME > $NODE_ID_FILE_NAME
+
+FROM configure-server as set-node-id
+ARG NODE_ID_FILE_NAME
+COPY --from=get-node-id $NODE_ID_FILE_NAME .
+RUN sed -i "s/\(^service {\)/\1\n\tnode-id $(cat $NODE_ID_FILE_NAME)/" $AEROSPIKE_CONF_TEMPLATE_PATH
+RUN rm $NODE_ID_FILE_NAME

.github/workflows/docker-build-context/roster.smd

@@ -0,0 +1,12 @@
+[
+   [
+      97107025374203,
+      1
+   ],
+   {
+      "key": "test",
+      "value": "a1",
+      "generation": 1,
+      "timestamp": 465602976982
+   }
+]

.github/workflows/tests.yml

@@ -127,6 +127,7 @@ jobs:
         mkdir -p nullobject/src/main/nullobject
         mkdir -p query/src/main/query
         mkdir -p scan/src/main/scan
+        mkdir -p transaction/src/main/transaction
 
         cd ../../../../
         cp src/main/*.c build/temp*/src/main/src/main
@@ -138,6 +139,7 @@ jobs:
         cp src/main/nullobject/*.c build/temp*/src/main/nullobject/src/main/nullobject/
         cp src/main/query/*.c build/temp*/src/main/query/src/main/query/
         cp src/main/scan/*.c build/temp*/src/main/scan/src/main/scan/
+        cp src/main/transaction/*.c build/temp*/src/main/transaction/src/main/transaction/
 
     - name: Generate coverage report for all object files
       if: ${{ !cancelled() }}
@@ -372,7 +374,7 @@ jobs:
         docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }}
 
     - name: Run tests
-      run: python -m pytest ./new_tests/test_admin_*.py
+      run: python -m pytest ./new_tests/test_{mrt_functionality,admin_*}.py
       working-directory: test
 
     - name: Show logs if failed
@@ -412,30 +414,14 @@ jobs:
       run: sphinx-build -b linkcheck . links
       working-directory: doc
 
-  test-metrics-node-close-listener:
-    needs: build
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with:
-        python-version: ${{ env.LOWEST_SUPPORTED_PY_VERSION }}
-        architecture: 'x64'
-    - name: Download aerolab
-      run: wget https://github.com/aerospike/aerolab/releases/download/7.5.2/aerolab-linux-amd64-7.5.2.deb
-    - name: Install aerolab
-      run: sudo dpkg -i *.deb
-    - name: Tell aerolab to use Docker
-      run: aerolab config backend -t docker
-    - uses: actions/download-artifact@v3
-      with:
-        name: wheel-${{ env.LOWEST_SUPPORTED_PY_VERSION }}
-    - run: python3 -m pip install *.whl
-    - run: python3 test_node_close_listener.py
-      working-directory: test/metrics
-
-  test-metrics-cluster-name:
+  test-metrics:
     needs: build
+    strategy:
+      matrix:
+        suffix:
+        - node_close_listener
+        - cluster_name
+      fail-fast: false
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
@@ -449,5 +435,5 @@ jobs:
     - run: python3 -m pip install *.whl
     - run: python3 -m pip install -r requirements.txt
       working-directory: test/metrics
-    - run: python3 test_cluster_name.py
+    - run: python3 test_${{ matrix.suffix }}.py
       working-directory: test/metrics

.github/workflows/valgrind.yml

@@ -12,6 +12,9 @@ on:
         description: 'Use server release candidate?'
         required: true
         default: false
+      server-tag:
+        required: false
+        default: latest
       massif:
         type: boolean
         description: 'Use massif for testing memory usage'
@@ -57,6 +60,7 @@ jobs:
       uses: ./.github/actions/run-ee-server
       with:
         use-server-rc: ${{ inputs.use-server-rc }}
+        server-tag: ${{ inputs.server-tag }}
         docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }}
         docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }}
 

.github/workflows/wait-for-as-server-to-start.bash

@@ -6,6 +6,7 @@ set -o pipefail
 
 container_name=$1
 is_security_enabled=$2
+is_strong_consistency_enabled=$3
 
 if [[ $is_security_enabled == true ]]; then
     # We need to pass credentials to asinfo if server requires it
@@ -38,7 +39,14 @@ done
 while true; do
     echo "Waiting for server to stabilize (i.e return a cluster key)..."
     # We assume that when an ERROR is returned, the cluster is not stable yet (i.e not fully initialized)
-    if docker exec "$container_name" asinfo $user_credentials -v cluster-stable 2>&1 | (! grep -qE "^ERROR"); then
+    cluster_stable_info_cmd="cluster-stable"
+    if [[ $is_strong_consistency_enabled == true ]]; then
+        # The Dockerfile uses a roster from a previously running Aerospike server in a Docker container
+        # When we reuse this roster, the server assumes all of its partitions are dead because it's running on a new
+        # storage device.
+        cluster_stable_info_cmd="$cluster_stable_info_cmd:ignore-migrations=true"
+    fi
+    if docker exec "$container_name" asinfo $user_credentials -v $cluster_stable_info_cmd 2>&1 | (! grep -qE "^ERROR"); then
         echo "Server is in a stable state."
         break
     fi

.gitmodules

@@ -2,4 +2,4 @@
 	path = aerospike-client-c
 	# url = git@github.com:aerospike/aerospike-client-c.git
 	url = https://github.com/aerospike/aerospike-client-c.git
-	branch = stage
+	branch = CLIENT-2294