Merge branch 'dev' into CLIENT-2512-sanitizer-update

aerospike · Dec 16, 2024 · 5151898 · 5151898
2 parents 4243b9c + 323f3a1
commit 5151898
Show file tree

Hide file tree

Showing 96 changed files with 3,187 additions and 2,106 deletions.
diff --git a/.build.yml b/.build.yml
@@ -8,7 +8,7 @@ container:
 build:
   - name: build
     environment:
-      PYTHONS: /opt/python/cp38-cp38/bin,/opt/python/cp39-cp39/bin,/opt/python/cp310-cp310/bin,/opt/python/cp311-cp311/bin,/opt/python/cp312-cp312/bin
+      PYTHONS: /opt/python/cp38-cp38/bin,/opt/python/cp39-cp39/bin,/opt/python/cp310-cp310/bin,/opt/python/cp311-cp311/bin,/opt/python/cp312-cp312/bin,/opt/python/cp313-cp313/bin
     script:
       - scripts/manylinux2014build.sh
     artifact:

diff --git a/.github/actions/run-ee-server-for-ext-container/action.yml b/.github/actions/run-ee-server-for-ext-container/action.yml
diff --git a/.github/actions/run-ee-server/action.yml b/.github/actions/run-ee-server/action.yml
@@ -1,49 +1,34 @@
-name: 'Run EE Server'
+name: 'Run EE Server in a Docker container'
 description: 'Run EE server. Returns once server is ready. Only tested on Linux and macOS'
 # NOTE: do not share this server container with others
 # since it's using the default admin / admin credentials
 inputs:
   # All inputs in composite actions are strings
   use-server-rc:
     required: true
-    default: false
+    description: Deploy server release candidate?
+    default: 'false'
   server-tag:
     required: true
+    description: Specify Docker tag
     default: 'latest'
   # Github Composite Actions can't access secrets
   # so we need to pass them in as inputs
   docker-hub-username:
+    description: Required for using release candidates
     required: false
   docker-hub-password:
+    description: Required for using release candidates
     required: false
+  where-is-client-connecting-from:
+    required: false
+    description: 'docker-host, separate-docker-container, "remote-connection" via DOCKER_HOST'
+    default: 'docker-host'
 
 runs:
   using: "composite"
   steps:
-  - name: Install crudini to manipulate config.conf
-    # This will only work on the Github hosted runners.
-    # TODO: mac m1 self hosted runners do not have pipx installed by default
-    run: pipx install crudini --pip-args "-c ${{ github.workspace }}/.github/workflows/requirements.txt"
-    working-directory: .github/workflows
-    shell: bash
-
-  - name: Create config.conf
-    run: cp config.conf.template config.conf
-    working-directory: test
-    shell: bash
-
-  - name: Use enterprise edition instead of community edition in config.conf
-    run: |
-      crudini --existing=param --set config.conf enterprise-edition hosts ''
-      crudini --existing=param --set config.conf enterprise-edition hosts 127.0.0.1:3000
-      crudini --existing=param --set config.conf enterprise-edition user superuser
-      crudini --existing=param --set config.conf enterprise-edition password superuser
-    working-directory: test
-    shell: bash
-
-  - name: Create config folder to store configs in
-    run: mkdir configs
-    shell: bash
+  # Start up server
 
   - name: Log into Docker Hub to get server RC
     if: ${{ inputs.use-server-rc == 'true' }}
@@ -53,30 +38,133 @@ runs:
   - run: echo IMAGE_NAME=aerospike/aerospike-server-enterprise${{ inputs.use-server-rc == 'true' && '-rc' || '' }}:${{ inputs.server-tag }} >> $GITHUB_ENV
     shell: bash
 
-  - run: echo SECURITY_IMAGE_NAME=${{ env.IMAGE_NAME }}-security >> $GITHUB_ENV
+  - run: echo NEW_IMAGE_NAME=${{ env.IMAGE_NAME }}-python-client-testing >> $GITHUB_ENV
     shell: bash
 
   # macOS Github runners and Windows self-hosted runners don't have buildx installed by default
   - if: ${{ runner.os == 'Windows' || runner.os == 'macOS' }}
     uses: docker/setup-buildx-action@v3
 
-  - name: Build and push
+  - run: echo CA_CERT_FILE_NAME="ca.cer" >> $GITHUB_ENV
+    shell: bash
+
+  - run: echo CA_KEY_FILE_NAME="ca.pem" >> $GITHUB_ENV
+    shell: bash
+
+  - name: Create a certificate authority
+    run: openssl req -x509 -newkey rsa:2048 -keyout ${{ env.CA_KEY_FILE_NAME }} -out ${{ env.CA_CERT_FILE_NAME }} -nodes -subj '/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=mydummyca'
+    working-directory: .github/workflows/docker-build-context
+    shell: bash
+    env:
+      # Makes sure that the subject isn't interpreted as a path
+      MSYS_NO_PATHCONV: 1
+
+  - run: echo TLS_PORT="4333" >> $GITHUB_ENV
+    shell: bash
+
+  - name: Build Aerospike server Docker image for testing
+    # We enable TLS standard authentication to verify that the OpenSSL library bundled with the wheel works
+    # You can manually verify this by enabling debug logging in the client and checking that the server certificate was verified
     uses: docker/build-push-action@v6
     with:
       # Don't want to use default Git context or else it will clone the whole Python client repo again
-      context: .github/workflows
+      context: .github/workflows/docker-build-context
       build-args: |
-        image=${{ env.IMAGE_NAME }}
-      tags: ${{ env.SECURITY_IMAGE_NAME }}
+        SERVER_IMAGE=${{ env.IMAGE_NAME }}
+        CA_KEY_FILE_NAME=${{ env.CA_KEY_FILE_NAME }}
+        CA_CERT_FILE_NAME=${{ env.CA_CERT_FILE_NAME }}
+        TLS_PORT=${{ env.TLS_PORT }}
+      tags: ${{ env.NEW_IMAGE_NAME }}
       # setup-buildx-action configures Docker to use the docker-container build driver
       # This driver doesn't publish an image locally by default
       # so we have to manually enable it
       load: true
 
-  - run: docker run -d --name aerospike -p 3000:3000 ${{ env.SECURITY_IMAGE_NAME }}
+  - run: echo SERVER_CONTAINER_NAME="aerospike" >> $GITHUB_ENV
+    shell: bash
+
+  - run: docker run -d --name ${{ env.SERVER_CONTAINER_NAME }} -e DEFAULT_TTL=2592000 -p 3000:3000 -p ${{ env.TLS_PORT }}:${{ env.TLS_PORT }} ${{ env.NEW_IMAGE_NAME }}
     shell: bash
 
   - uses: ./.github/actions/wait-for-as-server-to-start
     with:
-      container-name: aerospike
+      container-name: ${{ env.SERVER_CONTAINER_NAME }}
       is-security-enabled: true
+      is-strong-consistency-enabled: true
+
+  - run: echo SUPERUSER_NAME_AND_PASSWORD="superuser" >> $GITHUB_ENV
+    shell: bash
+
+  - run: echo ASADM_AUTH_FLAGS="--user=${{ env.SUPERUSER_NAME_AND_PASSWORD }} --password=${{ env.SUPERUSER_NAME_AND_PASSWORD }}" >> $GITHUB_ENV
+    shell: bash
+
+  # All the partitions are assumed to be dead when reusing a roster file
+  - run: docker exec ${{ env.SERVER_CONTAINER_NAME }} asadm $ASADM_AUTH_FLAGS --enable --execute "manage revive ns test"
+    shell: bash
+
+  # Apply changes
+  - run: docker exec ${{ env.SERVER_CONTAINER_NAME }} asadm $ASADM_AUTH_FLAGS --enable --execute "manage recluster"
+    shell: bash
+
+  # For debugging
+  - run: docker logs ${{ env.SERVER_CONTAINER_NAME }}
+    shell: bash
+
+  # Configure tests
+
+  - name: Install crudini to manipulate config.conf
+    run: pipx install crudini --pip-args "-c ${{ github.workspace }}/.github/workflows/requirements.txt"
+    working-directory: .github/workflows
+    shell: bash
+
+  - name: Create config.conf
+    run: cp config.conf.template config.conf
+    working-directory: test
+    shell: bash
+
+  - name: Disable community edition connection
+    run: crudini --existing=param --set config.conf community-edition hosts ''
+    working-directory: test
+    shell: bash
+
+  - name: Set credentials in config file
+    run: |
+      crudini --existing=param --set config.conf enterprise-edition user ${{ env.SUPERUSER_NAME_AND_PASSWORD }}
+      crudini --existing=param --set config.conf enterprise-edition password ${{ env.SUPERUSER_NAME_AND_PASSWORD }}
+      crudini --set config.conf tls enable true
+      crudini --set config.conf tls cafile ../.github/workflows/docker-build-context/${{ env.CA_CERT_FILE_NAME }}
+    working-directory: test
+    shell: bash
+
+  - name: Set IP address to localhost
+    if: ${{ inputs.where-is-client-connecting-from == 'docker-host' }}
+    run: echo SERVER_IP=127.0.0.1 >> $GITHUB_ENV
+    working-directory: test
+    shell: bash
+
+  - name: Set IP address to remote machine running the Docker daemon
+    if: ${{ inputs.where-is-client-connecting-from == 'remote-connection' }}
+    run: |
+      SERVER_IP=${DOCKER_HOST/tcp:\/\//}
+      echo SERVER_IP=${SERVER_IP/:2375/} >> $GITHUB_ENV
+    working-directory: test
+    shell: bash
+
+  - name: Set IP address to Docker container for the server
+    if: ${{ inputs.where-is-client-connecting-from == 'separate-docker-container' }}
+    run: echo SERVER_IP=$(docker container inspect -f '{{ .NetworkSettings.IPAddress }}' ${{ env.SERVER_CONTAINER_NAME }}) >> $GITHUB_ENV
+    shell: bash
+
+  - name: Invalid input
+    if: ${{ env.SERVER_IP == '' }}
+    run: exit 1
+    shell: bash
+
+  - name: Get cluster name
+    run: echo CLUSTER_NAME=$(docker exec ${{ env.SERVER_CONTAINER_NAME }} asinfo $ASADM_AUTH_FLAGS -v "get-config:context=service" -l | grep -i cluster-name | cut -d = -f 2) >> $GITHUB_ENV
+    shell: bash
+
+  - name: Set EE server's IP address
+    run: crudini --existing=param --set config.conf enterprise-edition hosts "${{ env.SERVER_IP }}:${{ env.TLS_PORT }}|${{ env.CLUSTER_NAME }}"
+    working-directory: test
+    shell: bash
diff --git a/.github/actions/setup-docker-on-macos/action.yml b/.github/actions/setup-docker-on-macos/action.yml
@@ -0,0 +1,17 @@
+name: 'Install Docker on macOS runner'
+description: 'Install Docker using colima'
+
+runs:
+  using: "composite"
+  steps:
+  - name: Install Docker Engine
+    run: brew install colima
+    shell: bash
+
+  - name: Install Docker client
+    run: brew install docker
+    shell: bash
+
+  - name: Start Docker Engine
+    run: colima start
+    shell: bash
diff --git a/.github/actions/wait-for-as-server-to-start/action.yml b/.github/actions/wait-for-as-server-to-start/action.yml
@@ -6,6 +6,9 @@ inputs:
   is-security-enabled:
     required: false
     default: 'false'
+  is-strong-consistency-enabled:
+    required: false
+    default: 'false'
 
 runs:
   using: "composite"
@@ -21,5 +24,5 @@ runs:
   # Also, we don't want to fail if we timeout in case the server *did* finish starting up but the script couldn't detect it due to a bug
   # Effectively, this composite action is like calling "sleep" that is optimized to exit early when it detects an ok from the server
   - name: Wait for EE server to start
-    run: timeout 30 bash ./.github/workflows/wait-for-as-server-to-start.bash ${{ inputs.container-name }} ${{ inputs.is-security-enabled }} || true
+    run: timeout 30 bash ./.github/workflows/wait-for-as-server-to-start.bash ${{ inputs.container-name }} ${{ inputs.is-security-enabled }} ${{ inputs.is-strong-consistency-enabled }} || true
     shell: bash
diff --git a/.github/workflows/Dockerfile b/.github/workflows/Dockerfile
diff --git a/.github/workflows/build-and-run-stage-tests.yml b/.github/workflows/build-and-run-stage-tests.yml
@@ -21,12 +21,32 @@ on:
         description: 'Test macOS x86 wheels (unstable)'
 
 jobs:
-  build-wheels:
+  build-select-wheels:
+    strategy:
+      matrix:
+        platform-tag: [
+          "manylinux_x86_64",
+          "manylinux_aarch64",
+          "macosx_x86_64"
+        ]
+      # Need all the artifacts to run all the stage tests, so fail fast
     uses: ./.github/workflows/build-wheels.yml
+    with:
+      platform-tag: ${{ matrix.platform-tag }}
+      sha-to-build-and-test: ${{ github.sha }}
+    secrets: inherit
+
+  build-sdist:
+    uses: ./.github/workflows/build-sdist.yml
+    with:
+      sha_to_build: ${{ github.sha }}
 
   run-stage-tests:
     uses: ./.github/workflows/stage-tests.yml
-    needs: build-wheels
+    needs: [
+      build-select-wheels,
+      build-sdist
+    ]
     secrets: inherit
     with:
       use_jfrog_builds: false

diff --git a/.github/workflows/build-and-upload-wheels-for-qe.yml b/.github/workflows/build-and-upload-wheels-for-qe.yml
@@ -17,11 +17,18 @@ on:
 
 jobs:
   build-artifacts:
+    strategy:
+      matrix:
+        platform-tag: [
+          "manylinux_x86_64",
+          "manylinux_aarch64"
+        ]
     uses: ./.github/workflows/build-wheels.yml
     with:
-      # In a push event, any input values default to ''
-      # https://github.com/orgs/community/discussions/29242#discussioncomment-5063461
-      apply-no-optimizations: ${{ github.event_name == 'workflow_dispatch' && inputs.disable-optimizations || false }}
+      platform-tag: ${{ matrix.platform-tag }}
+      unoptimized: ${{ github.event_name == 'workflow_dispatch' && inputs.disable-optimizations || false }}
+      sha-to-build-and-test: ${{ github.sha }}
+    secrets: inherit
 
   upload-to-jfrog:
     needs: build-artifacts