diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index eabb1ad22..5291b8568 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -239,7 +239,7 @@ jobs:
           echo "VALIDATOR_IMAGE=ghcr.io/${LOWERCASE_REPO_OWNER}/gpu-operator/gpu-operator-validator"  >> $GITHUB_ENV
 
           echo "instance_hostname=ubuntu@${{ steps.get_public_dns_name.outputs.result }}" >> $GITHUB_ENV
-          echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
+          echo "private_key=${{ github.workspace }}/.cache/key" >> $GITHUB_ENV
       - name: Run e2e tests
         env:
           GPU_PRODUCT_NAME: "Tesla-T4"
@@ -291,7 +291,7 @@ jobs:
           echo "VALIDATOR_IMAGE=ghcr.io/${LOWERCASE_REPO_OWNER}/gpu-operator/gpu-operator-validator"  >> $GITHUB_ENV
 
           echo "instance_hostname=ubuntu@${{ steps.get_public_dns_name.outputs.result }}" >> $GITHUB_ENV
-          echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
+          echo "private_key=${{ github.workspace }}/.cache/key" >> $GITHUB_ENV
       - name: Run e2e tests
         env:
           GPU_PRODUCT_NAME: "Tesla-T4"
diff --git a/tests/scripts/.rsync-excludes b/tests/scripts/.rsync-excludes
index 3a945297f..d481cd6bb 100644
--- a/tests/scripts/.rsync-excludes
+++ b/tests/scripts/.rsync-excludes
@@ -3,3 +3,4 @@ vendor/
 aws-kube-ci
 cnt-ci
 key.pem
+key