From d4ed587ca2b83877f262922e451e3a75934ef1ad Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:03:22 -0700 Subject: [PATCH 01/58] wip --- .github/workflows/valgrind.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 15de27cee..76396cffa 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -19,7 +19,21 @@ on: default: false jobs: + look-for-wheel-in-jfrog: + runs-on: ubuntu-22.04 + steps: + - uses: jfrog/setup-jfrog-cli@v4 + env: + JF_URL: ${{ secrets.JFROG_PLATFORM_URL }} + JF_ACCESS_TOKEN: ${{ secrets.JFROG_ACCESS_TOKEN }} + + - name: Look for wheel built with default settings in JFrog + # TODO: need to reduce sha to certain number of chars + run: jf rt search ${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/${{ github.sha }} + build-manylinux-wheel: + needs: look-for-wheel-in-jfrog + if: ${{ !cancelled() && needs.look-for-wheel-in-jfrog.result == 'failure' }} uses: ./.github/workflows/build-wheels.yml with: python-tags: '["cp38"]' From ce311f2ecb000d576fdad558bcf108d88aaea8c5 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:21:44 -0700 Subject: [PATCH 02/58] wip --- .github/workflows/valgrind.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 76396cffa..429e63e30 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -18,6 +18,9 @@ on: required: false default: false +env: + PYTHON_TAG: cp38 + jobs: look-for-wheel-in-jfrog: runs-on: ubuntu-22.04 @@ -29,7 +32,8 @@ jobs: - name: Look for wheel built with default settings in JFrog # TODO: need to reduce sha to certain number of chars - run: jf rt search ${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/${{ github.sha }} + # TODO: must match exactly one artifact + run: jf rt search ${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/*${{ env.PYTHON_TAG }}*manylinux*x86_64*${{ github.sha }}* build-manylinux-wheel: needs: look-for-wheel-in-jfrog From 7600ac3cddbc0fbc1d90192a4bf4727f5bb4c734 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 11 Sep 2024 16:54:27 -0700 Subject: [PATCH 03/58] wip --- .github/workflows/valgrind.yml | 40 +++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 429e63e30..06f055b60 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -20,9 +20,12 @@ on: env: PYTHON_TAG: cp38 + PLATFORM_TAG: manylinux_x86_64 jobs: look-for-wheel-in-jfrog: + outputs: + num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} runs-on: ubuntu-22.04 steps: - uses: jfrog/setup-jfrog-cli@v4 @@ -30,14 +33,38 @@ jobs: JF_URL: ${{ secrets.JFROG_PLATFORM_URL }} JF_ACCESS_TOKEN: ${{ secrets.JFROG_ACCESS_TOKEN }} + - name: Get shortened commit hash of this workflow run + # versioningit commit sha is always 8 chars long it seems + # We assume this prints out a sha that is unique and at most 8 chars + run: echo SHORT_GITHUB_SHA=$(git rev-parse --short ${{ github.sha }}) >> $GITHUB_ENV + + # TODO: will fail if one commit has two builds with different labels + # TODO: might be code in stage tests to check exact file name matches? + - run: echo JF_SEARCH_STRING="${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/*${{ env.SHORT_GITHUB_SHA }}*${{ env.PYTHON_TAG }}*manylinux*x86_64*" >> $GITHUB_ENV + - name: Look for wheel built with default settings in JFrog - # TODO: need to reduce sha to certain number of chars - # TODO: must match exactly one artifact - run: jf rt search ${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/*${{ env.PYTHON_TAG }}*manylinux*x86_64*${{ github.sha }}* + id: count_num_artifacts_found + run: echo num_artifacts=$(jf rt search --count "${{ env.JF_SEARCH_STRING }}") >> $GITHUB_OUTPUT + + - name: Cannot determine which artifact to use, fail out + if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts > 1 }} + run: exit 1 + + - name: Found the exact artifact in JFrog, downloading... + if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} + run: jf rt download --flat --fail-no-op "${{ env.JF_SEARCH_STRING }}" + + - name: Pass to valgrind job + if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} + uses: actions/upload-artifact@v4 + with: + if-no-files-found: error + name: ${{ env.PYTHON_TAG }}-${{ env.PLATFORM_TAG }}.build + path: '*.whl' build-manylinux-wheel: needs: look-for-wheel-in-jfrog - if: ${{ !cancelled() && needs.look-for-wheel-in-jfrog.result == 'failure' }} + if: ${{ needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 }} uses: ./.github/workflows/build-wheels.yml with: python-tags: '["cp38"]' @@ -49,6 +76,7 @@ jobs: env: MASSIF_REPORT_FILE_NAME: massif.out needs: build-manylinux-wheel + if: ${{ !cancelled() && needs.build-manylinux-wheel.result == 'success' || needs.build-manylinux-wheel.result == 'skipped' }} runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 @@ -58,12 +86,13 @@ jobs: - uses: actions/setup-python@v2 with: + # TODO: can convert from python tag python-version: '3.8' architecture: 'x64' - uses: actions/download-artifact@v4 with: - name: cp38-manylinux_x86_64.build + name: ${{ env.PYTHON_TAG }}-${{ env.PLATFORM_TAG }}.build - name: Install client run: pip install ./*.whl @@ -90,6 +119,7 @@ jobs: - run: PYTHONMALLOC=malloc valgrind --error-exitcode=1 ${{ env.VALGRIND_ARGS }} python3 -m pytest -v new_tests/${{ github.event.inputs.test-file }} working-directory: test + # TODO: upload report as artifact - run: ms_print ./${{ env.MASSIF_REPORT_FILE_NAME }} if: ${{ !cancelled() && inputs.massif }} working-directory: test From 745d6a37bd568a06164fdf9ddb403439b6acf0a6 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 12 Sep 2024 08:17:23 -0700 Subject: [PATCH 04/58] wip --- .github/workflows/valgrind.yml | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 06f055b60..2778eb060 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -26,6 +26,10 @@ jobs: look-for-wheel-in-jfrog: outputs: num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} + python-tag: ${{ env.PYTHON_TAG }} + platform-tag: ${{ env.PLATFORM_TAG }} + env: + JFROG_FILESPEC_NAME: jfrog-filespec.json runs-on: ubuntu-22.04 steps: - uses: jfrog/setup-jfrog-cli@v4 @@ -38,13 +42,15 @@ jobs: # We assume this prints out a sha that is unique and at most 8 chars run: echo SHORT_GITHUB_SHA=$(git rev-parse --short ${{ github.sha }}) >> $GITHUB_ENV - # TODO: will fail if one commit has two builds with different labels - # TODO: might be code in stage tests to check exact file name matches? - - run: echo JF_SEARCH_STRING="${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/*${{ env.SHORT_GITHUB_SHA }}*${{ env.PYTHON_TAG }}*manylinux*x86_64*" >> $GITHUB_ENV + - name: Set filespec variables + # TODO: set using env var? + run: echo SPEC_VARS_FLAG='--spec-vars="repo_name=${{ vars.JFROG_GENERIC_REPO_NAME }};branch=${{ github.ref_name }};commit_sha=${{ env.SHORT_GITHUB_SHA }};python_tag=${{ env.PYTHON_TAG }}"' >> $GITHUB_ENV - name: Look for wheel built with default settings in JFrog id: count_num_artifacts_found - run: echo num_artifacts=$(jf rt search --count "${{ env.JF_SEARCH_STRING }}") >> $GITHUB_OUTPUT + # We use AQL because we want to exclude builds if they have special labels, like unoptimized builds + run: echo num_artifacts=$(jf rt search --count --spec=./${{ env.JFROG_FILESPEC_NAME }} ${{ env.SPEC_VARS_FLAG }}) >> $GITHUB_OUTPUT + working-directory: .github/workflows - name: Cannot determine which artifact to use, fail out if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts > 1 }} @@ -52,7 +58,7 @@ jobs: - name: Found the exact artifact in JFrog, downloading... if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} - run: jf rt download --flat --fail-no-op "${{ env.JF_SEARCH_STRING }}" + run: jf rt download --flat --fail-no-op --spec=./${{ env.JFROG_FILESPEC_NAME }} ${{ env.SPEC_VARS_FLAG }} - name: Pass to valgrind job if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} @@ -67,8 +73,8 @@ jobs: if: ${{ needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 }} uses: ./.github/workflows/build-wheels.yml with: - python-tags: '["cp38"]' - platform-tag: manylinux_x86_64 + python-tags: '["${{ needs.look-for-wheel-in-jfrog.outputs.python-tag }}"]' + platform-tag: ${{ needs.look-for-wheel-in-jfrog.outputs.platform-tag }} sha-to-build-and-test: ${{ github.sha }} secrets: inherit @@ -84,16 +90,24 @@ jobs: submodules: recursive fetch-depth: 0 + - name: Convert Python tag to Python version + run: echo PYTHON_VERSION=$(echo ${{ env.PYTHON_TAG }} | sed -e "s/cp3/cp3./" -e "s/cp//") >> $GITHUB_ENV + shell: bash + - uses: actions/setup-python@v2 with: - # TODO: can convert from python tag - python-version: '3.8' + python-version: '${{ env.PYTHON_VERSION }}' architecture: 'x64' - uses: actions/download-artifact@v4 with: name: ${{ env.PYTHON_TAG }}-${{ env.PLATFORM_TAG }}.build + - name: Remove artifact from Github + uses: geekyeggo/delete-artifact@v5 + with: + name: ${{ env.PYTHON_TAG }}-${{ env.PLATFORM_TAG }}.build + - name: Install client run: pip install ./*.whl From f0c5952e05987a023db096ffc786d384e4ba5564 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 12 Sep 2024 08:18:00 -0700 Subject: [PATCH 05/58] wip --- .github/workflows/jfrog-filespec.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/jfrog-filespec.json diff --git a/.github/workflows/jfrog-filespec.json b/.github/workflows/jfrog-filespec.json new file mode 100644 index 000000000..3161ca656 --- /dev/null +++ b/.github/workflows/jfrog-filespec.json @@ -0,0 +1,16 @@ +{ + "files": [ + { + "aql": { + "items.find": { + "repo": "${repo_name}", + "path": "${branch}", + "name": { + "$nmatch": "*(+|.)[a-z]+*", + "$match": "*${commit_sha}*${python_tag}*manylinux*x86_64*.whl" + } + } + } + } + ] +} From 26e63e7dc43e4324a662d97a981670775762e4c9 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 12 Sep 2024 08:33:27 -0700 Subject: [PATCH 06/58] checkout --- .github/workflows/valgrind.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 2778eb060..03157cd66 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -37,6 +37,11 @@ jobs: JF_URL: ${{ secrets.JFROG_PLATFORM_URL }} JF_ACCESS_TOKEN: ${{ secrets.JFROG_ACCESS_TOKEN }} + - uses: actions/checkout@v4 + with: + sparse-checkout-cone-mode: false + sparse-checkout: .github/workflows/${{ env.JFROG_FILESPEC_NAME }} + - name: Get shortened commit hash of this workflow run # versioningit commit sha is always 8 chars long it seems # We assume this prints out a sha that is unique and at most 8 chars From 0f5a0fd9594eacdb8b0a0711830da3897a5393f5 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 12 Sep 2024 08:51:00 -0700 Subject: [PATCH 07/58] fix syntax --- .github/workflows/jfrog-filespec.json | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/jfrog-filespec.json b/.github/workflows/jfrog-filespec.json index 3161ca656..a7fadf14f 100644 --- a/.github/workflows/jfrog-filespec.json +++ b/.github/workflows/jfrog-filespec.json @@ -5,10 +5,18 @@ "items.find": { "repo": "${repo_name}", "path": "${branch}", - "name": { - "$nmatch": "*(+|.)[a-z]+*", - "$match": "*${commit_sha}*${python_tag}*manylinux*x86_64*.whl" - } + "$and": [ + { + "name": { + "$nmatch": "*(+|.)[a-z]+*" + } + }, + { + "name": { + "$match": "*${commit_sha}*${python_tag}*manylinux*x86_64*.whl" + } + } + ] } } } From e9a3344f0ac77ea653b2f1c80022927d6d28405a Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 12 Sep 2024 09:01:14 -0700 Subject: [PATCH 08/58] cache wheel using jfrog --- .github/workflows/valgrind.yml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 03157cd66..d878c2ece 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -83,6 +83,13 @@ jobs: sha-to-build-and-test: ${{ github.sha }} secrets: inherit + upload-built-wheel-to-jfrog: + needs: build-manylinux-wheel + uses: ./.github/workflows/upload-to-jfrog.yml + with: + jfrog-repo-name: ${{ vars.JFROG_GENERIC_REPO_NAME }} + secrets: inherit + valgrind: env: MASSIF_REPORT_FILE_NAME: massif.out @@ -108,11 +115,6 @@ jobs: with: name: ${{ env.PYTHON_TAG }}-${{ env.PLATFORM_TAG }}.build - - name: Remove artifact from Github - uses: geekyeggo/delete-artifact@v5 - with: - name: ${{ env.PYTHON_TAG }}-${{ env.PLATFORM_TAG }}.build - - name: Install client run: pip install ./*.whl @@ -142,3 +144,15 @@ jobs: - run: ms_print ./${{ env.MASSIF_REPORT_FILE_NAME }} if: ${{ !cancelled() && inputs.massif }} working-directory: test + + # See reason for deleting artifacts in dev-workflow-p2.yml + # TODO: probably not correct + delete-artifacts: + needs: [ + # These jobs must have downloaded the artifact from Github before we can delete it + upload-built-wheel-to-jfrog, + valgrind + ] + # Workflow run must clean up after itself even if cancelled + if: ${{ always() }} + uses: ./.github/workflows/delete-artifacts.yml From a38524dd65fbbacd9253f28da6c5e6e4311558f2 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 12 Sep 2024 09:09:40 -0700 Subject: [PATCH 09/58] update comments --- .github/workflows/valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index d878c2ece..8bc2326cb 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -54,6 +54,7 @@ jobs: - name: Look for wheel built with default settings in JFrog id: count_num_artifacts_found # We use AQL because we want to exclude builds if they have special labels, like unoptimized builds + # We can't use a search pattern to exclude strings, as far as I know run: echo num_artifacts=$(jf rt search --count --spec=./${{ env.JFROG_FILESPEC_NAME }} ${{ env.SPEC_VARS_FLAG }}) >> $GITHUB_OUTPUT working-directory: .github/workflows @@ -146,7 +147,6 @@ jobs: working-directory: test # See reason for deleting artifacts in dev-workflow-p2.yml - # TODO: probably not correct delete-artifacts: needs: [ # These jobs must have downloaded the artifact from Github before we can delete it From 68f4f6832848ecc850ad5c6cf02abf9183f8ab05 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 12 Sep 2024 09:15:04 -0700 Subject: [PATCH 10/58] fix working dir --- .github/workflows/valgrind.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 8bc2326cb..f0112f6d8 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -65,6 +65,7 @@ jobs: - name: Found the exact artifact in JFrog, downloading... if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} run: jf rt download --flat --fail-no-op --spec=./${{ env.JFROG_FILESPEC_NAME }} ${{ env.SPEC_VARS_FLAG }} + working-directory: .github/workflows - name: Pass to valgrind job if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} From 604e07a684663e3e45d6ddde3579200cf95a0629 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 12 Sep 2024 09:24:39 -0700 Subject: [PATCH 11/58] fix --- .github/workflows/valgrind.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index f0112f6d8..ada22439c 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -73,7 +73,7 @@ jobs: with: if-no-files-found: error name: ${{ env.PYTHON_TAG }}-${{ env.PLATFORM_TAG }}.build - path: '*.whl' + path: './.github/workflows/*.whl' build-manylinux-wheel: needs: look-for-wheel-in-jfrog @@ -96,7 +96,7 @@ jobs: env: MASSIF_REPORT_FILE_NAME: massif.out needs: build-manylinux-wheel - if: ${{ !cancelled() && needs.build-manylinux-wheel.result == 'success' || needs.build-manylinux-wheel.result == 'skipped' }} + if: ${{ !cancelled() && (needs.build-manylinux-wheel.result == 'success' || needs.build-manylinux-wheel.result == 'skipped') }} runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 From 05011350cb549b1bb84bb9f1fdb9c4c1aa040ae6 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 12 Sep 2024 09:36:45 -0700 Subject: [PATCH 12/58] todo --- .github/workflows/valgrind.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index ada22439c..a769926c6 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -23,6 +23,7 @@ env: PLATFORM_TAG: manylinux_x86_64 jobs: + # TODO: handle case where we run valgrind on a "central" branch look-for-wheel-in-jfrog: outputs: num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} From 0242e354bce388843a37fda1e02975699339f561 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 16 Sep 2024 12:56:07 -0700 Subject: [PATCH 13/58] wip will break --- .github/workflows/valgrind.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index a769926c6..36adcc1b7 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -24,7 +24,8 @@ env: jobs: # TODO: handle case where we run valgrind on a "central" branch - look-for-wheel-in-jfrog: + look-for-wheel-in-generic-jfrog-repo: + if: ${{ github.ref_name != 'dev' && github.ref_name != 'stage' && github.ref_name != 'master' }} outputs: num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} python-tag: ${{ env.PYTHON_TAG }} @@ -77,8 +78,8 @@ jobs: path: './.github/workflows/*.whl' build-manylinux-wheel: - needs: look-for-wheel-in-jfrog - if: ${{ needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 }} + needs: look-for-wheel-in-generic-jfrog-repo + if: ${{ !cancelled() && needs.look-for-wheel-in-generic-jfrog-repo.result == 'skipped' || (needs.look-for-wheel-in-generic-jfrog-repo.result == 'success' && needs.look-for-wheel-in-generic-jfrog-repo.outputs.num_artifacts_found == 0) }} uses: ./.github/workflows/build-wheels.yml with: python-tags: '["${{ needs.look-for-wheel-in-jfrog.outputs.python-tag }}"]' From 53c5deb9cf3ef3758ff75896c6aeca53e03e9db3 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 18 Sep 2024 10:14:47 -0700 Subject: [PATCH 14/58] Revert "wip will break" This reverts commit 0242e354bce388843a37fda1e02975699339f561. --- .github/workflows/valgrind.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 36adcc1b7..a769926c6 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -24,8 +24,7 @@ env: jobs: # TODO: handle case where we run valgrind on a "central" branch - look-for-wheel-in-generic-jfrog-repo: - if: ${{ github.ref_name != 'dev' && github.ref_name != 'stage' && github.ref_name != 'master' }} + look-for-wheel-in-jfrog: outputs: num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} python-tag: ${{ env.PYTHON_TAG }} @@ -78,8 +77,8 @@ jobs: path: './.github/workflows/*.whl' build-manylinux-wheel: - needs: look-for-wheel-in-generic-jfrog-repo - if: ${{ !cancelled() && needs.look-for-wheel-in-generic-jfrog-repo.result == 'skipped' || (needs.look-for-wheel-in-generic-jfrog-repo.result == 'success' && needs.look-for-wheel-in-generic-jfrog-repo.outputs.num_artifacts_found == 0) }} + needs: look-for-wheel-in-jfrog + if: ${{ needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 }} uses: ./.github/workflows/build-wheels.yml with: python-tags: '["${{ needs.look-for-wheel-in-jfrog.outputs.python-tag }}"]' From 4bd345daa80511ecfe0174f95af9c5c47074a35f Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 23 Sep 2024 12:27:50 -0700 Subject: [PATCH 15/58] mv --- .../action.yml | 0 .github/workflows/stage-tests.yml | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename .github/actions/{get-artifact-for-stage-tests => get-artifact-for-testing}/action.yml (100%) diff --git a/.github/actions/get-artifact-for-stage-tests/action.yml b/.github/actions/get-artifact-for-testing/action.yml similarity index 100% rename from .github/actions/get-artifact-for-stage-tests/action.yml rename to .github/actions/get-artifact-for-testing/action.yml diff --git a/.github/workflows/stage-tests.yml b/.github/workflows/stage-tests.yml index a7664a3e0..bf7217d2c 100644 --- a/.github/workflows/stage-tests.yml +++ b/.github/workflows/stage-tests.yml @@ -82,7 +82,7 @@ jobs: - run: echo "DISTRO_DOCKER_IMAGE_PLATFORM=${{ matrix.test-case[2] }}" >> $GITHUB_ENV - run: echo "PYTHON_VERSION=${{ matrix.test-case[3] }}" >> $GITHUB_ENV - - uses: ./.github/actions/get-artifact-for-stage-tests + - uses: ./.github/actions/get-artifact-for-testing with: get_from_jfrog: ${{ inputs.use_jfrog_builds }} # This input is only used if above input is true @@ -169,7 +169,7 @@ jobs: test .github - - uses: ./.github/actions/get-artifact-for-stage-tests + - uses: ./.github/actions/get-artifact-for-testing with: # See comments in linux stage tests for how this works get_from_jfrog: ${{ inputs.use_jfrog_builds }} From fb00ca3b592add9c7f18459aaabfddab886c0f8e Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:07:57 -0700 Subject: [PATCH 16/58] finish --- .../action.yml | 0 .github/workflows/valgrind.yml | 18 +++++++++--------- 2 files changed, 9 insertions(+), 9 deletions(-) rename .github/actions/{get-artifact-for-testing => get-artifact-for-stage-tests}/action.yml (100%) diff --git a/.github/actions/get-artifact-for-testing/action.yml b/.github/actions/get-artifact-for-stage-tests/action.yml similarity index 100% rename from .github/actions/get-artifact-for-testing/action.yml rename to .github/actions/get-artifact-for-stage-tests/action.yml diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index a769926c6..749c37d9e 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -20,15 +20,16 @@ on: env: PYTHON_TAG: cp38 - PLATFORM_TAG: manylinux_x86_64 + WHEEL_OS: manylinux + WHEEL_ARCH: x86_64 jobs: # TODO: handle case where we run valgrind on a "central" branch look-for-wheel-in-jfrog: + if: ${{ github.ref_name != 'dev' && github.ref_name != 'stage' && github.ref_name != 'master' }} outputs: num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} python-tag: ${{ env.PYTHON_TAG }} - platform-tag: ${{ env.PLATFORM_TAG }} env: JFROG_FILESPEC_NAME: jfrog-filespec.json runs-on: ubuntu-22.04 @@ -72,13 +73,16 @@ jobs: if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} uses: actions/upload-artifact@v4 with: + # Artifact name doesn't matter. Valgrind job downloads all artifacts to get the one wheel if-no-files-found: error - name: ${{ env.PYTHON_TAG }}-${{ env.PLATFORM_TAG }}.build path: './.github/workflows/*.whl' build-manylinux-wheel: needs: look-for-wheel-in-jfrog - if: ${{ needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 }} + # If previous job was skipped, we are running on a central branch + # Always build wheels from a central branch for now + # Or if we are running on a feature branch and could not find any wheels in JFrog, build and upload it to JFrog + if: ${{ !cancelled() && (needs.look-for-wheel-in-jfrog.result == 'skipped' || (needs.look-for-wheel-in-jfrog.result == 'success' && needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0)) }} uses: ./.github/workflows/build-wheels.yml with: python-tags: '["${{ needs.look-for-wheel-in-jfrog.outputs.python-tag }}"]' @@ -105,10 +109,6 @@ jobs: submodules: recursive fetch-depth: 0 - - name: Convert Python tag to Python version - run: echo PYTHON_VERSION=$(echo ${{ env.PYTHON_TAG }} | sed -e "s/cp3/cp3./" -e "s/cp//") >> $GITHUB_ENV - shell: bash - - uses: actions/setup-python@v2 with: python-version: '${{ env.PYTHON_VERSION }}' @@ -116,7 +116,7 @@ jobs: - uses: actions/download-artifact@v4 with: - name: ${{ env.PYTHON_TAG }}-${{ env.PLATFORM_TAG }}.build + merge-multiple: true - name: Install client run: pip install ./*.whl From 850495c213cf01056d5104c4f4182db4ba630e95 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:09:20 -0700 Subject: [PATCH 17/58] rev --- .github/workflows/stage-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stage-tests.yml b/.github/workflows/stage-tests.yml index bf7217d2c..a7664a3e0 100644 --- a/.github/workflows/stage-tests.yml +++ b/.github/workflows/stage-tests.yml @@ -82,7 +82,7 @@ jobs: - run: echo "DISTRO_DOCKER_IMAGE_PLATFORM=${{ matrix.test-case[2] }}" >> $GITHUB_ENV - run: echo "PYTHON_VERSION=${{ matrix.test-case[3] }}" >> $GITHUB_ENV - - uses: ./.github/actions/get-artifact-for-testing + - uses: ./.github/actions/get-artifact-for-stage-tests with: get_from_jfrog: ${{ inputs.use_jfrog_builds }} # This input is only used if above input is true @@ -169,7 +169,7 @@ jobs: test .github - - uses: ./.github/actions/get-artifact-for-testing + - uses: ./.github/actions/get-artifact-for-stage-tests with: # See comments in linux stage tests for how this works get_from_jfrog: ${{ inputs.use_jfrog_builds }} From b4951203324f992f931109ef0e2eaddad497df4e Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:11:15 -0700 Subject: [PATCH 18/58] rv --- .github/workflows/valgrind.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 749c37d9e..f4cbc81a2 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -20,8 +20,7 @@ on: env: PYTHON_TAG: cp38 - WHEEL_OS: manylinux - WHEEL_ARCH: x86_64 + PLATFORM_TAG: manylinux_x86_64 jobs: # TODO: handle case where we run valgrind on a "central" branch @@ -30,6 +29,7 @@ jobs: outputs: num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} python-tag: ${{ env.PYTHON_TAG }} + platform-tag: ${{ env.PLATFORM_TAG }} env: JFROG_FILESPEC_NAME: jfrog-filespec.json runs-on: ubuntu-22.04 From a7c3b1186a9c1108f880469bc020f5f769e526c3 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:13:53 -0700 Subject: [PATCH 19/58] rm useless env var --- .github/workflows/valgrind.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index f4cbc81a2..f698178ce 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -20,7 +20,6 @@ on: env: PYTHON_TAG: cp38 - PLATFORM_TAG: manylinux_x86_64 jobs: # TODO: handle case where we run valgrind on a "central" branch @@ -28,8 +27,9 @@ jobs: if: ${{ github.ref_name != 'dev' && github.ref_name != 'stage' && github.ref_name != 'master' }} outputs: num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} + # We want this environment variable to be available when calling a reusable workflow + # so we convert it to a job output so the reusable workflow can access it python-tag: ${{ env.PYTHON_TAG }} - platform-tag: ${{ env.PLATFORM_TAG }} env: JFROG_FILESPEC_NAME: jfrog-filespec.json runs-on: ubuntu-22.04 @@ -86,7 +86,7 @@ jobs: uses: ./.github/workflows/build-wheels.yml with: python-tags: '["${{ needs.look-for-wheel-in-jfrog.outputs.python-tag }}"]' - platform-tag: ${{ needs.look-for-wheel-in-jfrog.outputs.platform-tag }} + platform-tag: manylinux_x86_64 sha-to-build-and-test: ${{ github.sha }} secrets: inherit From 538b2d28d0f6c51be3cf0d9675c843c9c3a499d2 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:34:34 -0700 Subject: [PATCH 20/58] add abck --- .github/workflows/valgrind.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index f698178ce..6432b0def 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -46,7 +46,7 @@ jobs: - name: Get shortened commit hash of this workflow run # versioningit commit sha is always 8 chars long it seems - # We assume this prints out a sha that is unique and at most 8 chars + # We assume this prints out a sha that is at most 8 chars run: echo SHORT_GITHUB_SHA=$(git rev-parse --short ${{ github.sha }}) >> $GITHUB_ENV - name: Set filespec variables @@ -109,6 +109,10 @@ jobs: submodules: recursive fetch-depth: 0 + - name: Convert Python tag to Python version + run: echo PYTHON_VERSION=$(echo ${{ env.PYTHON_TAG }} | sed -e "s/cp3/cp3./" -e "s/cp//") >> $GITHUB_ENV + shell: bash + - uses: actions/setup-python@v2 with: python-version: '${{ env.PYTHON_VERSION }}' From 33a70d310a60eb66251c7e612241b963ec20b3c1 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 25 Sep 2024 16:05:10 -0700 Subject: [PATCH 21/58] Fix --- .github/workflows/jfrog-filespec.json | 24 --------------------- .github/workflows/valgrind.yml | 31 +++++++++++---------------- 2 files changed, 12 insertions(+), 43 deletions(-) delete mode 100644 .github/workflows/jfrog-filespec.json diff --git a/.github/workflows/jfrog-filespec.json b/.github/workflows/jfrog-filespec.json deleted file mode 100644 index a7fadf14f..000000000 --- a/.github/workflows/jfrog-filespec.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "files": [ - { - "aql": { - "items.find": { - "repo": "${repo_name}", - "path": "${branch}", - "$and": [ - { - "name": { - "$nmatch": "*(+|.)[a-z]+*" - } - }, - { - "name": { - "$match": "*${commit_sha}*${python_tag}*manylinux*x86_64*.whl" - } - } - ] - } - } - } - ] -} diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 6432b0def..210d2ede8 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -30,8 +30,6 @@ jobs: # We want this environment variable to be available when calling a reusable workflow # so we convert it to a job output so the reusable workflow can access it python-tag: ${{ env.PYTHON_TAG }} - env: - JFROG_FILESPEC_NAME: jfrog-filespec.json runs-on: ubuntu-22.04 steps: - uses: jfrog/setup-jfrog-cli@v4 @@ -39,34 +37,29 @@ jobs: JF_URL: ${{ secrets.JFROG_PLATFORM_URL }} JF_ACCESS_TOKEN: ${{ secrets.JFROG_ACCESS_TOKEN }} - - uses: actions/checkout@v4 - with: - sparse-checkout-cone-mode: false - sparse-checkout: .github/workflows/${{ env.JFROG_FILESPEC_NAME }} - - name: Get shortened commit hash of this workflow run # versioningit commit sha is always 8 chars long it seems - # We assume this prints out a sha that is at most 8 chars - run: echo SHORT_GITHUB_SHA=$(git rev-parse --short ${{ github.sha }}) >> $GITHUB_ENV - - - name: Set filespec variables - # TODO: set using env var? - run: echo SPEC_VARS_FLAG='--spec-vars="repo_name=${{ vars.JFROG_GENERIC_REPO_NAME }};branch=${{ github.ref_name }};commit_sha=${{ env.SHORT_GITHUB_SHA }};python_tag=${{ env.PYTHON_TAG }}"' >> $GITHUB_ENV + run: echo SHORT_GITHUB_SHA=$(echo ${{ github.sha }} | cut -c1-8) >> $GITHUB_ENV - name: Look for wheel built with default settings in JFrog + # AQL has the option to exclude patterns from search results + # but it doesn't allow regex, so we can't filter out any type of label in a wheel name + # Example: we want to filter out "unoptimized" and "dsym" but in case we add more labels, we want to use regex + # to handle those new labels without updating the regex. + run: echo $(jf rt search "${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/*${{ env.SHORT_GITHUB_SHA }}*${{ env.PYTHON_TAG }}*manylinux*x86_64*.whl" | jq '.[].path' | grep --invert-match "${{ env.SHORT_GITHUB_SHA }}\.") > file_matches.txt + shell: bash + + - name: Count artifacts id: count_num_artifacts_found - # We use AQL because we want to exclude builds if they have special labels, like unoptimized builds - # We can't use a search pattern to exclude strings, as far as I know - run: echo num_artifacts=$(jf rt search --count --spec=./${{ env.JFROG_FILESPEC_NAME }} ${{ env.SPEC_VARS_FLAG }}) >> $GITHUB_OUTPUT - working-directory: .github/workflows + run: echo num_artifacts=$(echo file_matches.txt | wc -l) >> $GITHUB_OUTPUT - - name: Cannot determine which artifact to use, fail out + - name: Multiple artifacts found, not sure which one to use. Fail out if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts > 1 }} run: exit 1 - name: Found the exact artifact in JFrog, downloading... if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} - run: jf rt download --flat --fail-no-op --spec=./${{ env.JFROG_FILESPEC_NAME }} ${{ env.SPEC_VARS_FLAG }} + run: jf rt download --flat --fail-no-op "$(cat file_matches.txt)" working-directory: .github/workflows - name: Pass to valgrind job From 529e015ea449f961d580ede9e6b6e69012fcb2b3 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Nov 2024 08:42:04 -0800 Subject: [PATCH 22/58] Fix --- .github/workflows/valgrind.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 210d2ede8..4a33e5021 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -60,7 +60,6 @@ jobs: - name: Found the exact artifact in JFrog, downloading... if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} run: jf rt download --flat --fail-no-op "$(cat file_matches.txt)" - working-directory: .github/workflows - name: Pass to valgrind job if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} @@ -68,7 +67,7 @@ jobs: with: # Artifact name doesn't matter. Valgrind job downloads all artifacts to get the one wheel if-no-files-found: error - path: './.github/workflows/*.whl' + path: './*.whl' build-manylinux-wheel: needs: look-for-wheel-in-jfrog From 487a552493bcf9828732e56b7ec22c57e631e55e Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:09:29 -0800 Subject: [PATCH 23/58] Split up --- .github/workflows/valgrind.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 4a33e5021..da9fadad0 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -31,6 +31,8 @@ jobs: # so we convert it to a job output so the reusable workflow can access it python-tag: ${{ env.PYTHON_TAG }} runs-on: ubuntu-22.04 + env: + JF_SEARCH_RESULTS_FILE_NAME: wheel_commit_matches.txt steps: - uses: jfrog/setup-jfrog-cli@v4 env: @@ -46,12 +48,15 @@ jobs: # but it doesn't allow regex, so we can't filter out any type of label in a wheel name # Example: we want to filter out "unoptimized" and "dsym" but in case we add more labels, we want to use regex # to handle those new labels without updating the regex. - run: echo $(jf rt search "${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/*${{ env.SHORT_GITHUB_SHA }}*${{ env.PYTHON_TAG }}*manylinux*x86_64*.whl" | jq '.[].path' | grep --invert-match "${{ env.SHORT_GITHUB_SHA }}\.") > file_matches.txt + run: jf rt search "${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/*${{ env.SHORT_GITHUB_SHA }}*${{ env.PYTHON_TAG }}*manylinux*x86_64*.whl" | jq -r '.[].path' > ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} shell: bash + - name: Filter out wheels with labels in results + run: sed -i "s/${{ env.SHORT_GITHUB_SHA }}\./d" ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + - name: Count artifacts id: count_num_artifacts_found - run: echo num_artifacts=$(echo file_matches.txt | wc -l) >> $GITHUB_OUTPUT + run: echo num_artifacts=$(echo ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | wc -l) >> $GITHUB_OUTPUT - name: Multiple artifacts found, not sure which one to use. Fail out if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts > 1 }} @@ -59,7 +64,7 @@ jobs: - name: Found the exact artifact in JFrog, downloading... if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} - run: jf rt download --flat --fail-no-op "$(cat file_matches.txt)" + run: jf rt download --flat --fail-no-op < file_matches.txt - name: Pass to valgrind job if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} From 4e240364c36ed3aecb5d4138a7678fde2946fd68 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:21:54 -0800 Subject: [PATCH 24/58] not an s command --- .github/workflows/valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index da9fadad0..ada6a59f4 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -52,7 +52,7 @@ jobs: shell: bash - name: Filter out wheels with labels in results - run: sed -i "s/${{ env.SHORT_GITHUB_SHA }}\./d" ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + run: sed -i "/${{ env.SHORT_GITHUB_SHA }}\./d" ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - name: Count artifacts id: count_num_artifacts_found From ebeb5645aec99251f676ad768b1b5e43417a7d26 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:26:00 -0800 Subject: [PATCH 25/58] fix --- .github/workflows/valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index ada6a59f4..33b95368f 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -64,7 +64,7 @@ jobs: - name: Found the exact artifact in JFrog, downloading... if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} - run: jf rt download --flat --fail-no-op < file_matches.txt + run: jf rt download --flat --fail-no-op < ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - name: Pass to valgrind job if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} From def538ad29f6a1273d03623f8910c60c36a88e89 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:35:07 -0800 Subject: [PATCH 26/58] fix --- .github/workflows/valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 33b95368f..a77c77aad 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -64,7 +64,7 @@ jobs: - name: Found the exact artifact in JFrog, downloading... if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} - run: jf rt download --flat --fail-no-op < ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + run: jf rt download --flat --fail-no-op <${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - name: Pass to valgrind job if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} From 6cf163373370642a1db2ced684d4f58fc9f0c02b Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:41:04 -0800 Subject: [PATCH 27/58] debug --- .github/workflows/valgrind.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index a77c77aad..a901ac8de 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -51,9 +51,13 @@ jobs: run: jf rt search "${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/*${{ env.SHORT_GITHUB_SHA }}*${{ env.PYTHON_TAG }}*manylinux*x86_64*.whl" | jq -r '.[].path' > ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} shell: bash + - run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + - name: Filter out wheels with labels in results run: sed -i "/${{ env.SHORT_GITHUB_SHA }}\./d" ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + - run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + - name: Count artifacts id: count_num_artifacts_found run: echo num_artifacts=$(echo ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | wc -l) >> $GITHUB_OUTPUT From fa1e76ea230a3895100c2724f4d6a4ba68a5ef78 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:05:31 -0800 Subject: [PATCH 28/58] Fix --- .github/workflows/valgrind.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index a901ac8de..da9adf649 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -48,19 +48,18 @@ jobs: # but it doesn't allow regex, so we can't filter out any type of label in a wheel name # Example: we want to filter out "unoptimized" and "dsym" but in case we add more labels, we want to use regex # to handle those new labels without updating the regex. - run: jf rt search "${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/*${{ env.SHORT_GITHUB_SHA }}*${{ env.PYTHON_TAG }}*manylinux*x86_64*.whl" | jq -r '.[].path' > ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - shell: bash + run: jf rt search "${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/*${{ env.SHORT_GITHUB_SHA }}*${{ env.PYTHON_TAG }}*manylinux*x86_64*.whl" > ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - name: Filter out wheels with labels in results - run: sed -i "/${{ env.SHORT_GITHUB_SHA }}\./d" ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + run: jq '.[] | select(.path | test("${{ env.SHORT_GITHUB_SHA }}") | not)' ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - name: Count artifacts id: count_num_artifacts_found - run: echo num_artifacts=$(echo ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | wc -l) >> $GITHUB_OUTPUT + run: echo num_artifacts=$(jq length ${{ env.JF_SEARCH_RESULTS_FILE_NAME }}) >> $GITHUB_OUTPUT - name: Multiple artifacts found, not sure which one to use. Fail out if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts > 1 }} @@ -68,7 +67,7 @@ jobs: - name: Found the exact artifact in JFrog, downloading... if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} - run: jf rt download --flat --fail-no-op <${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + run: jf rt download --flat --fail-no-op < ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - name: Pass to valgrind job if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} From 1bd4bbf91ef0b791b699b47df303d73d8df99c93 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:11:10 -0800 Subject: [PATCH 29/58] just do this.. --- .github/workflows/valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index da9adf649..15eefcb6e 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -67,7 +67,7 @@ jobs: - name: Found the exact artifact in JFrog, downloading... if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} - run: jf rt download --flat --fail-no-op < ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + run: jf rt download --flat --fail-no-op $(cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }}) - name: Pass to valgrind job if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} From 0c71f0a4d77b0daab663bd52b336bafab4eb3a7f Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:16:33 -0800 Subject: [PATCH 30/58] Fix --- .github/workflows/valgrind.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 15eefcb6e..0955ec5e3 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -65,9 +65,11 @@ jobs: if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts > 1 }} run: exit 1 - - name: Found the exact artifact in JFrog, downloading... + - name: Found the exact artifact in JFrog. Get the name if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} - run: jf rt download --flat --fail-no-op $(cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }}) + run: echo ARTIFACT_PATH=$(jq -r .[0].path ${{ env.JF_SEARCH_RESULTS_FILE_NAME }}) >> $GITHUB_ENV + + - run: jf rt download --flat --fail-no-op ${{ env.ARTIFACT_PATH }} - name: Pass to valgrind job if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} From 7f4c2107e86f5b6032856031974c5e218f3ea4bf Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:17:43 -0800 Subject: [PATCH 31/58] Fix --- .github/workflows/valgrind.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 0955ec5e3..5a862562e 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -65,11 +65,13 @@ jobs: if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts > 1 }} run: exit 1 - - name: Found the exact artifact in JFrog. Get the name + - name: Found the exact artifact in JFrog. Get the artifact name if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} run: echo ARTIFACT_PATH=$(jq -r .[0].path ${{ env.JF_SEARCH_RESULTS_FILE_NAME }}) >> $GITHUB_ENV - - run: jf rt download --flat --fail-no-op ${{ env.ARTIFACT_PATH }} + - name: Then download artifact from JFrog + if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} + run: jf rt download --flat --fail-no-op ${{ env.ARTIFACT_PATH }} - name: Pass to valgrind job if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} From 6f3b74e25a7cb06bc88521066eef4bf306578a9d Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:32:01 -0800 Subject: [PATCH 32/58] this works --- .github/workflows/valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 5a862562e..32c8487f3 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -53,7 +53,7 @@ jobs: - run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - name: Filter out wheels with labels in results - run: jq '.[] | select(.path | test("${{ env.SHORT_GITHUB_SHA }}") | not)' ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + run: jq '.[] | select(.path | test("${{ env.SHORT_GITHUB_SHA }}\\.") | not)' ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} From ccecb52740357962627a67d5c4fb5df44883f9a3 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:37:05 -0800 Subject: [PATCH 33/58] this is better --- .github/workflows/valgrind.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 32c8487f3..25f105930 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -104,8 +104,11 @@ jobs: valgrind: env: MASSIF_REPORT_FILE_NAME: massif.out - needs: build-manylinux-wheel - if: ${{ !cancelled() && (needs.build-manylinux-wheel.result == 'success' || needs.build-manylinux-wheel.result == 'skipped') }} + needs: [ + look-for-wheel-in-jfrog, + build-manylinux-wheel + ] + if: ${{ !cancelled() && (needs.look-for-wheel-in-jfrog.result == 'success' || needs.build-manylinux-wheel.result == 'success') }} runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 From efcdb6c6f98211036bf845295501614c5a31d945 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:48:05 -0800 Subject: [PATCH 34/58] jq doesnt edit file in place --- .github/workflows/valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 25f105930..fcd930d94 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -53,7 +53,7 @@ jobs: - run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - name: Filter out wheels with labels in results - run: jq '.[] | select(.path | test("${{ env.SHORT_GITHUB_SHA }}\\.") | not)' ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + run: jq '.[] | select(.path | test("${{ env.SHORT_GITHUB_SHA }}\\.") | not)' ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | sponge ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} From a2bbbe2335c04a94bc9784e6e36154a9e55e16dd Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:49:00 -0800 Subject: [PATCH 35/58] install --- .github/workflows/valgrind.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index fcd930d94..ef21a911f 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -52,6 +52,8 @@ jobs: - run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + - run: sudo apt install -y moreutils + - name: Filter out wheels with labels in results run: jq '.[] | select(.path | test("${{ env.SHORT_GITHUB_SHA }}\\.") | not)' ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | sponge ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} From 257cdec2b048c23013c97049cf78f991455db2d2 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:55:11 -0800 Subject: [PATCH 36/58] keep the array the same --- .github/workflows/valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index ef21a911f..66dbd990a 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -55,7 +55,7 @@ jobs: - run: sudo apt install -y moreutils - name: Filter out wheels with labels in results - run: jq '.[] | select(.path | test("${{ env.SHORT_GITHUB_SHA }}\\.") | not)' ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | sponge ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + run: jq 'map(select(.path | test("${{ env.SHORT_GITHUB_SHA }}\\.") | not))' ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | sponge ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} From 61d5f7175adf50d29c17cbaac9e807c18ebfa6a6 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Nov 2024 14:49:05 -0800 Subject: [PATCH 37/58] coment --- .github/workflows/valgrind.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 66dbd990a..ad13da738 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -57,7 +57,8 @@ jobs: - name: Filter out wheels with labels in results run: jq 'map(select(.path | test("${{ env.SHORT_GITHUB_SHA }}\\.") | not))' ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | sponge ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - - run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + - name: Check if artifacts with labels were filtered out + run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - name: Count artifacts id: count_num_artifacts_found From 06343ecf8b4baa2db267ddf2ce7a51fc73b7caf8 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 7 Nov 2024 08:57:37 -0800 Subject: [PATCH 38/58] Be more specific --- .github/workflows/valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index ad13da738..89e3993ec 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -111,7 +111,7 @@ jobs: look-for-wheel-in-jfrog, build-manylinux-wheel ] - if: ${{ !cancelled() && (needs.look-for-wheel-in-jfrog.result == 'success' || needs.build-manylinux-wheel.result == 'success') }} + if: ${{ !cancelled() && (needs.look-for-wheel-in-jfrog.result == 'success' || (needs.look-for-wheel-in-jfrog.result == 'skipped' && needs.build-manylinux-wheel.result == 'success')) }} runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 From e082960cc34af3f45cb08615824ac22d5011128f Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 7 Nov 2024 09:05:42 -0800 Subject: [PATCH 39/58] make sure cmds aren't failing early on --- .github/workflows/valgrind.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 89e3993ec..33d561334 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -56,6 +56,7 @@ jobs: - name: Filter out wheels with labels in results run: jq 'map(select(.path | test("${{ env.SHORT_GITHUB_SHA }}\\.") | not))' ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | sponge ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + shell: bash - name: Check if artifacts with labels were filtered out run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} From d9749950faeedb5b11ce19db512633ef18f30e62 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Thu, 7 Nov 2024 09:12:06 -0800 Subject: [PATCH 40/58] fix --- .github/workflows/valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 33d561334..a26de55bf 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -24,7 +24,7 @@ env: jobs: # TODO: handle case where we run valgrind on a "central" branch look-for-wheel-in-jfrog: - if: ${{ github.ref_name != 'dev' && github.ref_name != 'stage' && github.ref_name != 'master' }} + if: ${{ !startsWith(github.ref_name, 'dev') && !startsWith(github.ref_name, 'stage') && !startsWith(github.ref_name, 'master') }} outputs: num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} # We want this environment variable to be available when calling a reusable workflow From e1184a02dbac4ba9cafcd1f9288e97cc66997fdd Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 18 Nov 2024 09:54:23 -0800 Subject: [PATCH 41/58] clear up --- .github/workflows/valgrind.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index b87fd6639..f64e9f774 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -55,7 +55,8 @@ jobs: - run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - - run: sudo apt install -y moreutils + - name: Install sponge + run: sudo apt install -y moreutils - name: Filter out wheels with labels in results run: jq 'map(select(.path | test("${{ env.SHORT_GITHUB_SHA }}\\.") | not))' ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | sponge ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} From e85c717ca89ef0152e748ff9dd226ddf3552fb56 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 18 Nov 2024 09:54:49 -0800 Subject: [PATCH 42/58] clear up --- .github/workflows/valgrind.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index f64e9f774..27ca26b65 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -53,7 +53,8 @@ jobs: # to handle those new labels without updating the regex. run: jf rt search "${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/*${{ env.SHORT_GITHUB_SHA }}*${{ env.PYTHON_TAG }}*manylinux*x86_64*.whl" > ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - - run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} + - name: Show unfiltered results + run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} - name: Install sponge run: sudo apt install -y moreutils From 106c607adb64a7066f9f01049b559ed93992dbd6 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 18 Nov 2024 10:10:58 -0800 Subject: [PATCH 43/58] just hardcode --- .github/workflows/valgrind.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 27ca26b65..dff3bf15a 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -30,9 +30,6 @@ jobs: if: ${{ !startsWith(github.ref_name, 'dev') && !startsWith(github.ref_name, 'stage') && !startsWith(github.ref_name, 'master') }} outputs: num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} - # We want this environment variable to be available when calling a reusable workflow - # so we convert it to a job output so the reusable workflow can access it - python-tag: ${{ env.PYTHON_TAG }} runs-on: ubuntu-22.04 env: JF_SEARCH_RESULTS_FILE_NAME: wheel_commit_matches.txt @@ -98,7 +95,9 @@ jobs: if: ${{ !cancelled() && (needs.look-for-wheel-in-jfrog.result == 'skipped' || (needs.look-for-wheel-in-jfrog.result == 'success' && needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0)) }} uses: ./.github/workflows/build-wheels.yml with: - python-tags: '["${{ needs.look-for-wheel-in-jfrog.outputs.python-tag }}"]' + # TODO: need to update this manually each time we change the Python version + # Get this dynamically, instead + python-tags: '["cp38"]' platform-tag: manylinux_x86_64 sha-to-build-and-test: ${{ github.sha }} secrets: inherit From 11c05b3d5882b27a0b98fd846664e9236f8e0723 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 18 Nov 2024 10:17:30 -0800 Subject: [PATCH 44/58] Fix --- .github/workflows/valgrind.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index dff3bf15a..42ffae1f1 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -87,6 +87,8 @@ jobs: if-no-files-found: error path: './*.whl' + # TODO: combine these two below jobs into one using a reusable workflow + build-manylinux-wheel: needs: look-for-wheel-in-jfrog # If previous job was skipped, we are running on a central branch @@ -104,6 +106,8 @@ jobs: upload-built-wheel-to-jfrog: needs: build-manylinux-wheel + # By default, this job will be skipped if look-for-wheel-in-jfrog (ancestor job) was skipped + if: ${{ !cancelled() && needs.build-manylinux-wheel.result == 'success' }} uses: ./.github/workflows/upload-to-jfrog.yml with: jfrog-repo-name: ${{ vars.JFROG_GENERIC_REPO_NAME }} From 81d3e8db3f90c618bed2fcb50f997266ab5e4a60 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 18 Nov 2024 10:40:37 -0800 Subject: [PATCH 45/58] see if this works --- .github/workflows/valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 42ffae1f1..84532ba53 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -27,7 +27,7 @@ env: jobs: # TODO: handle case where we run valgrind on a "central" branch look-for-wheel-in-jfrog: - if: ${{ !startsWith(github.ref_name, 'dev') && !startsWith(github.ref_name, 'stage') && !startsWith(github.ref_name, 'master') }} + if: ${{ startsWith(github.ref_name, 'dev') == false && startsWith(github.ref_name, 'stage') == false && startsWith(github.ref_name, 'master') == false }} outputs: num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} runs-on: ubuntu-22.04 From 0cd2ffecac4fcbeee2f948470637d03d1df1cd7a Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 18 Nov 2024 10:43:47 -0800 Subject: [PATCH 46/58] try --- .github/workflows/valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 84532ba53..6d68f7a9d 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -27,7 +27,7 @@ env: jobs: # TODO: handle case where we run valgrind on a "central" branch look-for-wheel-in-jfrog: - if: ${{ startsWith(github.ref_name, 'dev') == false && startsWith(github.ref_name, 'stage') == false && startsWith(github.ref_name, 'master') == false }} + if: ${{ !cancelled() && !startsWith(github.ref_name, 'dev') && !startsWith(github.ref_name, 'stage') && !startsWith(github.ref_name, 'master') }} outputs: num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} runs-on: ubuntu-22.04 From 92be52fc868f081040bf8b2395aba8da9fa04aea Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 18 Nov 2024 10:49:06 -0800 Subject: [PATCH 47/58] make logic better --- .github/workflows/valgrind.yml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 6d68f7a9d..91e43ddd3 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -25,9 +25,7 @@ env: PYTHON_TAG: cp38 jobs: - # TODO: handle case where we run valgrind on a "central" branch look-for-wheel-in-jfrog: - if: ${{ !cancelled() && !startsWith(github.ref_name, 'dev') && !startsWith(github.ref_name, 'stage') && !startsWith(github.ref_name, 'master') }} outputs: num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} runs-on: ubuntu-22.04 @@ -87,14 +85,9 @@ jobs: if-no-files-found: error path: './*.whl' - # TODO: combine these two below jobs into one using a reusable workflow - build-manylinux-wheel: needs: look-for-wheel-in-jfrog - # If previous job was skipped, we are running on a central branch - # Always build wheels from a central branch for now - # Or if we are running on a feature branch and could not find any wheels in JFrog, build and upload it to JFrog - if: ${{ !cancelled() && (needs.look-for-wheel-in-jfrog.result == 'skipped' || (needs.look-for-wheel-in-jfrog.result == 'success' && needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0)) }} + if: ${{ needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 }} uses: ./.github/workflows/build-wheels.yml with: # TODO: need to update this manually each time we change the Python version @@ -106,8 +99,6 @@ jobs: upload-built-wheel-to-jfrog: needs: build-manylinux-wheel - # By default, this job will be skipped if look-for-wheel-in-jfrog (ancestor job) was skipped - if: ${{ !cancelled() && needs.build-manylinux-wheel.result == 'success' }} uses: ./.github/workflows/upload-to-jfrog.yml with: jfrog-repo-name: ${{ vars.JFROG_GENERIC_REPO_NAME }} @@ -120,7 +111,9 @@ jobs: look-for-wheel-in-jfrog, build-manylinux-wheel ] - if: ${{ !cancelled() && (needs.look-for-wheel-in-jfrog.result == 'success' || (needs.look-for-wheel-in-jfrog.result == 'skipped' && needs.build-manylinux-wheel.result == 'success')) }} + # Case 1: Found artifact in JFrog + # Case 2: Did not find artifact in JFrog, had to build it in GHA + if: ${{ !cancelled() && (needs.look-for-wheel-in-jfrog.result == 'success' && (needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 1) || (needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 && needs.build-manylinux-wheel.result == 'success')) }} runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 From 8fa19e544debb9b71b41d80bf4e4f6bff75b3000 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:06:22 -0800 Subject: [PATCH 48/58] put back --- .github/workflows/valgrind.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 91e43ddd3..275055640 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -28,6 +28,7 @@ jobs: look-for-wheel-in-jfrog: outputs: num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} + python-tag: ${{ env.PYTHON_TAG }} runs-on: ubuntu-22.04 env: JF_SEARCH_RESULTS_FILE_NAME: wheel_commit_matches.txt @@ -92,7 +93,7 @@ jobs: with: # TODO: need to update this manually each time we change the Python version # Get this dynamically, instead - python-tags: '["cp38"]' + python-tags: '["${{ needs.look-for-wheel-in-jfrog.outputs.python-tag }}"]' platform-tag: manylinux_x86_64 sha-to-build-and-test: ${{ github.sha }} secrets: inherit From fd8b2e628672d4c4acc9636ff0605314e7157d2b Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:07:29 -0800 Subject: [PATCH 49/58] fix --- .github/workflows/valgrind.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 275055640..f94d62300 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -28,6 +28,7 @@ jobs: look-for-wheel-in-jfrog: outputs: num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} + # So we can pass the python tag to a reusable workflow python-tag: ${{ env.PYTHON_TAG }} runs-on: ubuntu-22.04 env: @@ -91,8 +92,6 @@ jobs: if: ${{ needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 }} uses: ./.github/workflows/build-wheels.yml with: - # TODO: need to update this manually each time we change the Python version - # Get this dynamically, instead python-tags: '["${{ needs.look-for-wheel-in-jfrog.outputs.python-tag }}"]' platform-tag: manylinux_x86_64 sha-to-build-and-test: ${{ github.sha }} From a881a158faf32b07daa965433ce35f8978127b11 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:22:05 -0800 Subject: [PATCH 50/58] Don't upload to JFrog for central branches --- .github/workflows/valgrind.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index f94d62300..3c94d8b21 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -99,6 +99,7 @@ jobs: upload-built-wheel-to-jfrog: needs: build-manylinux-wheel + if: ${{ !startsWith(github.ref_name, 'dev') && !startsWith(github.ref_name, 'stage') && !startsWith(github.ref_name, 'master') }} uses: ./.github/workflows/upload-to-jfrog.yml with: jfrog-repo-name: ${{ vars.JFROG_GENERIC_REPO_NAME }} From a1d2ca467f0d201a4aa3f855b222a4619cf13501 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:42:05 -0800 Subject: [PATCH 51/58] space? --- .github/workflows/valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 3c94d8b21..5514edc12 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -99,7 +99,7 @@ jobs: upload-built-wheel-to-jfrog: needs: build-manylinux-wheel - if: ${{ !startsWith(github.ref_name, 'dev') && !startsWith(github.ref_name, 'stage') && !startsWith(github.ref_name, 'master') }} + if: ${{ ! startsWith(github.ref_name, 'dev') && ! startsWith(github.ref_name, 'stage') && ! startsWith(github.ref_name, 'master') }} uses: ./.github/workflows/upload-to-jfrog.yml with: jfrog-repo-name: ${{ vars.JFROG_GENERIC_REPO_NAME }} From 33ef486c9058f79510a58012fa5ea595531d9271 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:52:02 -0800 Subject: [PATCH 52/58] ref_name only works for push/prs --- .github/workflows/valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 5514edc12..7f58acce3 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -99,7 +99,7 @@ jobs: upload-built-wheel-to-jfrog: needs: build-manylinux-wheel - if: ${{ ! startsWith(github.ref_name, 'dev') && ! startsWith(github.ref_name, 'stage') && ! startsWith(github.ref_name, 'master') }} + if: ${{ !startsWith(github.event.ref, 'dev') && !startsWith(github.event.ref, 'stage') && !startsWith(github.event.ref, 'master') }} uses: ./.github/workflows/upload-to-jfrog.yml with: jfrog-repo-name: ${{ vars.JFROG_GENERIC_REPO_NAME }} From 962e5b73e38558edab4940c502150a4ca0da8901 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 18 Nov 2024 16:36:12 -0800 Subject: [PATCH 53/58] fully formed ref --- .github/workflows/valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 7f58acce3..070cd4a83 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -99,7 +99,7 @@ jobs: upload-built-wheel-to-jfrog: needs: build-manylinux-wheel - if: ${{ !startsWith(github.event.ref, 'dev') && !startsWith(github.event.ref, 'stage') && !startsWith(github.event.ref, 'master') }} + if: ${{ !contains(github.event.ref, 'dev') && !contains(github.event.ref, 'stage') && !contains(github.event.ref, 'master') }} uses: ./.github/workflows/upload-to-jfrog.yml with: jfrog-repo-name: ${{ vars.JFROG_GENERIC_REPO_NAME }} From f5065b62a26e309f275dd0894c34e997f9eb9779 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 18 Nov 2024 16:46:21 -0800 Subject: [PATCH 54/58] Add explanation --- .github/workflows/valgrind.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 070cd4a83..500579dd6 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -99,7 +99,9 @@ jobs: upload-built-wheel-to-jfrog: needs: build-manylinux-wheel - if: ${{ !contains(github.event.ref, 'dev') && !contains(github.event.ref, 'stage') && !contains(github.event.ref, 'master') }} + # github.ref_name does not work for workflow_dispatch. Have to use this instead + # Cannot use format() and env var together for prefixing each branch name + if: ${{ !contains(github.ref, '/refs/heads/dev') && !contains(github.ref, '/refs/heads/stage') && !contains(github.ref, '/refs/heads/master') }} uses: ./.github/workflows/upload-to-jfrog.yml with: jfrog-repo-name: ${{ vars.JFROG_GENERIC_REPO_NAME }} From 0e645a136d2eff56112fe443fcf0c550494d89fe Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 20 Nov 2024 08:31:11 -0800 Subject: [PATCH 55/58] better --- .github/workflows/valgrind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 500579dd6..1569a82ea 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -101,7 +101,7 @@ jobs: needs: build-manylinux-wheel # github.ref_name does not work for workflow_dispatch. Have to use this instead # Cannot use format() and env var together for prefixing each branch name - if: ${{ !contains(github.ref, '/refs/heads/dev') && !contains(github.ref, '/refs/heads/stage') && !contains(github.ref, '/refs/heads/master') }} + if: ${{ !startsWith(github.ref, 'refs/heads/dev') && !startsWith(github.ref, 'refs/heads/stage') && !startsWith(github.ref, 'refs/heads/master') }} uses: ./.github/workflows/upload-to-jfrog.yml with: jfrog-repo-name: ${{ vars.JFROG_GENERIC_REPO_NAME }} From dbab95aaaac48d5c415e59c06b13023b0c7e9ec3 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:00:26 -0800 Subject: [PATCH 56/58] add debug --- .github/workflows/valgrind.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 1569a82ea..49a4f2a96 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -87,6 +87,8 @@ jobs: if-no-files-found: error path: './*.whl' + - run: echo ${{ github.ref }} + build-manylinux-wheel: needs: look-for-wheel-in-jfrog if: ${{ needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 }} From fb094ae13a92e8678ede18c458e15cfc9efd7b53 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:16:16 -0800 Subject: [PATCH 57/58] Add note --- .github/workflows/valgrind.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 49a4f2a96..4ceb5e173 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -101,9 +101,10 @@ jobs: upload-built-wheel-to-jfrog: needs: build-manylinux-wheel - # github.ref_name does not work for workflow_dispatch. Have to use this instead - # Cannot use format() and env var together for prefixing each branch name - if: ${{ !startsWith(github.ref, 'refs/heads/dev') && !startsWith(github.ref, 'refs/heads/stage') && !startsWith(github.ref, 'refs/heads/master') }} + # TODO: this job should skip when this workflow is run on central branches + # We already have artifacts available for central branches in the PyPI-type JFrog repo + # The problem is we have to conditionally skip this job, but using the github context to get the branch name + # doesn't work for some reason. Just leave this alone for now. uses: ./.github/workflows/upload-to-jfrog.yml with: jfrog-repo-name: ${{ vars.JFROG_GENERIC_REPO_NAME }} From c2f438c70ae681ff7ecc7db404ae7779f40b51be Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:18:14 -0800 Subject: [PATCH 58/58] rm debug --- .github/workflows/valgrind.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 4ceb5e173..f01175f2e 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -87,8 +87,6 @@ jobs: if-no-files-found: error path: './*.whl' - - run: echo ${{ github.ref }} - build-manylinux-wheel: needs: look-for-wheel-in-jfrog if: ${{ needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 }}