From 420684460400e96182cac0ab0b6c1b41c8d6e480 Mon Sep 17 00:00:00 2001 From: Robert Fratto Date: Sat, 27 Apr 2024 15:28:32 -0400 Subject: [PATCH] split into reusable --- .github/workflows/fuzz-find-go.yml | 61 ++++++++++ .github/workflows/fuzz-go-pr.yml | 20 ++++ .github/workflows/fuzz-go-scheduled.yml | 70 ++++++++++++ .github/workflows/fuzz-run-go.yml | 114 +++++++++++++++++++ .github/workflows/fuzz.yml | 143 ------------------------ 5 files changed, 265 insertions(+), 143 deletions(-) create mode 100644 .github/workflows/fuzz-find-go.yml create mode 100644 .github/workflows/fuzz-go-pr.yml create mode 100644 .github/workflows/fuzz-go-scheduled.yml create mode 100644 .github/workflows/fuzz-run-go.yml delete mode 100644 .github/workflows/fuzz.yml diff --git a/.github/workflows/fuzz-find-go.yml b/.github/workflows/fuzz-find-go.yml new file mode 100644 index 0000000000..86ea128069 --- /dev/null +++ b/.github/workflows/fuzz-find-go.yml @@ -0,0 +1,61 @@ +# Find Go fuzz tests is a reusable workflow that will find all Go fuzz tests in +# a given directory. +# +# This workflow assumes that the code has been checked out. +# +# The output is a JSON array of objects with two keys: +# * package: The package path of the test, relative to the search directory. +# * function: The name of the fuzz function within the package. + +name: Find Go fuzz tests + +on: + workflow_call: + inputs: + directory: + description: "Directory to search for Go fuzz tests in." + default: '.' + required: false + type: string + outputs: + tests: + description: A JSON array of objects of tests, containing the keys "package" and "function". + value: "${{ jobs.find-tests.outputs.tests }}" + +jobs: + find-tests: + runs-on: ubuntu-latest + outputs: + tests: ${{ steps.find-tests.outputs.tests }} + steps: + - uses: actions/checkout@v4 + - name: Find fuzz tests + id: find-tests + run: | + TEST_FILES=$(find "${{ inputs.directory }}" -name '*_test.go' -not -path './vendor/*') + + RESULTS=() + + for FILE in $TEST_FILES; do + FUZZ_FUNC=$(grep -E 'func Fuzz\w*' $FILE | sed 's/func //' | sed 's/(.*$//') + if [ -z "$FUZZ_FUNC" ]; then + continue + fi + + PACKAGE_PATH=$(dirname ${FILE#${{ inputs.directory }}/}) + RESULTS+=("{\"package\":\"$PACKAGE_PATH\",\"function\":\"$FUZZ_FUNC\"}") + + echo "Found $FUZZ_FUNC in $PACKAGE_PATH" + done + + NUM_RESULTS=${#RESULTS[@]} + INCLUDE_STRING="" + for (( i=0; i<$NUM_RESULTS; i++ )); do + INCLUDE_STRING+="${RESULTS[$i]}" + + if [[ $i -lt $(($NUM_RESULTS-1)) ]]; then + INCLUDE_STRING+="," + fi + done + + echo 'tests=['$INCLUDE_STRING']' >> $GITHUB_OUTPUT diff --git a/.github/workflows/fuzz-go-pr.yml b/.github/workflows/fuzz-go-pr.yml new file mode 100644 index 0000000000..200ab6c2f9 --- /dev/null +++ b/.github/workflows/fuzz-go-pr.yml @@ -0,0 +1,20 @@ +name: Run Go fuzz tests (PR) +on: + pull_request: +jobs: + find-tests: + name: Find fuzz tests + uses: ./.github/workflows/fuzz-find-go.yml + + fuzz: + name: "${{ matrix.package }}: ${{ matrix.function }}" + needs: [find-tests] + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.find-tests.outputs.tests) }} + uses: ./.github/workflows/fuzz-run-go.yml + with: + package-path: ${{ matrix.package }} + test-name: ${{ matrix.function }} + fuzz-time: 5s diff --git a/.github/workflows/fuzz-go-scheduled.yml b/.github/workflows/fuzz-go-scheduled.yml new file mode 100644 index 0000000000..ab17bfe9a1 --- /dev/null +++ b/.github/workflows/fuzz-go-scheduled.yml @@ -0,0 +1,70 @@ +name: Run Go fuzz tests (scheduled) +on: + workflow_dispatch: {} +jobs: + find-tests: + name: Find fuzz tests + uses: ./.github/workflows/fuzz-find-go.yml + + fuzz: + name: "${{ matrix.package }}: ${{ matrix.function }}" + runs-on: ubuntu-latest + needs: [find-tests] + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.find-tests.outputs.tests) }} + steps: + - uses: actions/checkout@v4 + + - name: Set up Go 1.22 + uses: actions/setup-go@v5 + with: + go-version: "1.22" + cache: false + + - name: Run fuzz test + id: fuzz + uses: ./.github/workflows/fuzz-run-go.yml + with: + package-path: ${{ matrix.package }} + test-name: ${{ matrix.function }} + fuzz-time: 5s + + - name: Create new issue + if: ${{ failure() && steps.fuzz.outputs.failure-name != '' }} + uses: actions/github-script@v7 + with: + script: | + const failureName = "${{ steps.fuzz.outputs.failure-name }}"; + const issueTitle = `${{ matrix.package }}: ${{ matrix.function }} failed (${failureName})`; + + // Look for existing issue first with the same title. + const issues = await github.rest.search.issuesAndPullRequests({ + q: `is:issue is:open repo:${context.repo.owner}/${context.repo.repo} in:title "${failureName}"` + }) + console.log(issues); + const issue = issues.data.items.find((issue) => issue.title === issue.title); + if (issue) { + return; + } + + // Create a new issue. + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: issueTitle, + body: ` + A new fuzz test failure was found in ${{ matrix.package }}. + + To reproduce the failure locally, run the following command using the GitHub CLI to download the corpus entry: + + 
gh run download --repo ${{ github.repository }} ${{ github.run_id }} -n ${{ steps.fuzz.outputs.artifact-name }} --dir ${{ steps.fuzz.outputs.failure-path }}
+ + When opening a PR with the fix, please include in the corpus entry in your commit to prevent regressions. + + [Link to failed run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) + `, + + labels: ['bug'], + }) diff --git a/.github/workflows/fuzz-run-go.yml b/.github/workflows/fuzz-run-go.yml new file mode 100644 index 0000000000..1067f94a4d --- /dev/null +++ b/.github/workflows/fuzz-run-go.yml @@ -0,0 +1,114 @@ +# Run Go fuzz test is a reusable workflow that runs a single Go fuzz test. +# +# The workflow runs until fuzz-time has elapsed or a test case which causes the +# test to fail is found. If a new failure is found, the workflow will fail and +# upload the failing test case as an artifact. +# +# The workflow assumes that the code has already been checked out. + +name: Run Go fuzz test +on: + workflow_call: + inputs: + package-path: + description: "Directory of the Go package to run a Fuzz test in." + required: true + type: string + test-name: + description: "Full name of the Fuzz test to run." + required: true + type: string + fuzz-time: + description: "Time to run the Fuzz test for. (for example, 5m)" + required: true + type: string + + outputs: + arifact-name: + description: "Name of the artifact that was uploaded. Only use when the workflow fails." + value: ${{ jobs.fuzz.outputs.artifact-name }} + failure-name: + description: "Name of the test case that failed. Only use when the workflow fails." + value: ${{ jobs.fuzz.outputs.failure-name }} + failure-path: + description: "Path to place the failure artifact. Only use when the workflow fails." + value: ${{ inputs.package-path }}/testdata/fuzz/${{ inputs.test-name }} + +jobs: + fuzz: + runs-on: ubuntu-latest + outputs: + artifact-name: failure-${{ steps.new-failure.outputs.package }}-${{ steps.new-failure.outputs.function }} + failure-name: ${{ steps.new-failure.outputs.name }} + steps: + - uses: actions/checkout@v4 + + - name: Set up Go 1.22 + uses: actions/setup-go@v5 + with: + go-version: "1.22" + cache: false + + - name: Find cache location + run: + echo "FUZZ_CACHE=$(go env GOCACHE)/fuzz" >> $GITHUB_ENV + + - name: Restore fuzz cache + uses: actions/cache@v4 + with: + path: ${{ env.FUZZ_CACHE }} + key: fuzz-${{ inputs.package-path }}-${{ inputs.test-name }}-${{ github.sha }} + restore-keys: | + fuzz-${{ inputs.package-path }}-${{ inputs.test-name }}- + + - name: Fuzz + run: | + # Change directory to the package first, since go test doesn't + # support cross-module testing, and the provided directory may be in + # a different module. + cd "${{ inputs.package-path }}" + go test -fuzz="${{ inputs.test-name }}\$" -run="${{ inputs.test-name }}\$" -fuzztime="${{ inputs.fuzz-time }}" . + + # Fuzzing may have failed because of an existing bug, or it may have + # found a new one and written a new corpus entry in testdata/ relative to + # the package. + # + # If that file was written, we should save it as an artifact and then + # create an issue. + + - name: Check for new fuzz failure + id: new-failure + if: ${{ failure() }} + run: | + UNTRACKED=$(git ls-files . --exclude-standard --others) + if [ -z "$UNTRACKED" ]; then + exit 0 + fi + echo "Found new fuzz failure: $UNTRACKED" + echo "file=$UNTRACKED" >> $GITHUB_OUTPUT + echo "name=$(basename $UNTRACKED)" >> $GITHUB_OUTPUT + echo "package=$(echo ${{ inputs.package-path }} | sed 's/\//_/g')" >> $GITHUB_OUTPUT + echo "function=${{ inputs.test-name }}" >> $GITHUB_OUTPUT + + - name: Upload fuzz failure as artifact + id: artifact + if: ${{ failure() && steps.new-failure.outputs.file != '' }} + uses: actions/upload-artifact@v4 + with: + name: failure-${{ steps.new-failure.outputs.package }}-${{ steps.new-failure.outputs.function }} + path: ${{ steps.new-failure.outputs.file }} + + - name: Generate reproduction instructions + if: ${{ failure() && steps.new-failure.outputs.file != '' }} + run: | + cat >>$GITHUB_STEP_SUMMARY <gh run download --repo ${{ github.repository }} ${{ github.run_id }} -n failure-${{ steps.new-failure.outputs.package }}-${{ steps.new-failure.outputs.function }} --dir ${{ inputs.package-path }}/testdata/fuzz/${{ inputs.test-name }} + + When opening a PR with the fix, please include the test case file in your PR to prevent regressions. + EOF diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml deleted file mode 100644 index ff956a325a..0000000000 --- a/.github/workflows/fuzz.yml +++ /dev/null @@ -1,143 +0,0 @@ -name: Fuzz test -on: - pull_request: -jobs: - find-tests: - name: Find fuzz tests - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v4 - - name: Find fuzz tests - id: set-matrix - run: | - TEST_FILES=$(find . -name '*_test.go' -not -path './vendor/*') - - RESULTS=() - - for FILE in $TEST_FILES; do - FUZZ_FUNC=$(grep -E 'func Fuzz\w*' $FILE | sed 's/func //' | sed 's/(.*$//') - if [ -z "$FUZZ_FUNC" ]; then - continue - fi - - PACKAGE_PATH=$(dirname ${FILE#./}) - RESULTS+=("{\"package\":\"$PACKAGE_PATH\",\"function\":\"$FUZZ_FUNC\"}") - - echo "Found $PACKAGE_PATH :: $FUZZ_FUNC" - done - - NUM_RESULTS=${#RESULTS[@]} - INCLUDE_STRING="" - for (( i=0; i<$NUM_RESULTS; i++ )); do - INCLUDE_STRING+="${RESULTS[$i]}" - - if [[ $i -lt $(($NUM_RESULTS-1)) ]]; then - INCLUDE_STRING+="," - fi - done - - echo 'matrix={"include": ['$INCLUDE_STRING']}' >> $GITHUB_OUTPUT - - fuzz: - name: "${{ matrix.package }} :: ${{ matrix.function }}" - runs-on: ubuntu-latest - if: needs.find-tests.outputs.matrix != '' - needs: [find-tests] - strategy: - fail-fast: false # Allow other jobs in the matrix to run even if a single one fails. - matrix: ${{fromJson(needs.find-tests.outputs.matrix)}} - steps: - - uses: actions/checkout@v4 - - - name: Set up Go 1.22 - uses: actions/setup-go@v5 - with: - go-version: "1.22" - cache: false - - - name: Find cache location - run: - echo "FUZZ_CACHE=$(go env GOCACHE)/fuzz" >> $GITHUB_ENV - - - name: Restore corpus - uses: actions/cache@v4 - with: - path: ${{ env.FUZZ_CACHE }} - key: fuzz-${{ matrix.package }}-${{ matrix.function }}-${{ github.sha }} - restore-keys: | - fuzz-${{ matrix.package }}-${{ matrix.function }}- - save-always: true - - - name: Fuzz - run: | - cd "${{ matrix.package }}" - go test -fuzz="${{ matrix.function }}\$" -run="${{ matrix.function }}\$" -fuzztime=5s . - - # Fuzzing may have failed because of an existing bug, or it may have - # found a new one and written a new corpus entry in testdata/ relative to - # the package. - # - # If that file was written, we should save it as an artifact and then - # create an issue. - - - name: Check for new corpus entry - id: new-entry - if: ${{ failure() }} - run: | - UNTRACKED=$(git ls-files . --exclude-standard --others) - if [ -z "$UNTRACKED" ]; then - exit 0 - fi - echo "Found new corpus entry: $UNTRACKED" - echo "file=$UNTRACKED" >> $GITHUB_OUTPUT - echo "name=$(basename $UNTRACKED)" >> $GITHUB_OUTPUT - echo "package=$(echo ${{ matrix.package }} | sed 's/\//_/g')" >> $GITHUB_OUTPUT - echo "function=${{ matrix.function }}" >> $GITHUB_OUTPUT - - - name: Upload corpus entry - id: artifact - if: ${{ failure() && steps.new-entry.outputs.file != '' }} - uses: actions/upload-artifact@v4 - with: - name: failure-${{ steps.new-entry.outputs.package }}-${{ steps.new-entry.outputs.function }} - path: ${{ steps.new-entry.outputs.file }} - - - name: Create new issue - if: ${{ failure() && steps.new-entry.outputs.file != '' }} - uses: actions/github-script@v7 - with: - script: | - const failureName = "${{ steps.new-entry.outputs.name }}"; - const issueTitle = `${{ matrix.package }}: ${{ matrix.function }} failed (${failureName})`; - - // Look for existing issue first with the same title. - const issues = await github.rest.search.issuesAndPullRequests({ - q: `is:issue is:open repo:${context.repo.owner}/${context.repo.repo} in:title "${failureName}"` - }) - console.log(issues); - const issue = issues.data.items.find((issue) => issue.title === issue.title); - if (issue) { - return; - } - - // Create a new issue. - await github.rest.issues.create({ - owner: context.repo.owner, - repo: context.repo.repo, - title: issueTitle, - body: ` - A new fuzz test failure was found in ${{ matrix.package }}. - - To reproduce the failure locally, run the following command using the GitHub CLI to download the corpus entry: - - 
gh run download --repo ${{ github.repository }} ${{ github.run_id }} -n failure-${{ steps.new-entry.outputs.package }}-${{ steps.new-entry.outputs.function }} --dir ${{ matrix.package }}/testdata/fuzz/${{ matrix.function }}
- - When opening a PR with the fix, please include in the corpus entry in your commit to prevent regressions. - - [Link to failed run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) - `, - - labels: ['bug'], - })