Test automated fuzz testing #37
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Fuzz test | |
| on: | |
| pull_request: | |
| jobs: | |
| find-tests: | |
| name: Find fuzz tests | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Find fuzz tests | |
| id: set-matrix | |
| run: | | |
| TEST_FILES=$(find . -name '*_test.go' -not -path './vendor/*') | |
| RESULTS=() | |
| for FILE in $TEST_FILES; do | |
| FUZZ_FUNC=$(grep -E 'func Fuzz\w*' $FILE | sed 's/func //' | sed 's/(.*$//') | |
| if [ -z "$FUZZ_FUNC" ]; then | |
| continue | |
| fi | |
| PACKAGE_PATH=$(dirname ${FILE#./}) | |
| RESULTS+=("{\"package\":\"$PACKAGE_PATH\",\"function\":\"$FUZZ_FUNC\"}") | |
| echo "Found $PACKAGE_PATH :: $FUZZ_FUNC" | |
| done | |
| NUM_RESULTS=${#RESULTS[@]} | |
| INCLUDE_STRING="" | |
| for (( i=0; i<$NUM_RESULTS; i++ )); do | |
| INCLUDE_STRING+="${RESULTS[$i]}" | |
| if [[ $i -lt $(($NUM_RESULTS-1)) ]]; then | |
| INCLUDE_STRING+="," | |
| fi | |
| done | |
| echo 'matrix={"include": ['$INCLUDE_STRING']}' >> $GITHUB_OUTPUT | |
| fuzz: | |
| name: "${{ matrix.package }} :: ${{ matrix.function }}" | |
| runs-on: ubuntu-latest | |
| if: needs.find-tests.outputs.matrix != '' | |
| needs: [find-tests] | |
| strategy: | |
| fail-fast: false # Allow other jobs in the matrix to run even if a single one fails. | |
| matrix: ${{fromJson(needs.find-tests.outputs.matrix)}} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Go 1.22 | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.22" | |
| cache: false | |
| - name: Find cache location | |
| run: | |
| echo "FUZZ_CACHE=$(go env GOCACHE)/fuzz" >> $GITHUB_ENV | |
| - name: Restore corpus | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.FUZZ_CACHE }} | |
| key: fuzz-${{ matrix.package }}-${{ matrix.function }}-${{ github.sha }} | |
| restore-keys: | | |
| fuzz-${{ matrix.package }}-${{ matrix.function }}- | |
| save-always: true | |
| - name: Fuzz | |
| run: | | |
| cd "${{ matrix.package }}" | |
| go test -fuzz="${{ matrix.function }}\$" -run="${{ matrix.function }}\$" -fuzztime=5s . | |
| # Fuzzing may have failed because of an existing bug, or it may have | |
| # found a new one and written a new corpus entry in testdata/ relative to | |
| # the package. | |
| # | |
| # If that file was written, we should save it as an artifact and then | |
| # create an issue. | |
| - name: Check for new corpus entry | |
| id: new-entry | |
| if: ${{ failure() }} | |
| run: | | |
| UNTRACKED=$(git ls-files . --exclude-standard --others) | |
| if [ -z "$UNTRACKED" ]; then | |
| exit 0 | |
| fi | |
| echo "Found new corpus entry: $UNTRACKED" | |
| echo "file=$UNTRACKED" >> $GITHUB_OUTPUT | |
| echo "name=$(basename $UNTRACKED)" >> $GITHUB_OUTPUT | |
| echo "package=$(echo ${{ matrix.package }} | sed 's/\//_/g')" >> $GITHUB_OUTPUT | |
| echo "function=${{ matrix.function }}" >> $GITHUB_OUTPUT | |
| - name: Upload corpus entry | |
| id: artifact | |
| if: ${{ failure() && steps.new-entry.outputs.file != '' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: failure-${{ steps.new-entry.outputs.package }}-${{ steps.new-entry.outputs.function }} | |
| path: ${{ steps.new-entry.outputs.file }} | |
| - name: Create new issue | |
| if: ${{ failure() && steps.new-entry.outputs.file != '' && steps.artifact.output.artifact-url != '' }} | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const failureName = "${{ steps.new-entry.outputs.name }}"; | |
| const issueTitle = `${{ matrix.package }}: Fuzz test ${{ matrix.function }} failed (${failureName})`; | |
| // Look for existing issue first with the same title. | |
| const issues = github.search.issuesAndPullRequests({ | |
| q: `is:issue is:open repo:${context.repo.owner}/${context.repo.repo} in:title "${failureName}"` | |
| }) | |
| const issue = issues.data.items.find((issue) => issue.title === issue.title); | |
| if (issue) { | |
| return; | |
| } | |
| // Create a new issue. | |
| github.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title: issueTitle, | |
| body: `A new fuzz test failure was found in ${{ matrix.package }}`. | |
| To reproduce the failure locally: | |
| 1. Download the corpus entry from the artifact: ${{ steps.artifact.outputs.artifact-url }}. | |
| 2. Unzip the artifact. | |
| 3. Place the corpus entry at <pre>${{ matrix.package }}/testdata/fuzz/${{ matrix.function }}/${failureName}</pre>. | |
| 4. Run tests in the <pre>${{ matrix.package }}</pre> package. | |
| When opening a PR with the fix, please include in the corpus entry in your commit to prevent regressions.`, | |
| labels: ['bug'], | |
| }) |