From 44dbcefee449d05b81a3dc39488e9033e9e1f7c8 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 16 Dec 2024 11:55:37 +0100 Subject: [PATCH] feat: add test report for upgrade tests --- .../workflows/reusable-build-test-release.yml | 146 +++++++++++------- 1 file changed, 91 insertions(+), 55 deletions(-) diff --git a/.github/workflows/reusable-build-test-release.yml b/.github/workflows/reusable-build-test-release.yml index 09307a4a..16bf60bb 100644 --- a/.github/workflows/reusable-build-test-release.yml +++ b/.github/workflows/reusable-build-test-release.yml @@ -526,7 +526,6 @@ jobs: if: ${{ !cancelled() && (needs.run-unit-tests.result == 'success' || needs.run-unit-tests.result == 'skipped') && (needs.validate-custom-version.result == 'success' || needs.validate-custom-version.result == 'skipped') }} outputs: buildname: ${{ steps.buildupload.outputs.name }} - app-id: ${{ steps.app-id.outputs.app-id }} permissions: contents: write packages: read @@ -698,13 +697,7 @@ jobs: name: package-deployment path: build/package/deployment** if: ${{ !cancelled() }} - - name: app-id - id: app-id - run: | - APP_ID=$(grep SPLUNKBASE_ID .splunkbase | cut -d'=' -f2) - export APP_ID - echo "APP ID $APP_ID" - echo "app-id=$APP_ID" >> "$GITHUB_OUTPUT" + build-3_9: runs-on: ubuntu-latest needs: @@ -2293,9 +2286,9 @@ jobs: with: name: | summary-modinput* - - run-upgrade-tests: - if: ${{ !cancelled() && needs.build.result == 'success' && needs.test-inventory.outputs.upgrade == 'true' }} + + run-ucc-modinput-tests: + if: ${{ !cancelled() && needs.build.result == 'success' && needs.test-inventory.outputs.ucc_modinput_functional == 'true' && needs.setup-workflow.outputs.execute-ucc-modinput-labeled == 'true' }} needs: - build - test-inventory @@ -2308,7 +2301,7 @@ jobs: matrix: splunk: ${{ fromJson(needs.meta.outputs.matrix_supportedSplunk) }} vendor-version: ${{ fromJson(needs.meta.outputs.matrix_supportedModinputFunctionalVendors) }} - ta-version-from-upgrade: ${{ fromJson(inputs.upgrade-tests-ta-versions) }} + marker: ${{ fromJson(inputs.ucc-modinput-marker) }} container: image: ghcr.io/splunk/workflow-engine-base:4.1.0 env: @@ -2318,7 +2311,7 @@ jobs: ARGO_BASE_HREF: ${{ needs.setup.outputs.argo-href }} ARGO_NAMESPACE: ${{ needs.setup.outputs.argo-namespace }} SPLUNK_VERSION_BASE: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} - TEST_TYPE: "upgrade" + TEST_TYPE: "ucc_modinput_functional" TEST_ARGS: "" permissions: actions: read @@ -2359,21 +2352,33 @@ jobs: run: | RANDOM_STRING=$(head -3 /dev/urandom | tr -cd '[:lower:]' | cut -c -4) JOB_NAME=${{ needs.setup.outputs.job-name }}-${RANDOM_STRING} - JOB_NAME=${JOB_NAME//TEST-TYPE/${{ env.TEST_TYPE }}} + JOB_NAME=${JOB_NAME//TEST-TYPE/ucc_modinput} JOB_NAME=${JOB_NAME//[_.]/-} JOB_NAME=$(echo "$JOB_NAME" | tr '[:upper:]' '[:lower:]') echo "job-name=$JOB_NAME" >> "$GITHUB_OUTPUT" + - name: create test argument + id: create-test-arg + shell: bash + run: | + TEST_ARG_M="" + EMPTY_MARKER="[]" + + if [[ "${{ inputs.ucc-modinput-marker }}" != "$EMPTY_MARKER" ]]; then + TEST_ARG_M="-m" + fi + + echo "test-arg=$TEST_ARG_M" >> "$GITHUB_OUTPUT" - name: run-tests id: run-tests timeout-minutes: 340 continue-on-error: true env: ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - uses: splunk/wfe-test-runner-action@feat/ADDON-73868-add-inputs-for-upgrade-tests # TODO: add correct branch name + uses: splunk/wfe-test-runner-action@v5.0 with: splunk: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} test-type: ${{ env.TEST_TYPE }} - test-args: ${{ env.TEST_ARGS }} + test-args: ${{ env.TEST_ARGS }} ${{ steps.create-test-arg.outputs.test-arg }} ${{ matrix.marker }} job-name: ${{ steps.create-job-name.outputs.job-name }} labels: ${{ needs.setup.outputs.labels }} workflow-tmpl-name: ${{ needs.setup.outputs.argo-workflow-tmpl-name }} @@ -2383,8 +2388,6 @@ jobs: vendor-version: ${{ matrix.vendor-version.image }} sc4s-version: "No" k8s-manifests-branch: ${{ needs.setup.outputs.k8s-manifests-branch }} - app-id: ${{ needs.build.outputs.app-id }} - ta-upgrade-version: ${{ matrix.ta-version-from-upgrade }} - name: Read secrets from AWS Secrets Manager again into environment variables in case credential rotation id: update-argo-token if: ${{ !cancelled() }} @@ -2481,17 +2484,17 @@ jobs: fi echo "pulling logs" mkdir -p ${{ needs.setup.outputs.directory-path }}/argo-logs - aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/${WORKFLOW_NAME}/ ${{ needs.setup.outputs.directory-path }}/argo-logs/ --recursive + aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/workflows/${WORKFLOW_NAME}/ ${{ needs.setup.outputs.directory-path }}/argo-logs/ --recursive - uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.ta-version-from-upgrade }} tests artifacts + name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests artifacts path: | ${{ needs.setup.outputs.directory-path }}/test-results - uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.ta-version-from-upgrade }} tests logs + name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests logs path: | ${{ needs.setup.outputs.directory-path }}/argo-logs - name: Test Report @@ -2499,9 +2502,32 @@ jobs: uses: dorny/test-reporter@v1.9.1 if: ${{ !cancelled() }} with: - name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} test report + name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} test report path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" reporter: java-junit + - name: Parse JUnit XML + if: ${{ !cancelled() }} + run: | + apt-get install -y libxml2-utils + junit_xml_path="${{ needs.setup.outputs.directory-path }}/test-results" + junit_xml_file=$(find "$junit_xml_path" -name "*.xml" -type f 2>/dev/null | head -n 1) + if [ -n "$junit_xml_file" ]; then + total_tests=$(xmllint --xpath "count(//testcase)" "$junit_xml_file") + failures=$(xmllint --xpath "count(//testcase[failure])" "$junit_xml_file") + errors=$(xmllint --xpath "count(//testcase[error])" "$junit_xml_file") + skipped=$(xmllint --xpath "count(//testcase[skipped])" "$junit_xml_file") + passed=$((total_tests - failures - errors - skipped)) + echo "splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ matrix.marker }} ${{ matrix.vendor-version.image }} |$total_tests |$passed |$failures |$errors | $skipped |${{steps.test_report.outputs.url_html}}" > job_summary.txt + else + echo "no XML File found, exiting" + exit 1 + fi + - name: Upload-artifact-for-github-summary + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: summary-${{ env.TEST_TYPE }}-${{ matrix.splunk.version }}-${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}-${{ matrix.vendor-version.image }}-${{ matrix.marker }}-artifact + path: job_summary.txt - name: pull diag from s3 bucket if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} run: | @@ -2510,12 +2536,33 @@ jobs: - uses: actions/upload-artifact@v4 if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} tests diag + name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests diag path: | ${{ needs.setup.outputs.directory-path }}/diag* - run-ucc-modinput-tests: - if: ${{ !cancelled() && needs.build.result == 'success' && needs.test-inventory.outputs.ucc_modinput_functional == 'true' && needs.setup-workflow.outputs.execute-ucc-modinput-labeled == 'true' }} + ucc-modinput-tests-report: + needs: run-ucc-modinput-tests + runs-on: ubuntu-latest + if: ${{ !cancelled() && needs.run-ucc-modinput-tests.result != 'skipped' }} + steps: + - name: Download all summaries + uses: actions/download-artifact@v4 + with: + pattern: summary-ucc_modinput* + - name: Combine summaries into a table + run: | + echo "| Job | Total Tests | Passed Tests | Failed Tests | Errored Tests | Skipped Tests | Report Link" >> "$GITHUB_STEP_SUMMARY" + echo "| ---------- | ----------- | ------ | ------ | ------ | ------- | ------ |" >> "$GITHUB_STEP_SUMMARY" + for file in summary-ucc_modinput*/job_summary.txt; do + cat "$file" >> "$GITHUB_STEP_SUMMARY" + done + - uses: geekyeggo/delete-artifact@v5 + with: + name: | + summary-ucc_modinput* + + run-upgrade-tests: + if: ${{ !cancelled() && needs.build.result == 'success' && needs.test-inventory.outputs.upgrade == 'true' }} needs: - build - test-inventory @@ -2528,7 +2575,7 @@ jobs: matrix: splunk: ${{ fromJson(needs.meta.outputs.matrix_supportedSplunk) }} vendor-version: ${{ fromJson(needs.meta.outputs.matrix_supportedModinputFunctionalVendors) }} - marker: ${{ fromJson(inputs.ucc-modinput-marker) }} + ta-version-from-upgrade: ${{ fromJson(inputs.upgrade-tests-ta-versions) }} container: image: ghcr.io/splunk/workflow-engine-base:4.1.0 env: @@ -2538,7 +2585,7 @@ jobs: ARGO_BASE_HREF: ${{ needs.setup.outputs.argo-href }} ARGO_NAMESPACE: ${{ needs.setup.outputs.argo-namespace }} SPLUNK_VERSION_BASE: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} - TEST_TYPE: "ucc_modinput_functional" + TEST_TYPE: "upgrade" TEST_ARGS: "" permissions: actions: read @@ -2579,33 +2626,21 @@ jobs: run: | RANDOM_STRING=$(head -3 /dev/urandom | tr -cd '[:lower:]' | cut -c -4) JOB_NAME=${{ needs.setup.outputs.job-name }}-${RANDOM_STRING} - JOB_NAME=${JOB_NAME//TEST-TYPE/ucc_modinput} + JOB_NAME=${JOB_NAME//TEST-TYPE/${{ env.TEST_TYPE }}} JOB_NAME=${JOB_NAME//[_.]/-} JOB_NAME=$(echo "$JOB_NAME" | tr '[:upper:]' '[:lower:]') echo "job-name=$JOB_NAME" >> "$GITHUB_OUTPUT" - - name: create test argument - id: create-test-arg - shell: bash - run: | - TEST_ARG_M="" - EMPTY_MARKER="[]" - - if [[ "${{ inputs.ucc-modinput-marker }}" != "$EMPTY_MARKER" ]]; then - TEST_ARG_M="-m" - fi - - echo "test-arg=$TEST_ARG_M" >> "$GITHUB_OUTPUT" - name: run-tests id: run-tests timeout-minutes: 340 continue-on-error: true env: ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - uses: splunk/wfe-test-runner-action@v5.0 + uses: splunk/wfe-test-runner-action@feat/ADDON-73868-add-inputs-for-upgrade-tests # TODO: add correct branch name with: splunk: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} test-type: ${{ env.TEST_TYPE }} - test-args: ${{ env.TEST_ARGS }} ${{ steps.create-test-arg.outputs.test-arg }} ${{ matrix.marker }} + test-args: ${{ env.TEST_ARGS }} job-name: ${{ steps.create-job-name.outputs.job-name }} labels: ${{ needs.setup.outputs.labels }} workflow-tmpl-name: ${{ needs.setup.outputs.argo-workflow-tmpl-name }} @@ -2615,6 +2650,7 @@ jobs: vendor-version: ${{ matrix.vendor-version.image }} sc4s-version: "No" k8s-manifests-branch: ${{ needs.setup.outputs.k8s-manifests-branch }} + ta-upgrade-version: ${{ matrix.ta-version-from-upgrade }} - name: Read secrets from AWS Secrets Manager again into environment variables in case credential rotation id: update-argo-token if: ${{ !cancelled() }} @@ -2711,17 +2747,17 @@ jobs: fi echo "pulling logs" mkdir -p ${{ needs.setup.outputs.directory-path }}/argo-logs - aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/workflows/${WORKFLOW_NAME}/ ${{ needs.setup.outputs.directory-path }}/argo-logs/ --recursive + aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/${WORKFLOW_NAME}/ ${{ needs.setup.outputs.directory-path }}/argo-logs/ --recursive - uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests artifacts + name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.ta-version-from-upgrade }} tests artifacts path: | ${{ needs.setup.outputs.directory-path }}/test-results - uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests logs + name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.ta-version-from-upgrade }} tests logs path: | ${{ needs.setup.outputs.directory-path }}/argo-logs - name: Test Report @@ -2729,7 +2765,7 @@ jobs: uses: dorny/test-reporter@v1.9.1 if: ${{ !cancelled() }} with: - name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} test report + name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} test report path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" reporter: java-junit - name: Parse JUnit XML @@ -2744,7 +2780,7 @@ jobs: errors=$(xmllint --xpath "count(//testcase[error])" "$junit_xml_file") skipped=$(xmllint --xpath "count(//testcase[skipped])" "$junit_xml_file") passed=$((total_tests - failures - errors - skipped)) - echo "splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ matrix.marker }} ${{ matrix.vendor-version.image }} |$total_tests |$passed |$failures |$errors | $skipped |${{steps.test_report.outputs.url_html}}" > job_summary.txt + echo "splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ matrix.ta-version-from-upgrade }} ${{ matrix.vendor-version.image }} |$total_tests |$passed |$failures |$errors | $skipped |${{steps.test_report.outputs.url_html}}" > job_summary.txt else echo "no XML File found, exiting" exit 1 @@ -2753,7 +2789,7 @@ jobs: uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} with: - name: summary-${{ env.TEST_TYPE }}-${{ matrix.splunk.version }}-${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}-${{ matrix.vendor-version.image }}-${{ matrix.marker }}-artifact + name: summary-${{ env.TEST_TYPE }}-${{ matrix.splunk.version }}-${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}-${{ matrix.vendor-version.image }}-${{ matrix.ta-version-from-upgrade }}-artifact path: job_summary.txt - name: pull diag from s3 bucket if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} @@ -2763,30 +2799,30 @@ jobs: - uses: actions/upload-artifact@v4 if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests diag + name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} tests diag path: | ${{ needs.setup.outputs.directory-path }}/diag* - ucc-modinput-tests-report: - needs: run-ucc-modinput-tests + upgrade-tests-report: + needs: run-upgrade-tests runs-on: ubuntu-latest - if: ${{ !cancelled() && needs.run-ucc-modinput-tests.result != 'skipped' }} + if: ${{ !cancelled() && needs.run-upgrade-tests.result != 'skipped' }} steps: - name: Download all summaries uses: actions/download-artifact@v4 with: - pattern: summary-ucc_modinput* + pattern: summary-upgrade* - name: Combine summaries into a table run: | echo "| Job | Total Tests | Passed Tests | Failed Tests | Errored Tests | Skipped Tests | Report Link" >> "$GITHUB_STEP_SUMMARY" echo "| ---------- | ----------- | ------ | ------ | ------ | ------- | ------ |" >> "$GITHUB_STEP_SUMMARY" - for file in summary-ucc_modinput*/job_summary.txt; do + for file in summary-upgrade*/job_summary.txt; do cat "$file" >> "$GITHUB_STEP_SUMMARY" done - uses: geekyeggo/delete-artifact@v5 with: name: | - summary-ucc_modinput* + summary-upgrade* run-scripted-input-tests-full-matrix: if: ${{ !cancelled() && needs.build.result == 'success' && needs.test-inventory.outputs.scripted_inputs == 'true' && needs.setup-workflow.outputs.execute-scripted_inputs-labeled == 'true' }}