Skip to content

Commit

Permalink
ARCH-2011 - Add different trx files for failures we ignore vs allow
Browse files Browse the repository at this point in the history
  • Loading branch information
danielle-casella-adams committed Feb 26, 2024
1 parent b6addfd commit 1b8edb6
Show file tree
Hide file tree
Showing 6 changed files with 203 additions and 79 deletions.
23 changes: 12 additions & 11 deletions .github/workflows/build-and-review-pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -606,7 +606,7 @@ jobs:
if: always()
run: |
# Verify the trx-files list matches what we expect
failure="./test/files/failing-test/failure.trx"
failure="./test/files/failing-test-ignore/failure.trx"
./test/assert-values-match.sh --name "trx-files output" --expected "[$failure]" --actual "${{ steps.failing-test.outputs.trx-files }}"
# Verify the test-outcome is Failed
Expand All @@ -631,7 +631,7 @@ jobs:
# - The Counter stats are included in the report
# - The failure details are included
expectedFileName="./test/files/failing-test/expected-test-results.md"
expectedFileName="./test/files/failing-test-ignore/expected-test-results.md"
actualFileName="./test-results.md"
./test/assert-file-contents-match.sh --expectedFileName $expectedFileName --actualFileName $actualFileName
Expand All @@ -642,7 +642,8 @@ jobs:
runs-on: ubuntu-latest
env:
NO_FAILURES_DIR: './test/files/multiple-trx'
FAILURES_DIR: './test/files/failing-test'
ALLOW_FAILURES_DIR: './test/files/failing-test-allow'
IGNORE_FAILURES_DIR: './test/files/failing-test-ignore'

steps:
- name: '-------------------------------------------------------------------------------------------------------------'
Expand Down Expand Up @@ -732,7 +733,7 @@ jobs:
uses: ./
with:
github-token: '${{ secrets.GITHUB_TOKEN }}'
base-directory: ${{ env.FAILURES_DIR }}
base-directory: ${{ env.IGNORE_FAILURES_DIR }}
create-results-file: true # Keep this so we have an "actual" output to compare against
create-status-check: true
ignore-test-failures: true
Expand Down Expand Up @@ -764,13 +765,13 @@ jobs:
const expectedBody = fs.readFileSync('./test-results.md', 'utf8');
const expectedValues = {
name: 'status check - dotnet unit tests (purposely.failing.tests)',
name: 'status check - dotnet unit tests (ignore.failures)',
status: 'completed',
conclusion: 'neutral',
title: 'PURPOSELY.FAILING.TESTS',
title: 'IGNORE.FAILURES',
text: expectedBody
};
const actualCheck = actualStatusChecks.find(check => check.name.includes('purposely'));
const actualCheck = actualStatusChecks.find(check => check.name.includes('ignore'));
assertStatusCheckMatchesExpectations(core, actualCheck, expectedValues);
- name: '-------------------------------------------------------------------------------------------------------------'
Expand All @@ -784,7 +785,7 @@ jobs:
uses: ./
with:
github-token: '${{ secrets.GITHUB_TOKEN }}'
base-directory: ${{ env.FAILURES_DIR }}
base-directory: ${{ env.ALLOW_FAILURES_DIR }}
create-results-file: true # Keep this so we have an "actual" output to compare against
create-status-check: true
ignore-test-failures: false
Expand Down Expand Up @@ -816,13 +817,13 @@ jobs:
const expectedBody = fs.readFileSync('./test-results.md', 'utf8');
const expectedValues = {
name: 'status check - dotnet unit tests (purposely.failing.tests)',
name: 'status check - dotnet unit tests (allow.failures)',
status: 'completed',
conclusion: 'failure',
title: 'PURPOSELY.FAILING.TESTS',
title: 'ALLOW.FAILURES',
text: expectedBody
};
const actualCheck = actualStatusChecks.find(check => check.name.includes('widget'));
const actualCheck = actualStatusChecks.find(check => check.name.includes('allow'));
assertStatusCheckMatchesExpectations(core, actualCheck, expectedValues);
- name: '-------------------------------------------------------------------------------------------------------------'
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@

# PURPOSELY.FAILING.TESTS
# Allow.Failures

![Generic badge](https://img.shields.io/badge/1/4-FAILED-red.svg)
<details>
Expand Down Expand Up @@ -28,19 +28,19 @@
</table>
</details>
<details>
<summary>Outcome: Failed | Total Tests: 4 | Passed: 3 | Failed: 1</summary>
<summary>Outcome: Failed | Total Tests: 1 | Passed: 0 | Failed: 1</summary>
<table>
<tr>
<th>Total:</th>
<td>4</td>
<td>1</td>
</tr>
<tr>
<th>Executed:</th>
<td>4</td>
<td>1</td>
</tr>
<tr>
<th>Passed:</th>
<td>3</td>
<td>0</td>
</tr>
<tr>
<th>Failed:</th>
Expand All @@ -49,11 +49,11 @@
</table>
</details>
<details>
<summary>:x: Purposely.Failing.Tests.MathTests.OnePlusOneShouldNotEqualFive</summary>
<summary>:x: Allow.Failures.MathTests.OnePlusOneShouldNotEqualFive</summary>
<table>
<tr>
<th>Name:</th>
<td><code>Purposely.Failing.Tests.MathTests.OnePlusOneShouldNotEqualFive</code></td>
<td><code>Allow.Failures.MathTests.OnePlusOneShouldNotEqualFive</code></td>
</tr>
<tr>
<th>Outcome:</th>
Expand All @@ -73,11 +73,11 @@
</tr>
<tr>
<th>Code Base</th>
<td><code>C:\code\widget\bin\Release\net6.0\Purposely.Failing.Tests.dll</code></td>
<td><code>C:\code\widget\bin\Release\net6.0\Allow.Failures.dll</code></td>
</tr>
<tr>
<th>Class Name</th>
<td><code>Purposely.Failing.Tests.MathTests</code></td>
<td><code>Allow.Failures.MathTests</code></td>
</tr>
<tr>
<th>Method Name</th>
Expand All @@ -92,4 +92,4 @@
<td><pre>StackTrace</pre></td>
</tr>
</table>
</details>
</details>
40 changes: 40 additions & 0 deletions test/files/failing-test-allow/failure.trx
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
<?xml version="1.0" encoding="utf-8"?>
<TestRun id="808a5446-8133-4d5a-9a2b-a58b5db3a047" name="myuser@MY-MACHINE 2024-02-21 13:43:46" runUser="MYDOMAIN\myuser" xmlns="http://microsoft.com/schemas/VisualStudio/TeamTest/2010">
<Times creation="2024-02-21T13:43:46.5143122-07:00" queuing="2024-02-21T13:43:46.5143127-07:00" start="2024-02-21T13:43:45.7198761-07:00" finish="2024-02-21T13:43:46.5245604-07:00" />
<TestSettings name="default" id="8ee24c85-02e2-49c5-bd6c-8c6c3f49e76a">
<Deployment runDeploymentRoot="myuser_MY-MACHINE_2024-02-21_13_43_46" />
</TestSettings>
<Results>
<UnitTestResult executionId="49f53981-a391-4a96-b0d1-dfc1f2efbd58" testId="09917f6f-ce84-0a6f-b407-bec561fda4c1" testName="Allow.Failures.MathTests.OnePlusOneShouldNotEqualFive" computerName="MY-MACHINE" duration="00:00:00.0065475" startTime="2024-02-21T13:43:46.3841020-07:00" endTime="2024-02-21T13:43:46.3841023-07:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Failed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="49f53981-a391-4a96-b0d1-dfc1f2efbd58">
<Output>
<ErrorInfo>
<Message>Assert.NotEqual() Failure: Values are equal&#xD;
Expected: Not 2&#xD;
Actual: 2</Message>
<StackTrace> at Allow.Failures.MathTests.OnePlusOneShouldNotEqualFive() in C:\code\widget\MathTests.cs:line 19</StackTrace>
</ErrorInfo>
</Output>
</UnitTestResult>
</Results>
<TestDefinitions>
<UnitTest name="Allow.Failures.MathTests.OnePlusOneShouldNotEqualFive" storage="c:\code\widget\bin\release\net6.0\allow.failures.dll" id="09917f6f-ce84-0a6f-b407-bec561fda4c1">
<Execution id="49f53981-a391-4a96-b0d1-dfc1f2efbd58" />
<TestMethod codeBase="C:\code\widget\bin\Release\net6.0\Allow.Failures.dll" adapterTypeName="executor://xunit/VsTestRunner2/netcoreapp" className="Allow.Failures.MathTests" name="OnePlusOneShouldNotEqualFive" />
</UnitTest>
</TestDefinitions>
<TestEntries>
<TestEntry testId="09917f6f-ce84-0a6f-b407-bec561fda4c1" executionId="49f53981-a391-4a96-b0d1-dfc1f2efbd58" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" />
</TestEntries>
<TestLists>
<TestList name="Results Not in a List" id="8c84fa94-04c1-424b-9868-57a2d4851a1d" />
<TestList name="All Loaded Results" id="19431567-8539-422a-85d7-44ee4e166bda" />
</TestLists>
<ResultSummary outcome="Failed">
<Counters total="1" executed="1" passed="0" failed="1" error="0" timeout="0" aborted="0" inconclusive="0" passedButRunAborted="0" notRunnable="0" notExecuted="0" disconnected="0" warning="0" completed="0" inProgress="0" pending="0" />
<RunInfos>
<RunInfo computerName="MY-MACHINE" outcome="Error" timestamp="2024-02-21T13:43:46.3794356-07:00">
<Text>[xUnit.net 00:00:00.26] Allow.Failures.MathTests.OnePlusOneShouldNotEqualFive [FAIL]</Text>
</RunInfo>
</RunInfos>
</ResultSummary>
</TestRun>
95 changes: 95 additions & 0 deletions test/files/failing-test-ignore/expected-test-results.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@

# Ignore.Failures

![Generic badge](https://img.shields.io/badge/1/4-FAILED-red.svg)
<details>
<summary>Duration: 0.805 seconds</summary>
<table>
<tr>
<th>Start:</th>
<td><code>2024-02-21 20:43:45.719 UTC</code></td>
</tr>
<tr>
<th>Creation:</th>
<td><code>2024-02-21 20:43:46.514 UTC</code></td>
</tr>
<tr>
<th>Queuing:</th>
<td><code>2024-02-21 20:43:46.514 UTC</code></td>
</tr>
<tr>
<th>Finish:</th>
<td><code>2024-02-21 20:43:46.524 UTC</code></td>
</tr>
<tr>
<th>Duration:</th>
<td><code>0.805 seconds</code></td>
</tr>
</table>
</details>
<details>
<summary>Outcome: Failed | Total Tests: 2 | Passed: 1 | Failed: 1</summary>
<table>
<tr>
<th>Total:</th>
<td>2</td>
</tr>
<tr>
<th>Executed:</th>
<td>2</td>
</tr>
<tr>
<th>Passed:</th>
<td>1</td>
</tr>
<tr>
<th>Failed:</th>
<td>1</td>
</tr>
</table>
</details>
<details>
<summary>:x: Ignore.Failures.MathTests.OnePlusOneShouldNotEqualFive</summary>
<table>
<tr>
<th>Name:</th>
<td><code>Ignore.Failures.MathTests.OnePlusOneShouldNotEqualFive</code></td>
</tr>
<tr>
<th>Outcome:</th>
<td><code>Failed</code></td>
</tr>
<tr>
<th>Start:</th>
<td><code>2024-02-21 20:43:46.384 UTC</code></td>
</tr>
<tr>
<th>End:</th>
<td><code>2024-02-21 20:43:46.384 UTC</code></td>
</tr>
<tr>
<th>Duration:</th>
<td><code>00:00:00.0065475</code></td>
</tr>
<tr>
<th>Code Base</th>
<td><code>C:\code\widget\bin\Release\net6.0\Ignore.Failures.dll</code></td>
</tr>
<tr>
<th>Class Name</th>
<td><code>Ignore.Failures.MathTests</code></td>
</tr>
<tr>
<th>Method Name</th>
<td><code>OnePlusOneShouldNotEqualFive</code></td>
</tr>
<tr>
<th>Error Message:</th>
<td><pre>Message</pre></td>
</tr>
<tr>
<th>Stack Trace:</th>
<td><pre>StackTrace</pre></td>
</tr>
</table>
</details>
46 changes: 46 additions & 0 deletions test/files/failing-test-ignore/failure.trx
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
<?xml version="1.0" encoding="utf-8"?>
<TestRun id="808a5446-8133-4d5a-9a2b-a58b5db3a047" name="myuser@MY-MACHINE 2024-02-21 13:43:46" runUser="MYDOMAIN\myuser" xmlns="http://microsoft.com/schemas/VisualStudio/TeamTest/2010">
<Times creation="2024-02-21T13:43:46.5143122-07:00" queuing="2024-02-21T13:43:46.5143127-07:00" start="2024-02-21T13:43:45.7198761-07:00" finish="2024-02-21T13:43:46.5245604-07:00" />
<TestSettings name="default" id="8ee24c85-02e2-49c5-bd6c-8c6c3f49e76a">
<Deployment runDeploymentRoot="myuser_MY-MACHINE_2024-02-21_13_43_46" />
</TestSettings>
<Results>
<UnitTestResult executionId="1da09059-c2e3-488d-9b7b-ce42d8098ef3" testId="be281ffb-a4d6-67ab-d9af-941232c31283" testName="Ignore.Failures.MathTests.AddShouldReturnCorrectValue(number1: 1, number2: 2, expectedValue: 3)" computerName="MY-MACHINE" duration="00:00:00.0000121" startTime="2024-02-21T13:43:46.3595538-07:00" endTime="2024-02-21T13:43:46.3595539-07:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Passed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="1da09059-c2e3-488d-9b7b-ce42d8098ef3" />
<UnitTestResult executionId="49f53981-a391-4a96-b0d1-dfc1f2efbd58" testId="09917f6f-ce84-0a6f-b407-bec561fda4c1" testName="Ignore.Failures.MathTests.OnePlusOneShouldNotEqualFive" computerName="MY-MACHINE" duration="00:00:00.0065475" startTime="2024-02-21T13:43:46.3841020-07:00" endTime="2024-02-21T13:43:46.3841023-07:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Failed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="49f53981-a391-4a96-b0d1-dfc1f2efbd58">
<Output>
<ErrorInfo>
<Message>Assert.NotEqual() Failure: Values are equal&#xD;
Expected: Not 2&#xD;
Actual: 2</Message>
<StackTrace> at Ignore.Failures.MathTests.OnePlusOneShouldNotEqualFive() in C:\code\widget\MathTests.cs:line 19</StackTrace>
</ErrorInfo>
</Output>
</UnitTestResult>
</Results>
<TestDefinitions>
<UnitTest name="Ignore.Failures.MathTests.AddShouldReturnCorrectValue(number1: 1, number2: 2, expectedValue: 3)" storage="c:\code\widget\bin\release\net6.0\ignore.failures.dll" id="be281ffb-a4d6-67ab-d9af-941232c31283">
<Execution id="1da09059-c2e3-488d-9b7b-ce42d8098ef3" />
<TestMethod codeBase="C:\code\widget\bin\Release\net6.0\Ignore.Failures.dll" adapterTypeName="executor://xunit/VsTestRunner2/netcoreapp" className="Ignore.Failures.MathTests" name="AddShouldReturnCorrectValue" />
</UnitTest>
<UnitTest name="Ignore.Failures.MathTests.OnePlusOneShouldNotEqualFive" storage="c:\code\widget\bin\release\net6.0\ignore.failures.dll" id="09917f6f-ce84-0a6f-b407-bec561fda4c1">
<Execution id="49f53981-a391-4a96-b0d1-dfc1f2efbd58" />
<TestMethod codeBase="C:\code\widget\bin\Release\net6.0\Ignore.Failures.dll" adapterTypeName="executor://xunit/VsTestRunner2/netcoreapp" className="Ignore.Failures.MathTests" name="OnePlusOneShouldNotEqualFive" />
</UnitTest>
</TestDefinitions>
<TestEntries>
<TestEntry testId="be281ffb-a4d6-67ab-d9af-941232c31283" executionId="1da09059-c2e3-488d-9b7b-ce42d8098ef3" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" />
<TestEntry testId="09917f6f-ce84-0a6f-b407-bec561fda4c1" executionId="49f53981-a391-4a96-b0d1-dfc1f2efbd58" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" />
</TestEntries>
<TestLists>
<TestList name="Results Not in a List" id="8c84fa94-04c1-424b-9868-57a2d4851a1d" />
<TestList name="All Loaded Results" id="19431567-8539-422a-85d7-44ee4e166bda" />
</TestLists>
<ResultSummary outcome="Failed">
<Counters total="2" executed="2" passed="1" failed="1" error="0" timeout="0" aborted="0" inconclusive="0" passedButRunAborted="0" notRunnable="0" notExecuted="0" disconnected="0" warning="0" completed="0" inProgress="0" pending="0" />
<RunInfos>
<RunInfo computerName="MY-MACHINE" outcome="Error" timestamp="2024-02-21T13:43:46.3794356-07:00">
<Text>[xUnit.net 00:00:00.26] Ignore.Failures.MathTests.OnePlusOneShouldNotEqualFive [FAIL]</Text>
</RunInfo>
</RunInfos>
</ResultSummary>
</TestRun>
Loading

0 comments on commit 1b8edb6

Please sign in to comment.