Skip to content

Commit

Permalink
Merge pull request #219 from GannettDigital/QE-6631
Browse files Browse the repository at this point in the history 
Added deferFailureReports configuration option - cutting a release for testing
  • Loading branch information
@scottgunther
scottgunther authored Aug 4, 2020
2 parents f435009 + f559257 commit 6f79ee4
Show file tree
Hide file tree
Showing 3 changed files with 150 additions and 2 deletions.
7 changes: 7 additions & 0 deletions docs/pages/documentation/configuration-file.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ This section documents utilization of the configuration file in place of CLI opt
* Use to customize the selenium driver
* See [driver configuration](#driver-configuration)
* Example: `driver: {<CUSTOM DRIVER CAPABILITIES>}`

### sauce connect arguments
* Use to pass options to the embedded suace connect tunnel
* documentation for options can be found [here](https://www.npmjs.com/package/sauce-connect-launcher)
Expand All @@ -51,6 +52,12 @@ This section documents utilization of the configuration file in place of CLI opt
* Values Allowed: `JSON`, `actionJSON`, `JUnit`
* Example: `reportFormat: 'JSON'`

### deferFailureReports
* Whether or not to wait until all tests are finished to show test failure reports
* When paralellism is set to zero, this has no effect
* Default is `false`
* Example: `deferFailureReports: true`

### jUnitReportSpecificity
* The levels of detail
* Default value is `testReport`
Expand Down
12 changes: 12 additions & 0 deletions lib/runner/test-runner/test-report-handler.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ module.exports = testReportHandler = {
time: null,
testReports: [],
},
_failedReports: [],
startReportHandler() {
testReportHandler._report.time = process.hrtime();
},
Expand Down Expand Up @@ -57,17 +58,28 @@ module.exports = testReportHandler = {
if (rerun !== -1 && testRun.report.status === 'fail') {
testRun.report.status = 'rerun';
testReport.rerunCount++;
} else if (testRun.report.status === 'fail' && configHandler.get('deferFailureReports') === true) {
testReportHandler._failedReports.push(testRun.report);
return;
}
testReportHandler._handleTestReport(testRun.report);
},
finalizeReport() {
if (configHandler.get('deferFailureReports') === true) {
testReportHandler._printFailedReports();
}
testReportHandler._report.time = process.hrtime(testReportHandler._report.time);
if (!testReportHandler._report.failedTestCount) {
testReportHandler._report.status = 'pass';
}
testReportHandler._handleTestReportSummary();
testReportHandler.emit('testReportHandler.reportFinalized');
},
_printFailedReports() {
for (let report of testReportHandler._failedReports) {
testReportHandler._handleTestReport(report);
}
},
_handleTestReport(report) {
switch (configHandler.get('reporter')) {
case 'basic':
Expand Down
133 changes: 131 additions & 2 deletions test/unit/lib/runner/test-runner/test-report-handler-tests.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,12 +263,17 @@ describe('lib/runner/test-runner/test-report-handler.js', function() {

describe('finalizeTestReport', function() {
let testReportHandler;
let configHandler;
let Emitter;

beforeEach(function() {
mockery.enable({useCleanCache: true});
mockery.registerAllowable('../../../../../lib/runner/test-runner/test-report-handler.js');

configHandler = {
get: sinon.stub(),
};

Emitter = {
mixIn: function(myObject) {
myObject.on = sinon.stub();
Expand All @@ -279,7 +284,7 @@ describe('lib/runner/test-runner/test-report-handler.js', function() {

mockery.registerMock('../../util/emitter.js', Emitter);
mockery.registerMock('../runner-event-dispatch/runner-event-dispatch.js', {});
mockery.registerMock('../../util/config/config-handler.js', {});
mockery.registerMock('../../util/config/config-handler.js', configHandler);

testReportHandler = require('../../../../../lib/runner/test-runner/test-report-handler.js');
testReportHandler._handleTestReport = sinon.stub();
Expand Down Expand Up @@ -395,6 +400,65 @@ describe('lib/runner/test-runner/test-report-handler.js', function() {
});
});

describe('if the most recent testRun report status is \'fail\' '
+ 'and the passed in rerunCount is falsey', function() {
describe('when config for deferFailureReports is true', function() {
it('should add the report to _failedReports', function() {
configHandler.get.returns(true);
testReportHandler._failedReports = [{
status: 'fail',
label: 'report1',
}];
testReportHandler._report = {
failedTestCount: 3,
testReports: [{
status: 'fail',
rerunCount: 0,
testRuns: [{
report: {
status: 'fail',
label: 'report2',
},
}],
}],
};

testReportHandler.finalizeTestReport(0, -1);

expect(testReportHandler._failedReports).to.eql([
{
status: 'fail',
label: 'report1',
},
{
status: 'fail',
label: 'report2',
},
]);
});

it('should not call _handleTestReport', function() {
configHandler.get.returns(true);
testReportHandler._report = {
failedTestCount: 3,
testReports: [{
status: 'fail',
rerunCount: 0,
testRuns: [{
report: {
status: 'fail',
},
}],
}],
};

testReportHandler.finalizeTestReport(0, -1);

expect(testReportHandler._handleTestReport.args).to.eql([]);
});
});
});

it('should call _handleTestReport with the report of the passed in testNumber most recent testRun', function() {
testReportHandler._report.testReports[0] = {
rerunCount: 0,
Expand All @@ -415,12 +479,17 @@ describe('lib/runner/test-runner/test-report-handler.js', function() {

describe('finalizeReport', function() {
let testReportHandler;
let configHandler;
let Emitter;

beforeEach(function() {
mockery.enable({useCleanCache: true});
mockery.registerAllowable('../../../../../lib/runner/test-runner/test-report-handler.js');

configHandler = {
get: sinon.stub(),
};

Emitter = {
mixIn: function(myObject) {
myObject.on = sinon.stub();
Expand All @@ -433,9 +502,10 @@ describe('lib/runner/test-runner/test-report-handler.js', function() {

mockery.registerMock('../../util/emitter.js', Emitter);
mockery.registerMock('../runner-event-dispatch/runner-event-dispatch.js', {});
mockery.registerMock('../../util/config/config-handler.js', {});
mockery.registerMock('../../util/config/config-handler.js', configHandler);

testReportHandler = require('../../../../../lib/runner/test-runner/test-report-handler.js');
testReportHandler._printFailedReports = sinon.stub();
testReportHandler._handleTestReportSummary = sinon.stub();
});

Expand All @@ -446,6 +516,18 @@ describe('lib/runner/test-runner/test-report-handler.js', function() {
process.hrtime.restore();
});

describe('when the config for deferFailureReports is true', function() {
it('should call printFailedReports', function() {
configHandler.get.returns(true);

testReportHandler.finalizeReport();

expect(testReportHandler._printFailedReports.args).to.eql([
[],
]);
});
});

it('should call process.hrtime once', function() {
testReportHandler.finalizeReport();

Expand Down Expand Up @@ -493,6 +575,53 @@ describe('lib/runner/test-runner/test-report-handler.js', function() {
});
});

describe('_printFailedReports', function() {
let testReportHandler;
let Emitter;

beforeEach(function() {
mockery.enable({useCleanCache: true});
mockery.registerAllowable('../../../../../lib/runner/test-runner/test-report-handler.js');

Emitter = {
mixIn: function(myObject) {
myObject.on = sinon.stub();
myObject.emit = sinon.stub();
},
};
sinon.spy(Emitter, 'mixIn');

sinon.stub(process, 'hrtime').returns([4, 54321]);

mockery.registerMock('../../util/emitter.js', Emitter);
mockery.registerMock('../runner-event-dispatch/runner-event-dispatch.js', {});
mockery.registerMock('../../util/config/config-handler.js', {});

testReportHandler = require('../../../../../lib/runner/test-runner/test-report-handler.js');
testReportHandler._handleTestReport = sinon.stub();
});

afterEach(function() {
mockery.resetCache();
mockery.deregisterAll();
mockery.disable();
process.hrtime.restore();
});

it('should call _handleTestReport for every failed report', function() {
testReportHandler._failedReports = [
'report1', 'report2',
];

testReportHandler._printFailedReports();

expect(testReportHandler._handleTestReport.args).to.eql([
['report1'],
['report2'],
]);
});
});

describe('_handleTestReport', function() {
let testReportHandler;
let Emitter;
Expand Down

0 comments on commit 6f79ee4

Please sign in to comment.