From ff5bdc6f2ac74ac7721e12b6d41ef2fa38afaa02 Mon Sep 17 00:00:00 2001 From: Zach Knox Date: Fri, 24 Jul 2020 13:26:09 -0400 Subject: [PATCH 1/5] QE-6631 added deferFailureReports --- lib/runner/test-runner/test-report-handler.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/runner/test-runner/test-report-handler.js b/lib/runner/test-runner/test-report-handler.js index c189b77..168cd18 100644 --- a/lib/runner/test-runner/test-report-handler.js +++ b/lib/runner/test-runner/test-report-handler.js @@ -13,6 +13,7 @@ module.exports = testReportHandler = { time: null, testReports: [], }, + _failedReports: [], startReportHandler() { testReportHandler._report.time = process.hrtime(); }, @@ -57,10 +58,16 @@ module.exports = testReportHandler = { if (rerun !== -1 && testRun.report.status === 'fail') { testRun.report.status = 'rerun'; testReport.rerunCount++; + } else if (testRun.report.status === 'fail' && configHandler.get('deferFailureReports') === true) { + _failedReports.push(testRun.report); + return; } testReportHandler._handleTestReport(testRun.report); }, finalizeReport() { + if (configHandler.get('deferFailureReports') === true) { + printFailedReports(); + } testReportHandler._report.time = process.hrtime(testReportHandler._report.time); if (!testReportHandler._report.failedTestCount) { testReportHandler._report.status = 'pass'; @@ -68,6 +75,11 @@ module.exports = testReportHandler = { testReportHandler._handleTestReportSummary(); testReportHandler.emit('testReportHandler.reportFinalized'); }, + printFailedReports() { + for (let report of _failedReports) { + _handleTestReport(report); + } + }, _handleTestReport(report) { switch (configHandler.get('reporter')) { case 'basic': From d9598148a77a889634fbb17051ced12185a93367 Mon Sep 17 00:00:00 2001 From: Zach Knox Date: Fri, 24 Jul 2020 15:34:02 -0400 Subject: [PATCH 2/5] passing unit tests --- docs/pages/documentation/configuration-file.md | 7 +++++++ .../test-runner/test-report-handler-tests.js | 14 ++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/pages/documentation/configuration-file.md b/docs/pages/documentation/configuration-file.md index 6e3a1c5..9403892 100644 --- a/docs/pages/documentation/configuration-file.md +++ b/docs/pages/documentation/configuration-file.md @@ -28,6 +28,7 @@ This section documents utilization of the configuration file in place of CLI opt * Use to customize the selenium driver * See [driver configuration](#driver-configuration) * Example: `driver: {}` + ### sauce connect arguments * Use to pass options to the embedded suace connect tunnel * documentation for options can be found [here](https://www.npmjs.com/package/sauce-connect-launcher) @@ -51,6 +52,12 @@ This section documents utilization of the configuration file in place of CLI opt * Values Allowed: `JSON`, `actionJSON`, `JUnit` * Example: `reportFormat: 'JSON'` +### deferFailureReports + * Whether or not to wait until all tests are finished to show test failure reports + * When paralellism is set to zero, this has no effect + * Default is `false` + * Example: `deferFailureReports: true` + ### jUnitReportSpecificity * The levels of detail * Default value is `testReport` diff --git a/test/unit/lib/runner/test-runner/test-report-handler-tests.js b/test/unit/lib/runner/test-runner/test-report-handler-tests.js index f62841a..b8f7f87 100644 --- a/test/unit/lib/runner/test-runner/test-report-handler-tests.js +++ b/test/unit/lib/runner/test-runner/test-report-handler-tests.js @@ -263,12 +263,17 @@ describe('lib/runner/test-runner/test-report-handler.js', function() { describe('finalizeTestReport', function() { let testReportHandler; + let configHandler; let Emitter; beforeEach(function() { mockery.enable({useCleanCache: true}); mockery.registerAllowable('../../../../../lib/runner/test-runner/test-report-handler.js'); + configHandler = { + get: sinon.stub(), + }; + Emitter = { mixIn: function(myObject) { myObject.on = sinon.stub(); @@ -279,7 +284,7 @@ describe('lib/runner/test-runner/test-report-handler.js', function() { mockery.registerMock('../../util/emitter.js', Emitter); mockery.registerMock('../runner-event-dispatch/runner-event-dispatch.js', {}); - mockery.registerMock('../../util/config/config-handler.js', {}); + mockery.registerMock('../../util/config/config-handler.js', configHandler); testReportHandler = require('../../../../../lib/runner/test-runner/test-report-handler.js'); testReportHandler._handleTestReport = sinon.stub(); @@ -415,12 +420,17 @@ describe('lib/runner/test-runner/test-report-handler.js', function() { describe('finalizeReport', function() { let testReportHandler; + let configHandler; let Emitter; beforeEach(function() { mockery.enable({useCleanCache: true}); mockery.registerAllowable('../../../../../lib/runner/test-runner/test-report-handler.js'); + configHandler = { + get: sinon.stub(), + }; + Emitter = { mixIn: function(myObject) { myObject.on = sinon.stub(); @@ -433,7 +443,7 @@ describe('lib/runner/test-runner/test-report-handler.js', function() { mockery.registerMock('../../util/emitter.js', Emitter); mockery.registerMock('../runner-event-dispatch/runner-event-dispatch.js', {}); - mockery.registerMock('../../util/config/config-handler.js', {}); + mockery.registerMock('../../util/config/config-handler.js', configHandler); testReportHandler = require('../../../../../lib/runner/test-runner/test-report-handler.js'); testReportHandler._handleTestReportSummary = sinon.stub(); From 2ef7a6b04bc449f6596f4fa7bffcb28a02f6ac2e Mon Sep 17 00:00:00 2001 From: Zach Knox Date: Mon, 27 Jul 2020 11:03:18 -0400 Subject: [PATCH 3/5] fixing bugs and adding in more tests --- lib/runner/test-runner/test-report-handler.js | 10 +- .../test-runner/test-report-handler-tests.js | 122 ++++++++++++++++++ 2 files changed, 127 insertions(+), 5 deletions(-) diff --git a/lib/runner/test-runner/test-report-handler.js b/lib/runner/test-runner/test-report-handler.js index 168cd18..50d4a21 100644 --- a/lib/runner/test-runner/test-report-handler.js +++ b/lib/runner/test-runner/test-report-handler.js @@ -59,14 +59,14 @@ module.exports = testReportHandler = { testRun.report.status = 'rerun'; testReport.rerunCount++; } else if (testRun.report.status === 'fail' && configHandler.get('deferFailureReports') === true) { - _failedReports.push(testRun.report); + testReportHandler._failedReports.push(testRun.report); return; } testReportHandler._handleTestReport(testRun.report); }, finalizeReport() { if (configHandler.get('deferFailureReports') === true) { - printFailedReports(); + testReportHandler._printFailedReports(); } testReportHandler._report.time = process.hrtime(testReportHandler._report.time); if (!testReportHandler._report.failedTestCount) { @@ -75,9 +75,9 @@ module.exports = testReportHandler = { testReportHandler._handleTestReportSummary(); testReportHandler.emit('testReportHandler.reportFinalized'); }, - printFailedReports() { - for (let report of _failedReports) { - _handleTestReport(report); + _printFailedReports() { + for (let report of testReportHandler._failedReports) { + testReportHandler._handleTestReport(report); } }, _handleTestReport(report) { diff --git a/test/unit/lib/runner/test-runner/test-report-handler-tests.js b/test/unit/lib/runner/test-runner/test-report-handler-tests.js index b8f7f87..cdc9378 100644 --- a/test/unit/lib/runner/test-runner/test-report-handler-tests.js +++ b/test/unit/lib/runner/test-runner/test-report-handler-tests.js @@ -400,6 +400,68 @@ describe('lib/runner/test-runner/test-report-handler.js', function() { }); }); + describe('if the most recent testRun report status is \'fail\' ' + + 'and the passed in rerunCount is falsey', function() { + describe('when config for deferFailureReports is true', function() { + it('should add the report to _failedReports', function() { + configHandler.get.returns(true); + + testReportHandler._failedReports = [{ + status: 'fail', + label: 'report1', + }]; + + testReportHandler._report = { + failedTestCount: 3, + testReports: [{ + status: 'fail', + rerunCount: 0, + testRuns: [{ + report: { + status: 'fail', + label: 'report2', + }, + }], + }], + }; + + testReportHandler.finalizeTestReport(0, -1); + + expect(testReportHandler._failedReports).to.eql([ + { + status: 'fail', + label: 'report1', + }, + { + status: 'fail', + label: 'report2', + }, + ]); + }); + + it('should not call _handleTestReport', function() { + configHandler.get.returns(true); + + testReportHandler._report = { + failedTestCount: 3, + testReports: [{ + status: 'fail', + rerunCount: 0, + testRuns: [{ + report: { + status: 'fail', + }, + }], + }], + }; + + testReportHandler.finalizeTestReport(0, -1); + + expect(testReportHandler._handleTestReport.args).to.eql([]); + }); + }); + }); + it('should call _handleTestReport with the report of the passed in testNumber most recent testRun', function() { testReportHandler._report.testReports[0] = { rerunCount: 0, @@ -446,6 +508,7 @@ describe('lib/runner/test-runner/test-report-handler.js', function() { mockery.registerMock('../../util/config/config-handler.js', configHandler); testReportHandler = require('../../../../../lib/runner/test-runner/test-report-handler.js'); + testReportHandler._printFailedReports = sinon.stub(); testReportHandler._handleTestReportSummary = sinon.stub(); }); @@ -456,6 +519,18 @@ describe('lib/runner/test-runner/test-report-handler.js', function() { process.hrtime.restore(); }); + describe('when the config for deferFailureReports is true', function() { + it('should call printFailedReports', function() { + configHandler.get.returns(true); + + testReportHandler.finalizeReport(); + + expect(testReportHandler._printFailedReports.args).to.eql([ + [], + ]); + }); + }); + it('should call process.hrtime once', function() { testReportHandler.finalizeReport(); @@ -503,6 +578,53 @@ describe('lib/runner/test-runner/test-report-handler.js', function() { }); }); + describe('_printFailedReports', function() { + let testReportHandler; + let Emitter; + + beforeEach(function() { + mockery.enable({useCleanCache: true}); + mockery.registerAllowable('../../../../../lib/runner/test-runner/test-report-handler.js'); + + Emitter = { + mixIn: function(myObject) { + myObject.on = sinon.stub(); + myObject.emit = sinon.stub(); + }, + }; + sinon.spy(Emitter, 'mixIn'); + + sinon.stub(process, 'hrtime').returns([4, 54321]); + + mockery.registerMock('../../util/emitter.js', Emitter); + mockery.registerMock('../runner-event-dispatch/runner-event-dispatch.js', {}); + mockery.registerMock('../../util/config/config-handler.js', {}); + + testReportHandler = require('../../../../../lib/runner/test-runner/test-report-handler.js'); + testReportHandler._handleTestReport = sinon.stub(); + }); + + afterEach(function() { + mockery.resetCache(); + mockery.deregisterAll(); + mockery.disable(); + process.hrtime.restore(); + }); + + it('should call _handleTestReport for every failed report', function() { + testReportHandler._failedReports = [ + 'report1', 'report2', + ]; + + testReportHandler._printFailedReports(); + + expect(testReportHandler._handleTestReport.args).to.eql([ + ['report1'], + ['report2'] + ]); + }); + }); + describe('_handleTestReport', function() { let testReportHandler; let Emitter; From 14b17e3aad04339e08cadf8704cf0b25174148cb Mon Sep 17 00:00:00 2001 From: Zach Knox Date: Mon, 27 Jul 2020 11:10:07 -0400 Subject: [PATCH 4/5] added a comma to appease the linter --- test/unit/lib/runner/test-runner/test-report-handler-tests.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/lib/runner/test-runner/test-report-handler-tests.js b/test/unit/lib/runner/test-runner/test-report-handler-tests.js index cdc9378..5e17cb8 100644 --- a/test/unit/lib/runner/test-runner/test-report-handler-tests.js +++ b/test/unit/lib/runner/test-runner/test-report-handler-tests.js @@ -620,7 +620,7 @@ describe('lib/runner/test-runner/test-report-handler.js', function() { expect(testReportHandler._handleTestReport.args).to.eql([ ['report1'], - ['report2'] + ['report2'], ]); }); }); From f5592572daeb6c1f058a2b005d4e0787261677a5 Mon Sep 17 00:00:00 2001 From: Zach Knox Date: Tue, 4 Aug 2020 09:39:12 -0400 Subject: [PATCH 5/5] AAA format --- test/unit/lib/runner/test-runner/test-report-handler-tests.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/unit/lib/runner/test-runner/test-report-handler-tests.js b/test/unit/lib/runner/test-runner/test-report-handler-tests.js index 5e17cb8..d65e9e0 100644 --- a/test/unit/lib/runner/test-runner/test-report-handler-tests.js +++ b/test/unit/lib/runner/test-runner/test-report-handler-tests.js @@ -405,12 +405,10 @@ describe('lib/runner/test-runner/test-report-handler.js', function() { describe('when config for deferFailureReports is true', function() { it('should add the report to _failedReports', function() { configHandler.get.returns(true); - testReportHandler._failedReports = [{ status: 'fail', label: 'report1', }]; - testReportHandler._report = { failedTestCount: 3, testReports: [{ @@ -441,7 +439,6 @@ describe('lib/runner/test-runner/test-report-handler.js', function() { it('should not call _handleTestReport', function() { configHandler.get.returns(true); - testReportHandler._report = { failedTestCount: 3, testReports: [{